fix(blueprints): fail closed on marker ownership for apply and withdraw (#1694)

* fix(blueprints): fail closed on marker ownership for apply and withdraw

Require a matching .blueprint.json under the stack lock, persist required_blueprint_id on deletion intents, remove the legacy remote apply fallback, and protect the marker in the file explorer.

* fix(blueprints): add CodeQL path barriers on ownership probes

Use the canonical resolve-and-startsWith sanitizer inline at the marker and stack-directory fs sinks so js/path-injection clears.

* fix(blueprints): block delete on failed withdraw and defer marker write

Refuse Blueprint DELETE when pre-delete withdraw does not complete, and write .blueprint.json only after a successful deploy so failed applies cannot orphan stacks or claim an unapplied revision.

* test(blueprints): align lock-order assert with deferred marker write

Update the per-stack lock ordering expectations to compose, cleanup, deploy, then marker after the partial-apply fix.

* fix(deps): bump postcss past GHSA-r28c-9q8g-f849 for npm audit

Raise the Vitest/Vite transitive postcss to 8.5.23 so Backend CI audit --audit-level=high passes.
This commit is contained in:
Anso
2026-07-24 15:57:18 -04:00
committed by GitHub
parent e33eda3c38
commit 17a8dc8a94
19 changed files with 1092 additions and 286 deletions
+244 -181
View File
@@ -19,8 +19,12 @@ import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlo
import { enforcePolicyForImageRefs } from './PolicyEnforcement';
import { BlueprintAnalyzer } from './BlueprintAnalyzer';
import { sanitizeForLog } from '../utils/safeLog';
const MARKER_FILENAME = '.blueprint.json';
import { isPathWithinBase } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
type BlueprintMarker,
} from '../helpers/blueprintMarker';
/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */
const COMPOSE_FILENAME = 'compose.yaml';
const REMOTE_HTTP_TIMEOUT_MS = 30_000;
@@ -41,10 +45,32 @@ function diagnosticLog(message: string, fields: Record<string, string | number |
console.info(`[BlueprintService:diag] ${message}`, safeFields);
}
export interface BlueprintMarker {
blueprintId: number;
revision: number;
lastApplied: number;
export type { BlueprintMarker };
export class BlueprintNameConflictError extends Error {
readonly code = 'name_conflict' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintNameConflictError';
}
}
/** Thrown when a remote node lacks the atomic apply/withdraw endpoints. */
export class BlueprintRemoteUpgradeRequiredError extends Error {
readonly code = 'remote_upgrade_required' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintRemoteUpgradeRequiredError';
}
}
/** Thrown when ownership cannot be verified (non-ENOENT I/O or remote probe failure). */
export class BlueprintOwnershipProbeError extends Error {
readonly code = 'ownership_probe_failed' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintOwnershipProbeError';
}
}
export interface DeployOutcome {
@@ -52,11 +78,17 @@ export interface DeployOutcome {
error?: string;
}
type LocalMarkerRead =
| { kind: 'missing' }
| { kind: 'present'; marker: BlueprintMarker }
| { kind: 'failed'; error: string };
/**
* BlueprintService is the orchestration layer between the reconciler and the
* concrete deploy/withdraw primitives. It owns:
* - per-target marker-file management (writes, reads, validates ownership)
* - name-conflict guard (refuses to touch a stack directory missing the marker)
* - name-conflict guard (refuses apply/withdraw when the directory lacks a matching
* `.blueprint.json` for this blueprint ID)
* - local deploy via ComposeService + FileSystemService
* - remote deploy via direct HTTP calls to the remote Sencho instance
* - per-(blueprint,node) concurrency lock so overlapping ticks don't collide
@@ -128,15 +160,12 @@ export class BlueprintService {
async readMarker(blueprintName: string, node: Node): Promise<BlueprintMarker | null> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME);
if (!markerPath.startsWith(path.resolve(baseDir))) return null;
const content = await fsPromises.readFile(markerPath, 'utf-8');
return BlueprintService.parseMarker(content);
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
return markerRead.kind === 'present' ? markerRead.marker : null;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return null;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`;
const res = await axios.get(url, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
@@ -146,7 +175,7 @@ export class BlueprintService {
const body = res.data;
const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null);
if (content == null) return null;
return BlueprintService.parseMarker(content);
return parseBlueprintMarker(content);
} catch {
return null;
}
@@ -154,47 +183,77 @@ export class BlueprintService {
/**
* Returns true when a stack directory by this name exists on the target
* node but does not carry our marker file. The reconciler must not
* deploy in that case: there is a real user-authored stack with the
* same name and we must not overwrite it.
* node and the on-disk marker is missing, malformed, or references a
* different blueprint ID. Throws BlueprintOwnershipProbeError when the
* directory or marker cannot be probed (non-ENOENT I/O or remote list failure).
*/
async hasNameConflict(blueprintName: string, node: Node): Promise<boolean> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch {
return false; // directory doesn't exist → no conflict
}
const markerPath = path.join(stackDir, MARKER_FILENAME);
try {
await fsPromises.stat(markerPath);
return false; // marker present → ours
} catch {
return true; // directory exists but no marker → conflict
}
async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise<boolean> {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!isPathWithinBase(stackDir, baseDir)) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err));
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return false;
const baseUrl = target.apiUrl.replace(/\/$/, '');
const listUrl = `${baseUrl}/api/stacks`;
const listRes = await axios.get(listUrl, {
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
if (markerRead.kind === 'failed') {
throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error);
}
return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`,
);
}
const baseUrl = target.apiUrl.replace(/\/$/, '');
let listRes;
try {
listRes = await axios.get(`${baseUrl}/api/stacks`, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
});
if (listRes.status !== 200) return false;
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null;
} catch {
return false;
} catch (err) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`,
);
}
if (listRes.status !== 200) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`,
);
}
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null || marker.blueprintId !== blueprintId;
}
/** Read and parse a local on-disk marker without going through the remote HTTP path. */
private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise<LocalMarkerRead> {
try {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
const content = await fsPromises.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker) return { kind: 'missing' };
return { kind: 'present', marker };
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'missing' };
return { kind: 'failed', error: BlueprintService.formatError(err) };
}
}
@@ -222,7 +281,7 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'deploying');
if (await this.hasNameConflict(blueprint.name, node)) {
if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`,
});
@@ -249,6 +308,12 @@ export class BlueprintService {
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'active' };
} catch (err) {
if (err instanceof BlueprintNameConflictError) {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message });
console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s',
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'name_conflict', error: 'name_conflict' };
}
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s',
@@ -280,20 +345,15 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'withdrawing');
// Refuse to withdraw a directory we do not own
const marker = await this.readMarker(blueprint.name, node);
if (marker && marker.blueprintId !== blueprint.id) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`,
});
return { status: 'name_conflict' };
}
// Ownership is validated on the node that owns the stack, inside the delete lock.
if (node.type === 'local') {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' });
await this.withdrawLocal(blueprint, node);
const localOutcome = await this.withdrawLocal(blueprint, node);
if (localOutcome.status !== 'withdrawn') return localOutcome;
} else {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' });
await this.withdrawRemote(blueprint, node);
const remoteOutcome = await this.withdrawRemote(blueprint, node);
if (remoteOutcome.status !== 'withdrawn') return remoteOutcome;
}
DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id);
console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s',
@@ -382,15 +442,22 @@ export class BlueprintService {
// ---- local primitives ----
/** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */
private async stackDirExists(nodeId: number, blueprintName: string): Promise<boolean> {
const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return false;
if (!isPathWithinBase(stackDir, baseDir)) {
throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`);
}
try {
const stat = await fsPromises.stat(stackDir);
return stat.isDirectory();
} catch {
return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw new BlueprintOwnershipProbeError(
`Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`,
);
}
}
@@ -423,14 +490,14 @@ export class BlueprintService {
}
/**
* Create the stack if needed, write the compose and marker files, run the
* deploy policy gate, and deploy, all under the per-stack operation lock so
* none of it can race a manual deploy/update/rollback/backup on the same
* stack and node. Runs on the node that owns the stack: deployLocal calls it
* for the hub's own node, and the /api/blueprints/apply-local route calls it
* on a remote node receiving a blueprint apply from its hub (so the file
* writes hold the remote's lock, not just the deploy). On lock conflict
* nothing is written and { ran: false } is returned.
* Create the stack if needed, write the compose file, run the deploy policy
* gate and deploy, then write the marker, all under the per-stack operation
* lock. The marker is written only after a successful deploy so a failed
* apply cannot claim an applied revision that never ran. Runs on the node
* that owns the stack: deployLocal calls it for the hub's own node, and the
* /api/blueprints/apply-local route calls it on a remote node receiving a
* blueprint apply from its hub. On lock conflict nothing is written and
* { ran: false } is returned.
*/
async applyLocalUnderLock(
nodeId: number,
@@ -439,47 +506,83 @@ export class BlueprintService {
markerContent: string,
auditPath: string,
): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> {
const expected = parseBlueprintMarker(markerContent);
if (!expected) {
throw new Error('Invalid blueprint marker');
}
const fs = FileSystemService.getInstance(nodeId);
const lock = await StackOpLockService.getInstance().runExclusive(
nodeId, stackName, 'deploy', 'system',
async () => {
if (!(await this.stackDirExists(nodeId, stackName))) {
let createdStack = false;
if (await this.stackDirExists(nodeId, stackName)) {
const existing = await this.readLocalMarkerFromDisk(nodeId, stackName);
if (existing.kind === 'failed') {
throw new BlueprintOwnershipProbeError(
`Cannot verify ownership of stack "${stackName}": ${existing.error}`,
);
}
if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) {
throw new BlueprintNameConflictError(
`A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`,
);
}
} else {
await fs.createStack(stackName);
createdStack = true;
}
await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent);
await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent);
// Clear lower-priority compose siblings so discovery cannot shadow compose.yaml.
// Local + modern apply-local only; legacy remote has no sibling DELETE.
await fs.removeAlternateRootComposeFiles(stackName);
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
try {
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent);
} catch (err) {
if (createdStack) {
try {
await fs.deleteStack(stackName);
} catch (cleanupErr) {
console.warn(
'[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s',
sanitizeForLog(stackName),
sanitizeForLog(BlueprintService.formatError(cleanupErr)),
);
}
}
throw err;
}
},
);
return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action };
}
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: node.id,
stackName: blueprint.name,
pruneVolumes: false,
actor: 'system:blueprint',
requireBlueprintId: blueprint.id,
});
if (!result.ok) {
if (result.code === 'lock_conflict') {
throw new Error(result.error);
}
throw new Error(result.error);
if (result.ok) {
return { status: 'withdrawn' };
}
if (result.code === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error });
return { status: 'name_conflict' };
}
this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error });
return { status: 'failed', error: result.error };
}
// ---- remote primitives ----
@@ -500,10 +603,7 @@ export class BlueprintService {
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// Atomic apply: the remote runs create + write compose/marker + deploy
// under its own per-stack lock, so the file writes cannot race a manual
// operation on that node. Older nodes without this route answer 404; we
// fall back to the legacy multi-call flow there (not lock-atomic).
// Atomic apply: the remote validates ownership and writes under its stack lock.
const res = await axios.post(
`${baseUrl}/api/blueprints/apply-local`,
{
@@ -514,11 +614,17 @@ export class BlueprintService {
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status === 404) {
console.warn(`[BlueprintService] remote node ${node.id} lacks /api/blueprints/apply-local; using legacy non-atomic apply`);
await this.deployRemoteLegacy(blueprint, node, marker);
return;
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`,
);
}
if (res.status === 409) {
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
throw new BlueprintNameConflictError(
BlueprintService.extractApiError(res.data)
|| `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`,
);
}
throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`);
}
if (res.status >= 400) {
@@ -526,105 +632,62 @@ export class BlueprintService {
}
}
/**
* Legacy remote apply for nodes that predate /api/blueprints/apply-local:
* create, write compose, write marker, deploy as separate calls. The remote
* deploy locks, but the preceding file writes do not, so this is not atomic
* against a concurrent manual operation on that node. Kept only as a
* compatibility fallback.
*/
private async deployRemoteLegacy(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success.
const createRes = await axios.post(`${baseUrl}/api/stacks`,
{ stackName: blueprint.name },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (createRes.status >= 400 && createRes.status !== 409) {
throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`);
}
// 2. Write the compose file
await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
// 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state)
await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
// 4. Deploy
const deployRes = await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`,
{},
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (deployRes.status >= 400) {
throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`);
}
}
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<void> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// down (best-effort)
let res;
try {
await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`,
{},
res = await axios.post(
`${baseUrl}/api/blueprints/withdraw-local`,
{ stackName: blueprint.name, blueprintId: blueprint.id },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
} catch (err) {
console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
// delete the stack directory entirely
const delRes = await axios.delete(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`,
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (delRes.status >= 400 && delRes.status !== 404) {
throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`);
if (res.status === 404) {
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`,
);
}
}
private async remotePutFile(
baseUrl: string,
headers: Record<string, string>,
stackName: string,
relPath: string,
content: string,
): Promise<void> {
const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`;
const res = await axios.put(url,
{ content },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status >= 400) {
throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`);
if (res.status === 200) {
return { status: 'withdrawn' };
}
if (res.status === 409) {
const error = BlueprintService.extractApiError(res.data) || 'withdraw refused';
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error });
return { status: 'name_conflict' };
}
// stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed
this.setStatus(blueprint.id, node.id, 'failed', { last_error: error });
return { status: 'failed', error };
}
const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`;
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
static parseMarker(content: string): BlueprintMarker | null {
try {
const parsed = JSON.parse(content);
if (parsed && typeof parsed === 'object'
&& typeof parsed.blueprintId === 'number'
&& typeof parsed.revision === 'number') {
return {
blueprintId: parsed.blueprintId,
revision: parsed.revision,
lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0,
};
}
} catch {
// fall through
}
return null;
return parseBlueprintMarker(content);
}
private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError {
return new BlueprintOwnershipProbeError(
`Cannot verify stack ownership for "${blueprintName}": ${detail}`,
);
}
static extractApiCode(body: unknown): string {
if (!body || typeof body !== 'object') return '';
const code = (body as Record<string, unknown>).code;
return typeof code === 'string' ? code : '';
}
static formatError(err: unknown): string {
+7 -3
View File
@@ -251,6 +251,8 @@ export interface StackUpdateCleanupPendingRow {
rollback_tags_json: string;
override_paths_json: string;
prune_volumes_requested: number;
/** Blueprint ID that authorized this deletion; null for manual deletes. */
required_blueprint_id: number | null;
created_at: number;
updated_at: number;
}
@@ -1828,6 +1830,7 @@ export class DatabaseService {
`);
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
// Distributed API model columns
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
@@ -3933,12 +3936,12 @@ export class DatabaseService {
this.db.prepare(
`INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
override_paths_json, prune_volumes_requested, required_blueprint_id, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
row.created_at, row.updated_at,
row.required_blueprint_id, row.created_at, row.updated_at,
);
}
@@ -4409,6 +4412,7 @@ export class DatabaseService {
rollback_tags_json: JSON.stringify(localCleanup.tags),
override_paths_json: JSON.stringify(localCleanup.overridePaths),
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -23,6 +23,10 @@ import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
} from '../helpers/blueprintMarker';
/**
* Directory that may contain recovery override files for a tombstone sweep.
@@ -46,13 +50,30 @@ export interface DeleteDeployedStackInput {
stackName: string;
pruneVolumes: boolean;
actor: string;
/** When set, deletion requires an on-disk .blueprint.json matching this blueprint ID. */
requireBlueprintId?: number;
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
continuationIntentId?: string;
}
export type DeleteDeployedStackResult =
| { ok: true }
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
| { ok: true; status: 'deleted' | 'already_absent' }
| {
ok: false;
code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed' | 'name_conflict' | 'failed';
error: string;
existingAction?: string;
};
type DirProbe = { kind: 'absent' } | { kind: 'present' } | { kind: 'error'; error: string };
type MarkerProbe =
| { kind: 'match' }
| { kind: 'name_conflict'; error: string }
| { kind: 'failed'; error: string };
function blueprintMarkerMismatchError(stackName: string): string {
return `Stack "${stackName}" exists without a matching blueprint marker; refusing to withdraw.`;
}
function collectArtifactsFromGenerations(
generations: Array<{ override_path: string | null; services_json: string }>,
@@ -100,6 +121,51 @@ function parseJsonStringArray(raw: string): string[] {
}
}
async function probeStackDirectory(nodeId: number, stackName: string): Promise<DirProbe> {
// Canonical js/path-injection barrier inline with the stat sink.
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
const safePath = path.resolve(baseResolved, stackName);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'error', error: 'Invalid stack path' };
}
try {
const stat = await fs.stat(safePath);
return stat.isDirectory() ? { kind: 'present' } : { kind: 'absent' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'absent' };
return { kind: 'error', error: getErrorMessage(error, 'Failed to access stack directory') };
}
}
async function probeBlueprintMarkerOwnership(
nodeId: number,
stackName: string,
requireBlueprintId: number,
): Promise<MarkerProbe> {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
try {
const content = await fs.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker || marker.blueprintId !== requireBlueprintId) {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'match' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'failed', error: getErrorMessage(error, 'Failed to read blueprint marker') };
}
}
export class DeployedStackDeletionService {
private static instance: DeployedStackDeletionService;
@@ -164,6 +230,42 @@ export class DeployedStackDeletionService {
const { nodeId, stackName, pruneVolumes } = input;
const db = DatabaseService.getInstance();
// Continuation loads ownership from the persisted intent; first call uses input.
let requiredBlueprintId: number | null =
typeof input.requireBlueprintId === 'number' ? input.requireBlueprintId : null;
if (existingIntentId) {
const existing = db.getDeletionIntentById(existingIntentId);
if (!existing || existing.status !== 'prepared') {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
if (existing.required_blueprint_id != null) {
requiredBlueprintId = existing.required_blueprint_id;
}
}
let skipPhysical = false;
if (requiredBlueprintId != null) {
const dirProbe = await probeStackDirectory(nodeId, stackName);
if (dirProbe.kind === 'error') {
return { ok: false, code: 'failed', error: dirProbe.error };
}
if (dirProbe.kind === 'absent') {
skipPhysical = true;
} else {
const ownership = await probeBlueprintMarkerOwnership(nodeId, stackName, requiredBlueprintId);
if (ownership.kind === 'failed') {
return { ok: false, code: 'failed', error: ownership.error };
}
if (ownership.kind === 'name_conflict') {
if (existingIntentId) {
db.updateCleanupPendingStatus(existingIntentId, 'cancelled');
}
return { ok: false, code: 'name_conflict', error: ownership.error };
}
}
}
let intentId = existingIntentId;
if (!intentId) {
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
@@ -177,6 +279,7 @@ export class DeployedStackDeletionService {
rollback_tags_json: JSON.stringify(tags),
override_paths_json: JSON.stringify(overridePaths),
prune_volumes_requested: pruneVolumes ? 1 : 0,
required_blueprint_id: requiredBlueprintId,
created_at: now,
updated_at: now,
};
@@ -197,38 +300,53 @@ export class DeployedStackDeletionService {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
try {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
if (!skipPhysical) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
pruneErr,
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
sanitizeForLog(stackName),
pruneErr,
);
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
const finalized = await this.finalizeLogicalDeletion(input, intentId);
if (!finalized.ok) return finalized;
return { ok: true, status: skipPhysical ? 'already_absent' : 'deleted' };
}
/** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */
private async finalizeLogicalDeletion(
input: DeleteDeployedStackInput,
intentId: string,
): Promise<DeleteDeployedStackResult> {
const { nodeId, stackName } = input;
const db = DatabaseService.getInstance();
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
return {
@@ -282,7 +400,7 @@ export class DeployedStackDeletionService {
stackName,
ts: Date.now(),
});
return { ok: true };
return { ok: true, status: 'deleted' };
}
/**
@@ -497,6 +615,31 @@ export class DeployedStackDeletionService {
continue;
}
if (intent.required_blueprint_id != null) {
const ownership = await probeBlueprintMarkerOwnership(
nodeId,
stackName,
intent.required_blueprint_id,
);
if (ownership.kind === 'name_conflict') {
db.updateCleanupPendingStatus(intent.id, 'cancelled');
console.warn(
'[DeployedStackDeletion] Startup cancelled blueprint deletion for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
if (ownership.kind === 'failed') {
console.warn(
'[DeployedStackDeletion] Startup ownership probe failed for %s (leaving prepared): %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
}
const result = await this.deleteDeployedStack({
nodeId,
stackName,
+10 -3
View File
@@ -58,6 +58,13 @@ const PROTECTED_STACK_FILES = new Set([
'.env',
]);
// Explorer-only protection: includes the blueprint ownership marker without
// putting it in PROTECTED_STACK_FILES (backup/rollback orphan removal).
const EXPLORER_PROTECTED_STACK_FILES = new Set([
...PROTECTED_STACK_FILES,
'.blueprint.json',
]);
// Bookkeeping markers Sencho writes into the backup slot. They are never copied
// back into the stack directory on restore: `.timestamp` records when the backup
// was taken; `.checksums` is the integrity manifest verified before a restore.
@@ -169,7 +176,7 @@ function isProtectedRelPath(relPath: string): boolean {
if (normalized.includes('/')) return false;
// Fold case so e.g. a request for COMPOSE.YAML cannot dodge the gate on a
// case-insensitive filesystem where it resolves to the real compose.yaml.
return PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
return EXPLORER_PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
}
function protectedFileError(relPath: string): Error & { code: string } {
@@ -1643,7 +1650,7 @@ export class FileSystemService {
type,
size,
mtime,
isProtected: protectedEnabled && PROTECTED_STACK_FILES.has(dirent.name),
isProtected: protectedEnabled && EXPLORER_PROTECTED_STACK_FILES.has(dirent.name),
};
})
);
@@ -2096,7 +2103,7 @@ export class FileSystemService {
type,
size: stat.isDirectory() ? 0 : stat.size,
mtime: stat.mtimeMs,
isProtected: (scope?.protectedEnabled ?? true) && PROTECTED_STACK_FILES.has(name),
isProtected: (scope?.protectedEnabled ?? true) && EXPLORER_PROTECTED_STACK_FILES.has(name),
};
}
}
@@ -457,15 +457,30 @@ function asApprovedBlueprint(blueprint: Blueprint): Blueprint & BlueprintApprova
}
/** Upgrade create rows to blockers when an unmanaged same-name stack already exists. */
async function applyCreateNameConflictBlockers(blueprintName: string, raw: RawAction[]): Promise<void> {
function blockCreateForOwnership(row: RawAction, detail: string): void {
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = detail;
}
async function applyCreateNameConflictBlockers(
blueprintName: string,
blueprintId: number,
raw: RawAction[],
): Promise<void> {
const { BlueprintService } = await import('./BlueprintService');
const svc = BlueprintService.getInstance();
for (const row of raw) {
if (row.action !== 'create') continue;
if (!(await svc.hasNameConflict(blueprintName, row.node))) continue;
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = 'Unmanaged stack with this name already exists on this node';
try {
if (!(await svc.hasNameConflict(blueprintName, row.node, blueprintId))) continue;
blockCreateForOwnership(row, 'Unmanaged stack with this name already exists on this node');
} catch (err) {
blockCreateForOwnership(
row,
err instanceof Error ? err.message : 'Cannot verify stack ownership on this node',
);
}
}
}
@@ -477,7 +492,7 @@ export async function buildBlueprintPreview(blueprintId: number): Promise<Bluepr
const deployments = db.listDeployments(blueprintId);
const decision = BlueprintReconciler.getInstance().computeDecisionForPreview(blueprint, allNodes);
const raw = projectActions(blueprint, allNodes, deployments, decision);
await applyCreateNameConflictBlockers(blueprint.name, raw);
await applyCreateNameConflictBlockers(blueprint.name, blueprint.id, raw);
const changes: PreviewChangeRow[] = [];
for (const row of raw) {