mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-18 06:23:18 +00:00
f5178889eb
* feat(recovery): capture complete authored Compose project for atomic rollback Replace the root-compose-only backup slot with staged recovery generations that record the managed inventory, exact Compose invocation, and prior image identity, and wire the same engine through deploy, update, manual rollback, and Git apply. * fix(recovery): satisfy CodeQL path barriers and update-guard mock Inline resolve+startsWith checks at generation/inventory fs sinks and stub getCurrentStackUpdateRecovery in UpdateGuardService tests. * fix(recovery): drop unused FileSystemService import in generation store test * fix(recovery): harden authored-project rollback for upgrade and restore safety Preserve legacy UUID backup rows, restore Git deploy state with files, make multi-file restore recoverable, evaluate policy on the restored target, and fail closed when Git capture cannot cover an apply. * fix(recovery): unblock Git apply unit tests and CodeQL pre-restore TOCTOU Mock recovery capture in git-source-service tests after fail-closed apply capture, and re-resolve live paths immediately before pre-restore snapshot reads. * fix(recovery): fall back to authored inventory when Git manifesto is missing First Git apply captures before promote, so a missing managed-project manifesto must not block rollback capture when the live stack already has authored files. * fix(recovery): make authored-project rollback atomic across Git state Restore the managed-project manifesto with files, keep nullable Git identity on first-apply captures, persist Git side-state in restore intents for startup reconcile, compensate legacy materialize failures, and refuse directory collisions before mutation. * fix(recovery): satisfy CodeQL path and TOCTOU barriers on manifesto restore Add inline resolve barriers for manifesto read/clear sinks and remove the access-then-read race when restoring a generation manifesto snapshot. * fix(recovery): close third-audit rollback generation blockers Fail closed on incomplete Git inventory fallbacks, execute captured Compose invocation during recovery, refuse startup and mutations while restore intents remain unresolved, propagate legacy stale-delete failures, and add Docker-level exact prior-image coverage plus regression tests. * fix(recovery): mark acquired before handoff in prior-image Docker test Match the production updateStack CAS sequence so the exact prior-image integration test does not fail handoff from the captured phase. * fix(recovery): close fourth-audit rollback safety blockers Evaluate policy against held images, use index-based pre-restore snapshots, hold the shared stack lock across Git apply, replay Mesh and empty captured invocations exactly, restore POSIX modes with fail-closed sensitive permissions, keep case-sensitive paths, and link Git auto-deploy health gates. Add regression coverage for these cases. * test(recovery): fix mocks for health-gate link and authored compose args Add linkGateOrRetain to the Git apply recovery mock, and mock authoredComposeArgs so the case-collision inventory test is not masked by a missing getComposeDir stub. * fix(recovery): close fifth-audit rollback safety blockers Share git_apply locking for webhook auto-apply, fail closed on malformed recovery service records, refuse mixed-image capture, and require exact probe counts with hold-tag eligibility checks. * fix(recovery): close sixth-audit rollback safety blockers Preserve the legacy backup slot during generation capture, encrypt sensitive pre-restore snapshots, revert files on a failed health probe without committing Git, fail closed when an absent-file revert would delete a directory, skip Compose one-offs, route manual and scheduled backup through the current generation, and persist runtime image platform identity. * fix(recovery): close seventh-audit rollback safety blockers Fleet snapshot restore and restore-all now capture a recovery generation under the stack lock before any authored file write, including on remote nodes. * fix(recovery): keep pre-deploy generations during health-gate observe Link deploy recovery generations to the observing gate so backup cannot replace them mid-observe. Distinguish missing hold tags from probe failures, refuse generation release when services metadata is corrupt, classify mixed-replica and coverage refusals, and toast the backend rollback message. * fix(recovery): wrap webhook deploy case for eslint const bindings in an unbraced switch case trip no-case-declarations. Match the pull case block.
392 lines
18 KiB
TypeScript
392 lines
18 KiB
TypeScript
import crypto from 'crypto';
|
|
import { ComposeService } from './ComposeService';
|
|
import { StackUpdateOrchestrator } from './StackUpdateOrchestrator';
|
|
import { StackOpLockService, stackOpSkipMessage, type StackOpAction } from './StackOpLockService';
|
|
import { DatabaseService, type Webhook } from './DatabaseService';
|
|
import { FileSystemService } from './FileSystemService';
|
|
import { GitSourceService } from './GitSourceService';
|
|
import { HealthGateService } from './HealthGateService';
|
|
import { LicenseService } from './LicenseService';
|
|
import { PROXY_TIER_HEADER } from './license-headers';
|
|
import { NodeRegistry } from './NodeRegistry';
|
|
import { getErrorMessage } from '../utils/errors';
|
|
import { redactSensitiveText } from '../utils/safeLog';
|
|
import { isValidStackName } from '../utils/validation';
|
|
import { assertPolicyGateAllows, buildSystemPolicyGateOptions } from '../helpers/policyGate';
|
|
|
|
type ExecutionResult = { success: boolean; error?: string; duration_ms: number };
|
|
type ExecutionStatus = 'success' | 'failure';
|
|
|
|
const REMOTE_WEBHOOK_REQUEST_TIMEOUT_MS = 30_000;
|
|
|
|
// Maps a webhook lifecycle action to the per-stack lock action. 'pull' updates,
|
|
// so it locks as 'update'; 'git-pull' is excluded (it locks inside GitSourceService).
|
|
const WEBHOOK_LOCK_ACTION: Record<string, StackOpAction | undefined> = {
|
|
deploy: 'deploy',
|
|
restart: 'restart',
|
|
stop: 'stop',
|
|
start: 'start',
|
|
pull: 'update',
|
|
};
|
|
|
|
export class WebhookService {
|
|
private static instance: WebhookService;
|
|
// Stable per-process decoy secret used to keep HMAC work non-skippable on
|
|
// reject paths (unknown webhook id, disabled, etc.). Never accepts a
|
|
// signature: the trigger handler decides the final 202 / 404 outcome from
|
|
// independent conditions and only consults the HMAC result when every
|
|
// other check has already passed.
|
|
private static decoySecret: string | null = null;
|
|
|
|
public static getInstance(): WebhookService {
|
|
if (!WebhookService.instance) {
|
|
WebhookService.instance = new WebhookService();
|
|
}
|
|
return WebhookService.instance;
|
|
}
|
|
|
|
public static getDecoySecret(): string {
|
|
if (!WebhookService.decoySecret) {
|
|
WebhookService.decoySecret = crypto.randomBytes(32).toString('hex');
|
|
}
|
|
return WebhookService.decoySecret;
|
|
}
|
|
|
|
public generateSecret(): string {
|
|
return crypto.randomBytes(32).toString('hex');
|
|
}
|
|
|
|
public validateSignature(payload: string, secret: string, signature: string): boolean {
|
|
// Always perform HMAC and timingSafeEqual against a fixed-length
|
|
// buffer so the wall-clock cost of this call is independent of the
|
|
// shape of the input signature. Without this, the early-return paths
|
|
// (missing header, wrong prefix, malformed hex) would skip the HMAC
|
|
// and let an attacker distinguish those cases from a real-shape
|
|
// wrong-secret case through repeated near-rate-limit probes with a
|
|
// large attacker-controlled body. Timing now depends only on the
|
|
// size of `payload`, which the attacker already controls and which
|
|
// does not reveal anything about the webhook id.
|
|
const expected = crypto.createHmac('sha256', secret).update(payload).digest();
|
|
const provided = Buffer.alloc(32);
|
|
let formatOk = false;
|
|
|
|
const parts = signature.split('=');
|
|
if (parts.length === 2 && parts[0] === 'sha256' && /^[0-9a-fA-F]{64}$/.test(parts[1])) {
|
|
provided.write(parts[1], 'hex');
|
|
formatOk = true;
|
|
}
|
|
|
|
const sigEq = crypto.timingSafeEqual(expected, provided);
|
|
return formatOk && sigEq;
|
|
}
|
|
|
|
public async gitSourceExists(stackName: string, nodeId: number): Promise<boolean> {
|
|
const node = NodeRegistry.getInstance().getNode(nodeId);
|
|
if (!node) return false;
|
|
if (node.type !== 'remote') return GitSourceService.getInstance().get(stackName) !== undefined;
|
|
|
|
const response = await this.remoteStackRequest(nodeId, stackName, 'git-source', 'GET');
|
|
return response.ok;
|
|
}
|
|
|
|
public async execute(
|
|
webhook: Webhook,
|
|
action: string,
|
|
triggerSource: string | null,
|
|
atomic?: boolean,
|
|
): Promise<ExecutionResult> {
|
|
if (webhook.id === undefined) {
|
|
throw new Error('Webhook must be loaded from the database before execution');
|
|
}
|
|
const webhookId = webhook.id;
|
|
|
|
const nodeId = webhook.node_id || NodeRegistry.getInstance().getDefaultNodeId();
|
|
const node = NodeRegistry.getInstance().getNode(nodeId);
|
|
if (!node) {
|
|
const error = `Node for webhook "${webhook.name}" was not found`;
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, 0, error);
|
|
return { success: false, error, duration_ms: 0 };
|
|
}
|
|
|
|
if (node.type === 'remote') {
|
|
return this.executeRemote(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic);
|
|
}
|
|
|
|
return this.executeLocal(webhookId, nodeId, webhook.stack_name, action, triggerSource, atomic);
|
|
}
|
|
|
|
public maskSecret(secret: string): string {
|
|
if (secret.length <= 8) return '********';
|
|
return '********' + secret.slice(-4);
|
|
}
|
|
|
|
private async executeLocal(
|
|
webhookId: number,
|
|
nodeId: number,
|
|
stackName: string,
|
|
action: string,
|
|
triggerSource: string | null,
|
|
atomic?: boolean,
|
|
): Promise<ExecutionResult> {
|
|
const stacks = await FileSystemService.getInstance(nodeId).getStacks();
|
|
if (!stacks.includes(stackName)) {
|
|
const error = `Stack "${stackName}" not found`;
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, 0, error);
|
|
return { success: false, error, duration_ms: 0 };
|
|
}
|
|
|
|
const startTime = Date.now();
|
|
try {
|
|
// git-pull pulls then deploys through GitSourceService, which holds
|
|
// the per-stack lock itself; locking here too would self-conflict.
|
|
if (action === 'git-pull') {
|
|
return this.executeLocalGitPull(webhookId, stackName, action, triggerSource, startTime);
|
|
}
|
|
const lockAction = WEBHOOK_LOCK_ACTION[action];
|
|
if (!lockAction) throw new Error(`Unknown action: ${action}`);
|
|
const compose = ComposeService.getInstance(nodeId);
|
|
// Run the lifecycle op under the per-stack lock so a webhook cannot
|
|
// race a manual deploy/update/rollback/backup on the same stack.
|
|
const lock = await StackOpLockService.getInstance().runExclusive(
|
|
nodeId, stackName, lockAction, 'system',
|
|
async () => {
|
|
switch (action) {
|
|
case 'deploy': {
|
|
await assertPolicyGateAllows(
|
|
stackName,
|
|
nodeId,
|
|
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
|
);
|
|
const deployResult = await compose.deployStack(
|
|
stackName,
|
|
undefined,
|
|
atomic,
|
|
{ source: 'webhook', actor: 'system:webhook' },
|
|
);
|
|
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'deploy', 'system:webhook');
|
|
if (deployResult.recoveryId) {
|
|
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
|
StackUpdateRecoveryService.getInstance().linkGateOrRetain(deployResult.recoveryId, healthGateId);
|
|
}
|
|
break;
|
|
}
|
|
case 'restart':
|
|
await compose.runCommand(stackName, 'restart');
|
|
break;
|
|
case 'stop':
|
|
await compose.runCommand(stackName, 'stop');
|
|
break;
|
|
case 'start':
|
|
await compose.runCommand(stackName, 'start');
|
|
break;
|
|
case 'pull': {
|
|
await assertPolicyGateAllows(
|
|
stackName,
|
|
nodeId,
|
|
buildSystemPolicyGateOptions('webhook', { auditPath: `/api/webhooks/${webhookId}/execute` }),
|
|
);
|
|
const orchResult = await StackUpdateOrchestrator.getInstance().execute(
|
|
{ nodeId, stackName, target: { scope: 'stack' }, trigger: 'webhook', actor: 'system:webhook' },
|
|
{ atomic: atomic ?? false, terminalWs: null },
|
|
);
|
|
const healthGateId = HealthGateService.getInstance().beginStack(nodeId, stackName, 'update', 'system:webhook');
|
|
const recoveryId = orchResult.kind === 'stack_compose_done' ? orchResult.recoveryId : null;
|
|
if (recoveryId) {
|
|
const { StackUpdateRecoveryService } = await import('./StackUpdateRecoveryService');
|
|
StackUpdateRecoveryService.getInstance().linkGateOrRetain(recoveryId, healthGateId);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
},
|
|
);
|
|
|
|
const durationMs = Date.now() - startTime;
|
|
if (!lock.ran) {
|
|
const error = stackOpSkipMessage(stackName, lock.existing.action);
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
|
|
return { success: false, error, duration_ms: durationMs };
|
|
}
|
|
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, null);
|
|
return { success: true, duration_ms: durationMs };
|
|
} catch (err) {
|
|
const durationMs = Date.now() - startTime;
|
|
const error = getErrorMessage(err, 'Unknown error');
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
|
|
return { success: false, error, duration_ms: durationMs };
|
|
}
|
|
}
|
|
|
|
private async executeLocalGitPull(
|
|
webhookId: number,
|
|
stackName: string,
|
|
action: string,
|
|
triggerSource: string | null,
|
|
startTime: number,
|
|
): Promise<ExecutionResult> {
|
|
const result = await GitSourceService.getInstance().handleWebhookPull(stackName);
|
|
const durationMs = Date.now() - startTime;
|
|
if (result.status === 'error') {
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, result.message);
|
|
return { success: false, error: result.message, duration_ms: durationMs };
|
|
}
|
|
|
|
const skipped = result.status === 'skipped';
|
|
// A debounced pull is rate-limited, not failed (the route answers 202
|
|
// Accepted). Record it as a success carrying the debounce note so it
|
|
// does not pollute the webhook's failure history.
|
|
this.recordExecution(
|
|
webhookId,
|
|
action,
|
|
'success',
|
|
triggerSource,
|
|
durationMs,
|
|
skipped ? result.message : null,
|
|
);
|
|
return { success: true, error: undefined, duration_ms: durationMs };
|
|
}
|
|
|
|
private async executeRemote(
|
|
webhookId: number,
|
|
nodeId: number,
|
|
stackName: string,
|
|
action: string,
|
|
triggerSource: string | null,
|
|
atomic?: boolean,
|
|
): Promise<ExecutionResult> {
|
|
const startTime = Date.now();
|
|
try {
|
|
const endpoint = action === 'git-pull'
|
|
? 'git-source/webhook-pull'
|
|
: action === 'pull'
|
|
? 'update'
|
|
: action;
|
|
const body = atomic === undefined ? undefined : { atomic };
|
|
const response = await this.remoteStackRequest(nodeId, stackName, endpoint, 'POST', body);
|
|
const durationMs = Date.now() - startTime;
|
|
const payload = await response.json().catch(() => ({})) as { error?: string; message?: string; status?: string };
|
|
|
|
if (!response.ok || payload.status === 'error') {
|
|
const error = payload.error || payload.message || `Remote ${action} failed with status ${response.status}`;
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
|
|
return { success: false, error, duration_ms: durationMs };
|
|
}
|
|
|
|
// A debounced remote pull comes back 202 with status "skipped": it was
|
|
// accepted and rate-limited, not failed. Record it as a success with
|
|
// the debounce note rather than failure noise.
|
|
const skipped = payload.status === 'skipped';
|
|
this.recordExecution(webhookId, action, 'success', triggerSource, durationMs, skipped ? (payload.message ?? null) : null);
|
|
return { success: true, duration_ms: durationMs };
|
|
} catch (err) {
|
|
const durationMs = Date.now() - startTime;
|
|
const error = getErrorMessage(err, 'Remote node operation failed');
|
|
this.recordExecution(webhookId, action, 'failure', triggerSource, durationMs, error);
|
|
return { success: false, error, duration_ms: durationMs };
|
|
}
|
|
}
|
|
|
|
private async remoteStackRequest(
|
|
nodeId: number,
|
|
stackName: string,
|
|
endpoint: string,
|
|
method: 'GET' | 'POST',
|
|
body?: unknown,
|
|
): Promise<Response> {
|
|
const target = NodeRegistry.getInstance().getProxyTarget(nodeId);
|
|
if (!target) throw new Error('Remote node is unreachable or not configured');
|
|
|
|
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
|
if (target.apiToken) headers.Authorization = `Bearer ${target.apiToken}`;
|
|
|
|
const licenseHeaders = LicenseService.getInstance().getProxyHeaders();
|
|
headers[PROXY_TIER_HEADER] = licenseHeaders.tier;
|
|
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), REMOTE_WEBHOOK_REQUEST_TIMEOUT_MS);
|
|
try {
|
|
// nodeId selects a server-controlled entry from the registry
|
|
// (CodeQL GOOD pattern: user input maps to known values, not concatenated into the URL).
|
|
const targetBase = new URL(target.apiUrl);
|
|
const protocol = targetBase.protocol;
|
|
const host = targetBase.host;
|
|
const hostname = targetBase.hostname;
|
|
|
|
// Verify the hostname is in the configured-node allow-list.
|
|
const allowedHosts = DatabaseService.getInstance().getNodes()
|
|
.filter(n => n.api_url)
|
|
.map(n => new URL(n.api_url!).hostname);
|
|
if (!allowedHosts.includes(hostname)) {
|
|
throw new Error('Remote node hostname is not a configured node');
|
|
}
|
|
|
|
// Restrict protocol to http/https (prevents file://, ftp://, etc.).
|
|
if (protocol !== 'http:' && protocol !== 'https:') {
|
|
throw new Error('Remote node URL must use http:// or https://');
|
|
}
|
|
|
|
// Validate path components to prevent traversal.
|
|
if (!isValidStackName(stackName)) {
|
|
throw new Error('Invalid stack name');
|
|
}
|
|
if (!/^[a-z][a-z0-9/-]*$/.test(endpoint) || endpoint.includes('..')) {
|
|
throw new Error('Invalid endpoint');
|
|
}
|
|
|
|
// Build URL from validated, server-controlled components.
|
|
const url = `${protocol}//${host}/api/stacks/${encodeURIComponent(stackName)}/${endpoint}`;
|
|
return await fetch(url, {
|
|
method,
|
|
headers,
|
|
body: method === 'GET' || body === undefined ? undefined : JSON.stringify(body),
|
|
signal: controller.signal,
|
|
});
|
|
} catch (err) {
|
|
if (controller.signal.aborted) {
|
|
throw new Error('Remote node request timed out', { cause: err });
|
|
}
|
|
throw err;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
private recordExecution(
|
|
webhookId: number,
|
|
action: string,
|
|
status: ExecutionStatus,
|
|
triggerSource: string | null,
|
|
durationMs: number,
|
|
error: string | null,
|
|
): void {
|
|
// Execution history is readable in the UI; scrub bearer tokens,
|
|
// JWTs, URL credentials, and homedir paths before persisting so a
|
|
// compose / remote-node error surfacing on the dashboard cannot leak
|
|
// operator secrets or infrastructure details.
|
|
const safeError = error === null ? null : redactSensitiveText(error);
|
|
try {
|
|
DatabaseService.getInstance().addWebhookExecution({
|
|
webhook_id: webhookId,
|
|
action,
|
|
status,
|
|
trigger_source: triggerSource,
|
|
duration_ms: durationMs,
|
|
error: safeError,
|
|
executed_at: Date.now(),
|
|
});
|
|
} catch (err) {
|
|
// The webhook_executions table has ON DELETE CASCADE on webhook_id,
|
|
// so a delete that races an in-flight execution removes the parent
|
|
// row and any insert here fails the FK constraint. Swallow that
|
|
// race: the trigger already returned 202 and the action either
|
|
// ran or failed before reaching this point. Other write errors
|
|
// are still worth logging as warnings so a structural problem
|
|
// does not go silent.
|
|
console.warn(
|
|
`[Webhooks] Could not record execution for webhook ${webhookId} ` +
|
|
`(parent webhook may have been deleted mid-flight): ${getErrorMessage(err, 'Unknown error')}`,
|
|
);
|
|
}
|
|
}
|
|
}
|