mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-19 14:56:27 +00:00
feat(blueprints): backend foundation for fleet-wide compose templates (#860)
* feat(blueprints): add backend foundation for fleet-wide compose templates
Introduces the Blueprint Model: a docker-compose.yml plus a node selector
(labels or explicit IDs) that Sencho reconciles across the fleet. Backend
foundation only; the frontend tab and documentation follow.
Schema (DatabaseService):
- node_labels table for fleet-level orchestration tagging
- blueprints table with compose content, selector, drift_mode, classification
- blueprint_deployments table for per-node materialized state
- New idempotent migrate methods following the existing pattern
Services:
- BlueprintAnalyzer: pure compose-YAML classifier (stateless / stateful /
unknown) with 17 covered cases including named volumes, bind mounts,
external volumes, and tmpfs
- NodeLabelService: label CRUD plus selector matching helper (any/all/ids)
- BlueprintService: local + remote deploy/withdraw orchestration, marker
file management, name-conflict guard, per-(blueprint,node) lock
- BlueprintReconciler: 60-second loop with three-mode drift policy
(observe/suggest/enforce), state-aware guards, and Enforce-downgrade for
volume-destroying drift
Routes (gated requirePaid + requireAdmin on mutations):
- /api/blueprints (CRUD + apply + withdraw + accept + preview + analyze)
- /api/node-labels (CRUD + listAll + listDistinct)
Notifications: four new categories registered in NotificationService for
deploy/failure/drift events.
Bootstrap: reconciler start/stop wired in startup and shutdown.
Tests: 45 new Vitest cases covering selector matching, classifier rules,
state-aware guards, drift-mode branching, and marker parsing. Full backend
suite (1625 tests) passes; tsc clean.
* fix(lint): replace bare Function type in blueprint reconciler tests
Replace 8 occurrences of `as unknown as { computeDecision: Function }`
with a properly typed `ReconcilerWithCompute` alias that mirrors the
real method signature. Export `ReconcileDecision` from
BlueprintReconciler so the test can reference it.
Resolves @typescript-eslint/no-unsafe-function-type errors that were
failing the Backend (Lint) CI step.
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
import { parse as parseYaml } from 'yaml';
|
||||
import type { BlueprintClassification } from './DatabaseService';
|
||||
|
||||
export interface AnalyzerResult {
|
||||
classification: BlueprintClassification;
|
||||
reasons: string[];
|
||||
hasNamedVolumes: boolean;
|
||||
hasBindMounts: boolean;
|
||||
hasExternalVolumes: boolean;
|
||||
hasTmpfsOnly: boolean;
|
||||
parseError?: string;
|
||||
}
|
||||
|
||||
interface ComposeShape {
|
||||
services?: Record<string, ComposeService> | null;
|
||||
volumes?: Record<string, ComposeVolume | null> | null;
|
||||
}
|
||||
|
||||
interface ComposeService {
|
||||
volumes?: Array<string | ComposeServiceVolume> | null;
|
||||
tmpfs?: string | string[] | null;
|
||||
}
|
||||
|
||||
interface ComposeServiceVolume {
|
||||
type?: string;
|
||||
source?: string;
|
||||
target?: string;
|
||||
bind?: { propagation?: string };
|
||||
volume?: { nocopy?: boolean };
|
||||
}
|
||||
|
||||
interface ComposeVolume {
|
||||
external?: boolean | { name?: string };
|
||||
driver?: string;
|
||||
name?: string;
|
||||
}
|
||||
|
||||
const STATEFUL_BIND_TARGETS = [
|
||||
'/var/lib',
|
||||
'/var/log',
|
||||
'/data',
|
||||
'/db',
|
||||
'/etc',
|
||||
];
|
||||
|
||||
export class BlueprintAnalyzer {
|
||||
static analyze(composeContent: string): AnalyzerResult {
|
||||
const result: AnalyzerResult = {
|
||||
classification: 'unknown',
|
||||
reasons: [],
|
||||
hasNamedVolumes: false,
|
||||
hasBindMounts: false,
|
||||
hasExternalVolumes: false,
|
||||
hasTmpfsOnly: true,
|
||||
};
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = parseYaml(composeContent);
|
||||
} catch (err) {
|
||||
result.parseError = err instanceof Error ? err.message : String(err);
|
||||
result.classification = 'unknown';
|
||||
result.reasons.push('compose YAML did not parse');
|
||||
result.hasTmpfsOnly = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (parsed == null || typeof parsed !== 'object') {
|
||||
result.parseError = 'compose document is empty or not an object';
|
||||
result.classification = 'unknown';
|
||||
result.reasons.push('compose document is empty');
|
||||
result.hasTmpfsOnly = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
const doc = parsed as ComposeShape;
|
||||
if (!doc.services || Object.keys(doc.services).length === 0) {
|
||||
result.classification = 'unknown';
|
||||
result.reasons.push('compose document has no services');
|
||||
result.hasTmpfsOnly = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
const topVolumes = doc.volumes ?? {};
|
||||
const services = doc.services ?? {};
|
||||
|
||||
const externalVolumeNames = new Set<string>();
|
||||
const declaredNamedVolumes = new Set<string>();
|
||||
for (const [volName, volDef] of Object.entries(topVolumes)) {
|
||||
if (volDef && typeof volDef === 'object' && 'external' in volDef && volDef.external) {
|
||||
externalVolumeNames.add(volName);
|
||||
result.hasExternalVolumes = true;
|
||||
result.reasons.push(`external volume "${volName}": Sencho cannot reason about portability`);
|
||||
result.hasTmpfsOnly = false;
|
||||
} else {
|
||||
declaredNamedVolumes.add(volName);
|
||||
}
|
||||
}
|
||||
|
||||
let sawAnyMount = false;
|
||||
|
||||
for (const [serviceName, serviceDef] of Object.entries(services)) {
|
||||
if (!serviceDef || typeof serviceDef !== 'object') continue;
|
||||
const volumes = serviceDef.volumes ?? [];
|
||||
for (const v of volumes) {
|
||||
sawAnyMount = true;
|
||||
if (typeof v === 'string') {
|
||||
const parts = v.split(':');
|
||||
const source = parts[0];
|
||||
if (!source) continue;
|
||||
if (source.startsWith('/') || source.startsWith('.') || source.startsWith('~')) {
|
||||
result.hasBindMounts = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`bind mount "${source}" in service "${serviceName}"`);
|
||||
} else if (externalVolumeNames.has(source)) {
|
||||
// already accounted for in topVolumes loop
|
||||
} else {
|
||||
result.hasNamedVolumes = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`named volume "${source}" in service "${serviceName}"`);
|
||||
}
|
||||
} else if (v && typeof v === 'object') {
|
||||
const t = (v.type ?? '').toLowerCase();
|
||||
const source = v.source ?? '';
|
||||
if (t === 'bind') {
|
||||
result.hasBindMounts = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`bind mount "${source}" in service "${serviceName}"`);
|
||||
} else if (t === 'volume') {
|
||||
if (externalVolumeNames.has(source)) {
|
||||
// counted above
|
||||
} else {
|
||||
result.hasNamedVolumes = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`named volume "${source}" in service "${serviceName}"`);
|
||||
}
|
||||
} else if (t === 'tmpfs') {
|
||||
// tmpfs is ephemeral; do not flip hasTmpfsOnly
|
||||
} else if (source) {
|
||||
// type omitted; fall back to source heuristic
|
||||
if (source.startsWith('/') || source.startsWith('.') || source.startsWith('~')) {
|
||||
result.hasBindMounts = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`bind mount "${source}" in service "${serviceName}"`);
|
||||
} else {
|
||||
result.hasNamedVolumes = true;
|
||||
result.hasTmpfsOnly = false;
|
||||
result.reasons.push(`named volume "${source}" in service "${serviceName}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const tmpfs = serviceDef.tmpfs;
|
||||
if (tmpfs && (typeof tmpfs === 'string' || Array.isArray(tmpfs))) {
|
||||
sawAnyMount = true;
|
||||
// tmpfs alone keeps hasTmpfsOnly true unless other mounts already flipped it
|
||||
}
|
||||
}
|
||||
|
||||
if (!sawAnyMount) {
|
||||
result.hasTmpfsOnly = false;
|
||||
}
|
||||
|
||||
// Classification rules (apply in order):
|
||||
// 1. Named volumes or bind mounts → stateful
|
||||
// 2. External volumes (no other persistence) → unknown (Sencho cannot prove portability)
|
||||
// 3. Only tmpfs or no volumes → stateless
|
||||
if (result.hasNamedVolumes || result.hasBindMounts) {
|
||||
result.classification = 'stateful';
|
||||
} else if (result.hasExternalVolumes) {
|
||||
result.classification = 'unknown';
|
||||
} else {
|
||||
result.classification = 'stateless';
|
||||
if (result.reasons.length === 0) {
|
||||
result.reasons.push('no persistent volumes detected');
|
||||
}
|
||||
}
|
||||
|
||||
// Helpful annotation: if any bind mount targets a known stateful path, surface it
|
||||
for (const [serviceName, serviceDef] of Object.entries(services)) {
|
||||
if (!serviceDef || typeof serviceDef !== 'object') continue;
|
||||
for (const v of serviceDef.volumes ?? []) {
|
||||
if (typeof v !== 'string') continue;
|
||||
const parts = v.split(':');
|
||||
const target = parts[1];
|
||||
if (!target) continue;
|
||||
if (STATEFUL_BIND_TARGETS.some(prefix => target.startsWith(prefix))) {
|
||||
result.reasons.push(`mount target "${target}" in service "${serviceName}" looks data-bearing`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when applying a new compose to an existing deployment would
|
||||
* destroy named volumes (rename or removal of a top-level named volume).
|
||||
* Used by the reconciler to downgrade Enforce → Suggest for volume-destroying
|
||||
* drift events on stateful blueprints.
|
||||
*/
|
||||
static wouldDestroyVolumes(currentCompose: string, nextCompose: string): boolean {
|
||||
const current = BlueprintAnalyzer.extractNamedVolumes(currentCompose);
|
||||
const next = BlueprintAnalyzer.extractNamedVolumes(nextCompose);
|
||||
for (const name of current) {
|
||||
if (!next.has(name)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static extractNamedVolumes(composeContent: string): Set<string> {
|
||||
try {
|
||||
const doc = (parseYaml(composeContent) ?? {}) as ComposeShape;
|
||||
const out = new Set<string>();
|
||||
for (const [name, def] of Object.entries(doc.volumes ?? {})) {
|
||||
if (def && typeof def === 'object' && 'external' in def && def.external) continue;
|
||||
out.add(name);
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return new Set();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
import {
|
||||
DatabaseService,
|
||||
type Blueprint,
|
||||
type BlueprintDeployment,
|
||||
type Node,
|
||||
} from './DatabaseService';
|
||||
import { BlueprintService } from './BlueprintService';
|
||||
import { BlueprintAnalyzer } from './BlueprintAnalyzer';
|
||||
import { NodeLabelService } from './NodeLabelService';
|
||||
import { NotificationService } from './NotificationService';
|
||||
|
||||
const RECONCILER_INTERVAL_MS = 60_000;
|
||||
const RECONCILER_INITIAL_DELAY_MS = 5_000;
|
||||
|
||||
export interface ReconcileDecision {
|
||||
deploy: Node[];
|
||||
withdraw: Node[];
|
||||
check: Node[];
|
||||
stateReview: Node[];
|
||||
evictBlocked: Node[];
|
||||
}
|
||||
|
||||
/**
|
||||
* BlueprintReconciler is the desired-state loop. Every tick it reads each
|
||||
* enabled blueprint, resolves its selector, and reconciles the per-node
|
||||
* state against the desired set. It honors the state-aware guards
|
||||
* (stateful blueprints get pending_state_review on first deploy and
|
||||
* evict_blocked on un-target) and the three-mode drift policy.
|
||||
*/
|
||||
export class BlueprintReconciler {
|
||||
private static instance: BlueprintReconciler | null = null;
|
||||
private intervalHandle: ReturnType<typeof setInterval> | null = null;
|
||||
private initialTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private running = false;
|
||||
private stopped = false;
|
||||
|
||||
static getInstance(): BlueprintReconciler {
|
||||
if (!BlueprintReconciler.instance) {
|
||||
BlueprintReconciler.instance = new BlueprintReconciler();
|
||||
}
|
||||
return BlueprintReconciler.instance;
|
||||
}
|
||||
|
||||
private constructor() { /* singleton */ }
|
||||
|
||||
start(): void {
|
||||
if (this.intervalHandle || this.initialTimer) return;
|
||||
this.stopped = false;
|
||||
this.initialTimer = setTimeout(() => {
|
||||
this.initialTimer = null;
|
||||
// Guard against a stop() that fired during the initial delay.
|
||||
if (this.stopped) return;
|
||||
void this.evaluate();
|
||||
this.intervalHandle = setInterval(() => void this.evaluate(), RECONCILER_INTERVAL_MS);
|
||||
}, RECONCILER_INITIAL_DELAY_MS);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
if (this.initialTimer) { clearTimeout(this.initialTimer); this.initialTimer = null; }
|
||||
if (this.intervalHandle) { clearInterval(this.intervalHandle); this.intervalHandle = null; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Force one tick. Useful for the /apply endpoint and tests.
|
||||
*/
|
||||
async tick(): Promise<void> {
|
||||
await this.evaluate();
|
||||
}
|
||||
|
||||
/**
|
||||
* Force reconciliation for a single blueprint. Invoked by the /apply
|
||||
* endpoint so users get immediate action without waiting for the
|
||||
* interval.
|
||||
*/
|
||||
async reconcileOne(blueprintId: number): Promise<void> {
|
||||
const blueprint = DatabaseService.getInstance().getBlueprint(blueprintId);
|
||||
if (!blueprint || !blueprint.enabled) return;
|
||||
const nodes = DatabaseService.getInstance().getNodes();
|
||||
await this.reconcileBlueprint(blueprint, nodes);
|
||||
}
|
||||
|
||||
private async evaluate(): Promise<void> {
|
||||
if (this.running) return; // prevent overlap on slow ticks
|
||||
this.running = true;
|
||||
try {
|
||||
const db = DatabaseService.getInstance();
|
||||
const blueprints = db.listEnabledBlueprints();
|
||||
if (blueprints.length === 0) return;
|
||||
const nodes = db.getNodes();
|
||||
for (const blueprint of blueprints) {
|
||||
try {
|
||||
await this.reconcileBlueprint(blueprint, nodes);
|
||||
} catch (err) {
|
||||
console.error(`[BlueprintReconciler] failed for blueprint "${blueprint.name}":`, err);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
|
||||
private async reconcileBlueprint(blueprint: Blueprint, allNodes: Node[]): Promise<void> {
|
||||
const decision = this.computeDecision(blueprint, allNodes);
|
||||
|
||||
// 1. State-review guard for stateful blueprints reaching new nodes.
|
||||
for (const node of decision.stateReview) {
|
||||
DatabaseService.getInstance().upsertDeployment({
|
||||
blueprint_id: blueprint.id,
|
||||
node_id: node.id,
|
||||
status: 'pending_state_review',
|
||||
last_checked_at: Date.now(),
|
||||
drift_summary: 'Stateful blueprint awaiting operator confirmation before first deploy',
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Eviction guard for stateful blueprints leaving the selector.
|
||||
for (const node of decision.evictBlocked) {
|
||||
DatabaseService.getInstance().upsertDeployment({
|
||||
blueprint_id: blueprint.id,
|
||||
node_id: node.id,
|
||||
status: 'evict_blocked',
|
||||
last_checked_at: Date.now(),
|
||||
drift_summary: 'Stateful blueprint eviction requires operator confirmation',
|
||||
});
|
||||
}
|
||||
|
||||
// 3. Deploy missing/stale entries (stateless or pre-accepted stateful).
|
||||
const svc = BlueprintService.getInstance();
|
||||
for (const node of decision.deploy) {
|
||||
await svc.deployToNode(blueprint, node);
|
||||
}
|
||||
|
||||
// 4. Withdraw stateless deployments leaving the selector.
|
||||
for (const node of decision.withdraw) {
|
||||
await svc.withdrawFromNode(blueprint, node);
|
||||
}
|
||||
|
||||
// 5. Drift check for active deployments.
|
||||
for (const node of decision.check) {
|
||||
const driftResult = await svc.checkForDrift(blueprint, node);
|
||||
if (!driftResult.drifted) continue;
|
||||
const reason = driftResult.reason ?? 'unknown drift';
|
||||
DatabaseService.getInstance().upsertDeployment({
|
||||
blueprint_id: blueprint.id,
|
||||
node_id: node.id,
|
||||
status: 'drifted',
|
||||
last_checked_at: Date.now(),
|
||||
last_drift_at: Date.now(),
|
||||
drift_summary: reason,
|
||||
});
|
||||
await this.handleDrift(blueprint, node, reason);
|
||||
}
|
||||
}
|
||||
|
||||
private computeDecision(blueprint: Blueprint, allNodes: Node[]): ReconcileDecision {
|
||||
const labelSvc = NodeLabelService.getInstance();
|
||||
const desiredNodes = labelSvc.matchSelector(blueprint.selector, allNodes);
|
||||
const desiredIds = new Set(desiredNodes.map(n => n.id));
|
||||
|
||||
const existingDeployments = DatabaseService.getInstance().listDeployments(blueprint.id);
|
||||
const deploymentByNode = new Map<number, BlueprintDeployment>();
|
||||
for (const dep of existingDeployments) deploymentByNode.set(dep.node_id, dep);
|
||||
|
||||
const decision: ReconcileDecision = {
|
||||
deploy: [],
|
||||
withdraw: [],
|
||||
check: [],
|
||||
stateReview: [],
|
||||
evictBlocked: [],
|
||||
};
|
||||
|
||||
// Desired but not active or stale
|
||||
for (const node of desiredNodes) {
|
||||
const dep = deploymentByNode.get(node.id);
|
||||
if (!dep) {
|
||||
if (blueprint.classification === 'stateful' || blueprint.classification === 'unknown') {
|
||||
decision.stateReview.push(node);
|
||||
} else {
|
||||
decision.deploy.push(node);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Operator-blocking states must not be auto-acted on
|
||||
if (dep.status === 'pending_state_review' || dep.status === 'evict_blocked' || dep.status === 'name_conflict') {
|
||||
continue;
|
||||
}
|
||||
if (dep.status === 'active' && dep.applied_revision === blueprint.revision) {
|
||||
decision.check.push(node);
|
||||
continue;
|
||||
}
|
||||
if (dep.applied_revision !== blueprint.revision) {
|
||||
// revision drift: re-deploy (stateful never auto-redeploys volume-destroying changes; handled in handleDrift)
|
||||
decision.deploy.push(node);
|
||||
continue;
|
||||
}
|
||||
if (dep.status === 'failed' || dep.status === 'pending') {
|
||||
decision.deploy.push(node);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Active on a node that is no longer desired
|
||||
for (const dep of existingDeployments) {
|
||||
if (desiredIds.has(dep.node_id)) continue;
|
||||
if (dep.status === 'withdrawn') continue;
|
||||
const node = allNodes.find(n => n.id === dep.node_id);
|
||||
if (!node) continue;
|
||||
if (blueprint.classification === 'stateful' || blueprint.classification === 'unknown') {
|
||||
if (dep.status !== 'evict_blocked') decision.evictBlocked.push(node);
|
||||
} else {
|
||||
decision.withdraw.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
return decision;
|
||||
}
|
||||
|
||||
private async handleDrift(blueprint: Blueprint, node: Node, reason: string): Promise<void> {
|
||||
const notifications = NotificationService.getInstance();
|
||||
switch (blueprint.drift_mode) {
|
||||
case 'observe':
|
||||
return; // detection only; UI surfaces the drift
|
||||
|
||||
case 'suggest':
|
||||
notifications.dispatchAlert(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
`Blueprint "${blueprint.name}" drifted on node "${node.name}": ${reason}`,
|
||||
{ stackName: blueprint.name },
|
||||
);
|
||||
return;
|
||||
|
||||
case 'enforce': {
|
||||
// Stateful safeguard: if the upcoming redeploy would destroy named volumes,
|
||||
// downgrade to suggest semantics for this drift event.
|
||||
if (blueprint.classification === 'stateful') {
|
||||
const marker = await BlueprintService.getInstance().readMarker(blueprint.name, node);
|
||||
if (!marker) {
|
||||
notifications.dispatchAlert(
|
||||
'warning',
|
||||
'blueprint_drift_detected',
|
||||
`Blueprint "${blueprint.name}" lost its marker on node "${node.name}"; auto-fix declined to avoid stomping unowned data. Reason: ${reason}`,
|
||||
{ stackName: blueprint.name },
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
DatabaseService.getInstance().upsertDeployment({
|
||||
blueprint_id: blueprint.id,
|
||||
node_id: node.id,
|
||||
status: 'correcting',
|
||||
last_checked_at: Date.now(),
|
||||
});
|
||||
const result = await BlueprintService.getInstance().deployToNode(blueprint, node);
|
||||
if (result.status !== 'active') {
|
||||
notifications.dispatchAlert(
|
||||
'error',
|
||||
'blueprint_drift_correction_failed',
|
||||
`Auto-fix for "${blueprint.name}" on node "${node.name}" failed: ${result.error ?? 'unknown error'}`,
|
||||
{ stackName: blueprint.name },
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Used by an operator-confirmed redeploy of a deployment in a guard
|
||||
* state. Re-reads the deployment row and refuses unless it is in a
|
||||
* transition-eligible state, so a TOCTOU window between the route
|
||||
* handler's check and the actual deploy can't smuggle a name_conflict
|
||||
* row through.
|
||||
*/
|
||||
async forceDeploy(blueprintId: number, nodeId: number): Promise<void> {
|
||||
const blueprint = DatabaseService.getInstance().getBlueprint(blueprintId);
|
||||
if (!blueprint) return;
|
||||
const node = DatabaseService.getInstance().getNode(nodeId);
|
||||
if (!node) return;
|
||||
const dep = DatabaseService.getInstance().getDeployment(blueprintId, nodeId);
|
||||
// Allow forceDeploy when:
|
||||
// - dep is missing (operator-driven first deploy outside selector)
|
||||
// - dep.status is pending_state_review (operator accepted)
|
||||
// - dep.status is failed (manual retry)
|
||||
// Refuse when dep is name_conflict (must be cleared explicitly) or evict_blocked
|
||||
// (operator must use the withdraw flow first).
|
||||
if (dep && (dep.status === 'name_conflict' || dep.status === 'evict_blocked')) {
|
||||
console.warn(`[BlueprintReconciler] forceDeploy refused for blueprint ${blueprintId} on node ${nodeId}: status=${dep.status}`);
|
||||
return;
|
||||
}
|
||||
await BlueprintService.getInstance().deployToNode(blueprint, node);
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to react to a compose change that introduces volume-destroying
|
||||
* differences. Returns true when the change would destroy data on the
|
||||
* given deployment. Reconciler uses this to refuse Enforce on a
|
||||
* stateful drift that would wipe volumes.
|
||||
*/
|
||||
static wouldDestroyVolumes(blueprint: Blueprint, priorCompose: string): boolean {
|
||||
if (blueprint.classification !== 'stateful') return false;
|
||||
return BlueprintAnalyzer.wouldDestroyVolumes(priorCompose, blueprint.compose_content);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,489 @@
|
||||
import path from 'path';
|
||||
import { promises as fsPromises } from 'fs';
|
||||
import axios, { AxiosError } from 'axios';
|
||||
import {
|
||||
DatabaseService,
|
||||
type Blueprint,
|
||||
type BlueprintDeployment,
|
||||
type BlueprintDeploymentStatus,
|
||||
type Node,
|
||||
} from './DatabaseService';
|
||||
import { ComposeService } from './ComposeService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { LicenseService, PROXY_TIER_HEADER, PROXY_VARIANT_HEADER } from './LicenseService';
|
||||
|
||||
const MARKER_FILENAME = '.blueprint.json';
|
||||
const COMPOSE_FILENAME = 'docker-compose.yml';
|
||||
const REMOTE_HTTP_TIMEOUT_MS = 30_000;
|
||||
|
||||
export interface BlueprintMarker {
|
||||
blueprintId: number;
|
||||
revision: number;
|
||||
lastApplied: number;
|
||||
}
|
||||
|
||||
export interface DeployOutcome {
|
||||
status: BlueprintDeploymentStatus;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* BlueprintService is the orchestration layer between the reconciler and the
|
||||
* concrete deploy/withdraw primitives. It owns:
|
||||
* - per-target marker-file management (writes, reads, validates ownership)
|
||||
* - name-conflict guard (refuses to touch a stack directory missing the marker)
|
||||
* - local deploy via ComposeService + FileSystemService
|
||||
* - remote deploy via direct HTTP calls to the remote Sencho instance
|
||||
* - per-(blueprint,node) concurrency lock so overlapping ticks don't collide
|
||||
*
|
||||
* The reconciler decides *what* needs to happen; this service performs it.
|
||||
*/
|
||||
export class BlueprintService {
|
||||
private static instance: BlueprintService | null = null;
|
||||
private readonly inflight = new Set<string>();
|
||||
|
||||
static getInstance(): BlueprintService {
|
||||
if (!BlueprintService.instance) {
|
||||
BlueprintService.instance = new BlueprintService();
|
||||
}
|
||||
return BlueprintService.instance;
|
||||
}
|
||||
|
||||
private constructor() { /* singleton */ }
|
||||
|
||||
private lockKey(blueprintId: number, nodeId: number): string {
|
||||
return `${blueprintId}:${nodeId}`;
|
||||
}
|
||||
|
||||
private acquireLock(blueprintId: number, nodeId: number): boolean {
|
||||
const key = this.lockKey(blueprintId, nodeId);
|
||||
if (this.inflight.has(key)) return false;
|
||||
this.inflight.add(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
private releaseLock(blueprintId: number, nodeId: number): void {
|
||||
this.inflight.delete(this.lockKey(blueprintId, nodeId));
|
||||
}
|
||||
|
||||
private buildMarker(blueprint: Blueprint): BlueprintMarker {
|
||||
return {
|
||||
blueprintId: blueprint.id,
|
||||
revision: blueprint.revision,
|
||||
lastApplied: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
private setStatus(
|
||||
blueprintId: number,
|
||||
nodeId: number,
|
||||
status: BlueprintDeploymentStatus,
|
||||
extras: Partial<{
|
||||
applied_revision: number | null;
|
||||
last_deployed_at: number | null;
|
||||
last_drift_at: number | null;
|
||||
drift_summary: string | null;
|
||||
last_error: string | null;
|
||||
}> = {},
|
||||
): BlueprintDeployment {
|
||||
return DatabaseService.getInstance().upsertDeployment({
|
||||
blueprint_id: blueprintId,
|
||||
node_id: nodeId,
|
||||
status,
|
||||
last_checked_at: Date.now(),
|
||||
...extras,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the marker file from a target node. Returns null when missing,
|
||||
* malformed, or unreadable. The reconciler treats null as "we do not
|
||||
* own this directory" and refuses to touch it.
|
||||
*/
|
||||
async readMarker(blueprintName: string, node: Node): Promise<BlueprintMarker | null> {
|
||||
try {
|
||||
if (node.type === 'local') {
|
||||
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
|
||||
const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME);
|
||||
if (!markerPath.startsWith(path.resolve(baseDir))) return null;
|
||||
const content = await fsPromises.readFile(markerPath, 'utf-8');
|
||||
return BlueprintService.parseMarker(content);
|
||||
}
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) return null;
|
||||
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`;
|
||||
const res = await axios.get(url, {
|
||||
headers: this.remoteHeaders(target.apiToken),
|
||||
timeout: REMOTE_HTTP_TIMEOUT_MS,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
if (res.status !== 200) return null;
|
||||
const body = res.data;
|
||||
const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null);
|
||||
if (content == null) return null;
|
||||
return BlueprintService.parseMarker(content);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true when a stack directory by this name exists on the target
|
||||
* node but does not carry our marker file. The reconciler must not
|
||||
* deploy in that case: there is a real user-authored stack with the
|
||||
* same name and we must not overwrite it.
|
||||
*/
|
||||
async hasNameConflict(blueprintName: string, node: Node): Promise<boolean> {
|
||||
try {
|
||||
if (node.type === 'local') {
|
||||
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
|
||||
const stackDir = path.resolve(baseDir, blueprintName);
|
||||
if (!stackDir.startsWith(path.resolve(baseDir))) return true;
|
||||
try {
|
||||
const stat = await fsPromises.stat(stackDir);
|
||||
if (!stat.isDirectory()) return false;
|
||||
} catch {
|
||||
return false; // directory doesn't exist → no conflict
|
||||
}
|
||||
const markerPath = path.join(stackDir, MARKER_FILENAME);
|
||||
try {
|
||||
await fsPromises.stat(markerPath);
|
||||
return false; // marker present → ours
|
||||
} catch {
|
||||
return true; // directory exists but no marker → conflict
|
||||
}
|
||||
}
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) return false;
|
||||
const baseUrl = target.apiUrl.replace(/\/$/, '');
|
||||
const listUrl = `${baseUrl}/api/stacks`;
|
||||
const listRes = await axios.get(listUrl, {
|
||||
headers: this.remoteHeaders(target.apiToken),
|
||||
timeout: REMOTE_HTTP_TIMEOUT_MS,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
if (listRes.status !== 200) return false;
|
||||
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
|
||||
const exists = stacks.some(s => s?.name === blueprintName);
|
||||
if (!exists) return false;
|
||||
const marker = await this.readMarker(blueprintName, node);
|
||||
return marker == null;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deploy this blueprint to the given target node. Caller must have already
|
||||
* resolved that the target should receive this blueprint (selector match
|
||||
* passed, no state-review pending, etc.). This method handles the
|
||||
* name-conflict guard and the local/remote dispatch.
|
||||
*/
|
||||
async deployToNode(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
|
||||
if (!this.acquireLock(blueprint.id, node.id)) {
|
||||
return { status: 'pending' };
|
||||
}
|
||||
try {
|
||||
this.setStatus(blueprint.id, node.id, 'deploying');
|
||||
if (await this.hasNameConflict(blueprint.name, node)) {
|
||||
this.setStatus(blueprint.id, node.id, 'name_conflict', {
|
||||
last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`,
|
||||
});
|
||||
return { status: 'name_conflict', error: 'name_conflict' };
|
||||
}
|
||||
const marker = this.buildMarker(blueprint);
|
||||
if (node.type === 'local') {
|
||||
await this.deployLocal(blueprint, node, marker);
|
||||
} else {
|
||||
await this.deployRemote(blueprint, node, marker);
|
||||
}
|
||||
this.setStatus(blueprint.id, node.id, 'active', {
|
||||
applied_revision: blueprint.revision,
|
||||
last_deployed_at: Date.now(),
|
||||
last_drift_at: null,
|
||||
drift_summary: null,
|
||||
last_error: null,
|
||||
});
|
||||
return { status: 'active' };
|
||||
} catch (err) {
|
||||
const message = BlueprintService.formatError(err);
|
||||
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
|
||||
return { status: 'failed', error: message };
|
||||
} finally {
|
||||
this.releaseLock(blueprint.id, node.id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Withdraw a blueprint from the target node: docker compose down, delete
|
||||
* the directory. Caller must have already cleared the eviction guard
|
||||
* (stateful blueprints require explicit operator confirmation).
|
||||
*/
|
||||
async withdrawFromNode(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
|
||||
if (!this.acquireLock(blueprint.id, node.id)) {
|
||||
return { status: 'pending' };
|
||||
}
|
||||
try {
|
||||
this.setStatus(blueprint.id, node.id, 'withdrawing');
|
||||
// Refuse to withdraw a directory we do not own
|
||||
const marker = await this.readMarker(blueprint.name, node);
|
||||
if (marker && marker.blueprintId !== blueprint.id) {
|
||||
this.setStatus(blueprint.id, node.id, 'name_conflict', {
|
||||
last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`,
|
||||
});
|
||||
return { status: 'name_conflict' };
|
||||
}
|
||||
if (node.type === 'local') {
|
||||
await this.withdrawLocal(blueprint, node);
|
||||
} else {
|
||||
await this.withdrawRemote(blueprint, node);
|
||||
}
|
||||
DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id);
|
||||
return { status: 'withdrawn' };
|
||||
} catch (err) {
|
||||
const message = BlueprintService.formatError(err);
|
||||
this.setStatus(blueprint.id, node.id, 'failed', { last_error: `withdraw failed: ${message}` });
|
||||
return { status: 'failed', error: message };
|
||||
} finally {
|
||||
this.releaseLock(blueprint.id, node.id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inspect the actual state of a deployment on its node and report
|
||||
* whether it has drifted from the desired state. The reconciler decides
|
||||
* what to do with the result based on drift_mode.
|
||||
*/
|
||||
async checkForDrift(blueprint: Blueprint, node: Node): Promise<{ drifted: boolean; reason?: string }> {
|
||||
try {
|
||||
const marker = await this.readMarker(blueprint.name, node);
|
||||
if (!marker) {
|
||||
return { drifted: true, reason: 'marker file missing on node' };
|
||||
}
|
||||
if (marker.blueprintId !== blueprint.id) {
|
||||
return { drifted: true, reason: 'marker references a different blueprint' };
|
||||
}
|
||||
if (marker.revision !== blueprint.revision) {
|
||||
return { drifted: true, reason: `revision drift (node has ${marker.revision}, blueprint is ${blueprint.revision})` };
|
||||
}
|
||||
// Check container state
|
||||
const containerState = await this.containerHealth(blueprint.name, node);
|
||||
if (!containerState.allRunning) {
|
||||
return { drifted: true, reason: containerState.detail };
|
||||
}
|
||||
return { drifted: false };
|
||||
} catch (err) {
|
||||
return { drifted: true, reason: BlueprintService.formatError(err) };
|
||||
}
|
||||
}
|
||||
|
||||
private async containerHealth(blueprintName: string, node: Node): Promise<{ allRunning: boolean; detail: string }> {
|
||||
try {
|
||||
// Docker Compose normalizes the project name to lowercase. Match the same canonical form.
|
||||
const projectName = blueprintName.toLowerCase();
|
||||
if (node.type === 'local') {
|
||||
const docker = NodeRegistry.getInstance().getDocker(node.id);
|
||||
const containers = await docker.listContainers({
|
||||
all: true,
|
||||
filters: { label: [`com.docker.compose.project=${projectName}`] },
|
||||
});
|
||||
if (containers.length === 0) return { allRunning: false, detail: 'no containers running for this blueprint' };
|
||||
const notRunning = containers.filter(c => c.State !== 'running');
|
||||
if (notRunning.length > 0) {
|
||||
const first = notRunning[0];
|
||||
return { allRunning: false, detail: `container "${first.Names[0] ?? first.Id.slice(0, 12)}" is ${first.State}` };
|
||||
}
|
||||
return { allRunning: true, detail: '' };
|
||||
}
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) return { allRunning: false, detail: 'remote node not reachable (no proxy target)' };
|
||||
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/containers`;
|
||||
const res = await axios.get(url, {
|
||||
headers: this.remoteHeaders(target.apiToken),
|
||||
timeout: REMOTE_HTTP_TIMEOUT_MS,
|
||||
validateStatus: () => true,
|
||||
});
|
||||
if (res.status !== 200) {
|
||||
return { allRunning: false, detail: `remote stack lookup returned HTTP ${res.status}` };
|
||||
}
|
||||
const list = Array.isArray(res.data) ? res.data as Array<{ State?: string; Names?: string[]; Id?: string }> : [];
|
||||
if (list.length === 0) return { allRunning: false, detail: 'remote stack has no containers' };
|
||||
const notRunning = list.filter(c => (c.State ?? '') !== 'running');
|
||||
if (notRunning.length > 0) {
|
||||
const first = notRunning[0];
|
||||
return { allRunning: false, detail: `remote container "${first.Names?.[0] ?? first.Id?.slice(0, 12)}" is ${first.State}` };
|
||||
}
|
||||
return { allRunning: true, detail: '' };
|
||||
} catch (err) {
|
||||
return { allRunning: false, detail: BlueprintService.formatError(err) };
|
||||
}
|
||||
}
|
||||
|
||||
// ---- local primitives ----
|
||||
|
||||
private async stackDirExists(node: Node, blueprintName: string): Promise<boolean> {
|
||||
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
|
||||
const stackDir = path.resolve(baseDir, blueprintName);
|
||||
if (!stackDir.startsWith(path.resolve(baseDir))) return false;
|
||||
try {
|
||||
const stat = await fsPromises.stat(stackDir);
|
||||
return stat.isDirectory();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async deployLocal(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
|
||||
const fs = FileSystemService.getInstance(node.id);
|
||||
if (!(await this.stackDirExists(node, blueprint.name))) {
|
||||
await fs.createStack(blueprint.name);
|
||||
}
|
||||
await fs.writeStackFile(blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
|
||||
await fs.writeStackFile(blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
|
||||
await ComposeService.getInstance(node.id).deployStack(blueprint.name, undefined, false);
|
||||
}
|
||||
|
||||
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
|
||||
try {
|
||||
await ComposeService.getInstance(node.id).downStack(blueprint.name);
|
||||
} catch (err) {
|
||||
// best-effort: continue to delete the directory even if down fails
|
||||
console.warn(`[BlueprintService] downStack failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
|
||||
}
|
||||
if (await this.stackDirExists(node, blueprint.name)) {
|
||||
await FileSystemService.getInstance(node.id).deleteStack(blueprint.name);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- remote primitives ----
|
||||
|
||||
private remoteHeaders(apiToken: string): Record<string, string> {
|
||||
const proxy = LicenseService.getInstance().getProxyHeaders();
|
||||
return {
|
||||
Authorization: `Bearer ${apiToken}`,
|
||||
[PROXY_TIER_HEADER]: proxy.tier,
|
||||
[PROXY_VARIANT_HEADER]: proxy.variant ?? '',
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
}
|
||||
|
||||
private async deployRemote(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
|
||||
const baseUrl = target.apiUrl.replace(/\/$/, '');
|
||||
const headers = this.remoteHeaders(target.apiToken);
|
||||
|
||||
// 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success.
|
||||
const createRes = await axios.post(`${baseUrl}/api/stacks`,
|
||||
{ stackName: blueprint.name },
|
||||
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
|
||||
);
|
||||
if (createRes.status >= 400 && createRes.status !== 409) {
|
||||
throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`);
|
||||
}
|
||||
|
||||
// 2. Write the compose file
|
||||
await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
|
||||
|
||||
// 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state)
|
||||
await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
|
||||
|
||||
// 4. Deploy
|
||||
const deployRes = await axios.post(
|
||||
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`,
|
||||
{},
|
||||
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
|
||||
);
|
||||
if (deployRes.status >= 400) {
|
||||
throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<void> {
|
||||
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
|
||||
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
|
||||
const baseUrl = target.apiUrl.replace(/\/$/, '');
|
||||
const headers = this.remoteHeaders(target.apiToken);
|
||||
|
||||
// down (best-effort)
|
||||
try {
|
||||
await axios.post(
|
||||
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`,
|
||||
{},
|
||||
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
|
||||
}
|
||||
|
||||
// delete the stack directory entirely
|
||||
const delRes = await axios.delete(
|
||||
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`,
|
||||
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
|
||||
);
|
||||
if (delRes.status >= 400 && delRes.status !== 404) {
|
||||
throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`);
|
||||
}
|
||||
}
|
||||
|
||||
private async remotePutFile(
|
||||
baseUrl: string,
|
||||
headers: Record<string, string>,
|
||||
stackName: string,
|
||||
relPath: string,
|
||||
content: string,
|
||||
): Promise<void> {
|
||||
const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`;
|
||||
const res = await axios.put(url,
|
||||
{ content },
|
||||
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
|
||||
);
|
||||
if (res.status >= 400) {
|
||||
throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`);
|
||||
}
|
||||
}
|
||||
|
||||
static parseMarker(content: string): BlueprintMarker | null {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
if (parsed && typeof parsed === 'object'
|
||||
&& typeof parsed.blueprintId === 'number'
|
||||
&& typeof parsed.revision === 'number') {
|
||||
return {
|
||||
blueprintId: parsed.blueprintId,
|
||||
revision: parsed.revision,
|
||||
lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static formatError(err: unknown): string {
|
||||
if (axios.isAxiosError(err)) {
|
||||
const ax = err as AxiosError<{ error?: string; message?: string }>;
|
||||
if (ax.response?.data) {
|
||||
const body = ax.response.data;
|
||||
if (body && typeof body === 'object') {
|
||||
if (typeof body.error === 'string') return body.error;
|
||||
if (typeof body.message === 'string') return body.message;
|
||||
}
|
||||
}
|
||||
if (ax.code) return `${ax.code}: ${ax.message}`;
|
||||
return ax.message;
|
||||
}
|
||||
if (err instanceof Error) return err.message;
|
||||
return String(err);
|
||||
}
|
||||
|
||||
static extractApiError(body: unknown): string {
|
||||
if (!body || typeof body !== 'object') return '';
|
||||
const obj = body as Record<string, unknown>;
|
||||
if (typeof obj.error === 'string') return obj.error;
|
||||
if (typeof obj.message === 'string') return obj.message;
|
||||
return '';
|
||||
}
|
||||
}
|
||||
@@ -226,6 +226,61 @@ export interface FleetSnapshotFile {
|
||||
content: string;
|
||||
}
|
||||
|
||||
export type DriftMode = 'observe' | 'suggest' | 'enforce';
|
||||
export type BlueprintClassification = 'stateless' | 'stateful' | 'unknown';
|
||||
export type BlueprintDeploymentStatus =
|
||||
| 'pending'
|
||||
| 'pending_state_review'
|
||||
| 'deploying'
|
||||
| 'active'
|
||||
| 'drifted'
|
||||
| 'correcting'
|
||||
| 'failed'
|
||||
| 'withdrawing'
|
||||
| 'withdrawn'
|
||||
| 'evict_blocked'
|
||||
| 'name_conflict';
|
||||
|
||||
export type BlueprintSelector =
|
||||
| { type: 'labels'; any: string[]; all: string[] }
|
||||
| { type: 'nodes'; ids: number[] };
|
||||
|
||||
export interface NodeLabelRow {
|
||||
id: number;
|
||||
node_id: number;
|
||||
label: string;
|
||||
created_at: number;
|
||||
}
|
||||
|
||||
export interface Blueprint {
|
||||
id: number;
|
||||
name: string;
|
||||
description: string | null;
|
||||
compose_content: string;
|
||||
selector: BlueprintSelector;
|
||||
drift_mode: DriftMode;
|
||||
classification: BlueprintClassification;
|
||||
classification_reasons: string[];
|
||||
enabled: boolean;
|
||||
revision: number;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
export interface BlueprintDeployment {
|
||||
id: number;
|
||||
blueprint_id: number;
|
||||
node_id: number;
|
||||
status: BlueprintDeploymentStatus;
|
||||
applied_revision: number | null;
|
||||
last_deployed_at: number | null;
|
||||
last_checked_at: number | null;
|
||||
last_drift_at: number | null;
|
||||
drift_summary: string | null;
|
||||
last_error: string | null;
|
||||
}
|
||||
|
||||
export interface AuditLogEntry {
|
||||
id: number;
|
||||
timestamp: number;
|
||||
@@ -520,6 +575,8 @@ export class DatabaseService {
|
||||
this.migrateNotificationCategory();
|
||||
this.migrateNotificationActor();
|
||||
this.migrateMeshTables();
|
||||
this.migrateNodeLabels();
|
||||
this.migrateBlueprints();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1272,6 +1329,74 @@ export class DatabaseService {
|
||||
this.tryAddColumn('nodes', 'mesh_enabled', 'INTEGER NOT NULL DEFAULT 0');
|
||||
}
|
||||
|
||||
private migrateNodeLabels(): void {
|
||||
try {
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS node_labels (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
UNIQUE(node_id, label),
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
)
|
||||
`).run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_node_labels_node ON node_labels(node_id)').run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_node_labels_label ON node_labels(label)').run();
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] Could not create node_labels:', (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
private migrateBlueprints(): void {
|
||||
try {
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS blueprints (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
compose_content TEXT NOT NULL,
|
||||
selector_json TEXT NOT NULL,
|
||||
drift_mode TEXT NOT NULL DEFAULT 'suggest',
|
||||
classification TEXT NOT NULL DEFAULT 'unknown',
|
||||
classification_reasons TEXT,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
revision INTEGER NOT NULL DEFAULT 1,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
created_by TEXT
|
||||
)
|
||||
`).run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_blueprints_enabled ON blueprints(enabled)').run();
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] Could not create blueprints:', (e as Error).message);
|
||||
}
|
||||
try {
|
||||
this.db.prepare(`
|
||||
CREATE TABLE IF NOT EXISTS blueprint_deployments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
blueprint_id INTEGER NOT NULL,
|
||||
node_id INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
applied_revision INTEGER,
|
||||
last_deployed_at INTEGER,
|
||||
last_checked_at INTEGER,
|
||||
last_drift_at INTEGER,
|
||||
drift_summary TEXT,
|
||||
last_error TEXT,
|
||||
UNIQUE(blueprint_id, node_id),
|
||||
FOREIGN KEY (blueprint_id) REFERENCES blueprints(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE
|
||||
)
|
||||
`).run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_blueprint_deployments_blueprint ON blueprint_deployments(blueprint_id)').run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_blueprint_deployments_node ON blueprint_deployments(node_id)').run();
|
||||
this.db.prepare('CREATE INDEX IF NOT EXISTS idx_blueprint_deployments_status ON blueprint_deployments(status)').run();
|
||||
} catch (e) {
|
||||
console.warn('[DatabaseService] Could not create blueprint_deployments:', (e as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
// --- Sencho Mesh ---
|
||||
|
||||
public listMeshStacks(nodeId?: number): Array<{ id: number; node_id: number; stack_name: string; created_at: number; created_by: string | null }> {
|
||||
@@ -3621,4 +3746,244 @@ export class DatabaseService {
|
||||
).run(nodeId, ...validStackNames);
|
||||
return result.changes;
|
||||
}
|
||||
|
||||
// --- Node Labels (fleet-level orchestration) ---
|
||||
|
||||
public listNodeLabels(nodeId?: number): NodeLabelRow[] {
|
||||
const sql = nodeId !== undefined
|
||||
? 'SELECT id, node_id, label, created_at FROM node_labels WHERE node_id = ? ORDER BY label'
|
||||
: 'SELECT id, node_id, label, created_at FROM node_labels ORDER BY node_id, label';
|
||||
const rows = nodeId !== undefined
|
||||
? this.db.prepare(sql).all(nodeId)
|
||||
: this.db.prepare(sql).all();
|
||||
return rows as NodeLabelRow[];
|
||||
}
|
||||
|
||||
public listDistinctNodeLabels(): string[] {
|
||||
const rows = this.db.prepare('SELECT DISTINCT label FROM node_labels ORDER BY label').all() as { label: string }[];
|
||||
return rows.map(r => r.label);
|
||||
}
|
||||
|
||||
public addNodeLabel(nodeId: number, label: string): NodeLabelRow {
|
||||
const trimmed = label.trim();
|
||||
if (!trimmed) throw new Error('Label must not be empty');
|
||||
const now = Date.now();
|
||||
const result = this.db.prepare(
|
||||
'INSERT OR IGNORE INTO node_labels (node_id, label, created_at) VALUES (?, ?, ?)'
|
||||
).run(nodeId, trimmed, now);
|
||||
const row = result.changes > 0
|
||||
? this.db.prepare('SELECT id, node_id, label, created_at FROM node_labels WHERE id = ?').get(result.lastInsertRowid)
|
||||
: this.db.prepare('SELECT id, node_id, label, created_at FROM node_labels WHERE node_id = ? AND label = ?').get(nodeId, trimmed);
|
||||
return row as NodeLabelRow;
|
||||
}
|
||||
|
||||
public removeNodeLabel(nodeId: number, label: string): boolean {
|
||||
const result = this.db.prepare('DELETE FROM node_labels WHERE node_id = ? AND label = ?').run(nodeId, label);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
public getNodeLabelsMap(): Record<number, string[]> {
|
||||
const rows = this.db.prepare('SELECT node_id, label FROM node_labels ORDER BY node_id, label').all() as { node_id: number; label: string }[];
|
||||
const map: Record<number, string[]> = {};
|
||||
for (const row of rows) {
|
||||
if (!map[row.node_id]) map[row.node_id] = [];
|
||||
map[row.node_id].push(row.label);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// --- Blueprints ---
|
||||
|
||||
private parseBlueprint(row: Record<string, unknown>): Blueprint {
|
||||
return {
|
||||
id: row.id as number,
|
||||
name: row.name as string,
|
||||
description: (row.description as string | null) ?? null,
|
||||
compose_content: row.compose_content as string,
|
||||
selector: JSON.parse(row.selector_json as string) as BlueprintSelector,
|
||||
drift_mode: row.drift_mode as DriftMode,
|
||||
classification: row.classification as BlueprintClassification,
|
||||
classification_reasons: row.classification_reasons
|
||||
? (JSON.parse(row.classification_reasons as string) as string[])
|
||||
: [],
|
||||
enabled: row.enabled === 1,
|
||||
revision: row.revision as number,
|
||||
created_at: row.created_at as number,
|
||||
updated_at: row.updated_at as number,
|
||||
created_by: (row.created_by as string | null) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
public listBlueprints(): Blueprint[] {
|
||||
return this.db.prepare('SELECT * FROM blueprints ORDER BY name')
|
||||
.all()
|
||||
.map(row => this.parseBlueprint(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public listEnabledBlueprints(): Blueprint[] {
|
||||
return this.db.prepare('SELECT * FROM blueprints WHERE enabled = 1 ORDER BY name')
|
||||
.all()
|
||||
.map(row => this.parseBlueprint(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public getBlueprint(id: number): Blueprint | undefined {
|
||||
const row = this.db.prepare('SELECT * FROM blueprints WHERE id = ?').get(id) as Record<string, unknown> | undefined;
|
||||
return row ? this.parseBlueprint(row) : undefined;
|
||||
}
|
||||
|
||||
public getBlueprintByName(name: string): Blueprint | undefined {
|
||||
const row = this.db.prepare('SELECT * FROM blueprints WHERE name = ?').get(name) as Record<string, unknown> | undefined;
|
||||
return row ? this.parseBlueprint(row) : undefined;
|
||||
}
|
||||
|
||||
public createBlueprint(input: {
|
||||
name: string;
|
||||
description: string | null;
|
||||
compose_content: string;
|
||||
selector: BlueprintSelector;
|
||||
drift_mode: DriftMode;
|
||||
classification: BlueprintClassification;
|
||||
classification_reasons: string[];
|
||||
enabled: boolean;
|
||||
created_by: string | null;
|
||||
}): Blueprint {
|
||||
const now = Date.now();
|
||||
const result = this.db.prepare(
|
||||
`INSERT INTO blueprints (name, description, compose_content, selector_json, drift_mode, classification, classification_reasons, enabled, revision, created_at, updated_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)`
|
||||
).run(
|
||||
input.name,
|
||||
input.description,
|
||||
input.compose_content,
|
||||
JSON.stringify(input.selector),
|
||||
input.drift_mode,
|
||||
input.classification,
|
||||
JSON.stringify(input.classification_reasons),
|
||||
input.enabled ? 1 : 0,
|
||||
now,
|
||||
now,
|
||||
input.created_by,
|
||||
);
|
||||
const created = this.getBlueprint(result.lastInsertRowid as number);
|
||||
if (!created) throw new Error('Failed to fetch created blueprint');
|
||||
return created;
|
||||
}
|
||||
|
||||
public updateBlueprint(id: number, updates: {
|
||||
name?: string;
|
||||
description?: string | null;
|
||||
compose_content?: string;
|
||||
selector?: BlueprintSelector;
|
||||
drift_mode?: DriftMode;
|
||||
classification?: BlueprintClassification;
|
||||
classification_reasons?: string[];
|
||||
enabled?: boolean;
|
||||
bumpRevision?: boolean;
|
||||
}): Blueprint | undefined {
|
||||
const existing = this.getBlueprint(id);
|
||||
if (!existing) return undefined;
|
||||
const fields: string[] = [];
|
||||
const values: unknown[] = [];
|
||||
if (updates.name !== undefined) { fields.push('name = ?'); values.push(updates.name); }
|
||||
if (updates.description !== undefined) { fields.push('description = ?'); values.push(updates.description); }
|
||||
if (updates.compose_content !== undefined) { fields.push('compose_content = ?'); values.push(updates.compose_content); }
|
||||
if (updates.selector !== undefined) { fields.push('selector_json = ?'); values.push(JSON.stringify(updates.selector)); }
|
||||
if (updates.drift_mode !== undefined) { fields.push('drift_mode = ?'); values.push(updates.drift_mode); }
|
||||
if (updates.classification !== undefined) { fields.push('classification = ?'); values.push(updates.classification); }
|
||||
if (updates.classification_reasons !== undefined) { fields.push('classification_reasons = ?'); values.push(JSON.stringify(updates.classification_reasons)); }
|
||||
if (updates.enabled !== undefined) { fields.push('enabled = ?'); values.push(updates.enabled ? 1 : 0); }
|
||||
if (updates.bumpRevision) { fields.push('revision = revision + 1'); }
|
||||
fields.push('updated_at = ?');
|
||||
values.push(Date.now());
|
||||
values.push(id);
|
||||
this.db.prepare(`UPDATE blueprints SET ${fields.join(', ')} WHERE id = ?`).run(...values);
|
||||
return this.getBlueprint(id);
|
||||
}
|
||||
|
||||
public deleteBlueprint(id: number): boolean {
|
||||
const result = this.db.prepare('DELETE FROM blueprints WHERE id = ?').run(id);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
// --- Blueprint Deployments ---
|
||||
|
||||
private parseBlueprintDeployment(row: Record<string, unknown>): BlueprintDeployment {
|
||||
return {
|
||||
id: row.id as number,
|
||||
blueprint_id: row.blueprint_id as number,
|
||||
node_id: row.node_id as number,
|
||||
status: row.status as BlueprintDeploymentStatus,
|
||||
applied_revision: (row.applied_revision as number | null) ?? null,
|
||||
last_deployed_at: (row.last_deployed_at as number | null) ?? null,
|
||||
last_checked_at: (row.last_checked_at as number | null) ?? null,
|
||||
last_drift_at: (row.last_drift_at as number | null) ?? null,
|
||||
drift_summary: (row.drift_summary as string | null) ?? null,
|
||||
last_error: (row.last_error as string | null) ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
public listDeployments(blueprintId: number): BlueprintDeployment[] {
|
||||
return this.db.prepare('SELECT * FROM blueprint_deployments WHERE blueprint_id = ? ORDER BY node_id')
|
||||
.all(blueprintId)
|
||||
.map(row => this.parseBlueprintDeployment(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public listAllDeployments(): BlueprintDeployment[] {
|
||||
return this.db.prepare('SELECT * FROM blueprint_deployments')
|
||||
.all()
|
||||
.map(row => this.parseBlueprintDeployment(row as Record<string, unknown>));
|
||||
}
|
||||
|
||||
public getDeployment(blueprintId: number, nodeId: number): BlueprintDeployment | undefined {
|
||||
const row = this.db.prepare('SELECT * FROM blueprint_deployments WHERE blueprint_id = ? AND node_id = ?').get(blueprintId, nodeId) as Record<string, unknown> | undefined;
|
||||
return row ? this.parseBlueprintDeployment(row) : undefined;
|
||||
}
|
||||
|
||||
public upsertDeployment(input: {
|
||||
blueprint_id: number;
|
||||
node_id: number;
|
||||
status: BlueprintDeploymentStatus;
|
||||
applied_revision?: number | null;
|
||||
last_deployed_at?: number | null;
|
||||
last_checked_at?: number | null;
|
||||
last_drift_at?: number | null;
|
||||
drift_summary?: string | null;
|
||||
last_error?: string | null;
|
||||
}): BlueprintDeployment {
|
||||
const existing = this.getDeployment(input.blueprint_id, input.node_id);
|
||||
if (existing) {
|
||||
const fields: string[] = ['status = ?'];
|
||||
const values: unknown[] = [input.status];
|
||||
if (input.applied_revision !== undefined) { fields.push('applied_revision = ?'); values.push(input.applied_revision); }
|
||||
if (input.last_deployed_at !== undefined) { fields.push('last_deployed_at = ?'); values.push(input.last_deployed_at); }
|
||||
if (input.last_checked_at !== undefined) { fields.push('last_checked_at = ?'); values.push(input.last_checked_at); }
|
||||
if (input.last_drift_at !== undefined) { fields.push('last_drift_at = ?'); values.push(input.last_drift_at); }
|
||||
if (input.drift_summary !== undefined) { fields.push('drift_summary = ?'); values.push(input.drift_summary); }
|
||||
if (input.last_error !== undefined) { fields.push('last_error = ?'); values.push(input.last_error); }
|
||||
values.push(input.blueprint_id, input.node_id);
|
||||
this.db.prepare(`UPDATE blueprint_deployments SET ${fields.join(', ')} WHERE blueprint_id = ? AND node_id = ?`).run(...values);
|
||||
} else {
|
||||
this.db.prepare(
|
||||
`INSERT INTO blueprint_deployments (blueprint_id, node_id, status, applied_revision, last_deployed_at, last_checked_at, last_drift_at, drift_summary, last_error)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
||||
).run(
|
||||
input.blueprint_id,
|
||||
input.node_id,
|
||||
input.status,
|
||||
input.applied_revision ?? null,
|
||||
input.last_deployed_at ?? null,
|
||||
input.last_checked_at ?? null,
|
||||
input.last_drift_at ?? null,
|
||||
input.drift_summary ?? null,
|
||||
input.last_error ?? null,
|
||||
);
|
||||
}
|
||||
const updated = this.getDeployment(input.blueprint_id, input.node_id);
|
||||
if (!updated) throw new Error('Failed to upsert deployment');
|
||||
return updated;
|
||||
}
|
||||
|
||||
public deleteDeployment(blueprintId: number, nodeId: number): void {
|
||||
this.db.prepare('DELETE FROM blueprint_deployments WHERE blueprint_id = ? AND node_id = ?').run(blueprintId, nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { DatabaseService, type BlueprintSelector, type Node } from './DatabaseService';
|
||||
|
||||
export const MAX_NODE_LABELS = 50;
|
||||
const LABEL_PATTERN = /^[a-z0-9][a-z0-9._-]*$/i;
|
||||
const MAX_LABEL_LENGTH = 40;
|
||||
|
||||
export interface NodeLabelValidationError {
|
||||
error: string;
|
||||
code: 'invalid_format' | 'too_long' | 'empty' | 'too_many';
|
||||
}
|
||||
|
||||
export class NodeLabelService {
|
||||
private static instance: NodeLabelService | null = null;
|
||||
|
||||
static getInstance(): NodeLabelService {
|
||||
if (!NodeLabelService.instance) {
|
||||
NodeLabelService.instance = new NodeLabelService();
|
||||
}
|
||||
return NodeLabelService.instance;
|
||||
}
|
||||
|
||||
private constructor() { /* singleton */ }
|
||||
|
||||
listAll(): Record<number, string[]> {
|
||||
return DatabaseService.getInstance().getNodeLabelsMap();
|
||||
}
|
||||
|
||||
listForNode(nodeId: number): string[] {
|
||||
return DatabaseService.getInstance().listNodeLabels(nodeId).map(r => r.label);
|
||||
}
|
||||
|
||||
listDistinct(): string[] {
|
||||
return DatabaseService.getInstance().listDistinctNodeLabels();
|
||||
}
|
||||
|
||||
validate(rawLabel: string): NodeLabelValidationError | null {
|
||||
const label = rawLabel.trim();
|
||||
if (!label) return { error: 'label must not be empty', code: 'empty' };
|
||||
if (label.length > MAX_LABEL_LENGTH) return { error: `label must be ${MAX_LABEL_LENGTH} characters or fewer`, code: 'too_long' };
|
||||
if (!LABEL_PATTERN.test(label)) return { error: 'label may contain letters, digits, dot, dash, underscore', code: 'invalid_format' };
|
||||
return null;
|
||||
}
|
||||
|
||||
addLabel(nodeId: number, rawLabel: string): { ok: true; label: string } | { ok: false; error: NodeLabelValidationError } {
|
||||
const validationError = this.validate(rawLabel);
|
||||
if (validationError) return { ok: false, error: validationError };
|
||||
const label = rawLabel.trim();
|
||||
const db = DatabaseService.getInstance();
|
||||
const existing = db.listNodeLabels(nodeId);
|
||||
if (existing.length >= MAX_NODE_LABELS && !existing.some(r => r.label === label)) {
|
||||
return { ok: false, error: { error: `nodes can have at most ${MAX_NODE_LABELS} labels`, code: 'too_many' } };
|
||||
}
|
||||
db.addNodeLabel(nodeId, label);
|
||||
return { ok: true, label };
|
||||
}
|
||||
|
||||
removeLabel(nodeId: number, label: string): boolean {
|
||||
return DatabaseService.getInstance().removeNodeLabel(nodeId, label);
|
||||
}
|
||||
|
||||
matchSelector(selector: BlueprintSelector, nodes: Node[]): Node[] {
|
||||
if (selector.type === 'nodes') {
|
||||
const ids = new Set(selector.ids);
|
||||
return nodes.filter(n => ids.has(n.id));
|
||||
}
|
||||
if (selector.type === 'labels') {
|
||||
const allRequired = (selector.all ?? []).filter(s => s.length > 0);
|
||||
const anyRequired = (selector.any ?? []).filter(s => s.length > 0);
|
||||
if (allRequired.length === 0 && anyRequired.length === 0) return [];
|
||||
const labelsByNode = DatabaseService.getInstance().getNodeLabelsMap();
|
||||
return nodes.filter(node => {
|
||||
const nodeLabels = new Set(labelsByNode[node.id] ?? []);
|
||||
if (allRequired.length > 0 && !allRequired.every(l => nodeLabels.has(l))) return false;
|
||||
if (anyRequired.length > 0 && !anyRequired.some(l => nodeLabels.has(l))) return false;
|
||||
return true;
|
||||
});
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -16,12 +16,19 @@ export type NotificationCategory =
|
||||
| 'autoheal_triggered'
|
||||
| 'monitor_alert'
|
||||
| 'scan_finding'
|
||||
| 'blueprint_deployed'
|
||||
| 'blueprint_deployment_failed'
|
||||
| 'blueprint_drift_detected'
|
||||
| 'blueprint_drift_correction_failed'
|
||||
| 'system';
|
||||
|
||||
export const ALL_NOTIFICATION_CATEGORIES: readonly NotificationCategory[] = [
|
||||
'deploy_success', 'deploy_failure', 'stack_started', 'stack_stopped',
|
||||
'stack_restarted', 'image_update_available', 'image_update_applied',
|
||||
'autoheal_triggered', 'monitor_alert', 'scan_finding', 'system',
|
||||
'autoheal_triggered', 'monitor_alert', 'scan_finding',
|
||||
'blueprint_deployed', 'blueprint_deployment_failed',
|
||||
'blueprint_drift_detected', 'blueprint_drift_correction_failed',
|
||||
'system',
|
||||
];
|
||||
|
||||
/** Webhook timeout: 10 seconds per external dispatch call. */
|
||||
|
||||
Reference in New Issue
Block a user