mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-18 06:23:18 +00:00
feat(stacks): persist a drift ledger with temporal source-change detection (#1333)
* feat(stacks): persist a drift ledger with temporal source-change detection Build on the read-only compose-vs-runtime drift check so a stack's drift is remembered over time, not just shown at a glance. - Record a deploy baseline: on a successful deploy, update, or rollback, store the deployed compose file's source and rendered-model hashes on the stack so the Drift tab can tell whether the file has changed since the last deploy. - Surface temporal drift in the Drift tab: "matches last deploy", "source changed since last deploy" (distinguishing a model change from a formatting-only edit), or "no deploy baseline yet". - Persist findings into a drift ledger: a re-check reconciles the current findings, recording newly detected ones and resolving cleared ones, and shows a short drift history under the findings. The drift report read stays side-effect-free; only an explicit re-check (and a deploy) writes the ledger. - Write drift detected/resolved events to the stack Activity timeline so the provenance sits alongside deploys and restarts. Node-local and available on the Community tier. Reconciliation is skipped when a check is not authoritative (Docker unreachable or a compose parse error) so an open finding is never falsely cleared. * fix(stacks): record the drift baseline for every deploy path and harden the ledger Address review feedback on the drift ledger: - Record the deploy baseline in ComposeService.deployStack/updateStack instead of only the manual route, so bulk, Git-source, App Store, scheduler, and webhook deploys all capture source/rendered hashes. Reconciliation stays on the explicit re-check. - Store no rendered baseline when the local parser cannot model the compose (for example a file over the parse cap) rather than a sentinel that would make a later real change read as unchanged. - Let temporal-overlay failures surface as a 500 instead of being hidden behind a neutral "no baseline"; only the compose read stays best-effort. - Omit the temporal card entirely when a report (for example from an older remote node) carries no temporal data, instead of showing a misleading "no baseline". - Keep drift_detected / drift_resolved history-only by excluding them from the routable-category whitelist, so they are never offered as a channel route that would never fire. - Use a JSON separator for the finding identity key so the source file is plain text (no embedded control byte). * fix(stacks): sanitize logged errors in the drift report handlers The drift report and re-check handlers logged the caught error object raw alongside the stack name, which a code scan flagged as a log-injection vector: a crafted stack name surfacing inside an error message or stack could forge log lines. Route the error through the log sanitizer so control characters are stripped before writing. Render it with util.inspect first so the stack trace, cause chain, and underlying error codes are preserved for debugging.
This commit is contained in:
Anso
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* Drift Ledger: the persistence layer on top of the read-only spatial engine.
|
||||
* Covers the deploy baseline + temporal comparison, the reconcile step that
|
||||
* records findings appearing and clearing (and the activity rows it writes), the
|
||||
* supporting database methods, and the POST re-check route that persists on demand.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import DockerController from '../services/DockerController';
|
||||
import type { StackDriftReport, StackDriftFinding, DriftFindingKind } from '../services/DriftDetectionService';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let DriftLedgerService: typeof import('../services/DriftLedgerService').DriftLedgerService;
|
||||
let computeStackHashes: typeof import('../services/DriftLedgerService').computeStackHashes;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let nodeId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ DriftLedgerService, computeStackHashes } = await import('../services/DriftLedgerService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
nodeId = (DatabaseService.getInstance().getDb().prepare('SELECT id FROM nodes WHERE is_default = 1').get() as { id: number }).id;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
function clearLedger(stack: string) {
|
||||
db().deleteStackDriftFindings(nodeId, stack);
|
||||
db().getDb().prepare('DELETE FROM notification_history WHERE node_id = ? AND stack_name = ?').run(nodeId, stack);
|
||||
db().deleteStackDossier(nodeId, stack);
|
||||
}
|
||||
|
||||
function finding(kind: DriftFindingKind, service: string, expected?: string, actual?: string): StackDriftFinding {
|
||||
return { kind, service, detail: `${service} ${kind}`, expected, actual };
|
||||
}
|
||||
|
||||
function reportWith(findings: StackDriftFinding[], over: Partial<StackDriftReport> = {}): StackDriftReport {
|
||||
return { stack: over.stack ?? 'web', status: findings.length ? 'drifted' : 'in-sync', hasComposeFile: true, hasContainers: true, findings, ...over };
|
||||
}
|
||||
|
||||
function driftActivity(stack: string) {
|
||||
return db().getStackActivity(nodeId, stack, { limit: 50 }).filter(e => e.category === 'drift_detected' || e.category === 'drift_resolved');
|
||||
}
|
||||
|
||||
describe('computeStackHashes', () => {
|
||||
it('is deterministic for identical content', () => {
|
||||
const a = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const b = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it('source hash differs but rendered hash matches when only comments/whitespace change', () => {
|
||||
const plain = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const commented = computeStackHashes('# a comment\nservices:\n web:\n image: nginx:1.27\n\n');
|
||||
expect(commented.sourceHash).not.toBe(plain.sourceHash);
|
||||
expect(commented.renderedHash).toBe(plain.renderedHash);
|
||||
});
|
||||
|
||||
it('rendered hash changes when the model changes', () => {
|
||||
const v1 = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const v2 = computeStackHashes('services:\n web:\n image: nginx:1.28\n');
|
||||
expect(v2.renderedHash).not.toBe(v1.renderedHash);
|
||||
});
|
||||
|
||||
it('returns a null rendered hash when the model cannot be parsed', () => {
|
||||
// No services => the local parser reports a parse error and cannot model it.
|
||||
const h = computeStackHashes('not_a_compose_key: true\n');
|
||||
expect(typeof h.sourceHash).toBe('string');
|
||||
expect(h.renderedHash).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('setStackDossierHashes', () => {
|
||||
beforeEach(() => clearLedger('hashstack'));
|
||||
|
||||
it('creates a dossier row with empty notes when none exists', () => {
|
||||
db().setStackDossierHashes(nodeId, 'hashstack', 'src1', 'rnd1');
|
||||
const row = db().getStackDossier(nodeId, 'hashstack');
|
||||
expect(row?.source_hash).toBe('src1');
|
||||
expect(row?.rendered_hash).toBe('rnd1');
|
||||
expect(row?.purpose).toBe('');
|
||||
});
|
||||
|
||||
it('preserves operator notes when updating hashes', () => {
|
||||
db().upsertStackDossier(nodeId, 'hashstack', {
|
||||
purpose: 'reverse proxy', owner: 'ops', access_urls: '', static_ip: '', vlan: '',
|
||||
firewall_notes: '', reverse_proxy_notes: '', backup_notes: '', upgrade_notes: '', recovery_notes: '', custom_notes: '',
|
||||
});
|
||||
db().setStackDossierHashes(nodeId, 'hashstack', 'src2', 'rnd2');
|
||||
const row = db().getStackDossier(nodeId, 'hashstack');
|
||||
expect(row?.purpose).toBe('reverse proxy');
|
||||
expect(row?.owner).toBe('ops');
|
||||
expect(row?.source_hash).toBe('src2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift finding store', () => {
|
||||
beforeEach(() => clearLedger('findstack'));
|
||||
|
||||
it('inserts open findings and resolves them', () => {
|
||||
const id = db().insertDriftFinding({
|
||||
node_id: nodeId, stack_name: 'findstack', service: 'web', finding_type: 'image-mismatch',
|
||||
severity: 'warning', message: 'm', expected_json: null, actual_json: null, detected_at: 1000,
|
||||
});
|
||||
expect(db().getOpenDriftFindings(nodeId, 'findstack')).toHaveLength(1);
|
||||
db().resolveDriftFinding(id, 2000);
|
||||
expect(db().getOpenDriftFindings(nodeId, 'findstack')).toHaveLength(0);
|
||||
const recent = db().getRecentDriftFindings(nodeId, 'findstack', 10);
|
||||
expect(recent).toHaveLength(1);
|
||||
expect(recent[0].resolved_at).toBe(2000);
|
||||
});
|
||||
|
||||
it('orders recent findings open-first, then resolved, each newest first', () => {
|
||||
const mk = (service: string, detected: number) => db().insertDriftFinding({
|
||||
node_id: nodeId, stack_name: 'findstack', service, finding_type: 'image-mismatch',
|
||||
severity: 'warning', message: service, expected_json: null, actual_json: null, detected_at: detected,
|
||||
});
|
||||
mk('a', 100);
|
||||
const b = mk('b', 300);
|
||||
mk('c', 200);
|
||||
db().resolveDriftFinding(b, 400); // b is the only resolved one (and the newest by detected_at)
|
||||
// Open (c@200, a@100 by detected_at DESC) come before the resolved b despite b being newest.
|
||||
expect(db().getRecentDriftFindings(nodeId, 'findstack', 10).map(r => r.service)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.computeTemporal', () => {
|
||||
beforeEach(() => clearLedger('tempstack'));
|
||||
const content = 'services:\n web:\n image: nginx:1.27\n';
|
||||
|
||||
it('reports no baseline before a deploy', () => {
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', content);
|
||||
expect(t).toEqual({ hasBaseline: false, sourceChanged: false, renderedChanged: false });
|
||||
});
|
||||
|
||||
it('reports a match when content is unchanged since baseline', () => {
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', sourceHash, renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', content);
|
||||
expect(t).toEqual({ hasBaseline: true, sourceChanged: false, renderedChanged: false });
|
||||
});
|
||||
|
||||
it('flags source and rendered changes after the file changes', () => {
|
||||
const baseline = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', baseline.sourceHash, baseline.renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', 'services:\n web:\n image: nginx:1.28\n');
|
||||
expect(t.hasBaseline).toBe(true);
|
||||
expect(t.sourceChanged).toBe(true);
|
||||
expect(t.renderedChanged).toBe(true);
|
||||
});
|
||||
|
||||
it('flags source changed but not rendered for a comments/whitespace-only edit', () => {
|
||||
const baseline = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', baseline.sourceHash, baseline.renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', `# a note\n${content}\n`);
|
||||
expect(t).toEqual({ hasBaseline: true, sourceChanged: true, renderedChanged: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.reconcile', () => {
|
||||
beforeEach(() => clearLedger('rec'));
|
||||
const ledger = () => DriftLedgerService.getInstance();
|
||||
|
||||
it('records newly detected findings and a single activity row', () => {
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web', 'nginx:1.27', 'nginx:1.26')], { stack: 'rec' }));
|
||||
expect(res).toEqual({ detected: 1, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
const activity = driftActivity('rec');
|
||||
expect(activity).toHaveLength(1);
|
||||
expect(activity[0].category).toBe('drift_detected');
|
||||
});
|
||||
|
||||
it('is idempotent: re-checking the same drift writes nothing new', () => {
|
||||
const report = reportWith([finding('image-mismatch', 'web')], { stack: 'rec' });
|
||||
ledger().reconcile(nodeId, 'rec', report);
|
||||
const res = ledger().reconcile(nodeId, 'rec', report);
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
expect(driftActivity('rec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('resolves a finding that has cleared and records a resolved activity row', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([], { stack: 'rec', status: 'in-sync' }));
|
||||
expect(res).toEqual({ detected: 0, resolved: 1 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(0);
|
||||
expect(driftActivity('rec').filter(e => e.category === 'drift_resolved')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records exactly one row per direction when one finding clears as another appears', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
// In a single check, 'web' clears while 'db' newly appears.
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([finding('service-missing', 'db')], { stack: 'rec' }));
|
||||
expect(res).toEqual({ detected: 1, resolved: 1 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
const cats = driftActivity('rec').map(e => e.category);
|
||||
expect(cats.filter(c => c === 'drift_detected')).toHaveLength(2); // first reconcile + this one
|
||||
expect(cats.filter(c => c === 'drift_resolved')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile an unreachable report (open findings are not falsely resolved)', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', { stack: 'rec', status: 'unreachable', hasComposeFile: true, hasContainers: false, findings: [] });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile a parse-error report', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', { stack: 'rec', status: 'drifted', hasComposeFile: false, hasContainers: false, findings: [], parseError: 'bad yaml' });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.recordBaseline', () => {
|
||||
beforeEach(() => clearLedger('baseline'));
|
||||
|
||||
it('hashes the on-disk compose and stores it as the dossier baseline', async () => {
|
||||
const stackDir = path.join(process.env.COMPOSE_DIR as string, 'baseline');
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
try {
|
||||
await DriftLedgerService.getInstance().recordBaseline(nodeId, 'baseline');
|
||||
const row = db().getStackDossier(nodeId, 'baseline');
|
||||
const expected = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
expect(row?.source_hash).toBe(expected.sourceHash);
|
||||
expect(row?.rendered_hash).toBe(expected.renderedHash);
|
||||
} finally {
|
||||
fs.rmSync(stackDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift route (GET read-only, POST recheck persists)', () => {
|
||||
const STACK = 'recheckroute';
|
||||
let stackDir: string;
|
||||
|
||||
// A running container on a different image than compose declares => image-mismatch.
|
||||
const stubDriftedDocker = () => vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{
|
||||
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
|
||||
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
}],
|
||||
networks: [], volumes: [],
|
||||
}),
|
||||
} as unknown as DockerController);
|
||||
|
||||
beforeEach(() => {
|
||||
clearLedger(STACK);
|
||||
stackDir = path.join(process.env.COMPOSE_DIR as string, STACK);
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(stackDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('GET reports drift without writing the ledger or activity timeline', async () => {
|
||||
stubDriftedDocker();
|
||||
const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('drifted');
|
||||
expect(res.body.temporal).toBeDefined();
|
||||
// A passive read must not persist anything.
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK)).toHaveLength(0);
|
||||
expect(driftActivity(STACK)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('POST recheck persists the current drift and returns temporal + ledger', async () => {
|
||||
stubDriftedDocker();
|
||||
const res = await request(app).post(`/api/stacks/${STACK}/drift/recheck`).set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('drifted');
|
||||
expect(res.body.temporal).toBeDefined();
|
||||
expect(Array.isArray(res.body.ledger)).toBe(true);
|
||||
expect(res.body.ledger).toHaveLength(1);
|
||||
expect(res.body.ledger[0]).toMatchObject({ service: 'web', kind: 'image-mismatch', resolvedAt: null });
|
||||
|
||||
// The transition was recorded exactly once in the activity timeline.
|
||||
const acts = driftActivity(STACK);
|
||||
expect(acts).toHaveLength(1);
|
||||
expect(acts[0].category).toBe('drift_detected');
|
||||
// And persisted as an open finding.
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import { z } from 'zod';
|
||||
import path from 'path';
|
||||
import { inspect } from 'node:util';
|
||||
import YAML from 'yaml';
|
||||
import multer from 'multer';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
@@ -12,7 +13,8 @@ import { CacheService } from '../services/CacheService';
|
||||
import { UpdatePreviewService } from '../services/UpdatePreviewService';
|
||||
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
|
||||
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
|
||||
import { buildStackDriftReport } from '../services/DriftDetectionService';
|
||||
import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService';
|
||||
import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService';
|
||||
import { requirePermission, checkPermission } from '../middleware/permissions';
|
||||
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
|
||||
import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService';
|
||||
@@ -939,6 +941,7 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
|
||||
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
|
||||
DatabaseService.getInstance().deleteGitSource(stackName);
|
||||
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName });
|
||||
} catch (dbErr) {
|
||||
console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr);
|
||||
@@ -1009,18 +1012,80 @@ stacksRouter.get('/:stackName/services', async (req: Request, res: Response) =>
|
||||
}
|
||||
});
|
||||
|
||||
/** A persisted finding as the Drift tab consumes it (no internal column names). */
|
||||
interface DriftLedgerEntry {
|
||||
service: string;
|
||||
kind: DriftFindingKind;
|
||||
message: string;
|
||||
detectedAt: number;
|
||||
resolvedAt: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the full Drift tab payload: the spatial report (compose vs runtime),
|
||||
* the temporal overlay (source changed since last deploy), and the persisted
|
||||
* ledger history. When `reconcile` is set, the current findings are persisted
|
||||
* into the ledger (new ones recorded, cleared ones resolved) before the ledger
|
||||
* is read back, so the returned history reflects the just-observed state.
|
||||
*/
|
||||
async function buildDriftPayload(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
reconcile: boolean,
|
||||
): Promise<StackDriftReport & { temporal: DriftTemporal; ledger: DriftLedgerEntry[] }> {
|
||||
const report = await buildStackDriftReport(nodeId, stackName);
|
||||
// Only the on-disk read is best-effort: an unreadable compose is already surfaced
|
||||
// by the report as a parse error, so temporal degrades to neutral. computeTemporal
|
||||
// runs outside the try so a real ledger fault (a DB or hashing error) surfaces as a
|
||||
// 500 instead of being hidden behind a misleading "no baseline".
|
||||
let content: string | null = null;
|
||||
try {
|
||||
content = await FileSystemService.getInstance(nodeId).getStackContent(stackName);
|
||||
} catch {
|
||||
// Unreadable compose: the report carries the parseError; temporal stays neutral.
|
||||
}
|
||||
const temporal: DriftTemporal = content !== null
|
||||
? DriftLedgerService.getInstance().computeTemporal(nodeId, stackName, content)
|
||||
: { hasBaseline: false, sourceChanged: false, renderedChanged: false };
|
||||
if (reconcile) {
|
||||
DriftLedgerService.getInstance().reconcile(nodeId, stackName, report);
|
||||
}
|
||||
// finding_type is a free-text column, but reconcile only ever writes a DriftFindingKind.
|
||||
const ledger: DriftLedgerEntry[] = DatabaseService.getInstance()
|
||||
.getRecentDriftFindings(nodeId, stackName, 20)
|
||||
.map(r => ({ service: r.service, kind: r.finding_type as DriftFindingKind, message: r.message, detectedAt: r.detected_at, resolvedAt: r.resolved_at }));
|
||||
return { ...report, temporal, ledger };
|
||||
}
|
||||
|
||||
stacksRouter.get('/:stackName/drift', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
const report = await buildStackDriftReport(req.nodeId, stackName);
|
||||
res.json(report);
|
||||
res.json(await buildDriftPayload(req.nodeId, stackName, false));
|
||||
} catch (error) {
|
||||
console.error('[Stacks] Failed to build drift report for %s:', sanitizeForLog(stackName), error);
|
||||
console.error('[Stacks] Failed to build drift report for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(inspect(error, { depth: 4 })));
|
||||
res.status(500).json({ error: 'Failed to build drift report' });
|
||||
}
|
||||
});
|
||||
|
||||
// Re-check is the one place a passive drift view becomes a ledger write: it
|
||||
// reconciles the current findings into stack_drift_findings (recording newly
|
||||
// detected and newly resolved ones) before returning the fresh payload, so the
|
||||
// GET above can stay a side-effect-free read.
|
||||
stacksRouter.post('/:stackName/drift/recheck', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
res.json(await buildDriftPayload(req.nodeId, stackName, true));
|
||||
} catch (error) {
|
||||
console.error('[Stacks] Failed to re-check drift for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(inspect(error, { depth: 4 })));
|
||||
res.status(500).json({ error: 'Failed to re-check drift' });
|
||||
}
|
||||
});
|
||||
|
||||
stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
|
||||
@@ -10,6 +10,7 @@ import { MeshService } from './MeshService';
|
||||
import { LogFormatter } from './LogFormatter';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { RegistryService } from './RegistryService';
|
||||
import { DriftLedgerService } from './DriftLedgerService';
|
||||
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
@@ -359,6 +360,11 @@ export class ComposeService {
|
||||
}
|
||||
throw deployError;
|
||||
}
|
||||
// Reached only on a successful deploy (the catch above always rethrows). Record
|
||||
// the drift baseline here so every deploy path gets one, not just the manual
|
||||
// route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel
|
||||
// through this method. Internally guarded; awaited so it cannot race later work.
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
}
|
||||
|
||||
streamLogs(stackName: string, ws: WebSocket) {
|
||||
@@ -554,6 +560,9 @@ export class ComposeService {
|
||||
}
|
||||
throw updateError;
|
||||
}
|
||||
// Reached only on a successful update; re-baseline so temporal drift compares
|
||||
// against what is now deployed (see deployStack for why this lives here).
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
}
|
||||
|
||||
public async downStack(stackName: string): Promise<void> {
|
||||
|
||||
@@ -85,10 +85,29 @@ export interface StackDossier extends StackDossierFields {
|
||||
id?: number;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
/** SHA-256 of the compose file's UTF-8 text at the last deploy through Sencho (baseline for temporal drift). */
|
||||
source_hash?: string | null;
|
||||
/** SHA-256 of the parsed compose model at the last deploy (ignores comments/whitespace). */
|
||||
rendered_hash?: string | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
/** A persisted drift finding: one service-scoped divergence, open until resolved. */
|
||||
export interface StackDriftFindingRow {
|
||||
id: number;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
service: string;
|
||||
finding_type: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
expected_json: string | null;
|
||||
actual_json: string | null;
|
||||
detected_at: number;
|
||||
resolved_at: number | null;
|
||||
}
|
||||
|
||||
export interface Node {
|
||||
id: number;
|
||||
name: string;
|
||||
@@ -697,6 +716,7 @@ export class DatabaseService {
|
||||
this.migrateAddBlueprintPinnedNode();
|
||||
this.migrateAutoHealNodeId();
|
||||
this.migrateFleetSyncStickyError();
|
||||
this.migrateStackDossierHashes();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1175,6 +1195,22 @@ export class DatabaseService {
|
||||
UNIQUE(node_id, stack_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_drift_findings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
service TEXT NOT NULL,
|
||||
finding_type TEXT NOT NULL,
|
||||
severity TEXT NOT NULL DEFAULT 'warning',
|
||||
message TEXT NOT NULL,
|
||||
expected_json TEXT,
|
||||
actual_json TEXT,
|
||||
detected_at INTEGER NOT NULL,
|
||||
resolved_at INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_drift_findings_open
|
||||
ON stack_drift_findings(node_id, stack_name, resolved_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -1473,6 +1509,11 @@ export class DatabaseService {
|
||||
this.tryAddColumn('notification_history', 'container_name', 'TEXT');
|
||||
}
|
||||
|
||||
private migrateStackDossierHashes(): void {
|
||||
this.tryAddColumn('stack_dossiers', 'source_hash', 'TEXT');
|
||||
this.tryAddColumn('stack_dossiers', 'rendered_hash', 'TEXT');
|
||||
}
|
||||
|
||||
private migrateScanPolicyFleetColumns(): void {
|
||||
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
|
||||
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
|
||||
@@ -2109,6 +2150,55 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM stack_dossiers WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the deploy-time baseline hashes for a stack. Creates a dossier row
|
||||
* with empty operator notes if none exists; on conflict updates only the hash
|
||||
* columns so operator-authored notes and their updated_at are left untouched.
|
||||
*/
|
||||
public setStackDossierHashes(nodeId: number, stackName: string, sourceHash: string, renderedHash: string | null): void {
|
||||
const now = Date.now();
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_dossiers (node_id, stack_name, source_hash, rendered_hash, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(node_id, stack_name) DO UPDATE SET
|
||||
source_hash = excluded.source_hash,
|
||||
rendered_hash = excluded.rendered_hash`
|
||||
).run(nodeId, stackName, sourceHash, renderedHash, now, now);
|
||||
}
|
||||
|
||||
// --- Stack Drift Findings (the persisted drift ledger) ---
|
||||
|
||||
public insertDriftFinding(f: Omit<StackDriftFindingRow, 'id' | 'resolved_at'>): number {
|
||||
const res = this.db.prepare(
|
||||
`INSERT INTO stack_drift_findings
|
||||
(node_id, stack_name, service, finding_type, severity, message, expected_json, actual_json, detected_at, resolved_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`
|
||||
).run(f.node_id, f.stack_name, f.service, f.finding_type, f.severity, f.message, f.expected_json, f.actual_json, f.detected_at);
|
||||
return res.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
public resolveDriftFinding(id: number, resolvedAt: number): void {
|
||||
this.db.prepare('UPDATE stack_drift_findings SET resolved_at = ? WHERE id = ? AND resolved_at IS NULL').run(resolvedAt, id);
|
||||
}
|
||||
|
||||
/** Open (unresolved) findings for a stack, oldest first. */
|
||||
public getOpenDriftFindings(nodeId: number, stackName: string): StackDriftFindingRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_drift_findings WHERE node_id = ? AND stack_name = ? AND resolved_at IS NULL ORDER BY detected_at ASC, id ASC'
|
||||
).all(nodeId, stackName) as StackDriftFindingRow[];
|
||||
}
|
||||
|
||||
/** Recent findings for a stack: open ones first, then resolved, each newest first. */
|
||||
public getRecentDriftFindings(nodeId: number, stackName: string, limit: number): StackDriftFindingRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_drift_findings WHERE node_id = ? AND stack_name = ? ORDER BY (resolved_at IS NOT NULL) ASC, detected_at DESC, id DESC LIMIT ?'
|
||||
).all(nodeId, stackName, limit) as StackDriftFindingRow[];
|
||||
}
|
||||
|
||||
public deleteStackDriftFindings(nodeId: number, stackName: string): void {
|
||||
this.db.prepare('DELETE FROM stack_drift_findings WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
private mapNotificationRow(row: any): NotificationHistory {
|
||||
@@ -2446,6 +2536,7 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM stack_label_assignments WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_labels WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_dossiers WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_drift_findings WHERE node_id = ?').run(id);
|
||||
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
|
||||
this.deleteRoleAssignmentsByResource('node', String(id));
|
||||
this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id);
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import type { StackDriftFindingRow } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { parseComposeDependencies } from '../helpers/composeDependencyParse';
|
||||
import type { DeclaredCompose } from '../helpers/composeDependencyParse';
|
||||
import { sha256Hex } from '../utils/hashing';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import type { StackDriftReport, StackDriftFinding } from './DriftDetectionService';
|
||||
|
||||
/**
|
||||
* The persistence-backed Drift Ledger that builds on the read-only spatial
|
||||
* engine (DriftDetectionService). It adds the two things the engine deliberately
|
||||
* leaves out: a deploy-time baseline (so "the source changed since you deployed"
|
||||
* can be answered) and a persisted history of findings (so drift can be seen to
|
||||
* appear and resolve over time). Node-local: it reads and writes the database of
|
||||
* whichever node owns the stack.
|
||||
*/
|
||||
|
||||
/** Temporal alignment of the on-disk compose against the last deploy baseline. */
|
||||
export interface DriftTemporal {
|
||||
/** True once a deploy through Sencho has recorded baseline hashes. */
|
||||
hasBaseline: boolean;
|
||||
/** The compose file's text differs from the last deploy. */
|
||||
sourceChanged: boolean;
|
||||
/** Parsed compose model differs from the last deploy (ignores comments/whitespace). */
|
||||
renderedChanged: boolean;
|
||||
}
|
||||
|
||||
export interface DriftReconcileResult {
|
||||
detected: number;
|
||||
resolved: number;
|
||||
}
|
||||
|
||||
/** Stable identity for a finding across checks: same service + kind is the same finding. */
|
||||
function findingKey(service: string, kind: string): string {
|
||||
return JSON.stringify([service, kind]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order-independent serialization of the parsed model so two compose files that
|
||||
* differ only in comments, whitespace, or key order hash equal, while a real
|
||||
* change to images/ports/services/networks/volumes changes the hash. Returns null
|
||||
* when the local parser cannot produce a model (for example a file over the parse
|
||||
* size cap), so the caller stores no rendered baseline rather than a sentinel that
|
||||
* would make a later real change read as unchanged.
|
||||
*/
|
||||
function stableModelString(model: DeclaredCompose): string | null {
|
||||
if (model.parseError) return null;
|
||||
const services = [...model.services]
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map(s => ({
|
||||
name: s.name,
|
||||
image: s.image ?? null,
|
||||
dependsOn: [...s.dependsOn].sort(),
|
||||
networks: [...s.networks].sort(),
|
||||
volumes: [...s.volumes].sort(),
|
||||
ports: s.ports.map(p => `${p.hostIp}:${p.publishedPort}/${p.protocol}`).sort(),
|
||||
}));
|
||||
const networks = Object.keys(model.networks).sort().map(k => ({ key: k, ...model.networks[k] }));
|
||||
const volumes = Object.keys(model.volumes).sort().map(k => ({ key: k, ...model.volumes[k] }));
|
||||
return JSON.stringify({ services, networks, volumes });
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes a compose file two ways: the raw text (source) and the parsed model
|
||||
* (rendered). renderedHash is null when the model cannot be parsed, so a
|
||||
* model-level comparison is simply skipped rather than forced to a false equal.
|
||||
*/
|
||||
export function computeStackHashes(content: string): { sourceHash: string; renderedHash: string | null } {
|
||||
const model = stableModelString(parseComposeDependencies(content));
|
||||
return {
|
||||
sourceHash: sha256Hex(content),
|
||||
renderedHash: model === null ? null : sha256Hex(model),
|
||||
};
|
||||
}
|
||||
|
||||
export class DriftLedgerService {
|
||||
private static instance: DriftLedgerService | null = null;
|
||||
|
||||
static getInstance(): DriftLedgerService {
|
||||
if (!DriftLedgerService.instance) DriftLedgerService.instance = new DriftLedgerService();
|
||||
return DriftLedgerService.instance;
|
||||
}
|
||||
|
||||
private constructor() { /* singleton */ }
|
||||
|
||||
/**
|
||||
* Record the deploy-time baseline hashes for a stack. Best-effort: a read or
|
||||
* hash failure is logged and swallowed so it never fails the deploy that
|
||||
* triggered it.
|
||||
*/
|
||||
async recordBaseline(nodeId: number, stackName: string): Promise<void> {
|
||||
try {
|
||||
const content = await FileSystemService.getInstance(nodeId).getStackContent(stackName);
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
DatabaseService.getInstance().setStackDossierHashes(nodeId, stackName, sourceHash, renderedHash);
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] Failed to record baseline for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
|
||||
/** Compare the current compose content against the stored deploy baseline. */
|
||||
computeTemporal(nodeId: number, stackName: string, content: string): DriftTemporal {
|
||||
const dossier = DatabaseService.getInstance().getStackDossier(nodeId, stackName);
|
||||
const storedSource = dossier?.source_hash ?? null;
|
||||
const storedRendered = dossier?.rendered_hash ?? null;
|
||||
if (!storedSource) {
|
||||
return { hasBaseline: false, sourceChanged: false, renderedChanged: false };
|
||||
}
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
return {
|
||||
hasBaseline: true,
|
||||
sourceChanged: sourceHash !== storedSource,
|
||||
// Only a real model change counts; if either side has no parseable model, skip it.
|
||||
renderedChanged: storedRendered != null && renderedHash != null && renderedHash !== storedRendered,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile the spatial report's current findings against the persisted ledger:
|
||||
* insert findings that are newly seen, resolve ones that have cleared. Idempotent
|
||||
* (a repeat check with no change writes nothing). Skipped when the report is not
|
||||
* authoritative (Docker unreachable or a compose parse error) so open findings are
|
||||
* never falsely resolved. On a real transition it records one summary activity row
|
||||
* per direction so the stack Activity timeline shows drift appearing and clearing.
|
||||
*/
|
||||
reconcile(nodeId: number, stackName: string, report: StackDriftReport): DriftReconcileResult {
|
||||
if (report.status === 'unreachable' || report.parseError) {
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const openByKey = new Map(db.getOpenDriftFindings(nodeId, stackName).map(r => [findingKey(r.service, r.finding_type), r]));
|
||||
const currentByKey = new Map(report.findings.map(f => [findingKey(f.service, f.kind), f]));
|
||||
|
||||
const toInsert: StackDriftFinding[] = [];
|
||||
for (const [key, f] of currentByKey) {
|
||||
if (!openByKey.has(key)) toInsert.push(f);
|
||||
}
|
||||
const toResolve: StackDriftFindingRow[] = [];
|
||||
for (const [key, row] of openByKey) {
|
||||
if (!currentByKey.has(key)) toResolve.push(row);
|
||||
}
|
||||
if (toInsert.length === 0 && toResolve.length === 0) {
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
db.getDb().transaction(() => {
|
||||
for (const f of toInsert) {
|
||||
db.insertDriftFinding({
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
service: f.service,
|
||||
finding_type: f.kind,
|
||||
severity: 'warning',
|
||||
message: f.detail,
|
||||
expected_json: f.expected !== undefined ? JSON.stringify(f.expected) : null,
|
||||
actual_json: f.actual !== undefined ? JSON.stringify(f.actual) : null,
|
||||
detected_at: now,
|
||||
});
|
||||
}
|
||||
for (const row of toResolve) {
|
||||
db.resolveDriftFinding(row.id, now);
|
||||
}
|
||||
})();
|
||||
|
||||
if (toInsert.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_detected', 'warning',
|
||||
`Drift detected on ${stackName}: ${toInsert.length} new finding${toInsert.length === 1 ? '' : 's'}`, now);
|
||||
}
|
||||
if (toResolve.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_resolved', 'info',
|
||||
`Drift resolved on ${stackName}: ${toResolve.length} finding${toResolve.length === 1 ? '' : 's'} cleared`, now);
|
||||
}
|
||||
return { detected: toInsert.length, resolved: toResolve.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a drift transition to the stack activity timeline. History-only (no
|
||||
* external channel dispatch): a drift signal belongs in the activity feed, not
|
||||
* in every configured Discord/Slack webhook.
|
||||
*/
|
||||
private recordActivity(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
category: 'drift_detected' | 'drift_resolved',
|
||||
level: 'info' | 'warning',
|
||||
message: string,
|
||||
timestamp: number,
|
||||
): void {
|
||||
try {
|
||||
DatabaseService.getInstance().addNotificationHistory(nodeId, {
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
timestamp,
|
||||
stack_name: stackName,
|
||||
actor_username: null,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] Failed to record activity for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,11 @@ export type NotificationCategory =
|
||||
| 'blueprint_deployment_failed'
|
||||
| 'blueprint_drift_detected'
|
||||
| 'blueprint_drift_correction_failed'
|
||||
// Stack drift ledger transitions. Written to history only (the Activity
|
||||
// timeline), never dispatched to channels, so they are deliberately excluded
|
||||
// from ALL_NOTIFICATION_CATEGORIES (the routable-category whitelist) below.
|
||||
| 'drift_detected'
|
||||
| 'drift_resolved'
|
||||
| 'system';
|
||||
|
||||
export const ALL_NOTIFICATION_CATEGORIES: readonly NotificationCategory[] = [
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
/** Hex-encoded SHA-256 of a UTF-8 string. Stable across runs and platforms. */
|
||||
export function sha256Hex(content: string): string {
|
||||
return createHash('sha256').update(content, 'utf8').digest('hex');
|
||||
}
|
||||
Reference in New Issue
Block a user