mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-08-04 07:57:54 +00:00
feat(stacks): persist a drift ledger with temporal source-change detection (#1333)
* feat(stacks): persist a drift ledger with temporal source-change detection Build on the read-only compose-vs-runtime drift check so a stack's drift is remembered over time, not just shown at a glance. - Record a deploy baseline: on a successful deploy, update, or rollback, store the deployed compose file's source and rendered-model hashes on the stack so the Drift tab can tell whether the file has changed since the last deploy. - Surface temporal drift in the Drift tab: "matches last deploy", "source changed since last deploy" (distinguishing a model change from a formatting-only edit), or "no deploy baseline yet". - Persist findings into a drift ledger: a re-check reconciles the current findings, recording newly detected ones and resolving cleared ones, and shows a short drift history under the findings. The drift report read stays side-effect-free; only an explicit re-check (and a deploy) writes the ledger. - Write drift detected/resolved events to the stack Activity timeline so the provenance sits alongside deploys and restarts. Node-local and available on the Community tier. Reconciliation is skipped when a check is not authoritative (Docker unreachable or a compose parse error) so an open finding is never falsely cleared. * fix(stacks): record the drift baseline for every deploy path and harden the ledger Address review feedback on the drift ledger: - Record the deploy baseline in ComposeService.deployStack/updateStack instead of only the manual route, so bulk, Git-source, App Store, scheduler, and webhook deploys all capture source/rendered hashes. Reconciliation stays on the explicit re-check. - Store no rendered baseline when the local parser cannot model the compose (for example a file over the parse cap) rather than a sentinel that would make a later real change read as unchanged. - Let temporal-overlay failures surface as a 500 instead of being hidden behind a neutral "no baseline"; only the compose read stays best-effort. - Omit the temporal card entirely when a report (for example from an older remote node) carries no temporal data, instead of showing a misleading "no baseline". - Keep drift_detected / drift_resolved history-only by excluding them from the routable-category whitelist, so they are never offered as a channel route that would never fire. - Use a JSON separator for the finding identity key so the source file is plain text (no embedded control byte). * fix(stacks): sanitize logged errors in the drift report handlers The drift report and re-check handlers logged the caught error object raw alongside the stack name, which a code scan flagged as a log-injection vector: a crafted stack name surfacing inside an error message or stack could forge log lines. Route the error through the log sanitizer so control characters are stripped before writing. Render it with util.inspect first so the stack trace, cause chain, and underlying error codes are preserved for debugging.
This commit is contained in:
Anso
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* Drift Ledger: the persistence layer on top of the read-only spatial engine.
|
||||
* Covers the deploy baseline + temporal comparison, the reconcile step that
|
||||
* records findings appearing and clearing (and the activity rows it writes), the
|
||||
* supporting database methods, and the POST re-check route that persists on demand.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import request from 'supertest';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
|
||||
import DockerController from '../services/DockerController';
|
||||
import type { StackDriftReport, StackDriftFinding, DriftFindingKind } from '../services/DriftDetectionService';
|
||||
|
||||
let tmpDir: string;
|
||||
let app: import('express').Express;
|
||||
let authHeader: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let DriftLedgerService: typeof import('../services/DriftLedgerService').DriftLedgerService;
|
||||
let computeStackHashes: typeof import('../services/DriftLedgerService').computeStackHashes;
|
||||
let LicenseService: typeof import('../services/LicenseService').LicenseService;
|
||||
let nodeId: number;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ app } = await import('../index'));
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ DriftLedgerService, computeStackHashes } = await import('../services/DriftLedgerService'));
|
||||
({ LicenseService } = await import('../services/LicenseService'));
|
||||
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '5m' });
|
||||
authHeader = `Bearer ${token}`;
|
||||
nodeId = (DatabaseService.getInstance().getDb().prepare('SELECT id FROM nodes WHERE is_default = 1').get() as { id: number }).id;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
cleanupTestDb(tmpDir);
|
||||
});
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
function clearLedger(stack: string) {
|
||||
db().deleteStackDriftFindings(nodeId, stack);
|
||||
db().getDb().prepare('DELETE FROM notification_history WHERE node_id = ? AND stack_name = ?').run(nodeId, stack);
|
||||
db().deleteStackDossier(nodeId, stack);
|
||||
}
|
||||
|
||||
function finding(kind: DriftFindingKind, service: string, expected?: string, actual?: string): StackDriftFinding {
|
||||
return { kind, service, detail: `${service} ${kind}`, expected, actual };
|
||||
}
|
||||
|
||||
function reportWith(findings: StackDriftFinding[], over: Partial<StackDriftReport> = {}): StackDriftReport {
|
||||
return { stack: over.stack ?? 'web', status: findings.length ? 'drifted' : 'in-sync', hasComposeFile: true, hasContainers: true, findings, ...over };
|
||||
}
|
||||
|
||||
function driftActivity(stack: string) {
|
||||
return db().getStackActivity(nodeId, stack, { limit: 50 }).filter(e => e.category === 'drift_detected' || e.category === 'drift_resolved');
|
||||
}
|
||||
|
||||
describe('computeStackHashes', () => {
|
||||
it('is deterministic for identical content', () => {
|
||||
const a = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const b = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
expect(a).toEqual(b);
|
||||
});
|
||||
|
||||
it('source hash differs but rendered hash matches when only comments/whitespace change', () => {
|
||||
const plain = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const commented = computeStackHashes('# a comment\nservices:\n web:\n image: nginx:1.27\n\n');
|
||||
expect(commented.sourceHash).not.toBe(plain.sourceHash);
|
||||
expect(commented.renderedHash).toBe(plain.renderedHash);
|
||||
});
|
||||
|
||||
it('rendered hash changes when the model changes', () => {
|
||||
const v1 = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
const v2 = computeStackHashes('services:\n web:\n image: nginx:1.28\n');
|
||||
expect(v2.renderedHash).not.toBe(v1.renderedHash);
|
||||
});
|
||||
|
||||
it('returns a null rendered hash when the model cannot be parsed', () => {
|
||||
// No services => the local parser reports a parse error and cannot model it.
|
||||
const h = computeStackHashes('not_a_compose_key: true\n');
|
||||
expect(typeof h.sourceHash).toBe('string');
|
||||
expect(h.renderedHash).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('setStackDossierHashes', () => {
|
||||
beforeEach(() => clearLedger('hashstack'));
|
||||
|
||||
it('creates a dossier row with empty notes when none exists', () => {
|
||||
db().setStackDossierHashes(nodeId, 'hashstack', 'src1', 'rnd1');
|
||||
const row = db().getStackDossier(nodeId, 'hashstack');
|
||||
expect(row?.source_hash).toBe('src1');
|
||||
expect(row?.rendered_hash).toBe('rnd1');
|
||||
expect(row?.purpose).toBe('');
|
||||
});
|
||||
|
||||
it('preserves operator notes when updating hashes', () => {
|
||||
db().upsertStackDossier(nodeId, 'hashstack', {
|
||||
purpose: 'reverse proxy', owner: 'ops', access_urls: '', static_ip: '', vlan: '',
|
||||
firewall_notes: '', reverse_proxy_notes: '', backup_notes: '', upgrade_notes: '', recovery_notes: '', custom_notes: '',
|
||||
});
|
||||
db().setStackDossierHashes(nodeId, 'hashstack', 'src2', 'rnd2');
|
||||
const row = db().getStackDossier(nodeId, 'hashstack');
|
||||
expect(row?.purpose).toBe('reverse proxy');
|
||||
expect(row?.owner).toBe('ops');
|
||||
expect(row?.source_hash).toBe('src2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift finding store', () => {
|
||||
beforeEach(() => clearLedger('findstack'));
|
||||
|
||||
it('inserts open findings and resolves them', () => {
|
||||
const id = db().insertDriftFinding({
|
||||
node_id: nodeId, stack_name: 'findstack', service: 'web', finding_type: 'image-mismatch',
|
||||
severity: 'warning', message: 'm', expected_json: null, actual_json: null, detected_at: 1000,
|
||||
});
|
||||
expect(db().getOpenDriftFindings(nodeId, 'findstack')).toHaveLength(1);
|
||||
db().resolveDriftFinding(id, 2000);
|
||||
expect(db().getOpenDriftFindings(nodeId, 'findstack')).toHaveLength(0);
|
||||
const recent = db().getRecentDriftFindings(nodeId, 'findstack', 10);
|
||||
expect(recent).toHaveLength(1);
|
||||
expect(recent[0].resolved_at).toBe(2000);
|
||||
});
|
||||
|
||||
it('orders recent findings open-first, then resolved, each newest first', () => {
|
||||
const mk = (service: string, detected: number) => db().insertDriftFinding({
|
||||
node_id: nodeId, stack_name: 'findstack', service, finding_type: 'image-mismatch',
|
||||
severity: 'warning', message: service, expected_json: null, actual_json: null, detected_at: detected,
|
||||
});
|
||||
mk('a', 100);
|
||||
const b = mk('b', 300);
|
||||
mk('c', 200);
|
||||
db().resolveDriftFinding(b, 400); // b is the only resolved one (and the newest by detected_at)
|
||||
// Open (c@200, a@100 by detected_at DESC) come before the resolved b despite b being newest.
|
||||
expect(db().getRecentDriftFindings(nodeId, 'findstack', 10).map(r => r.service)).toEqual(['c', 'a', 'b']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.computeTemporal', () => {
|
||||
beforeEach(() => clearLedger('tempstack'));
|
||||
const content = 'services:\n web:\n image: nginx:1.27\n';
|
||||
|
||||
it('reports no baseline before a deploy', () => {
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', content);
|
||||
expect(t).toEqual({ hasBaseline: false, sourceChanged: false, renderedChanged: false });
|
||||
});
|
||||
|
||||
it('reports a match when content is unchanged since baseline', () => {
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', sourceHash, renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', content);
|
||||
expect(t).toEqual({ hasBaseline: true, sourceChanged: false, renderedChanged: false });
|
||||
});
|
||||
|
||||
it('flags source and rendered changes after the file changes', () => {
|
||||
const baseline = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', baseline.sourceHash, baseline.renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', 'services:\n web:\n image: nginx:1.28\n');
|
||||
expect(t.hasBaseline).toBe(true);
|
||||
expect(t.sourceChanged).toBe(true);
|
||||
expect(t.renderedChanged).toBe(true);
|
||||
});
|
||||
|
||||
it('flags source changed but not rendered for a comments/whitespace-only edit', () => {
|
||||
const baseline = computeStackHashes(content);
|
||||
db().setStackDossierHashes(nodeId, 'tempstack', baseline.sourceHash, baseline.renderedHash);
|
||||
const t = DriftLedgerService.getInstance().computeTemporal(nodeId, 'tempstack', `# a note\n${content}\n`);
|
||||
expect(t).toEqual({ hasBaseline: true, sourceChanged: true, renderedChanged: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.reconcile', () => {
|
||||
beforeEach(() => clearLedger('rec'));
|
||||
const ledger = () => DriftLedgerService.getInstance();
|
||||
|
||||
it('records newly detected findings and a single activity row', () => {
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web', 'nginx:1.27', 'nginx:1.26')], { stack: 'rec' }));
|
||||
expect(res).toEqual({ detected: 1, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
const activity = driftActivity('rec');
|
||||
expect(activity).toHaveLength(1);
|
||||
expect(activity[0].category).toBe('drift_detected');
|
||||
});
|
||||
|
||||
it('is idempotent: re-checking the same drift writes nothing new', () => {
|
||||
const report = reportWith([finding('image-mismatch', 'web')], { stack: 'rec' });
|
||||
ledger().reconcile(nodeId, 'rec', report);
|
||||
const res = ledger().reconcile(nodeId, 'rec', report);
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
expect(driftActivity('rec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('resolves a finding that has cleared and records a resolved activity row', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([], { stack: 'rec', status: 'in-sync' }));
|
||||
expect(res).toEqual({ detected: 0, resolved: 1 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(0);
|
||||
expect(driftActivity('rec').filter(e => e.category === 'drift_resolved')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('records exactly one row per direction when one finding clears as another appears', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
// In a single check, 'web' clears while 'db' newly appears.
|
||||
const res = ledger().reconcile(nodeId, 'rec', reportWith([finding('service-missing', 'db')], { stack: 'rec' }));
|
||||
expect(res).toEqual({ detected: 1, resolved: 1 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
const cats = driftActivity('rec').map(e => e.category);
|
||||
expect(cats.filter(c => c === 'drift_detected')).toHaveLength(2); // first reconcile + this one
|
||||
expect(cats.filter(c => c === 'drift_resolved')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile an unreachable report (open findings are not falsely resolved)', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', { stack: 'rec', status: 'unreachable', hasComposeFile: true, hasContainers: false, findings: [] });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not reconcile a parse-error report', () => {
|
||||
ledger().reconcile(nodeId, 'rec', reportWith([finding('image-mismatch', 'web')], { stack: 'rec' }));
|
||||
const res = ledger().reconcile(nodeId, 'rec', { stack: 'rec', status: 'drifted', hasComposeFile: false, hasContainers: false, findings: [], parseError: 'bad yaml' });
|
||||
expect(res).toEqual({ detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, 'rec')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.recordBaseline', () => {
|
||||
beforeEach(() => clearLedger('baseline'));
|
||||
|
||||
it('hashes the on-disk compose and stores it as the dossier baseline', async () => {
|
||||
const stackDir = path.join(process.env.COMPOSE_DIR as string, 'baseline');
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
try {
|
||||
await DriftLedgerService.getInstance().recordBaseline(nodeId, 'baseline');
|
||||
const row = db().getStackDossier(nodeId, 'baseline');
|
||||
const expected = computeStackHashes('services:\n web:\n image: nginx:1.27\n');
|
||||
expect(row?.source_hash).toBe(expected.sourceHash);
|
||||
expect(row?.rendered_hash).toBe(expected.renderedHash);
|
||||
} finally {
|
||||
fs.rmSync(stackDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift route (GET read-only, POST recheck persists)', () => {
|
||||
const STACK = 'recheckroute';
|
||||
let stackDir: string;
|
||||
|
||||
// A running container on a different image than compose declares => image-mismatch.
|
||||
const stubDriftedDocker = () => vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({
|
||||
containers: [{
|
||||
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
|
||||
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
|
||||
}],
|
||||
networks: [], volumes: [],
|
||||
}),
|
||||
} as unknown as DockerController);
|
||||
|
||||
beforeEach(() => {
|
||||
clearLedger(STACK);
|
||||
stackDir = path.join(process.env.COMPOSE_DIR as string, STACK);
|
||||
fs.mkdirSync(stackDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(stackDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('GET reports drift without writing the ledger or activity timeline', async () => {
|
||||
stubDriftedDocker();
|
||||
const res = await request(app).get(`/api/stacks/${STACK}/drift`).set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('drifted');
|
||||
expect(res.body.temporal).toBeDefined();
|
||||
// A passive read must not persist anything.
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK)).toHaveLength(0);
|
||||
expect(driftActivity(STACK)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('POST recheck persists the current drift and returns temporal + ledger', async () => {
|
||||
stubDriftedDocker();
|
||||
const res = await request(app).post(`/api/stacks/${STACK}/drift/recheck`).set('Authorization', authHeader);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.status).toBe('drifted');
|
||||
expect(res.body.temporal).toBeDefined();
|
||||
expect(Array.isArray(res.body.ledger)).toBe(true);
|
||||
expect(res.body.ledger).toHaveLength(1);
|
||||
expect(res.body.ledger[0]).toMatchObject({ service: 'web', kind: 'image-mismatch', resolvedAt: null });
|
||||
|
||||
// The transition was recorded exactly once in the activity timeline.
|
||||
const acts = driftActivity(STACK);
|
||||
expect(acts).toHaveLength(1);
|
||||
expect(acts[0].category).toBe('drift_detected');
|
||||
// And persisted as an open finding.
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Router, type Request, type Response, type NextFunction } from 'express';
|
||||
import { z } from 'zod';
|
||||
import path from 'path';
|
||||
import { inspect } from 'node:util';
|
||||
import YAML from 'yaml';
|
||||
import multer from 'multer';
|
||||
import { FileSystemService } from '../services/FileSystemService';
|
||||
@@ -12,7 +13,8 @@ import { CacheService } from '../services/CacheService';
|
||||
import { UpdatePreviewService } from '../services/UpdatePreviewService';
|
||||
import { GitSourceService, GitSourceError, repoHost as gitRepoHost } from '../services/GitSourceService';
|
||||
import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement';
|
||||
import { buildStackDriftReport } from '../services/DriftDetectionService';
|
||||
import { buildStackDriftReport, type DriftFindingKind, type StackDriftReport } from '../services/DriftDetectionService';
|
||||
import { DriftLedgerService, type DriftTemporal } from '../services/DriftLedgerService';
|
||||
import { requirePermission, checkPermission } from '../middleware/permissions';
|
||||
import { NotificationService, type NotificationCategory } from '../services/NotificationService';
|
||||
import { StackOpLockService, type StackOpAction } from '../services/StackOpLockService';
|
||||
@@ -939,6 +941,7 @@ stacksRouter.delete('/:stackName', async (req: Request, res: Response) => {
|
||||
DatabaseService.getInstance().deleteRoleAssignmentsByResource('stack', stackName);
|
||||
DatabaseService.getInstance().deleteGitSource(stackName);
|
||||
DatabaseService.getInstance().deleteStackDossier(req.nodeId, stackName);
|
||||
DatabaseService.getInstance().deleteStackDriftFindings(req.nodeId, stackName);
|
||||
if (debug) console.debug(`[Stacks:debug] Delete: db OK`, { stackName: sanitizedName });
|
||||
} catch (dbErr) {
|
||||
console.error('[Stacks] Database cleanup failed for %s; files already removed:', sanitizeForLog(stackName), dbErr);
|
||||
@@ -1009,18 +1012,80 @@ stacksRouter.get('/:stackName/services', async (req: Request, res: Response) =>
|
||||
}
|
||||
});
|
||||
|
||||
/** A persisted finding as the Drift tab consumes it (no internal column names). */
|
||||
interface DriftLedgerEntry {
|
||||
service: string;
|
||||
kind: DriftFindingKind;
|
||||
message: string;
|
||||
detectedAt: number;
|
||||
resolvedAt: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the full Drift tab payload: the spatial report (compose vs runtime),
|
||||
* the temporal overlay (source changed since last deploy), and the persisted
|
||||
* ledger history. When `reconcile` is set, the current findings are persisted
|
||||
* into the ledger (new ones recorded, cleared ones resolved) before the ledger
|
||||
* is read back, so the returned history reflects the just-observed state.
|
||||
*/
|
||||
async function buildDriftPayload(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
reconcile: boolean,
|
||||
): Promise<StackDriftReport & { temporal: DriftTemporal; ledger: DriftLedgerEntry[] }> {
|
||||
const report = await buildStackDriftReport(nodeId, stackName);
|
||||
// Only the on-disk read is best-effort: an unreadable compose is already surfaced
|
||||
// by the report as a parse error, so temporal degrades to neutral. computeTemporal
|
||||
// runs outside the try so a real ledger fault (a DB or hashing error) surfaces as a
|
||||
// 500 instead of being hidden behind a misleading "no baseline".
|
||||
let content: string | null = null;
|
||||
try {
|
||||
content = await FileSystemService.getInstance(nodeId).getStackContent(stackName);
|
||||
} catch {
|
||||
// Unreadable compose: the report carries the parseError; temporal stays neutral.
|
||||
}
|
||||
const temporal: DriftTemporal = content !== null
|
||||
? DriftLedgerService.getInstance().computeTemporal(nodeId, stackName, content)
|
||||
: { hasBaseline: false, sourceChanged: false, renderedChanged: false };
|
||||
if (reconcile) {
|
||||
DriftLedgerService.getInstance().reconcile(nodeId, stackName, report);
|
||||
}
|
||||
// finding_type is a free-text column, but reconcile only ever writes a DriftFindingKind.
|
||||
const ledger: DriftLedgerEntry[] = DatabaseService.getInstance()
|
||||
.getRecentDriftFindings(nodeId, stackName, 20)
|
||||
.map(r => ({ service: r.service, kind: r.finding_type as DriftFindingKind, message: r.message, detectedAt: r.detected_at, resolvedAt: r.resolved_at }));
|
||||
return { ...report, temporal, ledger };
|
||||
}
|
||||
|
||||
stacksRouter.get('/:stackName/drift', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
const report = await buildStackDriftReport(req.nodeId, stackName);
|
||||
res.json(report);
|
||||
res.json(await buildDriftPayload(req.nodeId, stackName, false));
|
||||
} catch (error) {
|
||||
console.error('[Stacks] Failed to build drift report for %s:', sanitizeForLog(stackName), error);
|
||||
console.error('[Stacks] Failed to build drift report for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(inspect(error, { depth: 4 })));
|
||||
res.status(500).json({ error: 'Failed to build drift report' });
|
||||
}
|
||||
});
|
||||
|
||||
// Re-check is the one place a passive drift view becomes a ledger write: it
|
||||
// reconciles the current findings into stack_drift_findings (recording newly
|
||||
// detected and newly resolved ones) before returning the fresh payload, so the
|
||||
// GET above can stay a side-effect-free read.
|
||||
stacksRouter.post('/:stackName/drift/recheck', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:read', 'stack', stackName)) return;
|
||||
if (!(await requireStackExists(req.nodeId, stackName, res))) return;
|
||||
try {
|
||||
res.json(await buildDriftPayload(req.nodeId, stackName, true));
|
||||
} catch (error) {
|
||||
console.error('[Stacks] Failed to re-check drift for %s:', sanitizeForLog(stackName),
|
||||
sanitizeForLog(inspect(error, { depth: 4 })));
|
||||
res.status(500).json({ error: 'Failed to re-check drift' });
|
||||
}
|
||||
});
|
||||
|
||||
stacksRouter.post('/:stackName/deploy', async (req: Request, res: Response) => {
|
||||
const stackName = req.params.stackName as string;
|
||||
if (!requirePermission(req, res, 'stack:deploy', 'stack', stackName)) return;
|
||||
|
||||
@@ -10,6 +10,7 @@ import { MeshService } from './MeshService';
|
||||
import { LogFormatter } from './LogFormatter';
|
||||
import { NodeRegistry } from './NodeRegistry';
|
||||
import { RegistryService } from './RegistryService';
|
||||
import { DriftLedgerService } from './DriftLedgerService';
|
||||
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
@@ -359,6 +360,11 @@ export class ComposeService {
|
||||
}
|
||||
throw deployError;
|
||||
}
|
||||
// Reached only on a successful deploy (the catch above always rethrows). Record
|
||||
// the drift baseline here so every deploy path gets one, not just the manual
|
||||
// route: bulk, Git-source, App Store, scheduler, and webhook deploys all funnel
|
||||
// through this method. Internally guarded; awaited so it cannot race later work.
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
}
|
||||
|
||||
streamLogs(stackName: string, ws: WebSocket) {
|
||||
@@ -554,6 +560,9 @@ export class ComposeService {
|
||||
}
|
||||
throw updateError;
|
||||
}
|
||||
// Reached only on a successful update; re-baseline so temporal drift compares
|
||||
// against what is now deployed (see deployStack for why this lives here).
|
||||
await DriftLedgerService.getInstance().recordBaseline(this.nodeId, stackName);
|
||||
}
|
||||
|
||||
public async downStack(stackName: string): Promise<void> {
|
||||
|
||||
@@ -85,10 +85,29 @@ export interface StackDossier extends StackDossierFields {
|
||||
id?: number;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
/** SHA-256 of the compose file's UTF-8 text at the last deploy through Sencho (baseline for temporal drift). */
|
||||
source_hash?: string | null;
|
||||
/** SHA-256 of the parsed compose model at the last deploy (ignores comments/whitespace). */
|
||||
rendered_hash?: string | null;
|
||||
created_at: number;
|
||||
updated_at: number;
|
||||
}
|
||||
|
||||
/** A persisted drift finding: one service-scoped divergence, open until resolved. */
|
||||
export interface StackDriftFindingRow {
|
||||
id: number;
|
||||
node_id: number;
|
||||
stack_name: string;
|
||||
service: string;
|
||||
finding_type: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
expected_json: string | null;
|
||||
actual_json: string | null;
|
||||
detected_at: number;
|
||||
resolved_at: number | null;
|
||||
}
|
||||
|
||||
export interface Node {
|
||||
id: number;
|
||||
name: string;
|
||||
@@ -697,6 +716,7 @@ export class DatabaseService {
|
||||
this.migrateAddBlueprintPinnedNode();
|
||||
this.migrateAutoHealNodeId();
|
||||
this.migrateFleetSyncStickyError();
|
||||
this.migrateStackDossierHashes();
|
||||
|
||||
// Reset the cache once at end of constructor in case any migration
|
||||
// populated it via getGlobalSettings() and a subsequent migration
|
||||
@@ -1175,6 +1195,22 @@ export class DatabaseService {
|
||||
UNIQUE(node_id, stack_name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS stack_drift_findings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
node_id INTEGER NOT NULL,
|
||||
stack_name TEXT NOT NULL,
|
||||
service TEXT NOT NULL,
|
||||
finding_type TEXT NOT NULL,
|
||||
severity TEXT NOT NULL DEFAULT 'warning',
|
||||
message TEXT NOT NULL,
|
||||
expected_json TEXT,
|
||||
actual_json TEXT,
|
||||
detected_at INTEGER NOT NULL,
|
||||
resolved_at INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_stack_drift_findings_open
|
||||
ON stack_drift_findings(node_id, stack_name, resolved_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
@@ -1473,6 +1509,11 @@ export class DatabaseService {
|
||||
this.tryAddColumn('notification_history', 'container_name', 'TEXT');
|
||||
}
|
||||
|
||||
private migrateStackDossierHashes(): void {
|
||||
this.tryAddColumn('stack_dossiers', 'source_hash', 'TEXT');
|
||||
this.tryAddColumn('stack_dossiers', 'rendered_hash', 'TEXT');
|
||||
}
|
||||
|
||||
private migrateScanPolicyFleetColumns(): void {
|
||||
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
|
||||
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
|
||||
@@ -2109,6 +2150,55 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM stack_dossiers WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the deploy-time baseline hashes for a stack. Creates a dossier row
|
||||
* with empty operator notes if none exists; on conflict updates only the hash
|
||||
* columns so operator-authored notes and their updated_at are left untouched.
|
||||
*/
|
||||
public setStackDossierHashes(nodeId: number, stackName: string, sourceHash: string, renderedHash: string | null): void {
|
||||
const now = Date.now();
|
||||
this.db.prepare(
|
||||
`INSERT INTO stack_dossiers (node_id, stack_name, source_hash, rendered_hash, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(node_id, stack_name) DO UPDATE SET
|
||||
source_hash = excluded.source_hash,
|
||||
rendered_hash = excluded.rendered_hash`
|
||||
).run(nodeId, stackName, sourceHash, renderedHash, now, now);
|
||||
}
|
||||
|
||||
// --- Stack Drift Findings (the persisted drift ledger) ---
|
||||
|
||||
public insertDriftFinding(f: Omit<StackDriftFindingRow, 'id' | 'resolved_at'>): number {
|
||||
const res = this.db.prepare(
|
||||
`INSERT INTO stack_drift_findings
|
||||
(node_id, stack_name, service, finding_type, severity, message, expected_json, actual_json, detected_at, resolved_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)`
|
||||
).run(f.node_id, f.stack_name, f.service, f.finding_type, f.severity, f.message, f.expected_json, f.actual_json, f.detected_at);
|
||||
return res.lastInsertRowid as number;
|
||||
}
|
||||
|
||||
public resolveDriftFinding(id: number, resolvedAt: number): void {
|
||||
this.db.prepare('UPDATE stack_drift_findings SET resolved_at = ? WHERE id = ? AND resolved_at IS NULL').run(resolvedAt, id);
|
||||
}
|
||||
|
||||
/** Open (unresolved) findings for a stack, oldest first. */
|
||||
public getOpenDriftFindings(nodeId: number, stackName: string): StackDriftFindingRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_drift_findings WHERE node_id = ? AND stack_name = ? AND resolved_at IS NULL ORDER BY detected_at ASC, id ASC'
|
||||
).all(nodeId, stackName) as StackDriftFindingRow[];
|
||||
}
|
||||
|
||||
/** Recent findings for a stack: open ones first, then resolved, each newest first. */
|
||||
public getRecentDriftFindings(nodeId: number, stackName: string, limit: number): StackDriftFindingRow[] {
|
||||
return this.db.prepare(
|
||||
'SELECT * FROM stack_drift_findings WHERE node_id = ? AND stack_name = ? ORDER BY (resolved_at IS NOT NULL) ASC, detected_at DESC, id DESC LIMIT ?'
|
||||
).all(nodeId, stackName, limit) as StackDriftFindingRow[];
|
||||
}
|
||||
|
||||
public deleteStackDriftFindings(nodeId: number, stackName: string): void {
|
||||
this.db.prepare('DELETE FROM stack_drift_findings WHERE node_id = ? AND stack_name = ?').run(nodeId, stackName);
|
||||
}
|
||||
|
||||
// --- Notification History ---
|
||||
|
||||
private mapNotificationRow(row: any): NotificationHistory {
|
||||
@@ -2446,6 +2536,7 @@ export class DatabaseService {
|
||||
this.db.prepare('DELETE FROM stack_label_assignments WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_labels WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_dossiers WHERE node_id = ?').run(id);
|
||||
this.db.prepare('DELETE FROM stack_drift_findings WHERE node_id = ?').run(id);
|
||||
this.db.prepare('UPDATE blueprints SET pinned_node_id = NULL WHERE pinned_node_id = ?').run(id);
|
||||
this.deleteRoleAssignmentsByResource('node', String(id));
|
||||
this.db.prepare('DELETE FROM fleet_sync_status WHERE node_id = ?').run(id);
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import type { StackDriftFindingRow } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import { parseComposeDependencies } from '../helpers/composeDependencyParse';
|
||||
import type { DeclaredCompose } from '../helpers/composeDependencyParse';
|
||||
import { sha256Hex } from '../utils/hashing';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import type { StackDriftReport, StackDriftFinding } from './DriftDetectionService';
|
||||
|
||||
/**
|
||||
* The persistence-backed Drift Ledger that builds on the read-only spatial
|
||||
* engine (DriftDetectionService). It adds the two things the engine deliberately
|
||||
* leaves out: a deploy-time baseline (so "the source changed since you deployed"
|
||||
* can be answered) and a persisted history of findings (so drift can be seen to
|
||||
* appear and resolve over time). Node-local: it reads and writes the database of
|
||||
* whichever node owns the stack.
|
||||
*/
|
||||
|
||||
/** Temporal alignment of the on-disk compose against the last deploy baseline. */
|
||||
export interface DriftTemporal {
|
||||
/** True once a deploy through Sencho has recorded baseline hashes. */
|
||||
hasBaseline: boolean;
|
||||
/** The compose file's text differs from the last deploy. */
|
||||
sourceChanged: boolean;
|
||||
/** Parsed compose model differs from the last deploy (ignores comments/whitespace). */
|
||||
renderedChanged: boolean;
|
||||
}
|
||||
|
||||
export interface DriftReconcileResult {
|
||||
detected: number;
|
||||
resolved: number;
|
||||
}
|
||||
|
||||
/** Stable identity for a finding across checks: same service + kind is the same finding. */
|
||||
function findingKey(service: string, kind: string): string {
|
||||
return JSON.stringify([service, kind]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order-independent serialization of the parsed model so two compose files that
|
||||
* differ only in comments, whitespace, or key order hash equal, while a real
|
||||
* change to images/ports/services/networks/volumes changes the hash. Returns null
|
||||
* when the local parser cannot produce a model (for example a file over the parse
|
||||
* size cap), so the caller stores no rendered baseline rather than a sentinel that
|
||||
* would make a later real change read as unchanged.
|
||||
*/
|
||||
function stableModelString(model: DeclaredCompose): string | null {
|
||||
if (model.parseError) return null;
|
||||
const services = [...model.services]
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.map(s => ({
|
||||
name: s.name,
|
||||
image: s.image ?? null,
|
||||
dependsOn: [...s.dependsOn].sort(),
|
||||
networks: [...s.networks].sort(),
|
||||
volumes: [...s.volumes].sort(),
|
||||
ports: s.ports.map(p => `${p.hostIp}:${p.publishedPort}/${p.protocol}`).sort(),
|
||||
}));
|
||||
const networks = Object.keys(model.networks).sort().map(k => ({ key: k, ...model.networks[k] }));
|
||||
const volumes = Object.keys(model.volumes).sort().map(k => ({ key: k, ...model.volumes[k] }));
|
||||
return JSON.stringify({ services, networks, volumes });
|
||||
}
|
||||
|
||||
/**
|
||||
* Hashes a compose file two ways: the raw text (source) and the parsed model
|
||||
* (rendered). renderedHash is null when the model cannot be parsed, so a
|
||||
* model-level comparison is simply skipped rather than forced to a false equal.
|
||||
*/
|
||||
export function computeStackHashes(content: string): { sourceHash: string; renderedHash: string | null } {
|
||||
const model = stableModelString(parseComposeDependencies(content));
|
||||
return {
|
||||
sourceHash: sha256Hex(content),
|
||||
renderedHash: model === null ? null : sha256Hex(model),
|
||||
};
|
||||
}
|
||||
|
||||
export class DriftLedgerService {
|
||||
private static instance: DriftLedgerService | null = null;
|
||||
|
||||
static getInstance(): DriftLedgerService {
|
||||
if (!DriftLedgerService.instance) DriftLedgerService.instance = new DriftLedgerService();
|
||||
return DriftLedgerService.instance;
|
||||
}
|
||||
|
||||
private constructor() { /* singleton */ }
|
||||
|
||||
/**
|
||||
* Record the deploy-time baseline hashes for a stack. Best-effort: a read or
|
||||
* hash failure is logged and swallowed so it never fails the deploy that
|
||||
* triggered it.
|
||||
*/
|
||||
async recordBaseline(nodeId: number, stackName: string): Promise<void> {
|
||||
try {
|
||||
const content = await FileSystemService.getInstance(nodeId).getStackContent(stackName);
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
DatabaseService.getInstance().setStackDossierHashes(nodeId, stackName, sourceHash, renderedHash);
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] Failed to record baseline for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
|
||||
/** Compare the current compose content against the stored deploy baseline. */
|
||||
computeTemporal(nodeId: number, stackName: string, content: string): DriftTemporal {
|
||||
const dossier = DatabaseService.getInstance().getStackDossier(nodeId, stackName);
|
||||
const storedSource = dossier?.source_hash ?? null;
|
||||
const storedRendered = dossier?.rendered_hash ?? null;
|
||||
if (!storedSource) {
|
||||
return { hasBaseline: false, sourceChanged: false, renderedChanged: false };
|
||||
}
|
||||
const { sourceHash, renderedHash } = computeStackHashes(content);
|
||||
return {
|
||||
hasBaseline: true,
|
||||
sourceChanged: sourceHash !== storedSource,
|
||||
// Only a real model change counts; if either side has no parseable model, skip it.
|
||||
renderedChanged: storedRendered != null && renderedHash != null && renderedHash !== storedRendered,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile the spatial report's current findings against the persisted ledger:
|
||||
* insert findings that are newly seen, resolve ones that have cleared. Idempotent
|
||||
* (a repeat check with no change writes nothing). Skipped when the report is not
|
||||
* authoritative (Docker unreachable or a compose parse error) so open findings are
|
||||
* never falsely resolved. On a real transition it records one summary activity row
|
||||
* per direction so the stack Activity timeline shows drift appearing and clearing.
|
||||
*/
|
||||
reconcile(nodeId: number, stackName: string, report: StackDriftReport): DriftReconcileResult {
|
||||
if (report.status === 'unreachable' || report.parseError) {
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
const db = DatabaseService.getInstance();
|
||||
const openByKey = new Map(db.getOpenDriftFindings(nodeId, stackName).map(r => [findingKey(r.service, r.finding_type), r]));
|
||||
const currentByKey = new Map(report.findings.map(f => [findingKey(f.service, f.kind), f]));
|
||||
|
||||
const toInsert: StackDriftFinding[] = [];
|
||||
for (const [key, f] of currentByKey) {
|
||||
if (!openByKey.has(key)) toInsert.push(f);
|
||||
}
|
||||
const toResolve: StackDriftFindingRow[] = [];
|
||||
for (const [key, row] of openByKey) {
|
||||
if (!currentByKey.has(key)) toResolve.push(row);
|
||||
}
|
||||
if (toInsert.length === 0 && toResolve.length === 0) {
|
||||
return { detected: 0, resolved: 0 };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
db.getDb().transaction(() => {
|
||||
for (const f of toInsert) {
|
||||
db.insertDriftFinding({
|
||||
node_id: nodeId,
|
||||
stack_name: stackName,
|
||||
service: f.service,
|
||||
finding_type: f.kind,
|
||||
severity: 'warning',
|
||||
message: f.detail,
|
||||
expected_json: f.expected !== undefined ? JSON.stringify(f.expected) : null,
|
||||
actual_json: f.actual !== undefined ? JSON.stringify(f.actual) : null,
|
||||
detected_at: now,
|
||||
});
|
||||
}
|
||||
for (const row of toResolve) {
|
||||
db.resolveDriftFinding(row.id, now);
|
||||
}
|
||||
})();
|
||||
|
||||
if (toInsert.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_detected', 'warning',
|
||||
`Drift detected on ${stackName}: ${toInsert.length} new finding${toInsert.length === 1 ? '' : 's'}`, now);
|
||||
}
|
||||
if (toResolve.length > 0) {
|
||||
this.recordActivity(nodeId, stackName, 'drift_resolved', 'info',
|
||||
`Drift resolved on ${stackName}: ${toResolve.length} finding${toResolve.length === 1 ? '' : 's'} cleared`, now);
|
||||
}
|
||||
return { detected: toInsert.length, resolved: toResolve.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a drift transition to the stack activity timeline. History-only (no
|
||||
* external channel dispatch): a drift signal belongs in the activity feed, not
|
||||
* in every configured Discord/Slack webhook.
|
||||
*/
|
||||
private recordActivity(
|
||||
nodeId: number,
|
||||
stackName: string,
|
||||
category: 'drift_detected' | 'drift_resolved',
|
||||
level: 'info' | 'warning',
|
||||
message: string,
|
||||
timestamp: number,
|
||||
): void {
|
||||
try {
|
||||
DatabaseService.getInstance().addNotificationHistory(nodeId, {
|
||||
level,
|
||||
category,
|
||||
message,
|
||||
timestamp,
|
||||
stack_name: stackName,
|
||||
actor_username: null,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] Failed to record activity for %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,11 @@ export type NotificationCategory =
|
||||
| 'blueprint_deployment_failed'
|
||||
| 'blueprint_drift_detected'
|
||||
| 'blueprint_drift_correction_failed'
|
||||
// Stack drift ledger transitions. Written to history only (the Activity
|
||||
// timeline), never dispatched to channels, so they are deliberately excluded
|
||||
// from ALL_NOTIFICATION_CATEGORIES (the routable-category whitelist) below.
|
||||
| 'drift_detected'
|
||||
| 'drift_resolved'
|
||||
| 'system';
|
||||
|
||||
export const ALL_NOTIFICATION_CATEGORIES: readonly NotificationCategory[] = [
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
/** Hex-encoded SHA-256 of a UTF-8 string. Stable across runs and platforms. */
|
||||
export function sha256Hex(content: string): string {
|
||||
return createHash('sha256').update(content, 'utf8').digest('hex');
|
||||
}
|
||||
@@ -5,7 +5,7 @@ description: See at a glance whether a stack's running containers still match th
|
||||
|
||||
The **Drift** tab in the right-hand **Anatomy** panel answers a single day-two question: does what is actually running still match the Compose file on disk? Sencho treats your Compose file as the source of truth, so it compares the file against the live Docker runtime and reports exactly where the two have diverged.
|
||||
|
||||
The check is read-only. It tells you what changed and never alters a stack on its own, so you can trust the report before deciding what to do about it. The report is built fresh each time you open the tab, with no separate state to maintain.
|
||||
The check is read-only. It tells you what changed and never alters a stack on its own, so you can trust the report before deciding what to do about it. Opening the tab builds the comparison fresh. When you deploy a stack through Sencho, it also records the Compose file it deployed as a baseline, so it can later tell you whether the file has changed since then, and it keeps a short history of the findings it has seen.
|
||||
|
||||
## Status
|
||||
|
||||
@@ -18,6 +18,18 @@ Every stack resolves to one of four states, shown as a badge at the top of the t
|
||||
| **Not running** | The stack is defined on disk but no containers are running. |
|
||||
| **Unreachable** | Docker could not be reached, so drift cannot be assessed right now. |
|
||||
|
||||
## Since your last deploy
|
||||
|
||||
Below the status badge, a second line compares the Compose file on disk against the version you last deployed through Sencho:
|
||||
|
||||
| Signal | Meaning |
|
||||
|--------|---------|
|
||||
| **Matches last deploy** | The Compose file is unchanged since you last deployed it. |
|
||||
| **Source changed** | The Compose file has been edited since the last deploy. If the change affects the model (an image, port, or service), Sencho says so; a comments or formatting only edit is called out separately. |
|
||||
| **No deploy baseline** | This stack has not been deployed through Sencho yet, so there is nothing to compare against. Deploy it once to start tracking. |
|
||||
|
||||
This is independent of the runtime status above: a stack can be **In sync** with its running containers while its file has already **changed** for the next deploy.
|
||||
|
||||
## Findings
|
||||
|
||||
When a stack is drifted, each reason is listed against the service it affects:
|
||||
@@ -31,6 +43,12 @@ When a stack is drifted, each reason is listed against the service it affects:
|
||||
|
||||
Image references are compared after normalizing the implicit Docker Hub registry and a missing tag to `:latest`, so `nginx` and `docker.io/library/nginx:latest` are treated as the same image. A running container pinned to a digest is compared against the declared tag as written.
|
||||
|
||||
## Drift history
|
||||
|
||||
Each time you **re-check** a stack, and after every deploy, Sencho records the findings it sees. The **Drift history** list under the findings shows recent entries with when each was first **detected** and, once it clears, when it was **resolved**. This turns a point-in-time check into a short ledger of how a stack has drifted and recovered over time.
|
||||
|
||||
Drift that appears or clears is also written to the stack's **Activity** timeline, so **Drift detected** and **Drift resolved** events sit alongside deploys and restarts.
|
||||
|
||||
## Accessing the Drift tab
|
||||
|
||||
1. Click any stack in the left sidebar to open it.
|
||||
@@ -52,6 +70,9 @@ On a phone, the same report appears under the **Compose** section of the stack d
|
||||
A Compose port range such as `8000-8002:8000-8002` is compared conservatively and can read as a ports difference even when the deployment is correct. Sencho errs toward surfacing a possible difference rather than hiding one. The status reflects this as drift you can confirm against the file.
|
||||
</Accordion>
|
||||
<Accordion title="The status says 'Unreachable'">
|
||||
Sencho could not reach Docker on the active node, so it cannot compare the runtime. Confirm the Docker engine is running and the node is online, then use **re-check**. Other stacks on the same node will show the same state until Docker responds.
|
||||
Sencho could not reach Docker on the active node, so it cannot compare the runtime. Confirm the Docker engine is running and the node is online, then use **re-check**. Other stacks on the same node will show the same state until Docker responds. While Docker is unreachable, Sencho does not change the recorded drift history, so an open finding is never cleared just because the check could not run.
|
||||
</Accordion>
|
||||
<Accordion title="The tab says 'No deploy baseline'">
|
||||
Sencho records a baseline the first time you deploy a stack through it. A stack you imported or have not yet deployed from Sencho has nothing to compare against. Deploy it once from Sencho and the line changes to **Matches last deploy**.
|
||||
</Accordion>
|
||||
</AccordionGroup>
|
||||
|
||||
@@ -21,6 +21,8 @@ interface DriftReport {
|
||||
hasContainers: boolean;
|
||||
findings: Array<{ kind: string; service: string; detail: string; expected?: string; actual?: string }>;
|
||||
parseError?: string;
|
||||
temporal?: { hasBaseline: boolean; sourceChanged: boolean; renderedChanged: boolean };
|
||||
ledger?: Array<{ service: string; kind: string; message: string; detectedAt: number; resolvedAt: number | null }>;
|
||||
}
|
||||
|
||||
function report(partial: Partial<DriftReport>): DriftReport {
|
||||
@@ -133,12 +135,79 @@ describe('DriftPanel', () => {
|
||||
expect(screen.queryByTestId('drift-retry-btn')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('re-checks on demand', async () => {
|
||||
it('re-checks on demand via the recheck endpoint (a POST), not the read GET', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ status: 'in-sync' })));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
await screen.findByTestId('drift-status');
|
||||
expect(apiFetch).toHaveBeenCalledTimes(1);
|
||||
expect(apiFetch).toHaveBeenLastCalledWith('/stacks/web/drift');
|
||||
fireEvent.click(screen.getByTestId('drift-recheck-btn'));
|
||||
await waitFor(() => expect(apiFetch).toHaveBeenCalledTimes(2));
|
||||
expect(apiFetch).toHaveBeenLastCalledWith('/stacks/web/drift/recheck', { method: 'POST' });
|
||||
});
|
||||
|
||||
it('omits the temporal card when the report carries no temporal field (older node)', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({ status: 'in-sync' }))); // no temporal field
|
||||
render(<DriftPanel stackName="web" />);
|
||||
await screen.findByTestId('drift-status');
|
||||
expect(screen.queryByTestId('drift-temporal')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows "no deploy baseline" when the report has no temporal baseline', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({
|
||||
status: 'in-sync',
|
||||
temporal: { hasBaseline: false, sourceChanged: false, renderedChanged: false },
|
||||
})));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
const temporal = await screen.findByTestId('drift-temporal');
|
||||
expect(temporal).toHaveAttribute('data-temporal', 'no-baseline');
|
||||
});
|
||||
|
||||
it('flags a source change since the last deploy', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({
|
||||
status: 'in-sync',
|
||||
temporal: { hasBaseline: true, sourceChanged: true, renderedChanged: true },
|
||||
})));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
const temporal = await screen.findByTestId('drift-temporal');
|
||||
expect(temporal).toHaveAttribute('data-temporal', 'source-changed');
|
||||
expect(screen.getByText(/changed since the last deploy/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('notes a formatting-only change when source changed but the model did not', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({
|
||||
status: 'in-sync',
|
||||
temporal: { hasBaseline: true, sourceChanged: true, renderedChanged: false },
|
||||
})));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
const temporal = await screen.findByTestId('drift-temporal');
|
||||
expect(temporal).toHaveAttribute('data-temporal', 'source-changed');
|
||||
expect(screen.getByText(/formatting only/i)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows "matches last deploy" when the source is unchanged', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({
|
||||
status: 'in-sync',
|
||||
temporal: { hasBaseline: true, sourceChanged: false, renderedChanged: false },
|
||||
})));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
const temporal = await screen.findByTestId('drift-temporal');
|
||||
expect(temporal).toHaveAttribute('data-temporal', 'matches');
|
||||
});
|
||||
|
||||
it('renders the persisted drift history with open and resolved entries', async () => {
|
||||
vi.mocked(apiFetch).mockResolvedValue(jsonRes(report({
|
||||
status: 'drifted',
|
||||
findings: [{ kind: 'image-mismatch', service: 'web', detail: 'image differs' }],
|
||||
ledger: [
|
||||
{ service: 'web', kind: 'image-mismatch', message: 'image differs', detectedAt: Date.now(), resolvedAt: null },
|
||||
{ service: 'db', kind: 'service-missing', message: 'db not running', detectedAt: Date.now() - 1000, resolvedAt: Date.now() },
|
||||
],
|
||||
})));
|
||||
render(<DriftPanel stackName="web" />);
|
||||
await screen.findByTestId('drift-status');
|
||||
expect(screen.getByText(/drift history/i)).toBeInTheDocument();
|
||||
expect(screen.getByText('open')).toBeInTheDocument();
|
||||
expect(screen.getByText('resolved')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { Check, TriangleAlert, CircleSlash, WifiOff, RefreshCw, type LucideIcon } from 'lucide-react';
|
||||
import {
|
||||
Check, TriangleAlert, CircleSlash, WifiOff, RefreshCw,
|
||||
FileClock, FileCheck2, FileQuestion, type LucideIcon,
|
||||
} from 'lucide-react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { formatTimeAgo } from '@/lib/relativeTime';
|
||||
import { useNodes } from '@/context/NodeContext';
|
||||
|
||||
// Mirrors the backend StackDriftReport shape (the frontend never imports backend).
|
||||
// Mirrors the backend payload shape (the frontend never imports backend).
|
||||
type StackDriftStatus = 'in-sync' | 'drifted' | 'missing-runtime' | 'unreachable';
|
||||
type DriftFindingKind = 'service-missing' | 'service-undeclared' | 'image-mismatch' | 'ports-mismatch';
|
||||
|
||||
@@ -17,6 +21,20 @@ interface StackDriftFinding {
|
||||
actual?: string;
|
||||
}
|
||||
|
||||
interface DriftTemporal {
|
||||
hasBaseline: boolean;
|
||||
sourceChanged: boolean;
|
||||
renderedChanged: boolean;
|
||||
}
|
||||
|
||||
interface DriftLedgerEntry {
|
||||
service: string;
|
||||
kind: DriftFindingKind;
|
||||
message: string;
|
||||
detectedAt: number;
|
||||
resolvedAt: number | null;
|
||||
}
|
||||
|
||||
interface StackDriftReport {
|
||||
stack: string;
|
||||
status: StackDriftStatus;
|
||||
@@ -24,11 +42,15 @@ interface StackDriftReport {
|
||||
hasContainers: boolean;
|
||||
findings: StackDriftFinding[];
|
||||
parseError?: string;
|
||||
// Optional so a report from an older remote node (no ledger layer) still renders.
|
||||
temporal?: DriftTemporal;
|
||||
ledger?: DriftLedgerEntry[];
|
||||
}
|
||||
|
||||
const LABEL_CLASS = 'font-mono text-[10px] uppercase tracking-[0.18em] text-stat-subtitle';
|
||||
const ACTION_CLASS =
|
||||
'inline-flex items-center gap-1 font-mono text-[10px] uppercase tracking-wide text-stat-subtitle hover:text-brand transition-colors disabled:opacity-40';
|
||||
const CARD_CLASS = 'rounded-lg border px-3 py-2.5';
|
||||
|
||||
const STATUS_META: Record<StackDriftStatus, { label: string; icon: LucideIcon; tone: string; line: string }> = {
|
||||
'in-sync': {
|
||||
@@ -64,6 +86,37 @@ const FINDING_LABEL: Record<DriftFindingKind, string> = {
|
||||
'ports-mismatch': 'ports',
|
||||
};
|
||||
|
||||
/** The temporal overlay: how the on-disk compose compares to the last deploy baseline. */
|
||||
function temporalMeta(temporal: DriftTemporal): { label: string; icon: LucideIcon; tone: string; line: string; key: string } {
|
||||
if (!temporal.hasBaseline) {
|
||||
return {
|
||||
key: 'no-baseline',
|
||||
label: 'no deploy baseline',
|
||||
icon: FileQuestion,
|
||||
tone: 'border-muted bg-card/40 text-stat-subtitle',
|
||||
line: 'Deploy through Sencho to start tracking changes since deploy.',
|
||||
};
|
||||
}
|
||||
if (temporal.sourceChanged) {
|
||||
return {
|
||||
key: 'source-changed',
|
||||
label: 'source changed',
|
||||
icon: FileClock,
|
||||
tone: 'border-warning/40 bg-warning/[0.06] text-warning',
|
||||
line: temporal.renderedChanged
|
||||
? 'The compose model changed since the last deploy.'
|
||||
: 'The compose file changed since the last deploy (formatting only).',
|
||||
};
|
||||
}
|
||||
return {
|
||||
key: 'matches',
|
||||
label: 'matches last deploy',
|
||||
icon: FileCheck2,
|
||||
tone: 'border-success/40 bg-success/[0.06] text-success',
|
||||
line: 'The compose source is unchanged since the last deploy.',
|
||||
};
|
||||
}
|
||||
|
||||
function Finding({ finding }: { finding: StackDriftFinding }) {
|
||||
return (
|
||||
<div className="border-t border-muted py-2 first:border-t-0">
|
||||
@@ -84,6 +137,26 @@ function Finding({ finding }: { finding: StackDriftFinding }) {
|
||||
);
|
||||
}
|
||||
|
||||
function LedgerRow({ entry }: { entry: DriftLedgerEntry }) {
|
||||
const resolved = entry.resolvedAt != null;
|
||||
return (
|
||||
<div className="border-t border-muted py-2 first:border-t-0">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="rounded-md bg-brand/15 px-1.5 py-0.5 font-mono text-[11px] text-brand">{entry.service}</span>
|
||||
<span className="font-mono text-[10px] uppercase tracking-wide text-stat-subtitle">{FINDING_LABEL[entry.kind] ?? entry.kind}</span>
|
||||
<span className={cn('font-mono text-[10px] uppercase tracking-wide', resolved ? 'text-success' : 'text-warning')}>
|
||||
{resolved ? 'resolved' : 'open'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 text-[12px] text-foreground/90">{entry.message}</div>
|
||||
<div className="mt-1 font-mono text-[10px] text-stat-subtitle">
|
||||
detected {formatTimeAgo(entry.detectedAt)}
|
||||
{entry.resolvedAt != null ? ` · resolved ${formatTimeAgo(entry.resolvedAt)}` : ''}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
const { activeNode } = useNodes();
|
||||
const nodeId = activeNode?.id;
|
||||
@@ -91,16 +164,16 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [loadError, setLoadError] = useState(false);
|
||||
const [reloadKey, setReloadKey] = useState(0);
|
||||
const [rechecking, setRechecking] = useState(false);
|
||||
|
||||
// Refetch when the stack OR the active node changes (the same stack can exist on
|
||||
// two nodes), and on an explicit re-check. Drift is a point-in-time snapshot, so
|
||||
// a failed load shows a distinct retry state rather than a stale or blank report.
|
||||
// Passive load when the stack OR active node changes (the same stack can exist on
|
||||
// two nodes), and on an explicit retry. Read-only: it never writes the ledger, so
|
||||
// opening the tab has no side effects. A failed load shows a distinct retry state
|
||||
// rather than a stale or blank report.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
const run = async () => {
|
||||
setLoading(true);
|
||||
// Clear any prior failure so an in-flight re-check shows the checking
|
||||
// affordance instead of leaving the error card up.
|
||||
setLoadError(false);
|
||||
try {
|
||||
const res = await apiFetch(`/stacks/${stackName}/drift`);
|
||||
@@ -125,8 +198,34 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
return () => { cancelled = true; };
|
||||
}, [stackName, nodeId, reloadKey]);
|
||||
|
||||
// Re-check reconciles the ledger server-side (recording newly detected / resolved
|
||||
// findings) and returns the fresh payload, so the history reflects this check.
|
||||
const recheck = async () => {
|
||||
setRechecking(true);
|
||||
try {
|
||||
const res = await apiFetch(`/stacks/${stackName}/drift/recheck`, { method: 'POST' });
|
||||
if (!res.ok) {
|
||||
toast.error('Failed to re-check drift.');
|
||||
return;
|
||||
}
|
||||
setReport((await res.json()) as StackDriftReport);
|
||||
setLoadError(false);
|
||||
} catch {
|
||||
toast.error('Failed to re-check drift.');
|
||||
} finally {
|
||||
setRechecking(false);
|
||||
}
|
||||
};
|
||||
|
||||
const meta = report ? STATUS_META[report.status] : null;
|
||||
const StatusIcon = meta?.icon;
|
||||
// Only render the temporal card when the payload actually carries it. A report
|
||||
// proxied from an older node without the ledger layer omits it; showing "no deploy
|
||||
// baseline" there would be misleading, so the card is left out entirely.
|
||||
const temporal = report?.temporal ? temporalMeta(report.temporal) : null;
|
||||
const TemporalIcon = temporal?.icon;
|
||||
const ledger = report?.ledger ?? [];
|
||||
const busy = loading || rechecking;
|
||||
|
||||
return (
|
||||
<div data-testid="drift-panel" className="flex-1 min-h-0 overflow-y-auto px-3 py-3 flex flex-col gap-4">
|
||||
@@ -135,11 +234,11 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
<button
|
||||
type="button"
|
||||
data-testid="drift-recheck-btn"
|
||||
onClick={() => setReloadKey(k => k + 1)}
|
||||
disabled={loading}
|
||||
onClick={recheck}
|
||||
disabled={busy}
|
||||
className={ACTION_CLASS}
|
||||
>
|
||||
<RefreshCw className={cn('h-3 w-3', loading && 'animate-spin')} strokeWidth={1.5} /> re-check
|
||||
<RefreshCw className={cn('h-3 w-3', busy && 'animate-spin')} strokeWidth={1.5} /> re-check
|
||||
</button>
|
||||
</div>
|
||||
|
||||
@@ -160,7 +259,7 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
) : (
|
||||
<>
|
||||
{meta && StatusIcon && (
|
||||
<div data-testid="drift-status" data-status={report.status} className={cn('rounded-lg border px-3 py-2.5', meta.tone)}>
|
||||
<div data-testid="drift-status" data-status={report.status} className={cn(CARD_CLASS, meta.tone)}>
|
||||
<div className="flex items-center gap-2">
|
||||
<StatusIcon className="h-4 w-4 shrink-0" strokeWidth={1.5} />
|
||||
<span className="font-mono text-[11px] uppercase tracking-wide">{meta.label}</span>
|
||||
@@ -174,6 +273,16 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{temporal && TemporalIcon && (
|
||||
<div data-testid="drift-temporal" data-temporal={temporal.key} className={cn(CARD_CLASS, temporal.tone)}>
|
||||
<div className="flex items-center gap-2">
|
||||
<TemporalIcon className="h-4 w-4 shrink-0" strokeWidth={1.5} />
|
||||
<span className="font-mono text-[11px] uppercase tracking-wide">{temporal.label}</span>
|
||||
</div>
|
||||
<div className="mt-1 font-mono text-[11px] leading-relaxed text-foreground/80">{temporal.line}</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{report.parseError && (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/[0.06] px-3 py-2 font-mono text-[11px] text-destructive">
|
||||
{report.parseError}
|
||||
@@ -190,6 +299,17 @@ export default function DriftPanel({ stackName }: { stackName: string }) {
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
|
||||
{ledger.length > 0 && (
|
||||
<section>
|
||||
<div className={cn(LABEL_CLASS, 'mb-1.5')}>drift history</div>
|
||||
<div className="rounded-lg border border-muted bg-card/40 px-3 py-1">
|
||||
{ledger.map((e, i) => (
|
||||
<LedgerRow key={`${e.service}-${e.kind}-${e.detectedAt}-${i}`} entry={e} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import {
|
||||
Rocket, RefreshCcw, CircleStop, Play, ArrowUp, Activity, Loader2, AlertCircle,
|
||||
TriangleAlert, CircleCheck,
|
||||
} from 'lucide-react';
|
||||
import type { LucideIcon } from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
@@ -33,6 +34,8 @@ const CATEGORY_ICON: Record<string, LucideIcon> = {
|
||||
stack_stopped: CircleStop,
|
||||
stack_started: Play,
|
||||
image_update_applied: ArrowUp,
|
||||
drift_detected: TriangleAlert,
|
||||
drift_resolved: CircleCheck,
|
||||
};
|
||||
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
Reference in New Issue
Block a user