mirror of
https://github.com/Studio-Saelix/sencho.git
synced 2026-07-26 11:49:16 +00:00
feat(drift): add an opt-in background drift scanner
Builds on the deploy-time reconcile and last-checked timestamp to keep the drift ledger current without an operator opening each Drift tab. A new DriftScanService (singleton, base-tick timer, off by default) periodically reconciles every stack on each local node, recording drift and surfacing it in the activity feed. Local nodes only (a remote node runs its own instance and scans itself); detection only, it never changes a running stack to match its compose. Configured under Settings, Automation, Drift detection (drift_scan_enabled, drift_scan_interval_minutes 15-1440), wired through the generic /settings API with matching Zod validation and a toggle plus interval control. Tests cover the scanner gating and local-only filter, reconcileNode (shared snapshot, unreachable skip, per-stack failure), and the settings and section payload.
This commit is contained in:
@@ -246,9 +246,11 @@ describe('DriftLedgerService.reconcile', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.reconcileStack', () => {
|
||||
describe('DriftLedgerService.reconcileStack / reconcileNode', () => {
|
||||
const STACK_A = 'recstacka';
|
||||
const STACK_B = 'recstackb';
|
||||
let dirA: string;
|
||||
let dirB: string;
|
||||
|
||||
const composeDir = () => process.env.COMPOSE_DIR as string;
|
||||
|
||||
@@ -260,14 +262,19 @@ describe('DriftLedgerService.reconcileStack', () => {
|
||||
|
||||
beforeEach(() => {
|
||||
clearLedger(STACK_A);
|
||||
clearLedger(STACK_B);
|
||||
dirA = path.join(composeDir(), STACK_A);
|
||||
fs.mkdirSync(dirA, { recursive: true });
|
||||
fs.writeFileSync(path.join(dirA, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
dirB = path.join(composeDir(), STACK_B);
|
||||
for (const d of [dirA, dirB]) {
|
||||
fs.mkdirSync(d, { recursive: true });
|
||||
fs.writeFileSync(path.join(d, 'compose.yaml'), 'services:\n web:\n image: nginx:1.27\n');
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
fs.rmSync(dirA, { recursive: true, force: true });
|
||||
fs.rmSync(dirB, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('reconcileStack builds the report, persists drift, and stamps last-checked', async () => {
|
||||
@@ -279,6 +286,47 @@ describe('DriftLedgerService.reconcileStack', () => {
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_A)).toHaveLength(1);
|
||||
expect(typeof db().getStackDossier(nodeId, STACK_A)?.last_drift_check_at).toBe('number');
|
||||
});
|
||||
|
||||
it('reconcileNode reconciles every stack against a single shared snapshot', async () => {
|
||||
const getSnap = vi.fn().mockResolvedValue({
|
||||
containers: [driftedContainer(STACK_A), driftedContainer(STACK_B)], networks: [], volumes: [],
|
||||
});
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: getSnap } as unknown as DockerController);
|
||||
const res = await DriftLedgerService.getInstance().reconcileNode(nodeId);
|
||||
expect(getSnap).toHaveBeenCalledTimes(1); // one snapshot for the whole node, not one per stack
|
||||
expect(res.detected).toBe(2);
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_A)).toHaveLength(1);
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_B)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('reconcileNode logs and skips a stack that fails mid-scan, still scanning the rest', async () => {
|
||||
// Make STACK_B unreadable (no compose file); STACK_A stays valid and drifted.
|
||||
fs.rmSync(path.join(dirB, 'compose.yaml'), { force: true });
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [driftedContainer(STACK_A)], networks: [], volumes: [] }),
|
||||
} as unknown as DockerController);
|
||||
const res = await DriftLedgerService.getInstance().reconcileNode(nodeId);
|
||||
// STACK_A scanned and recorded; STACK_B failed and is excluded, the scan continued.
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_A)).toHaveLength(1);
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_B)).toHaveLength(0);
|
||||
expect(res.detected).toBe(1);
|
||||
});
|
||||
|
||||
it('reconcileNode skips the whole node when Docker is unreachable (open findings are not falsely resolved)', async () => {
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [driftedContainer(STACK_A)], networks: [], volumes: [] }),
|
||||
} as unknown as DockerController);
|
||||
await DriftLedgerService.getInstance().reconcileStack(nodeId, STACK_A);
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_A)).toHaveLength(1);
|
||||
|
||||
vi.restoreAllMocks();
|
||||
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
|
||||
getDependencySnapshot: vi.fn().mockRejectedValue(new Error('docker down')),
|
||||
} as unknown as DockerController);
|
||||
const res = await DriftLedgerService.getInstance().reconcileNode(nodeId);
|
||||
expect(res).toEqual({ stacks: 0, detected: 0, resolved: 0 });
|
||||
expect(db().getOpenDriftFindings(nodeId, STACK_A)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('DriftLedgerService.recordBaseline', () => {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* DriftScanService: the opt-in background scanner that periodically reconciles
|
||||
* every local stack. These tests drive tick() directly (not the timer) and stub
|
||||
* reconcileNode, so they assert the gating and due-interval logic without touching
|
||||
* Docker: off => no scan, on+due => one reconcile per local node, and a second
|
||||
* tick inside the interval is skipped.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
|
||||
|
||||
let tmpDir: string;
|
||||
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
|
||||
let DriftLedgerService: typeof import('../services/DriftLedgerService').DriftLedgerService;
|
||||
let DriftScanService: typeof import('../services/DriftScanService').DriftScanService;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = await setupTestDb();
|
||||
({ DatabaseService } = await import('../services/DatabaseService'));
|
||||
({ DriftLedgerService } = await import('../services/DriftLedgerService'));
|
||||
({ DriftScanService } = await import('../services/DriftScanService'));
|
||||
});
|
||||
|
||||
afterAll(() => cleanupTestDb(tmpDir));
|
||||
|
||||
function db() {
|
||||
return DatabaseService.getInstance();
|
||||
}
|
||||
|
||||
function localNodeCount(): number {
|
||||
return db().getNodes().filter(n => n.type === 'local').length;
|
||||
}
|
||||
|
||||
describe('DriftScanService', () => {
|
||||
beforeEach(() => {
|
||||
// Reset the singleton so each test starts with lastScanAt = 0 (a scan is due).
|
||||
(DriftScanService as unknown as { instance: unknown }).instance = null;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
db().updateGlobalSetting('drift_scan_enabled', '0');
|
||||
db().updateGlobalSetting('drift_scan_interval_minutes', '60');
|
||||
});
|
||||
|
||||
it('does not scan when drift_scan_enabled is off', async () => {
|
||||
db().updateGlobalSetting('drift_scan_enabled', '0');
|
||||
const spy = vi.spyOn(DriftLedgerService.getInstance(), 'reconcileNode').mockResolvedValue({ stacks: 0, detected: 0, resolved: 0 });
|
||||
await DriftScanService.getInstance().tick();
|
||||
expect(spy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reconciles every local node when enabled and a scan is due', async () => {
|
||||
db().updateGlobalSetting('drift_scan_enabled', '1');
|
||||
db().updateGlobalSetting('drift_scan_interval_minutes', '60');
|
||||
const spy = vi.spyOn(DriftLedgerService.getInstance(), 'reconcileNode').mockResolvedValue({ stacks: 0, detected: 0, resolved: 0 });
|
||||
await DriftScanService.getInstance().tick();
|
||||
const locals = db().getNodes().filter(n => n.type === 'local');
|
||||
expect(locals.length).toBeGreaterThan(0);
|
||||
expect(spy).toHaveBeenCalledTimes(locals.length);
|
||||
for (const n of locals) expect(spy).toHaveBeenCalledWith(n.id);
|
||||
});
|
||||
|
||||
it('reconciles local nodes only, never a remote node', async () => {
|
||||
db().updateGlobalSetting('drift_scan_enabled', '1');
|
||||
db().updateGlobalSetting('drift_scan_interval_minutes', '60');
|
||||
const remoteId = db().getDb().prepare(
|
||||
"INSERT INTO nodes (name, type, compose_dir, is_default, status, created_at) VALUES ('remote-scan-x', 'remote', '', 0, 'online', ?)"
|
||||
).run(Date.now()).lastInsertRowid as number;
|
||||
const spy = vi.spyOn(DriftLedgerService.getInstance(), 'reconcileNode').mockResolvedValue({ stacks: 0, detected: 0, resolved: 0 });
|
||||
try {
|
||||
await DriftScanService.getInstance().tick();
|
||||
const localIds = db().getNodes().filter(n => n.type === 'local').map(n => n.id);
|
||||
expect(localIds.length).toBeGreaterThan(0);
|
||||
for (const id of localIds) expect(spy).toHaveBeenCalledWith(id);
|
||||
// A remote node runs its own Sencho instance and scans itself.
|
||||
expect(spy).not.toHaveBeenCalledWith(remoteId);
|
||||
} finally {
|
||||
db().getDb().prepare('DELETE FROM nodes WHERE id = ?').run(remoteId);
|
||||
}
|
||||
});
|
||||
|
||||
it('skips a second tick that fires before the interval elapses', async () => {
|
||||
db().updateGlobalSetting('drift_scan_enabled', '1');
|
||||
db().updateGlobalSetting('drift_scan_interval_minutes', '60');
|
||||
const spy = vi.spyOn(DriftLedgerService.getInstance(), 'reconcileNode').mockResolvedValue({ stacks: 0, detected: 0, resolved: 0 });
|
||||
const svc = DriftScanService.getInstance();
|
||||
await svc.tick(); // due (lastScanAt = 0) => scans
|
||||
await svc.tick(); // within the 60-minute interval => skipped
|
||||
expect(spy).toHaveBeenCalledTimes(localNodeCount()); // not doubled
|
||||
});
|
||||
});
|
||||
@@ -241,6 +241,60 @@ describe('prune_on_update (auto-prune after updates)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('drift detection scan settings', () => {
|
||||
it('seeds off with a 60 minute interval in a fresh database', () => {
|
||||
const s = DatabaseService.getInstance().getGlobalSettings();
|
||||
expect(s.drift_scan_enabled).toBe('0');
|
||||
expect(s.drift_scan_interval_minutes).toBe('60');
|
||||
});
|
||||
|
||||
it('exposes both keys through the settings GET projection', async () => {
|
||||
const res = await request(app).get('/api/settings').set('Cookie', adminCookie);
|
||||
expect(res.body.drift_scan_enabled).toBeDefined();
|
||||
expect(res.body.drift_scan_interval_minutes).toBeDefined();
|
||||
});
|
||||
|
||||
it('accepts a well-formed enable + interval write and persists it', async () => {
|
||||
const res = await request(app)
|
||||
.patch('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ drift_scan_enabled: '1', drift_scan_interval_minutes: '120' });
|
||||
expect(res.status).toBe(200);
|
||||
const s = DatabaseService.getInstance().getGlobalSettings();
|
||||
expect(s.drift_scan_enabled).toBe('1');
|
||||
expect(s.drift_scan_interval_minutes).toBe('120');
|
||||
// Restore the shipped defaults so later suites observe seeded behavior.
|
||||
DatabaseService.getInstance().updateGlobalSetting('drift_scan_enabled', '0');
|
||||
DatabaseService.getInstance().updateGlobalSetting('drift_scan_interval_minutes', '60');
|
||||
});
|
||||
|
||||
it('rejects a non-enum drift_scan_enabled value (400) and does not write it', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'drift_scan_enabled', value: 'maybe' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().drift_scan_enabled).not.toBe('maybe');
|
||||
});
|
||||
|
||||
it('rejects an out-of-range interval below the 15 minute floor (400)', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', adminCookie)
|
||||
.send({ key: 'drift_scan_interval_minutes', value: '5' });
|
||||
expect(res.status).toBe(400);
|
||||
expect(DatabaseService.getInstance().getGlobalSettings().drift_scan_interval_minutes).not.toBe('5');
|
||||
});
|
||||
|
||||
it('rejects a non-admin write (403)', async () => {
|
||||
const res = await request(app)
|
||||
.post('/api/settings')
|
||||
.set('Cookie', viewerCookie)
|
||||
.send({ key: 'drift_scan_enabled', value: '1' });
|
||||
expect(res.status).toBe(403);
|
||||
});
|
||||
});
|
||||
|
||||
describe('health gate settings', () => {
|
||||
it('seeds enabled with a 90 second window in a fresh database', () => {
|
||||
const settings = DatabaseService.getInstance().getGlobalSettings();
|
||||
|
||||
@@ -3,6 +3,7 @@ import { DatabaseService } from '../services/DatabaseService';
|
||||
import { LicenseService } from '../services/LicenseService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { DriftScanService } from '../services/DriftScanService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
@@ -32,6 +33,7 @@ export function installShutdownHandlers(server: Server): void {
|
||||
console.warn('[Shutdown] MonitorService cleanup failed:', (e as Error).message);
|
||||
}
|
||||
try { AutoHealService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] AutoHealService cleanup failed:', (e as Error).message); }
|
||||
try { DriftScanService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] DriftScanService cleanup failed:', (e as Error).message); }
|
||||
try { HealthGateService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] HealthGateService cleanup failed:', (e as Error).message); }
|
||||
try { FleetSyncRetryService.getInstance().stop(); } catch (e) { console.warn('[Shutdown] FleetSyncRetryService cleanup failed:', (e as Error).message); }
|
||||
try { DockerEventManager.getInstance().stop(); } catch (e) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import SelfUpdateService from '../services/SelfUpdateService';
|
||||
import SelfIdentityService from '../services/SelfIdentityService';
|
||||
import { MonitorService } from '../services/MonitorService';
|
||||
import { AutoHealService } from '../services/AutoHealService';
|
||||
import { DriftScanService } from '../services/DriftScanService';
|
||||
import { HealthGateService } from '../services/HealthGateService';
|
||||
import { FleetSyncRetryService } from '../services/FleetSyncRetryService';
|
||||
import { DockerEventManager } from '../services/DockerEventManager';
|
||||
@@ -119,6 +120,7 @@ export async function startServer(server: Server): Promise<void> {
|
||||
// safely run alongside the async initializers below.
|
||||
MonitorService.getInstance().start();
|
||||
AutoHealService.getInstance().start();
|
||||
DriftScanService.getInstance().start();
|
||||
HealthGateService.getInstance().start();
|
||||
FleetSyncRetryService.getInstance().start();
|
||||
ImageUpdateService.getInstance().start();
|
||||
|
||||
@@ -30,6 +30,8 @@ const ALLOWED_SETTING_KEYS = new Set([
|
||||
'health_gate_enabled',
|
||||
'health_gate_window_seconds',
|
||||
'env_block_deploy_on_missing_required',
|
||||
'drift_scan_enabled',
|
||||
'drift_scan_interval_minutes',
|
||||
]);
|
||||
|
||||
// Keys whose write requires a paid license, not just an admin role.
|
||||
@@ -58,6 +60,8 @@ const SettingsPatchSchema = z.object({
|
||||
health_gate_enabled: z.enum(['0', '1']),
|
||||
health_gate_window_seconds: z.coerce.number().int().min(15).max(600).transform(String),
|
||||
env_block_deploy_on_missing_required: z.enum(['0', '1']),
|
||||
drift_scan_enabled: z.enum(['0', '1']),
|
||||
drift_scan_interval_minutes: z.coerce.number().int().min(15).max(1440).transform(String),
|
||||
}).partial();
|
||||
|
||||
export const settingsRouter = Router();
|
||||
|
||||
@@ -1497,6 +1497,8 @@ export class DatabaseService {
|
||||
stmt.run('health_gate_window_seconds', '90');
|
||||
stmt.run('image_update_check_interval_minutes', '120');
|
||||
stmt.run('env_block_deploy_on_missing_required', '0');
|
||||
stmt.run('drift_scan_enabled', '0');
|
||||
stmt.run('drift_scan_interval_minutes', '60');
|
||||
|
||||
// Seed the default local node if none exists
|
||||
const nodeCount = (this.db.prepare('SELECT COUNT(*) as count FROM nodes').get() as any)?.count || 0;
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import type { StackDriftFindingRow } from './DatabaseService';
|
||||
import { FileSystemService } from './FileSystemService';
|
||||
import DockerController from './DockerController';
|
||||
import type { DependencySnapshot } from './DockerController';
|
||||
import { parseComposeDependencies } from '../helpers/composeDependencyParse';
|
||||
import type { DeclaredCompose } from '../helpers/composeDependencyParse';
|
||||
import { sha256Hex } from '../utils/hashing';
|
||||
import { sanitizeForLog } from '../utils/safeLog';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
import { buildStackDriftReport } from './DriftDetectionService';
|
||||
import { assembleStackDrift, buildStackDriftReport } from './DriftDetectionService';
|
||||
import type { StackDriftReport, StackDriftFinding } from './DriftDetectionService';
|
||||
|
||||
/**
|
||||
@@ -33,6 +35,13 @@ export interface DriftReconcileResult {
|
||||
resolved: number;
|
||||
}
|
||||
|
||||
/** A whole-node reconcile outcome: a stack result plus the number of stacks
|
||||
* scanned cleanly. A stack that throws mid-scan is logged and excluded from the
|
||||
* count, and the scan continues with the rest. */
|
||||
export interface DriftNodeReconcileResult extends DriftReconcileResult {
|
||||
stacks: number;
|
||||
}
|
||||
|
||||
/** Stable identity for a finding across checks: same service + kind is the same finding. */
|
||||
function findingKey(service: string, kind: string): string {
|
||||
return JSON.stringify([service, kind]);
|
||||
@@ -196,6 +205,48 @@ export class DriftLedgerService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile every stack on a node against a single Docker snapshot. The
|
||||
* background scanner drives this so drift is recorded (and its activity
|
||||
* surfaced) without an operator opening each Drift tab. One snapshot is shared
|
||||
* across all stacks to keep a full-node scan cheap. A Docker-unreachable node is
|
||||
* skipped wholesale rather than falsely resolving open findings; a single stack
|
||||
* failing (unreadable compose, etc.) is logged and the scan moves on.
|
||||
*/
|
||||
async reconcileNode(nodeId: number): Promise<DriftNodeReconcileResult> {
|
||||
const fs = FileSystemService.getInstance(nodeId);
|
||||
let stacks: string[];
|
||||
try {
|
||||
stacks = await fs.getStacks();
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] reconcileNode: failed to list stacks on node %d:', nodeId, sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
return { stacks: 0, detected: 0, resolved: 0 };
|
||||
}
|
||||
let snapshot: DependencySnapshot;
|
||||
try {
|
||||
snapshot = await DockerController.getInstance(nodeId).getDependencySnapshot(stacks);
|
||||
} catch (error) {
|
||||
console.warn('[DriftLedger] reconcileNode: snapshot unavailable on node %d; scan skipped:', nodeId, sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
return { stacks: 0, detected: 0, resolved: 0 };
|
||||
}
|
||||
let detected = 0, resolved = 0, scanned = 0;
|
||||
for (const stackName of stacks) {
|
||||
try {
|
||||
const content = await fs.getStackContent(stackName);
|
||||
const declared = parseComposeDependencies(content);
|
||||
const containers = snapshot.containers.filter(c => c.stack === stackName);
|
||||
const report = assembleStackDrift({ stack: stackName, declared, containers, networks: snapshot.networks, parseError: declared.parseError });
|
||||
const r = this.reconcile(nodeId, stackName, report);
|
||||
detected += r.detected;
|
||||
resolved += r.resolved;
|
||||
scanned += 1;
|
||||
} catch (error) {
|
||||
console.error('[DriftLedger] reconcileNode: failed for stack %s:', sanitizeForLog(stackName), sanitizeForLog(getErrorMessage(error, 'unknown')));
|
||||
}
|
||||
}
|
||||
return { stacks: scanned, detected, resolved };
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a drift transition to the stack activity timeline. History-only (no
|
||||
* external channel dispatch): a drift signal belongs in the activity feed, not
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { DatabaseService } from './DatabaseService';
|
||||
import { DriftLedgerService } from './DriftLedgerService';
|
||||
import { isDebugEnabled } from '../utils/debug';
|
||||
import { getErrorMessage } from '../utils/errors';
|
||||
|
||||
/**
|
||||
* Opt-in background drift scanner. The drift ledger (DriftLedgerService) only
|
||||
* advances when something reconciles a stack: a manual re-check or a deploy. With
|
||||
* neither, the persisted history and its activity entries never move, so the Drift
|
||||
* tab can show live drift the history never recorded. This service closes that gap:
|
||||
* when enabled, it periodically reconciles every stack on each local node (one
|
||||
* reconcileNode call per node) so drift is recorded and surfaced in the activity
|
||||
* feed without an operator opening each Drift tab.
|
||||
*
|
||||
* Local nodes only: a remote node runs its own Sencho instance and scans itself.
|
||||
* Off by default (drift_scan_enabled); the interval is drift_scan_interval_minutes.
|
||||
* A fixed base tick re-reads both settings each minute, so enabling, disabling, or
|
||||
* changing the interval takes effect on the next tick without a restart or a timer
|
||||
* reschedule, which is why the plain global-settings keys suffice (no side-effect
|
||||
* endpoint needed). Detection only: it never auto-reconciles the runtime to compose.
|
||||
*/
|
||||
|
||||
const BASE_TICK_MS = 60_000; // re-read settings and check whether a scan is due, once a minute
|
||||
const INITIAL_DELAY_MS = 45_000; // let Docker and the node registry settle before the first scan
|
||||
const DEFAULT_INTERVAL_MINUTES = 60;
|
||||
const MIN_INTERVAL_MINUTES = 15;
|
||||
|
||||
export class DriftScanService {
|
||||
private static instance: DriftScanService;
|
||||
private tickTimer: NodeJS.Timeout | null = null;
|
||||
private initialTimer: NodeJS.Timeout | null = null;
|
||||
private isScanning = false;
|
||||
private lastScanAt = 0;
|
||||
|
||||
private constructor() {}
|
||||
|
||||
static getInstance(): DriftScanService {
|
||||
if (!DriftScanService.instance) DriftScanService.instance = new DriftScanService();
|
||||
return DriftScanService.instance;
|
||||
}
|
||||
|
||||
start(): void {
|
||||
if (this.initialTimer || this.tickTimer) return;
|
||||
this.initialTimer = setTimeout(() => {
|
||||
void this.tick();
|
||||
this.tickTimer = setInterval(() => void this.tick(), BASE_TICK_MS);
|
||||
}, INITIAL_DELAY_MS);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
if (this.initialTimer) {
|
||||
clearTimeout(this.initialTimer);
|
||||
this.initialTimer = null;
|
||||
}
|
||||
if (this.tickTimer) {
|
||||
clearInterval(this.tickTimer);
|
||||
this.tickTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Fail safe: a settings read error disables scanning rather than scanning unasked. */
|
||||
private isEnabled(): boolean {
|
||||
try {
|
||||
return DatabaseService.getInstance().getGlobalSettings()['drift_scan_enabled'] === '1';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private intervalMs(): number {
|
||||
try {
|
||||
const raw = Number(DatabaseService.getInstance().getGlobalSettings()['drift_scan_interval_minutes']);
|
||||
const minutes = Number.isFinite(raw) && raw >= MIN_INTERVAL_MINUTES ? raw : DEFAULT_INTERVAL_MINUTES;
|
||||
return minutes * 60_000;
|
||||
} catch {
|
||||
return DEFAULT_INTERVAL_MINUTES * 60_000;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Base-tick worker: scan only when enabled and the configured interval has
|
||||
* elapsed since the last scan. The isScanning guard drops a tick that fires
|
||||
* while a slow scan is still running rather than overlapping it.
|
||||
*/
|
||||
async tick(): Promise<void> {
|
||||
if (this.isScanning || !this.isEnabled()) return;
|
||||
const now = Date.now();
|
||||
if (now - this.lastScanAt < this.intervalMs()) return;
|
||||
|
||||
this.isScanning = true;
|
||||
this.lastScanAt = now;
|
||||
try {
|
||||
const nodes = DatabaseService.getInstance().getNodes().filter(n => n.type === 'local');
|
||||
for (const node of nodes) {
|
||||
if (node.id === undefined) continue;
|
||||
const result = await DriftLedgerService.getInstance().reconcileNode(node.id);
|
||||
if (isDebugEnabled() && (result.detected > 0 || result.resolved > 0)) {
|
||||
console.log(`[DriftScan:diag] node ${node.id}: ${result.stacks} stack(s), +${result.detected} detected, -${result.resolved} resolved`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('[DriftScan] scan failed:', getErrorMessage(error, 'unknown'));
|
||||
} finally {
|
||||
this.isScanning = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,12 +45,16 @@ Image references are compared after normalizing the implicit Docker Hub registry
|
||||
|
||||
## Drift history
|
||||
|
||||
Each time you **re-check** a stack, and after every deploy, Sencho records the findings it sees. The **Drift history** list under the findings shows recent entries with when each was first **detected** and, once it clears, when it was **resolved**. This turns a point-in-time check into a short ledger of how a stack has drifted and recovered over time.
|
||||
Sencho records the findings it sees whenever a stack is checked: each time you **re-check**, after every deploy, and on each background scan when one is enabled. The **Drift history** list under the findings shows recent entries with when each was first **detected** and, once it clears, when it was **resolved**, turning a point-in-time check into a short ledger of how a stack has drifted and recovered over time.
|
||||
|
||||
The history is labelled with when it was last checked. The status badge at the top is always live, recomputed each time you open the tab, while the history reflects the last time the ledger was reconciled. When the two differ, re-check to bring the history up to date.
|
||||
|
||||
Drift that appears or clears is also written to the stack's **Activity** timeline, so **Drift detected** and **Drift resolved** events sit alongside deploys and restarts.
|
||||
|
||||
### Background drift detection
|
||||
|
||||
Out of the box the history advances when you re-check a stack or deploy it. To keep it current without opening each tab, turn on **Drift detection** under **Settings → Automation**: each node then reconciles every stack on a schedule, recording drift and surfacing it in the activity feed on its own. Choose how often the scan runs in the same place. The scan only detects and records drift; it never changes a running stack to match its Compose file. It is off by default and configured per node.
|
||||
|
||||
## Accessing the Drift tab
|
||||
|
||||
1. Click any stack in the left sidebar to open it.
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { Skeleton } from '@/components/ui/skeleton';
|
||||
import { RefreshCw } from 'lucide-react';
|
||||
import { apiFetch } from '@/lib/api';
|
||||
import { toast } from '@/components/ui/toast-store';
|
||||
import { useNodes } from '@/context/NodeContext';
|
||||
import { useAuth } from '@/context/AuthContext';
|
||||
import { DEFAULT_SETTINGS } from './types';
|
||||
import type { PatchableSettings } from './types';
|
||||
import { SettingsSection } from './SettingsSection';
|
||||
import { SettingsField } from './SettingsField';
|
||||
import { SettingsActions, SettingsPrimaryButton } from './SettingsActions';
|
||||
import { useMastheadStats } from './MastheadStatsContext';
|
||||
import { useSettingsDirty } from './useSettingsDirty';
|
||||
import { TogglePill } from '@/components/ui/toggle-pill';
|
||||
import { NumberChip } from './SystemControls';
|
||||
|
||||
interface DriftDetectionSectionProps {
|
||||
onDirtyChange?: (dirty: boolean) => void;
|
||||
}
|
||||
|
||||
function SectionSkeleton() {
|
||||
return (
|
||||
<div className="space-y-3 rounded-lg border border-glass-border bg-glass p-4">
|
||||
<Skeleton className="h-10 w-full" />
|
||||
<Skeleton className="h-10 w-full" />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type DriftScanFields = Pick<PatchableSettings, 'drift_scan_enabled' | 'drift_scan_interval_minutes'>;
|
||||
|
||||
const DEFAULT_DRIFT_SCAN: DriftScanFields = {
|
||||
drift_scan_enabled: DEFAULT_SETTINGS.drift_scan_enabled,
|
||||
drift_scan_interval_minutes: DEFAULT_SETTINGS.drift_scan_interval_minutes,
|
||||
};
|
||||
|
||||
export function DriftDetectionSection({ onDirtyChange }: DriftDetectionSectionProps) {
|
||||
const { activeNode } = useNodes();
|
||||
const { isAdmin } = useAuth();
|
||||
const readOnly = !isAdmin;
|
||||
const { settings, setSettings, dirtyCount, hasChanges, reset, markSaved } = useSettingsDirty<DriftScanFields>({ ...DEFAULT_DRIFT_SCAN });
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [isSaving, setIsSaving] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
onDirtyChange?.(hasChanges);
|
||||
}, [hasChanges, onDirtyChange]);
|
||||
|
||||
useMastheadStats(
|
||||
isLoading
|
||||
? null
|
||||
: [
|
||||
{
|
||||
label: 'EDITED',
|
||||
value: hasChanges ? `${dirtyCount} pending` : 'saved',
|
||||
tone: hasChanges ? 'warn' : 'value',
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchSettings = async () => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const res = await apiFetch('/settings');
|
||||
const data: Record<string, string> = res.ok ? await res.json() : {};
|
||||
reset({
|
||||
drift_scan_enabled: (data.drift_scan_enabled as '0' | '1') ?? DEFAULT_SETTINGS.drift_scan_enabled,
|
||||
drift_scan_interval_minutes: data.drift_scan_interval_minutes ?? DEFAULT_SETTINGS.drift_scan_interval_minutes,
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to fetch drift detection settings', e);
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
fetchSettings();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [activeNode?.id]);
|
||||
|
||||
const onSettingChange = <K extends keyof DriftScanFields>(key: K, value: DriftScanFields[K]) => {
|
||||
setSettings(prev => ({ ...prev, [key]: value }));
|
||||
};
|
||||
|
||||
const saveSettings = async () => {
|
||||
const submitted = { ...settings };
|
||||
setIsSaving(true);
|
||||
try {
|
||||
const res = await apiFetch('/settings', {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(submitted),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({}));
|
||||
toast.error(err?.error || err?.message || 'Failed to save settings.');
|
||||
return;
|
||||
}
|
||||
markSaved(submitted);
|
||||
toast.success('Drift detection settings saved.');
|
||||
} catch (e: unknown) {
|
||||
toast.error((e as Error)?.message || 'Something went wrong.');
|
||||
} finally {
|
||||
setIsSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (isLoading) return <SectionSkeleton />;
|
||||
|
||||
return (
|
||||
<fieldset disabled={readOnly} className="m-0 flex min-w-0 flex-col gap-10 border-0 p-0">
|
||||
<SettingsSection title="Background drift detection" kicker="node-scoped">
|
||||
<SettingsField
|
||||
label="Scan stacks for drift on a schedule"
|
||||
helper="When on, this node periodically reconciles every stack so configuration drift is recorded in each stack's drift history and surfaced in the activity feed, without opening the Drift tab. Detection only: Sencho never changes a running stack to match its compose. Off by default."
|
||||
>
|
||||
<TogglePill
|
||||
checked={settings.drift_scan_enabled === '1'}
|
||||
onChange={(next) => onSettingChange('drift_scan_enabled', next ? '1' : '0')}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField
|
||||
label="Scan every"
|
||||
helper="How often a background scan runs while drift detection is on. The Drift tab still re-checks on demand and after every deploy, regardless of this interval."
|
||||
>
|
||||
<NumberChip
|
||||
value={settings.drift_scan_interval_minutes || '60'}
|
||||
onChange={(v) => onSettingChange('drift_scan_interval_minutes', v)}
|
||||
suffix="min"
|
||||
min={15}
|
||||
max={1440}
|
||||
step={15}
|
||||
/>
|
||||
</SettingsField>
|
||||
</SettingsSection>
|
||||
|
||||
<SettingsActions hint={readOnly ? 'Read-only · admin access required to edit' : (hasChanges ? `${dirtyCount} unsaved` : undefined)}>
|
||||
{!readOnly && (
|
||||
<SettingsPrimaryButton onClick={saveSettings} disabled={isSaving || !hasChanges}>
|
||||
{isSaving ? (
|
||||
<>
|
||||
<RefreshCw className="w-4 h-4 animate-spin" />
|
||||
Saving
|
||||
</>
|
||||
) : (
|
||||
'Save settings'
|
||||
)}
|
||||
</SettingsPrimaryButton>
|
||||
)}
|
||||
</SettingsActions>
|
||||
</fieldset>
|
||||
);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
HostAlertsSection,
|
||||
DockerStorageSection,
|
||||
UpdatesSection,
|
||||
DriftDetectionSection,
|
||||
FleetMeshSection,
|
||||
NotificationsSection,
|
||||
DeveloperSection,
|
||||
@@ -84,6 +85,7 @@ function renderSection(
|
||||
case 'host-alerts': return <HostAlertsSection onDirtyChange={(d) => onDirtyChange('host-alerts', d)} />;
|
||||
case 'docker-storage': return <DockerStorageSection onDirtyChange={(d) => onDirtyChange('docker-storage', d)} />;
|
||||
case 'image-updates': return <UpdatesSection />;
|
||||
case 'drift-scan': return <DriftDetectionSection onDirtyChange={(d) => onDirtyChange('drift-scan', d)} />;
|
||||
case 'fleet-mesh': return <FleetMeshSection onDirtyChange={(d) => onDirtyChange('fleet-mesh', d)} />;
|
||||
case 'notifications': return <NotificationsSection />;
|
||||
case 'notification-routing': return <NotificationRoutingSection />;
|
||||
|
||||
@@ -26,6 +26,7 @@ import { DockerStorageSection } from '../DockerStorageSection';
|
||||
import { FleetMeshSection } from '../FleetMeshSection';
|
||||
import { DataRetentionSection } from '../DataRetentionSection';
|
||||
import { DeveloperSection } from '../DeveloperSection';
|
||||
import { DriftDetectionSection } from '../DriftDetectionSection';
|
||||
|
||||
const mockedFetch = apiFetch as unknown as ReturnType<typeof vi.fn>;
|
||||
const mockedLicense = useLicense as unknown as ReturnType<typeof vi.fn>;
|
||||
@@ -133,4 +134,13 @@ describe('split section save payloads', () => {
|
||||
await waitFor(() => expect(mockedFetch.mock.calls.some(c => c[1]?.method === 'PATCH')).toBe(true));
|
||||
expect(patchedKeys()).toEqual(['developer_mode']);
|
||||
});
|
||||
|
||||
it('DriftDetectionSection patches only the drift scan keys', async () => {
|
||||
render(<DriftDetectionSection />);
|
||||
const save = await screen.findByRole('button', { name: /save settings/i });
|
||||
fireEvent.click(screen.getByRole('switch')); // drift_scan_enabled
|
||||
fireEvent.click(save);
|
||||
await waitFor(() => expect(mockedFetch.mock.calls.some(c => c[1]?.method === 'PATCH')).toBe(true));
|
||||
expect(patchedKeys()).toEqual(['drift_scan_enabled', 'drift_scan_interval_minutes']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@ export { LicenseSection } from './LicenseSection';
|
||||
export { HostAlertsSection } from './HostAlertsSection';
|
||||
export { DockerStorageSection } from './DockerStorageSection';
|
||||
export { UpdatesSection } from './UpdatesSection';
|
||||
export { DriftDetectionSection } from './DriftDetectionSection';
|
||||
export { FleetMeshSection } from './FleetMeshSection';
|
||||
export { NotificationsSection } from './NotificationsSection';
|
||||
export { DeveloperSection } from './DeveloperSection';
|
||||
|
||||
@@ -221,6 +221,15 @@ export const SETTINGS_ITEMS: readonly SettingsItemMeta[] = [
|
||||
tier: null,
|
||||
scope: 'node',
|
||||
},
|
||||
{
|
||||
id: 'drift-scan',
|
||||
group: 'automation',
|
||||
label: 'Drift detection',
|
||||
description: 'Periodically reconcile each stack on this node so configuration drift is recorded and surfaced without opening every Drift tab.',
|
||||
keywords: ['drift', 'scan', 'reconcile', 'compose', 'runtime', 'detection', 'background', 'interval', 'ledger', 'history'],
|
||||
tier: null,
|
||||
scope: 'node',
|
||||
},
|
||||
{
|
||||
id: 'webhooks',
|
||||
group: 'automation',
|
||||
|
||||
@@ -18,6 +18,8 @@ export interface PatchableSettings {
|
||||
health_gate_enabled?: '0' | '1';
|
||||
health_gate_window_seconds?: string;
|
||||
env_block_deploy_on_missing_required?: '0' | '1';
|
||||
drift_scan_enabled?: '0' | '1';
|
||||
drift_scan_interval_minutes?: string;
|
||||
}
|
||||
|
||||
export const DEFAULT_SETTINGS: PatchableSettings = {
|
||||
@@ -40,6 +42,8 @@ export const DEFAULT_SETTINGS: PatchableSettings = {
|
||||
health_gate_enabled: '1',
|
||||
health_gate_window_seconds: '90',
|
||||
env_block_deploy_on_missing_required: '0',
|
||||
drift_scan_enabled: '0',
|
||||
drift_scan_interval_minutes: '60',
|
||||
};
|
||||
|
||||
export type SectionId =
|
||||
@@ -54,6 +58,7 @@ export type SectionId =
|
||||
| 'host-alerts'
|
||||
| 'docker-storage'
|
||||
| 'image-updates'
|
||||
| 'drift-scan'
|
||||
| 'fleet-mesh'
|
||||
| 'notifications'
|
||||
| 'webhooks'
|
||||
|
||||
Reference in New Issue
Block a user