Merge remote-tracking branch 'origin/main' into docs/tutorials-batch-1

This commit is contained in:
SaelixCode
2026-07-24 16:00:58 -04:00
146 changed files with 7526 additions and 885 deletions
+6 -6
View File
@@ -78,10 +78,10 @@ body:
label: Subsystems involved
description: Tick any that apply.
options:
- label: Trivy / vulnerability scanning
- label: Pilot Agent (NAT tunnel)
- label: Sencho Mesh (cross-node networking)
- label: Fleet Actions / Secrets / Snapshots
- label: Cloud Backup
- label: SSO / LDAP
- label: Stacks, Compose & Deploy
- label: Multi-node, Pilot & Mesh
- label: Fleet (actions, sync, secrets, backups, blueprints)
- label: Alerts, Notifications & Observability
- label: Security, Scanning & Auth (Trivy, SSO, MFA, RBAC)
- label: Updates, Schedules & Automation
- label: None of the above
+7 -7
View File
@@ -5081,9 +5081,9 @@
"optional": true
},
"node_modules/nanoid": {
"version": "3.3.12",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz",
"integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==",
"version": "3.3.16",
"resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz",
"integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==",
"dev": true,
"funding": [
{
@@ -5428,9 +5428,9 @@
}
},
"node_modules/postcss": {
"version": "8.5.15",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz",
"integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==",
"version": "8.5.23",
"resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz",
"integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==",
"dev": true,
"funding": [
{
@@ -5448,7 +5448,7 @@
],
"license": "MIT",
"dependencies": {
"nanoid": "^3.3.12",
"nanoid": "^3.3.16",
"picocolors": "^1.1.1",
"source-map-js": "^1.2.1"
},
+2 -1
View File
@@ -30,7 +30,8 @@
"node": ">=26.0.0"
},
"overrides": {
"brace-expansion": "^5.0.7"
"brace-expansion": "^5.0.7",
"postcss": "^8.5.23"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
@@ -55,7 +55,7 @@ beforeEach(() => {
const db = DatabaseService.getInstance();
db.getDb().prepare('DELETE FROM auto_heal_history').run();
db.getDb().prepare('DELETE FROM auto_heal_policies').run();
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue();
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
+146 -2
View File
@@ -57,8 +57,8 @@ describe('GET /api/alerts', () => {
it('filters alerts by stackName query param', async () => {
// Seed two alerts for different stacks
const db = DatabaseService.getInstance();
db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 });
db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 });
const res = await request(app)
.get('/api/alerts?stackName=web')
@@ -105,10 +105,101 @@ describe('POST /api/alerts', () => {
expect(res.status).toBe(201);
expect(res.body.id).toBeDefined();
expect(res.body.stack_name).toBe('new-stack');
expect(res.body.service_name).toBeNull();
expect(res.body.metric).toBe('memory_percent');
expect(res.body.threshold).toBe(85);
});
it('persists a valid dotted service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'svc-stack',
service_name: 'api.web',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(201);
expect(res.body.service_name).toBe('api.web');
});
it('normalizes empty service_name to null', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'empty-svc',
service_name: '',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(201);
expect(res.body.service_name).toBeNull();
});
it('rejects whitespace-only service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'bad-svc',
service_name: ' ',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
});
it('rejects reserved _unlabeled service_name', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'bad-svc',
service_name: '_unlabeled',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
});
it('rejects service_name when the capability is disabled', async () => {
const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } =
await import('../services/CapabilityRegistry');
disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
try {
const res = await request(app)
.post('/api/alerts')
.set('Cookie', authCookie)
.send({
stack_name: 'cap-stack',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
});
expect(res.status).toBe(400);
expect(res.body.code).toBe('capability_unavailable');
} finally {
enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
}
});
it('validates required fields and returns 400 for missing data', async () => {
const res = await request(app)
.post('/api/alerts')
@@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => {
// Create an alert to delete
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'delete-me',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
@@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => {
expect(res.status).toBe(200);
expect(res.body.success).toBe(true);
});
it('rejects leading-junk ids like 1abc without deleting alert 1', async () => {
const created = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-junk',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
duration_mins: 0,
cooldown_mins: 0,
});
expect(created.id).toBeDefined();
const res = await request(app)
.delete(`/api/alerts/${created.id}abc`)
.set('Cookie', authCookie);
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert id');
expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true);
});
it('rejects fractional ids like 2.5 without deleting alert 2', async () => {
const first = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-fraction',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
});
const second = DatabaseService.getInstance().addStackAlert({
stack_name: 'strict-id-fraction',
service_name: null,
metric: 'memory_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
});
expect(second.id).toBeDefined();
const res = await request(app)
.delete(`/api/alerts/${second.id}.5`)
.set('Cookie', authCookie);
expect(res.status).toBe(400);
expect(res.body.error).toBe('Invalid alert id');
const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id);
expect(remaining).toEqual(expect.arrayContaining([first.id, second.id]));
});
});
// --- POST /api/notifications/test ---
@@ -76,4 +76,8 @@ describe('WebSocket API-token scope enforcement', () => {
it('does not scope-block a full-admin token from a generic socket', async () => {
expect(await upgradeStatus(createToken('full-admin'), '/ws')).not.toBe(403);
});
it('blocks a full-admin token from the host console (403)', async () => {
expect(await upgradeStatus(createToken('full-admin'), '/api/system/host-console')).toBe(403);
});
});
@@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => {
mockTier('paid');
mockDeployStack.mockResolvedValue(undefined);
const { NotificationService } = await import('../services/NotificationService');
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
expect(res.status).toBe(200);
@@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => {
mockTier('paid');
mockDeployStack.mockRejectedValueOnce(new Error('restore failed'));
const { NotificationService } = await import('../services/NotificationService');
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie);
expect(res.status).toBe(500);
+1 -1
View File
@@ -715,7 +715,7 @@ describe('GET /api/audit-log/export', () => {
const csvText = res.text;
const lines = csvText.split('\n');
expect(lines[0]).toBe('id,timestamp,username,method,path,status_code,node_id,ip_address,summary');
expect(lines[0]).toBe('id,timestamp,username,acting_as,method,path,status_code,node_id,ip_address,summary');
expect(lines.length).toBeGreaterThan(1);
});
+2 -1
View File
@@ -115,7 +115,8 @@ describe('POST /api/system/console-token', () => {
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
.post('/api/system/console-token')
.set('Authorization', `Bearer ${token}`);
.set('Authorization', `Bearer ${token}`)
.send({ path: 'host-console' });
expect(res.status).toBe(200);
expect(typeof res.body.token).toBe('string');
});
@@ -89,6 +89,7 @@ describe('Blueprint compose apply (real filesystem)', () => {
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent);
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent);
const resolved = await FileSystemService.getInstance(nodeId).getComposeFilename(stackName);
expect(resolved).toBe('compose.yaml');
@@ -104,6 +105,10 @@ describe('Blueprint compose apply (real filesystem)', () => {
await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n a:\n image: a\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'services:\n b:\n image: b\n');
await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n c:\n image: c\n');
await fsPromises.writeFile(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: 2, revision: 2, lastApplied: 1 }, null, 2),
);
const composeContent = 'services:\n app:\n image: redis:7\n';
const markerContent = JSON.stringify({ blueprintId: 2, revision: 3, lastApplied: Date.now() }, null, 2);
@@ -131,8 +136,83 @@ describe('Blueprint compose apply (real filesystem)', () => {
await expectMissing(path.join(stackDir, 'compose.yml'));
await expectMissing(path.join(stackDir, 'docker-compose.yaml'));
await expectMissing(path.join(stackDir, 'docker-compose.yml'));
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent);
expect(deploySpy).toHaveBeenCalledTimes(1);
});
it('does not write a new marker when deploy fails; rolls back a newly created stack', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-partial-${counter}`;
const composeContent = 'services:\n web:\n image: traefik:v3\n';
const markerContent = JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: Date.now() }, null, 2);
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('docker unavailable'));
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
composeContent,
markerContent,
'/api/blueprints/test/apply',
),
).rejects.toThrow(/docker unavailable/);
await expectMissing(path.join(stackDir, '.blueprint.json'));
await expectMissing(stackDir);
});
it('keeps the prior marker when a re-apply deploy fails', async () => {
const nodeId = seedLocalNode();
const stackName = `bp-reapply-fail-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
const priorMarker = JSON.stringify({ blueprintId: 8, revision: 2, lastApplied: 1 }, null, 2);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n old:\n image: nginx\n');
await fsPromises.writeFile(path.join(stackDir, '.blueprint.json'), priorMarker);
vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('deploy blew up'));
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
'services:\n new:\n image: redis:7\n',
JSON.stringify({ blueprintId: 8, revision: 3, lastApplied: Date.now() }, null, 2),
'/api/blueprints/test/apply',
),
).rejects.toThrow(/deploy blew up/);
expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(priorMarker);
expect(await fsPromises.access(stackDir).then(() => true, () => false)).toBe(true);
});
it('refuses to overwrite an existing unmanaged stack directory inside the lock', async () => {
const { BlueprintNameConflictError } = await import('../services/BlueprintService');
const nodeId = seedLocalNode();
const stackName = `bp-hijack-${counter}`;
const stackDir = path.join(process.env.COMPOSE_DIR!, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
const original = 'services:\n mine:\n image: nginx:alpine\n';
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original);
const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined);
await expect(
BlueprintService.getInstance().applyLocalUnderLock(
nodeId,
stackName,
'services:\n bp:\n image: redis:7\n',
JSON.stringify({ blueprintId: 99, revision: 1, lastApplied: Date.now() }, null, 2),
'/api/blueprints/test/apply',
),
).rejects.toBeInstanceOf(BlueprintNameConflictError);
expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(original);
await expectMissing(path.join(stackDir, '.blueprint.json'));
expect(deploySpy).not.toHaveBeenCalled();
});
});
describe('FileSystemService.removeAlternateRootComposeFiles', () => {
@@ -111,9 +111,9 @@ describe('Blueprint route edge cases', () => {
describe('Blueprint delete guard', () => {
// Rows with no stack of ours on the node must not block delete, AND the route must not run the
// withdraw primitive for them: withdrawFromNode proceeds on a missing marker and would
// down/delete a same-name stack Sencho never owned. A name_conflict is exactly that unmanaged
// stack, so it is excluded even though it carries a last_deployed_at timestamp.
// withdraw primitive for them: there is nothing Sencho owns to remove. A name_conflict is an
// unmanaged same-name stack, so it is excluded even though it may carry a last_deployed_at
// timestamp. When withdraw does run, ownership is enforced under the delete lock.
it.each([
{ label: 'never-deployed pending review', status: 'pending_state_review' as const, last_deployed_at: null },
{ label: 'first-deploy failure', status: 'failed' as const, last_deployed_at: null },
@@ -184,6 +184,34 @@ describe('Blueprint delete guard', () => {
expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeUndefined();
});
it('refuses to delete a stateless blueprint when pre-delete withdraw fails', async () => {
const node = seedNode();
const bp = seedBlueprint([node.id]);
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'failed',
applied_revision: bp.revision,
last_deployed_at: Date.now(),
last_error: 'Remote node lacks withdraw-local',
});
const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({
status: 'failed',
error: 'Remote node does not support atomic blueprint withdraw',
});
const res = await request(app)
.delete(`/api/blueprints/${bp.id}`)
.set('Cookie', adminCookie);
expect(res.status).toBe(409);
expect(res.body.code).toBe('withdraw_failed_blocking_delete');
expect(res.body.nodeId).toBe(node.id);
expect(withdrawSpy).toHaveBeenCalledTimes(1);
expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeDefined();
expect(DatabaseService.getInstance().listDeployments(bp.id)).toHaveLength(1);
});
it('refuses to delete when a pending review still has a deployed stack (revision drift)', async () => {
const node = seedNode();
const bp = seedBlueprint([node.id], 'stateful');
@@ -253,38 +281,54 @@ describe('BlueprintService marker edge cases', () => {
});
it('refuses to withdraw when the marker belongs to a different blueprint', async () => {
const fs = await import('fs');
const path = await import('path');
const composeDir = process.env.COMPOSE_DIR!;
const localNode = DatabaseService.getInstance().getNodes()[0];
const bp = seedBlueprint([localNode.id]);
DatabaseService.getInstance().getDb()
.prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?')
.run(composeDir, localNode.id);
const refreshed = DatabaseService.getInstance().getNode(localNode.id)!;
const bp = seedBlueprint([refreshed.id]);
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({
blueprintId: bp.id + 999,
revision: 1,
lastApplied: 0,
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: refreshed.id,
status: 'active',
applied_revision: bpObj.revision,
last_deployed_at: Date.now(),
});
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, localNode);
const stackDir = path.join(composeDir, bpObj.name);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n');
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: bp.id + 999, revision: 1, lastApplied: 0 }),
);
const { DeployedStackDeletionService } = await import('../services/DeployedStackDeletionService');
const deleteSpy = vi.spyOn(DeployedStackDeletionService.getInstance(), 'deleteDeployedStack');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, refreshed);
expect(result.status).toBe('name_conflict');
// The deployment row must record the conflict, not silently disappear.
const dep = DatabaseService.getInstance().getDeployment(bp.id, localNode.id);
// deleteDeployedStack is invoked but returns name_conflict without mutating.
expect(deleteSpy).toHaveBeenCalled();
const dep = DatabaseService.getInstance().getDeployment(bp.id, refreshed.id);
expect(dep).toBeDefined();
expect(dep?.status).toBe('name_conflict');
expect(fs.existsSync(path.join(stackDir, 'compose.yaml'))).toBe(true);
});
});
describe('BlueprintService developer-mode diagnostics', () => {
// withdrawFromNode emits its "withdraw inputs" diagnostic line before reading the
// marker, so a cross-blueprint marker stub lets us assert the gate without Docker.
// withdrawFromNode emits its "withdraw inputs" diagnostic before the deletion
// service runs. An absent stack directory is enough to exercise logging without Docker.
function arrangeWithdraw() {
const localNode = DatabaseService.getInstance().getNodes()[0];
const bp = seedBlueprint([localNode.id]);
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({
blueprintId: bp.id + 999,
revision: 1,
lastApplied: 0,
});
return { bpObj, localNode };
}
@@ -326,6 +326,7 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
const conflict = await BlueprintService.getInstance().hasNameConflict(
created.body.name as string,
DatabaseService.getInstance().getNode(node.id)!,
created.body.id as number,
);
expect(conflict).toBe(true);
@@ -342,6 +343,41 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => {
.toBe('services:\n app:\n image: nginx\n');
});
it('hasNameConflict is false for a matching marker and true for a foreign marker', async () => {
const node = seedNode();
counter += 1;
const created = await request(app)
.post('/api/blueprints')
.set('Cookie', adminCookie)
.send(validBlueprintBody(node.id));
expect(created.status).toBe(201);
const composeDir = process.env.COMPOSE_DIR!;
DatabaseService.getInstance().getDb()
.prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?')
.run(composeDir, node.id);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const stackName = created.body.name as string;
const blueprintId = created.body.id as number;
const stackDir = path.join(composeDir, stackName);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n');
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId, revision: 1, lastApplied: 1 }),
);
const { BlueprintService } = await import('../services/BlueprintService');
const svc = BlueprintService.getInstance();
expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(false);
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: blueprintId + 99, revision: 1, lastApplied: 1 }),
);
expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(true);
});
it('blocks rename while a non-withdrawn deployment exists', async () => {
const node = seedNode();
counter += 1;
@@ -101,7 +101,7 @@ describe('BlueprintService remote deploy', () => {
expect(dep?.applied_revision).toBe(bpObj.revision);
});
it('falls back to the legacy create/write/deploy flow when the remote lacks apply-local (404)', async () => {
it('fails closed when the remote lacks apply-local (404); no legacy mutations', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
@@ -110,25 +110,17 @@ describe('BlueprintService remote deploy', () => {
vi.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: [] });
const putSpy = vi.spyOn(axios, 'put').mockResolvedValue({ status: 200, data: {} });
const postSpy = vi.spyOn(axios, 'post')
.mockResolvedValueOnce({ status: 404, data: {} }) // apply-local missing on older node
.mockResolvedValueOnce({ status: 201, data: {} }) // legacy create stack
.mockResolvedValueOnce({ status: 200, data: {} }); // legacy deploy
.mockResolvedValueOnce({ status: 404, data: {} });
const result = await BlueprintService.getInstance().deployToNode(bpObj, nodeObj);
expect(result.status).toBe('active');
expect(result.status).toBe('failed');
expect(result.error).toMatch(/apply-local|Upgrade/i);
expect(postSpy).toHaveBeenCalledTimes(1);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/apply-local$/);
expect(postSpy.mock.calls[1][0]).toMatch(/\/api\/stacks$/);
expect(putSpy).toHaveBeenCalledTimes(2); // compose + marker
const composePutUrl = String(putSpy.mock.calls[0][0]);
const markerPutUrl = String(putSpy.mock.calls[1][0]);
expect(composePutUrl).toMatch(/[?&]path=compose\.yaml(?:&|$)/);
expect(markerPutUrl).toMatch(/[?&]path=\.blueprint\.json(?:&|$)/);
expect(postSpy.mock.calls[2][0]).toMatch(/\/deploy$/);
const [composePutOrder, markerPutOrder] = putSpy.mock.invocationCallOrder;
const deployOrder = postSpy.mock.invocationCallOrder[2];
expect(composePutOrder).toBeLessThan(markerPutOrder);
expect(markerPutOrder).toBeLessThan(deployOrder);
expect(putSpy).not.toHaveBeenCalled();
const dep = DatabaseService.getInstance().getDeployment(bp.id, node.id);
expect(dep?.status).toBe('failed');
});
it('maps a remote apply lock-conflict (409) to status=failed', async () => {
@@ -192,7 +184,7 @@ describe('BlueprintService remote deploy', () => {
expect(dep?.status).toBe('name_conflict');
});
it('withdraws a remote deployment by deleting the stack and removing the row', async () => {
it('withdraws a remote deployment via withdraw-local and removes the row', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
@@ -204,14 +196,107 @@ describe('BlueprintService remote deploy', () => {
applied_revision: bpObj.revision,
});
vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); // readMarker → null → proceed
vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); // remote down (best-effort)
const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} });
const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({
status: 200,
data: { status: 'withdrawn' },
});
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('withdrawn');
expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)).toBeUndefined();
});
it('maps remote withdraw-local lock conflict to failed without fallback delete', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'active',
applied_revision: bpObj.revision,
});
vi.spyOn(axios, 'post').mockResolvedValue({
status: 409,
data: {
error: `${bpObj.name} is busy: another operation (update) is already in progress`,
code: 'stack_op_in_progress',
},
});
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('failed');
expect(result.error).toMatch(/already in progress/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed');
});
it('fails closed when the remote lacks withdraw-local (404); no legacy delete', async () => {
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = DatabaseService.getInstance().getNode(node.id)!;
const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!;
DatabaseService.getInstance().upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'active',
applied_revision: bpObj.revision,
});
const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ status: 404, data: { error: 'Not Found' } });
const delSpy = vi.spyOn(axios, 'delete');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('failed');
expect(result.error).toMatch(/withdraw-local|Upgrade/i);
expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/);
expect(delSpy).not.toHaveBeenCalled();
expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed');
});
it('does not clear hub role assignments when withdrawing a remote deployment', async () => {
const bcrypt = await import('bcrypt');
const db = DatabaseService.getInstance();
const node = seedRemoteNode();
const bp = seedBlueprint([node.id]);
const nodeObj = db.getNode(node.id)!;
const bpObj = db.getBlueprint(bp.id)!;
db.upsertDeployment({
blueprint_id: bp.id,
node_id: node.id,
status: 'active',
applied_revision: bpObj.revision,
});
const hash = await bcrypt.hash('password123', 1);
const userId = db.addUser({
username: `remote-wd-rbac-${counter}`, password_hash: hash, role: 'viewer',
});
db.addRoleAssignment({
user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bpObj.name,
});
vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: { status: 'withdrawn' } });
const delSpy = vi.spyOn(axios, 'delete');
const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource');
const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj);
expect(result.status).toBe('withdrawn');
expect(delSpy).not.toHaveBeenCalled();
expect(rbacSpy).not.toHaveBeenCalled();
expect(db.getAllRoleAssignments(userId)
.some((a) => a.resource_type === 'stack' && a.resource_id === bpObj.name)).toBe(true);
db.deleteUser(userId);
});
});
@@ -129,3 +129,71 @@ describe('POST /api/blueprints/apply-local (node-to-node atomic apply)', () => {
expect(StackOpLockService.getInstance().get(1, 'apply-local-busy')?.action).toBe('update');
});
});
describe('POST /api/blueprints/withdraw-local', () => {
let viewerCookie: string;
beforeAll(async () => {
const bcrypt = (await import('bcrypt')).default;
const passwordHash = await bcrypt.hash('bp-wd-viewer-pass', 1);
DatabaseService.getInstance().addUser({
username: 'bp-wd-viewer',
password_hash: passwordHash,
role: 'viewer',
});
const res = await request(app)
.post('/api/auth/login')
.send({ username: 'bp-wd-viewer', password: 'bp-wd-viewer-pass' });
const cookies = res.headers['set-cookie'] as string | string[];
viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies;
});
it('rejects an invalid stack name', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: '../escape', blueprintId: 1 });
expect(res.status).toBe(400);
expect(res.body.error).toContain('Invalid stack name');
});
it('rejects a non-positive blueprintId', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: 'wd-local-stack', blueprintId: 0 });
expect(res.status).toBe(400);
expect(res.body.error).toMatch(/blueprintId/i);
});
it('returns 403 for a viewer without stack:delete', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', viewerCookie)
.send({ stackName: 'wd-local-stack', blueprintId: 1 });
expect(res.status).toBe(403);
});
it('returns 409 self_stack_protected for Sencho own stack', async () => {
const selfStackGuard = await import('../helpers/selfStackGuard');
vi.spyOn(selfStackGuard, 'refuseIfSelfStack').mockImplementation(async (_req, res) => {
res.status(409).json({ error: 'self', code: 'self_stack_protected' });
return true;
});
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: 'sencho-self', blueprintId: 1 });
expect(res.status).toBe(409);
expect(res.body.code).toBe('self_stack_protected');
});
it('returns already_absent when the stack directory is missing', async () => {
const res = await request(app)
.post('/api/blueprints/withdraw-local')
.set('Cookie', adminCookie)
.send({ stackName: `wd-absent-${Date.now()}`, blueprintId: 42 });
expect(res.status).toBe(200);
expect(res.body.status).toBe('already_absent');
});
});
+124 -8
View File
@@ -13,6 +13,7 @@
* lifecycle in the plan.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import bcrypt from 'bcrypt';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
import type { Blueprint, Node } from '../services/DatabaseService';
import type { ReconcileDecision } from '../services/BlueprintReconciler';
@@ -40,6 +41,7 @@ afterAll(() => cleanupTestDb(tmpDir));
beforeEach(() => {
const db = DatabaseService.getInstance().getDb();
db.prepare('DELETE FROM role_assignments').run();
db.prepare('DELETE FROM blueprint_deployments').run();
db.prepare('DELETE FROM blueprints').run();
db.prepare('DELETE FROM node_labels').run();
@@ -397,7 +399,7 @@ describe('BlueprintReconciler developer-mode diagnostics', () => {
});
describe('BlueprintService per-stack lock', () => {
it('deploy under a free lock writes compose then marker, then deploys', async () => {
it('deploy under a free lock writes compose, cleans siblings, deploys, then writes the marker', async () => {
const nodeId = seedNode();
const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -417,7 +419,7 @@ describe('BlueprintService per-stack lock', () => {
source: 'blueprint',
actor: 'system:blueprint',
});
// Compose is written first, then the marker, both before sibling cleanup and deploy.
// Compose is written first; the marker is deferred until after sibling cleanup and deploy.
expect(writeSpy).toHaveBeenCalledTimes(2);
expect(writeSpy.mock.calls[0][1]).toBe('compose.yaml');
expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content);
@@ -427,9 +429,9 @@ describe('BlueprintService per-stack lock', () => {
const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder;
const [cleanupOrder] = cleanupSpy.mock.invocationCallOrder;
const [deployOrder] = deploySpy.mock.invocationCallOrder;
expect(composeOrder).toBeLessThan(markerOrder);
expect(markerOrder).toBeLessThan(cleanupOrder);
expect(composeOrder).toBeLessThan(cleanupOrder);
expect(cleanupOrder).toBeLessThan(deployOrder);
expect(deployOrder).toBeLessThan(markerOrder);
});
it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => {
@@ -466,6 +468,120 @@ describe('BlueprintService per-stack lock', () => {
});
});
describe('BlueprintService local withdraw clears stack-scoped role assignments', () => {
function hasAssignment(userId: number, resourceType: string, resourceId: string): boolean {
return DatabaseService.getInstance().getAllRoleAssignments(userId)
.some((a) => a.resource_type === resourceType && a.resource_id === resourceId);
}
async function arrangeLocalWithdraw() {
const db = DatabaseService.getInstance();
const composeDir = process.env.COMPOSE_DIR!;
const nodeId = seedNode();
db.getDb().prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?').run(composeDir, nodeId);
const bp = seedBlueprint({ name: `rbac-wd-${counter}`, classification: 'stateless', nodeIds: [nodeId] });
const node = db.getNode(nodeId)!;
db.upsertDeployment({
blueprint_id: bp.id,
node_id: nodeId,
status: 'active',
applied_revision: bp.revision,
});
const hash = await bcrypt.hash('password123', 1);
const userId = db.addUser({ username: `bp-rbac-${counter}`, password_hash: hash, role: 'viewer' });
const otherNodeId = seedNode();
db.addRoleAssignment({
user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bp.name,
});
db.addRoleAssignment({
user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack',
});
db.addRoleAssignment({
user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId),
});
// Matching on-disk marker so ownership passes; FS/down are stubbed.
const fs = await import('fs');
const path = await import('path');
const stackDir = path.join(composeDir, bp.name);
fs.mkdirSync(stackDir, { recursive: true });
fs.writeFileSync(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }),
);
const { ComposeService } = await import('../services/ComposeService');
const { FileSystemService } = await import('../services/FileSystemService');
vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined);
const deleteStackSpy = vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined);
return { bp, node, nodeId, userId, deleteStackSpy, db };
}
it('clears the target assignment after successful filesystem deletion and preserves unrelated grants', async () => {
const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw();
const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);
expect(outcome.status).toBe('withdrawn');
expect(deleteStackSpy).toHaveBeenCalledWith(bp.name);
expect(db.getDeployment(bp.id, node.id)).toBeUndefined();
expect(hasAssignment(userId, 'stack', bp.name)).toBe(false);
expect(hasAssignment(userId, 'stack', 'other-stack')).toBe(true);
expect(db.getAllRoleAssignments(userId).some((a) => a.resource_type === 'node')).toBe(true);
db.deleteUser(userId);
});
it('clears the target assignment when the stack directory is already absent', async () => {
const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw();
const fs = await import('fs');
const path = await import('path');
fs.rmSync(path.join(process.env.COMPOSE_DIR!, bp.name), { recursive: true, force: true });
const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);
expect(outcome.status).toBe('withdrawn');
expect(deleteStackSpy).not.toHaveBeenCalled();
expect(hasAssignment(userId, 'stack', bp.name)).toBe(false);
expect(db.getAllRoleAssignments(userId)).toHaveLength(2);
db.deleteUser(userId);
});
it('preserves the grant when filesystem deletion fails with a non-ENOENT error', async () => {
const { bp, node, nodeId, userId, deleteStackSpy, db } = await arrangeLocalWithdraw();
const fsErr = Object.assign(new Error('permission denied'), { code: 'EACCES' });
deleteStackSpy.mockRejectedValue(fsErr);
const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource');
const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);
expect(outcome.status).toBe('failed');
expect(rbacSpy).not.toHaveBeenCalled();
expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed');
expect(hasAssignment(userId, 'stack', bp.name)).toBe(true);
db.deleteUser(userId);
});
it('fails withdraw and keeps the deployment when role-assignment cleanup throws', async () => {
const { bp, node, nodeId, userId, db } = await arrangeLocalWithdraw();
vi.spyOn(db, 'deleteRoleAssignmentsByResource')
.mockImplementation(() => { throw new Error('simulated rbac cleanup failure'); });
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node);
expect(outcome.status).toBe('failed');
expect(db.getDeployment(bp.id, nodeId)).toBeDefined();
expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed');
expect(errorSpy.mock.calls.some((args) =>
typeof args[0] === 'string' && args[0].includes('Secondary DB cleanup failed'),
)).toBe(true);
expect(hasAssignment(userId, 'stack', bp.name)).toBe(true);
db.deleteUser(userId);
});
});
describe('BlueprintService marker parsing + name-conflict guard', () => {
it('parseMarker accepts a well-formed marker', () => {
const marker = BlueprintService.parseMarker(JSON.stringify({ blueprintId: 7, revision: 3, lastApplied: 12345 }));
@@ -496,7 +612,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('suggest-mode local drift uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const nodeId = seedNode();
const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -514,7 +630,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('suggest-mode remote drift keeps the authoritative node name', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
const nodeId = seedRemoteNode('sencho-test-02');
const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] });
const node = DatabaseService.getInstance().getNode(nodeId)!;
@@ -532,7 +648,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('stateful marker-loss on local uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null);
const nodeId = seedNode();
const bp = seedBlueprint({
@@ -556,7 +672,7 @@ describe('BlueprintReconciler drift alert node wording', () => {
it('enforce correction-failure on local uses on this node', async () => {
const { NotificationService } = await import('../services/NotificationService');
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({
status: 'failed',
error: 'compose up failed',
@@ -70,15 +70,18 @@ describe('fetchRemoteMeta Authorization header', () => {
describe('applyPilotModeCapabilityFilter', () => {
afterEach(() => {
enableCapability('host-console');
enableCapability('host-console-community');
});
it('removes host-console from active capabilities', () => {
it('removes host-console and host-console-community from active capabilities', () => {
expect(CAPABILITIES).toContain('host-console');
expect(CAPABILITIES).toContain('host-console-community');
applyPilotModeCapabilityFilter();
const active = getActiveCapabilities();
expect(active).not.toContain('host-console');
expect(active).not.toContain('host-console-community');
expect(active).toContain('stacks');
});
@@ -97,6 +100,7 @@ describe('applyPilotModeCapabilityFilter', () => {
const active = getActiveCapabilities();
expect(active).not.toContain('host-console');
expect(active.length).toBe(CAPABILITIES.length - 1);
expect(active).not.toContain('host-console-community');
expect(active.length).toBe(CAPABILITIES.length - 2);
});
});
@@ -24,7 +24,7 @@ function effSvc(over: Partial<EffService> = {}): EffService {
function container(over: Partial<DependencyContainer> = {}): DependencyContainer {
return {
id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp',
state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over,
};
}
+54 -9
View File
@@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
it('adds and retrieves stack alerts', () => {
db.addStackAlert({
stack_name: 'alert-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => {
expect(found.threshold).toBe(80);
expect(found.duration_mins).toBe(5);
expect(found.cooldown_mins).toBe(15);
expect(found.service_name).toBeNull();
});
it('persists a named service_name', () => {
const created = db.addStackAlert({
stack_name: 'scoped-stack',
service_name: 'api.web',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 15,
});
expect(created.service_name).toBe('api.web');
expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web');
});
it('filters alerts by stack name', () => {
db.addStackAlert({
stack_name: 'filter-stack-a',
service_name: null,
metric: 'memory_mb',
operator: '>=',
threshold: 512,
@@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
});
db.addStackAlert({
stack_name: 'filter-stack-b',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 90,
@@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => {
it('updates last_fired_at timestamp', () => {
db.addStackAlert({
stack_name: 'fired-stack',
service_name: null,
metric: 'net_rx',
operator: '>',
threshold: 100,
@@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => {
const alerts = db.getStackAlerts('fired-stack');
const alert = alerts[0];
const fireTime = Date.now();
db.updateStackAlertLastFired(alert.id, fireTime);
db.updateStackAlertLastFired(alert.id!, fireTime);
const updated = db.getStackAlerts('fired-stack');
expect(updated[0].last_fired_at).toBe(fireTime);
});
it('deletes an alert by id', () => {
db.addStackAlert({
it('upserts and reads distinct per-service cooldowns', () => {
const alert = db.addStackAlert({
stack_name: 'cooldown-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 1,
cooldown_mins: 10,
});
const id = alert.id!;
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
db.upsertStackAlertServiceCooldown(id, 'api', 1000);
db.upsertStackAlertServiceCooldown(id, 'database', 2000);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000);
expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000);
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
db.upsertStackAlertServiceCooldown(id, 'api', 3000);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000);
});
it('deletes an alert by id and removes child cooldown rows', () => {
const alert = db.addStackAlert({
stack_name: 'delete-stack',
service_name: null,
metric: 'memory_percent',
operator: '>',
threshold: 95,
duration_mins: 1,
cooldown_mins: 5,
});
const id = alert.id!;
db.upsertStackAlertServiceCooldown(id, 'api', Date.now());
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true);
const before = db.getStackAlerts('delete-stack');
expect(before.length).toBe(1);
db.deleteStackAlert(id);
db.deleteStackAlert(before[0].id);
const after = db.getStackAlerts('delete-stack');
expect(after.length).toBe(0);
expect(db.getStackAlerts('delete-stack').length).toBe(0);
expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false);
expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull();
});
});
@@ -0,0 +1,64 @@
/**
* Legacy installations upgrade stack_update_cleanup_pending with nullable
* required_blueprint_id via maybeAddCol.
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb';
let tmpDir: string;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
});
afterAll(() => cleanupTestDb(tmpDir));
describe('stack_update_cleanup_pending required_blueprint_id migration', () => {
it('upgrades a legacy table without the column and preserves null for old rows', () => {
const db = DatabaseService.getInstance();
const raw = db.getDb();
// Simulate a pre-migration installation: drop the column if present, then
// re-add it the same way initSchema's maybeAddCol does.
try {
raw.exec('ALTER TABLE stack_update_cleanup_pending DROP COLUMN required_blueprint_id');
} catch {
// Column may already be absent in a hand-built fixture.
}
const now = Date.now();
raw.prepare(`
INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, 'prepared', 'local_socket', '[]', '[]', 0, ?, ?)
`).run('legacy-1', 1, 'legacy-stack', now, now);
try {
raw.exec('ALTER TABLE stack_update_cleanup_pending ADD COLUMN required_blueprint_id INTEGER');
} catch {
// Idempotent if a parallel path re-added it.
}
const legacy = db.getCleanupPending('legacy-1');
expect(legacy).toBeDefined();
expect(legacy?.required_blueprint_id ?? null).toBeNull();
db.insertCleanupPending({
id: 'owned-1',
node_id: 2,
stack_name: 'bp-stack',
status: 'prepared',
target_kind: 'local_socket',
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: 42,
created_at: now,
updated_at: now,
});
expect(db.getCleanupPending('owned-1')?.required_blueprint_id).toBe(42);
});
});
@@ -32,7 +32,7 @@ function snap(partial: Partial<DependencySnapshot>): DependencySnapshot {
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: null,
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -89,6 +89,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
};
@@ -111,6 +112,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -129,6 +131,7 @@ describe('DeployedStackDeletionService ready transaction', () => {
rollback_tags_json: '[]',
override_paths_json: '[]',
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -154,3 +157,39 @@ describe('overrideDeletionContainmentBase', () => {
});
});
describe('DeployedStackDeletionService blueprint ownership probe', () => {
it('returns failed (not name_conflict) when marker read fails with non-ENOENT I/O', async () => {
const { promises: fsPromises } = await import('fs');
const { vi } = await import('vitest');
const composeDir = process.env.COMPOSE_DIR!;
const stackName = `del-probe-${Date.now()}`;
const stackDir = path.join(composeDir, stackName);
await fsPromises.mkdir(stackDir, { recursive: true });
await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n');
await fsPromises.writeFile(
path.join(stackDir, '.blueprint.json'),
JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: 0 }),
);
const accessErr = Object.assign(new Error('EACCES'), { code: 'EACCES' });
const readSpy = vi.spyOn(fsPromises, 'readFile').mockRejectedValueOnce(accessErr);
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: NODE,
stackName,
pruneVolumes: false,
actor: 'test',
requireBlueprintId: 7,
});
expect(result.ok).toBe(false);
if (!result.ok) {
expect(result.code).toBe('failed');
expect(result.error).toMatch(/EACCES|Failed to read|permission/i);
}
expect(readSpy).toHaveBeenCalled();
readSpy.mockRestore();
await fsPromises.rm(stackDir, { recursive: true, force: true });
});
});
@@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => {
expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]);
expect(c.volumes).toEqual(['web_data']); // bind mount dropped
expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped
expect(c.exitCode).toBeNull();
expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true);
const frontend = snap.networks.find((n) => n.name === 'web_frontend');
@@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => {
expect(snap.containers[0].stack).toBeNull();
expect(snap.containers[0].composeProject).toBeNull();
});
it('parses exitCode from list Status for exited containers', async () => {
mockDocker.listContainers.mockResolvedValue([
{ Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
{ Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] },
]);
mockDocker.listNetworks.mockResolvedValue([]);
mockDocker.listVolumes.mockResolvedValue({ Volumes: [] });
const snap = await DockerController.getInstance(1).getDependencySnapshot([]);
expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0);
expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137);
expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull();
expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull();
});
});
// --- getBulkStackStatuses uptime (runningSince from StartedAt) -----------------
@@ -27,7 +27,7 @@ const {
mockGetDocker,
mockIsOwnContainer,
} = vi.hoisted(() => ({
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockBroadcastEvent: vi.fn(),
mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }),
mockGetEvents: vi.fn(),
@@ -97,6 +97,8 @@ let service: DockerEventService;
beforeEach(() => {
vi.clearAllMocks();
vi.useFakeTimers();
mockDispatchAlert.mockReset();
mockDispatchAlert.mockResolvedValue({ persisted: true });
mockGetGlobalSettings.mockReturnValue({ global_crash: '1' });
mockIsOwnContainer.mockReset();
mockIsOwnContainer.mockReturnValue(false);
@@ -119,6 +121,11 @@ afterEach(() => {
vi.useRealTimers();
});
async function flushMicrotasks(): Promise<void> {
await Promise.resolve();
await Promise.resolve();
}
// ── Classification via event stream ────────────────────────────────────
describe('DockerEventService - die classification', () => {
@@ -367,6 +374,7 @@ describe('DockerEventService - die classification', () => {
},
},
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledWith(
'error',
@@ -442,9 +450,586 @@ describe('DockerEventService - rate limiting', () => {
// After the rate window, a summary warning fires.
await vi.advanceTimersByTimeAsync(61_000);
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
typeof c[2] === 'string' && c[2].includes('additional containers crashed'));
typeof c[2] === 'string' && c[2].includes('rate-limited'));
expect(summaryCalls).toHaveLength(1);
expect(summaryCalls[0][2]).toContain('2 additional');
expect(summaryCalls[0][2]).toBe(
'2 additional container crash alerts were rate-limited in the last minute.',
);
});
});
// ── Health alert transition / dedup / rate sharing ─────────────────────
describe('DockerEventService - health alert gates', () => {
it('emits once for duplicate unhealthy while already unhealthy and keeps unhealthySince', async () => {
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: {
ID: 'h1',
Attributes: { name: 'api', 'com.docker.compose.project': 'web' },
},
});
await flushMicrotasks();
const first = service.getContainerState('h1');
expect(first?.healthStatus).toBe('unhealthy');
expect(first?.unhealthySince).toBeTypeOf('number');
const since = first!.unhealthySince!;
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h1', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(service.getContainerState('h1')?.unhealthySince).toBe(since);
expect(service.getContainerState('h1')?.healthStatus).toBe('unhealthy');
});
it('suppresses a second flap while the first health dispatch is still pending', async () => {
let resolveFirst!: (value: { persisted: boolean }) => void;
mockDispatchAlert.mockImplementationOnce(
() => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }),
);
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h2', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h2', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h2', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
resolveFirst({ persisted: true });
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
});
it('retries a deferred flap when the in-flight dispatch fails to persist', async () => {
let resolveFirst!: (value: { persisted: boolean }) => void;
mockDispatchAlert
.mockImplementationOnce(
() => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }),
)
.mockResolvedValue({ persisted: true });
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h2b', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
resolveFirst({ persisted: false });
await flushMicrotasks();
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
});
it('does not advance health dedup when persisted is false, allowing a later transition', async () => {
mockDispatchAlert
.mockResolvedValueOnce({ persisted: false })
.mockResolvedValue({ persisted: true });
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h3', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h3', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h3', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
});
it('keeps health dedup across the 10-minute prune window and re-emits after 60 minutes', async () => {
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h4', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Recover so lastActivity ages without fresh unhealthy events; prune
// may drop containerState after 10m but health dedup must survive.
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h4', Attributes: { name: 'api' } },
});
await flushMicrotasks();
await vi.advanceTimersByTimeAsync(11 * 60_000);
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h4', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(50 * 60_000);
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h4', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h4', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
});
it('does not let crash and health dedup suppress one another', async () => {
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'die',
Actor: { ID: 'h5', Attributes: { exitCode: '1', name: 'api' } },
});
await vi.advanceTimersByTimeAsync(600);
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
stream.push({
Type: 'container',
Action: 'start',
Actor: { ID: 'h5', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h5', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
expect(mockDispatchAlert.mock.calls[1][2]).toContain('Healthcheck failed');
});
it('does not consume rate tokens for deduped duplicate health events', async () => {
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h6', Attributes: { name: 'api' } },
});
await flushMicrotasks();
for (let i = 0; i < 5; i++) {
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'h6', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h6', Attributes: { name: 'api' } },
});
await flushMicrotasks();
}
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
for (let i = 0; i < 19; i++) {
stream.push({
Type: 'container',
Action: 'die',
Actor: { ID: `crash-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } },
});
}
await vi.advanceTimersByTimeAsync(600);
const crashCalls = mockDispatchAlert.mock.calls.filter(c =>
typeof c[2] === 'string' && c[2].includes('Crash'));
expect(crashCalls).toHaveLength(19);
});
it('emits an accurate health-only rate-limit roll-up', async () => {
service = new DockerEventService(1, 'local');
await service.start();
for (let i = 0; i < 22; i++) {
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: `hh-${i}`, Attributes: { name: `svc-${i}` } },
});
await flushMicrotasks();
}
const healthCalls = mockDispatchAlert.mock.calls.filter(c =>
typeof c[2] === 'string' && c[2].includes('Healthcheck failed'));
expect(healthCalls).toHaveLength(20);
await vi.advanceTimersByTimeAsync(61_000);
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
typeof c[2] === 'string' && c[2].includes('rate-limited'));
expect(summaryCalls).toHaveLength(1);
expect(summaryCalls[0][2]).toBe(
'2 additional container health alerts were rate-limited in the last minute.',
);
});
it('emits an accurate mixed crash and health rate-limit roll-up', async () => {
service = new DockerEventService(1, 'local');
await service.start();
for (let i = 0; i < 10; i++) {
stream.push({
Type: 'container',
Action: 'die',
Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } },
});
}
await vi.advanceTimersByTimeAsync(600);
for (let i = 0; i < 10; i++) {
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } },
});
await flushMicrotasks();
}
for (let i = 10; i < 12; i++) {
stream.push({
Type: 'container',
Action: 'die',
Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } },
});
}
await vi.advanceTimersByTimeAsync(600);
for (let i = 10; i < 13; i++) {
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } },
});
await flushMicrotasks();
}
await vi.advanceTimersByTimeAsync(61_000);
const summaryCalls = mockDispatchAlert.mock.calls.filter(c =>
typeof c[2] === 'string' && c[2].includes('rate-limited'));
expect(summaryCalls).toHaveLength(1);
expect(summaryCalls[0][2]).toBe(
'5 additional container alerts were rate-limited in the last minute (2 crash, 3 health).',
);
});
it('keeps Auto-Heal health state when global_crash is off', async () => {
mockGetGlobalSettings.mockReturnValue({ global_crash: '0' });
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'h7', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).not.toHaveBeenCalled();
expect(service.getContainerState('h7')).toMatchObject({
healthStatus: 'unhealthy',
});
expect(service.getContainerState('h7')?.unhealthySince).toBeTypeOf('number');
});
it('does not refund a rate token into the next fixed window', async () => {
let resolvePersist!: (value: { persisted: boolean }) => void;
mockDispatchAlert.mockImplementation(
() =>
new Promise<{ persisted: boolean }>((resolve) => {
resolvePersist = resolve;
}),
);
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'cross-window', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Cross the fixed window, then consume a token in the new window so
// refund would inflate W2's count if it ignored the issuing window.
await vi.advanceTimersByTimeAsync(60_000);
mockDispatchAlert.mockResolvedValue({ persisted: true });
stream.push({
Type: 'container',
Action: 'die',
Actor: {
ID: 'cw-roll',
Attributes: { exitCode: '1', name: 'roll' },
},
});
await vi.advanceTimersByTimeAsync(600);
expect(
mockDispatchAlert.mock.calls.filter(
(c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'),
),
).toHaveLength(1);
// Fail the original health persist: old code refunds into W2 (count 1→0).
resolvePersist({ persisted: false });
await flushMicrotasks();
// Fill the rest of W2 to the 20-alert cap (19 more crashes).
for (let i = 0; i < 19; i++) {
stream.push({
Type: 'container',
Action: 'die',
Actor: {
ID: `cw-${i}`,
Attributes: { exitCode: '1', name: `svc-${i}` },
},
});
}
await vi.advanceTimersByTimeAsync(600);
// Cap must still hold: a cross-window refund would have allowed a 21st.
stream.push({
Type: 'container',
Action: 'die',
Actor: {
ID: 'cw-overflow',
Attributes: { exitCode: '1', name: 'overflow' },
},
});
await vi.advanceTimersByTimeAsync(600);
const crashCalls = mockDispatchAlert.mock.calls.filter(
(c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'),
);
expect(crashCalls).toHaveLength(20);
});
it('clears pending health retry when the container recovers before persist fails', async () => {
let resolvePersist!: (value: { persisted: boolean }) => void;
mockDispatchAlert.mockImplementation(
() =>
new Promise<{ persisted: boolean }>((resolve) => {
resolvePersist = resolve;
}),
);
service = new DockerEventService(1, 'local');
await service.start();
// First unhealthy starts an in-flight dispatch.
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Flap while in flight: recover then re-fail so a pending-retry marker is set.
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Recover again before the first persist settles; marker must be dropped.
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
resolvePersist({ persisted: false });
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Later unrelated persist failure must not spuriously retry from a stale marker.
mockDispatchAlert.mockResolvedValueOnce({ persisted: false });
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'recover-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
});
it('does not dispatch deferred health retry after shutdown', async () => {
let resolvePersist!: (value: { persisted: boolean }) => void;
mockDispatchAlert.mockImplementation(
() =>
new Promise<{ persisted: boolean }>((resolve) => {
resolvePersist = resolve;
}),
);
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'shutdown-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
service.shutdown();
resolvePersist({ persisted: false });
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
});
it('clears health alert bookkeeping on destroy', async () => {
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
stream.push({
Type: 'container',
Action: 'destroy',
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
});
await flushMicrotasks();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
});
it('does not retry a deferred health alert after destroy', async () => {
let resolvePersist!: (value: { persisted: boolean }) => void;
mockDispatchAlert.mockImplementation(
() =>
new Promise<{ persisted: boolean }>((resolve) => {
resolvePersist = resolve;
}),
);
service = new DockerEventService(1, 'local');
await service.start();
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
// Set a pending-retry marker while the first dispatch is in flight.
stream.push({
Type: 'container',
Action: 'health_status: healthy',
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
});
stream.push({
Type: 'container',
Action: 'health_status: unhealthy',
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
stream.push({
Type: 'container',
Action: 'destroy',
Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } },
});
await flushMicrotasks();
resolvePersist({ persisted: false });
await flushMicrotasks();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
});
});
+218 -1
View File
@@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom
function container(p: Partial<DependencyContainer> & { id: string }): DependencyContainer {
return {
name: p.id, service: null, composeProject: null, stack: 'app',
state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p,
};
}
@@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => {
expect(report.findings).toEqual([]);
});
it('does not emit service-missing for a clean one-shot beside a running service', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(true);
});
it('still emits service-missing when a one-shot exits non-zero', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('still treats exited unless-stopped as missing even with exit 0', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'web', restart: 'unless-stopped' })]),
containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('missing-runtime');
expect(report.hasContainers).toBe(false);
});
it('fails closed when exitCode is null even with restart no', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'migrate', image: 'migrate:1', restart: 'no' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }),
container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate');
});
it('does not treat absent declared restart as a clean one-shot', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'daemon-default', image: 'daemon:1' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default');
});
it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: {
services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })],
networks: { jobs: { external: false } },
volumes: {},
projectName: 'app',
},
containers: [
container({
id: 'c1', service: 'migrate', state: 'exited', exitCode: 0,
networks: [{ name: 'app_jobs', id: 'j', ip: '' }],
}),
],
networks: [depNet('app_jobs')],
});
expect(report.status).not.toBe('missing-runtime');
expect(findingKinds(report)).not.toContain('service-missing');
expect(report.hasContainers).toBe(false);
expect(findingKinds(report)).toContain('network-missing');
expect(report.status).toBe('drifted');
});
it('all-one-shot stack without network findings is in-sync with hasContainers false', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([service({ name: 'migrate', restart: 'no' })]),
containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })],
});
expect(report.status).toBe('in-sync');
expect(report.hasContainers).toBe(false);
expect(report.findings).toEqual([]);
});
it('emits service-missing when normalized restart is always (deploy any)', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'always' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker');
});
it('emits service-missing when normalized restart is on-failure', () => {
const report = assembleStackDrift({
stack: 'app',
declared: declared([
service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }),
service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }),
]),
containers: [
container({ id: 'c1', service: 'app', image: 'app:1' }),
container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }),
],
});
expect(findingKinds(report)).toContain('service-missing');
});
it('counts a restarting container as deployed', () => {
const report = assembleStackDrift({
stack: 'app',
@@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => {
expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]);
});
it('preserves restart policy including no, unless-stopped, and absent', () => {
const withNo = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'migrate', restart: 'no' })],
networks: {},
volumes: {},
});
expect(withNo.services[0].restart).toBe('no');
const withUnless = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'web', restart: 'unless-stopped' })],
networks: {},
volumes: {},
});
expect(withUnless.services[0].restart).toBe('unless-stopped');
const absent = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({ name: 'job', restart: undefined })],
networks: {},
volumes: {},
});
expect(absent.services[0].restart).toBeNull();
});
it('normalizes deploy.restart_policy conditions with Compose precedence', () => {
const none = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'migrate',
restart: 'unless-stopped',
deploy: { restart_policy: { condition: 'none' } },
})],
networks: {},
volumes: {},
});
expect(none.services[0].restart).toBe('no');
const any = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: { condition: 'any' } },
})],
networks: {},
volumes: {},
});
expect(any.services[0].restart).toBe('always');
const onFailure = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: undefined,
deploy: { restart_policy: { condition: 'on-failure' } },
})],
networks: {},
volumes: {},
});
expect(onFailure.services[0].restart).toBe('on-failure');
const defaultAny = declaredFromEffectiveModel({
projectName: 'app',
services: [effSvc({
name: 'worker',
restart: 'no',
deploy: { restart_policy: {} },
})],
networks: {},
volumes: {},
});
expect(defaultAny.services[0].restart).toBe('always');
});
it('normalizes networks so drift matches the rendered model', () => {
const model: EffectiveModel = {
projectName: 'myapp',
+2 -2
View File
@@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => {
// A running container on a different image than compose declares => image-mismatch.
const driftedContainer = (stack: string) => ({
id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
});
beforeEach(() => {
@@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => {
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{
id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK,
state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [],
state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [],
}],
networks: [], volumes: [],
}),
@@ -95,9 +95,10 @@ describe('FileSystemService stack methods', () => {
expect(fileNames).toEqual(['.env', 'compose.yaml']);
});
it('marks compose.yaml and .env as protected', async () => {
it('marks compose.yaml, .env, and .blueprint.json as protected', async () => {
await fs.writeFile(path.join(stackDir, 'compose.yaml'), '');
await fs.writeFile(path.join(stackDir, '.env'), '');
await fs.writeFile(path.join(stackDir, '.blueprint.json'), '{}');
await fs.writeFile(path.join(stackDir, 'custom.conf'), '');
const service = FileSystemService.getInstance();
@@ -106,6 +107,7 @@ describe('FileSystemService stack methods', () => {
const byName = Object.fromEntries(entries.map(e => [e.name, e]));
expect(byName['compose.yaml'].isProtected).toBe(true);
expect(byName['.env'].isProtected).toBe(true);
expect(byName['.blueprint.json'].isProtected).toBe(true);
expect(byName['custom.conf'].isProtected).toBe(false);
});
@@ -18,7 +18,7 @@ const {
mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockPushResourceToNode: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
}));
vi.mock('../services/DatabaseService', () => ({
@@ -40,7 +40,7 @@ const {
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }),
}));
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
// AutoHeal suppression is exercised elsewhere; here we only need the calls to
// not throw when the gate finalizes a service run.
vi.mock('../services/AutoHealService', () => ({
@@ -85,6 +95,8 @@ type Fixture = {
restartCount?: number;
startedAt?: string;
imageId?: string;
exitCode?: number | null;
restartPolicy?: string | null;
};
function setContainers(fixtures: Fixture[]): void {
@@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void {
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:app',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } },
Config: { Labels: { 'com.docker.compose.service': f.service } },
});
});
}
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
async function ticks(n: number): Promise<void> {
@@ -137,6 +169,8 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' });
svc().start();
});
@@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => {
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited' },
{ id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
@@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => {
expect(report.failureSource).toBe('collateral');
});
it('passes when a collateral one-shot exits 0 with restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a collateral one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a collateral daemon with omitted restart exits 0', async () => {
setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined });
const token = await prepareService([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{ id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' },
]);
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' },
{
id: 's1', name: 'web-daemon-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.failureSource).toBe('collateral');
expect(report.reason).toContain('exited during observation');
});
it('passes when the primary service is a completed one-shot', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual unhealthy health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('passes a primary one-shot with residual starting health', async () => {
setDeclaredRestarts({ job: 'no' });
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{
id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting', imageId: 'sha256:app',
},
]);
await ticks(1);
expect(svc().getReport(0, 'web', runId!).status).toBe('observing');
await ticks(6);
expect(svc().getReport(0, 'web', runId!).status).toBe('passed');
});
it('fails when a primary one-shot exits with null exit code', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits 0 under unless-stopped', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails when a primary one-shot exits non-zero', async () => {
const token = await prepareService([
{ id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' },
], { serviceName: 'job', expectedReplicas: 1 });
svc().attachExpectedImage(token, 'sha256:app');
const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' });
await ticks(1);
setContainers([
{ id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' },
]);
await ticks(1);
const report = svc().getReport(0, 'web', runId!);
expect(report.status).toBe('failed');
expect(report.reason).toContain('exited during observation');
expect(report.failureSource).toBe('primary');
});
it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => {
// Sibling is healthy at prepare, then gone before arming. Seeding expected
// from the prepare baseline must still track it so the gate fails.
@@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({
settings: {} as Record<string, string>,
listContainers: vi.fn(),
inspect: vi.fn(),
renderConfig: vi.fn(),
},
}));
@@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({
},
}));
vi.mock('../services/ComposeService', () => ({
getComposeCommandTimeoutMs: () => 30_000,
ComposeService: {
getInstance: () => ({
renderConfig: state.renderConfig,
}),
},
}));
import { HealthGateService } from '../services/HealthGateService';
type ContainerFixture = {
@@ -89,25 +99,57 @@ type ContainerFixture = {
health?: string | null;
restartCount?: number;
startedAt?: string;
exitCode?: number | null;
restartPolicy?: string | null;
service?: string;
imageId?: string;
};
/** Configure the docker mocks from a simple fixture list. */
function setContainers(fixtures: ContainerFixture[]): void {
state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' })));
state.listContainers.mockResolvedValue(fixtures.map(f => ({
Id: f.id,
Names: [`/${f.name}`],
State: f.state ?? 'running',
Labels: f.service ? { 'com.docker.compose.service': f.service } : {},
})));
state.inspect.mockImplementation((id: string) => {
const f = fixtures.find(c => c.id === id);
if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 }));
return Promise.resolve({
State: {
Status: f.state ?? 'running',
ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode,
Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined,
StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z',
},
RestartCount: f.restartCount ?? 0,
Image: f.imageId ?? 'sha256:img',
HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } },
Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} },
});
});
}
/** Declared Compose restart map used for one-shot recognition (not inspect). */
function setDeclaredRestarts(services: Record<string, string | undefined>): void {
const rendered = {
name: 'web',
services: Object.fromEntries(
Object.entries(services).map(([name, restart]) => [
name,
restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart },
]),
),
};
state.renderConfig.mockResolvedValue({
rendered: JSON.stringify(rendered),
stderr: '',
code: 0,
timedOut: false,
});
}
const svc = () => HealthGateService.getInstance();
const latest = (stack = 'web') => svc().getReport(0, stack);
@@ -125,7 +167,9 @@ beforeEach(() => {
state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' };
state.listContainers.mockReset();
state.inspect.mockReset();
setContainers([{ id: 'aaa', name: 'web-app-1' }]);
state.renderConfig.mockReset();
setDeclaredRestarts({ app: 'unless-stopped' });
setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]);
svc().start();
});
@@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => {
it('fails fast when a container exits', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1); // baseline
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]);
await ticks(1);
const report = latest();
expect(report.status).toBe('failed');
@@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => {
expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true);
});
it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => {
// QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and
// explicit restart:"no". Declared intent must decide, not inspect.
setDeclaredRestarts({ 'daemon-default': undefined });
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'running', restartPolicy: 'no',
},
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default',
state: 'exited', exitCode: 0, restartPolicy: 'no',
},
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('passes a clean one-shot even when residual health is unhealthy', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'unhealthy',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('passes a clean one-shot even when residual health is still starting', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{
id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0,
restartPolicy: 'no', health: 'starting',
},
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('passed');
});
it('still fails unhealthy on a long-running container', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'unhealthy',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('unhealthy');
});
it('still ends unknown when a long-running healthcheck is starting at window end', async () => {
setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' });
setContainers([
{ id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' },
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' },
]);
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([
{
id: 'app', name: 'web-app-1', service: 'app', state: 'running',
restartPolicy: 'unless-stopped', health: 'starting',
},
{ id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' },
]);
await ticks(1);
expect(latest().status).toBe('observing');
await ticks(6);
expect(latest().status).toBe('unknown');
expect(latest().reason).toContain('still starting');
});
it('fails when exit 0 has unless-stopped restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when exit 0 has always restart policy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails closed when exit code is null on an exited container', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails when a one-shot exits non-zero', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]);
await ticks(1);
expect(latest().status).toBe('failed');
expect(latest().reason).toContain('exited during observation');
});
it('fails fast when a healthcheck reports unhealthy', async () => {
svc().beginStack(0, 'web', 'update', 'tester');
await ticks(1);
+132 -4
View File
@@ -21,8 +21,8 @@ describe('WebSocket upgrade - host console auth enforcement', () => {
beforeAll(async () => {
vi.restoreAllMocks();
tmpDir = await setupTestDb();
// Host console requires the paid tier; mock the license so the tier gate
// passes for the admin/accepted cases. Individual tests override as needed.
// Host console is available on every tier for admins; mock paid so other
// suites that share LicenseService state stay stable.
const { LicenseService } = await import('../services/LicenseService');
getTierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid');
const mod = await import('../index');
@@ -78,10 +78,28 @@ describe('WebSocket upgrade - host console auth enforcement', () => {
expect(await expectRejected(ws)).toBe(403);
});
it('rejects an admin on the Community tier (403)', async () => {
it('accepts a Community-tier admin', async () => {
getTierSpy.mockReturnValueOnce('community');
const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } });
expect(await expectRejected(ws)).toBe(403);
const opened = await new Promise<boolean>((resolve) => {
ws.on('open', () => { ws.close(); resolve(true); });
ws.on('error', () => resolve(false));
ws.on('unexpected-response', () => resolve(false));
});
expect(opened).toBe(true);
});
it('accepts a console_session Bearer on Community (remote bridge mint path)', async () => {
getTierSpy.mockReturnValueOnce('community');
const { mintConsoleSession } = await import('../helpers/consoleSession');
const token = mintConsoleSession({ path: 'host-console', actingAs: TEST_USERNAME });
const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } });
const opened = await new Promise<boolean>((resolve) => {
ws.on('open', () => { ws.close(); resolve(true); });
ws.on('error', () => resolve(false));
ws.on('unexpected-response', () => resolve(false));
});
expect(opened).toBe(true);
});
it('accepts an admin on Admiral and records an open audit row', async () => {
@@ -127,6 +145,116 @@ describe('WebSocket upgrade - host console auth enforcement', () => {
expect(firstMessage).toContain('Invalid stack path');
ws.close();
});
it('rejects an unknown nodeId without spawning a PTY (404)', async () => {
const { HostTerminalService } = await import('../services/HostTerminalService');
const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal');
try {
const ws = new WebSocket(wsUrl('?nodeId=99999999'), {
headers: { Cookie: `sencho_token=${adminToken()}` },
});
expect(await expectRejected(ws)).toBe(404);
expect(spawnSpy).not.toHaveBeenCalled();
} finally {
spawnSpy.mockRestore();
}
});
it('rejects a malformed nodeId that parseInt would coerce (404)', async () => {
const ws = new WebSocket(wsUrl('?nodeId=1abc'), {
headers: { Cookie: `sencho_token=${adminToken()}` },
});
expect(await expectRejected(ws)).toBe(404);
});
it('rejects zero and negative nodeId values (404)', async () => {
for (const id of ['0', '-1']) {
const ws = new WebSocket(wsUrl(`?nodeId=${id}`), {
headers: { Cookie: `sencho_token=${adminToken()}` },
});
expect(await expectRejected(ws)).toBe(404);
}
});
it('rejects a replayed console_session token on second Host Console upgrade', async () => {
const { mintConsoleSession } = await import('../helpers/consoleSession');
const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' });
const first = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } });
const firstOpened = await new Promise<boolean>((resolve) => {
first.on('open', () => { first.close(); resolve(true); });
first.on('error', () => resolve(false));
first.on('unexpected-response', () => resolve(false));
});
expect(firstOpened).toBe(true);
const second = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } });
expect(await expectRejected(second)).toBe(401);
});
it('rejects a host-console console_session on the container-exec /ws path', async () => {
const { mintConsoleSession } = await import('../helpers/consoleSession');
const token = mintConsoleSession({ path: 'host-console' });
const addr = server.address();
if (!addr || typeof addr === 'string') throw new Error('Server not listening');
const ws = new WebSocket(`ws://127.0.0.1:${addr.port}/ws`, {
headers: { Authorization: `Bearer ${token}` },
});
expect(await expectRejected(ws)).toBe(403);
});
it('records acting_as on console_session open audit rows', async () => {
const { mintConsoleSession } = await import('../helpers/consoleSession');
const { DatabaseService } = await import('../services/DatabaseService');
const db = DatabaseService.getInstance();
const insertSpy = vi.spyOn(db, 'insertAuditLog');
const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' });
try {
const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } });
const opened = await new Promise<boolean>((resolve) => {
ws.on('open', () => { ws.close(); resolve(true); });
ws.on('error', () => resolve(false));
ws.on('unexpected-response', () => resolve(false));
});
expect(opened).toBe(true);
await waitFor(() => insertSpy.mock.calls.some((c) => {
const e = c[0] as { summary?: string };
return typeof e.summary === 'string' && e.summary.includes('Opened host console');
}));
const openEntry = insertSpy.mock.calls
.map((c) => c[0] as { username: string; acting_as?: string | null; summary: string })
.find((e) => e.summary.includes('Opened host console'));
expect(openEntry?.username).toBe('console_session');
expect(openEntry?.acting_as).toBe('qaadmin');
} finally {
insertSpy.mockRestore();
}
});
it('closes without spawning a PTY when directory resolution throws (no default-node fallback)', async () => {
const { FileSystemService } = await import('../services/FileSystemService');
const { HostTerminalService } = await import('../services/HostTerminalService');
const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal');
const getInstanceSpy = vi.spyOn(FileSystemService, 'getInstance').mockImplementation(() => {
throw new Error('compose dir unavailable');
});
try {
const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } });
const firstMessage = await new Promise<string>((resolve) => {
ws.on('message', (data) => resolve(data.toString()));
ws.on('error', () => resolve(''));
ws.on('unexpected-response', () => resolve(''));
});
expect(firstMessage).toMatch(/Failed to resolve console directory/i);
expect(spawnSpy).not.toHaveBeenCalled();
// getInstance must not be retried against a fallback/default node id.
expect(getInstanceSpy).toHaveBeenCalledTimes(1);
ws.close();
} finally {
spawnSpy.mockRestore();
getInstanceSpy.mockRestore();
}
});
});
/** Poll a predicate up to ~1s; resolve true as soon as it passes. */
+34 -1
View File
@@ -375,9 +375,42 @@ describe('POST /api/system/console-token', () => {
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
.post('/api/system/console-token')
.set('Authorization', `Bearer ${token}`);
.set('Authorization', `Bearer ${token}`)
.send({ path: 'host-console' });
expect(res.status).toBe(200);
expect(typeof res.body.token).toBe('string');
const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string; jti?: string };
expect(decoded.scope).toBe('console_session');
expect(decoded.path).toBe('host-console');
expect(typeof decoded.jti).toBe('string');
});
it('returns 400 when path is missing', async () => {
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
.post('/api/system/console-token')
.set('Authorization', `Bearer ${token}`)
.send({});
expect(res.status).toBe(400);
});
it('returns 200 for an admin on Community (no paid gate)', async () => {
const { LicenseService } = await import('../services/LicenseService');
const spy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community');
try {
const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
const res = await request(app)
.post('/api/system/console-token')
.set('Authorization', `Bearer ${token}`)
.send({ path: 'container-exec' });
expect(res.status).toBe(200);
expect(typeof res.body.token).toBe('string');
const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string };
expect(decoded.scope).toBe('console_session');
expect(decoded.path).toBe('container-exec');
} finally {
spy.mockReturnValue('paid');
}
});
it('returns 403 for non-admin user (viewer role)', async () => {
@@ -25,7 +25,7 @@ const {
mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled
mockSetSystemState: vi.fn(),
mockAddNotificationHistory: vi.fn(),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetStacks: vi.fn().mockResolvedValue([]),
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')),
+443 -18
View File
@@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM
const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric,
mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs,
mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState,
mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown,
mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown,
mockGetSystemState, mockSetSystemState,
mockPruneScanHistoryPerImage, mockDeleteScansByImageRef,
mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream,
mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks,
@@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
mockCleanupOldNotifications: vi.fn(),
mockCleanupOldAuditLogs: vi.fn(),
mockUpdateStackAlertLastFired: vi.fn(),
mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null),
mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false),
mockUpsertStackAlertServiceCooldown: vi.fn(),
mockGetSystemState: vi.fn().mockReturnValue(null),
mockSetSystemState: vi.fn(),
mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0),
@@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine
}),
mockGetImages: vi.fn().mockResolvedValue([]),
mockGetStacks: vi.fn().mockResolvedValue([]),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }),
mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }),
mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]),
@@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({
cleanupOldNotifications: mockCleanupOldNotifications,
cleanupOldAuditLogs: mockCleanupOldAuditLogs,
updateStackAlertLastFired: mockUpdateStackAlertLastFired,
getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown,
hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown,
upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown,
getSystemState: mockGetSystemState,
setSystemState: mockSetSystemState,
pruneScanHistoryPerImage: mockPruneScanHistoryPerImage,
@@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => {
function setupAlertScenario(cpuPercent: number) {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([{
Id: 'c1',
Labels: { 'com.docker.compose.project': 'my-stack' },
Id: 'c1abcdefghijk',
Names: ['/my-stack-api-1'],
Labels: {
'com.docker.compose.project': 'my-stack',
'com.docker.compose.service': 'api',
},
}]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 },
@@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => {
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
}
it('fires alert when condition met and duration is 0', async () => {
@@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => {
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', actor: 'system:monitor' },
'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' },
);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
@@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => {
expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU'));
});
it('respects cooldown after firing', async () => {
it('respects cooldown after firing via persisted service cooldown', async () => {
setupAlertScenario(90);
mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
});
it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => {
setupAlertScenario(90);
// Simulate that alert was fired 30 minutes ago (within 60-min cooldown)
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => {
cooldown_mins: 60,
last_fired_at: Date.now() - 30 * 60 * 1000,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
});
it('resets breach state when condition clears', async () => {
it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => {
setupAlertScenario(90);
mockGetStackAlerts.mockReturnValue([{
id: 1,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: Date.now() - 90 * 60 * 1000,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
it('does not advance cooldown when notification history is not persisted', async () => {
setupAlertScenario(90);
mockDispatchAlert.mockResolvedValueOnce({ persisted: false });
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled();
expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled();
mockDispatchAlert.mockResolvedValueOnce({ persisted: true });
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(2);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number));
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number));
});
it('drops in-memory breach timers when a rule is deleted', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([]);
mockGetStackAlerts.mockReturnValue([]);
mockGetGlobalSettings.mockReturnValue({});
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false);
});
it('names unlabeled containers as unknown service in the alert body', async () => {
setupAlertScenario(90);
mockGetRunningContainers.mockResolvedValue([{
Id: 'orphan123456789',
Names: ['/orphan'],
Labels: {
'com.docker.compose.project': 'my-stack',
},
}]);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
expect.stringContaining('**unknown service**'),
expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }),
);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number));
});
it('resets breach state when condition clears for that container only', async () => {
const svc = MonitorService.getInstance();
// First: breach starts
@@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => {
mockGetStackAlerts.mockReturnValue([{
id: 42,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has(42)).toBe(true);
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true);
// Second: condition clears
setupAlertScenario(10);
mockGetStackAlerts.mockReturnValue([{
id: 42,
stack_name: 'my-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => {
last_fired_at: 0,
}]);
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has(42)).toBe(false);
expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false);
});
it('falls back to short container id when Names is absent', async () => {
setupAlertScenario(90);
mockGetRunningContainers.mockResolvedValue([{
Id: 'abcdef1234567890',
Labels: {
'com.docker.compose.project': 'my-stack',
'com.docker.compose.service': 'api',
},
}]);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).',
{ stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' },
);
});
});
@@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => {
expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1');
// restart_count=5 > threshold=3, should fire
expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' });
expect(mockDispatchAlert).toHaveBeenCalledWith(
'warning',
'monitor_alert',
expect.stringContaining('Restart count'),
{ stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' },
);
});
it('skips Docker inspect when no restart_count rules exist', async () => {
@@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => {
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlerts.mockReturnValue([]);
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
});
it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => {
@@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => {
});
it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => {
// 5 containers in the same stack, all breaching the same rule. Without
// the per-cycle dedup, parallel workers race past the cooldown check
// and each fire dispatchAlert before any DB write lands.
// 5 replicas of the same Compose service, all breaching. Dedup is per
// rule+service, so only one fire should land this cycle.
const containers = Array.from({ length: 5 }, (_, i) => ({
Id: `container-${i}`,
Labels: { 'com.docker.compose.project': 'shared-stack' },
Names: [`/shared-stack-web-${i}`],
Labels: {
'com.docker.compose.project': 'shared-stack',
'com.docker.compose.service': 'web',
},
}));
mockGetRunningContainers.mockResolvedValue(containers);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
@@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => {
mockGetStackAlerts.mockReturnValue([{
id: 7,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
@@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => {
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
@@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => {
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(1);
expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1);
});
it('allows different services on an All-services rule to fire in the same cycle', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 8,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
mockGetGlobalSettings.mockReturnValue({});
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(2);
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number));
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number));
});
it('does not let a healthy sibling clear another container breach timer', async () => {
const svc = MonitorService.getInstance();
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetStackAlerts.mockReturnValue([{
id: 9,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
const highCpu = JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
});
const lowCpu = JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-hot',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-cool',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true);
expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false);
});
it('ignores sibling services for a service-scoped rule', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 10,
stack_name: 'shared-stack',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockReturnValue(null);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(false);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockDispatchAlert).toHaveBeenCalledTimes(1);
expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**');
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number));
});
it('does not block service B after service A fires under new-schema cooldown', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 11,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics
}]);
// api already has a cooldown row; database does not, so database may still fire.
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
const svc = MonitorService.getInstance();
await (svc as any).evaluate();
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number));
});
it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => {
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'api-1',
Names: ['/stack-api-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'api-2',
Names: ['/stack-api-2'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
{
Id: 'db-1',
Names: ['/stack-db-1'],
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
mockGetStackAlerts.mockReturnValue([{
id: 13,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 0,
cooldown_mins: 60,
last_fired_at: 0,
}]);
mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) =>
service === 'api' ? Date.now() - 5 * 60 * 1000 : null,
);
mockHasAnyStackAlertServiceCooldown.mockReturnValue(true);
// beforeEach already cleared the singleton; this instance has empty maps.
const svc = MonitorService.getInstance();
expect((svc as any).activeBreaches.size).toBe(0);
expect((svc as any).firedThisCycle.size).toBe(0);
await (svc as any).evaluate();
const cpuDispatches = mockDispatchAlert.mock.calls.filter(
(args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'),
);
expect(cpuDispatches).toHaveLength(1);
expect(cpuDispatches[0][2]).toContain('**database**');
expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number));
expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number));
});
it('preserves breach timers when container enumeration fails', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlerts.mockReturnValue([{
id: 12,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockRejectedValue(new Error('docker down'));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('12:still-running')).toBe(true);
});
it('removes breach timers for stopped containers after successful enumeration', async () => {
const svc = MonitorService.getInstance();
(svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() });
mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]);
mockGetStackAlerts.mockReturnValue([{
id: 13,
stack_name: 'shared-stack',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 999,
cooldown_mins: 0,
last_fired_at: 0,
}]);
mockGetGlobalSettings.mockReturnValue({});
mockGetRunningContainers.mockResolvedValue([
{
Id: 'still-here',
Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' },
},
]);
mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({
cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 },
precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 },
memory_stats: { usage: 100e6, limit: 1e9 },
}));
await (svc as any).evaluate();
expect((svc as any).activeBreaches.has('13:gone')).toBe(false);
});
it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => {
@@ -142,7 +142,7 @@ describe('networking operator routes', () => {
it('blocks admin delete when network is attached', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }],
networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }],
volumes: [],
}),
@@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => {
it('blocks a network that still has an attached container', () => {
const snapshot = {
volumes: [],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }],
networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }],
};
expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached');
@@ -60,7 +60,7 @@ describe('networking summary', () => {
it('flags a stack with an undeclared runtime network as network drift', async () => {
vi.spyOn(DockerController, 'getInstance').mockReturnValue({
getDependencySnapshot: vi.fn().mockResolvedValue({
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }],
networks: [
{ id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
{ id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK },
@@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => {
mockFetch.mockRejectedValueOnce(new Error('Network timeout'));
// Should not throw
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined();
await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true });
});
it('does not dispatch to global agents when routes array is empty and no stackName', async () => {
@@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: false });
// No external dispatch and no routing read when there is no persisted row.
expect(mockFetch).not.toHaveBeenCalled();
@@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: true });
// The history row was still written; only routing failed, and it was logged.
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
@@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', ()
await expect(
svc.dispatchAlert('info', 'system', 'Host rebooted'),
).resolves.toBeUndefined();
).resolves.toEqual({ persisted: true });
expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1);
expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error));
@@ -0,0 +1,92 @@
import { describe, expect, it } from 'vitest';
import {
isCleanOneShotCompletion,
isNoRestartPolicy,
normalizeComposeRestartIntent,
} from '../utils/oneShotCompletion';
describe('isNoRestartPolicy', () => {
it('accepts only explicit no', () => {
expect(isNoRestartPolicy('no')).toBe(true);
});
it('rejects absent, empty, and restarting policies', () => {
expect(isNoRestartPolicy(undefined)).toBe(false);
expect(isNoRestartPolicy(null)).toBe(false);
expect(isNoRestartPolicy('')).toBe(false);
expect(isNoRestartPolicy('unless-stopped')).toBe(false);
expect(isNoRestartPolicy('always')).toBe(false);
expect(isNoRestartPolicy('on-failure')).toBe(false);
});
});
describe('isCleanOneShotCompletion', () => {
const clean = {
state: 'exited',
exitCode: 0 as number | null,
restartPolicy: 'no' as string | null | undefined,
};
it('returns true only for exited + exit 0 + explicit restart no', () => {
expect(isCleanOneShotCompletion(clean)).toBe(true);
});
it('returns false for absent or empty declared restart', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false);
});
it('returns false for non-exited states', () => {
expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false);
});
it('returns false for non-zero and null exit codes (fail closed)', () => {
expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false);
});
it('returns false for restarting policies even with exit 0', () => {
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false);
expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false);
});
});
describe('normalizeComposeRestartIntent', () => {
it('falls back to service restart when deploy.restart_policy is unset', () => {
expect(normalizeComposeRestartIntent('no')).toBe('no');
expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped');
expect(normalizeComposeRestartIntent(undefined)).toBeNull();
expect(normalizeComposeRestartIntent(null, {})).toBeNull();
expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always');
});
it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => {
expect(normalizeComposeRestartIntent('unless-stopped', {
restart_policy: { condition: 'none' },
})).toBe('no');
expect(normalizeComposeRestartIntent(null, {
restart_policy: { condition: 'any' },
})).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'on-failure' },
})).toBe('on-failure');
expect(normalizeComposeRestartIntent('no', {
restart_policy: {},
})).toBe('always');
});
it('fails closed on malformed restart_policy shapes', () => {
expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always');
expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always');
expect(normalizeComposeRestartIntent('no', {
restart_policy: { condition: 'weird' },
})).toBe('always');
});
});
@@ -212,7 +212,11 @@ describe('hygiene rules', () => {
});
it('flags a missing restart policy and healthcheck', () => {
const bare = model([svc({ restart: undefined, hasHealthcheck: false })]);
expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1);
const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy');
expect(restartFindings).toHaveLength(1);
expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i);
expect(restartFindings[0].remediation).toMatch(/restart: "no"/);
expect(restartFindings[0].remediation).toMatch(/unless-stopped/);
expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1);
const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]);
expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0);
@@ -0,0 +1,256 @@
/**
* Mixed-version gate for POST /api/alerts with a non-empty service_name.
*
* Remote-targeted requests skip express.json() so the raw stream stays
* pipeable. The hub must still buffer that body under the local JSON size
* cap, reject compressed bodies (cannot inspect safely), reject scoped
* creates when the remote lacks service-scoped-stack-alert, and forward
* unscoped identity-encoded JSON intact when allowed.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest';
import http from 'http';
import type { AddressInfo } from 'net';
import zlib from 'zlib';
import request from 'supertest';
import jwt from 'jsonwebtoken';
import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let listenServer: http.Server;
let listenPort: number;
let adminBearer: string;
let capServer: http.Server;
let noCapServer: http.Server;
let capNodeId: number;
let noCapNodeId: number;
const noCapPaths: string[] = [];
const capPaths: string[] = [];
let lastCapBody: Buffer | null = null;
let lastNoCapBody: Buffer | null = null;
function metaServer(
capabilities: string[],
seen: string[],
onBody: (body: Buffer) => void,
): http.Server {
return http.createServer((req, res) => {
if (req.url) seen.push(req.url);
const chunks: Buffer[] = [];
req.on('data', (chunk: Buffer) => chunks.push(chunk));
req.on('end', () => {
if (!req.url?.startsWith('/api/meta')) {
onBody(Buffer.concat(chunks));
}
res.writeHead(200, { 'Content-Type': 'application/json' });
if (req.url?.startsWith('/api/meta')) {
res.end(JSON.stringify({ version: '0.93.0', capabilities }));
} else {
res.end(JSON.stringify({ id: 1, ok: true }));
}
});
});
}
async function listen(server: http.Server): Promise<number> {
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
return (server.address() as AddressInfo).port;
}
beforeAll(async () => {
tmpDir = await setupTestDb();
({ app } = await import('../index'));
const { DatabaseService } = await import('../services/DatabaseService');
const db = DatabaseService.getInstance();
adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' });
capServer = metaServer(
['cross-node-rbac', 'service-scoped-stack-alert'],
capPaths,
(body) => { lastCapBody = body; },
);
noCapServer = metaServer(
['cross-node-rbac'],
noCapPaths,
(body) => { lastNoCapBody = body; },
);
const capPort = await listen(capServer);
const noCapPort = await listen(noCapServer);
capNodeId = db.addNode({
name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token',
});
noCapNodeId = db.addNode({
name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp',
is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token',
});
listenServer = http.createServer(app);
listenPort = await listen(listenServer);
});
afterAll(async () => {
await new Promise<void>((resolve) => listenServer.close(() => resolve()));
await new Promise<void>((resolve) => capServer.close(() => resolve()));
await new Promise<void>((resolve) => noCapServer.close(() => resolve()));
cleanupTestDb(tmpDir);
});
beforeEach(() => {
noCapPaths.length = 0;
capPaths.length = 0;
lastNoCapBody = null;
lastCapBody = null;
});
const scopedPayload = {
stack_name: 'web',
service_name: 'api',
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
};
const unscopedPayload = {
stack_name: 'web',
service_name: null,
metric: 'cpu_percent',
operator: '>',
threshold: 80,
duration_mins: 5,
cooldown_mins: 60,
};
describe('remote proxy service-scoped alert gate', () => {
it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(scopedPayload);
expect(res.status).toBe(400);
expect(res.body.code).toBe('capability_unavailable');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true);
expect(lastNoCapBody).toBeNull();
});
it('forwards unscoped alert JSON intact when remote lacks the capability', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(unscopedPayload);
expect(res.status).toBe(200);
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
expect(lastNoCapBody).not.toBeNull();
expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload);
});
it('forwards scoped alert JSON intact when remote advertises the capability', async () => {
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(capNodeId))
.set('Content-Type', 'application/json')
.send(scopedPayload);
expect(res.status).toBe(200);
expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true);
expect(lastCapBody).not.toBeNull();
expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload);
});
it('rejects oversized identity JSON via Content-Length before upstream', async () => {
const huge = {
...unscopedPayload,
stack_name: 'x'.repeat(120 * 1024),
};
const started = Date.now();
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.send(huge);
const elapsedMs = Date.now() - started;
expect(res.status).toBe(413);
expect(res.body.code).toBe('entity_too_large');
expect(elapsedMs).toBeLessThan(2000);
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => {
const oversized = Buffer.from(JSON.stringify({
...unscopedPayload,
stack_name: 'y'.repeat(120 * 1024),
}));
const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => {
const req = http.request({
hostname: '127.0.0.1',
port: listenPort,
method: 'POST',
path: '/api/alerts',
headers: {
Authorization: `Bearer ${adminBearer}`,
'x-node-id': String(noCapNodeId),
'Content-Type': 'application/json',
'Transfer-Encoding': 'chunked',
},
}, (res) => {
const chunks: Buffer[] = [];
res.on('data', (c: Buffer) => chunks.push(c));
res.on('end', () => {
const text = Buffer.concat(chunks).toString('utf-8');
try {
resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } });
} catch {
resolve({ status: res.statusCode ?? 0, body: {} });
}
});
});
req.on('error', reject);
// Write in small chunks so the hub crosses the limit mid-stream.
const step = 16 * 1024;
for (let offset = 0; offset < oversized.length; offset += step) {
req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length)));
}
req.end();
});
expect(result.status).toBe(413);
expect(result.body.code).toBe('entity_too_large');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => {
const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8'));
const res = await request(app)
.post('/api/alerts')
.set('Authorization', `Bearer ${adminBearer}`)
.set('x-node-id', String(noCapNodeId))
.set('Content-Type', 'application/json')
.set('Content-Encoding', 'gzip')
.send(compressed);
expect(res.status).toBe(415);
expect(res.body.code).toBe('encoding_unsupported');
expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false);
expect(lastNoCapBody).toBeNull();
});
});
@@ -30,10 +30,9 @@ describe('console_session token parity (HTTP route vs mint helper)', () => {
({ app } = await import('../index'));
adminCookie = await loginAsTestAdmin(app);
// POST /api/system/console-token is paid-gated. Seed an active license so the
// parity assertion can observe the token the route returns. The
// license_last_validated fallback is skipped when the state key is absent, so
// the active status alone drives the paid tier.
// POST /api/system/console-token is admin-gated (available on every tier).
// Seed an active license only so other suites that share LicenseService
// state stay stable if they expect paid.
const { DatabaseService } = await import('../services/DatabaseService');
DatabaseService.getInstance().setSystemState('license_status', 'active');
});
@@ -43,12 +42,13 @@ describe('console_session token parity (HTTP route vs mint helper)', () => {
});
it('POST /api/system/console-token produces a token with the same shape as mintConsoleSession()', async () => {
const directToken = mintConsoleSession();
const directToken = mintConsoleSession({ path: 'host-console', actingAs: 'testadmin' });
const directDecoded = jwt.verify(directToken, TEST_JWT_SECRET) as Record<string, unknown>;
const res = await request(app)
.post('/api/system/console-token')
.set('Cookie', adminCookie);
.set('Cookie', adminCookie)
.send({ path: 'host-console' });
expect(res.status).toBe(200);
expect(typeof res.body.token).toBe('string');
@@ -57,6 +57,12 @@ describe('console_session token parity (HTTP route vs mint helper)', () => {
// Identical scope so the remote's upgrade handler treats both the same.
expect(routeDecoded.scope).toBe('console_session');
expect(directDecoded.scope).toBe('console_session');
expect(routeDecoded.path).toBe('host-console');
expect(directDecoded.path).toBe('host-console');
expect(typeof routeDecoded.jti).toBe('string');
expect(typeof directDecoded.jti).toBe('string');
expect(routeDecoded.acting_as).toBe('testadmin');
expect(directDecoded.acting_as).toBe('testadmin');
// Same claim keys: if somebody adds or drops a claim on one path but not
// the other, remote upgrade behavior will diverge.
@@ -58,6 +58,13 @@ describe('remoteWsForwardAllowed', () => {
expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false);
});
it('denies every API-token scope for Host Console while preserving full-admin for container exec', () => {
expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(false);
expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false);
expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'read-only', decoded: { scope: 'api_token' } })).toBe(false);
expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(true);
});
it('allows an interactive path for a pre-gated console_session token', () => {
expect(remoteWsForwardAllowed(CONSOLE, { ...base, decoded: { scope: 'console_session' } })).toBe(true);
});
@@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => {
networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }],
volumes: [],
containers: [{
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx',
id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx',
networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [],
}],
};
@@ -34,7 +34,7 @@ const {
mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0),
mockDeleteOldScans: vi.fn().mockReturnValue(0),
mockGetTier: vi.fn().mockReturnValue('paid'),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetProxyTarget: vi.fn().mockReturnValue(null),
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
mockScanAllNodeImages: vi.fn(),
@@ -59,7 +59,7 @@ const {
mockGetStackContent: vi.fn().mockResolvedValue(''),
mockGetEnvContent: vi.fn().mockResolvedValue(''),
mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }),
mockDispatchAlert: vi.fn().mockResolvedValue(undefined),
mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }),
mockGetProxyTarget: vi.fn().mockReturnValue(null),
mockIsTrivyAvailable: vi.fn().mockReturnValue(true),
mockScanAllNodeImages: vi.fn().mockResolvedValue({
@@ -45,7 +45,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -78,7 +78,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -131,3 +131,32 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () =
expect(sawCascadeWarning).toBe(true);
});
});
describe('DELETE /api/stacks/:stackName clears stack-scoped role assignments', () => {
it('removes only the deleted stack assignment and preserves unrelated grants', async () => {
const db = DatabaseService.getInstance();
const bcrypt = await import('bcrypt');
const hash = await bcrypt.hash('password123', 1);
const userId = db.addUser({ username: 'stack-del-rbac', password_hash: hash, role: 'viewer' });
const otherNodeId = db.addNode({
name: 'stack-del-rbac-node', type: 'remote', api_url: 'http://test:1852',
api_token: '', compose_dir: '/tmp', is_default: false,
});
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'api' });
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack' });
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId) });
const res = await request(app)
.delete('/api/stacks/api')
.set('Cookie', adminCookie);
expect(res.status).toBe(200);
const remaining = db.getAllRoleAssignments(userId);
expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'api')).toBe(false);
expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'other-stack')).toBe(true);
expect(remaining.some((a) => a.resource_type === 'node' && a.resource_id === String(otherNodeId))).toBe(true);
db.deleteUser(userId);
db.deleteNode(otherNodeId);
});
});
@@ -0,0 +1,132 @@
/**
* DELETE /api/stacks/:stackName must purge notification_history for that
* (node_id, stack_name) so the bell panel, Activity ticker, Stack Activity,
* and Networking recent activity no longer show the deleted stack.
*/
import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest';
import request from 'supertest';
import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb';
let tmpDir: string;
let app: import('express').Express;
let DatabaseService: typeof import('../services/DatabaseService').DatabaseService;
let ComposeService: typeof import('../services/ComposeService').ComposeService;
let FileSystemService: typeof import('../services/FileSystemService').FileSystemService;
let MeshService: typeof import('../services/MeshService').MeshService;
let NotificationService: typeof import('../services/NotificationService').NotificationService;
let adminCookie: string;
beforeAll(async () => {
tmpDir = await setupTestDb();
({ DatabaseService } = await import('../services/DatabaseService'));
({ ComposeService } = await import('../services/ComposeService'));
({ FileSystemService } = await import('../services/FileSystemService'));
({ MeshService } = await import('../services/MeshService'));
({ NotificationService } = await import('../services/NotificationService'));
({ app } = await import('../index'));
adminCookie = await loginAsTestAdmin(app);
});
afterAll(() => cleanupTestDb(tmpDir));
beforeEach(() => {
vi.restoreAllMocks();
vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined);
vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined);
vi.spyOn(MeshService.getInstance(), 'optOutStack').mockResolvedValue(undefined);
const raw = DatabaseService.getInstance().getDb();
raw.prepare('DELETE FROM notification_history').run();
});
function seedNotif(
nodeId: number,
opts: { stack_name?: string; message?: string; timestamp?: number },
): void {
DatabaseService.getInstance().addNotificationHistory(nodeId, {
level: 'error',
message: opts.message ?? 'test',
timestamp: opts.timestamp ?? Date.now(),
stack_name: opts.stack_name ?? 'web',
category: 'deploy_failure',
});
}
function stackNamesForNode(nodeId: number): Array<string | null> {
const rows = DatabaseService.getInstance()
.getDb()
.prepare('SELECT stack_name FROM notification_history WHERE node_id = ? ORDER BY id')
.all(nodeId) as Array<{ stack_name: string | null }>;
return rows.map((r) => r.stack_name);
}
describe('DELETE /api/stacks/:stackName purges notifications', () => {
it('removes only the target node/stack rows and emits one invalidation', async () => {
const db = DatabaseService.getInstance();
const nodeA = db.getNodes()[0].id;
const nodeB = db.addNode({
name: 'other-node',
type: 'remote',
api_url: 'http://other:1852',
api_token: 't',
compose_dir: '/tmp',
is_default: false,
});
seedNotif(nodeA, { stack_name: 'web', message: 'a-web' });
seedNotif(nodeA, { stack_name: 'db', message: 'a-db' });
// Unattached (null stack_name) via SQL; addNotificationHistory omits null.
db.getDb()
.prepare(
`INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, category)
VALUES (?, 'info', 'a-unattached', ?, 0, NULL, 'system')`,
)
.run(nodeA, Date.now());
seedNotif(nodeB, { stack_name: 'web', message: 'b-web' });
const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent');
const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie);
expect(res.status).toBe(200);
expect(stackNamesForNode(nodeA).sort((a, b) => String(a).localeCompare(String(b)))).toEqual([
'db',
null,
]);
expect(stackNamesForNode(nodeB)).toEqual(['web']);
const notifInvalidations = broadcastSpy.mock.calls
.map((c) => c[0])
.filter(
(e) =>
e.type === 'state-invalidate' &&
e.scope === 'notifications' &&
e.action === 'stack-deleted',
);
expect(notifInvalidations).toHaveLength(1);
expect(notifInvalidations[0]).toMatchObject({
type: 'state-invalidate',
scope: 'notifications',
action: 'stack-deleted',
nodeId: nodeA,
stackName: 'web',
});
expect(typeof notifInvalidations[0].ts).toBe('number');
});
it('emits no notification invalidation when delete fails before success', async () => {
vi.spyOn(FileSystemService.prototype, 'deleteStack').mockRejectedValue(new Error('fs boom'));
const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent');
const db = DatabaseService.getInstance();
const nodeId = db.getNodes()[0].id;
seedNotif(nodeId, { stack_name: 'web' });
const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie);
expect(res.status).toBe(500);
expect(stackNamesForNode(nodeId)).toEqual(['web']);
const notifInvalidations = broadcastSpy.mock.calls
.map((c) => c[0])
.filter((e) => e.scope === 'notifications' && e.action === 'stack-deleted');
expect(notifInvalidations).toHaveLength(0);
});
});
@@ -90,7 +90,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -68,7 +68,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -1116,6 +1116,37 @@ describe('PUT /api/stacks/:stackName/files/content', () => {
expect(content).toBe('community-write');
});
it('blocks root trust file writes while a stack op lock is held', async () => {
const { StackOpLockService } = await import('../services/StackOpLockService');
StackOpLockService.getInstance().tryAcquire(1, STACK, 'deploy', 'admin');
try {
const composeRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: 'compose.yaml' })
.set('Cookie', adminCookie)
.send({ content: 'services:\n app:\n image: nginx\n' });
expect(composeRes.status).toBe(409);
expect(composeRes.body.code).toBe('stack_op_in_progress');
const markerRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: '.blueprint.json' })
.set('Cookie', adminCookie)
.send({ content: '{"blueprintId":1,"revision":1,"lastApplied":0}' });
expect(markerRes.status).toBe(409);
expect(markerRes.body.code).toBe('stack_op_in_progress');
const nestedRes = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
.query({ path: 'config/app.conf' })
.set('Cookie', adminCookie)
.send({ content: 'ok' });
expect(nestedRes.status).toBe(204);
} finally {
StackOpLockService.getInstance().release(1, STACK);
}
});
it('returns 400 when content is not a string', async () => {
const res = await request(app)
.put(`/api/stacks/${STACK}/files/content`)
@@ -1931,6 +1962,16 @@ describe('protected stack files', () => {
expect(res.body.code).toBe('PROTECTED_FILE');
});
it('DELETE /files refuses .blueprint.json with 409 PROTECTED_FILE', async () => {
await fs.writeFile(path.join(stacksDir, STACK, '.blueprint.json'), '{"blueprintId":1,"revision":1}\n');
const res = await request(app)
.delete(`/api/stacks/${STACK}/files`)
.query({ path: '.blueprint.json' })
.set('Cookie', adminCookie);
expect(res.status).toBe(409);
expect(res.body.code).toBe('PROTECTED_FILE');
});
it('PATCH /files/rename refuses compose.yaml as source with 409 PROTECTED_FILE', async () => {
const res = await request(app)
.patch(`/api/stacks/${STACK}/files/rename`)
@@ -89,7 +89,7 @@ beforeAll(async () => {
authCookie = await loginAsTestAdmin(app);
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -102,7 +102,7 @@ beforeAll(async () => {
writeStack('web');
const { NotificationService } = await import('../services/NotificationService');
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined);
vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true });
});
afterAll(() => {
@@ -132,7 +132,7 @@ beforeAll(async () => {
const { NotificationService } = await import('../services/NotificationService');
dispatchAlertSpy = vi
.spyOn(NotificationService.getInstance(), 'dispatchAlert')
.mockResolvedValue(undefined);
.mockResolvedValue({ persisted: true });
});
afterAll(() => {
+95 -1
View File
@@ -8,7 +8,7 @@
* product paths. This test pins each position by observing behavior that
* could only originate from the expected handler.
*/
import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest';
import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest';
import WebSocket from 'ws';
import jwt from 'jsonwebtoken';
import bcrypt from 'bcrypt';
@@ -420,6 +420,100 @@ describe('WebSocket upgrade dispatch order', () => {
expect(outcome.kind).toBe('unexpected');
if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403);
});
it('rejects a full-admin API token remote Host Console with 403 (container exec stays allowed)', async () => {
const { DatabaseService } = await import('../services/DatabaseService');
const adminId = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!.id;
const rawToken = createTestApiToken({
db: DatabaseService,
scope: 'full-admin',
userId: adminId,
name: `host-console-api-deny-${Date.now()}`,
});
const consoleWs = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { bearer: rawToken });
const consoleOutcome = await waitForOutcome(consoleWs);
expect(consoleOutcome.kind).toBe('unexpected');
if (consoleOutcome.kind === 'unexpected') expect(consoleOutcome.status).toBe(403);
const execWs = connect(`/ws?nodeId=${remoteNodeId}`, { bearer: rawToken });
const execOutcome = await waitForOutcome(execWs);
if (execOutcome.kind === 'unexpected') expect(execOutcome.status).not.toBe(403);
try { execWs.terminate(); } catch { /* ignore */ }
});
});
describe('remote Host Console Community capability probe', () => {
// Community hubs must probe host-console-community before forwarding Host
// Console; Admiral hubs skip the probe so legacy remotes still work.
// Fail closed when the probe errors or the capability is missing.
const meta = (capabilities: string[]): import('../services/CapabilityRegistry').RemoteMeta => ({
version: '0.95.0',
capabilities,
startedAt: 1,
updateError: null,
online: true,
imagePinKind: null,
updateBlocked: false,
imageChannel: null,
});
async function setLicense(status: string): Promise<void> {
const { DatabaseService } = await import('../services/DatabaseService');
DatabaseService.getInstance().setSystemState('license_status', status);
}
afterEach(async () => {
vi.restoreAllMocks();
await setLicense('active');
});
it('rejects Community hub remote Host Console when the remote lacks host-console-community', async () => {
await setLicense('community');
const { NodeRegistry } = await import('../services/NodeRegistry');
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta(['host-console']));
const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie });
const outcome = await waitForOutcome(ws);
expect(outcome.kind).toBe('unexpected');
if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403);
});
it('rejects Community hub remote Host Console when the capability probe fails', async () => {
await setLicense('community');
const { NodeRegistry } = await import('../services/NodeRegistry');
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockRejectedValue(new Error('offline'));
const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie });
const outcome = await waitForOutcome(ws);
expect(outcome.kind).toBe('unexpected');
if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403);
});
it('forwards Community hub remote Host Console when the remote advertises host-console-community', async () => {
await setLicense('community');
const { NodeRegistry } = await import('../services/NodeRegistry');
vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(
meta(['host-console-community']),
);
const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie });
const outcome = await waitForOutcome(ws);
if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403);
try { ws.terminate(); } catch { /* ignore */ }
});
it('skips the capability probe on an Admiral hub even when the remote lacks host-console-community', async () => {
await setLicense('active');
const { NodeRegistry } = await import('../services/NodeRegistry');
const spy = vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta([]));
const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie });
const outcome = await waitForOutcome(ws);
if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403);
expect(spy).not.toHaveBeenCalled();
try { ws.terminate(); } catch { /* ignore */ }
});
});
it('dispatches /api/pilot/tunnel to the pilot handler (rejects non-pilot bearer before path-based dispatch)', async () => {
+23
View File
@@ -714,6 +714,29 @@ describe('Orphaned role assignment cleanup', () => {
// Cleanup
db.deleteUser(userId);
});
it('deleteRoleAssignmentsByResource removes only the matching resource tuple', async () => {
const db = DatabaseService.getInstance();
const hash = await bcrypt.hash('password123', 1);
const userId = db.addUser({ username: 'tupleorphan', password_hash: hash, role: 'viewer' });
const nodeId = db.addNode({
name: 'tuple-cleanup-node', type: 'remote', api_url: 'http://test:1852',
api_token: '', compose_dir: '/tmp', is_default: false,
});
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'target-stack' });
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'keep-stack' });
db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(nodeId) });
db.deleteRoleAssignmentsByResource('stack', 'target-stack');
const after = db.getAllRoleAssignments(userId);
expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'target-stack')).toBe(false);
expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'keep-stack')).toBe(true);
expect(after.some((a) => a.resource_type === 'node' && a.resource_id === String(nodeId))).toBe(true);
db.deleteUser(userId);
db.deleteNode(nodeId);
});
});
// ---- Role-Based Permission Checks (via API) ----
+28
View File
@@ -0,0 +1,28 @@
/**
* Dependency-neutral blueprint marker parse helpers.
* Used by BlueprintService and DeployedStackDeletionService without a service import cycle.
*/
export const BLUEPRINT_MARKER_FILENAME = '.blueprint.json';
export interface BlueprintMarker {
blueprintId: number;
revision: number;
lastApplied: number;
}
export function parseBlueprintMarker(content: string): BlueprintMarker | null {
try {
const parsed: unknown = JSON.parse(content);
if (!parsed || typeof parsed !== 'object') return null;
const obj = parsed as Record<string, unknown>;
if (typeof obj.blueprintId !== 'number' || typeof obj.revision !== 'number') return null;
return {
blueprintId: obj.blueprintId,
revision: obj.revision,
lastApplied: typeof obj.lastApplied === 'number' ? obj.lastApplied : 0,
};
} catch {
return null;
}
}
@@ -26,6 +26,8 @@ export interface DeclaredService {
image?: string;
/** Service `network_mode:` as declared (e.g. `host`), or undefined. */
networkMode?: string;
/** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */
restart?: string | null;
}
/** A top-level networks:/volumes: entry. */
@@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose {
ports,
image: asString(svc.image),
networkMode: asString(svc.network_mode),
restart: asString(svc.restart) ?? null,
});
}
+64 -10
View File
@@ -1,17 +1,55 @@
import { randomUUID } from 'crypto';
import jwt from 'jsonwebtoken';
import { DatabaseService } from '../services/DatabaseService';
/**
* Console session token lifetime. Short on purpose: these tokens are only
* used to bridge an already-authenticated HTTP request into a WebSocket
* upgrade, and each one is consumed by a single `wss:ws(target)` call.
* Console session token lifetime. Short on purpose: these tokens bridge an
* already-authenticated hub request into a remote WebSocket upgrade. Each
* token is path-scoped and consumed on first interactive WebSocket upgrade
* acceptance (before PTY spawn).
*/
const CONSOLE_SESSION_TTL_SECONDS = 60;
const CONSOLE_SESSION_SCOPE = 'console_session';
/** Interactive surfaces that may mint or accept a console_session JWT. */
export type ConsoleSessionPath = 'host-console' | 'container-exec';
export interface MintConsoleSessionOptions {
path: ConsoleSessionPath;
/** Hub operator identity for remote audit (option B). Never used as the session principal. */
actingAs?: string;
}
export interface ConsoleSessionClaims {
scope: typeof CONSOLE_SESSION_SCOPE;
username?: string;
path: ConsoleSessionPath;
acting_as?: string;
}
const ACTING_AS_MAX = 64;
const ACTING_AS_RE = /^[A-Za-z0-9._@+-]+$/;
/** Sanitize an optional hub operator name for the acting_as claim. */
export function sanitizeActingAs(raw: unknown): string | undefined {
if (typeof raw !== 'string') return undefined;
const trimmed = raw.trim();
if (!trimmed || trimmed.length > ACTING_AS_MAX) return undefined;
if (!ACTING_AS_RE.test(trimmed)) return undefined;
return trimmed;
}
export function isConsoleSessionPath(value: unknown): value is ConsoleSessionPath {
return value === 'host-console' || value === 'container-exec';
}
/**
* Map a WebSocket pathname to the console_session path claim it requires.
* Returns null for surfaces that must not accept console_session tokens.
*/
export function consoleSessionPathForPathname(pathname: string): ConsoleSessionPath | null {
if (pathname.startsWith('/api/system/host-console')) return 'host-console';
if (pathname === '/ws') return 'container-exec';
return null;
}
/**
@@ -19,13 +57,19 @@ export interface ConsoleSessionClaims {
* instance without leaking the long-lived node api_token onto a
* machine-to-machine WebSocket. Throws if the JWT secret is not configured.
*/
export function mintConsoleSession(username?: string): string {
export function mintConsoleSession(opts: MintConsoleSessionOptions): string {
if (!isConsoleSessionPath(opts.path)) {
throw new Error('Invalid console session path');
}
const jwtSecret = DatabaseService.getInstance().getGlobalSettings().auth_jwt_secret;
if (!jwtSecret) throw new Error('No JWT secret configured');
const payload: ConsoleSessionClaims = username
? { scope: CONSOLE_SESSION_SCOPE, username }
: { scope: CONSOLE_SESSION_SCOPE };
return jwt.sign(payload, jwtSecret, { expiresIn: CONSOLE_SESSION_TTL_SECONDS });
const payload: ConsoleSessionClaims = { scope: CONSOLE_SESSION_SCOPE, path: opts.path };
const actingAs = sanitizeActingAs(opts.actingAs);
if (actingAs) payload.acting_as = actingAs;
return jwt.sign(payload, jwtSecret, {
expiresIn: CONSOLE_SESSION_TTL_SECONDS,
jwtid: randomUUID(),
});
}
/** True when `decoded.scope` is the console_session scope. */
@@ -33,4 +77,14 @@ export function isConsoleSessionScope(scope: unknown): boolean {
return scope === CONSOLE_SESSION_SCOPE;
}
export { CONSOLE_SESSION_SCOPE };
/**
* Mark a console_session jti as used. Returns false when the jti is missing,
* already consumed, or cannot be recorded (replay / malformed token).
*/
export function consumeConsoleSessionJti(jti: unknown, expiresAtMs: number): boolean {
if (typeof jti !== 'string' || !jti) return false;
if (!Number.isFinite(expiresAtMs)) return false;
return DatabaseService.getInstance().consumeConsoleSessionJti(jti, expiresAtMs);
}
export { CONSOLE_SESSION_SCOPE, CONSOLE_SESSION_TTL_SECONDS };
@@ -1,5 +1,6 @@
import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel';
import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse';
import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion';
/** Map one rendered network/volume entry to the declared shape drift expects. */
function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource {
@@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo
),
image: s.image,
networkMode: s.networkMode,
restart: normalizeComposeRestartIntent(s.restart, s.deploy),
}));
return {
+207 -4
View File
@@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY
import { LicenseService } from '../services/LicenseService';
import { isProxyExemptPath } from '../helpers/proxyExemptPaths';
import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry';
import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry';
import { getErrorMessage } from '../utils/errors';
import { DatabaseService } from '../services/DatabaseService';
import { redactSensitiveText } from '../utils/safeLog';
@@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
// Body forwarding: conditionalJsonParser skips parsing for remote
// requests (see middleware/jsonParser.ts), so req's raw stream is
// intact and http-proxy's req.pipe(proxyReq) forwards the body
// automatically.
// usually intact and http-proxy's req.pipe(proxyReq) forwards it.
// When a gate must inspect JSON (POST /alerts), we buffer into
// req.rawBody first; rewrite that buffer here because the stream is
// already consumed.
if (req.rawBody) {
proxyReq.removeHeader('Transfer-Encoding');
proxyReq.removeHeader('Content-Length');
if (!proxyReq.getHeader('Content-Type')) {
proxyReq.setHeader('Content-Type', 'application/json');
}
proxyReq.setHeader('Content-Length', req.rawBody.length);
proxyReq.write(req.rawBody);
}
},
proxyRes: (proxyRes, req) => {
// Mark every response forwarded from a remote node with a sentinel
@@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
}
// Mixed-version RBAC gate (non-admin only).
// Mixed-version RBAC gate (non-admin only). Runs before alert body
// buffering so an unauthorized client cannot force unbounded memory use.
if (req.user?.role !== 'admin') {
const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId);
if (!rbacSupported) {
@@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler {
}
}
// POST /alerts bodies are not on req.body for remote hops (JSON parsing
// is skipped so the stream can be piped). Buffer once under the same
// 100 KB cap as express.json(), gate on service_name, then rewrite
// rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot
// inspect them, and treating parse failure as unscoped would bypass the
// mixed-version gate.
if (isAlertCreateRoute(req)) {
if (hasNonIdentityContentEncoding(req)) {
await drainRequestBody(req);
res.status(415).json({
error: 'Compressed request bodies are not supported for remote alert creates',
code: 'encoding_unsupported',
});
return;
}
try {
req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT);
} catch (err) {
const status = Number((err as { status?: number }).status);
if (status === 413) {
console.error('[remoteNodeProxy] alert body rejected as too large:', err);
res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' });
return;
}
if (status === 400) {
console.error('[remoteNodeProxy] alert body incomplete:', err);
res.status(400).json({ error: 'Incomplete request body' });
return;
}
throw err;
}
if (alertCreateHasScopedService(req.rawBody)) {
const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY);
if (!supported) {
res.status(400).json({
error: 'Service-scoped alert rules are not supported on this node',
code: 'capability_unavailable',
});
return;
}
}
}
req.proxyTarget = target;
beginProxyTiming(req, res);
proxy(req, res, next);
@@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean {
}
return false;
}
/** POST /alerts (path is post-/api strip). */
function isAlertCreateRoute(req: Request): boolean {
return req.method === 'POST' && /^\/alerts\/?$/.test(req.path);
}
/** Same default as express.json(); remote alert creates must not exceed it. */
const ALERT_PROXY_BODY_LIMIT = 100 * 1024;
/** Max time to wait for leftover body bytes after a size/encoding reject. */
const DRAIN_TIMEOUT_MS = 5_000;
/** Error with HTTP status for the alert-body gate catch mapper. */
function alertBodyError(message: string, status: number): Error {
return Object.assign(new Error(message), { status, expose: true });
}
/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */
function hasNonIdentityContentEncoding(req: Request): boolean {
const raw = req.headers['content-encoding'];
if (raw == null) return false;
const value = Array.isArray(raw) ? raw.join(',') : raw;
return value.split(',').some((part) => {
const encoding = part.trim().toLowerCase();
return encoding.length > 0 && encoding !== 'identity';
});
}
/** True when the buffered JSON alert body targets a specific Compose service. */
function alertCreateHasScopedService(rawBody: Buffer): boolean {
if (rawBody.length === 0) return false;
try {
const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown };
return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== '';
} catch {
// Identity-encoded non-JSON is forwarded as-is; the remote rejects it.
// Encoded bodies never reach here (rejected earlier).
return false;
}
}
/**
* Consume remaining request bytes (or wait for abort/close) so the response
* can flush without leaving the socket half-open. Does not buffer into memory.
* Caps wait time so a stalled or endless chunked stream cannot hang the gate.
*/
function drainRequestBody(req: Request): Promise<void> {
return new Promise((resolve) => {
if (req.readableEnded || req.destroyed) {
resolve();
return;
}
let settled = false;
const done = () => {
if (settled) return;
settled = true;
clearTimeout(timer);
req.off('end', done);
req.off('error', done);
req.off('aborted', done);
req.off('close', done);
resolve();
};
// Bound hang time: destroy after timeout so mid-stream rejects still settle.
const timer = setTimeout(() => {
if (!req.destroyed) req.destroy();
done();
}, DRAIN_TIMEOUT_MS);
req.resume();
req.once('end', done);
req.once('error', done);
req.once('aborted', done);
req.once('close', done);
});
}
/**
* Buffer the request so a capability gate can inspect JSON without leaving
* http-proxy with an already-ended empty stream. Enforces `limit` on both
* declared Content-Length and streamed accumulation.
*/
async function bufferRequestBody(req: Request, limit: number): Promise<Buffer> {
if (req.rawBody) {
if (req.rawBody.length > limit) {
throw alertBodyError('Alert payload too large', 413);
}
return req.rawBody;
}
if (req.readableEnded) return Buffer.alloc(0);
const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10);
if (Number.isFinite(declared) && declared > limit) {
// Reject immediately so the client gets a structured 413; drain leftover
// bytes in the background so the socket can close without holding the gate.
void drainRequestBody(req);
throw alertBodyError('Alert payload too large', 413);
}
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
let total = 0;
let settled = false;
const settle = (fn: () => void) => {
if (settled) return;
settled = true;
cleanup();
fn();
};
const finish = (buf: Buffer) => settle(() => resolve(buf));
const fail = (err: Error) => settle(() => reject(err));
const onData = (chunk: Buffer) => {
const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
total += buf.length;
if (total > limit) {
chunks.length = 0;
// fail() removes listeners; drain leftover bytes so the socket can close.
fail(alertBodyError('Alert payload too large', 413));
void drainRequestBody(req);
return;
}
chunks.push(buf);
};
const onEnd = () => finish(Buffer.concat(chunks));
const onError = (err: Error) => fail(err);
const onAborted = () => fail(alertBodyError('Client aborted request body', 400));
const onClose = () => {
if (!settled && !req.readableEnded) {
fail(alertBodyError('Client closed request before body finished', 400));
}
};
function cleanup(): void {
req.off('data', onData);
req.off('end', onEnd);
req.off('error', onError);
req.off('aborted', onAborted);
req.off('close', onClose);
}
req.on('data', onData);
req.on('end', onEnd);
req.on('error', onError);
req.on('aborted', onAborted);
req.on('close', onClose);
});
}
+39 -4
View File
@@ -3,9 +3,21 @@ import { z } from 'zod';
import { DatabaseService } from '../services/DatabaseService';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin } from '../middleware/tierGates';
import { isValidServiceName } from '../utils/validation';
import {
getActiveCapabilities,
SERVICE_SCOPED_STACK_ALERT_CAPABILITY,
} from '../services/CapabilityRegistry';
const AlertCreateSchema = z.object({
stack_name: z.string().min(1).max(255),
service_name: z.preprocess(
(val) => (val === '' ? null : val),
z.string().max(255).nullable().optional().refine(
(val) => val == null || isValidServiceName(val),
{ message: 'Invalid service name' },
),
),
metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']),
operator: z.enum(['>', '>=', '<', '<=', '==']),
threshold: z.number().min(0),
@@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => {
const alerts = DatabaseService.getInstance().getStackAlerts(stackName);
res.json(alerts);
} catch {
} catch (error) {
console.error('Failed to fetch alerts:', error);
res.status(500).json({ error: 'Failed to fetch alerts' });
}
});
@@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors });
return;
}
const { service_name, ...alertFields } = parsed.data;
const serviceName = service_name ?? null;
if (
serviceName != null
&& !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY)
) {
res.status(400).json({
error: 'This node does not support service-scoped alert rules',
code: 'capability_unavailable',
});
return;
}
try {
const created = DatabaseService.getInstance().addStackAlert(parsed.data);
const created = DatabaseService.getInstance().addStackAlert({
...alertFields,
service_name: serviceName,
});
res.status(201).json(created);
} catch (error) {
console.error('Failed to add alert:', error);
@@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => {
alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => {
if (!requireAdmin(req, res)) return;
// Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2).
const rawId = String(req.params.id ?? '');
const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN;
if (!Number.isInteger(id) || id <= 0) {
res.status(400).json({ error: 'Invalid alert id' });
return;
}
try {
const id = parseInt(req.params.id as string, 10);
DatabaseService.getInstance().deleteStackAlert(id);
res.json({ success: true });
} catch {
} catch (error) {
console.error('Failed to delete alert:', error);
res.status(500).json({ error: 'Failed to delete alert' });
}
});
+1 -1
View File
@@ -103,7 +103,7 @@ auditLogRouter.get('/export', async (req: Request, res: Response): Promise<void>
res.setHeader('Content-Type', 'text/csv');
res.setHeader('Content-Disposition', `attachment; filename="audit-log-${timestamp}.csv"`);
const headers = ['id', 'timestamp', 'username', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary'];
const headers = ['id', 'timestamp', 'username', 'acting_as', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary'];
const rows = result.entries.map(e =>
headers.map(h => escapeCsvField(e[h as keyof typeof e])).join(','),
);
+79 -8
View File
@@ -7,7 +7,9 @@ import {
type BlueprintSelector,
type DriftMode,
} from '../services/DatabaseService';
import { BlueprintService } from '../services/BlueprintService';
import { BlueprintService, BlueprintNameConflictError, BlueprintOwnershipProbeError } from '../services/BlueprintService';
import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService';
import { refuseIfSelfStack } from '../helpers/selfStackGuard';
import {
BlueprintReconciler,
messageForConfirmedOutcomes,
@@ -317,12 +319,9 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise<voi
return;
}
}
// Best-effort cleanup before delete: withdraw exactly the rows a stateful delete would
// block, i.e. stacks Sencho deployed and still owns (last_deployed_at set, and neither a
// name_conflict nor an already-withdrawn row). Never run the withdraw primitive for a
// never-deployed or unmanaged row: withdrawFromNode proceeds on a missing marker and would
// down/delete a same-name stack Sencho does not own. The blueprint-delete cascade removes
// the rows the loop skips.
// Withdraw owned deployments before delete. Fail closed: if any withdraw does not
// complete as withdrawn, keep the blueprint (and its deployment rows) so the operator
// can retry. Never orphan a live stack by deleting the only control-plane record.
const nodes = DatabaseService.getInstance().getNodes();
const deployments = DatabaseService.getInstance().listDeployments(id);
for (const dep of deployments) {
@@ -330,9 +329,24 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise<voi
const node = nodes.find(n => n.id === dep.node_id);
if (!node) continue;
try {
await BlueprintService.getInstance().withdrawFromNode(blueprint, node);
const outcome = await BlueprintService.getInstance().withdrawFromNode(blueprint, node);
if (outcome.status !== 'withdrawn') {
res.status(409).json({
error: `Cannot delete blueprint: withdraw on node "${node.name}" ended as ${outcome.status}. Resolve that deployment, then retry.`,
code: 'withdraw_failed_blocking_delete',
nodeId: node.id,
withdrawStatus: outcome.status,
});
return;
}
} catch (err) {
console.warn(`[Blueprints] Pre-delete withdraw failed for blueprint ${id} on node ${node.id}:`, err);
res.status(409).json({
error: `Cannot delete blueprint: withdraw on node "${node.name}" failed. Resolve that deployment, then retry.`,
code: 'withdraw_failed_blocking_delete',
nodeId: node.id,
});
return;
}
}
DatabaseService.getInstance().deleteBlueprint(id);
@@ -384,11 +398,68 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi
}
res.json({ deployed: true });
} catch (error) {
if (error instanceof BlueprintNameConflictError) {
res.status(409).json({ error: error.message, code: 'name_conflict' });
return;
}
if (error instanceof BlueprintOwnershipProbeError) {
console.error('[Blueprints] apply-local ownership probe failed:', sanitizeForLog(error.message));
res.status(500).json({ error: error.message });
return;
}
console.error('[Blueprints] apply-local error:', sanitizeForLog(getErrorMessage(error, 'apply failed')));
res.status(500).json({ error: getErrorMessage(error, 'Blueprint apply failed') });
}
});
// Node-to-node atomic blueprint withdraw. Ownership is validated under the
// delete lock on this node. Requires stack:delete and refuses Sencho's own stack.
blueprintsRouter.post('/withdraw-local', async (req: Request, res: Response): Promise<void> => {
const body = (req.body ?? {}) as { stackName?: unknown; blueprintId?: unknown };
if (typeof body.stackName !== 'string' || !isValidStackName(body.stackName)) {
res.status(400).json({ error: 'Invalid stack name' });
return;
}
if (typeof body.blueprintId !== 'number' || !Number.isInteger(body.blueprintId) || body.blueprintId <= 0) {
res.status(400).json({ error: 'blueprintId must be a positive integer' });
return;
}
if (!requirePermission(req, res, 'stack:delete', 'stack', body.stackName)) return;
if (await refuseIfSelfStack(req, res, body.stackName)) return;
try {
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: req.nodeId,
stackName: body.stackName,
pruneVolumes: false,
actor: req.user?.username ?? 'system:blueprint',
requireBlueprintId: body.blueprintId,
});
if (result.ok) {
res.json({
status: result.status === 'already_absent' ? 'already_absent' : 'withdrawn',
});
return;
}
if (result.code === 'name_conflict') {
res.status(409).json({ error: result.error, code: 'name_conflict' });
return;
}
if (result.code === 'lock_conflict') {
res.status(409).json({
error: result.error,
code: 'stack_op_in_progress',
inProgress: { action: result.existingAction },
});
return;
}
res.status(500).json({ error: result.error });
} catch (error) {
console.error('[Blueprints] withdraw-local error:', sanitizeForLog(getErrorMessage(error, 'withdraw failed')));
res.status(500).json({ error: getErrorMessage(error, 'Blueprint withdraw failed') });
}
});
blueprintsRouter.post('/:id/apply', async (req: Request, res: Response): Promise<void> => {
if (!requireAdmin(req, res)) return;
const id = parseIntParam(req, res, 'id');
+21 -4
View File
@@ -1,8 +1,12 @@
import { Router, type Request, type Response } from 'express';
import { authMiddleware } from '../middleware/auth';
import { requireAdmin, requirePaid } from '../middleware/tierGates';
import { requireAdmin } from '../middleware/tierGates';
import { rejectApiTokenScope } from '../middleware/apiTokenScope';
import { mintConsoleSession } from '../helpers/consoleSession';
import {
isConsoleSessionPath,
mintConsoleSession,
sanitizeActingAs,
} from '../helpers/consoleSession';
/**
* Mint a short-lived `console_session` JWT. Used by the gateway when it
@@ -11,15 +15,28 @@ import { mintConsoleSession } from '../helpers/consoleSession';
* remote's upgrade handler on interactive paths, so the gateway authenticates
* with the long-lived token, asks for this short-lived one, then forwards
* the WS upgrade using it.
*
* Body:
* - `path` (required): `host-console` | `container-exec`
* - `acting_as` (optional): hub operator for remote audit. On node_proxy mints
* the body value is used (sanitized). On browser admin mints the signed-in
* username is stamped so a Bearer console_session cannot erase attribution.
*/
export const consoleRouter = Router();
consoleRouter.post('/console-token', authMiddleware, (req: Request, res: Response): void => {
if (rejectApiTokenScope(req, res, 'API tokens cannot generate console tokens.')) return;
if (!requireAdmin(req, res)) return;
if (!requirePaid(req, res)) return;
try {
res.json({ token: mintConsoleSession() });
const path = req.body?.path;
if (!isConsoleSessionPath(path)) {
res.status(400).json({ error: 'Invalid or missing console path' });
return;
}
const actingAs = req.machineAuthScope === 'node_proxy'
? sanitizeActingAs(req.body?.acting_as)
: sanitizeActingAs(req.user?.username);
res.json({ token: mintConsoleSession({ path, actingAs }) });
} catch (error) {
console.error('Failed to issue console token:', error);
res.status(500).json({ error: 'Failed to issue console token' });
+39
View File
@@ -135,6 +135,40 @@ function releaseStackOpLock(req: Request, stackName: string): void {
StackOpLockService.getInstance().release(req.nodeId, stackName);
}
/** Root compose + blueprint marker on the stack-source root must not change while a lifecycle op holds the stack lock. */
const STACK_OP_LOCKED_ROOT_TRUST_FILES = new Set([
'compose.yaml',
'compose.yml',
'docker-compose.yaml',
'docker-compose.yml',
'.blueprint.json',
]);
function rejectIfStackOpBlocksRootTrustFileWrite(
req: Request,
res: Response,
stackName: string,
relPath: string,
root: StackFileRoot,
): boolean {
if (root.kind !== 'stack-source') return false;
if (relPath.includes('/')) return false;
const base = relPath.toLowerCase();
if (!STACK_OP_LOCKED_ROOT_TRUST_FILES.has(base)) return false;
const existing = StackOpLockService.getInstance().get(req.nodeId, stackName);
if (!existing) return false;
res.status(409).json({
error: `${stackName} is busy: another operation (${existing.action}) is already in progress`,
code: 'stack_op_in_progress',
inProgress: {
action: existing.action,
startedAt: existing.startedAt,
user: existing.user,
},
});
return true;
}
function stackFileEtag(mtimeMs: number): string {
return `W/"${Math.floor(mtimeMs)}"`;
}
@@ -2884,6 +2918,10 @@ stacksRouter.post(
return res.status(400).json({ error: 'Invalid filename' });
}
const targetRelPath = relPath ? `${relPath}/${originalName}` : originalName;
if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, targetRelPath, root)) {
await cleanupUploadTemp(req);
return;
}
const overwrite = String(req.query.overwrite) === '1';
// The multer wrapper stashed the route-entry timestamp on the request so
// the success path and the rejection paths share one window. Fall back to
@@ -2987,6 +3025,7 @@ stacksRouter.put('/:stackName/files/content', async (req: Request, res: Response
const expectedVersion = req.header('if-match') || undefined;
const root = await resolveRootForOp(req, res, stackName, 'write');
if (!root) return;
if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, relPath, root)) return;
const startedAt = Date.now();
logFileDiag('write start', { stackName, relPath, nodeId: req.nodeId, bytes: Buffer.byteLength(content, 'utf-8'), hasIfMatch: expectedVersion !== undefined, rootKind: root.kind });
try {
+244 -181
View File
@@ -19,8 +19,12 @@ import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlo
import { enforcePolicyForImageRefs } from './PolicyEnforcement';
import { BlueprintAnalyzer } from './BlueprintAnalyzer';
import { sanitizeForLog } from '../utils/safeLog';
const MARKER_FILENAME = '.blueprint.json';
import { isPathWithinBase } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
type BlueprintMarker,
} from '../helpers/blueprintMarker';
/** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */
const COMPOSE_FILENAME = 'compose.yaml';
const REMOTE_HTTP_TIMEOUT_MS = 30_000;
@@ -41,10 +45,32 @@ function diagnosticLog(message: string, fields: Record<string, string | number |
console.info(`[BlueprintService:diag] ${message}`, safeFields);
}
export interface BlueprintMarker {
blueprintId: number;
revision: number;
lastApplied: number;
export type { BlueprintMarker };
export class BlueprintNameConflictError extends Error {
readonly code = 'name_conflict' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintNameConflictError';
}
}
/** Thrown when a remote node lacks the atomic apply/withdraw endpoints. */
export class BlueprintRemoteUpgradeRequiredError extends Error {
readonly code = 'remote_upgrade_required' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintRemoteUpgradeRequiredError';
}
}
/** Thrown when ownership cannot be verified (non-ENOENT I/O or remote probe failure). */
export class BlueprintOwnershipProbeError extends Error {
readonly code = 'ownership_probe_failed' as const;
constructor(message: string) {
super(message);
this.name = 'BlueprintOwnershipProbeError';
}
}
export interface DeployOutcome {
@@ -52,11 +78,17 @@ export interface DeployOutcome {
error?: string;
}
type LocalMarkerRead =
| { kind: 'missing' }
| { kind: 'present'; marker: BlueprintMarker }
| { kind: 'failed'; error: string };
/**
* BlueprintService is the orchestration layer between the reconciler and the
* concrete deploy/withdraw primitives. It owns:
* - per-target marker-file management (writes, reads, validates ownership)
* - name-conflict guard (refuses to touch a stack directory missing the marker)
* - name-conflict guard (refuses apply/withdraw when the directory lacks a matching
* `.blueprint.json` for this blueprint ID)
* - local deploy via ComposeService + FileSystemService
* - remote deploy via direct HTTP calls to the remote Sencho instance
* - per-(blueprint,node) concurrency lock so overlapping ticks don't collide
@@ -128,15 +160,12 @@ export class BlueprintService {
async readMarker(blueprintName: string, node: Node): Promise<BlueprintMarker | null> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME);
if (!markerPath.startsWith(path.resolve(baseDir))) return null;
const content = await fsPromises.readFile(markerPath, 'utf-8');
return BlueprintService.parseMarker(content);
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
return markerRead.kind === 'present' ? markerRead.marker : null;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return null;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`;
const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`;
const res = await axios.get(url, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
@@ -146,7 +175,7 @@ export class BlueprintService {
const body = res.data;
const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null);
if (content == null) return null;
return BlueprintService.parseMarker(content);
return parseBlueprintMarker(content);
} catch {
return null;
}
@@ -154,47 +183,77 @@ export class BlueprintService {
/**
* Returns true when a stack directory by this name exists on the target
* node but does not carry our marker file. The reconciler must not
* deploy in that case: there is a real user-authored stack with the
* same name and we must not overwrite it.
* node and the on-disk marker is missing, malformed, or references a
* different blueprint ID. Throws BlueprintOwnershipProbeError when the
* directory or marker cannot be probed (non-ENOENT I/O or remote list failure).
*/
async hasNameConflict(blueprintName: string, node: Node): Promise<boolean> {
try {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch {
return false; // directory doesn't exist → no conflict
}
const markerPath = path.join(stackDir, MARKER_FILENAME);
try {
await fsPromises.stat(markerPath);
return false; // marker present → ours
} catch {
return true; // directory exists but no marker → conflict
}
async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise<boolean> {
if (node.type === 'local') {
const baseDir = NodeRegistry.getInstance().getComposeDir(node.id);
const stackDir = path.resolve(baseDir, blueprintName);
if (!isPathWithinBase(stackDir, baseDir)) return true;
try {
const stat = await fsPromises.stat(stackDir);
if (!stat.isDirectory()) return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err));
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) return false;
const baseUrl = target.apiUrl.replace(/\/$/, '');
const listUrl = `${baseUrl}/api/stacks`;
const listRes = await axios.get(listUrl, {
const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName);
if (markerRead.kind === 'failed') {
throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error);
}
return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId;
}
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`,
);
}
const baseUrl = target.apiUrl.replace(/\/$/, '');
let listRes;
try {
listRes = await axios.get(`${baseUrl}/api/stacks`, {
headers: this.remoteHeaders(target.apiToken),
timeout: REMOTE_HTTP_TIMEOUT_MS,
validateStatus: () => true,
});
if (listRes.status !== 200) return false;
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null;
} catch {
return false;
} catch (err) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`,
);
}
if (listRes.status !== 200) {
throw new BlueprintOwnershipProbeError(
`Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`,
);
}
const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : [];
const exists = stacks.some(s => s?.name === blueprintName);
if (!exists) return false;
const marker = await this.readMarker(blueprintName, node);
return marker == null || marker.blueprintId !== blueprintId;
}
/** Read and parse a local on-disk marker without going through the remote HTTP path. */
private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise<LocalMarkerRead> {
try {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
const content = await fsPromises.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker) return { kind: 'missing' };
return { kind: 'present', marker };
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'missing' };
return { kind: 'failed', error: BlueprintService.formatError(err) };
}
}
@@ -222,7 +281,7 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'deploying');
if (await this.hasNameConflict(blueprint.name, node)) {
if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`,
});
@@ -249,6 +308,12 @@ export class BlueprintService {
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'active' };
} catch (err) {
if (err instanceof BlueprintNameConflictError) {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message });
console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s',
sanitizeForLog(blueprint.name), node.id, Date.now() - started);
return { status: 'name_conflict', error: 'name_conflict' };
}
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s',
@@ -280,20 +345,15 @@ export class BlueprintService {
});
try {
this.setStatus(blueprint.id, node.id, 'withdrawing');
// Refuse to withdraw a directory we do not own
const marker = await this.readMarker(blueprint.name, node);
if (marker && marker.blueprintId !== blueprint.id) {
this.setStatus(blueprint.id, node.id, 'name_conflict', {
last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`,
});
return { status: 'name_conflict' };
}
// Ownership is validated on the node that owns the stack, inside the delete lock.
if (node.type === 'local') {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' });
await this.withdrawLocal(blueprint, node);
const localOutcome = await this.withdrawLocal(blueprint, node);
if (localOutcome.status !== 'withdrawn') return localOutcome;
} else {
diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' });
await this.withdrawRemote(blueprint, node);
const remoteOutcome = await this.withdrawRemote(blueprint, node);
if (remoteOutcome.status !== 'withdrawn') return remoteOutcome;
}
DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id);
console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s',
@@ -382,15 +442,22 @@ export class BlueprintService {
// ---- local primitives ----
/** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */
private async stackDirExists(nodeId: number, blueprintName: string): Promise<boolean> {
const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId);
const stackDir = path.resolve(baseDir, blueprintName);
if (!stackDir.startsWith(path.resolve(baseDir))) return false;
if (!isPathWithinBase(stackDir, baseDir)) {
throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`);
}
try {
const stat = await fsPromises.stat(stackDir);
return stat.isDirectory();
} catch {
return false;
} catch (err) {
const code = (err as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return false;
throw new BlueprintOwnershipProbeError(
`Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`,
);
}
}
@@ -423,14 +490,14 @@ export class BlueprintService {
}
/**
* Create the stack if needed, write the compose and marker files, run the
* deploy policy gate, and deploy, all under the per-stack operation lock so
* none of it can race a manual deploy/update/rollback/backup on the same
* stack and node. Runs on the node that owns the stack: deployLocal calls it
* for the hub's own node, and the /api/blueprints/apply-local route calls it
* on a remote node receiving a blueprint apply from its hub (so the file
* writes hold the remote's lock, not just the deploy). On lock conflict
* nothing is written and { ran: false } is returned.
* Create the stack if needed, write the compose file, run the deploy policy
* gate and deploy, then write the marker, all under the per-stack operation
* lock. The marker is written only after a successful deploy so a failed
* apply cannot claim an applied revision that never ran. Runs on the node
* that owns the stack: deployLocal calls it for the hub's own node, and the
* /api/blueprints/apply-local route calls it on a remote node receiving a
* blueprint apply from its hub. On lock conflict nothing is written and
* { ran: false } is returned.
*/
async applyLocalUnderLock(
nodeId: number,
@@ -439,47 +506,83 @@ export class BlueprintService {
markerContent: string,
auditPath: string,
): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> {
const expected = parseBlueprintMarker(markerContent);
if (!expected) {
throw new Error('Invalid blueprint marker');
}
const fs = FileSystemService.getInstance(nodeId);
const lock = await StackOpLockService.getInstance().runExclusive(
nodeId, stackName, 'deploy', 'system',
async () => {
if (!(await this.stackDirExists(nodeId, stackName))) {
let createdStack = false;
if (await this.stackDirExists(nodeId, stackName)) {
const existing = await this.readLocalMarkerFromDisk(nodeId, stackName);
if (existing.kind === 'failed') {
throw new BlueprintOwnershipProbeError(
`Cannot verify ownership of stack "${stackName}": ${existing.error}`,
);
}
if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) {
throw new BlueprintNameConflictError(
`A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`,
);
}
} else {
await fs.createStack(stackName);
createdStack = true;
}
await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent);
await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent);
// Clear lower-priority compose siblings so discovery cannot shadow compose.yaml.
// Local + modern apply-local only; legacy remote has no sibling DELETE.
await fs.removeAlternateRootComposeFiles(stackName);
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
try {
await assertPolicyGateAllows(
stackName,
nodeId,
buildSystemPolicyGateOptions('blueprint', { auditPath }),
);
await ComposeService.getInstance(nodeId).deployStack(
stackName,
undefined,
false,
{ source: 'blueprint', actor: 'system:blueprint' },
);
await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent);
} catch (err) {
if (createdStack) {
try {
await fs.deleteStack(stackName);
} catch (cleanupErr) {
console.warn(
'[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s',
sanitizeForLog(stackName),
sanitizeForLog(BlueprintService.formatError(cleanupErr)),
);
}
}
throw err;
}
},
);
return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action };
}
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<void> {
private async withdrawLocal(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({
nodeId: node.id,
stackName: blueprint.name,
pruneVolumes: false,
actor: 'system:blueprint',
requireBlueprintId: blueprint.id,
});
if (!result.ok) {
if (result.code === 'lock_conflict') {
throw new Error(result.error);
}
throw new Error(result.error);
if (result.ok) {
return { status: 'withdrawn' };
}
if (result.code === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error });
return { status: 'name_conflict' };
}
this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error });
return { status: 'failed', error: result.error };
}
// ---- remote primitives ----
@@ -500,10 +603,7 @@ export class BlueprintService {
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// Atomic apply: the remote runs create + write compose/marker + deploy
// under its own per-stack lock, so the file writes cannot race a manual
// operation on that node. Older nodes without this route answer 404; we
// fall back to the legacy multi-call flow there (not lock-atomic).
// Atomic apply: the remote validates ownership and writes under its stack lock.
const res = await axios.post(
`${baseUrl}/api/blueprints/apply-local`,
{
@@ -514,11 +614,17 @@ export class BlueprintService {
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status === 404) {
console.warn(`[BlueprintService] remote node ${node.id} lacks /api/blueprints/apply-local; using legacy non-atomic apply`);
await this.deployRemoteLegacy(blueprint, node, marker);
return;
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`,
);
}
if (res.status === 409) {
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
throw new BlueprintNameConflictError(
BlueprintService.extractApiError(res.data)
|| `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`,
);
}
throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`);
}
if (res.status >= 400) {
@@ -526,105 +632,62 @@ export class BlueprintService {
}
}
/**
* Legacy remote apply for nodes that predate /api/blueprints/apply-local:
* create, write compose, write marker, deploy as separate calls. The remote
* deploy locks, but the preceding file writes do not, so this is not atomic
* against a concurrent manual operation on that node. Kept only as a
* compatibility fallback.
*/
private async deployRemoteLegacy(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise<void> {
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<DeployOutcome> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success.
const createRes = await axios.post(`${baseUrl}/api/stacks`,
{ stackName: blueprint.name },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (createRes.status >= 400 && createRes.status !== 409) {
throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`);
}
// 2. Write the compose file
await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content);
// 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state)
await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2));
// 4. Deploy
const deployRes = await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`,
{},
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (deployRes.status >= 400) {
throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`);
}
}
private async withdrawRemote(blueprint: Blueprint, node: Node): Promise<void> {
const target = NodeRegistry.getInstance().getProxyTarget(node.id);
if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`);
const baseUrl = target.apiUrl.replace(/\/$/, '');
const headers = this.remoteHeaders(target.apiToken);
// down (best-effort)
let res;
try {
await axios.post(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`,
{},
res = await axios.post(
`${baseUrl}/api/blueprints/withdraw-local`,
{ stackName: blueprint.name, blueprintId: blueprint.id },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
} catch (err) {
console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`);
const message = BlueprintService.formatError(err);
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
// delete the stack directory entirely
const delRes = await axios.delete(
`${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`,
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (delRes.status >= 400 && delRes.status !== 404) {
throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`);
if (res.status === 404) {
throw new BlueprintRemoteUpgradeRequiredError(
`Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`,
);
}
}
private async remotePutFile(
baseUrl: string,
headers: Record<string, string>,
stackName: string,
relPath: string,
content: string,
): Promise<void> {
const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`;
const res = await axios.put(url,
{ content },
{ headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true },
);
if (res.status >= 400) {
throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`);
if (res.status === 200) {
return { status: 'withdrawn' };
}
if (res.status === 409) {
const error = BlueprintService.extractApiError(res.data) || 'withdraw refused';
if (BlueprintService.extractApiCode(res.data) === 'name_conflict') {
this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error });
return { status: 'name_conflict' };
}
// stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed
this.setStatus(blueprint.id, node.id, 'failed', { last_error: error });
return { status: 'failed', error };
}
const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`;
this.setStatus(blueprint.id, node.id, 'failed', { last_error: message });
return { status: 'failed', error: message };
}
static parseMarker(content: string): BlueprintMarker | null {
try {
const parsed = JSON.parse(content);
if (parsed && typeof parsed === 'object'
&& typeof parsed.blueprintId === 'number'
&& typeof parsed.revision === 'number') {
return {
blueprintId: parsed.blueprintId,
revision: parsed.revision,
lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0,
};
}
} catch {
// fall through
}
return null;
return parseBlueprintMarker(content);
}
private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError {
return new BlueprintOwnershipProbeError(
`Cannot verify stack ownership for "${blueprintName}": ${detail}`,
);
}
static extractApiCode(body: unknown): string {
if (!body || typeof body !== 'object') return '';
const code = (body as Record<string, unknown>).code;
return typeof code === 'string' ? code : '';
}
static formatError(err: unknown): string {
@@ -38,6 +38,7 @@ export const CAPABILITIES = [
'notification-suppression',
'notification-suppression-schedule',
'host-console',
'host-console-community',
'container-exec',
'audit-log',
'scheduled-ops',
@@ -58,6 +59,7 @@ export const CAPABILITIES = [
'stack-down-remove-volumes',
'guided-external-network-preflight',
'service-scoped-update',
'service-scoped-stack-alert',
] as const;
/**
@@ -71,6 +73,12 @@ export const CROSS_NODE_RBAC_CAPABILITY = 'cross-node-rbac';
export type Capability = (typeof CAPABILITIES)[number];
/** Legacy Host Console advertisement (Admiral hubs still accept this on remotes). */
export const HOST_CONSOLE_CAPABILITY = 'host-console' as const satisfies Capability;
/** Host Console works without a paid license on this node. */
export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as const satisfies Capability;
/** Remotes that evaluate weekly maintenance windows on mute/suppression replicas. */
export const NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY =
'notification-suppression-schedule' as const satisfies Capability;
@@ -81,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes'
/** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */
export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability;
/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */
export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY =
'service-scoped-stack-alert' as const satisfies Capability;
/** Returns true when the string is a usable semver version. */
export function isValidVersion(v: string | null | undefined): v is string {
return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v);
@@ -165,6 +177,7 @@ export function getActiveCapabilities(): readonly string[] {
*/
const PILOT_DISABLED_CAPABILITIES: readonly Capability[] = [
'host-console',
'host-console-community',
];
/** Disable capabilities that require a central->pilot path that is not yet wired. */
+128 -14
View File
@@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe
export interface StackAlert {
id?: number;
stack_name: string;
service_name: string | null;
metric: string;
operator: string;
threshold: number;
@@ -250,6 +251,8 @@ export interface StackUpdateCleanupPendingRow {
rollback_tags_json: string;
override_paths_json: string;
prune_volumes_requested: number;
/** Blueprint ID that authorized this deletion; null for manual deletes. */
required_blueprint_id: number | null;
created_at: number;
updated_at: number;
}
@@ -613,6 +616,8 @@ export interface AuditLogEntry {
node_id: number | null;
ip_address: string;
summary: string;
/** Hub operator for remote console_session bridges; null/absent for direct sessions. */
acting_as?: string | null;
}
export interface SecretRow {
@@ -1060,6 +1065,7 @@ export class DatabaseService {
this.migrateStackDossierHashes();
this.migrateGitSourceMultiFile();
this.migrateNodeUpdateSkips();
this.migrateStackAlertServiceScope();
// Reset the cache once at end of constructor in case any migration
// populated it via getGlobalSettings() and a subsequent migration
@@ -1096,6 +1102,7 @@ export class DatabaseService {
CREATE TABLE IF NOT EXISTS stack_alerts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stack_name TEXT NOT NULL,
service_name TEXT,
metric TEXT NOT NULL,
operator TEXT NOT NULL,
threshold REAL NOT NULL,
@@ -1104,6 +1111,17 @@ export class DatabaseService {
last_fired_at INTEGER DEFAULT 0
);
-- FK cascade is declarative only: PRAGMA foreign_keys is never enabled
-- on this connection, so parent deletes must remove children explicitly
-- (see deleteStackAlert).
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS notification_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
node_id INTEGER NOT NULL DEFAULT 0,
@@ -1240,12 +1258,20 @@ export class DatabaseService {
status_code INTEGER NOT NULL DEFAULT 0,
node_id INTEGER,
ip_address TEXT NOT NULL DEFAULT '',
summary TEXT NOT NULL DEFAULT ''
summary TEXT NOT NULL DEFAULT '',
acting_as TEXT
);
CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp);
CREATE INDEX IF NOT EXISTS idx_audit_log_username ON audit_log(username);
CREATE TABLE IF NOT EXISTS console_session_jtis (
jti TEXT PRIMARY KEY,
used_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at);
CREATE TABLE IF NOT EXISTS api_tokens (
id INTEGER PRIMARY KEY AUTOINCREMENT,
token_hash TEXT NOT NULL UNIQUE,
@@ -1788,7 +1814,23 @@ export class DatabaseService {
try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ }
};
// Remote Host Console bridges record the hub operator separately from
// the console_session principal (username stays console_session).
maybeAddCol('audit_log', 'acting_as', 'TEXT');
// Cached INSERT may predate the column; rebuild on next flush.
this.auditLogInsertStmt = null;
this.db.exec(`
CREATE TABLE IF NOT EXISTS console_session_jtis (
jti TEXT PRIMARY KEY,
used_at INTEGER NOT NULL,
expires_at INTEGER NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at);
`);
maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0');
maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER');
// Distributed API model columns
maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''");
@@ -2255,6 +2297,25 @@ export class DatabaseService {
}
}
private migrateStackAlertServiceScope(): void {
this.tryAddColumn('stack_alerts', 'service_name', 'TEXT');
try {
// FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children.
this.db.prepare(`
CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns (
alert_id INTEGER NOT NULL,
service_name TEXT NOT NULL,
last_fired_at INTEGER NOT NULL,
PRIMARY KEY (alert_id, service_name),
FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE
)
`).run();
} catch (e) {
console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message);
throw e;
}
}
private migrateScanPolicyFleetColumns(): void {
this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''");
this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0');
@@ -3050,22 +3111,19 @@ export class DatabaseService {
// --- Stack Alerts ---
public getStackAlerts(stackName?: string): StackAlert[] {
let stmt;
if (stackName) {
stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?');
return stmt.all(stackName) as StackAlert[];
} else {
stmt = this.db.prepare('SELECT * FROM stack_alerts');
return stmt.all() as StackAlert[];
return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[];
}
return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[];
}
public addStackAlert(alert: StackAlert): StackAlert {
const stmt = this.db.prepare(
'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)'
'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
);
const result = stmt.run(
alert.stack_name,
alert.service_name ?? null,
alert.metric,
alert.operator,
alert.threshold,
@@ -3076,9 +3134,16 @@ export class DatabaseService {
return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert;
}
/**
* Delete an alert and its per-service cooldown rows.
* SQLite foreign_keys is not enabled here, so child rows are removed
* explicitly rather than relying on ON DELETE CASCADE.
*/
public deleteStackAlert(id: number): void {
const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?');
stmt.run(id);
this.transaction(() => {
this.deleteStackAlertServiceCooldowns(id);
this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id);
});
}
public updateStackAlertLastFired(id: number, timestamp: number): void {
@@ -3086,6 +3151,32 @@ export class DatabaseService {
stmt.run(timestamp, id);
}
public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null {
const row = this.db.prepare(
'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?'
).get(alertId, serviceName) as { last_fired_at: number } | undefined;
return row?.last_fired_at ?? null;
}
public hasAnyStackAlertServiceCooldown(alertId: number): boolean {
const row = this.db.prepare(
'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1'
).get(alertId) as { present: number } | undefined;
return !!row;
}
public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void {
this.db.prepare(`
INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at)
VALUES (?, ?, ?)
ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at
`).run(alertId, serviceName, timestamp);
}
public deleteStackAlertServiceCooldowns(alertId: number): void {
this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId);
}
// --- Auto-Heal Policies ---
public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] {
@@ -3845,12 +3936,12 @@ export class DatabaseService {
this.db.prepare(
`INSERT INTO stack_update_cleanup_pending (
id, node_id, stack_name, status, target_kind, rollback_tags_json,
override_paths_json, prune_volumes_requested, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
override_paths_json, prune_volumes_requested, required_blueprint_id, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
).run(
row.id, row.node_id, row.stack_name, row.status, row.target_kind,
row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested,
row.created_at, row.updated_at,
row.required_blueprint_id, row.created_at, row.updated_at,
);
}
@@ -4046,6 +4137,12 @@ export class DatabaseService {
stmt.run(nodeId);
}
/** Purge stack-scoped notification history when a stack is deleted. */
public deleteNotificationsForStack(nodeId: number, stackName: string): number {
const stmt = this.db.prepare('DELETE FROM notification_history WHERE node_id = ? AND stack_name = ?');
return stmt.run(nodeId, stackName).changes;
}
public updateNotificationDispatchError(id: number, error: string): void {
this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id);
}
@@ -4315,6 +4412,7 @@ export class DatabaseService {
rollback_tags_json: JSON.stringify(localCleanup.tags),
override_paths_json: JSON.stringify(localCleanup.overridePaths),
prune_volumes_requested: 0,
required_blueprint_id: null,
created_at: now,
updated_at: now,
});
@@ -4456,6 +4554,21 @@ export class DatabaseService {
this.db.prepare('DELETE FROM pilot_enrollments WHERE node_id = ?').run(nodeId);
}
/**
* One-time consume for a console_session JWT id. Inserts the jti on first
* use; returns false when the jti was already recorded (replay).
*/
public consumeConsoleSessionJti(jti: string, expiresAtMs: number): boolean {
const now = Date.now();
return this.db.transaction(() => {
this.db.prepare('DELETE FROM console_session_jtis WHERE expires_at < ?').run(now);
const result = this.db.prepare(
'INSERT OR IGNORE INTO console_session_jtis (jti, used_at, expires_at) VALUES (?, ?, ?)',
).run(jti, now, expiresAtMs);
return result.changes > 0;
})();
}
// --- Stack Update Status ---
/**
@@ -5159,7 +5272,7 @@ export class DatabaseService {
this.auditLogBuffer = [];
if (!this.auditLogInsertStmt) {
this.auditLogInsertStmt = this.db.prepare(
'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary, acting_as) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)',
);
}
const stmt = this.auditLogInsertStmt;
@@ -5174,6 +5287,7 @@ export class DatabaseService {
entry.node_id,
entry.ip_address,
entry.summary,
entry.acting_as ?? null,
);
}
});
@@ -18,10 +18,15 @@ import DockerController from './DockerController';
import { FileSystemService } from './FileSystemService';
import { NodeRegistry } from './NodeRegistry';
import { MeshService } from './MeshService';
import { NotificationService } from './NotificationService';
import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService';
import { getErrorMessage } from '../utils/errors';
import { sanitizeForLog } from '../utils/safeLog';
import { isPathWithinBase, isValidStackName } from '../utils/validation';
import {
BLUEPRINT_MARKER_FILENAME,
parseBlueprintMarker,
} from '../helpers/blueprintMarker';
/**
* Directory that may contain recovery override files for a tombstone sweep.
@@ -45,13 +50,30 @@ export interface DeleteDeployedStackInput {
stackName: string;
pruneVolumes: boolean;
actor: string;
/** When set, deletion requires an on-disk .blueprint.json matching this blueprint ID. */
requireBlueprintId?: number;
/** When true, skip acquiring a new lock (caller already holds delete via continuation). */
continuationIntentId?: string;
}
export type DeleteDeployedStackResult =
| { ok: true }
| { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string };
| { ok: true; status: 'deleted' | 'already_absent' }
| {
ok: false;
code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed' | 'name_conflict' | 'failed';
error: string;
existingAction?: string;
};
type DirProbe = { kind: 'absent' } | { kind: 'present' } | { kind: 'error'; error: string };
type MarkerProbe =
| { kind: 'match' }
| { kind: 'name_conflict'; error: string }
| { kind: 'failed'; error: string };
function blueprintMarkerMismatchError(stackName: string): string {
return `Stack "${stackName}" exists without a matching blueprint marker; refusing to withdraw.`;
}
function collectArtifactsFromGenerations(
generations: Array<{ override_path: string | null; services_json: string }>,
@@ -99,6 +121,51 @@ function parseJsonStringArray(raw: string): string[] {
}
}
async function probeStackDirectory(nodeId: number, stackName: string): Promise<DirProbe> {
// Canonical js/path-injection barrier inline with the stat sink.
const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir());
const safePath = path.resolve(baseResolved, stackName);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'error', error: 'Invalid stack path' };
}
try {
const stat = await fs.stat(safePath);
return stat.isDirectory() ? { kind: 'present' } : { kind: 'absent' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') return { kind: 'absent' };
return { kind: 'error', error: getErrorMessage(error, 'Failed to access stack directory') };
}
}
async function probeBlueprintMarkerOwnership(
nodeId: number,
stackName: string,
requireBlueprintId: number,
): Promise<MarkerProbe> {
// Canonical js/path-injection barrier inline with the read sink.
const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId));
const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME);
if (!safePath.startsWith(baseResolved + path.sep)) {
return { kind: 'failed', error: 'Invalid stack path for blueprint marker' };
}
try {
const content = await fs.readFile(safePath, 'utf-8');
const marker = parseBlueprintMarker(content);
if (!marker || marker.blueprintId !== requireBlueprintId) {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'match' };
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code === 'ENOENT') {
return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) };
}
return { kind: 'failed', error: getErrorMessage(error, 'Failed to read blueprint marker') };
}
}
export class DeployedStackDeletionService {
private static instance: DeployedStackDeletionService;
@@ -163,6 +230,42 @@ export class DeployedStackDeletionService {
const { nodeId, stackName, pruneVolumes } = input;
const db = DatabaseService.getInstance();
// Continuation loads ownership from the persisted intent; first call uses input.
let requiredBlueprintId: number | null =
typeof input.requireBlueprintId === 'number' ? input.requireBlueprintId : null;
if (existingIntentId) {
const existing = db.getDeletionIntentById(existingIntentId);
if (!existing || existing.status !== 'prepared') {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
if (existing.required_blueprint_id != null) {
requiredBlueprintId = existing.required_blueprint_id;
}
}
let skipPhysical = false;
if (requiredBlueprintId != null) {
const dirProbe = await probeStackDirectory(nodeId, stackName);
if (dirProbe.kind === 'error') {
return { ok: false, code: 'failed', error: dirProbe.error };
}
if (dirProbe.kind === 'absent') {
skipPhysical = true;
} else {
const ownership = await probeBlueprintMarkerOwnership(nodeId, stackName, requiredBlueprintId);
if (ownership.kind === 'failed') {
return { ok: false, code: 'failed', error: ownership.error };
}
if (ownership.kind === 'name_conflict') {
if (existingIntentId) {
db.updateCleanupPendingStatus(existingIntentId, 'cancelled');
}
return { ok: false, code: 'name_conflict', error: ownership.error };
}
}
}
let intentId = existingIntentId;
if (!intentId) {
const { tags, overridePaths } = collectArtifacts(nodeId, stackName);
@@ -176,6 +279,7 @@ export class DeployedStackDeletionService {
rollback_tags_json: JSON.stringify(tags),
override_paths_json: JSON.stringify(overridePaths),
prune_volumes_requested: pruneVolumes ? 1 : 0,
required_blueprint_id: requiredBlueprintId,
created_at: now,
updated_at: now,
};
@@ -196,38 +300,53 @@ export class DeployedStackDeletionService {
return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' };
}
try {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
if (!skipPhysical) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
await ComposeService.getInstance(nodeId).downStack(stackName);
} catch (downErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
'[DeployedStackDeletion] Compose down failed or no-op for %s:',
sanitizeForLog(stackName),
pruneErr,
downErr,
);
}
if (intent.prune_volumes_requested === 1) {
try {
await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]);
} catch (pruneErr) {
console.warn(
'[DeployedStackDeletion] Volume prune failed for %s, continuing delete:',
sanitizeForLog(stackName),
pruneErr,
);
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
}
try {
await FileSystemService.getInstance(nodeId).deleteStack(stackName);
} catch (fsErr) {
db.updateCleanupPendingStatus(intentId, 'cancelled');
return {
ok: false,
code: 'fs_failed',
error: getErrorMessage(fsErr, 'Failed to remove stack files'),
};
}
const finalized = await this.finalizeLogicalDeletion(input, intentId);
if (!finalized.ok) return finalized;
return { ok: true, status: skipPhysical ? 'already_absent' : 'deleted' };
}
/** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */
private async finalizeLogicalDeletion(
input: DeleteDeployedStackInput,
intentId: string,
): Promise<DeleteDeployedStackResult> {
const { nodeId, stackName } = input;
const db = DatabaseService.getInstance();
if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) {
return {
@@ -248,6 +367,7 @@ export class DeployedStackDeletionService {
db.deleteStackExposure(nodeId, stackName);
db.deleteStackProjectEnvFiles(nodeId, stackName);
db.deleteStackScans(nodeId, stackName);
db.deleteNotificationsForStack(nodeId, stackName);
} catch (dbErr) {
console.error(
'[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:',
@@ -272,7 +392,15 @@ export class DeployedStackDeletionService {
}
await this.sweepReadyIntent(intentId);
return { ok: true };
NotificationService.getInstance().broadcastEvent({
type: 'state-invalidate',
scope: 'notifications',
action: 'stack-deleted',
nodeId,
stackName,
ts: Date.now(),
});
return { ok: true, status: 'deleted' };
}
/**
@@ -487,6 +615,31 @@ export class DeployedStackDeletionService {
continue;
}
if (intent.required_blueprint_id != null) {
const ownership = await probeBlueprintMarkerOwnership(
nodeId,
stackName,
intent.required_blueprint_id,
);
if (ownership.kind === 'name_conflict') {
db.updateCleanupPendingStatus(intent.id, 'cancelled');
console.warn(
'[DeployedStackDeletion] Startup cancelled blueprint deletion for %s: %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
if (ownership.kind === 'failed') {
console.warn(
'[DeployedStackDeletion] Startup ownership probe failed for %s (leaving prepared): %s',
sanitizeForLog(stackName),
sanitizeForLog(ownership.error),
);
continue;
}
}
const result = await this.deleteDeployedStack({
nodeId,
stackName,
+7
View File
@@ -210,6 +210,12 @@ export interface DependencyContainer {
/** Resolved Sencho stack, or null when the container is not Sencho-managed. */
stack: string | null;
state: string;
/**
* Exit code from list Status when a parenthesized code is present
* (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited").
* Required so Drift can fail closed on unknown codes.
*/
exitCode: number | null;
image: string;
networks: { name: string; id: string; ip: string }[];
/** Named-volume sources mounted by the container (bind mounts excluded). */
@@ -1833,6 +1839,7 @@ class DockerController {
composeProject: c.Labels?.['com.docker.compose.project'] ?? null,
stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase),
state: c.State ?? 'unknown',
exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined),
image: c.Image ?? '',
networks,
volumes,
+223 -44
View File
@@ -45,13 +45,21 @@ export interface ContainerHealthSnapshot {
/** Grace window after a `die` before classifying, to absorb out-of-order kill events. */
const DIE_GRACE_WINDOW_MS = 500;
/** Max crash alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */
/** Max crash/health alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */
const RATE_LIMIT_MAX = 20;
const RATE_WINDOW_MS = 60_000;
/** Dedup window for repeat crash alerts of the same container. */
const CRASH_DEDUP_MS = 60 * 60_000;
/**
* Dedup window for repeat health alerts of the same container. Same duration as
* crash dedup, but stored outside containerState so the 10-minute prune cannot
* shrink the advertised 60-minute window. Unlike crash dedup, recovery (start /
* healthy) does not clear this stamp.
*/
const HEALTH_DEDUP_MS = CRASH_DEDUP_MS;
/** Interval for pruning stale container state from memory. */
const PRUNE_INTERVAL_MS = 60_000;
const STATE_STALE_AFTER_MS = 10 * 60_000;
@@ -109,6 +117,11 @@ interface DockerEventPayload {
type LifecycleStatus = 'disconnected' | 'connecting' | 'connected' | 'stopped';
/** Rate-limit token bound to the fixed window that issued it. */
interface RateToken {
readonly windowStart: number;
}
export class DockerEventService {
private readonly nodeId: number;
private readonly nodeName: string;
@@ -127,12 +140,35 @@ export class DockerEventService {
/** Pending die timers keyed by container ID (for the 500ms grace window). */
private pendingDieTimers: Map<string, NodeJS.Timeout> = new Map();
/** Rate-limiter bookkeeping. */
private rateWindowStart = 0;
private rateCount = 0;
private suppressedCount = 0;
/**
* Shared fixed-window rate limiter for crash and health alerts (per local
* node / this service instance). Dedup and transition checks run before
* consuming a token so suppressed duplicates do not count toward the cap.
* Tokens carry the issuing window start so a late refund cannot inflate a
* newer window's budget.
*/
private containerAlertRateWindowStart = 0;
private containerAlertRateCount = 0;
private suppressedCrashAlertCount = 0;
private suppressedHealthAlertCount = 0;
private summaryTimer: NodeJS.Timeout | null = null;
/**
* Process-local health-alert dedup keyed by container id. Kept outside
* containerState so pruneStaleState cannot drop an active 60m window.
* Entries expire after HEALTH_DEDUP_MS via pruneStaleState; the map clears
* on process restart. Not cleared on healthy/start recovery.
*/
private healthAlertDedupAt = new Map<string, number>();
/** In-flight health dispatches; prevents concurrent duplicate flaps. */
private healthAlertInFlight = new Set<string>();
/**
* Container ids that entered unhealthy again while a health dispatch was
* in flight. Retried once after the in-flight call finishes if history
* did not persist (so a failed write cannot permanently silence the alert).
*/
private healthAlertPendingRetry = new Set<string>();
/** Parse-error tracking for flooded bad payloads. */
private parseErrorWindowStart = 0;
private parseErrorCount = 0;
@@ -166,6 +202,8 @@ export class DockerEventService {
/** Close the stream, cancel timers, and clear state. */
public shutdown(): void {
// Flush overflow roll-up while still live so pending summaries are not dropped.
this.flushRateLimitSummaryNow();
this.status = 'stopped';
this.clearReconnectTimer();
if (this.pruneTimer) {
@@ -180,6 +218,17 @@ export class DockerEventService {
this.pendingDieTimers.clear();
this.detachStream();
this.containerState.clear();
this.healthAlertDedupAt.clear();
this.healthAlertInFlight.clear();
this.healthAlertPendingRetry.clear();
this.suppressedCrashAlertCount = 0;
this.suppressedHealthAlertCount = 0;
this.containerAlertRateCount = 0;
}
/** Status helper so await-crossing stopped checks are not narrowed away. */
private isStopped(): boolean {
return this.status === 'stopped';
}
// ========================================================================
@@ -460,27 +509,104 @@ export class DockerEventService {
state.lastActivityAt = Date.now();
if (action.includes('unhealthy')) {
if (state.healthStatus !== 'unhealthy') {
const enteringUnhealthy = state.healthStatus !== 'unhealthy';
if (enteringUnhealthy) {
state.unhealthySince = Date.now();
}
// Auto-Heal reads healthStatus / unhealthySince; update before any alert gate.
state.healthStatus = 'unhealthy';
if (!enteringUnhealthy) return;
if (!this.isCrashAlertsEnabled()) return;
void this.dispatchHealthAlert(id, state);
} else {
// Recovery clears Auto-Heal timing but must not erase health-alert dedup,
// or a healthy↔unhealthy flap would re-alert immediately.
state.unhealthySince = undefined;
state.healthStatus = action.includes('starting') ? 'starting' : 'healthy';
// Drop deferred-retry markers so a later unrelated persist failure cannot
// spuriously retry from a flap that already recovered.
this.healthAlertPendingRetry.delete(id);
}
}
/**
* Dispatch a healthcheck failure alert with transition already confirmed.
* Dedup and rate checks run before the token is consumed. Advance dedup
* only when dispatchAlert returns `{ persisted: true }` (history row written).
*/
private async dispatchHealthAlert(id: string, state: InternalContainerState): Promise<void> {
if (this.isStopped()) return;
const now = Date.now();
if (this.isWithinAlertDedupWindow(this.healthAlertDedupAt.get(id), now, HEALTH_DEDUP_MS)) {
return;
}
if (this.healthAlertInFlight.has(id)) {
this.healthAlertPendingRetry.add(id);
return;
}
const token = this.consumeRateToken();
if (!token) {
this.suppressedHealthAlertCount += 1;
this.scheduleSummary();
return;
}
this.healthAlertInFlight.add(id);
let persisted = false;
try {
if (this.isStopped()) {
this.refundRateToken(token);
this.healthAlertPendingRetry.delete(id);
return;
}
const name = state.name ?? id.slice(0, 12);
void this.emitError(
const result = await this.emitError(
'monitor_alert',
`Healthcheck failed: ${name} is unhealthy.`,
state.stackName,
state.name,
state.isSelf === true,
);
} else {
state.unhealthySince = undefined;
if (action.includes('starting')) {
state.healthStatus = 'starting';
persisted = result.persisted;
if (persisted) {
this.healthAlertDedupAt.set(id, Date.now());
this.healthAlertPendingRetry.delete(id);
} else {
state.healthStatus = 'healthy';
// Refund only into the window that issued the token so a late
// persist failure cannot inflate a newer window's budget.
this.refundRateToken(token);
console.error(
`[DockerEvents:${this.nodeName}] Health alert history not persisted for ${id}; dedup not advanced`,
);
}
} finally {
this.healthAlertInFlight.delete(id);
}
if (this.isStopped()) {
this.healthAlertPendingRetry.delete(id);
return;
}
// A concurrent healthy→unhealthy flap was deferred while we were in
// flight. If the first write failed, retry so the alert is not lost
// while the container stays unhealthy with no further transitions.
if (this.shouldRetryHealthAlert(id, state, persisted)) {
this.healthAlertPendingRetry.delete(id);
await this.dispatchHealthAlert(id, state);
}
}
private shouldRetryHealthAlert(
id: string,
state: InternalContainerState,
persisted: boolean,
): boolean {
return !this.isStopped()
&& !persisted
&& this.healthAlertPendingRetry.has(id)
&& state.healthStatus === 'unhealthy';
}
private onStart(id: string): void {
@@ -495,6 +621,7 @@ export class DockerEventService {
state.healthStatus = 'starting';
state.lastStartAt = Date.now();
state.lastActivityAt = Date.now();
this.healthAlertPendingRetry.delete(id);
}
private onDestroy(id: string): void {
@@ -504,6 +631,9 @@ export class DockerEventService {
clearTimeout(pending);
this.pendingDieTimers.delete(id);
}
this.healthAlertPendingRetry.delete(id);
this.healthAlertInFlight.delete(id);
this.healthAlertDedupAt.delete(id);
}
private async classifyDie(id: string, event: DockerEventPayload, dieAt: number): Promise<void> {
@@ -545,7 +675,7 @@ export class DockerEventService {
// Dedup early: crashloops repeatedly reach this point with exit 137,
// and the OOM fallback below issues a Docker inspect. Skipping the
// inspect on deduped crashes avoids hammering the daemon.
if (state.lastCrashAlertAt && now - state.lastCrashAlertAt < CRASH_DEDUP_MS) {
if (this.isWithinAlertDedupWindow(state.lastCrashAlertAt, now, CRASH_DEDUP_MS)) {
return;
}
@@ -584,6 +714,7 @@ export class DockerEventService {
state: InternalContainerState | null,
info: { name: string; exitCode: number; stackName?: string; isSelf?: boolean },
): Promise<void> {
if (this.isStopped()) return;
// Respect the existing global crash-alerts toggle so users who have
// disabled these notifications in Settings remain opted out.
if (!this.isCrashAlertsEnabled()) return;
@@ -592,8 +723,9 @@ export class DockerEventService {
? `Container OOM Kill: ${info.name} was killed by the OOM killer (out of memory).`
: `Container Crash Detected: ${info.name} exited unexpectedly (Code: ${info.exitCode}).`;
if (!this.consumeRateToken()) {
this.suppressedCount += 1;
const token = this.consumeRateToken();
if (!token) {
this.suppressedCrashAlertCount += 1;
this.scheduleSummary();
return;
}
@@ -625,32 +757,70 @@ export class DockerEventService {
return value;
}
/** Return true if an alert can be emitted now. Side effect: increments counters. */
private consumeRateToken(): boolean {
/**
* Consume one shared crash/health rate token for the current fixed window.
* Returns null when the window is exhausted. Rolling into a new window first
* flushes any pending overflow summary so counters cannot span windows.
*/
private consumeRateToken(): RateToken | null {
const now = Date.now();
if (now - this.rateWindowStart >= RATE_WINDOW_MS) {
this.rateWindowStart = now;
this.rateCount = 0;
if (now - this.containerAlertRateWindowStart >= RATE_WINDOW_MS) {
this.flushRateLimitSummaryNow();
this.containerAlertRateWindowStart = now;
this.containerAlertRateCount = 0;
}
if (this.rateCount >= RATE_LIMIT_MAX) return false;
this.rateCount += 1;
return true;
if (this.containerAlertRateCount >= RATE_LIMIT_MAX) return null;
this.containerAlertRateCount += 1;
return { windowStart: this.containerAlertRateWindowStart };
}
/** Refund a token only when the issuing window is still active. */
private refundRateToken(token: RateToken): void {
if (token.windowStart !== this.containerAlertRateWindowStart) return;
if (this.containerAlertRateCount > 0) this.containerAlertRateCount -= 1;
}
private isWithinAlertDedupWindow(
lastAlertAt: number | undefined,
now: number,
windowMs: number,
): boolean {
return lastAlertAt !== undefined && now - lastAlertAt < windowMs;
}
private buildRateLimitSummaryMessage(crash: number, health: number): string {
const total = crash + health;
if (crash > 0 && health > 0) {
return `${total} additional container alerts were rate-limited in the last minute (${crash} crash, ${health} health).`;
}
if (health > 0) {
return `${health} additional container health alerts were rate-limited in the last minute.`;
}
return `${crash} additional container crash alerts were rate-limited in the last minute.`;
}
/** Emit and clear any pending overflow roll-up immediately (e.g. on window roll). */
private flushRateLimitSummaryNow(): void {
if (this.summaryTimer) {
clearTimeout(this.summaryTimer);
this.summaryTimer = null;
}
const crash = this.suppressedCrashAlertCount;
const health = this.suppressedHealthAlertCount;
this.suppressedCrashAlertCount = 0;
this.suppressedHealthAlertCount = 0;
if (crash + health <= 0) return;
if (this.status === 'stopped') return;
void this.emitWarning('monitor_alert', this.buildRateLimitSummaryMessage(crash, health));
}
private scheduleSummary(): void {
if (this.summaryTimer) return;
const remaining = RATE_WINDOW_MS - (Date.now() - this.rateWindowStart);
const remaining = RATE_WINDOW_MS - (Date.now() - this.containerAlertRateWindowStart);
const delay = Math.max(1_000, remaining);
this.summaryTimer = setTimeout(() => {
const count = this.suppressedCount;
this.summaryTimer = null;
this.suppressedCount = 0;
if (count > 0) {
void this.emitWarning(
'monitor_alert',
`${count} additional containers crashed in the last minute.`,
);
}
this.flushRateLimitSummaryNow();
}, delay);
}
@@ -707,15 +877,24 @@ export class DockerEventService {
}
private pruneStaleState(): void {
if (this.containerState.size === 0) return;
const cutoff = Date.now() - STATE_STALE_AFTER_MS;
for (const [id, state] of this.containerState) {
// Keep state for a container with an unresolved crash so Auto-Heal can
// still act on it past the default stale window (policy thresholds run
// up to 24h). It is cleared on `start` and removed on `destroy`.
if (state.crashedAt !== undefined) continue;
if (state.lastActivityAt < cutoff) {
this.containerState.delete(id);
if (this.containerState.size > 0) {
const cutoff = Date.now() - STATE_STALE_AFTER_MS;
for (const [id, state] of this.containerState) {
// Keep state for a container with an unresolved crash so Auto-Heal can
// still act on it past the default stale window (policy thresholds run
// up to 24h). It is cleared on `start` and removed on `destroy`.
if (state.crashedAt !== undefined) continue;
if (state.lastActivityAt < cutoff) {
this.containerState.delete(id);
}
}
}
// Health dedup lives outside containerState so the 60m window survives
// ordinary 10m prune. Drop only entries whose window has fully expired.
if (this.healthAlertDedupAt.size > 0) {
const dedupCutoff = Date.now() - HEALTH_DEDUP_MS;
for (const [id, at] of this.healthAlertDedupAt) {
if (at < dedupCutoff) this.healthAlertDedupAt.delete(id);
}
}
}
@@ -734,15 +913,15 @@ export class DockerEventService {
: { stackName, containerName, actor: 'system:docker-events' };
}
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<void> {
private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<{ persisted: boolean }> {
return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly));
}
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> {
return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName));
}
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<void> {
private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> {
return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName));
}
+41 -25
View File
@@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel';
import { compareStackNetworks, fromDeclaredCompose } from './network/normalize';
import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
const MAX_RENDER_ERROR = 600;
@@ -178,16 +179,17 @@ function networkDriftFindings(
}
/**
* Pure diff step (no Docker / FS access) so it is directly unit-testable. Only
* running containers are compared, since a stopped container publishes no ports
* and is not "deployed": a declared service with no running container is
* service-missing, a running container with no matching service is
* service-undeclared, and image / port differences are checked only for services
* present on both sides so a missing/undeclared service is not double-reported.
* Pure diff step (no Docker / FS access) so it is directly unit-testable.
* Running/restarting containers drive image/port comparison. Clean one-shot
* completions (exit 0 + explicit declared restart "no", including normalized
* `deploy.restart_policy.condition: none`) satisfy service presence without
* counting as hasContainers. Omitting restart does not qualify. Network
* comparison still uses only running/restarting attachments.
*/
export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport {
const { stack, declared, containers, parseError } = input;
const networks = input.networks ?? [];
// Public contract: at least one running/restarting container (not "satisfied").
const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state));
// A parse failure means the declared model is untrustworthy: report drift
@@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError };
}
const runtimeByService = new Map<string, RuntimeService>();
for (const c of containers) {
if (!RUNNING_STATES.has(c.state)) continue;
const name = c.service ?? c.name;
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
if (c.image) agg.images.add(normalizeImageRef(c.image));
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
runtimeByService.set(name, agg);
}
// Nothing running: the stack is defined on disk but not deployed. One status
// conveys this; per-service findings would just be noise.
if (!hasContainers) {
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
}
const declaredByName = new Map<string, DeclaredService>();
for (const svc of declared.services) declaredByName.set(svc.name, svc);
const runtimeByService = new Map<string, RuntimeService>();
const oneShotSatisfied = new Set<string>();
for (const c of containers) {
const name = c.service ?? c.name;
if (RUNNING_STATES.has(c.state)) {
const agg = runtimeByService.get(name) ?? { images: new Set<string>(), ports: new Set<string>() };
if (c.image) agg.images.add(normalizeImageRef(c.image));
for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol));
runtimeByService.set(name, agg);
continue;
}
const declaredRestart = declaredByName.get(name)?.restart;
if (isCleanOneShotCompletion({
state: c.state,
exitCode: c.exitCode,
restartPolicy: declaredRestart,
})) {
oneShotSatisfied.add(name);
}
}
const servicePresent = (serviceName: string): boolean =>
runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName);
// Nothing running and no declared service satisfied by a clean one-shot: the
// stack is defined on disk but not deployed. One status conveys this.
if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) {
return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] };
}
const findings: StackDriftFinding[] = [];
// Declared service with no running container.
// Declared service with no running container and no clean one-shot completion.
for (const svc of declared.services) {
if (!runtimeByService.has(svc.name)) {
if (!servicePresent(svc.name)) {
findings.push({
kind: 'service-missing',
service: svc.name,
@@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe
}
}
// Image / port divergence for services present on both sides.
// Image / port divergence for services present on both sides (running only).
for (const [name, svc] of declaredByName) {
const runtime = runtimeByService.get(name);
if (!runtime) continue;
+10 -3
View File
@@ -58,6 +58,13 @@ const PROTECTED_STACK_FILES = new Set([
'.env',
]);
// Explorer-only protection: includes the blueprint ownership marker without
// putting it in PROTECTED_STACK_FILES (backup/rollback orphan removal).
const EXPLORER_PROTECTED_STACK_FILES = new Set([
...PROTECTED_STACK_FILES,
'.blueprint.json',
]);
// Bookkeeping markers Sencho writes into the backup slot. They are never copied
// back into the stack directory on restore: `.timestamp` records when the backup
// was taken; `.checksums` is the integrity manifest verified before a restore.
@@ -169,7 +176,7 @@ function isProtectedRelPath(relPath: string): boolean {
if (normalized.includes('/')) return false;
// Fold case so e.g. a request for COMPOSE.YAML cannot dodge the gate on a
// case-insensitive filesystem where it resolves to the real compose.yaml.
return PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
return EXPLORER_PROTECTED_STACK_FILES.has(fsCaseKey(normalized));
}
function protectedFileError(relPath: string): Error & { code: string } {
@@ -1643,7 +1650,7 @@ export class FileSystemService {
type,
size,
mtime,
isProtected: protectedEnabled && PROTECTED_STACK_FILES.has(dirent.name),
isProtected: protectedEnabled && EXPLORER_PROTECTED_STACK_FILES.has(dirent.name),
};
})
);
@@ -2096,7 +2103,7 @@ export class FileSystemService {
type,
size: stat.isDirectory() ? 0 : stat.size,
mtime: stat.mtimeMs,
isProtected: (scope?.protectedEnabled ?? true) && PROTECTED_STACK_FILES.has(name),
isProtected: (scope?.protectedEnabled ?? true) && EXPLORER_PROTECTED_STACK_FILES.has(name),
};
}
}
+112 -15
View File
@@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService';
import { sanitizeForLog } from '../utils/safeLog';
import { getErrorMessage } from '../utils/errors';
import { withTimeout } from '../utils/withTimeout';
import { isCleanOneShotCompletion } from '../utils/oneShotCompletion';
import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose';
import { parseEffectiveModel } from './preflight/effectiveModel';
import type { HealthGateContainer, HealthGateReport } from './updateGuard/types';
import { getComposeCommandTimeoutMs } from './ComposeService';
import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService';
const POLL_INTERVAL_MS = 5_000;
// A prepared-but-never-begun token (prepare called, mutation then failed before
@@ -47,6 +50,10 @@ interface ObservedContainer {
service: string | null;
/** Image id the container is running (service gates check convergence on it). */
imageId: string;
/** Inspect State.ExitCode; null when unavailable. */
exitCode: number | null;
/** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */
restartPolicy: string | null;
}
interface ActiveGate {
@@ -89,6 +96,13 @@ interface ActiveGate {
collateralBaselineByName: Map<string, ObservedContainer>;
/** Role of each expected container (service gates), for failure attribution. */
roleByName: Map<string, GateRole>;
/**
* Declared Compose restart intent by service name (normalized). Loaded once
* per gate; null until the first load attempt completes. Used for one-shot
* recognition instead of Docker inspect (which cannot distinguish omit vs
* explicit `restart: "no"`).
*/
declaredRestartByService: Map<string, string | null> | null;
}
/** A pre-mutation baseline container captured at prepare time. */
@@ -101,6 +115,8 @@ interface PreparedBaseline {
restartCount: number;
startedAt: string | null;
imageId: string;
exitCode: number | null;
restartPolicy: string | null;
}
function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean {
@@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta
health: baseline.health,
service: baseline.service,
imageId: baseline.imageId,
exitCode: baseline.exitCode,
restartPolicy: baseline.restartPolicy,
};
}
/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */
function isObservedContainerSatisfied(
gate: ActiveGate,
current: ObservedContainer | undefined,
): boolean {
if (!current) return false;
if (current.state === 'running') return true;
return isDeclaredCleanOneShot(gate, current);
}
/**
* One-shot recognition from declared Compose intent only. Unlabeled containers
* and services missing from the effective model fail closed.
*/
function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean {
const map = gate.declaredRestartByService;
if (!map || !current.service || !map.has(current.service)) return false;
return isCleanOneShotCompletion({
state: current.state,
exitCode: current.exitCode,
restartPolicy: map.get(current.service),
});
}
/** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */
interface PreparedGate {
token: string;
@@ -329,6 +371,7 @@ export class HealthGateService {
collateralEligibleNames: new Set(),
collateralBaselineByName: new Map(),
roleByName: new Map(),
declaredRestartByService: null,
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
@@ -488,6 +531,7 @@ export class HealthGateService {
prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)),
),
roleByName: new Map(),
declaredRestartByService: null,
};
this.active.set(key, gate);
this.scheduleNextPoll(gate);
@@ -587,6 +631,9 @@ export class HealthGateService {
if (gate.finalized || this.active.get(key) !== gate) return;
gate.consecutivePollErrors = 0;
await this.ensureDeclaredRestartMap(gate);
if (gate.finalized || this.active.get(key) !== gate) return;
const elapsedMs = Date.now() - gate.startedAt;
if (gate.expected === null) {
@@ -629,12 +676,16 @@ export class HealthGateService {
}
gate.missingLastPoll.delete(name);
if (current.state === 'exited' && baseline.restarts === current.restarts) {
// An exit with no restart attempt is terminal for the window.
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
// An exit with no restart attempt is terminal for the window, unless
// this is an expected one-shot (exit 0 + explicit declared restart "no").
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
this.finalize(gate, 'failed', `container ${name} exited during observation`, summary);
return;
}
if (current.health === 'unhealthy') {
// Residual Docker health on a completed one-shot is not a gate failure;
// long-running containers still fail on unhealthy.
if (!cleanOneShot && current.health === 'unhealthy') {
this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary);
return;
}
@@ -657,14 +708,19 @@ export class HealthGateService {
if (elapsedMs < gate.windowSeconds * 1000) return;
// Window complete: pass requires everything running and healthy wherever a
// healthcheck exists. A health state still 'starting' is not a pass.
const stillStarting = observed.filter(c => c.health === 'starting');
// Window complete: pass requires everything running (or a clean one-shot
// completion) and healthy wherever a healthcheck exists. A health state
// still 'starting' is not a pass (except residual health on a clean one-shot).
const stillStarting = observed.filter(
c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c),
);
if (stillStarting.length > 0) {
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
return;
}
const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running');
const notRunning = [...gate.expected.keys()].filter(
name => !isObservedContainerSatisfied(gate, byName.get(name)),
);
if (notRunning.length > 0) {
this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary);
return;
@@ -700,6 +756,9 @@ export class HealthGateService {
if (gate.finalized || this.active.get(key) !== gate) return;
gate.consecutivePollErrors = 0;
await this.ensureDeclaredRestartMap(gate);
if (gate.finalized || this.active.get(key) !== gate) return;
const elapsedMs = Date.now() - gate.startedAt;
const serviceName = gate.serviceName ?? '';
@@ -781,11 +840,12 @@ export class HealthGateService {
}
gate.missingLastPoll.delete(name);
if (current.state === 'exited' && baseline.restarts === current.restarts) {
const cleanOneShot = isDeclaredCleanOneShot(gate, current);
if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) {
this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role);
return;
}
if (current.health === 'unhealthy') {
if (!cleanOneShot && current.health === 'unhealthy') {
this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role);
return;
}
@@ -808,24 +868,28 @@ export class HealthGateService {
if (elapsedMs < gate.windowSeconds * 1000) return;
const stillStarting = observed.filter(
c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
c => c.health === 'starting'
&& !isDeclaredCleanOneShot(gate, c)
&& (c.service === serviceName || gate.collateralEligibleNames.has(c.name)),
);
if (stillStarting.length > 0) {
this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary);
return;
}
const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running');
if (runningPrimary.length !== gate.expectedReplicas) {
const satisfiedPrimary = observed.filter(
c => c.service === serviceName && isObservedContainerSatisfied(gate, c),
);
if (satisfiedPrimary.length !== gate.expectedReplicas) {
this.finalize(
gate, 'failed',
`service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`,
`service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`,
summary, 'primary',
);
return;
}
const collateralNotRunning = [...gate.expected.keys()]
.filter(name => gate.roleByName.get(name) === 'collateral')
.filter(name => byName.get(name)?.state !== 'running');
.filter(name => !isObservedContainerSatisfied(gate, byName.get(name)));
if (collateralNotRunning.length > 0) {
this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral');
return;
@@ -855,6 +919,35 @@ export class HealthGateService {
return this.listStackContainers(gate.nodeId, gate.stackName);
}
/**
* Load declared Compose restart intent once per gate. Fail closed to an empty
* map on render/parse errors so inspect "no" cannot false-qualify one-shots.
*/
private async ensureDeclaredRestartMap(gate: ActiveGate): Promise<void> {
if (gate.declaredRestartByService !== null) return;
gate.declaredRestartByService = new Map();
try {
const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName);
if (result.rendered === null) {
console.warn(
'[HealthGate] declared restart map unavailable for %s (compose render failed)',
sanitizeForLog(gate.stackName),
);
return;
}
const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName);
const declared = declaredFromEffectiveModel(model);
gate.declaredRestartByService = new Map(
declared.services.map((s) => [s.name, s.restart ?? null]),
);
} catch (error) {
console.warn(
'[HealthGate] declared restart map load failed for %s:',
sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'),
);
}
}
/** List and inspect a stack's containers into the gate's observation shape. */
private async listStackContainers(nodeId: number, stackName: string): Promise<ObservedContainer[]> {
const docker = DockerController.getInstance(nodeId).getDocker();
@@ -877,6 +970,8 @@ export class HealthGateService {
health: inspect.State?.Health?.Status ?? null,
service: labels['com.docker.compose.service'] ?? null,
imageId: inspect.Image ?? '',
exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null,
restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null,
};
} catch (e: unknown) {
// Removed between list and inspect; the missing-container logic will
@@ -903,6 +998,8 @@ export class HealthGateService {
restartCount: c.restartCount,
startedAt: c.startedAt,
imageId: c.imageId,
exitCode: c.exitCode,
restartPolicy: c.restartPolicy,
}));
}
@@ -18,6 +18,8 @@ export interface ConsoleAuditContext {
// path always supplies a concrete id.
readonly nodeId: number | null;
readonly ipAddress: string;
/** Hub operator for remote console_session bridges; null for direct sessions. */
readonly actingAs?: string | null;
}
const CONSOLE_AUDIT_PATH = '/api/system/host-console';
@@ -97,6 +99,7 @@ export class HostTerminalService {
node_id: audit.nodeId,
ip_address: audit.ipAddress,
summary,
acting_as: audit.actingAs ?? null,
});
} catch (err) {
console.error('[HostConsole] Failed to write session audit log:', err);
+120 -45
View File
@@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => {
return `triggered the operator ${operator}`;
};
/** Sentinel for containers without a Compose service label. Not a valid API target. */
const UNLABELED_SERVICE_KEY = '_unlabeled';
function displayServiceName(serviceName: string): string {
return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName;
}
function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null {
const sep = key.indexOf(':');
if (sep < 0) return null;
const ruleId = Number(key.slice(0, sep));
if (!Number.isFinite(ruleId)) return null;
return { ruleId, containerId: key.slice(sep + 1) };
}
/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */
function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number {
const ruleId = rule.id!;
let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName);
if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) {
lastFired = rule.last_fired_at || 0;
}
return lastFired || 0;
}
function normalizeContainerName(container: { Id: string; Names?: string[] }): string {
const raw = container.Names?.[0];
if (raw && raw.length > 0) {
return raw.startsWith('/') ? raw.slice(1) : raw;
}
return container.Id.slice(0, 12);
}
/** Shape of the JSON returned by Docker container stats (stream: false). */
interface DockerContainerStats {
cpu_stats?: {
@@ -153,21 +186,18 @@ export class MonitorService {
private janitorConsecutiveTimeouts = 0;
private janitorBreakerUntil = 0;
// Track the duration a specific stack alert rule has been in breach state
// key: rule_id, value: AlertState
private activeBreaches = new Map<number, AlertState>();
// Track the duration a specific stack alert rule+container has been in breach
// key: `${ruleId}:${containerId}`, value: AlertState
private activeBreaches = new Map<string, AlertState>();
// Track previous network counters per container for rate calculation.
// key: container_id, value: { rx bytes, tx bytes, sample timestamp }
private previousNetworkStats = new Map<string, { rx: number; tx: number; ts: number }>();
// Per-cycle dispatch dedup. Stack rules are shared across every container
// in the stack, so parallel processContainer calls can race past the
// cooldown check (DB write happens after the awaited dispatch) and fire
// the same alert N times on first breach. The synchronous check-and-add
// below is atomic in JS between awaits, so only the first worker wins.
// Per-cycle dispatch dedup keyed by rule+service so replicas of the same
// Compose service do not fire N times, while different services can.
// Reset at the start of each evaluateStackAlerts call.
private firedThisCycle = new Set<number>();
private firedThisCycle = new Set<string>();
// Crash and healthcheck detection live in DockerEventService (event-driven,
// causal classification). MonitorService no longer polls for container
@@ -604,6 +634,10 @@ export class MonitorService {
else alertsByStack.set(a.stack_name, [a]);
}
const activeRuleIds = new Set(alerts.map(a => a.id!));
const allCurrentIds = new Set<string>();
let enumerationFailed = false;
for (const node of nodes) {
if (!node.id) continue;
// Remote nodes are self-monitoring - skip direct Docker access
@@ -623,22 +657,39 @@ export class MonitorService {
(container) => this.processContainer(node, container, alertsByStack, docker, db),
);
for (const c of containers) allCurrentIds.add(c.Id);
// Clean up stale network stats for containers on this node that no longer run
if (this.previousNetworkStats.size > containers.length * 2) {
const currentIds = new Set(containers.map((c: { Id: string }) => c.Id));
const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id));
for (const key of this.previousNetworkStats.keys()) {
if (!currentIds.has(key)) this.previousNetworkStats.delete(key);
if (!nodeIds.has(key)) this.previousNetworkStats.delete(key);
}
}
} catch (err) {
// Enumeration failed: preserve existing breach timers.
enumerationFailed = true;
console.error(`Error fetching containers for node ${node.name}`, err);
}
}
// Clean up stale breach trackers for rules that have been deleted
const activeRuleIds = new Set(alerts.map(a => a.id!));
for (const key of this.activeBreaches.keys()) {
if (!activeRuleIds.has(key)) {
// After successful enumeration(s), drop breach timers for containers
// that are no longer running. Skip when any local enumeration failed
// so a transient Docker error cannot reset duration timers.
if (!enumerationFailed) {
for (const key of [...this.activeBreaches.keys()]) {
const parsed = parseAlertBreachKey(key);
if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) {
this.activeBreaches.delete(key);
}
}
}
// Clean up in-memory breach trackers for rules that have been deleted.
// Persisted cooldowns are removed only via transactional deleteStackAlert.
for (const key of [...this.activeBreaches.keys()]) {
const parsed = parseAlertBreachKey(key);
if (parsed && !activeRuleIds.has(parsed.ruleId)) {
this.activeBreaches.delete(key);
}
}
@@ -666,12 +717,14 @@ export class MonitorService {
*/
private async processContainer(
node: Node,
container: { Id: string; Labels?: Record<string, string> },
container: { Id: string; Names?: string[]; Labels?: Record<string, string> },
alertsByStack: Map<string, StackAlert[]>,
docker: DockerController,
db: DatabaseService,
): Promise<void> {
const stackName = container.Labels?.['com.docker.compose.project'] || 'system';
const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY;
const containerName = normalizeContainerName(container);
try {
const rawStats = await withTimeout(
@@ -727,7 +780,11 @@ export class MonitorService {
});
for (const rule of stackAlerts) {
if (rule.service_name && rule.service_name !== serviceName) continue;
const ruleId = rule.id!;
const breachKey = `${ruleId}:${container.Id}`;
const cooldownKey = `${ruleId}:${serviceName}`;
const currentValue = metrics[rule.metric as keyof typeof metrics];
if (currentValue === undefined) continue;
@@ -735,57 +792,75 @@ export class MonitorService {
const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold);
if (isBreaching) {
if (!this.activeBreaches.has(ruleId)) {
this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`);
if (!this.activeBreaches.has(breachKey)) {
this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() });
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`);
}
const breachState = this.activeBreaches.get(ruleId)!;
const breachState = this.activeBreaches.get(breachKey)!;
const durationMs = Date.now() - breachState.breachStartedAt;
const requiredDurationMs = rule.duration_mins * 60 * 1000;
if (durationMs >= requiredDurationMs) {
const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0);
const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db);
const requiredCooldownMs = rule.cooldown_mins * 60 * 1000;
if (timeSinceLastFired >= requiredCooldownMs) {
// Claim this rule for the cycle before awaiting
// dispatch. The check-and-add is synchronous, so
// sibling workers evaluating the same shared rule
// see the claim and skip — preventing N-fire when
// multiple containers in one stack all breach.
if (this.firedThisCycle.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`);
} else {
this.firedThisCycle.add(ruleId);
// Claim this rule+service for the cycle before awaiting
// dispatch so replicas of the same service skip.
if (this.firedThisCycle.has(cooldownKey)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`);
continue;
}
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
this.firedThisCycle.add(cooldownKey);
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const { name: metricName, unit } = getMetricDetails(rule.metric);
const operatorPhrase = getOperatorPhrase(rule.operator);
// Node-neutral body: the hub bell badge attributes remotes.
const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue;
const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold;
const serviceLabel = displayServiceName(serviceName);
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
await NotificationService.getInstance().dispatchAlert(
// Node-neutral body: the hub bell badge attributes remotes.
const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`;
try {
const { persisted } = await NotificationService.getInstance().dispatchAlert(
'warning',
'monitor_alert',
message,
{ stackName: rule.stack_name, actor: 'system:monitor' },
{ stackName: rule.stack_name, containerName, actor: 'system:monitor' },
);
if (!persisted) {
// History was not written; do not advance cooldown or we silence retries.
this.firedThisCycle.delete(cooldownKey);
console.error(
`[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`,
);
continue;
}
const firedAt = Date.now();
// Dual-write: per-service row for current code, parent last_fired_at
// so a downgrade that only reads the parent column still has a floor.
db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt);
db.updateStackAlertLastFired(ruleId, firedAt);
console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`);
} catch (fireErr) {
this.firedThisCycle.delete(cooldownKey);
console.error(
`[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`,
fireErr,
);
db.updateStackAlertLastFired(ruleId, Date.now());
}
} else if (isDebugEnabled()) {
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`);
}
}
} else {
if (this.activeBreaches.has(ruleId)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`);
this.activeBreaches.delete(ruleId);
if (this.activeBreaches.has(breachKey)) {
if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`);
this.activeBreaches.delete(breachKey);
}
}
}
+11 -5
View File
@@ -181,7 +181,7 @@ export class NotificationService {
category: NotificationCategory,
message: string,
options?: { stackName?: string; containerName?: string; actor?: string },
) {
): Promise<{ persisted: boolean }> {
const t0 = Date.now();
const { stackName, containerName, actor } = options ?? {};
@@ -191,6 +191,8 @@ export class NotificationService {
// WebSocket broadcast can all throw on an unhealthy DB, which would
// otherwise surface as an unhandledRejection and take the process down.
// The whole body is wrapped so the worst case is a dropped notification.
// Callers that gate cooldowns on history use `persisted` (true only after the row write).
let wroteHistory = false;
try {
// Internal writes use the middleware default so they share a row key
// with user-initiated requests; otherwise the UI and monitors split
@@ -216,11 +218,12 @@ export class NotificationService {
container_name: containerName,
actor_username: actor ?? null,
});
wroteHistory = true;
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true);
} catch (err) {
StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false);
console.error('[Notify] Failed to persist notification:', err);
return;
return { persisted: false };
}
// Separate [StackActivity:diag] namespace from the [Notify:diag] lines
// below so a single grep can pull every per-stack timeline write across
@@ -273,7 +276,7 @@ export class NotificationService {
}
if (suppressExternal) {
return;
return { persisted: wroteHistory };
}
// Resolve retry extras once for this dispatch (shared by all destinations).
@@ -298,14 +301,14 @@ export class NotificationService {
)
);
this.recordDispatchErrors(notification.id!, errors);
return;
return { persisted: wroteHistory };
}
// 4. Fall back to this instance's agents (keyed by this instance's default node id).
const agents = this.dbService.getEnabledAgents(localNodeId);
if (agents.length === 0) {
if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch');
return;
return { persisted: wroteHistory };
}
if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`);
@@ -322,8 +325,11 @@ export class NotificationService {
)
);
this.recordDispatchErrors(notification.id!, errors);
return { persisted: wroteHistory };
} catch (err) {
console.error('[Notify] dispatchAlert failed:', err);
// History may already be written; callers must not treat that as a miss.
return { persisted: wroteHistory };
}
}
@@ -457,15 +457,30 @@ function asApprovedBlueprint(blueprint: Blueprint): Blueprint & BlueprintApprova
}
/** Upgrade create rows to blockers when an unmanaged same-name stack already exists. */
async function applyCreateNameConflictBlockers(blueprintName: string, raw: RawAction[]): Promise<void> {
function blockCreateForOwnership(row: RawAction, detail: string): void {
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = detail;
}
async function applyCreateNameConflictBlockers(
blueprintName: string,
blueprintId: number,
raw: RawAction[],
): Promise<void> {
const { BlueprintService } = await import('./BlueprintService');
const svc = BlueprintService.getInstance();
for (const row of raw) {
if (row.action !== 'create') continue;
if (!(await svc.hasNameConflict(blueprintName, row.node))) continue;
row.action = 'blocked_name_conflict';
row.severity = 'blocker';
row.detail = 'Unmanaged stack with this name already exists on this node';
try {
if (!(await svc.hasNameConflict(blueprintName, row.node, blueprintId))) continue;
blockCreateForOwnership(row, 'Unmanaged stack with this name already exists on this node');
} catch (err) {
blockCreateForOwnership(
row,
err instanceof Error ? err.message : 'Cannot verify stack ownership on this node',
);
}
}
}
@@ -477,7 +492,7 @@ export async function buildBlueprintPreview(blueprintId: number): Promise<Bluepr
const deployments = db.listDeployments(blueprintId);
const decision = BlueprintReconciler.getInstance().computeDecisionForPreview(blueprint, allNodes);
const raw = projectActions(blueprint, allNodes, deployments, decision);
await applyCreateNameConflictBlockers(blueprint.name, raw);
await applyCreateNameConflictBlockers(blueprint.name, blueprint.id, raw);
const changes: PreviewChangeRow[] = [];
for (const row of raw) {
@@ -56,7 +56,7 @@ export interface EffService {
networkMode?: string;
restart?: string;
hasHealthcheck: boolean;
/** Raw deploy block (read for key presence only, never values; undefined = none). */
/** Raw deploy block (preflight uses key presence; Drift also reads restart_policy.condition). Undefined = none. */
deploy?: Record<string, unknown>;
containerName?: string;
user?: string;
+1 -1
View File
@@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = {
message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`,
sourcePath: s.name,
service: s.name,
remediation: 'Add restart: unless-stopped.',
remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.',
}));
},
};
+77
View File
@@ -0,0 +1,77 @@
/**
* Helpers for clean one-shot completion and Compose restart-intent normalization.
* Used by Health Gate and Drift service-presence only; does not change bulk
* status, Auto-Heal, or atomic-deploy helpers.
*/
/**
* True only for an explicit Compose `restart: "no"` (after normalization).
* Absent / null / empty do not qualify: Docker reports HostConfig restart
* "no" for both intentional one-shots and bare long-running services that
* omit `restart:`, so consumers must pass declared Compose intent, not inspect.
*/
export function isNoRestartPolicy(policy: string | null | undefined): boolean {
return policy === 'no';
}
/**
* Normalize Compose restart intent for Drift / declared-model consumers.
*
* Compose precedence: when `deploy.restart_policy` is set, its `condition`
* wins; otherwise the service-level `restart` field is used. Conditions map to
* Docker-like policy names so {@link isNoRestartPolicy} stays the single gate:
* `none` `no`, `any` `always`, `on-failure` `on-failure`. Missing
* condition defaults to `any` (Compose default). Unknown shapes fail closed.
* Absent service restart (no deploy policy) stays null and is not one-shot eligible.
*/
export function normalizeComposeRestartIntent(
serviceRestart: string | null | undefined,
deploy?: Record<string, unknown> | null,
): string | null {
if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) {
const policy = deploy['restart_policy'];
if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) {
return 'always';
}
const condition = (policy as Record<string, unknown>).condition;
// Missing, empty, or non-string → Compose default `any` → always.
if (typeof condition !== 'string' || condition === '') {
return 'always';
}
switch (condition.toLowerCase()) {
case 'none':
return 'no';
case 'on-failure':
return 'on-failure';
case 'any':
return 'always';
default:
return 'always';
}
}
if (serviceRestart == null || serviceRestart === '') return null;
return serviceRestart;
}
export interface OneShotCompletionInput {
state: string;
/** Exact Docker exit code; null means unknown (fail closed). */
exitCode: number | null;
/**
* Declared Compose restart intent after normalization (`"no"` for explicit
* one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker
* inspect HostConfig values here.
*/
restartPolicy: string | null | undefined;
}
/**
* True only for an exited container with exit code exactly 0 and explicit
* declared restart `"no"`. Null/absent restart, null exit codes, and restarting
* policies never qualify.
*/
export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean {
return input.state === 'exited'
&& input.exitCode === 0
&& isNoRestartPolicy(input.restartPolicy);
}
+4 -3
View File
@@ -54,9 +54,10 @@ export function handleGenericWs(
if (isProxyToken) return reject(socket, 403, 'Forbidden');
// Admin enforcement: container exec requires admin role.
// console_session tokens are already admin-gated at creation time.
// API tokens reaching this point have full-admin scope (read-only /
// deploy-only are blocked by the upgrade handler's scope gate).
// console_session tokens are already admin-gated at creation time and
// path/jti-gated in upgradeHandler. API tokens reaching this point have
// full-admin scope (read-only / deploy-only are blocked by the upgrade
// handler's scope gate).
if (!decoded.scope) {
const execUser = decoded.username ? DatabaseService.getInstance().getUserByUsername(decoded.username) : undefined;
if (!execUser) {
+33 -37
View File
@@ -2,22 +2,16 @@ import type { IncomingMessage } from 'http';
import type { Duplex } from 'stream';
import WebSocket, { WebSocketServer } from 'ws';
import { FileSystemService } from '../services/FileSystemService';
import { NodeRegistry } from '../services/NodeRegistry';
import { HostTerminalService } from '../services/HostTerminalService';
import { PROXY_TIER_HEADER } from '../services/license-headers';
import {
isLicenseTier,
normalizeTier,
} from '../services/license-normalize';
import { LicenseService } from '../services/LicenseService';
import { ROLE_PERMISSIONS, type PermissionAction } from '../middleware/permissions';
import { ROLE_PERMISSIONS } from '../middleware/permissions';
import type { UserRole } from '../services/DatabaseService';
import { getErrorMessage } from '../utils/errors';
import { rejectUpgrade as reject } from './reject';
import { isConsoleSessionScope } from '../helpers/consoleSession';
interface HostConsoleContext {
nodeId: number;
decoded: { scope?: string; username?: string };
decoded: { scope?: string; username?: string; acting_as?: string };
isProxyToken: boolean;
wsResolvedUser: { username: string; role: UserRole; token_version: number } | undefined;
stackParam: string | null;
@@ -26,15 +20,16 @@ interface HostConsoleContext {
/**
* Handle `/api/system/host-console` WebSocket upgrades.
*
* Enforces three gates before spawning the host PTY:
* Enforces two gates before spawning the host PTY:
* 1. Machine-credential rejection: node_proxy tokens cannot reach an
* interactive host shell.
* interactive host shell directly (remote forwarding mints a
* console_session via POST /console-token instead).
* 2. RBAC: user session tokens require the `system:console` permission.
* console_session tokens are pre-gated at issuance (see
* `routes/console.ts`) and skip this check.
* 3. License: host console requires the paid tier. For console_session
* tokens the tier is trusted from the gateway-supplied header;
* otherwise the local LicenseService is consulted.
*
* Path scoping and one-time jti consumption are enforced in upgradeHandler
* before this handler runs.
*/
export function handleHostConsoleWs(
req: IncomingMessage,
@@ -46,11 +41,10 @@ export function handleHostConsoleWs(
if (isProxyToken) return reject(socket, 403, 'Forbidden');
const isConsoleSession = decoded.scope === 'console_session';
const isConsoleSession = isConsoleSessionScope(decoded.scope);
if (!isConsoleSession) {
const userRole = wsResolvedUser?.role;
const consolePermission: PermissionAction = 'system:console';
if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes(consolePermission)) {
if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes('system:console')) {
console.log('[HostConsole] Access denied: insufficient permissions', {
username: wsResolvedUser?.username || decoded.username,
role: userRole,
@@ -59,18 +53,18 @@ export function handleHostConsoleWs(
}
}
const consoleTierHeader = req.headers[PROXY_TIER_HEADER] as string | undefined;
const ls = LicenseService.getInstance();
const consoleTier = (isConsoleSession && isLicenseTier(consoleTierHeader))
? normalizeTier(consoleTierHeader)
: ls.getTier();
if (consoleTier !== 'paid') {
return reject(socket, 403, 'Forbidden');
}
// Option B: console_session principal stays "console_session"; hub operator
// is recorded separately as acting_as. Browser sessions use the real user.
const consoleUsername = isConsoleSession
? 'console_session'
: (wsResolvedUser?.username || decoded.username || 'unknown');
const actingAs = isConsoleSession && typeof decoded.acting_as === 'string' && decoded.acting_as
? decoded.acting_as
: null;
const consoleUsername = wsResolvedUser?.username || decoded.username || 'console_session';
console.log('[HostConsole] WebSocket upgrade accepted', {
username: consoleUsername,
actingAs,
nodeId,
stack: stackParam || '(root)',
});
@@ -86,10 +80,6 @@ export function handleHostConsoleWs(
hostConsoleWss.handleUpgrade(req, socket, head, (ws) => {
hostConsoleWss.close();
let targetDirectory: string;
// The shell may end up rooted at a different node than requested if the
// requested node's directory cannot be resolved; the audit row must name
// the node the shell actually runs in, so track it alongside the directory.
let auditNodeId: number = nodeId;
try {
const baseDir = FileSystemService.getInstance(nodeId).getBaseDir();
const resolved = HostTerminalService.resolveConsoleDirectory(baseDir, stackParam);
@@ -100,22 +90,28 @@ export function handleHostConsoleWs(
}
targetDirectory = resolved;
} catch (error) {
const fallbackNodeId = NodeRegistry.getInstance().getDefaultNodeId();
console.error('[HostConsole] Failed to resolve console directory; falling back to the default node base dir', {
console.error('[HostConsole] Failed to resolve console directory', {
user: consoleUsername,
actingAs,
nodeId,
fallbackNodeId,
stack: stackParam || '(root)',
error: getErrorMessage(error, 'unknown'),
});
targetDirectory = FileSystemService.getInstance(fallbackNodeId).getBaseDir();
auditNodeId = fallbackNodeId;
if (ws.readyState === WebSocket.OPEN) {
ws.send('Error: Failed to resolve console directory.\r\n');
ws.close();
}
return;
}
const auditCtx = { username: consoleUsername, nodeId: auditNodeId, ipAddress };
const auditCtx = { username: consoleUsername, actingAs, nodeId, ipAddress };
try {
HostTerminalService.spawnTerminal(ws, targetDirectory, auditCtx);
} catch (error) {
console.error('[HostConsole] Unhandled spawn error:', { user: consoleUsername, error: getErrorMessage(error, 'unknown') });
console.error('[HostConsole] Unhandled spawn error:', {
user: consoleUsername,
actingAs,
error: getErrorMessage(error, 'unknown'),
});
if (ws.readyState === WebSocket.OPEN) {
ws.send('Error: Failed to start terminal session.\r\n');
ws.close();
+20 -5
View File
@@ -5,6 +5,7 @@ import { LicenseService } from '../services/LicenseService';
import { wsProxyServer } from '../proxy/websocketProxy';
import { getErrorMessage } from '../utils/errors';
import { rejectUpgrade as reject } from './reject';
import { consoleSessionPathForPathname } from '../helpers/consoleSession';
/**
* Forward a WebSocket upgrade to a remote Sencho instance. Handles the
@@ -23,9 +24,14 @@ export async function handleRemoteForwarder(
req: IncomingMessage,
socket: Duplex,
head: Buffer,
opts: { pathname: string; target: { apiUrl: string; apiToken: string } },
opts: {
pathname: string;
target: { apiUrl: string; apiToken: string };
/** Hub browser operator; recorded as acting_as on the remote audit trail. */
actingAs?: string;
},
): Promise<void> {
const { pathname, target } = opts;
const { pathname, target, actingAs } = opts;
if (!target.apiUrl) return reject(socket, 503, 'Service Unavailable');
const wsTarget = target.apiUrl.replace(/\/$/, '').replace(/^https?/, (m) => m === 'https' ? 'wss' : 'ws');
@@ -38,24 +44,33 @@ export async function handleRemoteForwarder(
// direct api_token access. Pilot loopback targets skip this: there is no
// long-lived api_token to exchange, and host-console is disabled on pilot
// mode at the capability registry anyway.
const isInteractiveConsolePath = pathname === '/api/system/host-console' || pathname === '/ws';
const sessionPath = consoleSessionPathForPathname(pathname);
let bearerTokenForProxy = target.apiToken;
if (isInteractiveConsolePath && !isPilotLoopback) {
if (sessionPath && !isPilotLoopback) {
try {
const consoleHeaders = LicenseService.getInstance().getProxyHeaders();
const tokenRes = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/console-token`, {
method: 'POST',
headers: {
'Authorization': `Bearer ${target.apiToken}`,
'Content-Type': 'application/json',
[PROXY_TIER_HEADER]: consoleHeaders.tier,
},
body: JSON.stringify({
path: sessionPath,
...(actingAs ? { acting_as: actingAs } : {}),
}),
});
if (!tokenRes.ok) {
console.error(`[WS Proxy] Remote console-token request failed: ${tokenRes.status}`);
return reject(socket, 502, 'Bad Gateway');
}
const data = await tokenRes.json() as { token?: string };
if (typeof data.token === 'string') bearerTokenForProxy = data.token;
if (typeof data.token !== 'string' || !data.token) {
console.error('[WS Proxy] Remote console-token response missing token');
return reject(socket, 502, 'Bad Gateway');
}
bearerTokenForProxy = data.token;
} catch (e) {
console.error('[WS Proxy] Failed to fetch remote console token:', getErrorMessage(e, 'unknown'));
return reject(socket, 502, 'Bad Gateway');
+90 -6
View File
@@ -20,6 +20,14 @@ import { validateApiToken, touchApiTokenLastUsed } from '../utils/apiTokenAuth';
import { isDebugEnabled } from '../utils/debug';
import { PROXY_TIER_HEADER } from '../services/license-headers';
import { isLicenseTier, normalizeTier } from '../services/license-normalize';
import { HOST_CONSOLE_COMMUNITY_CAPABILITY } from '../services/CapabilityRegistry';
import { remoteAdvertisesCapability } from '../helpers/remoteCapabilities';
import {
consoleSessionPathForPathname,
consumeConsoleSessionJti,
isConsoleSessionScope,
} from '../helpers/consoleSession';
function parseCookies(req: IncomingMessage): Record<string, string> {
const header = req.headers.cookie || '';
@@ -31,6 +39,17 @@ function parseCookies(req: IncomingMessage): Record<string, string> {
);
}
/**
* Parse ?nodeId= as a strict positive integer. Returns null when the param is
* absent (caller should use the default node). Returns undefined when the
* param is present but malformed (caller should 404).
*/
function parseStrictNodeIdParam(raw: string | null): number | null | undefined {
if (raw == null || raw === '') return null;
if (!/^[1-9][0-9]*$/.test(raw)) return undefined;
return Number(raw);
}
// The two WebSocket paths open to any authenticated user (read-only/deploy-only
// API tokens and non-admin sessions on a remote node). Defined once so the
// scope gate and the remote-forward gate cannot drift apart.
@@ -68,8 +87,15 @@ export function remoteWsForwardAllowed(
if (ctx.isProxyToken) return false;
// console_session is pre-gated as admin at issuance (routes/console.ts).
if (ctx.decoded.scope === 'console_session') return true;
// Reject opaque API tokens on remote Host Console forward (would mint
// console_session and get a host shell). Local denial is enforced in
// attachUpgrade before handleHostConsoleWs. Container exec (/ws) keeps
// its existing full-admin API-token allow.
if (pathname.startsWith('/api/system/host-console') && ctx.wsApiTokenScope) {
return false;
}
// A read-only/deploy-only api_token is already rejected for these paths by
// the scope gate; only full-admin survives to here.
// the scope gate; only full-admin survives to here (container exec /ws).
if (ctx.wsApiTokenScope) return ctx.wsApiTokenScope === 'full-admin';
const role = ctx.wsResolvedUser?.role;
if (!role) return false;
@@ -136,7 +162,16 @@ export function attachUpgrade(
try {
// Opaque sen_sk_ API tokens: handled before jwt.verify. Prefix +
// length + checksum reject malformed keys without touching SQLite.
let decoded: { username?: string; scope?: string; role?: string; tv?: number };
let decoded: {
username?: string;
scope?: string;
role?: string;
tv?: number;
path?: string;
acting_as?: string;
jti?: string;
exp?: number;
};
let wsApiTokenScope: string | null = null;
if (looksLikeApiToken(token)) {
const validation = validateApiToken(token);
@@ -151,7 +186,7 @@ export function attachUpgrade(
const settings = DatabaseService.getInstance().getGlobalSettings();
const jwtSecret = settings.auth_jwt_secret;
if (!jwtSecret) throw new Error('No JWT secret');
decoded = jwt.verify(token, jwtSecret) as { username?: string; scope?: string; role?: string; tv?: number };
decoded = jwt.verify(token, jwtSecret) as typeof decoded;
}
// Node proxy tokens are machine-to-machine credentials and must never be
@@ -180,6 +215,21 @@ export function attachUpgrade(
const parsedUrl = new URL(req.url || '', `http://${req.headers.host || 'localhost'}`);
const pathname = parsedUrl.pathname;
// Path-scoped, one-time console_session tokens for interactive surfaces.
// Consume runs at upgrade acceptance (before PTY spawn); missing exp/jti fail closed.
if (isConsoleSessionScope(decoded.scope)) {
const requiredPath = consoleSessionPathForPathname(pathname);
if (!requiredPath || decoded.path !== requiredPath) {
return reject(socket, 403, 'Forbidden');
}
if (typeof decoded.exp !== 'number' || typeof decoded.jti !== 'string' || !decoded.jti) {
return reject(socket, 401, 'Unauthorized');
}
if (!consumeConsoleSessionJti(decoded.jti, decoded.exp * 1000)) {
return reject(socket, 401, 'Unauthorized');
}
}
// Gate WebSocket paths by API token scope
if (wsApiTokenScope) {
if (wsApiTokenScope === 'read-only' || wsApiTokenScope === 'deploy-only') {
@@ -219,8 +269,11 @@ export function attachUpgrade(
return;
}
const nodeIdParam = parsedUrl.searchParams.get('nodeId');
const nodeId = nodeIdParam ? parseInt(nodeIdParam, 10) : NodeRegistry.getInstance().getDefaultNodeId();
const parsedNodeId = parseStrictNodeIdParam(parsedUrl.searchParams.get('nodeId'));
if (parsedNodeId === undefined) {
return reject(socket, 404, 'Not Found');
}
const nodeId = parsedNodeId ?? NodeRegistry.getInstance().getDefaultNodeId();
const node = NodeRegistry.getInstance().getNode(nodeId);
// Notification push channel: local only when no remote nodeId is
@@ -243,6 +296,23 @@ export function attachUpgrade(
if (!remoteWsForwardAllowed(pathname, { wsResolvedUser, wsApiTokenScope, isProxyToken, decoded })) {
return reject(socket, 403, 'Forbidden');
}
// Community hubs require host-console-community before forwarding Host
// Console (marks remotes that no longer paid-gate the console path).
// Admiral hubs skip the probe for legacy host-console peers. Fail
// closed on probe errors; never fall through to local handlers for a
// named remote.
if (
pathname.startsWith('/api/system/host-console')
&& LicenseService.getInstance().getTier() !== 'paid'
) {
const communityCapable = await remoteAdvertisesCapability(
nodeId,
HOST_CONSOLE_COMMUNITY_CAPABILITY,
);
if (!communityCapable) {
return reject(socket, 403, 'Forbidden');
}
}
// Resolve the proxy target through NodeRegistry so pilot-mode nodes
// (empty api_url + api_token, loopback bridge instead) and proxy-mode
// nodes share one dispatch path. Mirrors proxy/remoteNodeProxy.ts.
@@ -253,7 +323,11 @@ export function attachUpgrade(
// serve gateway-local data for a request that named a remote node.
return reject(socket, 503, 'Service Unavailable');
}
await handleRemoteForwarder(req, socket, head, { pathname, target });
await handleRemoteForwarder(req, socket, head, {
pathname,
target,
actingAs: wsResolvedUser?.username || decoded.username,
});
return;
}
@@ -264,6 +338,16 @@ export function attachUpgrade(
}
if (pathname.startsWith('/api/system/host-console')) {
// Opaque API tokens never open a local host shell (parity with the
// remote forward gate). Do not rely on missing wsResolvedUser alone.
if (wsApiTokenScope) {
return reject(socket, 403, 'Forbidden');
}
// Unknown / deleted nodeIds must not fall through to the hub compose
// root via getComposeDir's env default.
if (!node) {
return reject(socket, 404, 'Not Found');
}
handleHostConsoleWs(req, socket, head, {
nodeId,
decoded,
+27 -18
View File
@@ -1,10 +1,10 @@
---
title: Alerts & Notifications
sidebarTitle: Alerts and notifications
description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with per-stack rules and channel routing.
description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with stack and service rules and channel routing.
---
Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing per-stack threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention.
Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing stack and service threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention.
<Frame>
<img src="/images/alerts-notifications/notifications-settings.png" alt="Settings · Notifications · Channels panel with NODE Local in the header, a Delivery retries row showing Extra attempts 0 and a Save retries button, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, and Test beside Save." />
@@ -37,7 +37,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea
```json
{
"level": "warning",
"message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).",
"message": "The **CPU usage** for **api** in **plex** (container **plex-api-1**) has exceeded your threshold of **80%** (Currently: 91%).",
"timestamp": "2026-05-08T22:14:09.812Z",
"source": "sencho"
}
@@ -161,11 +161,17 @@ Every alert Sencho dispatches carries a category that you can filter on in the b
The four `blueprint_*` categories are accepted by routing rules but render as raw category strings in the bell because the frontend label map omits them.
## Per-stack alert rules
## Stack alert rules
Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. Rules live on the node where the stack runs and are evaluated locally on a 30-second tick.
Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. A rule can target **All services** in the stack or one Compose service. Metrics are evaluated per running container (not as a stack aggregate): each container has its own duration timer, and cooldown is tracked per Compose service so replicas of the same service share one silence window while different services can alert independently.
Open the rules editor by right-clicking a stack in the sidebar and choosing **Alerts**, or by pressing `A` while focused on a stack.
Rules live on the node where the stack runs and are evaluated locally on a 30-second tick.
Open the rules editor from any of these entry points:
- Stack header **More actions** → **Monitor** (Alerts tab)
- Container card **Monitor** (prefers that Compose service in add forms)
- Sidebar context menu **Alerts**, or press `A` while focused on a stack
<Frame>
<img src="/images/alerts-notifications/alert-panel.png" alt="The Stack PLEX MONITOR sheet with the Alerts tab active, a green notification-channel banner under the NOTIFICATION CHANNELS heading, an ACTIVE RULES section listing 'CPU Usage (%) > 80' with the secondary line 'Trigger after 5m • Cooldown 60m', and an ADD NEW RULE form with Metric, Operator, Threshold, Duration, Cooldown fields and an Add Rule submit button." />
@@ -175,13 +181,14 @@ Open the rules editor by right-clicking a stack in the sidebar and choosing **Al
| Field | Purpose |
|-------|---------|
| **Service** | **All services** (default) or one Compose service from the stack. All services evaluates every matching container independently. |
| **Metric** | The system resource or metric to monitor. |
| **Operator** | Comparison: `Greater than`, `Greater or eq`, `Less than`, `Less or eq`, `Equals`. |
| **Threshold** | A number ≥ 0. The unit follows the chosen metric. |
| **Duration (mins)** | How long the breach must persist before firing. Default `5`, range 0 to 1440. |
| **Cooldown (mins)** | Silence window between fires after a rule triggers. Default `60`, range 0 to 10080. |
| **Duration (mins)** | How long a container's breach must persist before firing. Default `5`, range 0 to 1440. |
| **Cooldown (mins)** | Silence window between fires for the same Compose service after a rule triggers. Default `60`, range 0 to 10080. |
Sencho tracks the start of each breach in memory; the rule fires only after the breach has lasted for the full **Duration**, and only if the previous fire is older than **Cooldown**.
Sencho tracks the start of each breach per container in memory; the rule fires only after that container's breach has lasted for the full **Duration**, and only if the previous fire for the same Compose service is older than **Cooldown**. The notification names the triggering service and container. Containers without a Compose service label are grouped under one shared cooldown key and appear as **unknown service** in the message; that label is not selectable as a rule target. A scoped rule whose service is no longer listed in the compose file shows **Not in compose** in the rules list; evaluation still matches running containers that carry that Compose service label.
### Available metrics
@@ -305,10 +312,11 @@ Real-time on the Docker event stream:
- **Crash** at `error`/`monitor_alert`: `Container Crash Detected: <name> exited unexpectedly (Code: <N>).`
- **OOM kill** at `error`/`monitor_alert`: `Container OOM Kill: <name> was killed by the OOM killer (out of memory).` When a `die` arrives with exit code 137 but no preceding `oom` event, Sencho inspects the container and reclassifies as OOM if the `OOMKilled` flag is set.
- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: <name> is unhealthy.`
- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: <name> is unhealthy.` Emitted only on the transition into Docker `unhealthy` (not on every repeated unhealthy event).
- **Mass exit** kicks in when a daemon-disconnect gap is followed by 20% or more of containers exiting on reconnect, in which case a single `info`/`system` summary `Docker daemon interruption detected: N containers exited during connection gap.` is emitted instead of N crash alerts.
- **Rate limit** caps crash dispatches at 20 per 60-second window per node, then a single `warning`/`monitor_alert` `N additional containers crashed in the last minute.`
- **Dedup** is 60 minutes per container after a non-rate-suppressed dispatch.
- **Rate limit** caps combined crash and health dispatches at 20 per fixed 60-second window per node. Overflow does not write individual history rows; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, `N additional container health alerts were rate-limited in the last minute.`, or `N additional container alerts were rate-limited in the last minute (X crash, Y health).`
- **Crash dedup** is 60 minutes per container after a non-rate-suppressed crash dispatch. Cleared when that container starts again.
- **Health dedup** is 60 minutes per container after a health alert that was persisted to notification history. It survives healthy/starting recovery and the 10-minute prune of in-memory container tracking state (dedup lives in a separate map), and clears only when the window expires or the Sencho process restarts.
### Daemon connectivity
@@ -403,7 +411,7 @@ Three retention controls live under **Settings · Operations · Data Retention**
A hard 100-row per-node cap inside `notification_history` always applies on top of the user-tunable retention; new inserts evict the oldest rows beyond 100 even if your retention window is longer.
A separate rate limit applies to crash and health alerts only: 20 emits per 60-second window per node, with a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` issued at the end of the window.
A separate rate limit applies to crash and health alerts only: 20 emits per fixed 60-second window per node (not a rolling window). When the cap is hit, individual overflow alerts are not written to history or the bell; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up that names crash-only, health-only, or mixed suppression.
## Crash detection toggle
@@ -429,8 +437,9 @@ The **Host Alerts** panel carries the **Host thresholds** rows (CPU limit, RAM l
| Notification fanout to channels | One attempt by default; optional 0-3 extra in-process attempts with a fixed 1s delay; 10-second timeout per attempt; no durable queue |
| Bell live updates | Pushed live over the notifications WebSocket per node |
| Bell safety-net reconcile | 60 seconds |
| Crash dedup window per container | 60 minutes |
| Crash rate-limit window | 20 alerts per 60 seconds per node, then a single roll-up |
| Crash dedup window per container | 60 minutes (cleared on container start) |
| Health dedup window per container | 60 minutes after a persisted health alert; survives recovery; process-local |
| Crash/health rate-limit window | 20 alerts per fixed 60 seconds per node, then a single typed roll-up (overflow individuals are not persisted) |
Switching the active node tears down per-stack rule editors and reloads channel state, but does not affect the bell, which keeps every node's WebSocket open in parallel.
@@ -441,7 +450,7 @@ Switching the active node tears down per-stack rule editors and reloads channel
Check three things in order. First, the channel toggle in **Settings · Notifications · Channels** must be on; the kicker on each tab reads `enabled` or `off`. Second, Discord, Slack, and webhook URLs must use HTTPS (the form rejects plain `http://` for those channels); Apprise endpoints may use HTTP or HTTPS. Third, a routing rule with unconstrained Node plus empty Stacks, Labels, Categories, and Severity matchers will intercept every alert and skip the global channels. Use the per-channel **Test** button to issue a one-shot dispatch and watch your endpoint for the literal message `🔌 Test Notification from Sencho!` Sencho records the failure reason in `notification_history.dispatch_error` when delivery throws, so a row that appears in the bell with no follow-up at the endpoint usually means a 4xx or timeout at the receiver.
</Accordion>
<Accordion title="An alert rule never fires even when the threshold is breached">
Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: the breach must persist for the full duration before the rule fires. Second, the rule is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging.
Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: that container's breach must persist for the full duration before the rule fires (a healthy sibling service does not clear another container's timer). Second, the same Compose service is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging.
</Accordion>
<Accordion title="I stopped a stack but got a crash alert anyway">
The Docker event service defers `die` classification by 500 ms to absorb out-of-order `kill` events from the daemon, then asks the lifecycle classifier whether the exit was intentional, clean (exit 0), a crash, or an OOM kill. If your stop happened far enough outside that window, or the daemon emitted the events without the kill marker the classifier looks for, the exit can be classified as a crash. The classifier favors avoiding silent crashes over avoiding noisy false positives. Compare the alert timestamp against your `docker compose down` time; entries within a second of each other are usually the same event seen from two angles.
@@ -452,8 +461,8 @@ Switching the active node tears down per-stack rule editors and reloads channel
<Accordion title="Crash alerts stopped arriving and nothing else looks wrong">
Check **Settings · Monitoring · Container Alerts · Container crash & health alerts**. The toggle is the master switch for the Docker event service. The panel cache reads the database every 500 ms; if the database read errors, Sencho defaults the toggle to off so the failure mode is silent rather than spammy. Repair the toggle, save, and the next Docker event reaches the dispatcher.
</Accordion>
<Accordion title="A burst of crashes happened but I only see about twenty alerts">
Sencho rate-limits crash and health dispatches to 20 emits per rolling 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` once the window closes. Every alert is still persisted to `notification_history` and visible in the bell up to the 100-row per-node cap; only the channel fanout is throttled.
<Accordion title="A burst of crashes or health flaps happened but I only see about twenty alerts">
Sencho rate-limits crash and health dispatches together to 20 emits per fixed 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, a health-only variant, or a mixed `(X crash, Y health)` summary once the window closes. Rate-limited overflow individuals are not written to `notification_history` and do not appear in the bell; only the roll-up is persisted for that overflow. Separately, health flaps on the same container are deduped for 60 minutes after a persisted unhealthy alert, even if Docker recovered and failed again inside that window.
</Accordion>
<Accordion title="My deploy success doesn't appear in the bell, only as a toast">
Sencho deliberately hides rows whose category is one of `deploy_success`, `stack_started`, `stack_stopped`, `stack_restarted`, or `image_update_applied` AND whose `actor_username` is set to a real user. The reasoning: those are confirmations of the action you just clicked and are already shown as a toast. The rows are still persisted to `notification_history` and still dispatched to global channels and matching routes; only the bell render hides them.
+2 -2
View File
@@ -171,7 +171,7 @@ Authorises every read and every write the API exposes, **except** the universal
| `/api/stacks/:stack/logs` (stack log stream) | yes | yes | yes |
| `/ws/notifications` (notification stream) | yes | yes | yes |
| `/ws` (generic exec / stats) | no | no | yes |
| `/api/system/host-console` (host shell) | no | no | yes |
| `/api/system/host-console` (host shell) | no | no | no |
A Read Only or Deploy Only token attempting an out-of-scope WebSocket upgrade is rejected with `403 Forbidden` before any frames flow.
@@ -268,7 +268,7 @@ API tokens are issued and revoked on the hub that authenticates the call. When y
- **One owner per token.** Tokens are not shareable across users at the data model; only the creating user can revoke a token through the UI.
- **Owner deletion breaks the token.** If the user who created a token is deleted, subsequent calls with that token return `401` because the auth path requires the creator to still exist. Rotate before deleting accounts.
- **No token introspection endpoint.** There is no `GET /api/me` for an API token; client code must know its own scope.
- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can.
- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can reach `/ws` (container exec / stats) but not Host Console; Host Console always requires a signed-in browser session.
## Common workflows
+1 -1
View File
@@ -89,7 +89,7 @@ Sencho uses three type contexts. The interface and data faces are yours to chang
The **Display** group holds layout and log-chip preferences for this browser:
- **Density** switches between **Comfortable** (roomy rows, the default) and **Compact** (tighter rows and tiles that fit more on screen for dense dashboards).
- **Log chip color** controls how service chips are colored in log views. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once.
- **Log chip color** controls how service chips are colored in log views on multi-service or multi-container stacks. Single-service, single-container stacks do not show chips. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once.
## Navigation
+6 -4
View File
@@ -28,10 +28,12 @@ Policies live next to your stack-level alert rules in the stack's **Monitor** sh
## Workflow
1. In the sidebar, right-click the stack you want to protect (or focus the stack and press **H**).
2. Click **Auto-Heal** in the **inspect** group. The stack's **Monitor** sheet opens directly on the **Auto-heal** tab.
3. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds.
4. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on.
1. Open the stack's **Monitor** sheet any of these ways:
- Stack header **More actions** → **Monitor**, then switch to the **Auto-heal** tab
- Container card **Monitor** (prefers that Compose service in **Add new policy**), then switch to **Auto-heal**
- Sidebar: right-click the stack (or focus it and press **H**) → **Auto-Heal** in the **inspect** group (opens directly on the **Auto-heal** tab)
2. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds.
3. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on.
<Frame>
<img src="/images/auto-heal-policies/sheet.png" alt="The Stack plex Monitor sheet on the Auto-heal tab, showing one policy scoped to the plex service in the Active policies section with its ON toggle, and the Add new policy form below with Service, Unhealthy for, Cooldown, Max restarts / hr, and Auto-disable after fields, plus the Add Policy button" />
+6 -3
View File
@@ -21,7 +21,7 @@ Three moving parts cooperate per blueprint.
1. **The declared spec.** A blueprint is a row in Sencho's database. It carries the compose YAML, the selector (labels or explicit node IDs), the drift policy, and a monotonic revision number. The revision auto-increments every time the compose changes.
2. **The reconciler.** A background loop on the controlling Sencho ticks every 60 seconds (with a 5-second initial delay after startup) and on demand after **Confirm Apply**. Each tick only mutates the fleet when the blueprint's current operational intent is approved. The tick resolves the selector, compares every desired-vs-live node, and runs only the place or remove outcomes the operator already confirmed.
3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the standard authenticated proxy to `/api/stacks` on remote nodes. Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp.
3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the authenticated proxy to `/api/blueprints/apply-local` and `/api/blueprints/withdraw-local` on remote nodes (older remotes fail closed until upgraded). Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp.
The marker is the trust root. If a directory by the blueprint's name already exists on a node and does not carry a matching marker, the reconciler refuses to touch it and surfaces a **Name conflict** on the deployment row. A Blueprint named `nginx` will never overwrite an existing user-authored `nginx` stack on any node.
@@ -155,7 +155,7 @@ Possible status values:
| **Withdrawing** | A withdraw is in flight. |
| **Withdrawn** | The deployment was successfully removed; the row is about to be cleared. |
| **Evict blocked** | A node left the selector while a stateful deployment was active. The reconciler refuses to evict until you choose an explicit mode. |
| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry our marker file. The reconciler will not touch it. |
| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry a matching `.blueprint.json` (missing, malformed, or another blueprint ID). The reconciler will not touch it. |
<Frame caption="A deployment row in Awaiting confirmation. The Confirm deploy action sits at the right; the notes column explains why the reconciler is waiting on the operator.">
<img src="/images/blueprint-model/detail-state-review.png" alt="Blueprint detail sheet with a deployment row in Awaiting confirmation state" />
@@ -309,7 +309,7 @@ A future Volume Migration feature will automate this with app-aware backup tooli
<AccordionGroup>
<Accordion title="A deployment row shows 'Name conflict' and refuses to deploy">
A directory by the blueprint's name already exists on that node and does not carry the `.blueprint.json` marker. The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry.
A directory by the blueprint's name already exists on that node without a matching `.blueprint.json` marker (missing, malformed, or owned by another blueprint). The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry.
</Accordion>
<Accordion title="A stateful deployment is stuck in 'Awaiting confirmation'">
The reconciler will not auto-deploy a stateful blueprint to a node it has never run on. Click **Confirm deploy** on the row, then choose **Deploy fresh** in the dialog. Sencho will create empty named volumes and start the stack.
@@ -323,6 +323,9 @@ A future Volume Migration feature will automate this with app-aware backup tooli
<Accordion title="A remote node is offline or disconnected during apply">
The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `compose.yaml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually.
</Accordion>
<Accordion title="Remote apply or withdraw fails asking for an upgrade">
Blueprint apply and withdraw on a remote node require that node's Sencho build to expose the atomic node-local endpoints (`/api/blueprints/apply-local` and `/api/blueprints/withdraw-local`). Older remotes refuse the mutation with a failed deployment row rather than writing files without ownership checks. Upgrade the remote instance to a matching Sencho release, then retry **Apply now** or withdraw.
</Accordion>
<Accordion title="A Docker daemon or registry failure surfaced during apply">
The deployment row moves to **Failed** and records the Docker or registry error. Resolve the daemon, socket, registry credentials, rate limit, disk, or volume-permission issue on the affected node, then click **Apply now**. Watch that node's stack activity and security scan status after retry.
</Accordion>
+3 -2
View File
@@ -23,7 +23,8 @@ Sencho encodes the active node, the view you are on, and the deep state that vie
| App Store | `/nodes/local/templates` | App Store on the active node |
| Logs | `/nodes/local/logs` | Cross-fleet log aggregation |
| Update | `/nodes/local/updates` | Fleet-wide update check |
| Console | `/nodes/local/host-console` | Host Console, a limited-availability operator surface documented on its own page when enabled on an instance |
| Console | `/nodes/local/host-console` | Host Console (admin role) |
| Console in a stack | `/nodes/local/host-console/radarr` | Host Console rooted in the Radarr stack directory |
| Audit | `/nodes/local/audit` | Audit history |
| Settings section | `/nodes/local/settings/nodes` | Settings on the Nodes section |
| Fleet tab | `/nodes/local/fleet/snapshots` | Fleet on the Snapshots tab (desktop) |
@@ -86,7 +87,7 @@ On desktop, opening the stack list at `/nodes/local/stacks` canonicalizes to `/n
Settings follows a similar split, but only on a phone: `/nodes/local/settings` opens the section list, and `/nodes/local/settings/<section>` opens a section directly. On desktop, `/nodes/local/settings` always normalizes straight to `/nodes/local/settings/appearance`; there is no bare section-list state to land on.
Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone, reflowing the terminal rather than switching to a dedicated phone screen.
Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone (`/nodes/<node>/host-console`), but Host Console is a desktop-oriented terminal: open it on a wider screen for a full interactive session.
On a phone, `/nodes/local/stacks/<stack>/files` opens the compose editor instead. Sencho does not expose a separate file-browser URL on a phone.
+1 -1
View File
@@ -139,7 +139,7 @@ Scope is a segmented control with two options:
### Live estimate and behaviour
- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim.
- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. A completed real prune that succeeds on at least one target re-triggers the same estimate so the toolbar total and per-node list reflect post-prune Docker state. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim.
- Each remote node receives one `POST /api/system/prune/system` call per selected target, with a 120-second timeout. If a transport error fires for one target, the remaining targets on that node are short-circuited with the same error rather than retried, so a dead node doesn't absorb the full multi-target timeout budget.
- Local nodes serialize against a per-node lock (`bulk-prune:<nodeId>`). A second fleet prune launched against the same local node while the first is still in flight returns *A prune is already running on this node* for each target.
- Reclaimed bytes are reported by the Docker daemon and are approximate. Per-node rows in the results panel sum the per-target reclaim; the per-target children show how much each individual prune actually freed.
+1 -1
View File
@@ -26,7 +26,7 @@ The palette groups results into three sections.
| Group | What it contains | What happens when you pick one |
|-------|------------------|--------------------------------|
| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, and **Schedules** appear for admins; **Console** is a limited-availability operator surface shown when enabled on an instance; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page |
| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, **Schedules**, and **Console** appear for admins; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page |
| **Nodes** | Every node in your fleet, with a green dot for online and a grey dot for offline. The currently active node carries a small **ACTIVE** chip on the right. | Switches the active node without leaving the current page |
| **Stacks** | Every compose stack on every online node, matched on the compose filename (extension included). | Switches to the stack's node and opens it in the editor |
+9 -7
View File
@@ -49,11 +49,13 @@ The 3-second crash probe from [atomic deployments](/features/atomic-deployments)
During the window, the gate checks every container that came up with the stack:
- all expected containers stay **running**,
- Docker healthchecks report **healthy** wherever a service defines one,
- no container **exits**, **disappears**, or enters a **restart loop**.
- all expected containers stay **running** (or finish cleanly as a one-shot; see below),
- Docker healthchecks report **healthy** on long-running containers wherever a service defines one (residual health on a completed one-shot is ignored),
- no container **crashes** (non-zero exit), **disappears**, or enters a **restart loop**.
A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success.
A one-shot or init service that finishes with exit code `0` and explicitly declares `restart: "no"` (or `deploy.restart_policy.condition: none`) counts as a successful completion, not a gate failure. Omitting `restart:` does not qualify: Compose defaults to no-restart, and Docker reports the same inspect value for both intentional jobs and bare long-running services. Residual Docker health state on a completed one-shot (`starting` or `unhealthy` from an inherited healthcheck) does not fail or leave the gate unknown. A clean exit under `unless-stopped`, `always`, or `on-failure`, or with no declared restart, still fails the gate. An exit whose code cannot be read also fails the gate.
A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period on a long-running container when the window ends records the verdict as unknown rather than claiming success.
After a **service-scoped** update or restore, the gate still watches the whole project, but it attributes failures: a problem on the updated service is a primary failure; a previously healthy sibling that regresses during the window is a collateral failure. Siblings that were already unhealthy before the update do not fail the gate on their own.
@@ -78,7 +80,7 @@ The gate also runs for updates you did not click: scheduled image updates, webho
Open **Settings > Infrastructure > Stacks > Deploy Guardrails** on the node you want to configure:
- **Observe health after updates** turns the gate on or off for that node. On by default; turning it off only stops the observation and its timeline events, never the update itself.
- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a healthcheck still starting at the end of the window records an unknown verdict instead of a pass.
- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a long-running healthcheck still starting at the end of the window records an unknown verdict instead of a pass.
<Frame>
<img src="/images/health-gated-updates/settings.png" alt="Stacks settings page showing the Deploy Guardrails subsection, with the Observe health after updates toggle set to On and the Observation window field set to 90 seconds" />
@@ -110,10 +112,10 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou
Unknown means one of the verdict-affecting signals could not be verified, most often because Docker or the node did not answer in time. The update path is never blocked by an unknown verdict; check the node's connection if it persists, and proceed when you are confident in the stack's state.
</Accordion>
<Accordion title="The health gate failed but my app seems fine">
The gate fails on the first clear problem it observes: a container exit, an unhealthy healthcheck, a restart loop, or a container disappearing mid-window. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period.
The gate fails on the first clear problem it observes: a non-zero exit, an unexpected clean exit of a service that is not an explicit one-shot, an unhealthy healthcheck on a long-running container, a restart loop, or a container disappearing mid-window. A one-shot that exits `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) does not fail the gate, including when residual Docker health on that completed container is still starting or unhealthy. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step that should stay up, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period.
</Accordion>
<Accordion title="The health gate result is unknown">
Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a healthcheck was still starting when the window ended, or no containers appeared to observe. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way.
Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a long-running healthcheck was still starting when the window ended, or no containers appeared to observe. Residual starting health on a completed one-shot does not produce unknown. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way.
</Accordion>
<Accordion title="The modal stays open saying the gate is observing">
The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever.
+13 -5
View File
@@ -6,7 +6,7 @@ description: An interactive terminal on your host OS, directly in the browser -
The **Console** tab opens a full interactive terminal session on the machine running Sencho. It behaves exactly like an SSH session, but without needing an SSH server, client, or key management.
<Note>
The Host Console is a limited-availability operator surface. When it is present on an instance, it requires the **admin** role. [Learn more about licensing](/features/licensing).
Host Console requires the **admin** role. [Learn more about roles](/features/rbac).
</Note>
<Frame>
@@ -27,6 +27,14 @@ The Host Console gives you a real terminal session on the Sencho host, streamed
Click **Console** in the top navigation bar. The session starts immediately in the `COMPOSE_DIR` root. If a stack is selected in the sidebar, the terminal opens directly inside that stack's directory instead, and a small back button appears in the masthead so you can return to the stack editor.
You can also open Console from a bookmark or shared link. Every Console view has a stable URL under [Deep links and URLs](/features/deep-links):
- `/nodes/local/host-console` opens Host Console on the local node at the compose root
- `/nodes/local/host-console/radarr` opens the same terminal rooted in the Radarr stack directory
- Remote nodes use their node slug the same way (for example `/nodes/nas-box-42/host-console`)
Console requires the **admin** role. The URL stays in the address bar on a phone, but Host Console is meant for a desktop browser.
## Cockpit layout
The Console page is a vertical stack of two surfaces.
@@ -74,9 +82,9 @@ Environment variables whose names suggest secrets (passwords, tokens, keys, cred
The Host Console is one of the most powerful features in Sencho and is treated as such:
- **Admin role required when present.** The Host Console is a limited-availability surface; only users with the **admin** role can open a console session.
- **Admin role required.** Only users with the **admin** role can open a console session.
- **Browser sessions only.** Console sessions are only available from a signed-in browser session, not from API tokens.
- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry capturing the user, node, client IP, and timestamp, so shell access is fully accountable.
- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry with the session principal, node, client IP, and timestamp. When the session is opened through a remote hub bridge, the principal is `console_session` and the hub operator is recorded in `acting_as`, so remote shell access stays accountable to the signed-in admin.
<Warning>
The Host Console provides unrestricted shell access to the machine running Sencho. Do not expose Sencho on a public network without HTTPS and strong authentication.
@@ -97,7 +105,7 @@ The Host Console is one of the most powerful features in Sencho and is treated a
</Accordion>
<Accordion title="Session ends immediately after connecting">
This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec <container> which bash` from the host. Also verify that your user has the **admin** role and that Console is available on this instance.
This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec <container> which bash` from the host. Also verify that your user has the **admin** role.
</Accordion>
<Accordion title="Connection drops after a short time">
@@ -105,6 +113,6 @@ The Host Console is one of the most powerful features in Sencho and is treated a
</Accordion>
<Accordion title="Console tab is not visible">
Console appears only when the surface is present on the instance, and only for users with the **admin** role. Verify those conditions are met, then refresh the page.
Console appears for users with the **admin** role. If you are an admin and still do not see it, refresh the page. On a remote node, the Console tab may show a lock card when that node does not support Host Console (for example a Pilot Agent node, or a Distributed API Proxy node that does not advertise Host Console).
</Accordion>
</AccordionGroup>
+1 -1
View File
@@ -236,7 +236,7 @@ Bearer tokens grant full control over the remote Sencho instance. Treat them lik
- **Rotate immediately** if a token is compromised: open the remote instance's **Settings → Infrastructure → Nodes** and click **Generate Token** to mint a new one. The previous token is invalidated instantly.
- Tokens are **encrypted at rest** in the local SQLite database.
- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access always requires a real browser session on that specific instance.
- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access requires a signed-in browser session. From the control plane you can open Host Console on a compatible Distributed API Proxy remote; Pilot Agent remotes do not offer Host Console.
### Transport encryption
+3 -2
View File
@@ -67,7 +67,8 @@ Every Sencho release ships with a static list of capabilities. The current list
| `notifications` | Alert notifications |
| `notification-routing` | Notification routing rules |
| `notification-suppression` | Mute rules |
| `host-console` | Host Console |
| `host-console` | Host Console (legacy advertisement retained for mixed-version fleets) |
| `host-console-community` | Host Console without a paid-license requirement on this node |
| `container-exec` | Container exec terminal |
| `audit-log` | Audit log |
| `scheduled-ops` | Scheduled operations |
@@ -94,7 +95,7 @@ Every Sencho release ships with a static list of capabilities. The current list
</Note>
<Note>
A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection.
A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console` or `host-console-community`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection.
</Note>
A handful of capabilities gate a smaller piece of behavior rather than a whole panel, so a missing one falls back to an older behavior instead of a lock card:
+1 -1
View File
@@ -210,7 +210,7 @@ When you manage nodes running different Sencho versions, the dashboard detects e
### Host console
Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Limited-availability surface when present; admin role required. [Learn more →](/features/host-console)
Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Admin role required. [Learn more →](/features/host-console)
## Security and access

Some files were not shown because too many files have changed in this diff Show More