diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index a019c923..99d72646 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -78,10 +78,10 @@ body: label: Subsystems involved description: Tick any that apply. options: - - label: Trivy / vulnerability scanning - - label: Pilot Agent (NAT tunnel) - - label: Sencho Mesh (cross-node networking) - - label: Fleet Actions / Secrets / Snapshots - - label: Cloud Backup - - label: SSO / LDAP + - label: Stacks, Compose & Deploy + - label: Multi-node, Pilot & Mesh + - label: Fleet (actions, sync, secrets, backups, blueprints) + - label: Alerts, Notifications & Observability + - label: Security, Scanning & Auth (Trivy, SSO, MFA, RBAC) + - label: Updates, Schedules & Automation - label: None of the above diff --git a/backend/package-lock.json b/backend/package-lock.json index 37e09d66..756d7fa9 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -5081,9 +5081,9 @@ "optional": true }, "node_modules/nanoid": { - "version": "3.3.12", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", - "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "version": "3.3.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.16.tgz", + "integrity": "sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==", "dev": true, "funding": [ { @@ -5428,9 +5428,9 @@ } }, "node_modules/postcss": { - "version": "8.5.15", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", - "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "dev": true, "funding": [ { @@ -5448,7 +5448,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, diff --git a/backend/package.json b/backend/package.json index c6bd4607..467de07d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,7 +30,8 @@ "node": ">=26.0.0" }, "overrides": { - "brace-expansion": "^5.0.7" + "brace-expansion": "^5.0.7", + "postcss": "^8.5.23" }, "devDependencies": { "@eslint/js": "^10.0.1", diff --git a/backend/src/__tests__/AutoHealService.evaluate.test.ts b/backend/src/__tests__/AutoHealService.evaluate.test.ts index a6085940..e9e530a8 100644 --- a/backend/src/__tests__/AutoHealService.evaluate.test.ts +++ b/backend/src/__tests__/AutoHealService.evaluate.test.ts @@ -55,7 +55,7 @@ beforeEach(() => { const db = DatabaseService.getInstance(); db.getDb().prepare('DELETE FROM auto_heal_history').run(); db.getDb().prepare('DELETE FROM auto_heal_policies').run(); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/alerts-api.test.ts b/backend/src/__tests__/alerts-api.test.ts index 53a3a600..693da057 100644 --- a/backend/src/__tests__/alerts-api.test.ts +++ b/backend/src/__tests__/alerts-api.test.ts @@ -57,8 +57,8 @@ describe('GET /api/alerts', () => { it('filters alerts by stackName query param', async () => { // Seed two alerts for different stacks const db = DatabaseService.getInstance(); - db.addStackAlert({ stack_name: 'web', metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); - db.addStackAlert({ stack_name: 'api', metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'web', service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, duration_mins: 5, cooldown_mins: 60 }); + db.addStackAlert({ stack_name: 'api', service_name: null, metric: 'memory_percent', operator: '>', threshold: 90, duration_mins: 5, cooldown_mins: 60 }); const res = await request(app) .get('/api/alerts?stackName=web') @@ -105,10 +105,101 @@ describe('POST /api/alerts', () => { expect(res.status).toBe(201); expect(res.body.id).toBeDefined(); expect(res.body.stack_name).toBe('new-stack'); + expect(res.body.service_name).toBeNull(); expect(res.body.metric).toBe('memory_percent'); expect(res.body.threshold).toBe(85); }); + it('persists a valid dotted service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'svc-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBe('api.web'); + }); + + it('normalizes empty service_name to null', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'empty-svc', + service_name: '', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(201); + expect(res.body.service_name).toBeNull(); + }); + + it('rejects whitespace-only service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: ' ', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects reserved _unlabeled service_name', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'bad-svc', + service_name: '_unlabeled', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + }); + + it('rejects service_name when the capability is disabled', async () => { + const { disableCapability, enableCapability, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } = + await import('../services/CapabilityRegistry'); + disableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + try { + const res = await request(app) + .post('/api/alerts') + .set('Cookie', authCookie) + .send({ + stack_name: 'cap-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }); + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + } finally { + enableCapability(SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + } + }); + it('validates required fields and returns 400 for missing data', async () => { const res = await request(app) .post('/api/alerts') @@ -202,6 +293,7 @@ describe('DELETE /api/alerts/:id', () => { // Create an alert to delete const created = DatabaseService.getInstance().addStackAlert({ stack_name: 'delete-me', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -216,6 +308,58 @@ describe('DELETE /api/alerts/:id', () => { expect(res.status).toBe(200); expect(res.body.success).toBe(true); }); + + it('rejects leading-junk ids like 1abc without deleting alert 1', async () => { + const created = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-junk', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 90, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(created.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${created.id}abc`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + expect(DatabaseService.getInstance().getStackAlerts('strict-id-junk').some((a) => a.id === created.id)).toBe(true); + }); + + it('rejects fractional ids like 2.5 without deleting alert 2', async () => { + const first = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + const second = DatabaseService.getInstance().addStackAlert({ + stack_name: 'strict-id-fraction', + service_name: null, + metric: 'memory_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + }); + expect(second.id).toBeDefined(); + + const res = await request(app) + .delete(`/api/alerts/${second.id}.5`) + .set('Cookie', authCookie); + + expect(res.status).toBe(400); + expect(res.body.error).toBe('Invalid alert id'); + const remaining = DatabaseService.getInstance().getStackAlerts('strict-id-fraction').map((a) => a.id); + expect(remaining).toEqual(expect.arrayContaining([first.id, second.id])); + }); }); // --- POST /api/notifications/test --- diff --git a/backend/src/__tests__/api-token-ws-scope.test.ts b/backend/src/__tests__/api-token-ws-scope.test.ts index 9d297b3e..eb61959f 100644 --- a/backend/src/__tests__/api-token-ws-scope.test.ts +++ b/backend/src/__tests__/api-token-ws-scope.test.ts @@ -76,4 +76,8 @@ describe('WebSocket API-token scope enforcement', () => { it('does not scope-block a full-admin token from a generic socket', async () => { expect(await upgradeStatus(createToken('full-admin'), '/ws')).not.toBe(403); }); + + it('blocks a full-admin token from the host console (403)', async () => { + expect(await upgradeStatus(createToken('full-admin'), '/api/system/host-console')).toBe(403); + }); }); diff --git a/backend/src/__tests__/atomic-deploy-hardening.test.ts b/backend/src/__tests__/atomic-deploy-hardening.test.ts index 7814549a..9008e501 100644 --- a/backend/src/__tests__/atomic-deploy-hardening.test.ts +++ b/backend/src/__tests__/atomic-deploy-hardening.test.ts @@ -168,7 +168,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockResolvedValue(undefined); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(200); @@ -186,7 +186,7 @@ describe('Rollback notifications (M-2)', () => { mockTier('paid'); mockDeployStack.mockRejectedValueOnce(new Error('restore failed')); const { NotificationService } = await import('../services/NotificationService'); - const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const spy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const res = await request(app).post('/api/stacks/web/rollback').set('Cookie', authCookie); expect(res.status).toBe(500); diff --git a/backend/src/__tests__/audit-log.test.ts b/backend/src/__tests__/audit-log.test.ts index 2ecfa57b..c983c578 100644 --- a/backend/src/__tests__/audit-log.test.ts +++ b/backend/src/__tests__/audit-log.test.ts @@ -715,7 +715,7 @@ describe('GET /api/audit-log/export', () => { const csvText = res.text; const lines = csvText.split('\n'); - expect(lines[0]).toBe('id,timestamp,username,method,path,status_code,node_id,ip_address,summary'); + expect(lines[0]).toBe('id,timestamp,username,acting_as,method,path,status_code,node_id,ip_address,summary'); expect(lines.length).toBeGreaterThan(1); }); diff --git a/backend/src/__tests__/auth.test.ts b/backend/src/__tests__/auth.test.ts index 412ff3f6..0672ea22 100644 --- a/backend/src/__tests__/auth.test.ts +++ b/backend/src/__tests__/auth.test.ts @@ -115,7 +115,8 @@ describe('POST /api/system/console-token', () => { const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) .post('/api/system/console-token') - .set('Authorization', `Bearer ${token}`); + .set('Authorization', `Bearer ${token}`) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); }); diff --git a/backend/src/__tests__/blueprints-compose-apply.test.ts b/backend/src/__tests__/blueprints-compose-apply.test.ts index 7e8d6b21..8baf72f6 100644 --- a/backend/src/__tests__/blueprints-compose-apply.test.ts +++ b/backend/src/__tests__/blueprints-compose-apply.test.ts @@ -89,6 +89,7 @@ describe('Blueprint compose apply (real filesystem)', () => { const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(composeContent); await expectMissing(path.join(stackDir, 'docker-compose.yml')); + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent); const resolved = await FileSystemService.getInstance(nodeId).getComposeFilename(stackName); expect(resolved).toBe('compose.yaml'); @@ -104,6 +105,10 @@ describe('Blueprint compose apply (real filesystem)', () => { await fsPromises.writeFile(path.join(stackDir, 'compose.yml'), 'services:\n a:\n image: a\n'); await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yaml'), 'services:\n b:\n image: b\n'); await fsPromises.writeFile(path.join(stackDir, 'docker-compose.yml'), 'services:\n c:\n image: c\n'); + await fsPromises.writeFile( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: 2, revision: 2, lastApplied: 1 }, null, 2), + ); const composeContent = 'services:\n app:\n image: redis:7\n'; const markerContent = JSON.stringify({ blueprintId: 2, revision: 3, lastApplied: Date.now() }, null, 2); @@ -131,8 +136,83 @@ describe('Blueprint compose apply (real filesystem)', () => { await expectMissing(path.join(stackDir, 'compose.yml')); await expectMissing(path.join(stackDir, 'docker-compose.yaml')); await expectMissing(path.join(stackDir, 'docker-compose.yml')); + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(markerContent); expect(deploySpy).toHaveBeenCalledTimes(1); }); + + it('does not write a new marker when deploy fails; rolls back a newly created stack', async () => { + const nodeId = seedLocalNode(); + const stackName = `bp-partial-${counter}`; + const composeContent = 'services:\n web:\n image: traefik:v3\n'; + const markerContent = JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: Date.now() }, null, 2); + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + + vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('docker unavailable')); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + composeContent, + markerContent, + '/api/blueprints/test/apply', + ), + ).rejects.toThrow(/docker unavailable/); + + await expectMissing(path.join(stackDir, '.blueprint.json')); + await expectMissing(stackDir); + }); + + it('keeps the prior marker when a re-apply deploy fails', async () => { + const nodeId = seedLocalNode(); + const stackName = `bp-reapply-fail-${counter}`; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + const priorMarker = JSON.stringify({ blueprintId: 8, revision: 2, lastApplied: 1 }, null, 2); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services:\n old:\n image: nginx\n'); + await fsPromises.writeFile(path.join(stackDir, '.blueprint.json'), priorMarker); + + vi.spyOn(ComposeService.prototype, 'deployStack').mockRejectedValue(new Error('deploy blew up')); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + 'services:\n new:\n image: redis:7\n', + JSON.stringify({ blueprintId: 8, revision: 3, lastApplied: Date.now() }, null, 2), + '/api/blueprints/test/apply', + ), + ).rejects.toThrow(/deploy blew up/); + + expect(await fsPromises.readFile(path.join(stackDir, '.blueprint.json'), 'utf-8')).toBe(priorMarker); + expect(await fsPromises.access(stackDir).then(() => true, () => false)).toBe(true); + }); + + it('refuses to overwrite an existing unmanaged stack directory inside the lock', async () => { + const { BlueprintNameConflictError } = await import('../services/BlueprintService'); + const nodeId = seedLocalNode(); + const stackName = `bp-hijack-${counter}`; + const stackDir = path.join(process.env.COMPOSE_DIR!, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + const original = 'services:\n mine:\n image: nginx:alpine\n'; + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), original); + + const deploySpy = vi.spyOn(ComposeService.prototype, 'deployStack').mockResolvedValue(undefined); + + await expect( + BlueprintService.getInstance().applyLocalUnderLock( + nodeId, + stackName, + 'services:\n bp:\n image: redis:7\n', + JSON.stringify({ blueprintId: 99, revision: 1, lastApplied: Date.now() }, null, 2), + '/api/blueprints/test/apply', + ), + ).rejects.toBeInstanceOf(BlueprintNameConflictError); + + expect(await fsPromises.readFile(path.join(stackDir, 'compose.yaml'), 'utf-8')).toBe(original); + await expectMissing(path.join(stackDir, '.blueprint.json')); + expect(deploySpy).not.toHaveBeenCalled(); + }); }); describe('FileSystemService.removeAlternateRootComposeFiles', () => { diff --git a/backend/src/__tests__/blueprints-edge-cases.test.ts b/backend/src/__tests__/blueprints-edge-cases.test.ts index f7beba38..026f2c22 100644 --- a/backend/src/__tests__/blueprints-edge-cases.test.ts +++ b/backend/src/__tests__/blueprints-edge-cases.test.ts @@ -111,9 +111,9 @@ describe('Blueprint route edge cases', () => { describe('Blueprint delete guard', () => { // Rows with no stack of ours on the node must not block delete, AND the route must not run the - // withdraw primitive for them: withdrawFromNode proceeds on a missing marker and would - // down/delete a same-name stack Sencho never owned. A name_conflict is exactly that unmanaged - // stack, so it is excluded even though it carries a last_deployed_at timestamp. + // withdraw primitive for them: there is nothing Sencho owns to remove. A name_conflict is an + // unmanaged same-name stack, so it is excluded even though it may carry a last_deployed_at + // timestamp. When withdraw does run, ownership is enforced under the delete lock. it.each([ { label: 'never-deployed pending review', status: 'pending_state_review' as const, last_deployed_at: null }, { label: 'first-deploy failure', status: 'failed' as const, last_deployed_at: null }, @@ -184,6 +184,34 @@ describe('Blueprint delete guard', () => { expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeUndefined(); }); + it('refuses to delete a stateless blueprint when pre-delete withdraw fails', async () => { + const node = seedNode(); + const bp = seedBlueprint([node.id]); + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'failed', + applied_revision: bp.revision, + last_deployed_at: Date.now(), + last_error: 'Remote node lacks withdraw-local', + }); + const withdrawSpy = vi.spyOn(BlueprintService.getInstance(), 'withdrawFromNode').mockResolvedValue({ + status: 'failed', + error: 'Remote node does not support atomic blueprint withdraw', + }); + + const res = await request(app) + .delete(`/api/blueprints/${bp.id}`) + .set('Cookie', adminCookie); + + expect(res.status).toBe(409); + expect(res.body.code).toBe('withdraw_failed_blocking_delete'); + expect(res.body.nodeId).toBe(node.id); + expect(withdrawSpy).toHaveBeenCalledTimes(1); + expect(DatabaseService.getInstance().getBlueprint(bp.id)).toBeDefined(); + expect(DatabaseService.getInstance().listDeployments(bp.id)).toHaveLength(1); + }); + it('refuses to delete when a pending review still has a deployed stack (revision drift)', async () => { const node = seedNode(); const bp = seedBlueprint([node.id], 'stateful'); @@ -253,38 +281,54 @@ describe('BlueprintService marker edge cases', () => { }); it('refuses to withdraw when the marker belongs to a different blueprint', async () => { + const fs = await import('fs'); + const path = await import('path'); + const composeDir = process.env.COMPOSE_DIR!; const localNode = DatabaseService.getInstance().getNodes()[0]; - const bp = seedBlueprint([localNode.id]); + DatabaseService.getInstance().getDb() + .prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?') + .run(composeDir, localNode.id); + const refreshed = DatabaseService.getInstance().getNode(localNode.id)!; + const bp = seedBlueprint([refreshed.id]); const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; - - vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({ - blueprintId: bp.id + 999, - revision: 1, - lastApplied: 0, + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: refreshed.id, + status: 'active', + applied_revision: bpObj.revision, + last_deployed_at: Date.now(), }); - const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, localNode); + const stackDir = path.join(composeDir, bpObj.name); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: bp.id + 999, revision: 1, lastApplied: 0 }), + ); + + const { DeployedStackDeletionService } = await import('../services/DeployedStackDeletionService'); + const deleteSpy = vi.spyOn(DeployedStackDeletionService.getInstance(), 'deleteDeployedStack'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, refreshed); expect(result.status).toBe('name_conflict'); - // The deployment row must record the conflict, not silently disappear. - const dep = DatabaseService.getInstance().getDeployment(bp.id, localNode.id); + // deleteDeployedStack is invoked but returns name_conflict without mutating. + expect(deleteSpy).toHaveBeenCalled(); + const dep = DatabaseService.getInstance().getDeployment(bp.id, refreshed.id); expect(dep).toBeDefined(); expect(dep?.status).toBe('name_conflict'); + expect(fs.existsSync(path.join(stackDir, 'compose.yaml'))).toBe(true); }); }); describe('BlueprintService developer-mode diagnostics', () => { - // withdrawFromNode emits its "withdraw inputs" diagnostic line before reading the - // marker, so a cross-blueprint marker stub lets us assert the gate without Docker. + // withdrawFromNode emits its "withdraw inputs" diagnostic before the deletion + // service runs. An absent stack directory is enough to exercise logging without Docker. function arrangeWithdraw() { const localNode = DatabaseService.getInstance().getNodes()[0]; const bp = seedBlueprint([localNode.id]); const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; - vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue({ - blueprintId: bp.id + 999, - revision: 1, - lastApplied: 0, - }); return { bpObj, localNode }; } diff --git a/backend/src/__tests__/blueprints-preview-apply.test.ts b/backend/src/__tests__/blueprints-preview-apply.test.ts index d466b367..6a084973 100644 --- a/backend/src/__tests__/blueprints-preview-apply.test.ts +++ b/backend/src/__tests__/blueprints-preview-apply.test.ts @@ -326,6 +326,7 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => { const conflict = await BlueprintService.getInstance().hasNameConflict( created.body.name as string, DatabaseService.getInstance().getNode(node.id)!, + created.body.id as number, ); expect(conflict).toBe(true); @@ -342,6 +343,41 @@ describe('POST /api/blueprints/:id/apply confirm binding', () => { .toBe('services:\n app:\n image: nginx\n'); }); + it('hasNameConflict is false for a matching marker and true for a foreign marker', async () => { + const node = seedNode(); + counter += 1; + const created = await request(app) + .post('/api/blueprints') + .set('Cookie', adminCookie) + .send(validBlueprintBody(node.id)); + expect(created.status).toBe(201); + + const composeDir = process.env.COMPOSE_DIR!; + DatabaseService.getInstance().getDb() + .prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?') + .run(composeDir, node.id); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const stackName = created.body.name as string; + const blueprintId = created.body.id as number; + const stackDir = path.join(composeDir, stackName); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync(path.join(stackDir, 'compose.yaml'), 'services:\n app:\n image: nginx\n'); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId, revision: 1, lastApplied: 1 }), + ); + + const { BlueprintService } = await import('../services/BlueprintService'); + const svc = BlueprintService.getInstance(); + expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(false); + + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: blueprintId + 99, revision: 1, lastApplied: 1 }), + ); + expect(await svc.hasNameConflict(stackName, nodeObj, blueprintId)).toBe(true); + }); + it('blocks rename while a non-withdrawn deployment exists', async () => { const node = seedNode(); counter += 1; diff --git a/backend/src/__tests__/blueprints-remote-deploy.test.ts b/backend/src/__tests__/blueprints-remote-deploy.test.ts index 2ce5ad3c..9bf91e72 100644 --- a/backend/src/__tests__/blueprints-remote-deploy.test.ts +++ b/backend/src/__tests__/blueprints-remote-deploy.test.ts @@ -101,7 +101,7 @@ describe('BlueprintService remote deploy', () => { expect(dep?.applied_revision).toBe(bpObj.revision); }); - it('falls back to the legacy create/write/deploy flow when the remote lacks apply-local (404)', async () => { + it('fails closed when the remote lacks apply-local (404); no legacy mutations', async () => { const node = seedRemoteNode(); const bp = seedBlueprint([node.id]); const nodeObj = DatabaseService.getInstance().getNode(node.id)!; @@ -110,25 +110,17 @@ describe('BlueprintService remote deploy', () => { vi.spyOn(axios, 'get').mockResolvedValue({ status: 200, data: [] }); const putSpy = vi.spyOn(axios, 'put').mockResolvedValue({ status: 200, data: {} }); const postSpy = vi.spyOn(axios, 'post') - .mockResolvedValueOnce({ status: 404, data: {} }) // apply-local missing on older node - .mockResolvedValueOnce({ status: 201, data: {} }) // legacy create stack - .mockResolvedValueOnce({ status: 200, data: {} }); // legacy deploy + .mockResolvedValueOnce({ status: 404, data: {} }); const result = await BlueprintService.getInstance().deployToNode(bpObj, nodeObj); - expect(result.status).toBe('active'); + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/apply-local|Upgrade/i); + expect(postSpy).toHaveBeenCalledTimes(1); expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/apply-local$/); - expect(postSpy.mock.calls[1][0]).toMatch(/\/api\/stacks$/); - expect(putSpy).toHaveBeenCalledTimes(2); // compose + marker - const composePutUrl = String(putSpy.mock.calls[0][0]); - const markerPutUrl = String(putSpy.mock.calls[1][0]); - expect(composePutUrl).toMatch(/[?&]path=compose\.yaml(?:&|$)/); - expect(markerPutUrl).toMatch(/[?&]path=\.blueprint\.json(?:&|$)/); - expect(postSpy.mock.calls[2][0]).toMatch(/\/deploy$/); - const [composePutOrder, markerPutOrder] = putSpy.mock.invocationCallOrder; - const deployOrder = postSpy.mock.invocationCallOrder[2]; - expect(composePutOrder).toBeLessThan(markerPutOrder); - expect(markerPutOrder).toBeLessThan(deployOrder); + expect(putSpy).not.toHaveBeenCalled(); + const dep = DatabaseService.getInstance().getDeployment(bp.id, node.id); + expect(dep?.status).toBe('failed'); }); it('maps a remote apply lock-conflict (409) to status=failed', async () => { @@ -192,7 +184,7 @@ describe('BlueprintService remote deploy', () => { expect(dep?.status).toBe('name_conflict'); }); - it('withdraws a remote deployment by deleting the stack and removing the row', async () => { + it('withdraws a remote deployment via withdraw-local and removes the row', async () => { const node = seedRemoteNode(); const bp = seedBlueprint([node.id]); const nodeObj = DatabaseService.getInstance().getNode(node.id)!; @@ -204,14 +196,107 @@ describe('BlueprintService remote deploy', () => { applied_revision: bpObj.revision, }); - vi.spyOn(axios, 'get').mockResolvedValue({ status: 404, data: {} }); // readMarker → null → proceed - vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: {} }); // remote down (best-effort) - const delSpy = vi.spyOn(axios, 'delete').mockResolvedValue({ status: 200, data: {} }); + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ + status: 200, + data: { status: 'withdrawn' }, + }); + const delSpy = vi.spyOn(axios, 'delete'); const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); expect(result.status).toBe('withdrawn'); - expect(delSpy.mock.calls[0][0]).toMatch(/\/api\/stacks\//); + expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/); + expect(delSpy).not.toHaveBeenCalled(); expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)).toBeUndefined(); }); + + it('maps remote withdraw-local lock conflict to failed without fallback delete', async () => { + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + vi.spyOn(axios, 'post').mockResolvedValue({ + status: 409, + data: { + error: `${bpObj.name} is busy: another operation (update) is already in progress`, + code: 'stack_op_in_progress', + }, + }); + const delSpy = vi.spyOn(axios, 'delete'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/already in progress/); + expect(delSpy).not.toHaveBeenCalled(); + expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed'); + }); + + it('fails closed when the remote lacks withdraw-local (404); no legacy delete', async () => { + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = DatabaseService.getInstance().getNode(node.id)!; + const bpObj = DatabaseService.getInstance().getBlueprint(bp.id)!; + DatabaseService.getInstance().upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + const postSpy = vi.spyOn(axios, 'post').mockResolvedValue({ status: 404, data: { error: 'Not Found' } }); + const delSpy = vi.spyOn(axios, 'delete'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('failed'); + expect(result.error).toMatch(/withdraw-local|Upgrade/i); + expect(postSpy.mock.calls[0][0]).toMatch(/\/api\/blueprints\/withdraw-local$/); + expect(delSpy).not.toHaveBeenCalled(); + expect(DatabaseService.getInstance().getDeployment(bp.id, node.id)?.status).toBe('failed'); + }); + + it('does not clear hub role assignments when withdrawing a remote deployment', async () => { + const bcrypt = await import('bcrypt'); + const db = DatabaseService.getInstance(); + const node = seedRemoteNode(); + const bp = seedBlueprint([node.id]); + const nodeObj = db.getNode(node.id)!; + const bpObj = db.getBlueprint(bp.id)!; + db.upsertDeployment({ + blueprint_id: bp.id, + node_id: node.id, + status: 'active', + applied_revision: bpObj.revision, + }); + + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ + username: `remote-wd-rbac-${counter}`, password_hash: hash, role: 'viewer', + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bpObj.name, + }); + + vi.spyOn(axios, 'post').mockResolvedValue({ status: 200, data: { status: 'withdrawn' } }); + const delSpy = vi.spyOn(axios, 'delete'); + const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource'); + + const result = await BlueprintService.getInstance().withdrawFromNode(bpObj, nodeObj); + + expect(result.status).toBe('withdrawn'); + expect(delSpy).not.toHaveBeenCalled(); + expect(rbacSpy).not.toHaveBeenCalled(); + expect(db.getAllRoleAssignments(userId) + .some((a) => a.resource_type === 'stack' && a.resource_id === bpObj.name)).toBe(true); + + db.deleteUser(userId); + }); }); diff --git a/backend/src/__tests__/blueprints-route-validation.test.ts b/backend/src/__tests__/blueprints-route-validation.test.ts index ce4de5dd..6848ff93 100644 --- a/backend/src/__tests__/blueprints-route-validation.test.ts +++ b/backend/src/__tests__/blueprints-route-validation.test.ts @@ -129,3 +129,71 @@ describe('POST /api/blueprints/apply-local (node-to-node atomic apply)', () => { expect(StackOpLockService.getInstance().get(1, 'apply-local-busy')?.action).toBe('update'); }); }); + +describe('POST /api/blueprints/withdraw-local', () => { + let viewerCookie: string; + + beforeAll(async () => { + const bcrypt = (await import('bcrypt')).default; + const passwordHash = await bcrypt.hash('bp-wd-viewer-pass', 1); + DatabaseService.getInstance().addUser({ + username: 'bp-wd-viewer', + password_hash: passwordHash, + role: 'viewer', + }); + const res = await request(app) + .post('/api/auth/login') + .send({ username: 'bp-wd-viewer', password: 'bp-wd-viewer-pass' }); + const cookies = res.headers['set-cookie'] as string | string[]; + viewerCookie = Array.isArray(cookies) ? cookies[0] : cookies; + }); + + it('rejects an invalid stack name', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: '../escape', blueprintId: 1 }); + expect(res.status).toBe(400); + expect(res.body.error).toContain('Invalid stack name'); + }); + + it('rejects a non-positive blueprintId', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: 'wd-local-stack', blueprintId: 0 }); + expect(res.status).toBe(400); + expect(res.body.error).toMatch(/blueprintId/i); + }); + + it('returns 403 for a viewer without stack:delete', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', viewerCookie) + .send({ stackName: 'wd-local-stack', blueprintId: 1 }); + expect(res.status).toBe(403); + }); + + it('returns 409 self_stack_protected for Sencho own stack', async () => { + const selfStackGuard = await import('../helpers/selfStackGuard'); + vi.spyOn(selfStackGuard, 'refuseIfSelfStack').mockImplementation(async (_req, res) => { + res.status(409).json({ error: 'self', code: 'self_stack_protected' }); + return true; + }); + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: 'sencho-self', blueprintId: 1 }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('self_stack_protected'); + }); + + it('returns already_absent when the stack directory is missing', async () => { + const res = await request(app) + .post('/api/blueprints/withdraw-local') + .set('Cookie', adminCookie) + .send({ stackName: `wd-absent-${Date.now()}`, blueprintId: 42 }); + expect(res.status).toBe(200); + expect(res.body.status).toBe('already_absent'); + }); +}); diff --git a/backend/src/__tests__/blueprints.test.ts b/backend/src/__tests__/blueprints.test.ts index c9f91c28..261a0e22 100644 --- a/backend/src/__tests__/blueprints.test.ts +++ b/backend/src/__tests__/blueprints.test.ts @@ -13,6 +13,7 @@ * lifecycle in the plan. */ import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import bcrypt from 'bcrypt'; import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; import type { Blueprint, Node } from '../services/DatabaseService'; import type { ReconcileDecision } from '../services/BlueprintReconciler'; @@ -40,6 +41,7 @@ afterAll(() => cleanupTestDb(tmpDir)); beforeEach(() => { const db = DatabaseService.getInstance().getDb(); + db.prepare('DELETE FROM role_assignments').run(); db.prepare('DELETE FROM blueprint_deployments').run(); db.prepare('DELETE FROM blueprints').run(); db.prepare('DELETE FROM node_labels').run(); @@ -397,7 +399,7 @@ describe('BlueprintReconciler developer-mode diagnostics', () => { }); describe('BlueprintService per-stack lock', () => { - it('deploy under a free lock writes compose then marker, then deploys', async () => { + it('deploy under a free lock writes compose, cleans siblings, deploys, then writes the marker', async () => { const nodeId = seedNode(); const bp = seedBlueprint({ classification: 'stateless', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -417,7 +419,7 @@ describe('BlueprintService per-stack lock', () => { source: 'blueprint', actor: 'system:blueprint', }); - // Compose is written first, then the marker, both before sibling cleanup and deploy. + // Compose is written first; the marker is deferred until after sibling cleanup and deploy. expect(writeSpy).toHaveBeenCalledTimes(2); expect(writeSpy.mock.calls[0][1]).toBe('compose.yaml'); expect(writeSpy.mock.calls[0][2]).toBe(bp.compose_content); @@ -427,9 +429,9 @@ describe('BlueprintService per-stack lock', () => { const [composeOrder, markerOrder] = writeSpy.mock.invocationCallOrder; const [cleanupOrder] = cleanupSpy.mock.invocationCallOrder; const [deployOrder] = deploySpy.mock.invocationCallOrder; - expect(composeOrder).toBeLessThan(markerOrder); - expect(markerOrder).toBeLessThan(cleanupOrder); + expect(composeOrder).toBeLessThan(cleanupOrder); expect(cleanupOrder).toBeLessThan(deployOrder); + expect(deployOrder).toBeLessThan(markerOrder); }); it('deploy skips, writes no stack files, and records failed when the stack lock is held', async () => { @@ -466,6 +468,120 @@ describe('BlueprintService per-stack lock', () => { }); }); +describe('BlueprintService local withdraw clears stack-scoped role assignments', () => { + function hasAssignment(userId: number, resourceType: string, resourceId: string): boolean { + return DatabaseService.getInstance().getAllRoleAssignments(userId) + .some((a) => a.resource_type === resourceType && a.resource_id === resourceId); + } + + async function arrangeLocalWithdraw() { + const db = DatabaseService.getInstance(); + const composeDir = process.env.COMPOSE_DIR!; + const nodeId = seedNode(); + db.getDb().prepare('UPDATE nodes SET compose_dir = ? WHERE id = ?').run(composeDir, nodeId); + + const bp = seedBlueprint({ name: `rbac-wd-${counter}`, classification: 'stateless', nodeIds: [nodeId] }); + const node = db.getNode(nodeId)!; + db.upsertDeployment({ + blueprint_id: bp.id, + node_id: nodeId, + status: 'active', + applied_revision: bp.revision, + }); + + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: `bp-rbac-${counter}`, password_hash: hash, role: 'viewer' }); + const otherNodeId = seedNode(); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: bp.name, + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack', + }); + db.addRoleAssignment({ + user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId), + }); + + // Matching on-disk marker so ownership passes; FS/down are stubbed. + const fs = await import('fs'); + const path = await import('path'); + const stackDir = path.join(composeDir, bp.name); + fs.mkdirSync(stackDir, { recursive: true }); + fs.writeFileSync( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: bp.id, revision: bp.revision, lastApplied: Date.now() }), + ); + const { ComposeService } = await import('../services/ComposeService'); + const { FileSystemService } = await import('../services/FileSystemService'); + vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined); + const deleteStackSpy = vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined); + + return { bp, node, nodeId, userId, deleteStackSpy, db }; + } + + it('clears the target assignment after successful filesystem deletion and preserves unrelated grants', async () => { + const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('withdrawn'); + expect(deleteStackSpy).toHaveBeenCalledWith(bp.name); + expect(db.getDeployment(bp.id, node.id)).toBeUndefined(); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(false); + expect(hasAssignment(userId, 'stack', 'other-stack')).toBe(true); + expect(db.getAllRoleAssignments(userId).some((a) => a.resource_type === 'node')).toBe(true); + db.deleteUser(userId); + }); + + it('clears the target assignment when the stack directory is already absent', async () => { + const { bp, node, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + const fs = await import('fs'); + const path = await import('path'); + fs.rmSync(path.join(process.env.COMPOSE_DIR!, bp.name), { recursive: true, force: true }); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('withdrawn'); + expect(deleteStackSpy).not.toHaveBeenCalled(); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(false); + expect(db.getAllRoleAssignments(userId)).toHaveLength(2); + db.deleteUser(userId); + }); + + it('preserves the grant when filesystem deletion fails with a non-ENOENT error', async () => { + const { bp, node, nodeId, userId, deleteStackSpy, db } = await arrangeLocalWithdraw(); + const fsErr = Object.assign(new Error('permission denied'), { code: 'EACCES' }); + deleteStackSpy.mockRejectedValue(fsErr); + const rbacSpy = vi.spyOn(db, 'deleteRoleAssignmentsByResource'); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('failed'); + expect(rbacSpy).not.toHaveBeenCalled(); + expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed'); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(true); + db.deleteUser(userId); + }); + + it('fails withdraw and keeps the deployment when role-assignment cleanup throws', async () => { + const { bp, node, nodeId, userId, db } = await arrangeLocalWithdraw(); + vi.spyOn(db, 'deleteRoleAssignmentsByResource') + .mockImplementation(() => { throw new Error('simulated rbac cleanup failure'); }); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + + const outcome = await BlueprintService.getInstance().withdrawFromNode(bp, node); + + expect(outcome.status).toBe('failed'); + expect(db.getDeployment(bp.id, nodeId)).toBeDefined(); + expect(db.getDeployment(bp.id, nodeId)?.status).toBe('failed'); + expect(errorSpy.mock.calls.some((args) => + typeof args[0] === 'string' && args[0].includes('Secondary DB cleanup failed'), + )).toBe(true); + expect(hasAssignment(userId, 'stack', bp.name)).toBe(true); + db.deleteUser(userId); + }); +}); + describe('BlueprintService marker parsing + name-conflict guard', () => { it('parseMarker accepts a well-formed marker', () => { const marker = BlueprintService.parseMarker(JSON.stringify({ blueprintId: 7, revision: 3, lastApplied: 12345 })); @@ -496,7 +612,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode local drift uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedNode(); const bp = seedBlueprint({ name: 'drift-local', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -514,7 +630,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('suggest-mode remote drift keeps the authoritative node name', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); const nodeId = seedRemoteNode('sencho-test-02'); const bp = seedBlueprint({ name: 'drift-remote', drift_mode: 'suggest', nodeIds: [nodeId] }); const node = DatabaseService.getInstance().getNode(nodeId)!; @@ -532,7 +648,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('stateful marker-loss on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'readMarker').mockResolvedValue(null); const nodeId = seedNode(); const bp = seedBlueprint({ @@ -556,7 +672,7 @@ describe('BlueprintReconciler drift alert node wording', () => { it('enforce correction-failure on local uses on this node', async () => { const { NotificationService } = await import('../services/NotificationService'); - const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + const dispatchSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); vi.spyOn(BlueprintService.getInstance(), 'deployToNode').mockResolvedValue({ status: 'failed', error: 'compose up failed', diff --git a/backend/src/__tests__/capability-registry-pilot.test.ts b/backend/src/__tests__/capability-registry-pilot.test.ts index b60ffb68..04231a62 100644 --- a/backend/src/__tests__/capability-registry-pilot.test.ts +++ b/backend/src/__tests__/capability-registry-pilot.test.ts @@ -70,15 +70,18 @@ describe('fetchRemoteMeta Authorization header', () => { describe('applyPilotModeCapabilityFilter', () => { afterEach(() => { enableCapability('host-console'); + enableCapability('host-console-community'); }); - it('removes host-console from active capabilities', () => { + it('removes host-console and host-console-community from active capabilities', () => { expect(CAPABILITIES).toContain('host-console'); + expect(CAPABILITIES).toContain('host-console-community'); applyPilotModeCapabilityFilter(); const active = getActiveCapabilities(); expect(active).not.toContain('host-console'); + expect(active).not.toContain('host-console-community'); expect(active).toContain('stacks'); }); @@ -97,6 +100,7 @@ describe('applyPilotModeCapabilityFilter', () => { const active = getActiveCapabilities(); expect(active).not.toContain('host-console'); - expect(active.length).toBe(CAPABILITIES.length - 1); + expect(active).not.toContain('host-console-community'); + expect(active.length).toBe(CAPABILITIES.length - 2); }); }); diff --git a/backend/src/__tests__/compose-network-inspector.test.ts b/backend/src/__tests__/compose-network-inspector.test.ts index d8d0d062..035d1599 100644 --- a/backend/src/__tests__/compose-network-inspector.test.ts +++ b/backend/src/__tests__/compose-network-inspector.test.ts @@ -24,7 +24,7 @@ function effSvc(over: Partial = {}): EffService { function container(over: Partial = {}): DependencyContainer { return { id: 'c1', name: 'web1', service: 'web', composeProject: 'myapp', stack: 'myapp', - state: 'running', image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, + state: 'running', exitCode: null, image: 'nginx:1.27', networks: [], volumes: [], ports: [], ...over, }; } diff --git a/backend/src/__tests__/database-metrics.test.ts b/backend/src/__tests__/database-metrics.test.ts index 522e0d87..0e4b37c5 100644 --- a/backend/src/__tests__/database-metrics.test.ts +++ b/backend/src/__tests__/database-metrics.test.ts @@ -324,6 +324,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('adds and retrieves stack alerts', () => { db.addStackAlert({ stack_name: 'alert-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -339,11 +340,27 @@ describe('DatabaseService - stack alerts CRUD', () => { expect(found.threshold).toBe(80); expect(found.duration_mins).toBe(5); expect(found.cooldown_mins).toBe(15); + expect(found.service_name).toBeNull(); + }); + + it('persists a named service_name', () => { + const created = db.addStackAlert({ + stack_name: 'scoped-stack', + service_name: 'api.web', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 15, + }); + expect(created.service_name).toBe('api.web'); + expect(db.getStackAlerts('scoped-stack')[0].service_name).toBe('api.web'); }); it('filters alerts by stack name', () => { db.addStackAlert({ stack_name: 'filter-stack-a', + service_name: null, metric: 'memory_mb', operator: '>=', threshold: 512, @@ -352,6 +369,7 @@ describe('DatabaseService - stack alerts CRUD', () => { }); db.addStackAlert({ stack_name: 'filter-stack-b', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 90, @@ -371,6 +389,7 @@ describe('DatabaseService - stack alerts CRUD', () => { it('updates last_fired_at timestamp', () => { db.addStackAlert({ stack_name: 'fired-stack', + service_name: null, metric: 'net_rx', operator: '>', threshold: 100, @@ -381,29 +400,55 @@ describe('DatabaseService - stack alerts CRUD', () => { const alerts = db.getStackAlerts('fired-stack'); const alert = alerts[0]; const fireTime = Date.now(); - db.updateStackAlertLastFired(alert.id, fireTime); + db.updateStackAlertLastFired(alert.id!, fireTime); const updated = db.getStackAlerts('fired-stack'); expect(updated[0].last_fired_at).toBe(fireTime); }); - it('deletes an alert by id', () => { - db.addStackAlert({ + it('upserts and reads distinct per-service cooldowns', () => { + const alert = db.addStackAlert({ + stack_name: 'cooldown-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 1, + cooldown_mins: 10, + }); + const id = alert.id!; + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + + db.upsertStackAlertServiceCooldown(id, 'api', 1000); + db.upsertStackAlertServiceCooldown(id, 'database', 2000); + + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(1000); + expect(db.getStackAlertServiceCooldown(id, 'database')).toBe(2000); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); + + db.upsertStackAlertServiceCooldown(id, 'api', 3000); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBe(3000); + }); + + it('deletes an alert by id and removes child cooldown rows', () => { + const alert = db.addStackAlert({ stack_name: 'delete-stack', + service_name: null, metric: 'memory_percent', operator: '>', threshold: 95, duration_mins: 1, cooldown_mins: 5, }); + const id = alert.id!; + db.upsertStackAlertServiceCooldown(id, 'api', Date.now()); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(true); - const before = db.getStackAlerts('delete-stack'); - expect(before.length).toBe(1); + db.deleteStackAlert(id); - db.deleteStackAlert(before[0].id); - - const after = db.getStackAlerts('delete-stack'); - expect(after.length).toBe(0); + expect(db.getStackAlerts('delete-stack').length).toBe(0); + expect(db.hasAnyStackAlertServiceCooldown(id)).toBe(false); + expect(db.getStackAlertServiceCooldown(id, 'api')).toBeNull(); }); }); diff --git a/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts b/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts new file mode 100644 index 00000000..9723050b --- /dev/null +++ b/backend/src/__tests__/database-stack-cleanup-intent-migration.test.ts @@ -0,0 +1,64 @@ +/** + * Legacy installations upgrade stack_update_cleanup_pending with nullable + * required_blueprint_id via maybeAddCol. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { setupTestDb, cleanupTestDb } from './helpers/setupTestDb'; + +let tmpDir: string; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +describe('stack_update_cleanup_pending required_blueprint_id migration', () => { + it('upgrades a legacy table without the column and preserves null for old rows', () => { + const db = DatabaseService.getInstance(); + const raw = db.getDb(); + + // Simulate a pre-migration installation: drop the column if present, then + // re-add it the same way initSchema's maybeAddCol does. + try { + raw.exec('ALTER TABLE stack_update_cleanup_pending DROP COLUMN required_blueprint_id'); + } catch { + // Column may already be absent in a hand-built fixture. + } + + const now = Date.now(); + raw.prepare(` + INSERT INTO stack_update_cleanup_pending ( + id, node_id, stack_name, status, target_kind, rollback_tags_json, + override_paths_json, prune_volumes_requested, created_at, updated_at + ) VALUES (?, ?, ?, 'prepared', 'local_socket', '[]', '[]', 0, ?, ?) + `).run('legacy-1', 1, 'legacy-stack', now, now); + + try { + raw.exec('ALTER TABLE stack_update_cleanup_pending ADD COLUMN required_blueprint_id INTEGER'); + } catch { + // Idempotent if a parallel path re-added it. + } + + const legacy = db.getCleanupPending('legacy-1'); + expect(legacy).toBeDefined(); + expect(legacy?.required_blueprint_id ?? null).toBeNull(); + + db.insertCleanupPending({ + id: 'owned-1', + node_id: 2, + stack_name: 'bp-stack', + status: 'prepared', + target_kind: 'local_socket', + rollback_tags_json: '[]', + override_paths_json: '[]', + prune_volumes_requested: 0, + required_blueprint_id: 42, + created_at: now, + updated_at: now, + }); + expect(db.getCleanupPending('owned-1')?.required_blueprint_id).toBe(42); + }); +}); diff --git a/backend/src/__tests__/dependency-graph-builder.test.ts b/backend/src/__tests__/dependency-graph-builder.test.ts index 7ec5982e..3bc0f3a7 100644 --- a/backend/src/__tests__/dependency-graph-builder.test.ts +++ b/backend/src/__tests__/dependency-graph-builder.test.ts @@ -32,7 +32,7 @@ function snap(partial: Partial): DependencySnapshot { function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: null, - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } diff --git a/backend/src/__tests__/deployed-stack-deletion-service.test.ts b/backend/src/__tests__/deployed-stack-deletion-service.test.ts index 27052fd8..d8dfc0b4 100644 --- a/backend/src/__tests__/deployed-stack-deletion-service.test.ts +++ b/backend/src/__tests__/deployed-stack-deletion-service.test.ts @@ -89,6 +89,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }; @@ -111,6 +112,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); @@ -129,6 +131,7 @@ describe('DeployedStackDeletionService ready transaction', () => { rollback_tags_json: '[]', override_paths_json: '[]', prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); @@ -154,3 +157,39 @@ describe('overrideDeletionContainmentBase', () => { }); }); +describe('DeployedStackDeletionService blueprint ownership probe', () => { + it('returns failed (not name_conflict) when marker read fails with non-ENOENT I/O', async () => { + const { promises: fsPromises } = await import('fs'); + const { vi } = await import('vitest'); + const composeDir = process.env.COMPOSE_DIR!; + const stackName = `del-probe-${Date.now()}`; + const stackDir = path.join(composeDir, stackName); + await fsPromises.mkdir(stackDir, { recursive: true }); + await fsPromises.writeFile(path.join(stackDir, 'compose.yaml'), 'services: {}\n'); + await fsPromises.writeFile( + path.join(stackDir, '.blueprint.json'), + JSON.stringify({ blueprintId: 7, revision: 1, lastApplied: 0 }), + ); + + const accessErr = Object.assign(new Error('EACCES'), { code: 'EACCES' }); + const readSpy = vi.spyOn(fsPromises, 'readFile').mockRejectedValueOnce(accessErr); + + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: NODE, + stackName, + pruneVolumes: false, + actor: 'test', + requireBlueprintId: 7, + }); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.code).toBe('failed'); + expect(result.error).toMatch(/EACCES|Failed to read|permission/i); + } + expect(readSpy).toHaveBeenCalled(); + readSpy.mockRestore(); + await fsPromises.rm(stackDir, { recursive: true, force: true }); + }); +}); + diff --git a/backend/src/__tests__/docker-controller.test.ts b/backend/src/__tests__/docker-controller.test.ts index 30cc4e99..e55e7082 100644 --- a/backend/src/__tests__/docker-controller.test.ts +++ b/backend/src/__tests__/docker-controller.test.ts @@ -1316,6 +1316,7 @@ describe('DockerController - getDependencySnapshot', () => { expect(c.networks).toEqual([{ name: 'web_frontend', id: 'net1', ip: '172.18.0.2' }]); expect(c.volumes).toEqual(['web_data']); // bind mount dropped expect(c.ports).toEqual([{ ip: '0.0.0.0', publishedPort: 8080, privatePort: 80, protocol: 'tcp' }]); // unpublished 9090 dropped + expect(c.exitCode).toBeNull(); expect(snap.networks.find((n) => n.name === 'bridge')?.isSystem).toBe(true); const frontend = snap.networks.find((n) => n.name === 'web_frontend'); @@ -1338,6 +1339,23 @@ describe('DockerController - getDependencySnapshot', () => { expect(snap.containers[0].stack).toBeNull(); expect(snap.containers[0].composeProject).toBeNull(); }); + + it('parses exitCode from list Status for exited containers', async () => { + mockDocker.listContainers.mockResolvedValue([ + { Id: 'a', Names: ['/job-0'], Image: 'busybox', State: 'exited', Status: 'Exited (0) 5 minutes ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'b', Names: ['/crash-0'], Image: 'busybox', State: 'exited', Status: 'Exited (137) 1 minute ago', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'c', Names: ['/up-0'], Image: 'busybox', State: 'running', Status: 'Up 3 hours', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + { Id: 'd', Names: ['/odd-0'], Image: 'busybox', State: 'exited', Status: 'Exited', Labels: {}, NetworkSettings: { Networks: {} }, Mounts: [], Ports: [] }, + ]); + mockDocker.listNetworks.mockResolvedValue([]); + mockDocker.listVolumes.mockResolvedValue({ Volumes: [] }); + + const snap = await DockerController.getInstance(1).getDependencySnapshot([]); + expect(snap.containers.find(c => c.id === 'a')?.exitCode).toBe(0); + expect(snap.containers.find(c => c.id === 'b')?.exitCode).toBe(137); + expect(snap.containers.find(c => c.id === 'c')?.exitCode).toBeNull(); + expect(snap.containers.find(c => c.id === 'd')?.exitCode).toBeNull(); + }); }); // --- getBulkStackStatuses uptime (runningSince from StartedAt) ----------------- diff --git a/backend/src/__tests__/docker-event-service.test.ts b/backend/src/__tests__/docker-event-service.test.ts index 2fb418e3..6e9f7c53 100644 --- a/backend/src/__tests__/docker-event-service.test.ts +++ b/backend/src/__tests__/docker-event-service.test.ts @@ -27,7 +27,7 @@ const { mockGetDocker, mockIsOwnContainer, } = vi.hoisted(() => ({ - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockBroadcastEvent: vi.fn(), mockGetGlobalSettings: vi.fn().mockReturnValue({ global_crash: '1' }), mockGetEvents: vi.fn(), @@ -97,6 +97,8 @@ let service: DockerEventService; beforeEach(() => { vi.clearAllMocks(); vi.useFakeTimers(); + mockDispatchAlert.mockReset(); + mockDispatchAlert.mockResolvedValue({ persisted: true }); mockGetGlobalSettings.mockReturnValue({ global_crash: '1' }); mockIsOwnContainer.mockReset(); mockIsOwnContainer.mockReturnValue(false); @@ -119,6 +121,11 @@ afterEach(() => { vi.useRealTimers(); }); +async function flushMicrotasks(): Promise { + await Promise.resolve(); + await Promise.resolve(); +} + // ── Classification via event stream ──────────────────────────────────── describe('DockerEventService - die classification', () => { @@ -367,6 +374,7 @@ describe('DockerEventService - die classification', () => { }, }, }); + await flushMicrotasks(); expect(mockDispatchAlert).toHaveBeenCalledWith( 'error', @@ -442,9 +450,586 @@ describe('DockerEventService - rate limiting', () => { // After the rate window, a summary warning fires. await vi.advanceTimersByTimeAsync(61_000); const summaryCalls = mockDispatchAlert.mock.calls.filter(c => - typeof c[2] === 'string' && c[2].includes('additional containers crashed')); + typeof c[2] === 'string' && c[2].includes('rate-limited')); expect(summaryCalls).toHaveLength(1); - expect(summaryCalls[0][2]).toContain('2 additional'); + expect(summaryCalls[0][2]).toBe( + '2 additional container crash alerts were rate-limited in the last minute.', + ); + }); +}); + +// ── Health alert transition / dedup / rate sharing ───────────────────── + +describe('DockerEventService - health alert gates', () => { + it('emits once for duplicate unhealthy while already unhealthy and keeps unhealthySince', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { + ID: 'h1', + Attributes: { name: 'api', 'com.docker.compose.project': 'web' }, + }, + }); + await flushMicrotasks(); + const first = service.getContainerState('h1'); + expect(first?.healthStatus).toBe('unhealthy'); + expect(first?.unhealthySince).toBeTypeOf('number'); + const since = first!.unhealthySince!; + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h1', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(service.getContainerState('h1')?.unhealthySince).toBe(since); + expect(service.getContainerState('h1')?.healthStatus).toBe('unhealthy'); + }); + + it('suppresses a second flap while the first health dispatch is still pending', async () => { + let resolveFirst!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementationOnce( + () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + resolveFirst({ persisted: true }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + }); + + it('retries a deferred flap when the in-flight dispatch fails to persist', async () => { + let resolveFirst!: (value: { persisted: boolean }) => void; + mockDispatchAlert + .mockImplementationOnce( + () => new Promise<{ persisted: boolean }>((resolve) => { resolveFirst = resolve; }), + ) + .mockResolvedValue({ persisted: true }); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h2b', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + resolveFirst({ persisted: false }); + await flushMicrotasks(); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not advance health dedup when persisted is false, allowing a later transition', async () => { + mockDispatchAlert + .mockResolvedValueOnce({ persisted: false }) + .mockResolvedValue({ persisted: true }); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h3', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('keeps health dedup across the 10-minute prune window and re-emits after 60 minutes', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Recover so lastActivity ages without fresh unhealthy events; prune + // may drop containerState after 10m but health dedup must survive. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + await vi.advanceTimersByTimeAsync(11 * 60_000); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(50 * 60_000); + + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h4', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not let crash and health dedup suppress one another', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: 'h5', Attributes: { exitCode: '1', name: 'api' } }, + }); + await vi.advanceTimersByTimeAsync(600); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'start', + Actor: { ID: 'h5', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h5', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + expect(mockDispatchAlert.mock.calls[1][2]).toContain('Healthcheck failed'); + }); + + it('does not consume rate tokens for deduped duplicate health events', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + for (let i = 0; i < 5; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h6', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + } + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + for (let i = 0; i < 19; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `crash-${i}`, Attributes: { exitCode: '1', name: `n-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + const crashCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('Crash')); + expect(crashCalls).toHaveLength(19); + }); + + it('emits an accurate health-only rate-limit roll-up', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + for (let i = 0; i < 22; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `hh-${i}`, Attributes: { name: `svc-${i}` } }, + }); + await flushMicrotasks(); + } + + const healthCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('Healthcheck failed')); + expect(healthCalls).toHaveLength(20); + + await vi.advanceTimersByTimeAsync(61_000); + const summaryCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('rate-limited')); + expect(summaryCalls).toHaveLength(1); + expect(summaryCalls[0][2]).toBe( + '2 additional container health alerts were rate-limited in the last minute.', + ); + }); + + it('emits an accurate mixed crash and health rate-limit roll-up', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + for (let i = 0; i < 10; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + for (let i = 0; i < 10; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, + }); + await flushMicrotasks(); + } + + for (let i = 10; i < 12; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { ID: `mc-${i}`, Attributes: { exitCode: '1', name: `c-${i}` } }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + for (let i = 10; i < 13; i++) { + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: `mh-${i}`, Attributes: { name: `h-${i}` } }, + }); + await flushMicrotasks(); + } + + await vi.advanceTimersByTimeAsync(61_000); + const summaryCalls = mockDispatchAlert.mock.calls.filter(c => + typeof c[2] === 'string' && c[2].includes('rate-limited')); + expect(summaryCalls).toHaveLength(1); + expect(summaryCalls[0][2]).toBe( + '5 additional container alerts were rate-limited in the last minute (2 crash, 3 health).', + ); + }); + + it('keeps Auto-Heal health state when global_crash is off', async () => { + mockGetGlobalSettings.mockReturnValue({ global_crash: '0' }); + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'h7', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + expect(mockDispatchAlert).not.toHaveBeenCalled(); + expect(service.getContainerState('h7')).toMatchObject({ + healthStatus: 'unhealthy', + }); + expect(service.getContainerState('h7')?.unhealthySince).toBeTypeOf('number'); + }); + + it('does not refund a rate token into the next fixed window', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'cross-window', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Cross the fixed window, then consume a token in the new window so + // refund would inflate W2's count if it ignored the issuing window. + await vi.advanceTimersByTimeAsync(60_000); + mockDispatchAlert.mockResolvedValue({ persisted: true }); + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: 'cw-roll', + Attributes: { exitCode: '1', name: 'roll' }, + }, + }); + await vi.advanceTimersByTimeAsync(600); + expect( + mockDispatchAlert.mock.calls.filter( + (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), + ), + ).toHaveLength(1); + + // Fail the original health persist: old code refunds into W2 (count 1→0). + resolvePersist({ persisted: false }); + await flushMicrotasks(); + + // Fill the rest of W2 to the 20-alert cap (19 more crashes). + for (let i = 0; i < 19; i++) { + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: `cw-${i}`, + Attributes: { exitCode: '1', name: `svc-${i}` }, + }, + }); + } + await vi.advanceTimersByTimeAsync(600); + + // Cap must still hold: a cross-window refund would have allowed a 21st. + stream.push({ + Type: 'container', + Action: 'die', + Actor: { + ID: 'cw-overflow', + Attributes: { exitCode: '1', name: 'overflow' }, + }, + }); + await vi.advanceTimersByTimeAsync(600); + + const crashCalls = mockDispatchAlert.mock.calls.filter( + (c) => typeof c[2] === 'string' && c[2].includes('Crash Detected'), + ); + expect(crashCalls).toHaveLength(20); + }); + + it('clears pending health retry when the container recovers before persist fails', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + // First unhealthy starts an in-flight dispatch. + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Flap while in flight: recover then re-fail so a pending-retry marker is set. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Recover again before the first persist settles; marker must be dropped. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + resolvePersist({ persisted: false }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Later unrelated persist failure must not spuriously retry from a stale marker. + mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'recover-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not dispatch deferred health retry after shutdown', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'shutdown-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + service.shutdown(); + resolvePersist({ persisted: false }); + await flushMicrotasks(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + }); + + it('clears health alert bookkeeping on destroy', async () => { + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + stream.push({ + Type: 'container', + Action: 'destroy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-dedup', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + }); + + it('does not retry a deferred health alert after destroy', async () => { + let resolvePersist!: (value: { persisted: boolean }) => void; + mockDispatchAlert.mockImplementation( + () => + new Promise<{ persisted: boolean }>((resolve) => { + resolvePersist = resolve; + }), + ); + + service = new DockerEventService(1, 'local'); + await service.start(); + + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + + // Set a pending-retry marker while the first dispatch is in flight. + stream.push({ + Type: 'container', + Action: 'health_status: healthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + stream.push({ + Type: 'container', + Action: 'health_status: unhealthy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + stream.push({ + Type: 'container', + Action: 'destroy', + Actor: { ID: 'destroy-pending', Attributes: { name: 'api' } }, + }); + await flushMicrotasks(); + + resolvePersist({ persisted: false }); + await flushMicrotasks(); + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); }); }); diff --git a/backend/src/__tests__/drift-detection.test.ts b/backend/src/__tests__/drift-detection.test.ts index e927c303..efb4694a 100644 --- a/backend/src/__tests__/drift-detection.test.ts +++ b/backend/src/__tests__/drift-detection.test.ts @@ -36,7 +36,7 @@ function declared(services: DeclaredService[], parseError?: string): DeclaredCom function container(p: Partial & { id: string }): DependencyContainer { return { name: p.id, service: null, composeProject: null, stack: 'app', - state: 'running', image: 'img:latest', networks: [], volumes: [], ports: [], ...p, + state: 'running', exitCode: null, image: 'img:latest', networks: [], volumes: [], ports: [], ...p, }; } @@ -108,6 +108,147 @@ describe('assembleStackDrift - status', () => { expect(report.findings).toEqual([]); }); + it('does not emit service-missing for a clean one-shot beside a running service', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(true); + }); + + it('still emits service-missing when a one-shot exits non-zero', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: 1 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('still treats exited unless-stopped as missing even with exit 0', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'web', restart: 'unless-stopped' })]), + containers: [container({ id: 'c1', service: 'web', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('missing-runtime'); + expect(report.hasContainers).toBe(false); + }); + + it('fails closed when exitCode is null even with restart no', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'migrate', image: 'migrate:1', restart: 'no' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1', state: 'running' }), + container({ id: 'c2', service: 'migrate', image: 'migrate:1', state: 'exited', exitCode: null }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find(f => f.kind === 'service-missing')?.service).toBe('migrate'); + }); + + it('does not treat absent declared restart as a clean one-shot', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'daemon-default', image: 'daemon:1' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'daemon-default', image: 'daemon:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('daemon-default'); + }); + + it('all-one-shot stack with dedicated network is not missing-runtime but keeps network-missing and hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: { + services: [service({ name: 'migrate', restart: 'no', networks: ['jobs'] })], + networks: { jobs: { external: false } }, + volumes: {}, + projectName: 'app', + }, + containers: [ + container({ + id: 'c1', service: 'migrate', state: 'exited', exitCode: 0, + networks: [{ name: 'app_jobs', id: 'j', ip: '' }], + }), + ], + networks: [depNet('app_jobs')], + }); + expect(report.status).not.toBe('missing-runtime'); + expect(findingKinds(report)).not.toContain('service-missing'); + expect(report.hasContainers).toBe(false); + expect(findingKinds(report)).toContain('network-missing'); + expect(report.status).toBe('drifted'); + }); + + it('all-one-shot stack without network findings is in-sync with hasContainers false', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([service({ name: 'migrate', restart: 'no' })]), + containers: [container({ id: 'c1', service: 'migrate', state: 'exited', exitCode: 0 })], + }); + expect(report.status).toBe('in-sync'); + expect(report.hasContainers).toBe(false); + expect(report.findings).toEqual([]); + }); + + it('emits service-missing when normalized restart is always (deploy any)', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'always' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + expect(report.findings.find((f) => f.kind === 'service-missing')?.service).toBe('worker'); + }); + + it('emits service-missing when normalized restart is on-failure', () => { + const report = assembleStackDrift({ + stack: 'app', + declared: declared([ + service({ name: 'app', image: 'app:1', restart: 'unless-stopped' }), + service({ name: 'worker', image: 'worker:1', restart: 'on-failure' }), + ]), + containers: [ + container({ id: 'c1', service: 'app', image: 'app:1' }), + container({ id: 'c2', service: 'worker', image: 'worker:1', state: 'exited', exitCode: 0 }), + ], + }); + expect(findingKinds(report)).toContain('service-missing'); + }); + it('counts a restarting container as deployed', () => { const report = assembleStackDrift({ stack: 'app', @@ -456,6 +597,82 @@ describe('declaredFromEffectiveModel', () => { expect(converted.services[0].ports).toEqual([{ hostIp: '127.0.0.1', publishedPort: 8080, protocol: 'tcp' }]); }); + it('preserves restart policy including no, unless-stopped, and absent', () => { + const withNo = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'migrate', restart: 'no' })], + networks: {}, + volumes: {}, + }); + expect(withNo.services[0].restart).toBe('no'); + + const withUnless = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'web', restart: 'unless-stopped' })], + networks: {}, + volumes: {}, + }); + expect(withUnless.services[0].restart).toBe('unless-stopped'); + + const absent = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ name: 'job', restart: undefined })], + networks: {}, + volumes: {}, + }); + expect(absent.services[0].restart).toBeNull(); + }); + + it('normalizes deploy.restart_policy conditions with Compose precedence', () => { + const none = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'migrate', + restart: 'unless-stopped', + deploy: { restart_policy: { condition: 'none' } }, + })], + networks: {}, + volumes: {}, + }); + expect(none.services[0].restart).toBe('no'); + + const any = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: { condition: 'any' } }, + })], + networks: {}, + volumes: {}, + }); + expect(any.services[0].restart).toBe('always'); + + const onFailure = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: undefined, + deploy: { restart_policy: { condition: 'on-failure' } }, + })], + networks: {}, + volumes: {}, + }); + expect(onFailure.services[0].restart).toBe('on-failure'); + + const defaultAny = declaredFromEffectiveModel({ + projectName: 'app', + services: [effSvc({ + name: 'worker', + restart: 'no', + deploy: { restart_policy: {} }, + })], + networks: {}, + volumes: {}, + }); + expect(defaultAny.services[0].restart).toBe('always'); + }); + it('normalizes networks so drift matches the rendered model', () => { const model: EffectiveModel = { projectName: 'myapp', diff --git a/backend/src/__tests__/drift-ledger.test.ts b/backend/src/__tests__/drift-ledger.test.ts index f81fbcd2..1bb4b7f6 100644 --- a/backend/src/__tests__/drift-ledger.test.ts +++ b/backend/src/__tests__/drift-ledger.test.ts @@ -323,7 +323,7 @@ describe('DriftLedgerService.reconcileStack', () => { // A running container on a different image than compose declares => image-mismatch. const driftedContainer = (stack: string) => ({ id: `${stack}-c1`, name: `${stack}-web-1`, service: 'web', composeProject: stack, stack, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }); beforeEach(() => { @@ -377,7 +377,7 @@ describe('drift route (GET read-only, POST recheck persists)', () => { getDependencySnapshot: vi.fn().mockResolvedValue({ containers: [{ id: 'c1', name: `${STACK}-web-1`, service: 'web', composeProject: STACK, stack: STACK, - state: 'running', image: 'nginx:1.26', networks: [], volumes: [], ports: [], + state: 'running', exitCode: null, image: 'nginx:1.26', networks: [], volumes: [], ports: [], }], networks: [], volumes: [], }), diff --git a/backend/src/__tests__/filesystem-stack-paths.test.ts b/backend/src/__tests__/filesystem-stack-paths.test.ts index abf7d2e7..00b29665 100644 --- a/backend/src/__tests__/filesystem-stack-paths.test.ts +++ b/backend/src/__tests__/filesystem-stack-paths.test.ts @@ -95,9 +95,10 @@ describe('FileSystemService stack methods', () => { expect(fileNames).toEqual(['.env', 'compose.yaml']); }); - it('marks compose.yaml and .env as protected', async () => { + it('marks compose.yaml, .env, and .blueprint.json as protected', async () => { await fs.writeFile(path.join(stackDir, 'compose.yaml'), ''); await fs.writeFile(path.join(stackDir, '.env'), ''); + await fs.writeFile(path.join(stackDir, '.blueprint.json'), '{}'); await fs.writeFile(path.join(stackDir, 'custom.conf'), ''); const service = FileSystemService.getInstance(); @@ -106,6 +107,7 @@ describe('FileSystemService stack methods', () => { const byName = Object.fromEntries(entries.map(e => [e.name, e])); expect(byName['compose.yaml'].isProtected).toBe(true); expect(byName['.env'].isProtected).toBe(true); + expect(byName['.blueprint.json'].isProtected).toBe(true); expect(byName['custom.conf'].isProtected).toBe(false); }); diff --git a/backend/src/__tests__/fleet-sync-retry-service.test.ts b/backend/src/__tests__/fleet-sync-retry-service.test.ts index b86c06c3..109c7acc 100644 --- a/backend/src/__tests__/fleet-sync-retry-service.test.ts +++ b/backend/src/__tests__/fleet-sync-retry-service.test.ts @@ -18,7 +18,7 @@ const { mockGetFleetSyncStatuses: vi.fn().mockReturnValue([]), mockGetSystemState: vi.fn().mockReturnValue(null), mockPushResourceToNode: vi.fn().mockResolvedValue(undefined), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), })); vi.mock('../services/DatabaseService', () => ({ diff --git a/backend/src/__tests__/fleet-sync-service.test.ts b/backend/src/__tests__/fleet-sync-service.test.ts index f30430d5..1787687c 100644 --- a/backend/src/__tests__/fleet-sync-service.test.ts +++ b/backend/src/__tests__/fleet-sync-service.test.ts @@ -40,7 +40,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockTransaction: vi.fn().mockImplementation((fn: () => unknown) => fn()), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockAxiosPost: vi.fn().mockResolvedValue({ data: { success: true } }), })); diff --git a/backend/src/__tests__/health-gate-prepare.test.ts b/backend/src/__tests__/health-gate-prepare.test.ts index 258be3e2..3a4a3cd4 100644 --- a/backend/src/__tests__/health-gate-prepare.test.ts +++ b/backend/src/__tests__/health-gate-prepare.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -68,6 +69,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + // AutoHeal suppression is exercised elsewhere; here we only need the calls to // not throw when the gate finalizes a service run. vi.mock('../services/AutoHealService', () => ({ @@ -85,6 +95,8 @@ type Fixture = { restartCount?: number; startedAt?: string; imageId?: string; + exitCode?: number | null; + restartPolicy?: string | null; }; function setContainers(fixtures: Fixture[]): void { @@ -100,16 +112,36 @@ function setContainers(fixtures: Fixture[]): void { return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, Image: f.imageId ?? 'sha256:app', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? 'unless-stopped' } }, Config: { Labels: { 'com.docker.compose.service': f.service } }, }); }); } +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); async function ticks(n: number): Promise { @@ -137,6 +169,8 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped', db: 'unless-stopped' }); svc().start(); }); @@ -216,7 +250,7 @@ describe('primary vs collateral attribution', () => { await ticks(1); setContainers([ { id: 'p1', name: 'web-app-1', service: 'app' }, - { id: 's1', name: 'web-db-1', service: 'db', state: 'exited' }, + { id: 's1', name: 'web-db-1', service: 'db', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }, ]); await ticks(1); const report = svc().getReport(0, 'web', runId!); @@ -224,6 +258,178 @@ describe('primary vs collateral attribution', () => { expect(report.failureSource).toBe('collateral'); }); + it('passes when a collateral one-shot exits 0 with restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a collateral one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-migrate-1', service: 'migrate', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a collateral daemon with omitted restart exits 0', async () => { + setDeclaredRestarts({ app: 'unless-stopped', 'daemon-default': undefined }); + const token = await prepareService([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { id: 's1', name: 'web-daemon-1', service: 'daemon-default', restartPolicy: 'no' }, + ]); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }, + { + id: 's1', name: 'web-daemon-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.failureSource).toBe('collateral'); + expect(report.reason).toContain('exited during observation'); + }); + + it('passes when the primary service is a completed one-shot', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual unhealthy health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('passes a primary one-shot with residual starting health', async () => { + setDeclaredRestarts({ job: 'no' }); + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { + id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', imageId: 'sha256:app', + }, + ]); + await ticks(1); + expect(svc().getReport(0, 'web', runId!).status).toBe('observing'); + await ticks(6); + expect(svc().getReport(0, 'web', runId!).status).toBe('passed'); + }); + + it('fails when a primary one-shot exits with null exit code', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: null, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits 0 under unless-stopped', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'unless-stopped' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + + it('fails when a primary one-shot exits non-zero', async () => { + const token = await prepareService([ + { id: 'p1', name: 'web-job-1', service: 'job', restartPolicy: 'no' }, + ], { serviceName: 'job', expectedReplicas: 1 }); + svc().attachExpectedImage(token, 'sha256:app'); + const { runId } = svc().beginPrepared({ prepareToken: token, actor: 'tester' }); + await ticks(1); + setContainers([ + { id: 'p1', name: 'web-job-1', service: 'job', state: 'exited', exitCode: 1, restartPolicy: 'no', imageId: 'sha256:app' }, + ]); + await ticks(1); + const report = svc().getReport(0, 'web', runId!); + expect(report.status).toBe('failed'); + expect(report.reason).toContain('exited during observation'); + expect(report.failureSource).toBe('primary'); + }); + it('fails with failureSource collateral when a healthy sibling vanishes before the first poll', async () => { // Sibling is healthy at prepare, then gone before arming. Seeding expected // from the prepare baseline must still track it so the gate fails. diff --git a/backend/src/__tests__/health-gate-service.test.ts b/backend/src/__tests__/health-gate-service.test.ts index 161bc570..0ba4458c 100644 --- a/backend/src/__tests__/health-gate-service.test.ts +++ b/backend/src/__tests__/health-gate-service.test.ts @@ -29,6 +29,7 @@ const { state } = vi.hoisted(() => ({ settings: {} as Record, listContainers: vi.fn(), inspect: vi.fn(), + renderConfig: vi.fn(), }, })); @@ -80,6 +81,15 @@ vi.mock('../services/DockerController', () => ({ }, })); +vi.mock('../services/ComposeService', () => ({ + getComposeCommandTimeoutMs: () => 30_000, + ComposeService: { + getInstance: () => ({ + renderConfig: state.renderConfig, + }), + }, +})); + import { HealthGateService } from '../services/HealthGateService'; type ContainerFixture = { @@ -89,25 +99,57 @@ type ContainerFixture = { health?: string | null; restartCount?: number; startedAt?: string; + exitCode?: number | null; + restartPolicy?: string | null; + service?: string; + imageId?: string; }; /** Configure the docker mocks from a simple fixture list. */ function setContainers(fixtures: ContainerFixture[]): void { - state.listContainers.mockResolvedValue(fixtures.map(f => ({ Id: f.id, Names: [`/${f.name}`], State: f.state ?? 'running' }))); + state.listContainers.mockResolvedValue(fixtures.map(f => ({ + Id: f.id, + Names: [`/${f.name}`], + State: f.state ?? 'running', + Labels: f.service ? { 'com.docker.compose.service': f.service } : {}, + }))); state.inspect.mockImplementation((id: string) => { const f = fixtures.find(c => c.id === id); if (!f) return Promise.reject(Object.assign(new Error('no such container'), { statusCode: 404 })); return Promise.resolve({ State: { Status: f.state ?? 'running', + ExitCode: f.exitCode === undefined ? (f.state === 'exited' ? 1 : 0) : f.exitCode, Health: f.health !== undefined && f.health !== null ? { Status: f.health } : undefined, StartedAt: f.startedAt ?? '2026-06-10T00:00:00Z', }, RestartCount: f.restartCount ?? 0, + Image: f.imageId ?? 'sha256:img', + HostConfig: { RestartPolicy: { Name: f.restartPolicy ?? '' } }, + Config: { Labels: f.service ? { 'com.docker.compose.service': f.service } : {} }, }); }); } +/** Declared Compose restart map used for one-shot recognition (not inspect). */ +function setDeclaredRestarts(services: Record): void { + const rendered = { + name: 'web', + services: Object.fromEntries( + Object.entries(services).map(([name, restart]) => [ + name, + restart === undefined ? { image: `${name}:1` } : { image: `${name}:1`, restart }, + ]), + ), + }; + state.renderConfig.mockResolvedValue({ + rendered: JSON.stringify(rendered), + stderr: '', + code: 0, + timedOut: false, + }); +} + const svc = () => HealthGateService.getInstance(); const latest = (stack = 'web') => svc().getReport(0, stack); @@ -125,7 +167,9 @@ beforeEach(() => { state.settings = { health_gate_enabled: '1', health_gate_window_seconds: '30' }; state.listContainers.mockReset(); state.inspect.mockReset(); - setContainers([{ id: 'aaa', name: 'web-app-1' }]); + state.renderConfig.mockReset(); + setDeclaredRestarts({ app: 'unless-stopped' }); + setContainers([{ id: 'aaa', name: 'web-app-1', service: 'app', restartPolicy: 'unless-stopped' }]); svc().start(); }); @@ -149,7 +193,7 @@ describe('HealthGateService verdicts', () => { it('fails fast when a container exits', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); // baseline - setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited' }]); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'unless-stopped' }]); await ticks(1); const report = latest(); expect(report.status).toBe('failed'); @@ -157,6 +201,167 @@ describe('HealthGateService verdicts', () => { expect(state.activity.some(a => a.category === 'health_gate_failed')).toBe(true); }); + it('passes when a clean one-shot exits 0 with explicit declared restart no', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('fails when a daemon with omitted Compose restart exits 0 (inspect also reports no)', async () => { + // QA P0: Docker HostConfig.RestartPolicy.Name is "no" for both omit and + // explicit restart:"no". Declared intent must decide, not inspect. + setDeclaredRestarts({ 'daemon-default': undefined }); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'running', restartPolicy: 'no', + }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'daemon', name: 'web-daemon-default-1', service: 'daemon-default', + state: 'exited', exitCode: 0, restartPolicy: 'no', + }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('passes a clean one-shot even when residual health is unhealthy', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'unhealthy', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('passes a clean one-shot even when residual health is still starting', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { + id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, + restartPolicy: 'no', health: 'starting', + }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('passed'); + }); + + it('still fails unhealthy on a long-running container', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'unhealthy', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('unhealthy'); + }); + + it('still ends unknown when a long-running healthcheck is starting at window end', async () => { + setDeclaredRestarts({ app: 'unless-stopped', migrate: 'no' }); + setContainers([ + { id: 'app', name: 'web-app-1', service: 'app', state: 'running', restartPolicy: 'unless-stopped' }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'running', restartPolicy: 'no' }, + ]); + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([ + { + id: 'app', name: 'web-app-1', service: 'app', state: 'running', + restartPolicy: 'unless-stopped', health: 'starting', + }, + { id: 'job', name: 'web-migrate-1', service: 'migrate', state: 'exited', exitCode: 0, restartPolicy: 'no' }, + ]); + await ticks(1); + expect(latest().status).toBe('observing'); + await ticks(6); + expect(latest().status).toBe('unknown'); + expect(latest().reason).toContain('still starting'); + }); + + it('fails when exit 0 has unless-stopped restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'unless-stopped' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when exit 0 has always restart policy', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 0, restartPolicy: 'always' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails closed when exit code is null on an exited container', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: null, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + + it('fails when a one-shot exits non-zero', async () => { + svc().beginStack(0, 'web', 'update', 'tester'); + await ticks(1); + setContainers([{ id: 'aaa', name: 'web-app-1', state: 'exited', exitCode: 1, restartPolicy: 'no' }]); + await ticks(1); + expect(latest().status).toBe('failed'); + expect(latest().reason).toContain('exited during observation'); + }); + it('fails fast when a healthcheck reports unhealthy', async () => { svc().beginStack(0, 'web', 'update', 'tester'); await ticks(1); diff --git a/backend/src/__tests__/host-console-ws.test.ts b/backend/src/__tests__/host-console-ws.test.ts index 3b0656ce..973ffa91 100644 --- a/backend/src/__tests__/host-console-ws.test.ts +++ b/backend/src/__tests__/host-console-ws.test.ts @@ -21,8 +21,8 @@ describe('WebSocket upgrade - host console auth enforcement', () => { beforeAll(async () => { vi.restoreAllMocks(); tmpDir = await setupTestDb(); - // Host console requires the paid tier; mock the license so the tier gate - // passes for the admin/accepted cases. Individual tests override as needed. + // Host console is available on every tier for admins; mock paid so other + // suites that share LicenseService state stay stable. const { LicenseService } = await import('../services/LicenseService'); getTierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); const mod = await import('../index'); @@ -78,10 +78,28 @@ describe('WebSocket upgrade - host console auth enforcement', () => { expect(await expectRejected(ws)).toBe(403); }); - it('rejects an admin on the Community tier (403)', async () => { + it('accepts a Community-tier admin', async () => { getTierSpy.mockReturnValueOnce('community'); const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } }); - expect(await expectRejected(ws)).toBe(403); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); + }); + + it('accepts a console_session Bearer on Community (remote bridge mint path)', async () => { + getTierSpy.mockReturnValueOnce('community'); + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console', actingAs: TEST_USERNAME }); + const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); }); it('accepts an admin on Admiral and records an open audit row', async () => { @@ -127,6 +145,116 @@ describe('WebSocket upgrade - host console auth enforcement', () => { expect(firstMessage).toContain('Invalid stack path'); ws.close(); }); + + + it('rejects an unknown nodeId without spawning a PTY (404)', async () => { + const { HostTerminalService } = await import('../services/HostTerminalService'); + const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal'); + try { + const ws = new WebSocket(wsUrl('?nodeId=99999999'), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + expect(spawnSpy).not.toHaveBeenCalled(); + } finally { + spawnSpy.mockRestore(); + } + }); + + it('rejects a malformed nodeId that parseInt would coerce (404)', async () => { + const ws = new WebSocket(wsUrl('?nodeId=1abc'), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + }); + + it('rejects zero and negative nodeId values (404)', async () => { + for (const id of ['0', '-1']) { + const ws = new WebSocket(wsUrl(`?nodeId=${id}`), { + headers: { Cookie: `sencho_token=${adminToken()}` }, + }); + expect(await expectRejected(ws)).toBe(404); + } + }); + + it('rejects a replayed console_session token on second Host Console upgrade', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' }); + const first = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const firstOpened = await new Promise((resolve) => { + first.on('open', () => { first.close(); resolve(true); }); + first.on('error', () => resolve(false)); + first.on('unexpected-response', () => resolve(false)); + }); + expect(firstOpened).toBe(true); + const second = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + expect(await expectRejected(second)).toBe(401); + }); + + it('rejects a host-console console_session on the container-exec /ws path', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const token = mintConsoleSession({ path: 'host-console' }); + const addr = server.address(); + if (!addr || typeof addr === 'string') throw new Error('Server not listening'); + const ws = new WebSocket(`ws://127.0.0.1:${addr.port}/ws`, { + headers: { Authorization: `Bearer ${token}` }, + }); + expect(await expectRejected(ws)).toBe(403); + }); + + it('records acting_as on console_session open audit rows', async () => { + const { mintConsoleSession } = await import('../helpers/consoleSession'); + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + const insertSpy = vi.spyOn(db, 'insertAuditLog'); + const token = mintConsoleSession({ path: 'host-console', actingAs: 'qaadmin' }); + try { + const ws = new WebSocket(wsUrl(), { headers: { Authorization: `Bearer ${token}` } }); + const opened = await new Promise((resolve) => { + ws.on('open', () => { ws.close(); resolve(true); }); + ws.on('error', () => resolve(false)); + ws.on('unexpected-response', () => resolve(false)); + }); + expect(opened).toBe(true); + await waitFor(() => insertSpy.mock.calls.some((c) => { + const e = c[0] as { summary?: string }; + return typeof e.summary === 'string' && e.summary.includes('Opened host console'); + })); + const openEntry = insertSpy.mock.calls + .map((c) => c[0] as { username: string; acting_as?: string | null; summary: string }) + .find((e) => e.summary.includes('Opened host console')); + expect(openEntry?.username).toBe('console_session'); + expect(openEntry?.acting_as).toBe('qaadmin'); + } finally { + insertSpy.mockRestore(); + } + }); + + it('closes without spawning a PTY when directory resolution throws (no default-node fallback)', async () => { + const { FileSystemService } = await import('../services/FileSystemService'); + const { HostTerminalService } = await import('../services/HostTerminalService'); + const spawnSpy = vi.spyOn(HostTerminalService, 'spawnTerminal'); + const getInstanceSpy = vi.spyOn(FileSystemService, 'getInstance').mockImplementation(() => { + throw new Error('compose dir unavailable'); + }); + + try { + const ws = new WebSocket(wsUrl(), { headers: { Cookie: `sencho_token=${adminToken()}` } }); + const firstMessage = await new Promise((resolve) => { + ws.on('message', (data) => resolve(data.toString())); + ws.on('error', () => resolve('')); + ws.on('unexpected-response', () => resolve('')); + }); + expect(firstMessage).toMatch(/Failed to resolve console directory/i); + expect(spawnSpy).not.toHaveBeenCalled(); + // getInstance must not be retried against a fallback/default node id. + expect(getInstanceSpy).toHaveBeenCalledTimes(1); + ws.close(); + } finally { + spawnSpy.mockRestore(); + getInstanceSpy.mockRestore(); + } + }); }); /** Poll a predicate up to ~1s; resolve true as soon as it passes. */ diff --git a/backend/src/__tests__/host-console.test.ts b/backend/src/__tests__/host-console.test.ts index fcbc3510..1187113e 100644 --- a/backend/src/__tests__/host-console.test.ts +++ b/backend/src/__tests__/host-console.test.ts @@ -375,9 +375,42 @@ describe('POST /api/system/console-token', () => { const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); const res = await request(app) .post('/api/system/console-token') - .set('Authorization', `Bearer ${token}`); + .set('Authorization', `Bearer ${token}`) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); + const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string; jti?: string }; + expect(decoded.scope).toBe('console_session'); + expect(decoded.path).toBe('host-console'); + expect(typeof decoded.jti).toBe('string'); + }); + + it('returns 400 when path is missing', async () => { + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/system/console-token') + .set('Authorization', `Bearer ${token}`) + .send({}); + expect(res.status).toBe(400); + }); + + it('returns 200 for an admin on Community (no paid gate)', async () => { + const { LicenseService } = await import('../services/LicenseService'); + const spy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + try { + const token = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + const res = await request(app) + .post('/api/system/console-token') + .set('Authorization', `Bearer ${token}`) + .send({ path: 'container-exec' }); + expect(res.status).toBe(200); + expect(typeof res.body.token).toBe('string'); + const decoded = jwt.verify(res.body.token, TEST_JWT_SECRET) as { scope?: string; path?: string }; + expect(decoded.scope).toBe('console_session'); + expect(decoded.path).toBe('container-exec'); + } finally { + spy.mockReturnValue('paid'); + } }); it('returns 403 for non-admin user (viewer role)', async () => { diff --git a/backend/src/__tests__/image-update-service.test.ts b/backend/src/__tests__/image-update-service.test.ts index 65a0b828..3645ba22 100644 --- a/backend/src/__tests__/image-update-service.test.ts +++ b/backend/src/__tests__/image-update-service.test.ts @@ -25,7 +25,7 @@ const { mockGetSystemState: vi.fn().mockReturnValue('1'), // default: backfilled mockSetSystemState: vi.fn(), mockAddNotificationHistory: vi.fn(), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetStacks: vi.fn().mockResolvedValue([]), mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockRejectedValue(new Error('no env')), diff --git a/backend/src/__tests__/monitor-service.test.ts b/backend/src/__tests__/monitor-service.test.ts index 4d51b855..39003a15 100644 --- a/backend/src/__tests__/monitor-service.test.ts +++ b/backend/src/__tests__/monitor-service.test.ts @@ -9,7 +9,9 @@ import { installArcstatsFsMock, arcstatsBody, DEFAULT_ARC_PATH, type ArcstatsFsM const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContainerMetric, mockCleanupOldMetrics, mockCleanupOldNotifications, mockCleanupOldAuditLogs, - mockUpdateStackAlertLastFired, mockGetSystemState, mockSetSystemState, + mockUpdateStackAlertLastFired, mockGetStackAlertServiceCooldown, + mockHasAnyStackAlertServiceCooldown, mockUpsertStackAlertServiceCooldown, + mockGetSystemState, mockSetSystemState, mockPruneScanHistoryPerImage, mockDeleteScansByImageRef, mockGetRunningContainers, mockGetAllContainers, mockGetContainerStatsStream, mockGetContainerRestartCount, mockGetDiskUsage, mockGetImages, mockGetStacks, @@ -29,6 +31,9 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine mockCleanupOldNotifications: vi.fn(), mockCleanupOldAuditLogs: vi.fn(), mockUpdateStackAlertLastFired: vi.fn(), + mockGetStackAlertServiceCooldown: vi.fn().mockReturnValue(null), + mockHasAnyStackAlertServiceCooldown: vi.fn().mockReturnValue(false), + mockUpsertStackAlertServiceCooldown: vi.fn(), mockGetSystemState: vi.fn().mockReturnValue(null), mockSetSystemState: vi.fn(), mockPruneScanHistoryPerImage: vi.fn().mockReturnValue(0), @@ -43,7 +48,7 @@ const { mockGetGlobalSettings, mockGetNodes, mockGetStackAlerts, mockAddContaine }), mockGetImages: vi.fn().mockResolvedValue([]), mockGetStacks: vi.fn().mockResolvedValue([]), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockCurrentLoad: vi.fn().mockResolvedValue({ currentLoad: 10 }), mockMem: vi.fn().mockResolvedValue({ total: 16e9, used: 4e9, active: 4e9, available: 12e9, free: 12e9, buffcache: 0 }), mockFsSize: vi.fn().mockResolvedValue([{ mount: '/', use: 30 }]), @@ -65,6 +70,9 @@ vi.mock('../services/DatabaseService', () => ({ cleanupOldNotifications: mockCleanupOldNotifications, cleanupOldAuditLogs: mockCleanupOldAuditLogs, updateStackAlertLastFired: mockUpdateStackAlertLastFired, + getStackAlertServiceCooldown: mockGetStackAlertServiceCooldown, + hasAnyStackAlertServiceCooldown: mockHasAnyStackAlertServiceCooldown, + upsertStackAlertServiceCooldown: mockUpsertStackAlertServiceCooldown, getSystemState: mockGetSystemState, setSystemState: mockSetSystemState, pruneScanHistoryPerImage: mockPruneScanHistoryPerImage, @@ -868,8 +876,12 @@ describe('MonitorService - breach state machine', () => { function setupAlertScenario(cpuPercent: number) { mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); mockGetRunningContainers.mockResolvedValue([{ - Id: 'c1', - Labels: { 'com.docker.compose.project': 'my-stack' }, + Id: 'c1abcdefghijk', + Names: ['/my-stack-api-1'], + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, }]); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ cpu_stats: { cpu_usage: { total_usage: 1000 + cpuPercent * 50 }, system_cpu_usage: 10000, online_cpus: 1 }, @@ -879,6 +891,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -887,6 +900,8 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); } it('fires alert when condition met and duration is 0', async () => { @@ -898,9 +913,10 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).toHaveBeenCalledWith( 'warning', 'monitor_alert', - 'The **CPU usage** for **my-stack** has exceeded your threshold of **80%** (Currently: 90%).', - { stackName: 'my-stack', actor: 'system:monitor' }, + 'The **CPU usage** for **api** in **my-stack** (container **my-stack-api-1**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'my-stack-api-1', actor: 'system:monitor' }, ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); }); @@ -913,12 +929,22 @@ describe('MonitorService - breach state machine', () => { expect(mockDispatchAlert).not.toHaveBeenCalledWith('warning', expect.stringContaining('CPU')); }); - it('respects cooldown after firing', async () => { + it('respects cooldown after firing via persisted service cooldown', async () => { + setupAlertScenario(90); + mockGetStackAlertServiceCooldown.mockReturnValue(Date.now() - 30 * 60 * 1000); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + }); + + it('respects legacy last_fired_at floor when no child cooldown rows exist', async () => { setupAlertScenario(90); - // Simulate that alert was fired 30 minutes ago (within 60-min cooldown) mockGetStackAlerts.mockReturnValue([{ id: 1, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -926,14 +952,94 @@ describe('MonitorService - breach state machine', () => { cooldown_mins: 60, last_fired_at: Date.now() - 30 * 60 * 1000, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); - expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); }); - it('resets breach state when condition clears', async () => { + it('fires after legacy last_fired_at cooldown expires and writes a child cooldown row', async () => { + setupAlertScenario(90); + mockGetStackAlerts.mockReturnValue([{ + id: 1, + stack_name: 'my-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 90 * 60 * 1000, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('does not advance cooldown when notification history is not persisted', async () => { + setupAlertScenario(90); + mockDispatchAlert.mockResolvedValueOnce({ persisted: false }); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalled(); + expect(mockUpdateStackAlertLastFired).not.toHaveBeenCalled(); + + mockDispatchAlert.mockResolvedValueOnce({ persisted: true }); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, 'api', expect.any(Number)); + expect(mockUpdateStackAlertLastFired).toHaveBeenCalledWith(1, expect.any(Number)); + }); + + it('drops in-memory breach timers when a rule is deleted', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('55:gone-container', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([]); + mockGetStackAlerts.mockReturnValue([]); + mockGetGlobalSettings.mockReturnValue({}); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('55:gone-container')).toBe(false); + }); + + it('names unlabeled containers as unknown service in the alert body', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'orphan123456789', + Names: ['/orphan'], + Labels: { + 'com.docker.compose.project': 'my-stack', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('**unknown service**'), + expect.objectContaining({ stackName: 'my-stack', containerName: 'orphan' }), + ); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(1, '_unlabeled', expect.any(Number)); + }); + + it('resets breach state when condition clears for that container only', async () => { const svc = MonitorService.getInstance(); // First: breach starts @@ -941,6 +1047,7 @@ describe('MonitorService - breach state machine', () => { mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -949,13 +1056,14 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(true); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(true); // Second: condition clears setupAlertScenario(10); mockGetStackAlerts.mockReturnValue([{ id: 42, stack_name: 'my-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -964,7 +1072,28 @@ describe('MonitorService - breach state machine', () => { last_fired_at: 0, }]); await (svc as any).evaluate(); - expect((svc as any).activeBreaches.has(42)).toBe(false); + expect((svc as any).activeBreaches.has('42:c1abcdefghijk')).toBe(false); + }); + + it('falls back to short container id when Names is absent', async () => { + setupAlertScenario(90); + mockGetRunningContainers.mockResolvedValue([{ + Id: 'abcdef1234567890', + Labels: { + 'com.docker.compose.project': 'my-stack', + 'com.docker.compose.service': 'api', + }, + }]); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + 'The **CPU usage** for **api** in **my-stack** (container **abcdef123456**) has exceeded your threshold of **80%** (Currently: 90%).', + { stackName: 'my-stack', containerName: 'abcdef123456', actor: 'system:monitor' }, + ); }); }); @@ -1074,7 +1203,12 @@ describe('MonitorService - restart_count metric', () => { expect(mockGetContainerRestartCount).toHaveBeenCalledWith('c1'); // restart_count=5 > threshold=3, should fire - expect(mockDispatchAlert).toHaveBeenCalledWith('warning', 'monitor_alert', expect.stringContaining('Restart count'), { stackName: 'my-stack', actor: 'system:monitor' }); + expect(mockDispatchAlert).toHaveBeenCalledWith( + 'warning', + 'monitor_alert', + expect.stringContaining('Restart count'), + { stackName: 'my-stack', containerName: 'c1', actor: 'system:monitor' }, + ); }); it('skips Docker inspect when no restart_count rules exist', async () => { @@ -1261,6 +1395,8 @@ describe('MonitorService - parallel container processing', () => { mockGetGlobalSettings.mockReturnValue({}); mockGetStackAlerts.mockReturnValue([]); mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); }); it('fans out per-container stats fetches in parallel (wall time ~ max not sum)', async () => { @@ -1311,12 +1447,15 @@ describe('MonitorService - parallel container processing', () => { }); it('dispatches a stack alert exactly once per cycle even when multiple containers in the stack breach', async () => { - // 5 containers in the same stack, all breaching the same rule. Without - // the per-cycle dedup, parallel workers race past the cooldown check - // and each fire dispatchAlert before any DB write lands. + // 5 replicas of the same Compose service, all breaching. Dedup is per + // rule+service, so only one fire should land this cycle. const containers = Array.from({ length: 5 }, (_, i) => ({ Id: `container-${i}`, - Labels: { 'com.docker.compose.project': 'shared-stack' }, + Names: [`/shared-stack-web-${i}`], + Labels: { + 'com.docker.compose.project': 'shared-stack', + 'com.docker.compose.service': 'web', + }, })); mockGetRunningContainers.mockResolvedValue(containers); mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ @@ -1328,6 +1467,7 @@ describe('MonitorService - parallel container processing', () => { mockGetStackAlerts.mockReturnValue([{ id: 7, stack_name: 'shared-stack', + service_name: null, metric: 'cpu_percent', operator: '>', threshold: 80, @@ -1335,6 +1475,8 @@ describe('MonitorService - parallel container processing', () => { cooldown_mins: 60, last_fired_at: 0, }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); const svc = MonitorService.getInstance(); await (svc as any).evaluate(); @@ -1343,7 +1485,290 @@ describe('MonitorService - parallel container processing', () => { (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), ); expect(cpuDispatches).toHaveLength(1); - expect(mockUpdateStackAlertLastFired).toHaveBeenCalledTimes(1); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledTimes(1); + }); + + it('allows different services on an All-services rule to fire in the same cycle', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 8, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + mockGetGlobalSettings.mockReturnValue({}); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(2); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'api', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(8, 'database', expect.any(Number)); + }); + + it('does not let a healthy sibling clear another container breach timer', async () => { + const svc = MonitorService.getInstance(); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetStackAlerts.mockReturnValue([{ + id: 9, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + + const highCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + const lowCpu = JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + }); + + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-hot', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-cool', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockImplementation(async (id: string) => (id === 'api-hot' ? highCpu : lowCpu)); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('9:api-hot')).toBe(true); + expect((svc as any).activeBreaches.has('9:db-cool')).toBe(false); + }); + + it('ignores sibling services for a service-scoped rule', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 10, + stack_name: 'shared-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockReturnValue(null); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(false); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockDispatchAlert).toHaveBeenCalledTimes(1); + expect(mockDispatchAlert.mock.calls[0][2]).toContain('**api**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(10, 'api', expect.any(Number)); + }); + + it('does not block service B after service A fires under new-schema cooldown', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 11, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: Date.now() - 5 * 60 * 1000, // would block under old rule-wide semantics + }]); + // api already has a cooldown row; database does not, so database may still fire. + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + const svc = MonitorService.getInstance(); + await (svc as any).evaluate(); + + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(11, 'database', expect.any(Number)); + }); + + it('honors persisted per-service cooldown after a fresh MonitorService instance', async () => { + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'api-1', + Names: ['/stack-api-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'api-2', + Names: ['/stack-api-2'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + { + Id: 'db-1', + Names: ['/stack-db-1'], + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'database' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 5500 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 0, + cooldown_mins: 60, + last_fired_at: 0, + }]); + mockGetStackAlertServiceCooldown.mockImplementation((_id: number, service: string) => + service === 'api' ? Date.now() - 5 * 60 * 1000 : null, + ); + mockHasAnyStackAlertServiceCooldown.mockReturnValue(true); + + // beforeEach already cleared the singleton; this instance has empty maps. + const svc = MonitorService.getInstance(); + expect((svc as any).activeBreaches.size).toBe(0); + expect((svc as any).firedThisCycle.size).toBe(0); + + await (svc as any).evaluate(); + + const cpuDispatches = mockDispatchAlert.mock.calls.filter( + (args: unknown[]) => args[1] === 'monitor_alert' && typeof args[2] === 'string' && args[2].includes('CPU'), + ); + expect(cpuDispatches).toHaveLength(1); + expect(cpuDispatches[0][2]).toContain('**database**'); + expect(mockUpsertStackAlertServiceCooldown).toHaveBeenCalledWith(13, 'database', expect.any(Number)); + expect(mockUpsertStackAlertServiceCooldown).not.toHaveBeenCalledWith(13, 'api', expect.any(Number)); + }); + + it('preserves breach timers when container enumeration fails', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('12:still-running', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 12, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockRejectedValue(new Error('docker down')); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('12:still-running')).toBe(true); + }); + + it('removes breach timers for stopped containers after successful enumeration', async () => { + const svc = MonitorService.getInstance(); + (svc as any).activeBreaches.set('13:gone', { breachStartedAt: Date.now() }); + mockGetNodes.mockReturnValue([{ id: 1, name: 'local', type: 'local' }]); + mockGetStackAlerts.mockReturnValue([{ + id: 13, + stack_name: 'shared-stack', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 999, + cooldown_mins: 0, + last_fired_at: 0, + }]); + mockGetGlobalSettings.mockReturnValue({}); + mockGetRunningContainers.mockResolvedValue([ + { + Id: 'still-here', + Labels: { 'com.docker.compose.project': 'shared-stack', 'com.docker.compose.service': 'api' }, + }, + ]); + mockGetContainerStatsStream.mockResolvedValue(JSON.stringify({ + cpu_stats: { cpu_usage: { total_usage: 1100 }, system_cpu_usage: 10000, online_cpus: 1 }, + precpu_stats: { cpu_usage: { total_usage: 1000 }, system_cpu_usage: 5000 }, + memory_stats: { usage: 100e6, limit: 1e9 }, + })); + + await (svc as any).evaluate(); + + expect((svc as any).activeBreaches.has('13:gone')).toBe(false); }); it('caps simultaneous Docker calls at MAX_CONTAINER_CONCURRENCY', async () => { diff --git a/backend/src/__tests__/networking-operator.test.ts b/backend/src/__tests__/networking-operator.test.ts index 466fcb95..7d2a6f6d 100644 --- a/backend/src/__tests__/networking-operator.test.ts +++ b/backend/src/__tests__/networking-operator.test.ts @@ -142,7 +142,7 @@ describe('networking operator routes', () => { it('blocks admin delete when network is attached', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'orphan_net', id: NET_ID, ip: '' }], volumes: [], ports: [] }], networks: [{ id: NET_ID, name: 'orphan_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: null, stack: null }], volumes: [], }), @@ -220,7 +220,7 @@ describe('evaluateNetworkDeleteGuard', () => { it('blocks a network that still has an attached container', () => { const snapshot = { volumes: [], - containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'img', networks: [{ name: 'app_net', id: 'n1', ip: '' }], volumes: [], ports: [] }], networks: [{ id: 'n1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }], }; expect(evaluateNetworkDeleteGuard('n1', snapshot, []).code).toBe('attached'); diff --git a/backend/src/__tests__/networking-summary.test.ts b/backend/src/__tests__/networking-summary.test.ts index c1ca54dc..caa3d011 100644 --- a/backend/src/__tests__/networking-summary.test.ts +++ b/backend/src/__tests__/networking-summary.test.ts @@ -60,7 +60,7 @@ describe('networking summary', () => { it('flags a stack with an undeclared runtime network as network drift', async () => { vi.spyOn(DockerController, 'getInstance').mockReturnValue({ getDependencySnapshot: vi.fn().mockResolvedValue({ - containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], + containers: [{ id: 'c1', name: 'web1', service: 'web', composeProject: STACK, stack: STACK, state: 'running', exitCode: null, image: 'nginx', networks: [{ name: `${STACK}_default`, id: 'd', ip: '' }, { name: `${STACK}_rogue`, id: 'r', ip: '' }], volumes: [], ports: [] }], networks: [ { id: 'd', name: `${STACK}_default`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, { id: 'r', name: `${STACK}_rogue`, driver: 'bridge', scope: 'local', isSystem: false, composeProject: STACK, stack: STACK }, diff --git a/backend/src/__tests__/notification-routing.test.ts b/backend/src/__tests__/notification-routing.test.ts index 1bb7dc6a..08125b96 100644 --- a/backend/src/__tests__/notification-routing.test.ts +++ b/backend/src/__tests__/notification-routing.test.ts @@ -217,7 +217,7 @@ describe('NotificationService - routing logic', () => { mockFetch.mockRejectedValueOnce(new Error('Network timeout')); // Should not throw - await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toBeUndefined(); + await expect(svc.dispatchAlert('error', 'monitor_alert', 'Crash', { stackName: 'my-app' })).resolves.toEqual({ persisted: true }); }); it('does not dispatch to global agents when routes array is empty and no stackName', async () => { @@ -553,7 +553,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: false }); // No external dispatch and no routing read when there is no persisted row. expect(mockFetch).not.toHaveBeenCalled(); @@ -580,7 +580,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('error', 'monitor_alert', 'Container crashed', { stackName: 'my-app' }), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); // The history row was still written; only routing failed, and it was logged. expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); @@ -598,7 +598,7 @@ describe('NotificationService - crash safety (dispatchAlert never rejects)', () await expect( svc.dispatchAlert('info', 'system', 'Host rebooted'), - ).resolves.toBeUndefined(); + ).resolves.toEqual({ persisted: true }); expect(mockAddNotificationHistory).toHaveBeenCalledTimes(1); expect(consoleErrorSpy).toHaveBeenCalledWith('[Notify] dispatchAlert failed:', expect.any(Error)); diff --git a/backend/src/__tests__/one-shot-completion.test.ts b/backend/src/__tests__/one-shot-completion.test.ts new file mode 100644 index 00000000..0faddd26 --- /dev/null +++ b/backend/src/__tests__/one-shot-completion.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from 'vitest'; +import { + isCleanOneShotCompletion, + isNoRestartPolicy, + normalizeComposeRestartIntent, +} from '../utils/oneShotCompletion'; + +describe('isNoRestartPolicy', () => { + it('accepts only explicit no', () => { + expect(isNoRestartPolicy('no')).toBe(true); + }); + + it('rejects absent, empty, and restarting policies', () => { + expect(isNoRestartPolicy(undefined)).toBe(false); + expect(isNoRestartPolicy(null)).toBe(false); + expect(isNoRestartPolicy('')).toBe(false); + expect(isNoRestartPolicy('unless-stopped')).toBe(false); + expect(isNoRestartPolicy('always')).toBe(false); + expect(isNoRestartPolicy('on-failure')).toBe(false); + }); +}); + +describe('isCleanOneShotCompletion', () => { + const clean = { + state: 'exited', + exitCode: 0 as number | null, + restartPolicy: 'no' as string | null | undefined, + }; + + it('returns true only for exited + exit 0 + explicit restart no', () => { + expect(isCleanOneShotCompletion(clean)).toBe(true); + }); + + it('returns false for absent or empty declared restart', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: undefined })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: null })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: '' })).toBe(false); + }); + + it('returns false for non-exited states', () => { + expect(isCleanOneShotCompletion({ ...clean, state: 'running' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'restarting' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'created' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, state: 'dead' })).toBe(false); + }); + + it('returns false for non-zero and null exit codes (fail closed)', () => { + expect(isCleanOneShotCompletion({ ...clean, exitCode: 1 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: 137 })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, exitCode: null })).toBe(false); + }); + + it('returns false for restarting policies even with exit 0', () => { + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'unless-stopped' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'always' })).toBe(false); + expect(isCleanOneShotCompletion({ ...clean, restartPolicy: 'on-failure' })).toBe(false); + }); +}); + +describe('normalizeComposeRestartIntent', () => { + it('falls back to service restart when deploy.restart_policy is unset', () => { + expect(normalizeComposeRestartIntent('no')).toBe('no'); + expect(normalizeComposeRestartIntent('unless-stopped')).toBe('unless-stopped'); + expect(normalizeComposeRestartIntent(undefined)).toBeNull(); + expect(normalizeComposeRestartIntent(null, {})).toBeNull(); + expect(normalizeComposeRestartIntent('always', { replicas: 2 })).toBe('always'); + }); + + it('maps deploy.restart_policy.condition with Compose defaults and precedence', () => { + expect(normalizeComposeRestartIntent('unless-stopped', { + restart_policy: { condition: 'none' }, + })).toBe('no'); + expect(normalizeComposeRestartIntent(null, { + restart_policy: { condition: 'any' }, + })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'on-failure' }, + })).toBe('on-failure'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: {}, + })).toBe('always'); + }); + + it('fails closed on malformed restart_policy shapes', () => { + expect(normalizeComposeRestartIntent('no', { restart_policy: null })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: 'none' })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { restart_policy: [] })).toBe('always'); + expect(normalizeComposeRestartIntent('no', { + restart_policy: { condition: 'weird' }, + })).toBe('always'); + }); +}); diff --git a/backend/src/__tests__/preflight-rules.test.ts b/backend/src/__tests__/preflight-rules.test.ts index aaab42cd..b6ffb191 100644 --- a/backend/src/__tests__/preflight-rules.test.ts +++ b/backend/src/__tests__/preflight-rules.test.ts @@ -212,7 +212,11 @@ describe('hygiene rules', () => { }); it('flags a missing restart policy and healthcheck', () => { const bare = model([svc({ restart: undefined, hasHealthcheck: false })]); - expect(ids(runRules(ctx({ model: bare })), 'no-restart-policy')).toHaveLength(1); + const restartFindings = ids(runRules(ctx({ model: bare })), 'no-restart-policy'); + expect(restartFindings).toHaveLength(1); + expect(restartFindings[0].remediation).toMatch(/one-shot|init jobs/i); + expect(restartFindings[0].remediation).toMatch(/restart: "no"/); + expect(restartFindings[0].remediation).toMatch(/unless-stopped/); expect(ids(runRules(ctx({ model: bare })), 'no-healthcheck')).toHaveLength(1); const withDeployRestart = model([svc({ restart: undefined, deploy: { restart_policy: { condition: 'any' } }})]); expect(ids(runRules(ctx({ model: withDeployRestart })), 'no-restart-policy')).toHaveLength(0); diff --git a/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts new file mode 100644 index 00000000..614ab340 --- /dev/null +++ b/backend/src/__tests__/proxy-service-scoped-alert-gate.test.ts @@ -0,0 +1,256 @@ +/** + * Mixed-version gate for POST /api/alerts with a non-empty service_name. + * + * Remote-targeted requests skip express.json() so the raw stream stays + * pipeable. The hub must still buffer that body under the local JSON size + * cap, reject compressed bodies (cannot inspect safely), reject scoped + * creates when the remote lacks service-scoped-stack-alert, and forward + * unscoped identity-encoded JSON intact when allowed. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import http from 'http'; +import type { AddressInfo } from 'net'; +import zlib from 'zlib'; +import request from 'supertest'; +import jwt from 'jsonwebtoken'; +import { setupTestDb, cleanupTestDb, TEST_USERNAME, TEST_JWT_SECRET } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let listenServer: http.Server; +let listenPort: number; +let adminBearer: string; +let capServer: http.Server; +let noCapServer: http.Server; +let capNodeId: number; +let noCapNodeId: number; + +const noCapPaths: string[] = []; +const capPaths: string[] = []; +let lastCapBody: Buffer | null = null; +let lastNoCapBody: Buffer | null = null; + +function metaServer( + capabilities: string[], + seen: string[], + onBody: (body: Buffer) => void, +): http.Server { + return http.createServer((req, res) => { + if (req.url) seen.push(req.url); + const chunks: Buffer[] = []; + req.on('data', (chunk: Buffer) => chunks.push(chunk)); + req.on('end', () => { + if (!req.url?.startsWith('/api/meta')) { + onBody(Buffer.concat(chunks)); + } + res.writeHead(200, { 'Content-Type': 'application/json' }); + if (req.url?.startsWith('/api/meta')) { + res.end(JSON.stringify({ version: '0.93.0', capabilities })); + } else { + res.end(JSON.stringify({ id: 1, ok: true })); + } + }); + }); +} + +async function listen(server: http.Server): Promise { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return (server.address() as AddressInfo).port; +} + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ app } = await import('../index')); + const { DatabaseService } = await import('../services/DatabaseService'); + const db = DatabaseService.getInstance(); + + adminBearer = jwt.sign({ username: TEST_USERNAME }, TEST_JWT_SECRET, { expiresIn: '1m' }); + + capServer = metaServer( + ['cross-node-rbac', 'service-scoped-stack-alert'], + capPaths, + (body) => { lastCapBody = body; }, + ); + noCapServer = metaServer( + ['cross-node-rbac'], + noCapPaths, + (body) => { lastNoCapBody = body; }, + ); + const capPort = await listen(capServer); + const noCapPort = await listen(noCapServer); + + capNodeId = db.addNode({ + name: 'alert-cap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${capPort}`, api_token: 'cap-token', + }); + noCapNodeId = db.addNode({ + name: 'alert-nocap-remote', type: 'remote', mode: 'proxy', compose_dir: '/tmp', + is_default: false, api_url: `http://127.0.0.1:${noCapPort}`, api_token: 'nocap-token', + }); + + listenServer = http.createServer(app); + listenPort = await listen(listenServer); +}); + +afterAll(async () => { + await new Promise((resolve) => listenServer.close(() => resolve())); + await new Promise((resolve) => capServer.close(() => resolve())); + await new Promise((resolve) => noCapServer.close(() => resolve())); + cleanupTestDb(tmpDir); +}); + +beforeEach(() => { + noCapPaths.length = 0; + capPaths.length = 0; + lastNoCapBody = null; + lastCapBody = null; +}); + +const scopedPayload = { + stack_name: 'web', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +const unscopedPayload = { + stack_name: 'web', + service_name: null, + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, +}; + +describe('remote proxy service-scoped alert gate', () => { + it('returns 400 for scoped alert create when remote lacks service-scoped-stack-alert', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(400); + expect(res.body.code).toBe('capability_unavailable'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(noCapPaths.some((p) => p.startsWith('/api/meta'))).toBe(true); + expect(lastNoCapBody).toBeNull(); + }); + + it('forwards unscoped alert JSON intact when remote lacks the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(unscopedPayload); + + expect(res.status).toBe(200); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastNoCapBody).not.toBeNull(); + expect(JSON.parse(lastNoCapBody!.toString('utf-8'))).toEqual(unscopedPayload); + }); + + it('forwards scoped alert JSON intact when remote advertises the capability', async () => { + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(capNodeId)) + .set('Content-Type', 'application/json') + .send(scopedPayload); + + expect(res.status).toBe(200); + expect(capPaths.some((p) => p.includes('/api/alerts'))).toBe(true); + expect(lastCapBody).not.toBeNull(); + expect(JSON.parse(lastCapBody!.toString('utf-8'))).toEqual(scopedPayload); + }); + + it('rejects oversized identity JSON via Content-Length before upstream', async () => { + const huge = { + ...unscopedPayload, + stack_name: 'x'.repeat(120 * 1024), + }; + + const started = Date.now(); + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .send(huge); + const elapsedMs = Date.now() - started; + + expect(res.status).toBe(413); + expect(res.body.code).toBe('entity_too_large'); + expect(elapsedMs).toBeLessThan(2000); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects chunked bodies once streamed bytes exceed the JSON limit', async () => { + const oversized = Buffer.from(JSON.stringify({ + ...unscopedPayload, + stack_name: 'y'.repeat(120 * 1024), + })); + + const result = await new Promise<{ status: number; body: { code?: string } }>((resolve, reject) => { + const req = http.request({ + hostname: '127.0.0.1', + port: listenPort, + method: 'POST', + path: '/api/alerts', + headers: { + Authorization: `Bearer ${adminBearer}`, + 'x-node-id': String(noCapNodeId), + 'Content-Type': 'application/json', + 'Transfer-Encoding': 'chunked', + }, + }, (res) => { + const chunks: Buffer[] = []; + res.on('data', (c: Buffer) => chunks.push(c)); + res.on('end', () => { + const text = Buffer.concat(chunks).toString('utf-8'); + try { + resolve({ status: res.statusCode ?? 0, body: JSON.parse(text) as { code?: string } }); + } catch { + resolve({ status: res.statusCode ?? 0, body: {} }); + } + }); + }); + req.on('error', reject); + // Write in small chunks so the hub crosses the limit mid-stream. + const step = 16 * 1024; + for (let offset = 0; offset < oversized.length; offset += step) { + req.write(oversized.subarray(offset, Math.min(offset + step, oversized.length))); + } + req.end(); + }); + + expect(result.status).toBe(413); + expect(result.body.code).toBe('entity_too_large'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); + + it('rejects gzip-encoded scoped JSON instead of forwarding as All-services', async () => { + const compressed = zlib.gzipSync(Buffer.from(JSON.stringify(scopedPayload), 'utf-8')); + + const res = await request(app) + .post('/api/alerts') + .set('Authorization', `Bearer ${adminBearer}`) + .set('x-node-id', String(noCapNodeId)) + .set('Content-Type', 'application/json') + .set('Content-Encoding', 'gzip') + .send(compressed); + + expect(res.status).toBe(415); + expect(res.body.code).toBe('encoding_unsupported'); + expect(noCapPaths.some((p) => p.includes('/api/alerts'))).toBe(false); + expect(lastNoCapBody).toBeNull(); + }); +}); diff --git a/backend/src/__tests__/remote-console-session.test.ts b/backend/src/__tests__/remote-console-session.test.ts index db53cbaa..ffc4f18a 100644 --- a/backend/src/__tests__/remote-console-session.test.ts +++ b/backend/src/__tests__/remote-console-session.test.ts @@ -30,10 +30,9 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { ({ app } = await import('../index')); adminCookie = await loginAsTestAdmin(app); - // POST /api/system/console-token is paid-gated. Seed an active license so the - // parity assertion can observe the token the route returns. The - // license_last_validated fallback is skipped when the state key is absent, so - // the active status alone drives the paid tier. + // POST /api/system/console-token is admin-gated (available on every tier). + // Seed an active license only so other suites that share LicenseService + // state stay stable if they expect paid. const { DatabaseService } = await import('../services/DatabaseService'); DatabaseService.getInstance().setSystemState('license_status', 'active'); }); @@ -43,12 +42,13 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { }); it('POST /api/system/console-token produces a token with the same shape as mintConsoleSession()', async () => { - const directToken = mintConsoleSession(); + const directToken = mintConsoleSession({ path: 'host-console', actingAs: 'testadmin' }); const directDecoded = jwt.verify(directToken, TEST_JWT_SECRET) as Record; const res = await request(app) .post('/api/system/console-token') - .set('Cookie', adminCookie); + .set('Cookie', adminCookie) + .send({ path: 'host-console' }); expect(res.status).toBe(200); expect(typeof res.body.token).toBe('string'); @@ -57,6 +57,12 @@ describe('console_session token parity (HTTP route vs mint helper)', () => { // Identical scope so the remote's upgrade handler treats both the same. expect(routeDecoded.scope).toBe('console_session'); expect(directDecoded.scope).toBe('console_session'); + expect(routeDecoded.path).toBe('host-console'); + expect(directDecoded.path).toBe('host-console'); + expect(typeof routeDecoded.jti).toBe('string'); + expect(typeof directDecoded.jti).toBe('string'); + expect(routeDecoded.acting_as).toBe('testadmin'); + expect(directDecoded.acting_as).toBe('testadmin'); // Same claim keys: if somebody adds or drops a claim on one path but not // the other, remote upgrade behavior will diverge. diff --git a/backend/src/__tests__/remote-ws-forward-gate.test.ts b/backend/src/__tests__/remote-ws-forward-gate.test.ts index 1b752ce8..6bd2594d 100644 --- a/backend/src/__tests__/remote-ws-forward-gate.test.ts +++ b/backend/src/__tests__/remote-ws-forward-gate.test.ts @@ -58,6 +58,13 @@ describe('remoteWsForwardAllowed', () => { expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false); }); + it('denies every API-token scope for Host Console while preserving full-admin for container exec', () => { + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'deploy-only', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(CONSOLE, { ...base, wsApiTokenScope: 'read-only', decoded: { scope: 'api_token' } })).toBe(false); + expect(remoteWsForwardAllowed(EXEC, { ...base, wsApiTokenScope: 'full-admin', decoded: { scope: 'api_token' } })).toBe(true); + }); + it('allows an interactive path for a pre-gated console_session token', () => { expect(remoteWsForwardAllowed(CONSOLE, { ...base, decoded: { scope: 'console_session' } })).toBe(true); }); diff --git a/backend/src/__tests__/sanitize-network-inspect.test.ts b/backend/src/__tests__/sanitize-network-inspect.test.ts index 316f5934..7358e52f 100644 --- a/backend/src/__tests__/sanitize-network-inspect.test.ts +++ b/backend/src/__tests__/sanitize-network-inspect.test.ts @@ -13,7 +13,7 @@ describe('sanitizeNetworkInspect connected containers', () => { networks: [{ id: 'net1', name: 'app_net', driver: 'bridge', scope: 'local', isSystem: false, composeProject: 'app', stack: 'app' }], volumes: [], containers: [{ - id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', image: 'nginx', + id: 'c1', name: 'app-web-1', service: 'web', composeProject: 'app', stack: 'app', state: 'running', exitCode: null, image: 'nginx', networks: [{ name: 'app_net', id: 'net1', ip: '172.20.0.5/16' }], volumes: [], ports: [], }], }; diff --git a/backend/src/__tests__/scheduler-policy.test.ts b/backend/src/__tests__/scheduler-policy.test.ts index 0edca7c4..0739c6af 100644 --- a/backend/src/__tests__/scheduler-policy.test.ts +++ b/backend/src/__tests__/scheduler-policy.test.ts @@ -34,7 +34,7 @@ const { mockMarkStaleRunsAsFailed: vi.fn().mockReturnValue(0), mockDeleteOldScans: vi.fn().mockReturnValue(0), mockGetTier: vi.fn().mockReturnValue('paid'), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn(), diff --git a/backend/src/__tests__/scheduler-service.test.ts b/backend/src/__tests__/scheduler-service.test.ts index 18f8ae4f..f629f717 100644 --- a/backend/src/__tests__/scheduler-service.test.ts +++ b/backend/src/__tests__/scheduler-service.test.ts @@ -59,7 +59,7 @@ const { mockGetStackContent: vi.fn().mockResolvedValue(''), mockGetEnvContent: vi.fn().mockResolvedValue(''), mockCheckImage: vi.fn().mockResolvedValue({ hasUpdate: false }), - mockDispatchAlert: vi.fn().mockResolvedValue(undefined), + mockDispatchAlert: vi.fn().mockResolvedValue({ persisted: true }), mockGetProxyTarget: vi.fn().mockReturnValue(null), mockIsTrivyAvailable: vi.fn().mockReturnValue(true), mockScanAllNodeImages: vi.fn().mockResolvedValue({ diff --git a/backend/src/__tests__/service-scoped-update-routes.test.ts b/backend/src/__tests__/service-scoped-update-routes.test.ts index f2564257..5811e4e9 100644 --- a/backend/src/__tests__/service-scoped-update-routes.test.ts +++ b/backend/src/__tests__/service-scoped-update-routes.test.ts @@ -45,7 +45,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-bulk-routes.test.ts b/backend/src/__tests__/stack-bulk-routes.test.ts index 3d9fe675..f62454c8 100644 --- a/backend/src/__tests__/stack-bulk-routes.test.ts +++ b/backend/src/__tests__/stack-bulk-routes.test.ts @@ -78,7 +78,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts index 037d6ed5..e1ad3b0a 100644 --- a/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts +++ b/backend/src/__tests__/stack-delete-cascades-mesh-opt-out.test.ts @@ -131,3 +131,32 @@ describe('DELETE /api/stacks/:stackName mesh opt-out cascade (F-1 / F-14)', () = expect(sawCascadeWarning).toBe(true); }); }); + +describe('DELETE /api/stacks/:stackName clears stack-scoped role assignments', () => { + it('removes only the deleted stack assignment and preserves unrelated grants', async () => { + const db = DatabaseService.getInstance(); + const bcrypt = await import('bcrypt'); + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: 'stack-del-rbac', password_hash: hash, role: 'viewer' }); + const otherNodeId = db.addNode({ + name: 'stack-del-rbac-node', type: 'remote', api_url: 'http://test:1852', + api_token: '', compose_dir: '/tmp', is_default: false, + }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'api' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'other-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(otherNodeId) }); + + const res = await request(app) + .delete('/api/stacks/api') + .set('Cookie', adminCookie); + + expect(res.status).toBe(200); + const remaining = db.getAllRoleAssignments(userId); + expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'api')).toBe(false); + expect(remaining.some((a) => a.resource_type === 'stack' && a.resource_id === 'other-stack')).toBe(true); + expect(remaining.some((a) => a.resource_type === 'node' && a.resource_id === String(otherNodeId))).toBe(true); + + db.deleteUser(userId); + db.deleteNode(otherNodeId); + }); +}); diff --git a/backend/src/__tests__/stack-delete-purges-notifications.test.ts b/backend/src/__tests__/stack-delete-purges-notifications.test.ts new file mode 100644 index 00000000..10aae69b --- /dev/null +++ b/backend/src/__tests__/stack-delete-purges-notifications.test.ts @@ -0,0 +1,132 @@ +/** + * DELETE /api/stacks/:stackName must purge notification_history for that + * (node_id, stack_name) so the bell panel, Activity ticker, Stack Activity, + * and Networking recent activity no longer show the deleted stack. + */ +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import request from 'supertest'; +import { setupTestDb, cleanupTestDb, loginAsTestAdmin } from './helpers/setupTestDb'; + +let tmpDir: string; +let app: import('express').Express; +let DatabaseService: typeof import('../services/DatabaseService').DatabaseService; +let ComposeService: typeof import('../services/ComposeService').ComposeService; +let FileSystemService: typeof import('../services/FileSystemService').FileSystemService; +let MeshService: typeof import('../services/MeshService').MeshService; +let NotificationService: typeof import('../services/NotificationService').NotificationService; +let adminCookie: string; + +beforeAll(async () => { + tmpDir = await setupTestDb(); + ({ DatabaseService } = await import('../services/DatabaseService')); + ({ ComposeService } = await import('../services/ComposeService')); + ({ FileSystemService } = await import('../services/FileSystemService')); + ({ MeshService } = await import('../services/MeshService')); + ({ NotificationService } = await import('../services/NotificationService')); + ({ app } = await import('../index')); + adminCookie = await loginAsTestAdmin(app); +}); + +afterAll(() => cleanupTestDb(tmpDir)); + +beforeEach(() => { + vi.restoreAllMocks(); + vi.spyOn(ComposeService.prototype, 'downStack').mockResolvedValue(undefined); + vi.spyOn(FileSystemService.prototype, 'deleteStack').mockResolvedValue(undefined); + vi.spyOn(MeshService.getInstance(), 'optOutStack').mockResolvedValue(undefined); + const raw = DatabaseService.getInstance().getDb(); + raw.prepare('DELETE FROM notification_history').run(); +}); + +function seedNotif( + nodeId: number, + opts: { stack_name?: string; message?: string; timestamp?: number }, +): void { + DatabaseService.getInstance().addNotificationHistory(nodeId, { + level: 'error', + message: opts.message ?? 'test', + timestamp: opts.timestamp ?? Date.now(), + stack_name: opts.stack_name ?? 'web', + category: 'deploy_failure', + }); +} + +function stackNamesForNode(nodeId: number): Array { + const rows = DatabaseService.getInstance() + .getDb() + .prepare('SELECT stack_name FROM notification_history WHERE node_id = ? ORDER BY id') + .all(nodeId) as Array<{ stack_name: string | null }>; + return rows.map((r) => r.stack_name); +} + +describe('DELETE /api/stacks/:stackName purges notifications', () => { + it('removes only the target node/stack rows and emits one invalidation', async () => { + const db = DatabaseService.getInstance(); + const nodeA = db.getNodes()[0].id; + const nodeB = db.addNode({ + name: 'other-node', + type: 'remote', + api_url: 'http://other:1852', + api_token: 't', + compose_dir: '/tmp', + is_default: false, + }); + + seedNotif(nodeA, { stack_name: 'web', message: 'a-web' }); + seedNotif(nodeA, { stack_name: 'db', message: 'a-db' }); + // Unattached (null stack_name) via SQL; addNotificationHistory omits null. + db.getDb() + .prepare( + `INSERT INTO notification_history (node_id, level, message, timestamp, is_read, stack_name, category) + VALUES (?, 'info', 'a-unattached', ?, 0, NULL, 'system')`, + ) + .run(nodeA, Date.now()); + seedNotif(nodeB, { stack_name: 'web', message: 'b-web' }); + + const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent'); + + const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie); + + expect(res.status).toBe(200); + expect(stackNamesForNode(nodeA).sort((a, b) => String(a).localeCompare(String(b)))).toEqual([ + 'db', + null, + ]); + expect(stackNamesForNode(nodeB)).toEqual(['web']); + + const notifInvalidations = broadcastSpy.mock.calls + .map((c) => c[0]) + .filter( + (e) => + e.type === 'state-invalidate' && + e.scope === 'notifications' && + e.action === 'stack-deleted', + ); + expect(notifInvalidations).toHaveLength(1); + expect(notifInvalidations[0]).toMatchObject({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: nodeA, + stackName: 'web', + }); + expect(typeof notifInvalidations[0].ts).toBe('number'); + }); + + it('emits no notification invalidation when delete fails before success', async () => { + vi.spyOn(FileSystemService.prototype, 'deleteStack').mockRejectedValue(new Error('fs boom')); + const broadcastSpy = vi.spyOn(NotificationService.getInstance(), 'broadcastEvent'); + const db = DatabaseService.getInstance(); + const nodeId = db.getNodes()[0].id; + seedNotif(nodeId, { stack_name: 'web' }); + + const res = await request(app).delete('/api/stacks/web').set('Cookie', adminCookie); + + expect(res.status).toBe(500); + expect(stackNamesForNode(nodeId)).toEqual(['web']); + const notifInvalidations = broadcastSpy.mock.calls + .map((c) => c[0]) + .filter((e) => e.scope === 'notifications' && e.action === 'stack-deleted'); + expect(notifInvalidations).toHaveLength(0); + }); +}); diff --git a/backend/src/__tests__/stack-docker-disconnect.test.ts b/backend/src/__tests__/stack-docker-disconnect.test.ts index 90dfcf8f..f1150bfa 100644 --- a/backend/src/__tests__/stack-docker-disconnect.test.ts +++ b/backend/src/__tests__/stack-docker-disconnect.test.ts @@ -90,7 +90,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-down-remove-volumes.test.ts b/backend/src/__tests__/stack-down-remove-volumes.test.ts index e390a3d5..117b8bb7 100644 --- a/backend/src/__tests__/stack-down-remove-volumes.test.ts +++ b/backend/src/__tests__/stack-down-remove-volumes.test.ts @@ -68,7 +68,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + dispatchAlertSpy = vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-files-routes.test.ts b/backend/src/__tests__/stack-files-routes.test.ts index 37373309..2195bb10 100644 --- a/backend/src/__tests__/stack-files-routes.test.ts +++ b/backend/src/__tests__/stack-files-routes.test.ts @@ -1116,6 +1116,37 @@ describe('PUT /api/stacks/:stackName/files/content', () => { expect(content).toBe('community-write'); }); + it('blocks root trust file writes while a stack op lock is held', async () => { + const { StackOpLockService } = await import('../services/StackOpLockService'); + StackOpLockService.getInstance().tryAcquire(1, STACK, 'deploy', 'admin'); + try { + const composeRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: 'compose.yaml' }) + .set('Cookie', adminCookie) + .send({ content: 'services:\n app:\n image: nginx\n' }); + expect(composeRes.status).toBe(409); + expect(composeRes.body.code).toBe('stack_op_in_progress'); + + const markerRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: '.blueprint.json' }) + .set('Cookie', adminCookie) + .send({ content: '{"blueprintId":1,"revision":1,"lastApplied":0}' }); + expect(markerRes.status).toBe(409); + expect(markerRes.body.code).toBe('stack_op_in_progress'); + + const nestedRes = await request(app) + .put(`/api/stacks/${STACK}/files/content`) + .query({ path: 'config/app.conf' }) + .set('Cookie', adminCookie) + .send({ content: 'ok' }); + expect(nestedRes.status).toBe(204); + } finally { + StackOpLockService.getInstance().release(1, STACK); + } + }); + it('returns 400 when content is not a string', async () => { const res = await request(app) .put(`/api/stacks/${STACK}/files/content`) @@ -1931,6 +1962,16 @@ describe('protected stack files', () => { expect(res.body.code).toBe('PROTECTED_FILE'); }); + it('DELETE /files refuses .blueprint.json with 409 PROTECTED_FILE', async () => { + await fs.writeFile(path.join(stacksDir, STACK, '.blueprint.json'), '{"blueprintId":1,"revision":1}\n'); + const res = await request(app) + .delete(`/api/stacks/${STACK}/files`) + .query({ path: '.blueprint.json' }) + .set('Cookie', adminCookie); + expect(res.status).toBe(409); + expect(res.body.code).toBe('PROTECTED_FILE'); + }); + it('PATCH /files/rename refuses compose.yaml as source with 409 PROTECTED_FILE', async () => { const res = await request(app) .patch(`/api/stacks/${STACK}/files/rename`) diff --git a/backend/src/__tests__/stack-op-lock-routes.test.ts b/backend/src/__tests__/stack-op-lock-routes.test.ts index 08a1fb45..4d76ea95 100644 --- a/backend/src/__tests__/stack-op-lock-routes.test.ts +++ b/backend/src/__tests__/stack-op-lock-routes.test.ts @@ -89,7 +89,7 @@ beforeAll(async () => { authCookie = await loginAsTestAdmin(app); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stack-self-protected-routes.test.ts b/backend/src/__tests__/stack-self-protected-routes.test.ts index 315a33e8..db7330d5 100644 --- a/backend/src/__tests__/stack-self-protected-routes.test.ts +++ b/backend/src/__tests__/stack-self-protected-routes.test.ts @@ -102,7 +102,7 @@ beforeAll(async () => { writeStack('web'); const { NotificationService } = await import('../services/NotificationService'); - vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue(undefined); + vi.spyOn(NotificationService.getInstance(), 'dispatchAlert').mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/stacks-failure-notifications.test.ts b/backend/src/__tests__/stacks-failure-notifications.test.ts index 168f0877..588e311b 100644 --- a/backend/src/__tests__/stacks-failure-notifications.test.ts +++ b/backend/src/__tests__/stacks-failure-notifications.test.ts @@ -132,7 +132,7 @@ beforeAll(async () => { const { NotificationService } = await import('../services/NotificationService'); dispatchAlertSpy = vi .spyOn(NotificationService.getInstance(), 'dispatchAlert') - .mockResolvedValue(undefined); + .mockResolvedValue({ persisted: true }); }); afterAll(() => { diff --git a/backend/src/__tests__/upgrade-order.test.ts b/backend/src/__tests__/upgrade-order.test.ts index dc5a6056..1f1cf15a 100644 --- a/backend/src/__tests__/upgrade-order.test.ts +++ b/backend/src/__tests__/upgrade-order.test.ts @@ -8,7 +8,7 @@ * product paths. This test pins each position by observing behavior that * could only originate from the expected handler. */ -import { describe, it, expect, beforeAll, afterAll, afterEach } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from 'vitest'; import WebSocket from 'ws'; import jwt from 'jsonwebtoken'; import bcrypt from 'bcrypt'; @@ -420,6 +420,100 @@ describe('WebSocket upgrade dispatch order', () => { expect(outcome.kind).toBe('unexpected'); if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); }); + + it('rejects a full-admin API token remote Host Console with 403 (container exec stays allowed)', async () => { + const { DatabaseService } = await import('../services/DatabaseService'); + const adminId = DatabaseService.getInstance().getUserByUsername(TEST_USERNAME)!.id; + const rawToken = createTestApiToken({ + db: DatabaseService, + scope: 'full-admin', + userId: adminId, + name: `host-console-api-deny-${Date.now()}`, + }); + const consoleWs = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { bearer: rawToken }); + const consoleOutcome = await waitForOutcome(consoleWs); + expect(consoleOutcome.kind).toBe('unexpected'); + if (consoleOutcome.kind === 'unexpected') expect(consoleOutcome.status).toBe(403); + + const execWs = connect(`/ws?nodeId=${remoteNodeId}`, { bearer: rawToken }); + const execOutcome = await waitForOutcome(execWs); + if (execOutcome.kind === 'unexpected') expect(execOutcome.status).not.toBe(403); + try { execWs.terminate(); } catch { /* ignore */ } + }); + }); + + describe('remote Host Console Community capability probe', () => { + // Community hubs must probe host-console-community before forwarding Host + // Console; Admiral hubs skip the probe so legacy remotes still work. + // Fail closed when the probe errors or the capability is missing. + + const meta = (capabilities: string[]): import('../services/CapabilityRegistry').RemoteMeta => ({ + version: '0.95.0', + capabilities, + startedAt: 1, + updateError: null, + online: true, + imagePinKind: null, + updateBlocked: false, + imageChannel: null, + }); + + async function setLicense(status: string): Promise { + const { DatabaseService } = await import('../services/DatabaseService'); + DatabaseService.getInstance().setSystemState('license_status', status); + } + + afterEach(async () => { + vi.restoreAllMocks(); + await setLicense('active'); + }); + + it('rejects Community hub remote Host Console when the remote lacks host-console-community', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta(['host-console'])); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + + it('rejects Community hub remote Host Console when the capability probe fails', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockRejectedValue(new Error('offline')); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + expect(outcome.kind).toBe('unexpected'); + if (outcome.kind === 'unexpected') expect(outcome.status).toBe(403); + }); + + it('forwards Community hub remote Host Console when the remote advertises host-console-community', async () => { + await setLicense('community'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue( + meta(['host-console-community']), + ); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403); + try { ws.terminate(); } catch { /* ignore */ } + }); + + it('skips the capability probe on an Admiral hub even when the remote lacks host-console-community', async () => { + await setLicense('active'); + const { NodeRegistry } = await import('../services/NodeRegistry'); + const spy = vi.spyOn(NodeRegistry.getInstance(), 'fetchMetaForNode').mockResolvedValue(meta([])); + + const ws = connect(`/api/system/host-console?nodeId=${remoteNodeId}`, { cookie: sessionCookie }); + const outcome = await waitForOutcome(ws); + if (outcome.kind === 'unexpected') expect(outcome.status).not.toBe(403); + expect(spy).not.toHaveBeenCalled(); + try { ws.terminate(); } catch { /* ignore */ } + }); }); it('dispatches /api/pilot/tunnel to the pilot handler (rejects non-pilot bearer before path-based dispatch)', async () => { diff --git a/backend/src/__tests__/users-rbac.test.ts b/backend/src/__tests__/users-rbac.test.ts index a6d01eca..c5917576 100644 --- a/backend/src/__tests__/users-rbac.test.ts +++ b/backend/src/__tests__/users-rbac.test.ts @@ -714,6 +714,29 @@ describe('Orphaned role assignment cleanup', () => { // Cleanup db.deleteUser(userId); }); + + it('deleteRoleAssignmentsByResource removes only the matching resource tuple', async () => { + const db = DatabaseService.getInstance(); + const hash = await bcrypt.hash('password123', 1); + const userId = db.addUser({ username: 'tupleorphan', password_hash: hash, role: 'viewer' }); + const nodeId = db.addNode({ + name: 'tuple-cleanup-node', type: 'remote', api_url: 'http://test:1852', + api_token: '', compose_dir: '/tmp', is_default: false, + }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'target-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'stack', resource_id: 'keep-stack' }); + db.addRoleAssignment({ user_id: userId, role: 'deployer', resource_type: 'node', resource_id: String(nodeId) }); + + db.deleteRoleAssignmentsByResource('stack', 'target-stack'); + + const after = db.getAllRoleAssignments(userId); + expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'target-stack')).toBe(false); + expect(after.some((a) => a.resource_type === 'stack' && a.resource_id === 'keep-stack')).toBe(true); + expect(after.some((a) => a.resource_type === 'node' && a.resource_id === String(nodeId))).toBe(true); + + db.deleteUser(userId); + db.deleteNode(nodeId); + }); }); // ---- Role-Based Permission Checks (via API) ---- diff --git a/backend/src/helpers/blueprintMarker.ts b/backend/src/helpers/blueprintMarker.ts new file mode 100644 index 00000000..d341f594 --- /dev/null +++ b/backend/src/helpers/blueprintMarker.ts @@ -0,0 +1,28 @@ +/** + * Dependency-neutral blueprint marker parse helpers. + * Used by BlueprintService and DeployedStackDeletionService without a service import cycle. + */ + +export const BLUEPRINT_MARKER_FILENAME = '.blueprint.json'; + +export interface BlueprintMarker { + blueprintId: number; + revision: number; + lastApplied: number; +} + +export function parseBlueprintMarker(content: string): BlueprintMarker | null { + try { + const parsed: unknown = JSON.parse(content); + if (!parsed || typeof parsed !== 'object') return null; + const obj = parsed as Record; + if (typeof obj.blueprintId !== 'number' || typeof obj.revision !== 'number') return null; + return { + blueprintId: obj.blueprintId, + revision: obj.revision, + lastApplied: typeof obj.lastApplied === 'number' ? obj.lastApplied : 0, + }; + } catch { + return null; + } +} diff --git a/backend/src/helpers/composeDependencyParse.ts b/backend/src/helpers/composeDependencyParse.ts index f8059fd0..e73f94ab 100644 --- a/backend/src/helpers/composeDependencyParse.ts +++ b/backend/src/helpers/composeDependencyParse.ts @@ -26,6 +26,8 @@ export interface DeclaredService { image?: string; /** Service `network_mode:` as declared (e.g. `host`), or undefined. */ networkMode?: string; + /** Service `restart` from raw YAML parse. The effective-model path may set this via normalizeComposeRestartIntent (including deploy.restart_policy). Null when unset. */ + restart?: string | null; } /** A top-level networks:/volumes: entry. */ @@ -200,6 +202,7 @@ export function parseComposeDependencies(content: string): DeclaredCompose { ports, image: asString(svc.image), networkMode: asString(svc.network_mode), + restart: asString(svc.restart) ?? null, }); } diff --git a/backend/src/helpers/consoleSession.ts b/backend/src/helpers/consoleSession.ts index 90565e31..6b32706b 100644 --- a/backend/src/helpers/consoleSession.ts +++ b/backend/src/helpers/consoleSession.ts @@ -1,17 +1,55 @@ +import { randomUUID } from 'crypto'; import jwt from 'jsonwebtoken'; import { DatabaseService } from '../services/DatabaseService'; /** - * Console session token lifetime. Short on purpose: these tokens are only - * used to bridge an already-authenticated HTTP request into a WebSocket - * upgrade, and each one is consumed by a single `wss:ws(target)` call. + * Console session token lifetime. Short on purpose: these tokens bridge an + * already-authenticated hub request into a remote WebSocket upgrade. Each + * token is path-scoped and consumed on first interactive WebSocket upgrade + * acceptance (before PTY spawn). */ const CONSOLE_SESSION_TTL_SECONDS = 60; const CONSOLE_SESSION_SCOPE = 'console_session'; +/** Interactive surfaces that may mint or accept a console_session JWT. */ +export type ConsoleSessionPath = 'host-console' | 'container-exec'; + +export interface MintConsoleSessionOptions { + path: ConsoleSessionPath; + /** Hub operator identity for remote audit (option B). Never used as the session principal. */ + actingAs?: string; +} + export interface ConsoleSessionClaims { scope: typeof CONSOLE_SESSION_SCOPE; - username?: string; + path: ConsoleSessionPath; + acting_as?: string; +} + +const ACTING_AS_MAX = 64; +const ACTING_AS_RE = /^[A-Za-z0-9._@+-]+$/; + +/** Sanitize an optional hub operator name for the acting_as claim. */ +export function sanitizeActingAs(raw: unknown): string | undefined { + if (typeof raw !== 'string') return undefined; + const trimmed = raw.trim(); + if (!trimmed || trimmed.length > ACTING_AS_MAX) return undefined; + if (!ACTING_AS_RE.test(trimmed)) return undefined; + return trimmed; +} + +export function isConsoleSessionPath(value: unknown): value is ConsoleSessionPath { + return value === 'host-console' || value === 'container-exec'; +} + +/** + * Map a WebSocket pathname to the console_session path claim it requires. + * Returns null for surfaces that must not accept console_session tokens. + */ +export function consoleSessionPathForPathname(pathname: string): ConsoleSessionPath | null { + if (pathname.startsWith('/api/system/host-console')) return 'host-console'; + if (pathname === '/ws') return 'container-exec'; + return null; } /** @@ -19,13 +57,19 @@ export interface ConsoleSessionClaims { * instance without leaking the long-lived node api_token onto a * machine-to-machine WebSocket. Throws if the JWT secret is not configured. */ -export function mintConsoleSession(username?: string): string { +export function mintConsoleSession(opts: MintConsoleSessionOptions): string { + if (!isConsoleSessionPath(opts.path)) { + throw new Error('Invalid console session path'); + } const jwtSecret = DatabaseService.getInstance().getGlobalSettings().auth_jwt_secret; if (!jwtSecret) throw new Error('No JWT secret configured'); - const payload: ConsoleSessionClaims = username - ? { scope: CONSOLE_SESSION_SCOPE, username } - : { scope: CONSOLE_SESSION_SCOPE }; - return jwt.sign(payload, jwtSecret, { expiresIn: CONSOLE_SESSION_TTL_SECONDS }); + const payload: ConsoleSessionClaims = { scope: CONSOLE_SESSION_SCOPE, path: opts.path }; + const actingAs = sanitizeActingAs(opts.actingAs); + if (actingAs) payload.acting_as = actingAs; + return jwt.sign(payload, jwtSecret, { + expiresIn: CONSOLE_SESSION_TTL_SECONDS, + jwtid: randomUUID(), + }); } /** True when `decoded.scope` is the console_session scope. */ @@ -33,4 +77,14 @@ export function isConsoleSessionScope(scope: unknown): boolean { return scope === CONSOLE_SESSION_SCOPE; } -export { CONSOLE_SESSION_SCOPE }; +/** + * Mark a console_session jti as used. Returns false when the jti is missing, + * already consumed, or cannot be recorded (replay / malformed token). + */ +export function consumeConsoleSessionJti(jti: unknown, expiresAtMs: number): boolean { + if (typeof jti !== 'string' || !jti) return false; + if (!Number.isFinite(expiresAtMs)) return false; + return DatabaseService.getInstance().consumeConsoleSessionJti(jti, expiresAtMs); +} + +export { CONSOLE_SESSION_SCOPE, CONSOLE_SESSION_TTL_SECONDS }; diff --git a/backend/src/helpers/effectiveToDeclaredCompose.ts b/backend/src/helpers/effectiveToDeclaredCompose.ts index 64325c07..bf73690d 100644 --- a/backend/src/helpers/effectiveToDeclaredCompose.ts +++ b/backend/src/helpers/effectiveToDeclaredCompose.ts @@ -1,5 +1,6 @@ import type { EffectiveModel, EffResource } from '../services/preflight/effectiveModel'; import type { DeclaredCompose, DeclaredPort, DeclaredResource, DeclaredService } from './composeDependencyParse'; +import { normalizeComposeRestartIntent } from '../utils/oneShotCompletion'; /** Map one rendered network/volume entry to the declared shape drift expects. */ function toDeclaredResource(key: string, res: EffResource, projectName: string): DeclaredResource { @@ -41,6 +42,7 @@ export function declaredFromEffectiveModel(model: EffectiveModel): DeclaredCompo ), image: s.image, networkMode: s.networkMode, + restart: normalizeComposeRestartIntent(s.restart, s.deploy), })); return { diff --git a/backend/src/proxy/remoteNodeProxy.ts b/backend/src/proxy/remoteNodeProxy.ts index b359dd0c..e3a35185 100644 --- a/backend/src/proxy/remoteNodeProxy.ts +++ b/backend/src/proxy/remoteNodeProxy.ts @@ -5,7 +5,7 @@ import { PROXY_TIER_HEADER, PROXY_ROLE_HEADER, PROXY_DEPLOY_SOURCE_HEADER, PROXY import { LicenseService } from '../services/LicenseService'; import { isProxyExemptPath } from '../helpers/proxyExemptPaths'; import { remoteSupportsCrossNodeRbac, remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; -import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY } from '../services/CapabilityRegistry'; +import { STACK_DOWN_REMOVE_VOLUMES_CAPABILITY, SERVICE_SCOPED_UPDATE_CAPABILITY, SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '../services/CapabilityRegistry'; import { getErrorMessage } from '../utils/errors'; import { DatabaseService } from '../services/DatabaseService'; import { redactSensitiveText } from '../utils/safeLog'; @@ -145,8 +145,19 @@ export function createRemoteProxyMiddleware(): RequestHandler { } // Body forwarding: conditionalJsonParser skips parsing for remote // requests (see middleware/jsonParser.ts), so req's raw stream is - // intact and http-proxy's req.pipe(proxyReq) forwards the body - // automatically. + // usually intact and http-proxy's req.pipe(proxyReq) forwards it. + // When a gate must inspect JSON (POST /alerts), we buffer into + // req.rawBody first; rewrite that buffer here because the stream is + // already consumed. + if (req.rawBody) { + proxyReq.removeHeader('Transfer-Encoding'); + proxyReq.removeHeader('Content-Length'); + if (!proxyReq.getHeader('Content-Type')) { + proxyReq.setHeader('Content-Type', 'application/json'); + } + proxyReq.setHeader('Content-Length', req.rawBody.length); + proxyReq.write(req.rawBody); + } }, proxyRes: (proxyRes, req) => { // Mark every response forwarded from a remote node with a sentinel @@ -246,7 +257,8 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } - // Mixed-version RBAC gate (non-admin only). + // Mixed-version RBAC gate (non-admin only). Runs before alert body + // buffering so an unauthorized client cannot force unbounded memory use. if (req.user?.role !== 'admin') { const rbacSupported = await remoteSupportsCrossNodeRbac(req.nodeId); if (!rbacSupported) { @@ -257,6 +269,49 @@ export function createRemoteProxyMiddleware(): RequestHandler { } } + // POST /alerts bodies are not on req.body for remote hops (JSON parsing + // is skipped so the stream can be piped). Buffer once under the same + // 100 KB cap as express.json(), gate on service_name, then rewrite + // rawBody in on.proxyReq. Compressed bodies are rejected: the hub cannot + // inspect them, and treating parse failure as unscoped would bypass the + // mixed-version gate. + if (isAlertCreateRoute(req)) { + if (hasNonIdentityContentEncoding(req)) { + await drainRequestBody(req); + res.status(415).json({ + error: 'Compressed request bodies are not supported for remote alert creates', + code: 'encoding_unsupported', + }); + return; + } + try { + req.rawBody = await bufferRequestBody(req, ALERT_PROXY_BODY_LIMIT); + } catch (err) { + const status = Number((err as { status?: number }).status); + if (status === 413) { + console.error('[remoteNodeProxy] alert body rejected as too large:', err); + res.status(413).json({ error: 'Alert payload too large', code: 'entity_too_large' }); + return; + } + if (status === 400) { + console.error('[remoteNodeProxy] alert body incomplete:', err); + res.status(400).json({ error: 'Incomplete request body' }); + return; + } + throw err; + } + if (alertCreateHasScopedService(req.rawBody)) { + const supported = await remoteAdvertisesCapability(req.nodeId, SERVICE_SCOPED_STACK_ALERT_CAPABILITY); + if (!supported) { + res.status(400).json({ + error: 'Service-scoped alert rules are not supported on this node', + code: 'capability_unavailable', + }); + return; + } + } + } + req.proxyTarget = target; beginProxyTiming(req, res); proxy(req, res, next); @@ -283,3 +338,151 @@ function isServiceScopedUpdateRoute(req: Request): boolean { } return false; } + +/** POST /alerts (path is post-/api strip). */ +function isAlertCreateRoute(req: Request): boolean { + return req.method === 'POST' && /^\/alerts\/?$/.test(req.path); +} + +/** Same default as express.json(); remote alert creates must not exceed it. */ +const ALERT_PROXY_BODY_LIMIT = 100 * 1024; + +/** Max time to wait for leftover body bytes after a size/encoding reject. */ +const DRAIN_TIMEOUT_MS = 5_000; + +/** Error with HTTP status for the alert-body gate catch mapper. */ +function alertBodyError(message: string, status: number): Error { + return Object.assign(new Error(message), { status, expose: true }); +} + +/** True when Content-Encoding is present and not identity (gzip/deflate/br/…). */ +function hasNonIdentityContentEncoding(req: Request): boolean { + const raw = req.headers['content-encoding']; + if (raw == null) return false; + const value = Array.isArray(raw) ? raw.join(',') : raw; + return value.split(',').some((part) => { + const encoding = part.trim().toLowerCase(); + return encoding.length > 0 && encoding !== 'identity'; + }); +} + +/** True when the buffered JSON alert body targets a specific Compose service. */ +function alertCreateHasScopedService(rawBody: Buffer): boolean { + if (rawBody.length === 0) return false; + try { + const parsed = JSON.parse(rawBody.toString('utf-8')) as { service_name?: unknown }; + return typeof parsed.service_name === 'string' && parsed.service_name.trim() !== ''; + } catch { + // Identity-encoded non-JSON is forwarded as-is; the remote rejects it. + // Encoded bodies never reach here (rejected earlier). + return false; + } +} + +/** + * Consume remaining request bytes (or wait for abort/close) so the response + * can flush without leaving the socket half-open. Does not buffer into memory. + * Caps wait time so a stalled or endless chunked stream cannot hang the gate. + */ +function drainRequestBody(req: Request): Promise { + return new Promise((resolve) => { + if (req.readableEnded || req.destroyed) { + resolve(); + return; + } + let settled = false; + const done = () => { + if (settled) return; + settled = true; + clearTimeout(timer); + req.off('end', done); + req.off('error', done); + req.off('aborted', done); + req.off('close', done); + resolve(); + }; + // Bound hang time: destroy after timeout so mid-stream rejects still settle. + const timer = setTimeout(() => { + if (!req.destroyed) req.destroy(); + done(); + }, DRAIN_TIMEOUT_MS); + req.resume(); + req.once('end', done); + req.once('error', done); + req.once('aborted', done); + req.once('close', done); + }); +} + +/** + * Buffer the request so a capability gate can inspect JSON without leaving + * http-proxy with an already-ended empty stream. Enforces `limit` on both + * declared Content-Length and streamed accumulation. + */ +async function bufferRequestBody(req: Request, limit: number): Promise { + if (req.rawBody) { + if (req.rawBody.length > limit) { + throw alertBodyError('Alert payload too large', 413); + } + return req.rawBody; + } + if (req.readableEnded) return Buffer.alloc(0); + + const declared = Number.parseInt(String(req.headers['content-length'] ?? ''), 10); + if (Number.isFinite(declared) && declared > limit) { + // Reject immediately so the client gets a structured 413; drain leftover + // bytes in the background so the socket can close without holding the gate. + void drainRequestBody(req); + throw alertBodyError('Alert payload too large', 413); + } + + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + cleanup(); + fn(); + }; + const finish = (buf: Buffer) => settle(() => resolve(buf)); + const fail = (err: Error) => settle(() => reject(err)); + + const onData = (chunk: Buffer) => { + const buf = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buf.length; + if (total > limit) { + chunks.length = 0; + // fail() removes listeners; drain leftover bytes so the socket can close. + fail(alertBodyError('Alert payload too large', 413)); + void drainRequestBody(req); + return; + } + chunks.push(buf); + }; + const onEnd = () => finish(Buffer.concat(chunks)); + const onError = (err: Error) => fail(err); + const onAborted = () => fail(alertBodyError('Client aborted request body', 400)); + const onClose = () => { + if (!settled && !req.readableEnded) { + fail(alertBodyError('Client closed request before body finished', 400)); + } + }; + + function cleanup(): void { + req.off('data', onData); + req.off('end', onEnd); + req.off('error', onError); + req.off('aborted', onAborted); + req.off('close', onClose); + } + + req.on('data', onData); + req.on('end', onEnd); + req.on('error', onError); + req.on('aborted', onAborted); + req.on('close', onClose); + }); +} diff --git a/backend/src/routes/alerts.ts b/backend/src/routes/alerts.ts index b6bea853..1b707c1a 100644 --- a/backend/src/routes/alerts.ts +++ b/backend/src/routes/alerts.ts @@ -3,9 +3,21 @@ import { z } from 'zod'; import { DatabaseService } from '../services/DatabaseService'; import { authMiddleware } from '../middleware/auth'; import { requireAdmin } from '../middleware/tierGates'; +import { isValidServiceName } from '../utils/validation'; +import { + getActiveCapabilities, + SERVICE_SCOPED_STACK_ALERT_CAPABILITY, +} from '../services/CapabilityRegistry'; const AlertCreateSchema = z.object({ stack_name: z.string().min(1).max(255), + service_name: z.preprocess( + (val) => (val === '' ? null : val), + z.string().max(255).nullable().optional().refine( + (val) => val == null || isValidServiceName(val), + { message: 'Invalid service name' }, + ), + ), metric: z.enum(['cpu_percent', 'memory_percent', 'memory_mb', 'net_rx', 'net_tx', 'restart_count']), operator: z.enum(['>', '>=', '<', '<=', '==']), threshold: z.number().min(0), @@ -22,7 +34,8 @@ alertsRouter.get('/', authMiddleware, async (req: Request, res: Response) => { const alerts = DatabaseService.getInstance().getStackAlerts(stackName); res.json(alerts); - } catch { + } catch (error) { + console.error('Failed to fetch alerts:', error); res.status(500).json({ error: 'Failed to fetch alerts' }); } }); @@ -34,8 +47,23 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { res.status(400).json({ error: 'Invalid alert data', details: parsed.error.flatten().fieldErrors }); return; } + const { service_name, ...alertFields } = parsed.data; + const serviceName = service_name ?? null; + if ( + serviceName != null + && !getActiveCapabilities().includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) + ) { + res.status(400).json({ + error: 'This node does not support service-scoped alert rules', + code: 'capability_unavailable', + }); + return; + } try { - const created = DatabaseService.getInstance().addStackAlert(parsed.data); + const created = DatabaseService.getInstance().addStackAlert({ + ...alertFields, + service_name: serviceName, + }); res.status(201).json(created); } catch (error) { console.error('Failed to add alert:', error); @@ -45,11 +73,18 @@ alertsRouter.post('/', authMiddleware, async (req: Request, res: Response) => { alertsRouter.delete('/:id', authMiddleware, async (req: Request, res: Response) => { if (!requireAdmin(req, res)) return; + // Reject leading-junk / fractional ids (parseInt("1abc") === 1, parseInt("2.5") === 2). + const rawId = String(req.params.id ?? ''); + const id = /^\d+$/.test(rawId) ? Number.parseInt(rawId, 10) : NaN; + if (!Number.isInteger(id) || id <= 0) { + res.status(400).json({ error: 'Invalid alert id' }); + return; + } try { - const id = parseInt(req.params.id as string, 10); DatabaseService.getInstance().deleteStackAlert(id); res.json({ success: true }); - } catch { + } catch (error) { + console.error('Failed to delete alert:', error); res.status(500).json({ error: 'Failed to delete alert' }); } }); diff --git a/backend/src/routes/auditLog.ts b/backend/src/routes/auditLog.ts index d4780b10..5f0760fd 100644 --- a/backend/src/routes/auditLog.ts +++ b/backend/src/routes/auditLog.ts @@ -103,7 +103,7 @@ auditLogRouter.get('/export', async (req: Request, res: Response): Promise res.setHeader('Content-Type', 'text/csv'); res.setHeader('Content-Disposition', `attachment; filename="audit-log-${timestamp}.csv"`); - const headers = ['id', 'timestamp', 'username', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary']; + const headers = ['id', 'timestamp', 'username', 'acting_as', 'method', 'path', 'status_code', 'node_id', 'ip_address', 'summary']; const rows = result.entries.map(e => headers.map(h => escapeCsvField(e[h as keyof typeof e])).join(','), ); diff --git a/backend/src/routes/blueprints.ts b/backend/src/routes/blueprints.ts index ecdef6db..84c113d8 100644 --- a/backend/src/routes/blueprints.ts +++ b/backend/src/routes/blueprints.ts @@ -7,7 +7,9 @@ import { type BlueprintSelector, type DriftMode, } from '../services/DatabaseService'; -import { BlueprintService } from '../services/BlueprintService'; +import { BlueprintService, BlueprintNameConflictError, BlueprintOwnershipProbeError } from '../services/BlueprintService'; +import { DeployedStackDeletionService } from '../services/DeployedStackDeletionService'; +import { refuseIfSelfStack } from '../helpers/selfStackGuard'; import { BlueprintReconciler, messageForConfirmedOutcomes, @@ -317,12 +319,9 @@ blueprintsRouter.delete('/:id', async (req: Request, res: Response): Promise n.id === dep.node_id); if (!node) continue; try { - await BlueprintService.getInstance().withdrawFromNode(blueprint, node); + const outcome = await BlueprintService.getInstance().withdrawFromNode(blueprint, node); + if (outcome.status !== 'withdrawn') { + res.status(409).json({ + error: `Cannot delete blueprint: withdraw on node "${node.name}" ended as ${outcome.status}. Resolve that deployment, then retry.`, + code: 'withdraw_failed_blocking_delete', + nodeId: node.id, + withdrawStatus: outcome.status, + }); + return; + } } catch (err) { console.warn(`[Blueprints] Pre-delete withdraw failed for blueprint ${id} on node ${node.id}:`, err); + res.status(409).json({ + error: `Cannot delete blueprint: withdraw on node "${node.name}" failed. Resolve that deployment, then retry.`, + code: 'withdraw_failed_blocking_delete', + nodeId: node.id, + }); + return; } } DatabaseService.getInstance().deleteBlueprint(id); @@ -384,11 +398,68 @@ blueprintsRouter.post('/apply-local', async (req: Request, res: Response): Promi } res.json({ deployed: true }); } catch (error) { + if (error instanceof BlueprintNameConflictError) { + res.status(409).json({ error: error.message, code: 'name_conflict' }); + return; + } + if (error instanceof BlueprintOwnershipProbeError) { + console.error('[Blueprints] apply-local ownership probe failed:', sanitizeForLog(error.message)); + res.status(500).json({ error: error.message }); + return; + } console.error('[Blueprints] apply-local error:', sanitizeForLog(getErrorMessage(error, 'apply failed'))); res.status(500).json({ error: getErrorMessage(error, 'Blueprint apply failed') }); } }); +// Node-to-node atomic blueprint withdraw. Ownership is validated under the +// delete lock on this node. Requires stack:delete and refuses Sencho's own stack. +blueprintsRouter.post('/withdraw-local', async (req: Request, res: Response): Promise => { + const body = (req.body ?? {}) as { stackName?: unknown; blueprintId?: unknown }; + if (typeof body.stackName !== 'string' || !isValidStackName(body.stackName)) { + res.status(400).json({ error: 'Invalid stack name' }); + return; + } + if (typeof body.blueprintId !== 'number' || !Number.isInteger(body.blueprintId) || body.blueprintId <= 0) { + res.status(400).json({ error: 'blueprintId must be a positive integer' }); + return; + } + if (!requirePermission(req, res, 'stack:delete', 'stack', body.stackName)) return; + if (await refuseIfSelfStack(req, res, body.stackName)) return; + + try { + const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ + nodeId: req.nodeId, + stackName: body.stackName, + pruneVolumes: false, + actor: req.user?.username ?? 'system:blueprint', + requireBlueprintId: body.blueprintId, + }); + if (result.ok) { + res.json({ + status: result.status === 'already_absent' ? 'already_absent' : 'withdrawn', + }); + return; + } + if (result.code === 'name_conflict') { + res.status(409).json({ error: result.error, code: 'name_conflict' }); + return; + } + if (result.code === 'lock_conflict') { + res.status(409).json({ + error: result.error, + code: 'stack_op_in_progress', + inProgress: { action: result.existingAction }, + }); + return; + } + res.status(500).json({ error: result.error }); + } catch (error) { + console.error('[Blueprints] withdraw-local error:', sanitizeForLog(getErrorMessage(error, 'withdraw failed'))); + res.status(500).json({ error: getErrorMessage(error, 'Blueprint withdraw failed') }); + } +}); + blueprintsRouter.post('/:id/apply', async (req: Request, res: Response): Promise => { if (!requireAdmin(req, res)) return; const id = parseIntParam(req, res, 'id'); diff --git a/backend/src/routes/console.ts b/backend/src/routes/console.ts index ae7fbe7f..7986c6bc 100644 --- a/backend/src/routes/console.ts +++ b/backend/src/routes/console.ts @@ -1,8 +1,12 @@ import { Router, type Request, type Response } from 'express'; import { authMiddleware } from '../middleware/auth'; -import { requireAdmin, requirePaid } from '../middleware/tierGates'; +import { requireAdmin } from '../middleware/tierGates'; import { rejectApiTokenScope } from '../middleware/apiTokenScope'; -import { mintConsoleSession } from '../helpers/consoleSession'; +import { + isConsoleSessionPath, + mintConsoleSession, + sanitizeActingAs, +} from '../helpers/consoleSession'; /** * Mint a short-lived `console_session` JWT. Used by the gateway when it @@ -11,15 +15,28 @@ import { mintConsoleSession } from '../helpers/consoleSession'; * remote's upgrade handler on interactive paths, so the gateway authenticates * with the long-lived token, asks for this short-lived one, then forwards * the WS upgrade using it. + * + * Body: + * - `path` (required): `host-console` | `container-exec` + * - `acting_as` (optional): hub operator for remote audit. On node_proxy mints + * the body value is used (sanitized). On browser admin mints the signed-in + * username is stamped so a Bearer console_session cannot erase attribution. */ export const consoleRouter = Router(); consoleRouter.post('/console-token', authMiddleware, (req: Request, res: Response): void => { if (rejectApiTokenScope(req, res, 'API tokens cannot generate console tokens.')) return; if (!requireAdmin(req, res)) return; - if (!requirePaid(req, res)) return; try { - res.json({ token: mintConsoleSession() }); + const path = req.body?.path; + if (!isConsoleSessionPath(path)) { + res.status(400).json({ error: 'Invalid or missing console path' }); + return; + } + const actingAs = req.machineAuthScope === 'node_proxy' + ? sanitizeActingAs(req.body?.acting_as) + : sanitizeActingAs(req.user?.username); + res.json({ token: mintConsoleSession({ path, actingAs }) }); } catch (error) { console.error('Failed to issue console token:', error); res.status(500).json({ error: 'Failed to issue console token' }); diff --git a/backend/src/routes/stacks.ts b/backend/src/routes/stacks.ts index 4fe318de..6df26f2a 100644 --- a/backend/src/routes/stacks.ts +++ b/backend/src/routes/stacks.ts @@ -135,6 +135,40 @@ function releaseStackOpLock(req: Request, stackName: string): void { StackOpLockService.getInstance().release(req.nodeId, stackName); } +/** Root compose + blueprint marker on the stack-source root must not change while a lifecycle op holds the stack lock. */ +const STACK_OP_LOCKED_ROOT_TRUST_FILES = new Set([ + 'compose.yaml', + 'compose.yml', + 'docker-compose.yaml', + 'docker-compose.yml', + '.blueprint.json', +]); + +function rejectIfStackOpBlocksRootTrustFileWrite( + req: Request, + res: Response, + stackName: string, + relPath: string, + root: StackFileRoot, +): boolean { + if (root.kind !== 'stack-source') return false; + if (relPath.includes('/')) return false; + const base = relPath.toLowerCase(); + if (!STACK_OP_LOCKED_ROOT_TRUST_FILES.has(base)) return false; + const existing = StackOpLockService.getInstance().get(req.nodeId, stackName); + if (!existing) return false; + res.status(409).json({ + error: `${stackName} is busy: another operation (${existing.action}) is already in progress`, + code: 'stack_op_in_progress', + inProgress: { + action: existing.action, + startedAt: existing.startedAt, + user: existing.user, + }, + }); + return true; +} + function stackFileEtag(mtimeMs: number): string { return `W/"${Math.floor(mtimeMs)}"`; } @@ -2884,6 +2918,10 @@ stacksRouter.post( return res.status(400).json({ error: 'Invalid filename' }); } const targetRelPath = relPath ? `${relPath}/${originalName}` : originalName; + if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, targetRelPath, root)) { + await cleanupUploadTemp(req); + return; + } const overwrite = String(req.query.overwrite) === '1'; // The multer wrapper stashed the route-entry timestamp on the request so // the success path and the rejection paths share one window. Fall back to @@ -2987,6 +3025,7 @@ stacksRouter.put('/:stackName/files/content', async (req: Request, res: Response const expectedVersion = req.header('if-match') || undefined; const root = await resolveRootForOp(req, res, stackName, 'write'); if (!root) return; + if (rejectIfStackOpBlocksRootTrustFileWrite(req, res, stackName, relPath, root)) return; const startedAt = Date.now(); logFileDiag('write start', { stackName, relPath, nodeId: req.nodeId, bytes: Buffer.byteLength(content, 'utf-8'), hasIfMatch: expectedVersion !== undefined, rootKind: root.kind }); try { diff --git a/backend/src/services/BlueprintService.ts b/backend/src/services/BlueprintService.ts index e9a8aa69..adddbee8 100644 --- a/backend/src/services/BlueprintService.ts +++ b/backend/src/services/BlueprintService.ts @@ -19,8 +19,12 @@ import { assertPolicyGateAllows, buildSystemPolicyGateOptions, describePolicyBlo import { enforcePolicyForImageRefs } from './PolicyEnforcement'; import { BlueprintAnalyzer } from './BlueprintAnalyzer'; import { sanitizeForLog } from '../utils/safeLog'; - -const MARKER_FILENAME = '.blueprint.json'; +import { isPathWithinBase } from '../utils/validation'; +import { + BLUEPRINT_MARKER_FILENAME, + parseBlueprintMarker, + type BlueprintMarker, +} from '../helpers/blueprintMarker'; /** On-disk compose name for Blueprint applies. Must match createStack scaffold and Sencho discovery priority. */ const COMPOSE_FILENAME = 'compose.yaml'; const REMOTE_HTTP_TIMEOUT_MS = 30_000; @@ -41,10 +45,32 @@ function diagnosticLog(message: string, fields: Record { try { if (node.type === 'local') { - const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); - const markerPath = path.resolve(baseDir, blueprintName, MARKER_FILENAME); - if (!markerPath.startsWith(path.resolve(baseDir))) return null; - const content = await fsPromises.readFile(markerPath, 'utf-8'); - return BlueprintService.parseMarker(content); + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + return markerRead.kind === 'present' ? markerRead.marker : null; } const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) return null; - const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(MARKER_FILENAME)}`; + const url = `${target.apiUrl.replace(/\/$/, '')}/api/stacks/${encodeURIComponent(blueprintName)}/files/content?path=${encodeURIComponent(BLUEPRINT_MARKER_FILENAME)}`; const res = await axios.get(url, { headers: this.remoteHeaders(target.apiToken), timeout: REMOTE_HTTP_TIMEOUT_MS, @@ -146,7 +175,7 @@ export class BlueprintService { const body = res.data; const content = typeof body === 'string' ? body : (typeof body?.content === 'string' ? body.content : null); if (content == null) return null; - return BlueprintService.parseMarker(content); + return parseBlueprintMarker(content); } catch { return null; } @@ -154,47 +183,77 @@ export class BlueprintService { /** * Returns true when a stack directory by this name exists on the target - * node but does not carry our marker file. The reconciler must not - * deploy in that case: there is a real user-authored stack with the - * same name and we must not overwrite it. + * node and the on-disk marker is missing, malformed, or references a + * different blueprint ID. Throws BlueprintOwnershipProbeError when the + * directory or marker cannot be probed (non-ENOENT I/O or remote list failure). */ - async hasNameConflict(blueprintName: string, node: Node): Promise { - try { - if (node.type === 'local') { - const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); - const stackDir = path.resolve(baseDir, blueprintName); - if (!stackDir.startsWith(path.resolve(baseDir))) return true; - try { - const stat = await fsPromises.stat(stackDir); - if (!stat.isDirectory()) return false; - } catch { - return false; // directory doesn't exist → no conflict - } - const markerPath = path.join(stackDir, MARKER_FILENAME); - try { - await fsPromises.stat(markerPath); - return false; // marker present → ours - } catch { - return true; // directory exists but no marker → conflict - } + async hasNameConflict(blueprintName: string, node: Node, blueprintId: number): Promise { + if (node.type === 'local') { + const baseDir = NodeRegistry.getInstance().getComposeDir(node.id); + const stackDir = path.resolve(baseDir, blueprintName); + if (!isPathWithinBase(stackDir, baseDir)) return true; + try { + const stat = await fsPromises.stat(stackDir); + if (!stat.isDirectory()) return false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw BlueprintService.ownershipProbeError(blueprintName, BlueprintService.formatError(err)); } - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) return false; - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const listUrl = `${baseUrl}/api/stacks`; - const listRes = await axios.get(listUrl, { + const markerRead = await this.readLocalMarkerFromDisk(node.id, blueprintName); + if (markerRead.kind === 'failed') { + throw BlueprintService.ownershipProbeError(blueprintName, markerRead.error); + } + return markerRead.kind === 'missing' || markerRead.marker.blueprintId !== blueprintId; + } + const target = NodeRegistry.getInstance().getProxyTarget(node.id); + if (!target) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": no proxy target configured`, + ); + } + const baseUrl = target.apiUrl.replace(/\/$/, ''); + let listRes; + try { + listRes = await axios.get(`${baseUrl}/api/stacks`, { headers: this.remoteHeaders(target.apiToken), timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true, }); - if (listRes.status !== 200) return false; - const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; - const exists = stacks.some(s => s?.name === blueprintName); - if (!exists) return false; - const marker = await this.readMarker(blueprintName, node); - return marker == null; - } catch { - return false; + } catch (err) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}": ${BlueprintService.formatError(err)}`, + ); + } + if (listRes.status !== 200) { + throw new BlueprintOwnershipProbeError( + `Cannot verify stack ownership on remote node "${node.name}" (HTTP ${listRes.status})`, + ); + } + const stacks = Array.isArray(listRes.data) ? listRes.data as Array<{ name?: string }> : []; + const exists = stacks.some(s => s?.name === blueprintName); + if (!exists) return false; + const marker = await this.readMarker(blueprintName, node); + return marker == null || marker.blueprintId !== blueprintId; + } + + /** Read and parse a local on-disk marker without going through the remote HTTP path. */ + private async readLocalMarkerFromDisk(nodeId: number, stackName: string): Promise { + try { + // Canonical js/path-injection barrier inline with the read sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; + } + const content = await fsPromises.readFile(safePath, 'utf-8'); + const marker = parseBlueprintMarker(content); + if (!marker) return { kind: 'missing' }; + return { kind: 'present', marker }; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'missing' }; + return { kind: 'failed', error: BlueprintService.formatError(err) }; } } @@ -222,7 +281,7 @@ export class BlueprintService { }); try { this.setStatus(blueprint.id, node.id, 'deploying'); - if (await this.hasNameConflict(blueprint.name, node)) { + if (await this.hasNameConflict(blueprint.name, node, blueprint.id)) { this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: `A stack named "${blueprint.name}" already exists on this node and is not managed by Sencho.`, }); @@ -249,6 +308,12 @@ export class BlueprintService { sanitizeForLog(blueprint.name), node.id, Date.now() - started); return { status: 'active' }; } catch (err) { + if (err instanceof BlueprintNameConflictError) { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: err.message }); + console.warn('[BlueprintService] deploy name conflict blueprint=%s node=%s durationMs=%s', + sanitizeForLog(blueprint.name), node.id, Date.now() - started); + return { status: 'name_conflict', error: 'name_conflict' }; + } const message = BlueprintService.formatError(err); this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); console.error('[BlueprintService] deploy failed blueprint=%s node=%s durationMs=%s error=%s', @@ -280,20 +345,15 @@ export class BlueprintService { }); try { this.setStatus(blueprint.id, node.id, 'withdrawing'); - // Refuse to withdraw a directory we do not own - const marker = await this.readMarker(blueprint.name, node); - if (marker && marker.blueprintId !== blueprint.id) { - this.setStatus(blueprint.id, node.id, 'name_conflict', { - last_error: `Marker on this node points to a different blueprint (id=${marker.blueprintId}); refusing to withdraw.`, - }); - return { status: 'name_conflict' }; - } + // Ownership is validated on the node that owns the stack, inside the delete lock. if (node.type === 'local') { diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'local' }); - await this.withdrawLocal(blueprint, node); + const localOutcome = await this.withdrawLocal(blueprint, node); + if (localOutcome.status !== 'withdrawn') return localOutcome; } else { diagnosticLog('withdraw branch', { blueprintId: blueprint.id, nodeId: node.id, target: 'remote' }); - await this.withdrawRemote(blueprint, node); + const remoteOutcome = await this.withdrawRemote(blueprint, node); + if (remoteOutcome.status !== 'withdrawn') return remoteOutcome; } DatabaseService.getInstance().deleteDeployment(blueprint.id, node.id); console.info('[BlueprintService] withdraw complete blueprint=%s node=%s durationMs=%s', @@ -382,15 +442,22 @@ export class BlueprintService { // ---- local primitives ---- + /** Returns whether the stack directory exists. Throws on non-ENOENT I/O. */ private async stackDirExists(nodeId: number, blueprintName: string): Promise { const baseDir = NodeRegistry.getInstance().getComposeDir(nodeId); const stackDir = path.resolve(baseDir, blueprintName); - if (!stackDir.startsWith(path.resolve(baseDir))) return false; + if (!isPathWithinBase(stackDir, baseDir)) { + throw new BlueprintOwnershipProbeError(`Invalid stack path for "${blueprintName}"`); + } try { const stat = await fsPromises.stat(stackDir); return stat.isDirectory(); - } catch { - return false; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return false; + throw new BlueprintOwnershipProbeError( + `Cannot access stack directory "${blueprintName}": ${BlueprintService.formatError(err)}`, + ); } } @@ -423,14 +490,14 @@ export class BlueprintService { } /** - * Create the stack if needed, write the compose and marker files, run the - * deploy policy gate, and deploy, all under the per-stack operation lock so - * none of it can race a manual deploy/update/rollback/backup on the same - * stack and node. Runs on the node that owns the stack: deployLocal calls it - * for the hub's own node, and the /api/blueprints/apply-local route calls it - * on a remote node receiving a blueprint apply from its hub (so the file - * writes hold the remote's lock, not just the deploy). On lock conflict - * nothing is written and { ran: false } is returned. + * Create the stack if needed, write the compose file, run the deploy policy + * gate and deploy, then write the marker, all under the per-stack operation + * lock. The marker is written only after a successful deploy so a failed + * apply cannot claim an applied revision that never ran. Runs on the node + * that owns the stack: deployLocal calls it for the hub's own node, and the + * /api/blueprints/apply-local route calls it on a remote node receiving a + * blueprint apply from its hub. On lock conflict nothing is written and + * { ran: false } is returned. */ async applyLocalUnderLock( nodeId: number, @@ -439,47 +506,83 @@ export class BlueprintService { markerContent: string, auditPath: string, ): Promise<{ ran: true } | { ran: false; existingAction: StackOpAction }> { + const expected = parseBlueprintMarker(markerContent); + if (!expected) { + throw new Error('Invalid blueprint marker'); + } const fs = FileSystemService.getInstance(nodeId); const lock = await StackOpLockService.getInstance().runExclusive( nodeId, stackName, 'deploy', 'system', async () => { - if (!(await this.stackDirExists(nodeId, stackName))) { + let createdStack = false; + if (await this.stackDirExists(nodeId, stackName)) { + const existing = await this.readLocalMarkerFromDisk(nodeId, stackName); + if (existing.kind === 'failed') { + throw new BlueprintOwnershipProbeError( + `Cannot verify ownership of stack "${stackName}": ${existing.error}`, + ); + } + if (existing.kind === 'missing' || existing.marker.blueprintId !== expected.blueprintId) { + throw new BlueprintNameConflictError( + `A stack named "${stackName}" already exists on this node and is not managed by this blueprint.`, + ); + } + } else { await fs.createStack(stackName); + createdStack = true; } await fs.writeStackFile(stackName, COMPOSE_FILENAME, composeContent); - await fs.writeStackFile(stackName, MARKER_FILENAME, markerContent); // Clear lower-priority compose siblings so discovery cannot shadow compose.yaml. - // Local + modern apply-local only; legacy remote has no sibling DELETE. await fs.removeAlternateRootComposeFiles(stackName); - await assertPolicyGateAllows( - stackName, - nodeId, - buildSystemPolicyGateOptions('blueprint', { auditPath }), - ); - await ComposeService.getInstance(nodeId).deployStack( - stackName, - undefined, - false, - { source: 'blueprint', actor: 'system:blueprint' }, - ); + try { + await assertPolicyGateAllows( + stackName, + nodeId, + buildSystemPolicyGateOptions('blueprint', { auditPath }), + ); + await ComposeService.getInstance(nodeId).deployStack( + stackName, + undefined, + false, + { source: 'blueprint', actor: 'system:blueprint' }, + ); + await fs.writeStackFile(stackName, BLUEPRINT_MARKER_FILENAME, markerContent); + } catch (err) { + if (createdStack) { + try { + await fs.deleteStack(stackName); + } catch (cleanupErr) { + console.warn( + '[BlueprintService] Failed to roll back newly created stack "%s" after apply error: %s', + sanitizeForLog(stackName), + sanitizeForLog(BlueprintService.formatError(cleanupErr)), + ); + } + } + throw err; + } }, ); return lock.ran ? { ran: true } : { ran: false, existingAction: lock.existing.action }; } - private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { + private async withdrawLocal(blueprint: Blueprint, node: Node): Promise { const result = await DeployedStackDeletionService.getInstance().deleteDeployedStack({ nodeId: node.id, stackName: blueprint.name, pruneVolumes: false, actor: 'system:blueprint', + requireBlueprintId: blueprint.id, }); - if (!result.ok) { - if (result.code === 'lock_conflict') { - throw new Error(result.error); - } - throw new Error(result.error); + if (result.ok) { + return { status: 'withdrawn' }; } + if (result.code === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: result.error }); + return { status: 'name_conflict' }; + } + this.setStatus(blueprint.id, node.id, 'failed', { last_error: result.error }); + return { status: 'failed', error: result.error }; } // ---- remote primitives ---- @@ -500,10 +603,7 @@ export class BlueprintService { const baseUrl = target.apiUrl.replace(/\/$/, ''); const headers = this.remoteHeaders(target.apiToken); - // Atomic apply: the remote runs create + write compose/marker + deploy - // under its own per-stack lock, so the file writes cannot race a manual - // operation on that node. Older nodes without this route answer 404; we - // fall back to the legacy multi-call flow there (not lock-atomic). + // Atomic apply: the remote validates ownership and writes under its stack lock. const res = await axios.post( `${baseUrl}/api/blueprints/apply-local`, { @@ -514,11 +614,17 @@ export class BlueprintService { { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, ); if (res.status === 404) { - console.warn(`[BlueprintService] remote node ${node.id} lacks /api/blueprints/apply-local; using legacy non-atomic apply`); - await this.deployRemoteLegacy(blueprint, node, marker); - return; + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint apply (/api/blueprints/apply-local). Upgrade that Sencho instance, then retry.`, + ); } if (res.status === 409) { + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + throw new BlueprintNameConflictError( + BlueprintService.extractApiError(res.data) + || `A stack named "${blueprint.name}" already exists on this node and is not managed by this blueprint.`, + ); + } throw new Error(`blueprint apply skipped: ${BlueprintService.extractApiError(res.data) || 'another operation is already in progress'}`); } if (res.status >= 400) { @@ -526,105 +632,62 @@ export class BlueprintService { } } - /** - * Legacy remote apply for nodes that predate /api/blueprints/apply-local: - * create, write compose, write marker, deploy as separate calls. The remote - * deploy locks, but the preceding file writes do not, so this is not atomic - * against a concurrent manual operation on that node. Kept only as a - * compatibility fallback. - */ - private async deployRemoteLegacy(blueprint: Blueprint, node: Node, marker: BlueprintMarker): Promise { + private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { const target = NodeRegistry.getInstance().getProxyTarget(node.id); if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); const baseUrl = target.apiUrl.replace(/\/$/, ''); const headers = this.remoteHeaders(target.apiToken); - // 1. Ensure stack exists. POST returns 409 when already exists; we treat that as success. - const createRes = await axios.post(`${baseUrl}/api/stacks`, - { stackName: blueprint.name }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (createRes.status >= 400 && createRes.status !== 409) { - throw new Error(`create stack: HTTP ${createRes.status} ${BlueprintService.extractApiError(createRes.data)}`); - } - - // 2. Write the compose file - await this.remotePutFile(baseUrl, headers, blueprint.name, COMPOSE_FILENAME, blueprint.compose_content); - - // 3. Write the marker (last so a partial failure leaves us in name_conflict-recoverable state) - await this.remotePutFile(baseUrl, headers, blueprint.name, MARKER_FILENAME, JSON.stringify(marker, null, 2)); - - // 4. Deploy - const deployRes = await axios.post( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/deploy`, - {}, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (deployRes.status >= 400) { - throw new Error(`deploy: HTTP ${deployRes.status} ${BlueprintService.extractApiError(deployRes.data)}`); - } - } - - private async withdrawRemote(blueprint: Blueprint, node: Node): Promise { - const target = NodeRegistry.getInstance().getProxyTarget(node.id); - if (!target) throw new Error(`Remote node "${node.name}" has no proxy target configured`); - const baseUrl = target.apiUrl.replace(/\/$/, ''); - const headers = this.remoteHeaders(target.apiToken); - - // down (best-effort) + let res; try { - await axios.post( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}/down`, - {}, + res = await axios.post( + `${baseUrl}/api/blueprints/withdraw-local`, + { stackName: blueprint.name, blueprintId: blueprint.id }, { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, ); } catch (err) { - console.warn(`[BlueprintService] remote down failed for "${blueprint.name}" on node ${node.id}: ${BlueprintService.formatError(err)}`); + const message = BlueprintService.formatError(err); + this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); + return { status: 'failed', error: message }; } - // delete the stack directory entirely - const delRes = await axios.delete( - `${baseUrl}/api/stacks/${encodeURIComponent(blueprint.name)}`, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (delRes.status >= 400 && delRes.status !== 404) { - throw new Error(`remote delete: HTTP ${delRes.status} ${BlueprintService.extractApiError(delRes.data)}`); + if (res.status === 404) { + throw new BlueprintRemoteUpgradeRequiredError( + `Remote node "${node.name}" does not support atomic blueprint withdraw (/api/blueprints/withdraw-local). Upgrade that Sencho instance, then retry.`, + ); } - } - - private async remotePutFile( - baseUrl: string, - headers: Record, - stackName: string, - relPath: string, - content: string, - ): Promise { - const url = `${baseUrl}/api/stacks/${encodeURIComponent(stackName)}/files/content?path=${encodeURIComponent(relPath)}`; - const res = await axios.put(url, - { content }, - { headers, timeout: REMOTE_HTTP_TIMEOUT_MS, validateStatus: () => true }, - ); - if (res.status >= 400) { - throw new Error(`PUT ${relPath}: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`); + if (res.status === 200) { + return { status: 'withdrawn' }; } + if (res.status === 409) { + const error = BlueprintService.extractApiError(res.data) || 'withdraw refused'; + if (BlueprintService.extractApiCode(res.data) === 'name_conflict') { + this.setStatus(blueprint.id, node.id, 'name_conflict', { last_error: error }); + return { status: 'name_conflict' }; + } + // stack_op_in_progress and any other 409: match local withdraw lock-conflict → failed + this.setStatus(blueprint.id, node.id, 'failed', { last_error: error }); + return { status: 'failed', error }; + } + const message = `blueprint withdraw: HTTP ${res.status} ${BlueprintService.extractApiError(res.data)}`; + this.setStatus(blueprint.id, node.id, 'failed', { last_error: message }); + return { status: 'failed', error: message }; } static parseMarker(content: string): BlueprintMarker | null { - try { - const parsed = JSON.parse(content); - if (parsed && typeof parsed === 'object' - && typeof parsed.blueprintId === 'number' - && typeof parsed.revision === 'number') { - return { - blueprintId: parsed.blueprintId, - revision: parsed.revision, - lastApplied: typeof parsed.lastApplied === 'number' ? parsed.lastApplied : 0, - }; - } - } catch { - // fall through - } - return null; + return parseBlueprintMarker(content); + } + + private static ownershipProbeError(blueprintName: string, detail: string): BlueprintOwnershipProbeError { + return new BlueprintOwnershipProbeError( + `Cannot verify stack ownership for "${blueprintName}": ${detail}`, + ); + } + + static extractApiCode(body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const code = (body as Record).code; + return typeof code === 'string' ? code : ''; } static formatError(err: unknown): string { diff --git a/backend/src/services/CapabilityRegistry.ts b/backend/src/services/CapabilityRegistry.ts index 1ce65b55..ba806c5d 100644 --- a/backend/src/services/CapabilityRegistry.ts +++ b/backend/src/services/CapabilityRegistry.ts @@ -38,6 +38,7 @@ export const CAPABILITIES = [ 'notification-suppression', 'notification-suppression-schedule', 'host-console', + 'host-console-community', 'container-exec', 'audit-log', 'scheduled-ops', @@ -58,6 +59,7 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; /** @@ -71,6 +73,12 @@ export const CROSS_NODE_RBAC_CAPABILITY = 'cross-node-rbac'; export type Capability = (typeof CAPABILITIES)[number]; +/** Legacy Host Console advertisement (Admiral hubs still accept this on remotes). */ +export const HOST_CONSOLE_CAPABILITY = 'host-console' as const satisfies Capability; + +/** Host Console works without a paid license on this node. */ +export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as const satisfies Capability; + /** Remotes that evaluate weekly maintenance windows on mute/suppression replicas. */ export const NOTIFICATION_SUPPRESSION_SCHEDULE_CAPABILITY = 'notification-suppression-schedule' as const satisfies Capability; @@ -81,6 +89,10 @@ export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' /** Capability for the nested per-service update/restore routes and the `effective-services` model they read. */ export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +/** Capability for nullable `service_name` on stack alert rules and per-service cooldown evaluation. */ +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = + 'service-scoped-stack-alert' as const satisfies Capability; + /** Returns true when the string is a usable semver version. */ export function isValidVersion(v: string | null | undefined): v is string { return !!v && v !== 'unknown' && v !== '0.0.0-dev' && !!semver.valid(v); @@ -165,6 +177,7 @@ export function getActiveCapabilities(): readonly string[] { */ const PILOT_DISABLED_CAPABILITIES: readonly Capability[] = [ 'host-console', + 'host-console-community', ]; /** Disable capabilities that require a central->pilot path that is not yet wired. */ diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index 03663fc0..ace540a4 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -101,6 +101,7 @@ function stringifyServicesJson(services: StackServiceStatus[], generation: numbe export interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -250,6 +251,8 @@ export interface StackUpdateCleanupPendingRow { rollback_tags_json: string; override_paths_json: string; prune_volumes_requested: number; + /** Blueprint ID that authorized this deletion; null for manual deletes. */ + required_blueprint_id: number | null; created_at: number; updated_at: number; } @@ -613,6 +616,8 @@ export interface AuditLogEntry { node_id: number | null; ip_address: string; summary: string; + /** Hub operator for remote console_session bridges; null/absent for direct sessions. */ + acting_as?: string | null; } export interface SecretRow { @@ -1060,6 +1065,7 @@ export class DatabaseService { this.migrateStackDossierHashes(); this.migrateGitSourceMultiFile(); this.migrateNodeUpdateSkips(); + this.migrateStackAlertServiceScope(); // Reset the cache once at end of constructor in case any migration // populated it via getGlobalSettings() and a subsequent migration @@ -1096,6 +1102,7 @@ export class DatabaseService { CREATE TABLE IF NOT EXISTS stack_alerts ( id INTEGER PRIMARY KEY AUTOINCREMENT, stack_name TEXT NOT NULL, + service_name TEXT, metric TEXT NOT NULL, operator TEXT NOT NULL, threshold REAL NOT NULL, @@ -1104,6 +1111,17 @@ export class DatabaseService { last_fired_at INTEGER DEFAULT 0 ); + -- FK cascade is declarative only: PRAGMA foreign_keys is never enabled + -- on this connection, so parent deletes must remove children explicitly + -- (see deleteStackAlert). + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ); + CREATE TABLE IF NOT EXISTS notification_history ( id INTEGER PRIMARY KEY AUTOINCREMENT, node_id INTEGER NOT NULL DEFAULT 0, @@ -1240,12 +1258,20 @@ export class DatabaseService { status_code INTEGER NOT NULL DEFAULT 0, node_id INTEGER, ip_address TEXT NOT NULL DEFAULT '', - summary TEXT NOT NULL DEFAULT '' + summary TEXT NOT NULL DEFAULT '', + acting_as TEXT ); CREATE INDEX IF NOT EXISTS idx_audit_log_timestamp ON audit_log(timestamp); CREATE INDEX IF NOT EXISTS idx_audit_log_username ON audit_log(username); + CREATE TABLE IF NOT EXISTS console_session_jtis ( + jti TEXT PRIMARY KEY, + used_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at); + CREATE TABLE IF NOT EXISTS api_tokens ( id INTEGER PRIMARY KEY AUTOINCREMENT, token_hash TEXT NOT NULL UNIQUE, @@ -1788,7 +1814,23 @@ export class DatabaseService { try { this.db.prepare(`ALTER TABLE ${table} ADD COLUMN ${col} ${def}`).run(); } catch (e) { /* ignore */ } }; + // Remote Host Console bridges record the hub operator separately from + // the console_session principal (username stays console_session). + maybeAddCol('audit_log', 'acting_as', 'TEXT'); + // Cached INSERT may predate the column; rebuild on next flush. + this.auditLogInsertStmt = null; + + this.db.exec(` + CREATE TABLE IF NOT EXISTS console_session_jtis ( + jti TEXT PRIMARY KEY, + used_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_console_session_jtis_expires ON console_session_jtis(expires_at); + `); + maybeAddCol('stack_update_recovery_generations', 'artifacts_retired', 'INTEGER NOT NULL DEFAULT 0'); + maybeAddCol('stack_update_cleanup_pending', 'required_blueprint_id', 'INTEGER'); // Distributed API model columns maybeAddCol('nodes', 'api_url', "TEXT DEFAULT ''"); @@ -2255,6 +2297,25 @@ export class DatabaseService { } } + private migrateStackAlertServiceScope(): void { + this.tryAddColumn('stack_alerts', 'service_name', 'TEXT'); + try { + // FK is not enforced (foreign_keys pragma off); deleteStackAlert removes children. + this.db.prepare(` + CREATE TABLE IF NOT EXISTS stack_alert_service_cooldowns ( + alert_id INTEGER NOT NULL, + service_name TEXT NOT NULL, + last_fired_at INTEGER NOT NULL, + PRIMARY KEY (alert_id, service_name), + FOREIGN KEY (alert_id) REFERENCES stack_alerts(id) ON DELETE CASCADE + ) + `).run(); + } catch (e) { + console.error('[DatabaseService] stack_alert_service_cooldowns migration failed:', (e as Error).message); + throw e; + } + } + private migrateScanPolicyFleetColumns(): void { this.tryAddColumn('scan_policies', 'node_identity', "TEXT NOT NULL DEFAULT ''"); this.tryAddColumn('scan_policies', 'replicated_from_control', 'INTEGER NOT NULL DEFAULT 0'); @@ -3050,22 +3111,19 @@ export class DatabaseService { // --- Stack Alerts --- public getStackAlerts(stackName?: string): StackAlert[] { - let stmt; if (stackName) { - stmt = this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?'); - return stmt.all(stackName) as StackAlert[]; - } else { - stmt = this.db.prepare('SELECT * FROM stack_alerts'); - return stmt.all() as StackAlert[]; + return this.db.prepare('SELECT * FROM stack_alerts WHERE stack_name = ?').all(stackName) as StackAlert[]; } + return this.db.prepare('SELECT * FROM stack_alerts').all() as StackAlert[]; } public addStackAlert(alert: StackAlert): StackAlert { const stmt = this.db.prepare( - 'INSERT INTO stack_alerts (stack_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO stack_alerts (stack_name, service_name, metric, operator, threshold, duration_mins, cooldown_mins, last_fired_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' ); const result = stmt.run( alert.stack_name, + alert.service_name ?? null, alert.metric, alert.operator, alert.threshold, @@ -3076,9 +3134,16 @@ export class DatabaseService { return this.db.prepare('SELECT * FROM stack_alerts WHERE id = ?').get(result.lastInsertRowid) as StackAlert; } + /** + * Delete an alert and its per-service cooldown rows. + * SQLite foreign_keys is not enabled here, so child rows are removed + * explicitly rather than relying on ON DELETE CASCADE. + */ public deleteStackAlert(id: number): void { - const stmt = this.db.prepare('DELETE FROM stack_alerts WHERE id = ?'); - stmt.run(id); + this.transaction(() => { + this.deleteStackAlertServiceCooldowns(id); + this.db.prepare('DELETE FROM stack_alerts WHERE id = ?').run(id); + }); } public updateStackAlertLastFired(id: number, timestamp: number): void { @@ -3086,6 +3151,32 @@ export class DatabaseService { stmt.run(timestamp, id); } + public getStackAlertServiceCooldown(alertId: number, serviceName: string): number | null { + const row = this.db.prepare( + 'SELECT last_fired_at FROM stack_alert_service_cooldowns WHERE alert_id = ? AND service_name = ?' + ).get(alertId, serviceName) as { last_fired_at: number } | undefined; + return row?.last_fired_at ?? null; + } + + public hasAnyStackAlertServiceCooldown(alertId: number): boolean { + const row = this.db.prepare( + 'SELECT 1 AS present FROM stack_alert_service_cooldowns WHERE alert_id = ? LIMIT 1' + ).get(alertId) as { present: number } | undefined; + return !!row; + } + + public upsertStackAlertServiceCooldown(alertId: number, serviceName: string, timestamp: number): void { + this.db.prepare(` + INSERT INTO stack_alert_service_cooldowns (alert_id, service_name, last_fired_at) + VALUES (?, ?, ?) + ON CONFLICT(alert_id, service_name) DO UPDATE SET last_fired_at = excluded.last_fired_at + `).run(alertId, serviceName, timestamp); + } + + public deleteStackAlertServiceCooldowns(alertId: number): void { + this.db.prepare('DELETE FROM stack_alert_service_cooldowns WHERE alert_id = ?').run(alertId); + } + // --- Auto-Heal Policies --- public getAutoHealPolicies(stackName?: string, nodeId?: number): AutoHealPolicy[] { @@ -3845,12 +3936,12 @@ export class DatabaseService { this.db.prepare( `INSERT INTO stack_update_cleanup_pending ( id, node_id, stack_name, status, target_kind, rollback_tags_json, - override_paths_json, prune_volumes_requested, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + override_paths_json, prune_volumes_requested, required_blueprint_id, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` ).run( row.id, row.node_id, row.stack_name, row.status, row.target_kind, row.rollback_tags_json, row.override_paths_json, row.prune_volumes_requested, - row.created_at, row.updated_at, + row.required_blueprint_id, row.created_at, row.updated_at, ); } @@ -4046,6 +4137,12 @@ export class DatabaseService { stmt.run(nodeId); } + /** Purge stack-scoped notification history when a stack is deleted. */ + public deleteNotificationsForStack(nodeId: number, stackName: string): number { + const stmt = this.db.prepare('DELETE FROM notification_history WHERE node_id = ? AND stack_name = ?'); + return stmt.run(nodeId, stackName).changes; + } + public updateNotificationDispatchError(id: number, error: string): void { this.db.prepare('UPDATE notification_history SET dispatch_error = ? WHERE id = ?').run(error, id); } @@ -4315,6 +4412,7 @@ export class DatabaseService { rollback_tags_json: JSON.stringify(localCleanup.tags), override_paths_json: JSON.stringify(localCleanup.overridePaths), prune_volumes_requested: 0, + required_blueprint_id: null, created_at: now, updated_at: now, }); @@ -4456,6 +4554,21 @@ export class DatabaseService { this.db.prepare('DELETE FROM pilot_enrollments WHERE node_id = ?').run(nodeId); } + /** + * One-time consume for a console_session JWT id. Inserts the jti on first + * use; returns false when the jti was already recorded (replay). + */ + public consumeConsoleSessionJti(jti: string, expiresAtMs: number): boolean { + const now = Date.now(); + return this.db.transaction(() => { + this.db.prepare('DELETE FROM console_session_jtis WHERE expires_at < ?').run(now); + const result = this.db.prepare( + 'INSERT OR IGNORE INTO console_session_jtis (jti, used_at, expires_at) VALUES (?, ?, ?)', + ).run(jti, now, expiresAtMs); + return result.changes > 0; + })(); + } + // --- Stack Update Status --- /** @@ -5159,7 +5272,7 @@ export class DatabaseService { this.auditLogBuffer = []; if (!this.auditLogInsertStmt) { this.auditLogInsertStmt = this.db.prepare( - 'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', + 'INSERT INTO audit_log (timestamp, username, method, path, status_code, node_id, ip_address, summary, acting_as) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', ); } const stmt = this.auditLogInsertStmt; @@ -5174,6 +5287,7 @@ export class DatabaseService { entry.node_id, entry.ip_address, entry.summary, + entry.acting_as ?? null, ); } }); diff --git a/backend/src/services/DeployedStackDeletionService.ts b/backend/src/services/DeployedStackDeletionService.ts index 3bddbd1a..b9ca7b92 100644 --- a/backend/src/services/DeployedStackDeletionService.ts +++ b/backend/src/services/DeployedStackDeletionService.ts @@ -18,10 +18,15 @@ import DockerController from './DockerController'; import { FileSystemService } from './FileSystemService'; import { NodeRegistry } from './NodeRegistry'; import { MeshService } from './MeshService'; +import { NotificationService } from './NotificationService'; import { StackOpLockService, stackOpSkipMessage } from './StackOpLockService'; import { getErrorMessage } from '../utils/errors'; import { sanitizeForLog } from '../utils/safeLog'; import { isPathWithinBase, isValidStackName } from '../utils/validation'; +import { + BLUEPRINT_MARKER_FILENAME, + parseBlueprintMarker, +} from '../helpers/blueprintMarker'; /** * Directory that may contain recovery override files for a tombstone sweep. @@ -45,13 +50,30 @@ export interface DeleteDeployedStackInput { stackName: string; pruneVolumes: boolean; actor: string; + /** When set, deletion requires an on-disk .blueprint.json matching this blueprint ID. */ + requireBlueprintId?: number; /** When true, skip acquiring a new lock (caller already holds delete via continuation). */ continuationIntentId?: string; } export type DeleteDeployedStackResult = - | { ok: true } - | { ok: false; code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed'; error: string; existingAction?: string }; + | { ok: true; status: 'deleted' | 'already_absent' } + | { + ok: false; + code: 'lock_conflict' | 'fs_failed' | 'tombstone_failed' | 'db_failed' | 'name_conflict' | 'failed'; + error: string; + existingAction?: string; + }; + +type DirProbe = { kind: 'absent' } | { kind: 'present' } | { kind: 'error'; error: string }; +type MarkerProbe = + | { kind: 'match' } + | { kind: 'name_conflict'; error: string } + | { kind: 'failed'; error: string }; + +function blueprintMarkerMismatchError(stackName: string): string { + return `Stack "${stackName}" exists without a matching blueprint marker; refusing to withdraw.`; +} function collectArtifactsFromGenerations( generations: Array<{ override_path: string | null; services_json: string }>, @@ -99,6 +121,51 @@ function parseJsonStringArray(raw: string): string[] { } } + +async function probeStackDirectory(nodeId: number, stackName: string): Promise { + // Canonical js/path-injection barrier inline with the stat sink. + const baseResolved = path.resolve(FileSystemService.getInstance(nodeId).getBaseDir()); + const safePath = path.resolve(baseResolved, stackName); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'error', error: 'Invalid stack path' }; + } + try { + const stat = await fs.stat(safePath); + return stat.isDirectory() ? { kind: 'present' } : { kind: 'absent' }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') return { kind: 'absent' }; + return { kind: 'error', error: getErrorMessage(error, 'Failed to access stack directory') }; + } +} + +async function probeBlueprintMarkerOwnership( + nodeId: number, + stackName: string, + requireBlueprintId: number, +): Promise { + // Canonical js/path-injection barrier inline with the read sink. + const baseResolved = path.resolve(NodeRegistry.getInstance().getComposeDir(nodeId)); + const safePath = path.resolve(baseResolved, stackName, BLUEPRINT_MARKER_FILENAME); + if (!safePath.startsWith(baseResolved + path.sep)) { + return { kind: 'failed', error: 'Invalid stack path for blueprint marker' }; + } + try { + const content = await fs.readFile(safePath, 'utf-8'); + const marker = parseBlueprintMarker(content); + if (!marker || marker.blueprintId !== requireBlueprintId) { + return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) }; + } + return { kind: 'match' }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return { kind: 'name_conflict', error: blueprintMarkerMismatchError(stackName) }; + } + return { kind: 'failed', error: getErrorMessage(error, 'Failed to read blueprint marker') }; + } +} + export class DeployedStackDeletionService { private static instance: DeployedStackDeletionService; @@ -163,6 +230,42 @@ export class DeployedStackDeletionService { const { nodeId, stackName, pruneVolumes } = input; const db = DatabaseService.getInstance(); + // Continuation loads ownership from the persisted intent; first call uses input. + let requiredBlueprintId: number | null = + typeof input.requireBlueprintId === 'number' ? input.requireBlueprintId : null; + + if (existingIntentId) { + const existing = db.getDeletionIntentById(existingIntentId); + if (!existing || existing.status !== 'prepared') { + return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' }; + } + if (existing.required_blueprint_id != null) { + requiredBlueprintId = existing.required_blueprint_id; + } + } + + let skipPhysical = false; + if (requiredBlueprintId != null) { + const dirProbe = await probeStackDirectory(nodeId, stackName); + if (dirProbe.kind === 'error') { + return { ok: false, code: 'failed', error: dirProbe.error }; + } + if (dirProbe.kind === 'absent') { + skipPhysical = true; + } else { + const ownership = await probeBlueprintMarkerOwnership(nodeId, stackName, requiredBlueprintId); + if (ownership.kind === 'failed') { + return { ok: false, code: 'failed', error: ownership.error }; + } + if (ownership.kind === 'name_conflict') { + if (existingIntentId) { + db.updateCleanupPendingStatus(existingIntentId, 'cancelled'); + } + return { ok: false, code: 'name_conflict', error: ownership.error }; + } + } + } + let intentId = existingIntentId; if (!intentId) { const { tags, overridePaths } = collectArtifacts(nodeId, stackName); @@ -176,6 +279,7 @@ export class DeployedStackDeletionService { rollback_tags_json: JSON.stringify(tags), override_paths_json: JSON.stringify(overridePaths), prune_volumes_requested: pruneVolumes ? 1 : 0, + required_blueprint_id: requiredBlueprintId, created_at: now, updated_at: now, }; @@ -196,38 +300,53 @@ export class DeployedStackDeletionService { return { ok: false, code: 'tombstone_failed', error: 'Deletion intent is not prepared' }; } - try { - await ComposeService.getInstance(nodeId).downStack(stackName); - } catch (downErr) { - console.warn( - '[DeployedStackDeletion] Compose down failed or no-op for %s:', - sanitizeForLog(stackName), - downErr, - ); - } - - if (intent.prune_volumes_requested === 1) { + if (!skipPhysical) { try { - await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]); - } catch (pruneErr) { + await ComposeService.getInstance(nodeId).downStack(stackName); + } catch (downErr) { console.warn( - '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:', + '[DeployedStackDeletion] Compose down failed or no-op for %s:', sanitizeForLog(stackName), - pruneErr, + downErr, ); } + + if (intent.prune_volumes_requested === 1) { + try { + await DockerController.getInstance(nodeId).pruneManagedOnly('volumes', [stackName]); + } catch (pruneErr) { + console.warn( + '[DeployedStackDeletion] Volume prune failed for %s, continuing delete:', + sanitizeForLog(stackName), + pruneErr, + ); + } + } + + try { + await FileSystemService.getInstance(nodeId).deleteStack(stackName); + } catch (fsErr) { + db.updateCleanupPendingStatus(intentId, 'cancelled'); + return { + ok: false, + code: 'fs_failed', + error: getErrorMessage(fsErr, 'Failed to remove stack files'), + }; + } } - try { - await FileSystemService.getInstance(nodeId).deleteStack(stackName); - } catch (fsErr) { - db.updateCleanupPendingStatus(intentId, 'cancelled'); - return { - ok: false, - code: 'fs_failed', - error: getErrorMessage(fsErr, 'Failed to remove stack files'), - }; - } + const finalized = await this.finalizeLogicalDeletion(input, intentId); + if (!finalized.ok) return finalized; + return { ok: true, status: skipPhysical ? 'already_absent' : 'deleted' }; + } + + /** Ready transaction, secondary DB/RBAC cleanup, mesh opt-out, sweep, invalidate. */ + private async finalizeLogicalDeletion( + input: DeleteDeployedStackInput, + intentId: string, + ): Promise { + const { nodeId, stackName } = input; + const db = DatabaseService.getInstance(); if (!db.commitStackDeletionReadyTransaction(intentId, nodeId, stackName)) { return { @@ -248,6 +367,7 @@ export class DeployedStackDeletionService { db.deleteStackExposure(nodeId, stackName); db.deleteStackProjectEnvFiles(nodeId, stackName); db.deleteStackScans(nodeId, stackName); + db.deleteNotificationsForStack(nodeId, stackName); } catch (dbErr) { console.error( '[DeployedStackDeletion] Secondary DB cleanup failed for %s; recovery rows already retired:', @@ -272,7 +392,15 @@ export class DeployedStackDeletionService { } await this.sweepReadyIntent(intentId); - return { ok: true }; + NotificationService.getInstance().broadcastEvent({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId, + stackName, + ts: Date.now(), + }); + return { ok: true, status: 'deleted' }; } /** @@ -487,6 +615,31 @@ export class DeployedStackDeletionService { continue; } + if (intent.required_blueprint_id != null) { + const ownership = await probeBlueprintMarkerOwnership( + nodeId, + stackName, + intent.required_blueprint_id, + ); + if (ownership.kind === 'name_conflict') { + db.updateCleanupPendingStatus(intent.id, 'cancelled'); + console.warn( + '[DeployedStackDeletion] Startup cancelled blueprint deletion for %s: %s', + sanitizeForLog(stackName), + sanitizeForLog(ownership.error), + ); + continue; + } + if (ownership.kind === 'failed') { + console.warn( + '[DeployedStackDeletion] Startup ownership probe failed for %s (leaving prepared): %s', + sanitizeForLog(stackName), + sanitizeForLog(ownership.error), + ); + continue; + } + } + const result = await this.deleteDeployedStack({ nodeId, stackName, diff --git a/backend/src/services/DockerController.ts b/backend/src/services/DockerController.ts index 8ee42120..7c46a91a 100644 --- a/backend/src/services/DockerController.ts +++ b/backend/src/services/DockerController.ts @@ -210,6 +210,12 @@ export interface DependencyContainer { /** Resolved Sencho stack, or null when the container is not Sencho-managed. */ stack: string | null; state: string; + /** + * Exit code from list Status when a parenthesized code is present + * (e.g. "Exited (0) …", "Restarting (1) …"); null when none (e.g. "Up …", bare "Exited"). + * Required so Drift can fail closed on unknown codes. + */ + exitCode: number | null; image: string; networks: { name: string; id: string; ip: string }[]; /** Named-volume sources mounted by the container (bind mounts excluded). */ @@ -1833,6 +1839,7 @@ class DockerController { composeProject: c.Labels?.['com.docker.compose.project'] ?? null, stack: DockerController.resolveContainerStack(c.Labels, projectToStack, knownSet, absDirToStack, resolvedBase), state: c.State ?? 'unknown', + exitCode: parseExitCode(typeof c.Status === 'string' ? c.Status : undefined), image: c.Image ?? '', networks, volumes, diff --git a/backend/src/services/DockerEventService.ts b/backend/src/services/DockerEventService.ts index 1bcc9a3f..7ea5033b 100644 --- a/backend/src/services/DockerEventService.ts +++ b/backend/src/services/DockerEventService.ts @@ -45,13 +45,21 @@ export interface ContainerHealthSnapshot { /** Grace window after a `die` before classifying, to absorb out-of-order kill events. */ const DIE_GRACE_WINDOW_MS = 500; -/** Max crash alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */ +/** Max crash/health alerts emitted per node within RATE_WINDOW_MS. Overflow is batched. */ const RATE_LIMIT_MAX = 20; const RATE_WINDOW_MS = 60_000; /** Dedup window for repeat crash alerts of the same container. */ const CRASH_DEDUP_MS = 60 * 60_000; +/** + * Dedup window for repeat health alerts of the same container. Same duration as + * crash dedup, but stored outside containerState so the 10-minute prune cannot + * shrink the advertised 60-minute window. Unlike crash dedup, recovery (start / + * healthy) does not clear this stamp. + */ +const HEALTH_DEDUP_MS = CRASH_DEDUP_MS; + /** Interval for pruning stale container state from memory. */ const PRUNE_INTERVAL_MS = 60_000; const STATE_STALE_AFTER_MS = 10 * 60_000; @@ -109,6 +117,11 @@ interface DockerEventPayload { type LifecycleStatus = 'disconnected' | 'connecting' | 'connected' | 'stopped'; +/** Rate-limit token bound to the fixed window that issued it. */ +interface RateToken { + readonly windowStart: number; +} + export class DockerEventService { private readonly nodeId: number; private readonly nodeName: string; @@ -127,12 +140,35 @@ export class DockerEventService { /** Pending die timers keyed by container ID (for the 500ms grace window). */ private pendingDieTimers: Map = new Map(); - /** Rate-limiter bookkeeping. */ - private rateWindowStart = 0; - private rateCount = 0; - private suppressedCount = 0; + /** + * Shared fixed-window rate limiter for crash and health alerts (per local + * node / this service instance). Dedup and transition checks run before + * consuming a token so suppressed duplicates do not count toward the cap. + * Tokens carry the issuing window start so a late refund cannot inflate a + * newer window's budget. + */ + private containerAlertRateWindowStart = 0; + private containerAlertRateCount = 0; + private suppressedCrashAlertCount = 0; + private suppressedHealthAlertCount = 0; private summaryTimer: NodeJS.Timeout | null = null; + /** + * Process-local health-alert dedup keyed by container id. Kept outside + * containerState so pruneStaleState cannot drop an active 60m window. + * Entries expire after HEALTH_DEDUP_MS via pruneStaleState; the map clears + * on process restart. Not cleared on healthy/start recovery. + */ + private healthAlertDedupAt = new Map(); + /** In-flight health dispatches; prevents concurrent duplicate flaps. */ + private healthAlertInFlight = new Set(); + /** + * Container ids that entered unhealthy again while a health dispatch was + * in flight. Retried once after the in-flight call finishes if history + * did not persist (so a failed write cannot permanently silence the alert). + */ + private healthAlertPendingRetry = new Set(); + /** Parse-error tracking for flooded bad payloads. */ private parseErrorWindowStart = 0; private parseErrorCount = 0; @@ -166,6 +202,8 @@ export class DockerEventService { /** Close the stream, cancel timers, and clear state. */ public shutdown(): void { + // Flush overflow roll-up while still live so pending summaries are not dropped. + this.flushRateLimitSummaryNow(); this.status = 'stopped'; this.clearReconnectTimer(); if (this.pruneTimer) { @@ -180,6 +218,17 @@ export class DockerEventService { this.pendingDieTimers.clear(); this.detachStream(); this.containerState.clear(); + this.healthAlertDedupAt.clear(); + this.healthAlertInFlight.clear(); + this.healthAlertPendingRetry.clear(); + this.suppressedCrashAlertCount = 0; + this.suppressedHealthAlertCount = 0; + this.containerAlertRateCount = 0; + } + + /** Status helper so await-crossing stopped checks are not narrowed away. */ + private isStopped(): boolean { + return this.status === 'stopped'; } // ======================================================================== @@ -460,27 +509,104 @@ export class DockerEventService { state.lastActivityAt = Date.now(); if (action.includes('unhealthy')) { - if (state.healthStatus !== 'unhealthy') { + const enteringUnhealthy = state.healthStatus !== 'unhealthy'; + if (enteringUnhealthy) { state.unhealthySince = Date.now(); } + // Auto-Heal reads healthStatus / unhealthySince; update before any alert gate. state.healthStatus = 'unhealthy'; + if (!enteringUnhealthy) return; if (!this.isCrashAlertsEnabled()) return; + void this.dispatchHealthAlert(id, state); + } else { + // Recovery clears Auto-Heal timing but must not erase health-alert dedup, + // or a healthy↔unhealthy flap would re-alert immediately. + state.unhealthySince = undefined; + state.healthStatus = action.includes('starting') ? 'starting' : 'healthy'; + // Drop deferred-retry markers so a later unrelated persist failure cannot + // spuriously retry from a flap that already recovered. + this.healthAlertPendingRetry.delete(id); + } + } + + /** + * Dispatch a healthcheck failure alert with transition already confirmed. + * Dedup and rate checks run before the token is consumed. Advance dedup + * only when dispatchAlert returns `{ persisted: true }` (history row written). + */ + private async dispatchHealthAlert(id: string, state: InternalContainerState): Promise { + if (this.isStopped()) return; + + const now = Date.now(); + if (this.isWithinAlertDedupWindow(this.healthAlertDedupAt.get(id), now, HEALTH_DEDUP_MS)) { + return; + } + if (this.healthAlertInFlight.has(id)) { + this.healthAlertPendingRetry.add(id); + return; + } + const token = this.consumeRateToken(); + if (!token) { + this.suppressedHealthAlertCount += 1; + this.scheduleSummary(); + return; + } + + this.healthAlertInFlight.add(id); + let persisted = false; + try { + if (this.isStopped()) { + this.refundRateToken(token); + this.healthAlertPendingRetry.delete(id); + return; + } const name = state.name ?? id.slice(0, 12); - void this.emitError( + const result = await this.emitError( 'monitor_alert', `Healthcheck failed: ${name} is unhealthy.`, state.stackName, state.name, state.isSelf === true, ); - } else { - state.unhealthySince = undefined; - if (action.includes('starting')) { - state.healthStatus = 'starting'; + persisted = result.persisted; + if (persisted) { + this.healthAlertDedupAt.set(id, Date.now()); + this.healthAlertPendingRetry.delete(id); } else { - state.healthStatus = 'healthy'; + // Refund only into the window that issued the token so a late + // persist failure cannot inflate a newer window's budget. + this.refundRateToken(token); + console.error( + `[DockerEvents:${this.nodeName}] Health alert history not persisted for ${id}; dedup not advanced`, + ); } + } finally { + this.healthAlertInFlight.delete(id); } + + if (this.isStopped()) { + this.healthAlertPendingRetry.delete(id); + return; + } + + // A concurrent healthy→unhealthy flap was deferred while we were in + // flight. If the first write failed, retry so the alert is not lost + // while the container stays unhealthy with no further transitions. + if (this.shouldRetryHealthAlert(id, state, persisted)) { + this.healthAlertPendingRetry.delete(id); + await this.dispatchHealthAlert(id, state); + } + } + + private shouldRetryHealthAlert( + id: string, + state: InternalContainerState, + persisted: boolean, + ): boolean { + return !this.isStopped() + && !persisted + && this.healthAlertPendingRetry.has(id) + && state.healthStatus === 'unhealthy'; } private onStart(id: string): void { @@ -495,6 +621,7 @@ export class DockerEventService { state.healthStatus = 'starting'; state.lastStartAt = Date.now(); state.lastActivityAt = Date.now(); + this.healthAlertPendingRetry.delete(id); } private onDestroy(id: string): void { @@ -504,6 +631,9 @@ export class DockerEventService { clearTimeout(pending); this.pendingDieTimers.delete(id); } + this.healthAlertPendingRetry.delete(id); + this.healthAlertInFlight.delete(id); + this.healthAlertDedupAt.delete(id); } private async classifyDie(id: string, event: DockerEventPayload, dieAt: number): Promise { @@ -545,7 +675,7 @@ export class DockerEventService { // Dedup early: crashloops repeatedly reach this point with exit 137, // and the OOM fallback below issues a Docker inspect. Skipping the // inspect on deduped crashes avoids hammering the daemon. - if (state.lastCrashAlertAt && now - state.lastCrashAlertAt < CRASH_DEDUP_MS) { + if (this.isWithinAlertDedupWindow(state.lastCrashAlertAt, now, CRASH_DEDUP_MS)) { return; } @@ -584,6 +714,7 @@ export class DockerEventService { state: InternalContainerState | null, info: { name: string; exitCode: number; stackName?: string; isSelf?: boolean }, ): Promise { + if (this.isStopped()) return; // Respect the existing global crash-alerts toggle so users who have // disabled these notifications in Settings remain opted out. if (!this.isCrashAlertsEnabled()) return; @@ -592,8 +723,9 @@ export class DockerEventService { ? `Container OOM Kill: ${info.name} was killed by the OOM killer (out of memory).` : `Container Crash Detected: ${info.name} exited unexpectedly (Code: ${info.exitCode}).`; - if (!this.consumeRateToken()) { - this.suppressedCount += 1; + const token = this.consumeRateToken(); + if (!token) { + this.suppressedCrashAlertCount += 1; this.scheduleSummary(); return; } @@ -625,32 +757,70 @@ export class DockerEventService { return value; } - /** Return true if an alert can be emitted now. Side effect: increments counters. */ - private consumeRateToken(): boolean { + /** + * Consume one shared crash/health rate token for the current fixed window. + * Returns null when the window is exhausted. Rolling into a new window first + * flushes any pending overflow summary so counters cannot span windows. + */ + private consumeRateToken(): RateToken | null { const now = Date.now(); - if (now - this.rateWindowStart >= RATE_WINDOW_MS) { - this.rateWindowStart = now; - this.rateCount = 0; + if (now - this.containerAlertRateWindowStart >= RATE_WINDOW_MS) { + this.flushRateLimitSummaryNow(); + this.containerAlertRateWindowStart = now; + this.containerAlertRateCount = 0; } - if (this.rateCount >= RATE_LIMIT_MAX) return false; - this.rateCount += 1; - return true; + if (this.containerAlertRateCount >= RATE_LIMIT_MAX) return null; + this.containerAlertRateCount += 1; + return { windowStart: this.containerAlertRateWindowStart }; + } + + /** Refund a token only when the issuing window is still active. */ + private refundRateToken(token: RateToken): void { + if (token.windowStart !== this.containerAlertRateWindowStart) return; + if (this.containerAlertRateCount > 0) this.containerAlertRateCount -= 1; + } + + private isWithinAlertDedupWindow( + lastAlertAt: number | undefined, + now: number, + windowMs: number, + ): boolean { + return lastAlertAt !== undefined && now - lastAlertAt < windowMs; + } + + private buildRateLimitSummaryMessage(crash: number, health: number): string { + const total = crash + health; + if (crash > 0 && health > 0) { + return `${total} additional container alerts were rate-limited in the last minute (${crash} crash, ${health} health).`; + } + if (health > 0) { + return `${health} additional container health alerts were rate-limited in the last minute.`; + } + return `${crash} additional container crash alerts were rate-limited in the last minute.`; + } + + /** Emit and clear any pending overflow roll-up immediately (e.g. on window roll). */ + private flushRateLimitSummaryNow(): void { + if (this.summaryTimer) { + clearTimeout(this.summaryTimer); + this.summaryTimer = null; + } + const crash = this.suppressedCrashAlertCount; + const health = this.suppressedHealthAlertCount; + this.suppressedCrashAlertCount = 0; + this.suppressedHealthAlertCount = 0; + if (crash + health <= 0) return; + if (this.status === 'stopped') return; + void this.emitWarning('monitor_alert', this.buildRateLimitSummaryMessage(crash, health)); } private scheduleSummary(): void { if (this.summaryTimer) return; - const remaining = RATE_WINDOW_MS - (Date.now() - this.rateWindowStart); + const remaining = RATE_WINDOW_MS - (Date.now() - this.containerAlertRateWindowStart); const delay = Math.max(1_000, remaining); this.summaryTimer = setTimeout(() => { - const count = this.suppressedCount; this.summaryTimer = null; - this.suppressedCount = 0; - if (count > 0) { - void this.emitWarning( - 'monitor_alert', - `${count} additional containers crashed in the last minute.`, - ); - } + this.flushRateLimitSummaryNow(); }, delay); } @@ -707,15 +877,24 @@ export class DockerEventService { } private pruneStaleState(): void { - if (this.containerState.size === 0) return; - const cutoff = Date.now() - STATE_STALE_AFTER_MS; - for (const [id, state] of this.containerState) { - // Keep state for a container with an unresolved crash so Auto-Heal can - // still act on it past the default stale window (policy thresholds run - // up to 24h). It is cleared on `start` and removed on `destroy`. - if (state.crashedAt !== undefined) continue; - if (state.lastActivityAt < cutoff) { - this.containerState.delete(id); + if (this.containerState.size > 0) { + const cutoff = Date.now() - STATE_STALE_AFTER_MS; + for (const [id, state] of this.containerState) { + // Keep state for a container with an unresolved crash so Auto-Heal can + // still act on it past the default stale window (policy thresholds run + // up to 24h). It is cleared on `start` and removed on `destroy`. + if (state.crashedAt !== undefined) continue; + if (state.lastActivityAt < cutoff) { + this.containerState.delete(id); + } + } + } + // Health dedup lives outside containerState so the 60m window survives + // ordinary 10m prune. Drop only entries whose window has fully expired. + if (this.healthAlertDedupAt.size > 0) { + const dedupCutoff = Date.now() - HEALTH_DEDUP_MS; + for (const [id, at] of this.healthAlertDedupAt) { + if (at < dedupCutoff) this.healthAlertDedupAt.delete(id); } } } @@ -734,15 +913,15 @@ export class DockerEventService { : { stackName, containerName, actor: 'system:docker-events' }; } - private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise { + private async emitError(category: NotificationCategory, message: string, stackName?: string, containerName?: string, systemOnly = false): Promise<{ persisted: boolean }> { return this.notifier.dispatchAlert('error', category, message, this.buildAlertOptions(stackName, containerName, systemOnly)); } - private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { + private async emitWarning(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> { return this.notifier.dispatchAlert('warning', category, message, this.buildAlertOptions(stackName, containerName)); } - private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise { + private async emitInfo(category: NotificationCategory, message: string, stackName?: string, containerName?: string): Promise<{ persisted: boolean }> { return this.notifier.dispatchAlert('info', category, message, this.buildAlertOptions(stackName, containerName)); } diff --git a/backend/src/services/DriftDetectionService.ts b/backend/src/services/DriftDetectionService.ts index 92fdab29..3151b4d2 100644 --- a/backend/src/services/DriftDetectionService.ts +++ b/backend/src/services/DriftDetectionService.ts @@ -8,6 +8,7 @@ import { parseEffectiveModel } from './preflight/effectiveModel'; import { compareStackNetworks, fromDeclaredCompose } from './network/normalize'; import { sanitizeForLog, redactSensitiveText } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; const MAX_RENDER_ERROR = 600; @@ -178,16 +179,17 @@ function networkDriftFindings( } /** - * Pure diff step (no Docker / FS access) so it is directly unit-testable. Only - * running containers are compared, since a stopped container publishes no ports - * and is not "deployed": a declared service with no running container is - * service-missing, a running container with no matching service is - * service-undeclared, and image / port differences are checked only for services - * present on both sides so a missing/undeclared service is not double-reported. + * Pure diff step (no Docker / FS access) so it is directly unit-testable. + * Running/restarting containers drive image/port comparison. Clean one-shot + * completions (exit 0 + explicit declared restart "no", including normalized + * `deploy.restart_policy.condition: none`) satisfy service presence without + * counting as hasContainers. Omitting restart does not qualify. Network + * comparison still uses only running/restarting attachments. */ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftReport { const { stack, declared, containers, parseError } = input; const networks = input.networks ?? []; + // Public contract: at least one running/restarting container (not "satisfied"). const hasContainers = containers.some((c) => RUNNING_STATES.has(c.state)); // A parse failure means the declared model is untrustworthy: report drift @@ -196,30 +198,44 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe return { stack, status: 'drifted', hasComposeFile: false, hasContainers, findings: [], parseError }; } - const runtimeByService = new Map(); - for (const c of containers) { - if (!RUNNING_STATES.has(c.state)) continue; - const name = c.service ?? c.name; - const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; - if (c.image) agg.images.add(normalizeImageRef(c.image)); - for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); - runtimeByService.set(name, agg); - } - - // Nothing running: the stack is defined on disk but not deployed. One status - // conveys this; per-service findings would just be noise. - if (!hasContainers) { - return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; - } - const declaredByName = new Map(); for (const svc of declared.services) declaredByName.set(svc.name, svc); + const runtimeByService = new Map(); + const oneShotSatisfied = new Set(); + for (const c of containers) { + const name = c.service ?? c.name; + if (RUNNING_STATES.has(c.state)) { + const agg = runtimeByService.get(name) ?? { images: new Set(), ports: new Set() }; + if (c.image) agg.images.add(normalizeImageRef(c.image)); + for (const p of c.ports) agg.ports.add(portKey(p.publishedPort, p.protocol)); + runtimeByService.set(name, agg); + continue; + } + const declaredRestart = declaredByName.get(name)?.restart; + if (isCleanOneShotCompletion({ + state: c.state, + exitCode: c.exitCode, + restartPolicy: declaredRestart, + })) { + oneShotSatisfied.add(name); + } + } + + const servicePresent = (serviceName: string): boolean => + runtimeByService.has(serviceName) || oneShotSatisfied.has(serviceName); + + // Nothing running and no declared service satisfied by a clean one-shot: the + // stack is defined on disk but not deployed. One status conveys this. + if (!hasContainers && !declared.services.some((svc) => servicePresent(svc.name))) { + return { stack, status: 'missing-runtime', hasComposeFile: true, hasContainers: false, findings: [] }; + } + const findings: StackDriftFinding[] = []; - // Declared service with no running container. + // Declared service with no running container and no clean one-shot completion. for (const svc of declared.services) { - if (!runtimeByService.has(svc.name)) { + if (!servicePresent(svc.name)) { findings.push({ kind: 'service-missing', service: svc.name, @@ -239,7 +255,7 @@ export function assembleStackDrift(input: AssembleStackDriftInput): StackDriftRe } } - // Image / port divergence for services present on both sides. + // Image / port divergence for services present on both sides (running only). for (const [name, svc] of declaredByName) { const runtime = runtimeByService.get(name); if (!runtime) continue; diff --git a/backend/src/services/FileSystemService.ts b/backend/src/services/FileSystemService.ts index f0e038b1..fecb9dd0 100644 --- a/backend/src/services/FileSystemService.ts +++ b/backend/src/services/FileSystemService.ts @@ -58,6 +58,13 @@ const PROTECTED_STACK_FILES = new Set([ '.env', ]); +// Explorer-only protection: includes the blueprint ownership marker without +// putting it in PROTECTED_STACK_FILES (backup/rollback orphan removal). +const EXPLORER_PROTECTED_STACK_FILES = new Set([ + ...PROTECTED_STACK_FILES, + '.blueprint.json', +]); + // Bookkeeping markers Sencho writes into the backup slot. They are never copied // back into the stack directory on restore: `.timestamp` records when the backup // was taken; `.checksums` is the integrity manifest verified before a restore. @@ -169,7 +176,7 @@ function isProtectedRelPath(relPath: string): boolean { if (normalized.includes('/')) return false; // Fold case so e.g. a request for COMPOSE.YAML cannot dodge the gate on a // case-insensitive filesystem where it resolves to the real compose.yaml. - return PROTECTED_STACK_FILES.has(fsCaseKey(normalized)); + return EXPLORER_PROTECTED_STACK_FILES.has(fsCaseKey(normalized)); } function protectedFileError(relPath: string): Error & { code: string } { @@ -1643,7 +1650,7 @@ export class FileSystemService { type, size, mtime, - isProtected: protectedEnabled && PROTECTED_STACK_FILES.has(dirent.name), + isProtected: protectedEnabled && EXPLORER_PROTECTED_STACK_FILES.has(dirent.name), }; }) ); @@ -2096,7 +2103,7 @@ export class FileSystemService { type, size: stat.isDirectory() ? 0 : stat.size, mtime: stat.mtimeMs, - isProtected: (scope?.protectedEnabled ?? true) && PROTECTED_STACK_FILES.has(name), + isProtected: (scope?.protectedEnabled ?? true) && EXPLORER_PROTECTED_STACK_FILES.has(name), }; } } diff --git a/backend/src/services/HealthGateService.ts b/backend/src/services/HealthGateService.ts index 0e51f1ce..a5c413df 100644 --- a/backend/src/services/HealthGateService.ts +++ b/backend/src/services/HealthGateService.ts @@ -5,8 +5,11 @@ import { AutoHealService } from './AutoHealService'; import { sanitizeForLog } from '../utils/safeLog'; import { getErrorMessage } from '../utils/errors'; import { withTimeout } from '../utils/withTimeout'; +import { isCleanOneShotCompletion } from '../utils/oneShotCompletion'; +import { declaredFromEffectiveModel } from '../helpers/effectiveToDeclaredCompose'; +import { parseEffectiveModel } from './preflight/effectiveModel'; import type { HealthGateContainer, HealthGateReport } from './updateGuard/types'; -import { getComposeCommandTimeoutMs } from './ComposeService'; +import { ComposeService, getComposeCommandTimeoutMs } from './ComposeService'; const POLL_INTERVAL_MS = 5_000; // A prepared-but-never-begun token (prepare called, mutation then failed before @@ -47,6 +50,10 @@ interface ObservedContainer { service: string | null; /** Image id the container is running (service gates check convergence on it). */ imageId: string; + /** Inspect State.ExitCode; null when unavailable. */ + exitCode: number | null; + /** HostConfig.RestartPolicy.Name (report/debug only; not used for one-shot intent). */ + restartPolicy: string | null; } interface ActiveGate { @@ -89,6 +96,13 @@ interface ActiveGate { collateralBaselineByName: Map; /** Role of each expected container (service gates), for failure attribution. */ roleByName: Map; + /** + * Declared Compose restart intent by service name (normalized). Loaded once + * per gate; null until the first load attempt completes. Used for one-shot + * recognition instead of Docker inspect (which cannot distinguish omit vs + * explicit `restart: "no"`). + */ + declaredRestartByService: Map | null; } /** A pre-mutation baseline container captured at prepare time. */ @@ -101,6 +115,8 @@ interface PreparedBaseline { restartCount: number; startedAt: string | null; imageId: string; + exitCode: number | null; + restartPolicy: string | null; } function isRegressionEligibleSibling(baseline: PreparedBaseline): boolean { @@ -118,9 +134,35 @@ function observedFromPreparedBaseline(baseline: PreparedBaseline): ObservedConta health: baseline.health, service: baseline.service, imageId: baseline.imageId, + exitCode: baseline.exitCode, + restartPolicy: baseline.restartPolicy, }; } +/** Running, or a clean one-shot exit (exit 0 + explicit declared restart "no"). */ +function isObservedContainerSatisfied( + gate: ActiveGate, + current: ObservedContainer | undefined, +): boolean { + if (!current) return false; + if (current.state === 'running') return true; + return isDeclaredCleanOneShot(gate, current); +} + +/** + * One-shot recognition from declared Compose intent only. Unlabeled containers + * and services missing from the effective model fail closed. + */ +function isDeclaredCleanOneShot(gate: ActiveGate, current: ObservedContainer): boolean { + const map = gate.declaredRestartByService; + if (!map || !current.service || !map.has(current.service)) return false; + return isCleanOneShotCompletion({ + state: current.state, + exitCode: current.exitCode, + restartPolicy: map.get(current.service), + }); +} + /** An in-memory prepare snapshot awaiting attachExpectedImage + beginPrepared. */ interface PreparedGate { token: string; @@ -329,6 +371,7 @@ export class HealthGateService { collateralEligibleNames: new Set(), collateralBaselineByName: new Map(), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -488,6 +531,7 @@ export class HealthGateService { prep.collateralBaseline.filter(b => prep.collateralEligibleNames.has(b.name)), ), roleByName: new Map(), + declaredRestartByService: null, }; this.active.set(key, gate); this.scheduleNextPoll(gate); @@ -587,6 +631,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; if (gate.expected === null) { @@ -629,12 +676,16 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { - // An exit with no restart attempt is terminal for the window. + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + // An exit with no restart attempt is terminal for the window, unless + // this is an expected one-shot (exit 0 + explicit declared restart "no"). + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `container ${name} exited during observation`, summary); return; } - if (current.health === 'unhealthy') { + // Residual Docker health on a completed one-shot is not a gate failure; + // long-running containers still fail on unhealthy. + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `container ${name} reported unhealthy`, summary); return; } @@ -657,14 +708,19 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; - // Window complete: pass requires everything running and healthy wherever a - // healthcheck exists. A health state still 'starting' is not a pass. - const stillStarting = observed.filter(c => c.health === 'starting'); + // Window complete: pass requires everything running (or a clean one-shot + // completion) and healthy wherever a healthcheck exists. A health state + // still 'starting' is not a pass (except residual health on a clean one-shot). + const stillStarting = observed.filter( + c => c.health === 'starting' && !isDeclaredCleanOneShot(gate, c), + ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const notRunning = [...gate.expected.keys()].filter(name => byName.get(name)?.state !== 'running'); + const notRunning = [...gate.expected.keys()].filter( + name => !isObservedContainerSatisfied(gate, byName.get(name)), + ); if (notRunning.length > 0) { this.finalize(gate, 'failed', `not running at the end of the window: ${notRunning.join(', ')}`, summary); return; @@ -700,6 +756,9 @@ export class HealthGateService { if (gate.finalized || this.active.get(key) !== gate) return; gate.consecutivePollErrors = 0; + await this.ensureDeclaredRestartMap(gate); + if (gate.finalized || this.active.get(key) !== gate) return; + const elapsedMs = Date.now() - gate.startedAt; const serviceName = gate.serviceName ?? ''; @@ -781,11 +840,12 @@ export class HealthGateService { } gate.missingLastPoll.delete(name); - if (current.state === 'exited' && baseline.restarts === current.restarts) { + const cleanOneShot = isDeclaredCleanOneShot(gate, current); + if (current.state === 'exited' && baseline.restarts === current.restarts && !cleanOneShot) { this.finalize(gate, 'failed', `${noun} ${name} exited during observation`, summary, role); return; } - if (current.health === 'unhealthy') { + if (!cleanOneShot && current.health === 'unhealthy') { this.finalize(gate, 'failed', `${noun} ${name} reported unhealthy`, summary, role); return; } @@ -808,24 +868,28 @@ export class HealthGateService { if (elapsedMs < gate.windowSeconds * 1000) return; const stillStarting = observed.filter( - c => c.health === 'starting' && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), + c => c.health === 'starting' + && !isDeclaredCleanOneShot(gate, c) + && (c.service === serviceName || gate.collateralEligibleNames.has(c.name)), ); if (stillStarting.length > 0) { this.finalize(gate, 'unknown', 'a healthcheck was still starting when the observation window ended', summary); return; } - const runningPrimary = observed.filter(c => c.service === serviceName && c.state === 'running'); - if (runningPrimary.length !== gate.expectedReplicas) { + const satisfiedPrimary = observed.filter( + c => c.service === serviceName && isObservedContainerSatisfied(gate, c), + ); + if (satisfiedPrimary.length !== gate.expectedReplicas) { this.finalize( gate, 'failed', - `service ${serviceName} has ${runningPrimary.length} running replica(s), expected ${gate.expectedReplicas}`, + `service ${serviceName} has ${satisfiedPrimary.length} satisfied replica(s), expected ${gate.expectedReplicas}`, summary, 'primary', ); return; } const collateralNotRunning = [...gate.expected.keys()] .filter(name => gate.roleByName.get(name) === 'collateral') - .filter(name => byName.get(name)?.state !== 'running'); + .filter(name => !isObservedContainerSatisfied(gate, byName.get(name))); if (collateralNotRunning.length > 0) { this.finalize(gate, 'failed', `sibling(s) not running at the end of the window: ${collateralNotRunning.join(', ')}`, summary, 'collateral'); return; @@ -855,6 +919,35 @@ export class HealthGateService { return this.listStackContainers(gate.nodeId, gate.stackName); } + /** + * Load declared Compose restart intent once per gate. Fail closed to an empty + * map on render/parse errors so inspect "no" cannot false-qualify one-shots. + */ + private async ensureDeclaredRestartMap(gate: ActiveGate): Promise { + if (gate.declaredRestartByService !== null) return; + gate.declaredRestartByService = new Map(); + try { + const result = await ComposeService.getInstance(gate.nodeId).renderConfig(gate.stackName); + if (result.rendered === null) { + console.warn( + '[HealthGate] declared restart map unavailable for %s (compose render failed)', + sanitizeForLog(gate.stackName), + ); + return; + } + const model = parseEffectiveModel(JSON.parse(result.rendered), gate.stackName); + const declared = declaredFromEffectiveModel(model); + gate.declaredRestartByService = new Map( + declared.services.map((s) => [s.name, s.restart ?? null]), + ); + } catch (error) { + console.warn( + '[HealthGate] declared restart map load failed for %s:', + sanitizeForLog(gate.stackName), getErrorMessage(error, 'unknown'), + ); + } + } + /** List and inspect a stack's containers into the gate's observation shape. */ private async listStackContainers(nodeId: number, stackName: string): Promise { const docker = DockerController.getInstance(nodeId).getDocker(); @@ -877,6 +970,8 @@ export class HealthGateService { health: inspect.State?.Health?.Status ?? null, service: labels['com.docker.compose.service'] ?? null, imageId: inspect.Image ?? '', + exitCode: typeof inspect.State?.ExitCode === 'number' ? inspect.State.ExitCode : null, + restartPolicy: inspect.HostConfig?.RestartPolicy?.Name || null, }; } catch (e: unknown) { // Removed between list and inspect; the missing-container logic will @@ -903,6 +998,8 @@ export class HealthGateService { restartCount: c.restartCount, startedAt: c.startedAt, imageId: c.imageId, + exitCode: c.exitCode, + restartPolicy: c.restartPolicy, })); } diff --git a/backend/src/services/HostTerminalService.ts b/backend/src/services/HostTerminalService.ts index d5c8d927..20627322 100644 --- a/backend/src/services/HostTerminalService.ts +++ b/backend/src/services/HostTerminalService.ts @@ -18,6 +18,8 @@ export interface ConsoleAuditContext { // path always supplies a concrete id. readonly nodeId: number | null; readonly ipAddress: string; + /** Hub operator for remote console_session bridges; null for direct sessions. */ + readonly actingAs?: string | null; } const CONSOLE_AUDIT_PATH = '/api/system/host-console'; @@ -97,6 +99,7 @@ export class HostTerminalService { node_id: audit.nodeId, ip_address: audit.ipAddress, summary, + acting_as: audit.actingAs ?? null, }); } catch (err) { console.error('[HostConsole] Failed to write session audit log:', err); diff --git a/backend/src/services/MonitorService.ts b/backend/src/services/MonitorService.ts index 4840b08c..4df2a575 100644 --- a/backend/src/services/MonitorService.ts +++ b/backend/src/services/MonitorService.ts @@ -32,6 +32,39 @@ const getOperatorPhrase = (operator: string): string => { return `triggered the operator ${operator}`; }; +/** Sentinel for containers without a Compose service label. Not a valid API target. */ +const UNLABELED_SERVICE_KEY = '_unlabeled'; + +function displayServiceName(serviceName: string): string { + return serviceName === UNLABELED_SERVICE_KEY ? 'unknown service' : serviceName; +} + +function parseAlertBreachKey(key: string): { ruleId: number; containerId: string } | null { + const sep = key.indexOf(':'); + if (sep < 0) return null; + const ruleId = Number(key.slice(0, sep)); + if (!Number.isFinite(ruleId)) return null; + return { ruleId, containerId: key.slice(sep + 1) }; +} + +/** Per-service cooldown timestamp, with pre-migration fallback to rule.last_fired_at. */ +function resolveStackAlertLastFired(rule: StackAlert, serviceName: string, db: DatabaseService): number { + const ruleId = rule.id!; + let lastFired = db.getStackAlertServiceCooldown(ruleId, serviceName); + if (lastFired == null && !db.hasAnyStackAlertServiceCooldown(ruleId) && (rule.last_fired_at || 0) > 0) { + lastFired = rule.last_fired_at || 0; + } + return lastFired || 0; +} + +function normalizeContainerName(container: { Id: string; Names?: string[] }): string { + const raw = container.Names?.[0]; + if (raw && raw.length > 0) { + return raw.startsWith('/') ? raw.slice(1) : raw; + } + return container.Id.slice(0, 12); +} + /** Shape of the JSON returned by Docker container stats (stream: false). */ interface DockerContainerStats { cpu_stats?: { @@ -153,21 +186,18 @@ export class MonitorService { private janitorConsecutiveTimeouts = 0; private janitorBreakerUntil = 0; - // Track the duration a specific stack alert rule has been in breach state - // key: rule_id, value: AlertState - private activeBreaches = new Map(); + // Track the duration a specific stack alert rule+container has been in breach + // key: `${ruleId}:${containerId}`, value: AlertState + private activeBreaches = new Map(); // Track previous network counters per container for rate calculation. // key: container_id, value: { rx bytes, tx bytes, sample timestamp } private previousNetworkStats = new Map(); - // Per-cycle dispatch dedup. Stack rules are shared across every container - // in the stack, so parallel processContainer calls can race past the - // cooldown check (DB write happens after the awaited dispatch) and fire - // the same alert N times on first breach. The synchronous check-and-add - // below is atomic in JS between awaits, so only the first worker wins. + // Per-cycle dispatch dedup keyed by rule+service so replicas of the same + // Compose service do not fire N times, while different services can. // Reset at the start of each evaluateStackAlerts call. - private firedThisCycle = new Set(); + private firedThisCycle = new Set(); // Crash and healthcheck detection live in DockerEventService (event-driven, // causal classification). MonitorService no longer polls for container @@ -604,6 +634,10 @@ export class MonitorService { else alertsByStack.set(a.stack_name, [a]); } + const activeRuleIds = new Set(alerts.map(a => a.id!)); + const allCurrentIds = new Set(); + let enumerationFailed = false; + for (const node of nodes) { if (!node.id) continue; // Remote nodes are self-monitoring - skip direct Docker access @@ -623,22 +657,39 @@ export class MonitorService { (container) => this.processContainer(node, container, alertsByStack, docker, db), ); + for (const c of containers) allCurrentIds.add(c.Id); + // Clean up stale network stats for containers on this node that no longer run if (this.previousNetworkStats.size > containers.length * 2) { - const currentIds = new Set(containers.map((c: { Id: string }) => c.Id)); + const nodeIds = new Set(containers.map((c: { Id: string }) => c.Id)); for (const key of this.previousNetworkStats.keys()) { - if (!currentIds.has(key)) this.previousNetworkStats.delete(key); + if (!nodeIds.has(key)) this.previousNetworkStats.delete(key); } } } catch (err) { + // Enumeration failed: preserve existing breach timers. + enumerationFailed = true; console.error(`Error fetching containers for node ${node.name}`, err); } } - // Clean up stale breach trackers for rules that have been deleted - const activeRuleIds = new Set(alerts.map(a => a.id!)); - for (const key of this.activeBreaches.keys()) { - if (!activeRuleIds.has(key)) { + // After successful enumeration(s), drop breach timers for containers + // that are no longer running. Skip when any local enumeration failed + // so a transient Docker error cannot reset duration timers. + if (!enumerationFailed) { + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && activeRuleIds.has(parsed.ruleId) && !allCurrentIds.has(parsed.containerId)) { + this.activeBreaches.delete(key); + } + } + } + + // Clean up in-memory breach trackers for rules that have been deleted. + // Persisted cooldowns are removed only via transactional deleteStackAlert. + for (const key of [...this.activeBreaches.keys()]) { + const parsed = parseAlertBreachKey(key); + if (parsed && !activeRuleIds.has(parsed.ruleId)) { this.activeBreaches.delete(key); } } @@ -666,12 +717,14 @@ export class MonitorService { */ private async processContainer( node: Node, - container: { Id: string; Labels?: Record }, + container: { Id: string; Names?: string[]; Labels?: Record }, alertsByStack: Map, docker: DockerController, db: DatabaseService, ): Promise { const stackName = container.Labels?.['com.docker.compose.project'] || 'system'; + const serviceName = container.Labels?.['com.docker.compose.service'] || UNLABELED_SERVICE_KEY; + const containerName = normalizeContainerName(container); try { const rawStats = await withTimeout( @@ -727,7 +780,11 @@ export class MonitorService { }); for (const rule of stackAlerts) { + if (rule.service_name && rule.service_name !== serviceName) continue; + const ruleId = rule.id!; + const breachKey = `${ruleId}:${container.Id}`; + const cooldownKey = `${ruleId}:${serviceName}`; const currentValue = metrics[rule.metric as keyof typeof metrics]; if (currentValue === undefined) continue; @@ -735,57 +792,75 @@ export class MonitorService { const isBreaching = this.evaluateCondition(currentValue, rule.operator, rule.threshold); if (isBreaching) { - if (!this.activeBreaches.has(ruleId)) { - this.activeBreaches.set(ruleId, { breachStartedAt: Date.now() }); - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}"`); + if (!this.activeBreaches.has(breachKey)) { + this.activeBreaches.set(breachKey, { breachStartedAt: Date.now() }); + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach entered: rule ${ruleId} (${rule.metric} ${rule.operator} ${rule.threshold}) on stack "${rule.stack_name}" service "${serviceName}" container "${containerName}"`); } - const breachState = this.activeBreaches.get(ruleId)!; + const breachState = this.activeBreaches.get(breachKey)!; const durationMs = Date.now() - breachState.breachStartedAt; const requiredDurationMs = rule.duration_mins * 60 * 1000; if (durationMs >= requiredDurationMs) { - const timeSinceLastFired = Date.now() - (rule.last_fired_at || 0); + const timeSinceLastFired = Date.now() - resolveStackAlertLastFired(rule, serviceName, db); const requiredCooldownMs = rule.cooldown_mins * 60 * 1000; if (timeSinceLastFired >= requiredCooldownMs) { - // Claim this rule for the cycle before awaiting - // dispatch. The check-and-add is synchronous, so - // sibling workers evaluating the same shared rule - // see the claim and skip — preventing N-fire when - // multiple containers in one stack all breach. - if (this.firedThisCycle.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} (already fired this cycle by sibling container)`); - } else { - this.firedThisCycle.add(ruleId); + // Claim this rule+service for the cycle before awaiting + // dispatch so replicas of the same service skip. + if (this.firedThisCycle.has(cooldownKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Skipping duplicate dispatch for rule ${ruleId} service "${serviceName}" (already fired this cycle by sibling replica)`); + continue; + } - const { name: metricName, unit } = getMetricDetails(rule.metric); - const operatorPhrase = getOperatorPhrase(rule.operator); + this.firedThisCycle.add(cooldownKey); - const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; - const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const { name: metricName, unit } = getMetricDetails(rule.metric); + const operatorPhrase = getOperatorPhrase(rule.operator); - // Node-neutral body: the hub bell badge attributes remotes. - const message = `The **${metricName}** for **${rule.stack_name}** ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + const safeCurrent = typeof currentValue === 'number' ? Number(currentValue.toFixed(2)) : currentValue; + const safeThreshold = typeof rule.threshold === 'number' ? Number(rule.threshold.toFixed(2)) : rule.threshold; + const serviceLabel = displayServiceName(serviceName); - console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); - await NotificationService.getInstance().dispatchAlert( + // Node-neutral body: the hub bell badge attributes remotes. + const message = `The **${metricName}** for **${serviceLabel}** in **${rule.stack_name}** (container **${containerName}**) ${operatorPhrase} **${safeThreshold}${unit}** (Currently: ${safeCurrent}${unit}).`; + + try { + const { persisted } = await NotificationService.getInstance().dispatchAlert( 'warning', 'monitor_alert', message, - { stackName: rule.stack_name, actor: 'system:monitor' }, + { stackName: rule.stack_name, containerName, actor: 'system:monitor' }, + ); + if (!persisted) { + // History was not written; do not advance cooldown or we silence retries. + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Alert history not persisted for rule ${ruleId} service "${serviceName}"; cooldown not advanced`, + ); + continue; + } + const firedAt = Date.now(); + // Dual-write: per-service row for current code, parent last_fired_at + // so a downgrade that only reads the parent column still has a floor. + db.upsertStackAlertServiceCooldown(ruleId, serviceName, firedAt); + db.updateStackAlertLastFired(ruleId, firedAt); + console.log(`[MonitorService] Alert fired: rule ${ruleId} on stack "${rule.stack_name}" service "${serviceName}": ${metricName} ${operatorPhrase} ${safeThreshold}${unit}`); + } catch (fireErr) { + this.firedThisCycle.delete(cooldownKey); + console.error( + `[MonitorService] Failed to fire alert rule ${ruleId} service "${serviceName}" container "${containerName}":`, + fireErr, ); - - db.updateStackAlertLastFired(ruleId, Date.now()); } } else if (isDebugEnabled()) { - console.log(`[Monitor:diag] Cooldown active for rule ${ruleId}: ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); + console.log(`[Monitor:diag] Cooldown active for rule ${ruleId} service "${serviceName}": ${Math.round((requiredCooldownMs - timeSinceLastFired) / 1000)}s remaining`); } } } else { - if (this.activeBreaches.has(ruleId)) { - if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}"`); - this.activeBreaches.delete(ruleId); + if (this.activeBreaches.has(breachKey)) { + if (isDebugEnabled()) console.log(`[Monitor:diag] Breach cleared: rule ${ruleId} on stack "${rule.stack_name}" container "${containerName}"`); + this.activeBreaches.delete(breachKey); } } } diff --git a/backend/src/services/NotificationService.ts b/backend/src/services/NotificationService.ts index 2a6ebbee..ba414ce0 100644 --- a/backend/src/services/NotificationService.ts +++ b/backend/src/services/NotificationService.ts @@ -181,7 +181,7 @@ export class NotificationService { category: NotificationCategory, message: string, options?: { stackName?: string; containerName?: string; actor?: string }, - ) { + ): Promise<{ persisted: boolean }> { const t0 = Date.now(); const { stackName, containerName, actor } = options ?? {}; @@ -191,6 +191,8 @@ export class NotificationService { // WebSocket broadcast can all throw on an unhealthy DB, which would // otherwise surface as an unhandledRejection and take the process down. // The whole body is wrapped so the worst case is a dropped notification. + // Callers that gate cooldowns on history use `persisted` (true only after the row write). + let wroteHistory = false; try { // Internal writes use the middleware default so they share a row key // with user-initiated requests; otherwise the UI and monitors split @@ -216,11 +218,12 @@ export class NotificationService { container_name: containerName, actor_username: actor ?? null, }); + wroteHistory = true; StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, true); } catch (err) { StackActivityMetricsService.getInstance().record(localNodeId, 'write', Date.now() - t0, false); console.error('[Notify] Failed to persist notification:', err); - return; + return { persisted: false }; } // Separate [StackActivity:diag] namespace from the [Notify:diag] lines // below so a single grep can pull every per-stack timeline write across @@ -273,7 +276,7 @@ export class NotificationService { } if (suppressExternal) { - return; + return { persisted: wroteHistory }; } // Resolve retry extras once for this dispatch (shared by all destinations). @@ -298,14 +301,14 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); - return; + return { persisted: wroteHistory }; } // 4. Fall back to this instance's agents (keyed by this instance's default node id). const agents = this.dbService.getEnabledAgents(localNodeId); if (agents.length === 0) { if (isDebugEnabled()) console.log('[Notify:diag] No routes or agents matched; skipping external dispatch'); - return; + return { persisted: wroteHistory }; } if (isDebugEnabled()) console.log(`[Notify:diag] Falling back to ${agents.length} global agent(s)`); @@ -322,8 +325,11 @@ export class NotificationService { ) ); this.recordDispatchErrors(notification.id!, errors); + return { persisted: wroteHistory }; } catch (err) { console.error('[Notify] dispatchAlert failed:', err); + // History may already be written; callers must not treat that as a miss. + return { persisted: wroteHistory }; } } diff --git a/backend/src/services/blueprintPreviewProjection.ts b/backend/src/services/blueprintPreviewProjection.ts index f32e2c71..d4991406 100644 --- a/backend/src/services/blueprintPreviewProjection.ts +++ b/backend/src/services/blueprintPreviewProjection.ts @@ -457,15 +457,30 @@ function asApprovedBlueprint(blueprint: Blueprint): Blueprint & BlueprintApprova } /** Upgrade create rows to blockers when an unmanaged same-name stack already exists. */ -async function applyCreateNameConflictBlockers(blueprintName: string, raw: RawAction[]): Promise { +function blockCreateForOwnership(row: RawAction, detail: string): void { + row.action = 'blocked_name_conflict'; + row.severity = 'blocker'; + row.detail = detail; +} + +async function applyCreateNameConflictBlockers( + blueprintName: string, + blueprintId: number, + raw: RawAction[], +): Promise { const { BlueprintService } = await import('./BlueprintService'); const svc = BlueprintService.getInstance(); for (const row of raw) { if (row.action !== 'create') continue; - if (!(await svc.hasNameConflict(blueprintName, row.node))) continue; - row.action = 'blocked_name_conflict'; - row.severity = 'blocker'; - row.detail = 'Unmanaged stack with this name already exists on this node'; + try { + if (!(await svc.hasNameConflict(blueprintName, row.node, blueprintId))) continue; + blockCreateForOwnership(row, 'Unmanaged stack with this name already exists on this node'); + } catch (err) { + blockCreateForOwnership( + row, + err instanceof Error ? err.message : 'Cannot verify stack ownership on this node', + ); + } } } @@ -477,7 +492,7 @@ export async function buildBlueprintPreview(blueprintId: number): Promise; containerName?: string; user?: string; diff --git a/backend/src/services/preflight/rules.ts b/backend/src/services/preflight/rules.ts index e58ca8e3..2e39aef1 100644 --- a/backend/src/services/preflight/rules.ts +++ b/backend/src/services/preflight/rules.ts @@ -363,7 +363,7 @@ const noRestartPolicy: PreflightRule = { message: `Service "${s.name}" has no restart policy, so it will not come back after a crash or host reboot.`, sourcePath: s.name, service: s.name, - remediation: 'Add restart: unless-stopped.', + remediation: 'For long-running services, add restart: unless-stopped. For one-shot or init jobs that should finish and stay stopped, restart: "no" is appropriate.', })); }, }; diff --git a/backend/src/utils/oneShotCompletion.ts b/backend/src/utils/oneShotCompletion.ts new file mode 100644 index 00000000..4c5206bb --- /dev/null +++ b/backend/src/utils/oneShotCompletion.ts @@ -0,0 +1,77 @@ +/** + * Helpers for clean one-shot completion and Compose restart-intent normalization. + * Used by Health Gate and Drift service-presence only; does not change bulk + * status, Auto-Heal, or atomic-deploy helpers. + */ + +/** + * True only for an explicit Compose `restart: "no"` (after normalization). + * Absent / null / empty do not qualify: Docker reports HostConfig restart + * "no" for both intentional one-shots and bare long-running services that + * omit `restart:`, so consumers must pass declared Compose intent, not inspect. + */ +export function isNoRestartPolicy(policy: string | null | undefined): boolean { + return policy === 'no'; +} + +/** + * Normalize Compose restart intent for Drift / declared-model consumers. + * + * Compose precedence: when `deploy.restart_policy` is set, its `condition` + * wins; otherwise the service-level `restart` field is used. Conditions map to + * Docker-like policy names so {@link isNoRestartPolicy} stays the single gate: + * `none` → `no`, `any` → `always`, `on-failure` → `on-failure`. Missing + * condition defaults to `any` (Compose default). Unknown shapes fail closed. + * Absent service restart (no deploy policy) stays null and is not one-shot eligible. + */ +export function normalizeComposeRestartIntent( + serviceRestart: string | null | undefined, + deploy?: Record | null, +): string | null { + if (deploy && Object.prototype.hasOwnProperty.call(deploy, 'restart_policy')) { + const policy = deploy['restart_policy']; + if (policy === null || typeof policy !== 'object' || Array.isArray(policy)) { + return 'always'; + } + const condition = (policy as Record).condition; + // Missing, empty, or non-string → Compose default `any` → always. + if (typeof condition !== 'string' || condition === '') { + return 'always'; + } + switch (condition.toLowerCase()) { + case 'none': + return 'no'; + case 'on-failure': + return 'on-failure'; + case 'any': + return 'always'; + default: + return 'always'; + } + } + if (serviceRestart == null || serviceRestart === '') return null; + return serviceRestart; +} + +export interface OneShotCompletionInput { + state: string; + /** Exact Docker exit code; null means unknown (fail closed). */ + exitCode: number | null; + /** + * Declared Compose restart intent after normalization (`"no"` for explicit + * one-shots / `deploy.restart_policy.condition: none`). Do not pass Docker + * inspect HostConfig values here. + */ + restartPolicy: string | null | undefined; +} + +/** + * True only for an exited container with exit code exactly 0 and explicit + * declared restart `"no"`. Null/absent restart, null exit codes, and restarting + * policies never qualify. + */ +export function isCleanOneShotCompletion(input: OneShotCompletionInput): boolean { + return input.state === 'exited' + && input.exitCode === 0 + && isNoRestartPolicy(input.restartPolicy); +} diff --git a/backend/src/websocket/generic.ts b/backend/src/websocket/generic.ts index 9e2edfaf..bf93c84b 100644 --- a/backend/src/websocket/generic.ts +++ b/backend/src/websocket/generic.ts @@ -54,9 +54,10 @@ export function handleGenericWs( if (isProxyToken) return reject(socket, 403, 'Forbidden'); // Admin enforcement: container exec requires admin role. - // console_session tokens are already admin-gated at creation time. - // API tokens reaching this point have full-admin scope (read-only / - // deploy-only are blocked by the upgrade handler's scope gate). + // console_session tokens are already admin-gated at creation time and + // path/jti-gated in upgradeHandler. API tokens reaching this point have + // full-admin scope (read-only / deploy-only are blocked by the upgrade + // handler's scope gate). if (!decoded.scope) { const execUser = decoded.username ? DatabaseService.getInstance().getUserByUsername(decoded.username) : undefined; if (!execUser) { diff --git a/backend/src/websocket/hostConsole.ts b/backend/src/websocket/hostConsole.ts index eaf26ade..08c137d3 100644 --- a/backend/src/websocket/hostConsole.ts +++ b/backend/src/websocket/hostConsole.ts @@ -2,22 +2,16 @@ import type { IncomingMessage } from 'http'; import type { Duplex } from 'stream'; import WebSocket, { WebSocketServer } from 'ws'; import { FileSystemService } from '../services/FileSystemService'; -import { NodeRegistry } from '../services/NodeRegistry'; import { HostTerminalService } from '../services/HostTerminalService'; -import { PROXY_TIER_HEADER } from '../services/license-headers'; -import { - isLicenseTier, - normalizeTier, -} from '../services/license-normalize'; -import { LicenseService } from '../services/LicenseService'; -import { ROLE_PERMISSIONS, type PermissionAction } from '../middleware/permissions'; +import { ROLE_PERMISSIONS } from '../middleware/permissions'; import type { UserRole } from '../services/DatabaseService'; import { getErrorMessage } from '../utils/errors'; import { rejectUpgrade as reject } from './reject'; +import { isConsoleSessionScope } from '../helpers/consoleSession'; interface HostConsoleContext { nodeId: number; - decoded: { scope?: string; username?: string }; + decoded: { scope?: string; username?: string; acting_as?: string }; isProxyToken: boolean; wsResolvedUser: { username: string; role: UserRole; token_version: number } | undefined; stackParam: string | null; @@ -26,15 +20,16 @@ interface HostConsoleContext { /** * Handle `/api/system/host-console` WebSocket upgrades. * - * Enforces three gates before spawning the host PTY: + * Enforces two gates before spawning the host PTY: * 1. Machine-credential rejection: node_proxy tokens cannot reach an - * interactive host shell. + * interactive host shell directly (remote forwarding mints a + * console_session via POST /console-token instead). * 2. RBAC: user session tokens require the `system:console` permission. * console_session tokens are pre-gated at issuance (see * `routes/console.ts`) and skip this check. - * 3. License: host console requires the paid tier. For console_session - * tokens the tier is trusted from the gateway-supplied header; - * otherwise the local LicenseService is consulted. + * + * Path scoping and one-time jti consumption are enforced in upgradeHandler + * before this handler runs. */ export function handleHostConsoleWs( req: IncomingMessage, @@ -46,11 +41,10 @@ export function handleHostConsoleWs( if (isProxyToken) return reject(socket, 403, 'Forbidden'); - const isConsoleSession = decoded.scope === 'console_session'; + const isConsoleSession = isConsoleSessionScope(decoded.scope); if (!isConsoleSession) { const userRole = wsResolvedUser?.role; - const consolePermission: PermissionAction = 'system:console'; - if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes(consolePermission)) { + if (!userRole || !ROLE_PERMISSIONS[userRole]?.includes('system:console')) { console.log('[HostConsole] Access denied: insufficient permissions', { username: wsResolvedUser?.username || decoded.username, role: userRole, @@ -59,18 +53,18 @@ export function handleHostConsoleWs( } } - const consoleTierHeader = req.headers[PROXY_TIER_HEADER] as string | undefined; - const ls = LicenseService.getInstance(); - const consoleTier = (isConsoleSession && isLicenseTier(consoleTierHeader)) - ? normalizeTier(consoleTierHeader) - : ls.getTier(); - if (consoleTier !== 'paid') { - return reject(socket, 403, 'Forbidden'); - } + // Option B: console_session principal stays "console_session"; hub operator + // is recorded separately as acting_as. Browser sessions use the real user. + const consoleUsername = isConsoleSession + ? 'console_session' + : (wsResolvedUser?.username || decoded.username || 'unknown'); + const actingAs = isConsoleSession && typeof decoded.acting_as === 'string' && decoded.acting_as + ? decoded.acting_as + : null; - const consoleUsername = wsResolvedUser?.username || decoded.username || 'console_session'; console.log('[HostConsole] WebSocket upgrade accepted', { username: consoleUsername, + actingAs, nodeId, stack: stackParam || '(root)', }); @@ -86,10 +80,6 @@ export function handleHostConsoleWs( hostConsoleWss.handleUpgrade(req, socket, head, (ws) => { hostConsoleWss.close(); let targetDirectory: string; - // The shell may end up rooted at a different node than requested if the - // requested node's directory cannot be resolved; the audit row must name - // the node the shell actually runs in, so track it alongside the directory. - let auditNodeId: number = nodeId; try { const baseDir = FileSystemService.getInstance(nodeId).getBaseDir(); const resolved = HostTerminalService.resolveConsoleDirectory(baseDir, stackParam); @@ -100,22 +90,28 @@ export function handleHostConsoleWs( } targetDirectory = resolved; } catch (error) { - const fallbackNodeId = NodeRegistry.getInstance().getDefaultNodeId(); - console.error('[HostConsole] Failed to resolve console directory; falling back to the default node base dir', { + console.error('[HostConsole] Failed to resolve console directory', { user: consoleUsername, + actingAs, nodeId, - fallbackNodeId, stack: stackParam || '(root)', error: getErrorMessage(error, 'unknown'), }); - targetDirectory = FileSystemService.getInstance(fallbackNodeId).getBaseDir(); - auditNodeId = fallbackNodeId; + if (ws.readyState === WebSocket.OPEN) { + ws.send('Error: Failed to resolve console directory.\r\n'); + ws.close(); + } + return; } - const auditCtx = { username: consoleUsername, nodeId: auditNodeId, ipAddress }; + const auditCtx = { username: consoleUsername, actingAs, nodeId, ipAddress }; try { HostTerminalService.spawnTerminal(ws, targetDirectory, auditCtx); } catch (error) { - console.error('[HostConsole] Unhandled spawn error:', { user: consoleUsername, error: getErrorMessage(error, 'unknown') }); + console.error('[HostConsole] Unhandled spawn error:', { + user: consoleUsername, + actingAs, + error: getErrorMessage(error, 'unknown'), + }); if (ws.readyState === WebSocket.OPEN) { ws.send('Error: Failed to start terminal session.\r\n'); ws.close(); diff --git a/backend/src/websocket/remoteForwarder.ts b/backend/src/websocket/remoteForwarder.ts index 60fcbd93..37059c56 100644 --- a/backend/src/websocket/remoteForwarder.ts +++ b/backend/src/websocket/remoteForwarder.ts @@ -5,6 +5,7 @@ import { LicenseService } from '../services/LicenseService'; import { wsProxyServer } from '../proxy/websocketProxy'; import { getErrorMessage } from '../utils/errors'; import { rejectUpgrade as reject } from './reject'; +import { consoleSessionPathForPathname } from '../helpers/consoleSession'; /** * Forward a WebSocket upgrade to a remote Sencho instance. Handles the @@ -23,9 +24,14 @@ export async function handleRemoteForwarder( req: IncomingMessage, socket: Duplex, head: Buffer, - opts: { pathname: string; target: { apiUrl: string; apiToken: string } }, + opts: { + pathname: string; + target: { apiUrl: string; apiToken: string }; + /** Hub browser operator; recorded as acting_as on the remote audit trail. */ + actingAs?: string; + }, ): Promise { - const { pathname, target } = opts; + const { pathname, target, actingAs } = opts; if (!target.apiUrl) return reject(socket, 503, 'Service Unavailable'); const wsTarget = target.apiUrl.replace(/\/$/, '').replace(/^https?/, (m) => m === 'https' ? 'wss' : 'ws'); @@ -38,24 +44,33 @@ export async function handleRemoteForwarder( // direct api_token access. Pilot loopback targets skip this: there is no // long-lived api_token to exchange, and host-console is disabled on pilot // mode at the capability registry anyway. - const isInteractiveConsolePath = pathname === '/api/system/host-console' || pathname === '/ws'; + const sessionPath = consoleSessionPathForPathname(pathname); let bearerTokenForProxy = target.apiToken; - if (isInteractiveConsolePath && !isPilotLoopback) { + if (sessionPath && !isPilotLoopback) { try { const consoleHeaders = LicenseService.getInstance().getProxyHeaders(); const tokenRes = await fetch(`${target.apiUrl.replace(/\/$/, '')}/api/system/console-token`, { method: 'POST', headers: { 'Authorization': `Bearer ${target.apiToken}`, + 'Content-Type': 'application/json', [PROXY_TIER_HEADER]: consoleHeaders.tier, }, + body: JSON.stringify({ + path: sessionPath, + ...(actingAs ? { acting_as: actingAs } : {}), + }), }); if (!tokenRes.ok) { console.error(`[WS Proxy] Remote console-token request failed: ${tokenRes.status}`); return reject(socket, 502, 'Bad Gateway'); } const data = await tokenRes.json() as { token?: string }; - if (typeof data.token === 'string') bearerTokenForProxy = data.token; + if (typeof data.token !== 'string' || !data.token) { + console.error('[WS Proxy] Remote console-token response missing token'); + return reject(socket, 502, 'Bad Gateway'); + } + bearerTokenForProxy = data.token; } catch (e) { console.error('[WS Proxy] Failed to fetch remote console token:', getErrorMessage(e, 'unknown')); return reject(socket, 502, 'Bad Gateway'); diff --git a/backend/src/websocket/upgradeHandler.ts b/backend/src/websocket/upgradeHandler.ts index e23a7b55..201ba34a 100644 --- a/backend/src/websocket/upgradeHandler.ts +++ b/backend/src/websocket/upgradeHandler.ts @@ -20,6 +20,14 @@ import { validateApiToken, touchApiTokenLastUsed } from '../utils/apiTokenAuth'; import { isDebugEnabled } from '../utils/debug'; import { PROXY_TIER_HEADER } from '../services/license-headers'; import { isLicenseTier, normalizeTier } from '../services/license-normalize'; +import { HOST_CONSOLE_COMMUNITY_CAPABILITY } from '../services/CapabilityRegistry'; +import { remoteAdvertisesCapability } from '../helpers/remoteCapabilities'; + +import { + consoleSessionPathForPathname, + consumeConsoleSessionJti, + isConsoleSessionScope, +} from '../helpers/consoleSession'; function parseCookies(req: IncomingMessage): Record { const header = req.headers.cookie || ''; @@ -31,6 +39,17 @@ function parseCookies(req: IncomingMessage): Record { ); } +/** + * Parse ?nodeId= as a strict positive integer. Returns null when the param is + * absent (caller should use the default node). Returns undefined when the + * param is present but malformed (caller should 404). + */ +function parseStrictNodeIdParam(raw: string | null): number | null | undefined { + if (raw == null || raw === '') return null; + if (!/^[1-9][0-9]*$/.test(raw)) return undefined; + return Number(raw); +} + // The two WebSocket paths open to any authenticated user (read-only/deploy-only // API tokens and non-admin sessions on a remote node). Defined once so the // scope gate and the remote-forward gate cannot drift apart. @@ -68,8 +87,15 @@ export function remoteWsForwardAllowed( if (ctx.isProxyToken) return false; // console_session is pre-gated as admin at issuance (routes/console.ts). if (ctx.decoded.scope === 'console_session') return true; + // Reject opaque API tokens on remote Host Console forward (would mint + // console_session and get a host shell). Local denial is enforced in + // attachUpgrade before handleHostConsoleWs. Container exec (/ws) keeps + // its existing full-admin API-token allow. + if (pathname.startsWith('/api/system/host-console') && ctx.wsApiTokenScope) { + return false; + } // A read-only/deploy-only api_token is already rejected for these paths by - // the scope gate; only full-admin survives to here. + // the scope gate; only full-admin survives to here (container exec /ws). if (ctx.wsApiTokenScope) return ctx.wsApiTokenScope === 'full-admin'; const role = ctx.wsResolvedUser?.role; if (!role) return false; @@ -136,7 +162,16 @@ export function attachUpgrade( try { // Opaque sen_sk_ API tokens: handled before jwt.verify. Prefix + // length + checksum reject malformed keys without touching SQLite. - let decoded: { username?: string; scope?: string; role?: string; tv?: number }; + let decoded: { + username?: string; + scope?: string; + role?: string; + tv?: number; + path?: string; + acting_as?: string; + jti?: string; + exp?: number; + }; let wsApiTokenScope: string | null = null; if (looksLikeApiToken(token)) { const validation = validateApiToken(token); @@ -151,7 +186,7 @@ export function attachUpgrade( const settings = DatabaseService.getInstance().getGlobalSettings(); const jwtSecret = settings.auth_jwt_secret; if (!jwtSecret) throw new Error('No JWT secret'); - decoded = jwt.verify(token, jwtSecret) as { username?: string; scope?: string; role?: string; tv?: number }; + decoded = jwt.verify(token, jwtSecret) as typeof decoded; } // Node proxy tokens are machine-to-machine credentials and must never be @@ -180,6 +215,21 @@ export function attachUpgrade( const parsedUrl = new URL(req.url || '', `http://${req.headers.host || 'localhost'}`); const pathname = parsedUrl.pathname; + // Path-scoped, one-time console_session tokens for interactive surfaces. + // Consume runs at upgrade acceptance (before PTY spawn); missing exp/jti fail closed. + if (isConsoleSessionScope(decoded.scope)) { + const requiredPath = consoleSessionPathForPathname(pathname); + if (!requiredPath || decoded.path !== requiredPath) { + return reject(socket, 403, 'Forbidden'); + } + if (typeof decoded.exp !== 'number' || typeof decoded.jti !== 'string' || !decoded.jti) { + return reject(socket, 401, 'Unauthorized'); + } + if (!consumeConsoleSessionJti(decoded.jti, decoded.exp * 1000)) { + return reject(socket, 401, 'Unauthorized'); + } + } + // Gate WebSocket paths by API token scope if (wsApiTokenScope) { if (wsApiTokenScope === 'read-only' || wsApiTokenScope === 'deploy-only') { @@ -219,8 +269,11 @@ export function attachUpgrade( return; } - const nodeIdParam = parsedUrl.searchParams.get('nodeId'); - const nodeId = nodeIdParam ? parseInt(nodeIdParam, 10) : NodeRegistry.getInstance().getDefaultNodeId(); + const parsedNodeId = parseStrictNodeIdParam(parsedUrl.searchParams.get('nodeId')); + if (parsedNodeId === undefined) { + return reject(socket, 404, 'Not Found'); + } + const nodeId = parsedNodeId ?? NodeRegistry.getInstance().getDefaultNodeId(); const node = NodeRegistry.getInstance().getNode(nodeId); // Notification push channel: local only when no remote nodeId is @@ -243,6 +296,23 @@ export function attachUpgrade( if (!remoteWsForwardAllowed(pathname, { wsResolvedUser, wsApiTokenScope, isProxyToken, decoded })) { return reject(socket, 403, 'Forbidden'); } + // Community hubs require host-console-community before forwarding Host + // Console (marks remotes that no longer paid-gate the console path). + // Admiral hubs skip the probe for legacy host-console peers. Fail + // closed on probe errors; never fall through to local handlers for a + // named remote. + if ( + pathname.startsWith('/api/system/host-console') + && LicenseService.getInstance().getTier() !== 'paid' + ) { + const communityCapable = await remoteAdvertisesCapability( + nodeId, + HOST_CONSOLE_COMMUNITY_CAPABILITY, + ); + if (!communityCapable) { + return reject(socket, 403, 'Forbidden'); + } + } // Resolve the proxy target through NodeRegistry so pilot-mode nodes // (empty api_url + api_token, loopback bridge instead) and proxy-mode // nodes share one dispatch path. Mirrors proxy/remoteNodeProxy.ts. @@ -253,7 +323,11 @@ export function attachUpgrade( // serve gateway-local data for a request that named a remote node. return reject(socket, 503, 'Service Unavailable'); } - await handleRemoteForwarder(req, socket, head, { pathname, target }); + await handleRemoteForwarder(req, socket, head, { + pathname, + target, + actingAs: wsResolvedUser?.username || decoded.username, + }); return; } @@ -264,6 +338,16 @@ export function attachUpgrade( } if (pathname.startsWith('/api/system/host-console')) { + // Opaque API tokens never open a local host shell (parity with the + // remote forward gate). Do not rely on missing wsResolvedUser alone. + if (wsApiTokenScope) { + return reject(socket, 403, 'Forbidden'); + } + // Unknown / deleted nodeIds must not fall through to the hub compose + // root via getComposeDir's env default. + if (!node) { + return reject(socket, 404, 'Not Found'); + } handleHostConsoleWs(req, socket, head, { nodeId, decoded, diff --git a/docs/features/alerts-notifications.mdx b/docs/features/alerts-notifications.mdx index efa0acc8..97043bb4 100644 --- a/docs/features/alerts-notifications.mdx +++ b/docs/features/alerts-notifications.mdx @@ -1,10 +1,10 @@ --- title: Alerts & Notifications sidebarTitle: Alerts and notifications -description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with per-stack rules and channel routing. +description: Threshold and event alerts for your fleet, dispatched to Discord, Slack, Apprise, or any webhook, with stack and service rules and channel routing. --- -Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing per-stack threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. +Sencho watches each node it manages for container crashes, host pressure, scheduled-task results, and update availability, then surfaces every signal in two places: the in-app notification bell at the top of the shell and an external channel you configure. This page covers everything from configuring channels to writing stack and service threshold rules, routing alerts to dedicated channels with routing rules, and tuning retention. Settings · Notifications · Channels panel with NODE Local in the header, a Delivery retries row showing Extra attempts 0 and a Save retries button, Discord Slack Webhook and Apprise tabs with Apprise selected, Enabled off, an empty Apprise endpoint placeholder, and Test beside Save. @@ -37,7 +37,7 @@ Use the Generic Webhook tab when you have your own receiver: a Mattermost or Tea ```json { "level": "warning", - "message": "The **CPU usage** for **plex** has exceeded your threshold of **80%** (Currently: 91%).", + "message": "The **CPU usage** for **api** in **plex** (container **plex-api-1**) has exceeded your threshold of **80%** (Currently: 91%).", "timestamp": "2026-05-08T22:14:09.812Z", "source": "sencho" } @@ -161,11 +161,17 @@ Every alert Sencho dispatches carries a category that you can filter on in the b The four `blueprint_*` categories are accepted by routing rules but render as raw category strings in the bell because the frontend label map omits them. -## Per-stack alert rules +## Stack alert rules -Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. +Each stack carries its own set of threshold rules that fire when a metric stays above (or below) a value for a configurable window. A rule can target **All services** in the stack or one Compose service. Metrics are evaluated per running container (not as a stack aggregate): each container has its own duration timer, and cooldown is tracked per Compose service so replicas of the same service share one silence window while different services can alert independently. -Open the rules editor by right-clicking a stack in the sidebar and choosing **Alerts**, or by pressing `A` while focused on a stack. +Rules live on the node where the stack runs and are evaluated locally on a 30-second tick. + +Open the rules editor from any of these entry points: + +- Stack header **More actions** → **Monitor** (Alerts tab) +- Container card **Monitor** (prefers that Compose service in add forms) +- Sidebar context menu **Alerts**, or press `A` while focused on a stack The Stack › PLEX › MONITOR sheet with the Alerts tab active, a green notification-channel banner under the NOTIFICATION CHANNELS heading, an ACTIVE RULES section listing 'CPU Usage (%) > 80' with the secondary line 'Trigger after 5m • Cooldown 60m', and an ADD NEW RULE form with Metric, Operator, Threshold, Duration, Cooldown fields and an Add Rule submit button. @@ -175,13 +181,14 @@ Open the rules editor by right-clicking a stack in the sidebar and choosing **Al | Field | Purpose | |-------|---------| +| **Service** | **All services** (default) or one Compose service from the stack. All services evaluates every matching container independently. | | **Metric** | The system resource or metric to monitor. | | **Operator** | Comparison: `Greater than`, `Greater or eq`, `Less than`, `Less or eq`, `Equals`. | | **Threshold** | A number ≥ 0. The unit follows the chosen metric. | -| **Duration (mins)** | How long the breach must persist before firing. Default `5`, range 0 to 1440. | -| **Cooldown (mins)** | Silence window between fires after a rule triggers. Default `60`, range 0 to 10080. | +| **Duration (mins)** | How long a container's breach must persist before firing. Default `5`, range 0 to 1440. | +| **Cooldown (mins)** | Silence window between fires for the same Compose service after a rule triggers. Default `60`, range 0 to 10080. | -Sencho tracks the start of each breach in memory; the rule fires only after the breach has lasted for the full **Duration**, and only if the previous fire is older than **Cooldown**. +Sencho tracks the start of each breach per container in memory; the rule fires only after that container's breach has lasted for the full **Duration**, and only if the previous fire for the same Compose service is older than **Cooldown**. The notification names the triggering service and container. Containers without a Compose service label are grouped under one shared cooldown key and appear as **unknown service** in the message; that label is not selectable as a rule target. A scoped rule whose service is no longer listed in the compose file shows **Not in compose** in the rules list; evaluation still matches running containers that carry that Compose service label. ### Available metrics @@ -305,10 +312,11 @@ Real-time on the Docker event stream: - **Crash** at `error`/`monitor_alert`: `Container Crash Detected: exited unexpectedly (Code: ).` - **OOM kill** at `error`/`monitor_alert`: `Container OOM Kill: was killed by the OOM killer (out of memory).` When a `die` arrives with exit code 137 but no preceding `oom` event, Sencho inspects the container and reclassifies as OOM if the `OOMKilled` flag is set. -- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: is unhealthy.` +- **Healthcheck failure** at `error`/`monitor_alert`: `Healthcheck failed: is unhealthy.` Emitted only on the transition into Docker `unhealthy` (not on every repeated unhealthy event). - **Mass exit** kicks in when a daemon-disconnect gap is followed by 20% or more of containers exiting on reconnect, in which case a single `info`/`system` summary `Docker daemon interruption detected: N containers exited during connection gap.` is emitted instead of N crash alerts. -- **Rate limit** caps crash dispatches at 20 per 60-second window per node, then a single `warning`/`monitor_alert` `N additional containers crashed in the last minute.` -- **Dedup** is 60 minutes per container after a non-rate-suppressed dispatch. +- **Rate limit** caps combined crash and health dispatches at 20 per fixed 60-second window per node. Overflow does not write individual history rows; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, `N additional container health alerts were rate-limited in the last minute.`, or `N additional container alerts were rate-limited in the last minute (X crash, Y health).` +- **Crash dedup** is 60 minutes per container after a non-rate-suppressed crash dispatch. Cleared when that container starts again. +- **Health dedup** is 60 minutes per container after a health alert that was persisted to notification history. It survives healthy/starting recovery and the 10-minute prune of in-memory container tracking state (dedup lives in a separate map), and clears only when the window expires or the Sencho process restarts. ### Daemon connectivity @@ -403,7 +411,7 @@ Three retention controls live under **Settings · Operations · Data Retention** A hard 100-row per-node cap inside `notification_history` always applies on top of the user-tunable retention; new inserts evict the oldest rows beyond 100 even if your retention window is longer. -A separate rate limit applies to crash and health alerts only: 20 emits per 60-second window per node, with a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` issued at the end of the window. +A separate rate limit applies to crash and health alerts only: 20 emits per fixed 60-second window per node (not a rolling window). When the cap is hit, individual overflow alerts are not written to history or the bell; at the end of the window Sencho emits a single `warning`/`monitor_alert` roll-up that names crash-only, health-only, or mixed suppression. ## Crash detection toggle @@ -429,8 +437,9 @@ The **Host Alerts** panel carries the **Host thresholds** rows (CPU limit, RAM l | Notification fanout to channels | One attempt by default; optional 0-3 extra in-process attempts with a fixed 1s delay; 10-second timeout per attempt; no durable queue | | Bell live updates | Pushed live over the notifications WebSocket per node | | Bell safety-net reconcile | 60 seconds | -| Crash dedup window per container | 60 minutes | -| Crash rate-limit window | 20 alerts per 60 seconds per node, then a single roll-up | +| Crash dedup window per container | 60 minutes (cleared on container start) | +| Health dedup window per container | 60 minutes after a persisted health alert; survives recovery; process-local | +| Crash/health rate-limit window | 20 alerts per fixed 60 seconds per node, then a single typed roll-up (overflow individuals are not persisted) | Switching the active node tears down per-stack rule editors and reloads channel state, but does not affect the bell, which keeps every node's WebSocket open in parallel. @@ -441,7 +450,7 @@ Switching the active node tears down per-stack rule editors and reloads channel Check three things in order. First, the channel toggle in **Settings · Notifications · Channels** must be on; the kicker on each tab reads `enabled` or `off`. Second, Discord, Slack, and webhook URLs must use HTTPS (the form rejects plain `http://` for those channels); Apprise endpoints may use HTTP or HTTPS. Third, a routing rule with unconstrained Node plus empty Stacks, Labels, Categories, and Severity matchers will intercept every alert and skip the global channels. Use the per-channel **Test** button to issue a one-shot dispatch and watch your endpoint for the literal message `🔌 Test Notification from Sencho!` Sencho records the failure reason in `notification_history.dispatch_error` when delivery throws, so a row that appears in the bell with no follow-up at the endpoint usually means a 4xx or timeout at the receiver. - Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: the breach must persist for the full duration before the rule fires. Second, the rule is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. + Three causes account for almost every case. First, the rule's **Duration** has not elapsed yet: that container's breach must persist for the full duration before the rule fires (a healthy sibling service does not clear another container's timer). Second, the same Compose service is still in cooldown after a previous fire. Third, the panel's banner is not green: a remote-node banner means the rule was saved on a remote whose channels you may not have configured, and an amber `No notification channels configured` banner means the rule evaluates fine but Sencho has nowhere to send the alert. The evaluator runs on a 30-second tick, so expect up to 30 seconds of latency between the breach starting and the timer engaging. The Docker event service defers `die` classification by 500 ms to absorb out-of-order `kill` events from the daemon, then asks the lifecycle classifier whether the exit was intentional, clean (exit 0), a crash, or an OOM kill. If your stop happened far enough outside that window, or the daemon emitted the events without the kill marker the classifier looks for, the exit can be classified as a crash. The classifier favors avoiding silent crashes over avoiding noisy false positives. Compare the alert timestamp against your `docker compose down` time; entries within a second of each other are usually the same event seen from two angles. @@ -452,8 +461,8 @@ Switching the active node tears down per-stack rule editors and reloads channel Check **Settings · Monitoring · Container Alerts · Container crash & health alerts**. The toggle is the master switch for the Docker event service. The panel cache reads the database every 500 ms; if the database read errors, Sencho defaults the toggle to off so the failure mode is silent rather than spammy. Repair the toggle, save, and the next Docker event reaches the dispatcher. - - Sencho rate-limits crash and health dispatches to 20 emits per rolling 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up `N additional containers crashed in the last minute.` once the window closes. Every alert is still persisted to `notification_history` and visible in the bell up to the 100-row per-node cap; only the channel fanout is throttled. + + Sencho rate-limits crash and health dispatches together to 20 emits per fixed 60-second window per node, then emits a single `warning`/`monitor_alert` roll-up such as `N additional container crash alerts were rate-limited in the last minute.`, a health-only variant, or a mixed `(X crash, Y health)` summary once the window closes. Rate-limited overflow individuals are not written to `notification_history` and do not appear in the bell; only the roll-up is persisted for that overflow. Separately, health flaps on the same container are deduped for 60 minutes after a persisted unhealthy alert, even if Docker recovered and failed again inside that window. Sencho deliberately hides rows whose category is one of `deploy_success`, `stack_started`, `stack_stopped`, `stack_restarted`, or `image_update_applied` AND whose `actor_username` is set to a real user. The reasoning: those are confirmations of the action you just clicked and are already shown as a toast. The rows are still persisted to `notification_history` and still dispatched to global channels and matching routes; only the bell render hides them. diff --git a/docs/features/api-tokens.mdx b/docs/features/api-tokens.mdx index d81a5056..60b412aa 100644 --- a/docs/features/api-tokens.mdx +++ b/docs/features/api-tokens.mdx @@ -171,7 +171,7 @@ Authorises every read and every write the API exposes, **except** the universal | `/api/stacks/:stack/logs` (stack log stream) | yes | yes | yes | | `/ws/notifications` (notification stream) | yes | yes | yes | | `/ws` (generic exec / stats) | no | no | yes | -| `/api/system/host-console` (host shell) | no | no | yes | +| `/api/system/host-console` (host shell) | no | no | no | A Read Only or Deploy Only token attempting an out-of-scope WebSocket upgrade is rejected with `403 Forbidden` before any frames flow. @@ -268,7 +268,7 @@ API tokens are issued and revoked on the hub that authenticates the call. When y - **One owner per token.** Tokens are not shareable across users at the data model; only the creating user can revoke a token through the UI. - **Owner deletion breaks the token.** If the user who created a token is deleted, subsequent calls with that token return `401` because the auth path requires the creator to still exist. Rotate before deleting accounts. - **No token introspection endpoint.** There is no `GET /api/me` for an API token; client code must know its own scope. -- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can. +- **WebSocket scope restrictions.** Read Only and Deploy Only tokens cannot reach the generic `/ws` exec/stats endpoint or the host console. Full Admin can reach `/ws` (container exec / stats) but not Host Console; Host Console always requires a signed-in browser session. ## Common workflows diff --git a/docs/features/appearance.mdx b/docs/features/appearance.mdx index c2f5e32b..f66047e6 100644 --- a/docs/features/appearance.mdx +++ b/docs/features/appearance.mdx @@ -89,7 +89,7 @@ Sencho uses three type contexts. The interface and data faces are yours to chang The **Display** group holds layout and log-chip preferences for this browser: - **Density** switches between **Comfortable** (roomy rows, the default) and **Compact** (tighter rows and tiles that fit more on screen for dense dashboards). -- **Log chip color** controls how service chips are colored in log views. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once. +- **Log chip color** controls how service chips are colored in log views on multi-service or multi-container stacks. Single-service, single-container stacks do not show chips. **Unified** uses the accent color for all service chips. **Per service** assigns each service a stable label color for faster visual scanning when following multiple services at once. ## Navigation diff --git a/docs/features/auto-heal-policies.mdx b/docs/features/auto-heal-policies.mdx index 1b978208..0c8c77bd 100644 --- a/docs/features/auto-heal-policies.mdx +++ b/docs/features/auto-heal-policies.mdx @@ -28,10 +28,12 @@ Policies live next to your stack-level alert rules in the stack's **Monitor** sh ## Workflow -1. In the sidebar, right-click the stack you want to protect (or focus the stack and press **H**). -2. Click **Auto-Heal** in the **inspect** group. The stack's **Monitor** sheet opens directly on the **Auto-heal** tab. -3. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds. -4. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on. +1. Open the stack's **Monitor** sheet any of these ways: + - Stack header **More actions** → **Monitor**, then switch to the **Auto-heal** tab + - Container card **Monitor** (prefers that Compose service in **Add new policy**), then switch to **Auto-heal** + - Sidebar: right-click the stack (or focus it and press **H**) → **Auto-Heal** in the **inspect** group (opens directly on the **Auto-heal** tab) +2. In **Add new policy**, pick a **Service** (or leave the combobox on **All services** to cover every container in the stack) and tune the four thresholds. +3. Click **Add Policy**. The new policy appears in **Active policies** above the form, with its enable toggle already on. The Stack › plex › Monitor sheet on the Auto-heal tab, showing one policy scoped to the plex service in the Active policies section with its ON toggle, and the Add new policy form below with Service, Unhealthy for, Cooldown, Max restarts / hr, and Auto-disable after fields, plus the Add Policy button diff --git a/docs/features/blueprint-model.mdx b/docs/features/blueprint-model.mdx index 3f2957a9..b6af1467 100644 --- a/docs/features/blueprint-model.mdx +++ b/docs/features/blueprint-model.mdx @@ -21,7 +21,7 @@ Three moving parts cooperate per blueprint. 1. **The declared spec.** A blueprint is a row in Sencho's database. It carries the compose YAML, the selector (labels or explicit node IDs), the drift policy, and a monotonic revision number. The revision auto-increments every time the compose changes. 2. **The reconciler.** A background loop on the controlling Sencho ticks every 60 seconds (with a 5-second initial delay after startup) and on demand after **Confirm Apply**. Each tick only mutates the fleet when the blueprint's current operational intent is approved. The tick resolves the selector, compares every desired-vs-live node, and runs only the place or remove outcomes the operator already confirmed. -3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the standard authenticated proxy to `/api/stacks` on remote nodes. Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp. +3. **The executor.** Per-node deploy and withdraw run against the local Docker socket on local nodes, and through the authenticated proxy to `/api/blueprints/apply-local` and `/api/blueprints/withdraw-local` on remote nodes (older remotes fail closed until upgraded). Every blueprint deployment writes a `.blueprint.json` marker into the stack directory; the marker carries the blueprint ID, the revision, and the last-applied timestamp. The marker is the trust root. If a directory by the blueprint's name already exists on a node and does not carry a matching marker, the reconciler refuses to touch it and surfaces a **Name conflict** on the deployment row. A Blueprint named `nginx` will never overwrite an existing user-authored `nginx` stack on any node. @@ -155,7 +155,7 @@ Possible status values: | **Withdrawing** | A withdraw is in flight. | | **Withdrawn** | The deployment was successfully removed; the row is about to be cleared. | | **Evict blocked** | A node left the selector while a stateful deployment was active. The reconciler refuses to evict until you choose an explicit mode. | -| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry our marker file. The reconciler will not touch it. | +| **Name conflict** | A directory by the blueprint's name already exists on the node and does not carry a matching `.blueprint.json` (missing, malformed, or another blueprint ID). The reconciler will not touch it. | Blueprint detail sheet with a deployment row in Awaiting confirmation state @@ -309,7 +309,7 @@ A future Volume Migration feature will automate this with app-aware backup tooli - A directory by the blueprint's name already exists on that node and does not carry the `.blueprint.json` marker. The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry. + A directory by the blueprint's name already exists on that node without a matching `.blueprint.json` marker (missing, malformed, or owned by another blueprint). The most likely cause is a manually created stack with the same name. Resolution: rename either the existing stack or the blueprint, then click **Apply now** on the detail sheet to retry. The reconciler will not auto-deploy a stateful blueprint to a node it has never run on. Click **Confirm deploy** on the row, then choose **Deploy fresh** in the dialog. Sencho will create empty named volumes and start the stack. @@ -323,6 +323,9 @@ A future Volume Migration feature will automate this with app-aware backup tooli The row moves to **Failed** with the remote error. Reconnect the node (see [Pilot Agent](/features/pilot-agent) and [Multi-node management](/features/multi-node)), verify whether the stack directory contains `compose.yaml` and `.blueprint.json`, then click **Apply now**. If the directory exists without a matching marker, Sencho treats it as a name conflict until you rename or remove the remote stack manually. + + Blueprint apply and withdraw on a remote node require that node's Sencho build to expose the atomic node-local endpoints (`/api/blueprints/apply-local` and `/api/blueprints/withdraw-local`). Older remotes refuse the mutation with a failed deployment row rather than writing files without ownership checks. Upgrade the remote instance to a matching Sencho release, then retry **Apply now** or withdraw. + The deployment row moves to **Failed** and records the Docker or registry error. Resolve the daemon, socket, registry credentials, rate limit, disk, or volume-permission issue on the affected node, then click **Apply now**. Watch that node's stack activity and security scan status after retry. diff --git a/docs/features/deep-links.mdx b/docs/features/deep-links.mdx index 875a2a49..5e1ec9e1 100644 --- a/docs/features/deep-links.mdx +++ b/docs/features/deep-links.mdx @@ -23,7 +23,8 @@ Sencho encodes the active node, the view you are on, and the deep state that vie | App Store | `/nodes/local/templates` | App Store on the active node | | Logs | `/nodes/local/logs` | Cross-fleet log aggregation | | Update | `/nodes/local/updates` | Fleet-wide update check | -| Console | `/nodes/local/host-console` | Host Console, a limited-availability operator surface documented on its own page when enabled on an instance | +| Console | `/nodes/local/host-console` | Host Console (admin role) | +| Console in a stack | `/nodes/local/host-console/radarr` | Host Console rooted in the Radarr stack directory | | Audit | `/nodes/local/audit` | Audit history | | Settings section | `/nodes/local/settings/nodes` | Settings on the Nodes section | | Fleet tab | `/nodes/local/fleet/snapshots` | Fleet on the Snapshots tab (desktop) | @@ -86,7 +87,7 @@ On desktop, opening the stack list at `/nodes/local/stacks` canonicalizes to `/n Settings follows a similar split, but only on a phone: `/nodes/local/settings` opens the section list, and `/nodes/local/settings/
` opens a section directly. On desktop, `/nodes/local/settings` always normalizes straight to `/nodes/local/settings/appearance`; there is no bare section-list state to land on. -Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone, reflowing the terminal rather than switching to a dedicated phone screen. +Fleet, Resources, Security, App Store, Logs, Update, Audit, and Schedules keep the exact same URL between desktop and phone, each rendering a phone-optimized screen at that path. Console also keeps the same URL on a phone (`/nodes//host-console`), but Host Console is a desktop-oriented terminal: open it on a wider screen for a full interactive session. On a phone, `/nodes/local/stacks//files` opens the compose editor instead. Sencho does not expose a separate file-browser URL on a phone. diff --git a/docs/features/fleet-actions.mdx b/docs/features/fleet-actions.mdx index 4cbe6556..43b70633 100644 --- a/docs/features/fleet-actions.mdx +++ b/docs/features/fleet-actions.mdx @@ -139,7 +139,7 @@ Scope is a segmented control with two options: ### Live estimate and behaviour -- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim. +- Changing a target or the scope re-triggers a debounced call to `POST /api/fleet/prune/estimate`, which walks the same Docker enumeration the destructive path uses so the estimate matches what pruning would actually reclaim. A completed real prune that succeeds on at least one target re-triggers the same estimate so the toolbar total and per-node list reflect post-prune Docker state. **Prune fleet** stays disabled until the estimate resolves: you cannot confirm a destructive fleet-wide prune with no context on what it will reclaim. - Each remote node receives one `POST /api/system/prune/system` call per selected target, with a 120-second timeout. If a transport error fires for one target, the remaining targets on that node are short-circuited with the same error rather than retried, so a dead node doesn't absorb the full multi-target timeout budget. - Local nodes serialize against a per-node lock (`bulk-prune:`). A second fleet prune launched against the same local node while the first is still in flight returns *A prune is already running on this node* for each target. - Reclaimed bytes are reported by the Docker daemon and are approximate. Per-node rows in the results panel sum the per-target reclaim; the per-target children show how much each individual prune actually freed. diff --git a/docs/features/global-search.mdx b/docs/features/global-search.mdx index 8d0d4f25..4f14d137 100644 --- a/docs/features/global-search.mdx +++ b/docs/features/global-search.mdx @@ -26,7 +26,7 @@ The palette groups results into three sections. | Group | What it contains | What happens when you pick one | |-------|------------------|--------------------------------| -| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, and **Schedules** appear for admins; **Console** is a limited-availability operator surface shown when enabled on an instance; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page | +| **Pages** | The reachable page destinations for your tier and role (the same set Classic / Smart / mobile navigation use). **Home**, **Resources**, **Networking**, **Security**, and **App Store** appear for signed-in operators; **Fleet** appears when your role holds the `node:read` permission; **Logs**, **Update**, **Schedules**, and **Console** appear for admins; **Audit** appears on Admiral for any role with the `system:audit` permission. See [RBAC & User Management](/features/rbac) for the full permission matrix. | Navigates to that page | | **Nodes** | Every node in your fleet, with a green dot for online and a grey dot for offline. The currently active node carries a small **ACTIVE** chip on the right. | Switches the active node without leaving the current page | | **Stacks** | Every compose stack on every online node, matched on the compose filename (extension included). | Switches to the stack's node and opens it in the editor | diff --git a/docs/features/health-gated-updates.mdx b/docs/features/health-gated-updates.mdx index 8abafdae..be9c27fa 100644 --- a/docs/features/health-gated-updates.mdx +++ b/docs/features/health-gated-updates.mdx @@ -49,11 +49,13 @@ The 3-second crash probe from [atomic deployments](/features/atomic-deployments) During the window, the gate checks every container that came up with the stack: -- all expected containers stay **running**, -- Docker healthchecks report **healthy** wherever a service defines one, -- no container **exits**, **disappears**, or enters a **restart loop**. +- all expected containers stay **running** (or finish cleanly as a one-shot; see below), +- Docker healthchecks report **healthy** on long-running containers wherever a service defines one (residual health on a completed one-shot is ignored), +- no container **crashes** (non-zero exit), **disappears**, or enters a **restart loop**. -A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period when the window ends records the verdict as unknown rather than claiming success. +A one-shot or init service that finishes with exit code `0` and explicitly declares `restart: "no"` (or `deploy.restart_policy.condition: none`) counts as a successful completion, not a gate failure. Omitting `restart:` does not qualify: Compose defaults to no-restart, and Docker reports the same inspect value for both intentional jobs and bare long-running services. Residual Docker health state on a completed one-shot (`starting` or `unhealthy` from an inherited healthcheck) does not fail or leave the gate unknown. A clean exit under `unless-stopped`, `always`, or `on-failure`, or with no declared restart, still fails the gate. An exit whose code cannot be read also fails the gate. + +A clear failure ends the observation immediately. Passing requires the full window, and a healthcheck still in its start period on a long-running container when the window ends records the verdict as unknown rather than claiming success. After a **service-scoped** update or restore, the gate still watches the whole project, but it attributes failures: a problem on the updated service is a primary failure; a previously healthy sibling that regresses during the window is a collateral failure. Siblings that were already unhealthy before the update do not fail the gate on their own. @@ -78,7 +80,7 @@ The gate also runs for updates you did not click: scheduled image updates, webho Open **Settings > Infrastructure > Stacks > Deploy Guardrails** on the node you want to configure: - **Observe health after updates** turns the gate on or off for that node. On by default; turning it off only stops the observation and its timeline events, never the update itself. -- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a healthcheck still starting at the end of the window records an unknown verdict instead of a pass. +- **Observation window** sets how long containers are watched, from 15 to 600 seconds. The default is 90 seconds; raise it for stacks that take a while to settle, since a long-running healthcheck still starting at the end of the window records an unknown verdict instead of a pass. Stacks settings page showing the Deploy Guardrails subsection, with the Observe health after updates toggle set to On and the Observation window field set to 90 seconds @@ -110,10 +112,10 @@ When a deploy or update fails, Sencho classifies the failure from the compose ou Unknown means one of the verdict-affecting signals could not be verified, most often because Docker or the node did not answer in time. The update path is never blocked by an unknown verdict; check the node's connection if it persists, and proceed when you are confident in the stack's state. - The gate fails on the first clear problem it observes: a container exit, an unhealthy healthcheck, a restart loop, or a container disappearing mid-window. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. + The gate fails on the first clear problem it observes: a non-zero exit, an unexpected clean exit of a service that is not an explicit one-shot, an unhealthy healthcheck on a long-running container, a restart loop, or a container disappearing mid-window. A one-shot that exits `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) does not fail the gate, including when residual Docker health on that completed container is still starting or unhealthy. Check the named container's logs from the stack page. If the container legitimately restarts during startup (a migration step that should stay up, for example), raise the observation window under **Settings > Infrastructure > Stacks > Deploy Guardrails** so the gate watches past the settle period. - Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a healthcheck was still starting when the window ended, or no containers appeared to observe. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. + Unknown means the observation could not finish: Sencho restarted mid-window, a newer update superseded the observation, Docker became unreachable, a long-running healthcheck was still starting when the window ended, or no containers appeared to observe. Residual starting health on a completed one-shot does not produce unknown. The stack timeline records the reason. Check the containers directly; an unknown verdict makes no claim either way. The modal holds its auto-close while the gate observes so the verdict is not lost. You can close it at any time; the observation continues server-side and the verdict lands on the stack timeline. If the gate result repeatedly cannot be retrieved, the modal gives up with an unknown verdict instead of waiting forever. diff --git a/docs/features/host-console.mdx b/docs/features/host-console.mdx index 23ee6ea4..306fbd04 100644 --- a/docs/features/host-console.mdx +++ b/docs/features/host-console.mdx @@ -6,7 +6,7 @@ description: An interactive terminal on your host OS, directly in the browser - The **Console** tab opens a full interactive terminal session on the machine running Sencho. It behaves exactly like an SSH session, but without needing an SSH server, client, or key management. - The Host Console is a limited-availability operator surface. When it is present on an instance, it requires the **admin** role. [Learn more about licensing](/features/licensing). + Host Console requires the **admin** role. [Learn more about roles](/features/rbac). @@ -27,6 +27,14 @@ The Host Console gives you a real terminal session on the Sencho host, streamed Click **Console** in the top navigation bar. The session starts immediately in the `COMPOSE_DIR` root. If a stack is selected in the sidebar, the terminal opens directly inside that stack's directory instead, and a small back button appears in the masthead so you can return to the stack editor. +You can also open Console from a bookmark or shared link. Every Console view has a stable URL under [Deep links and URLs](/features/deep-links): + +- `/nodes/local/host-console` opens Host Console on the local node at the compose root +- `/nodes/local/host-console/radarr` opens the same terminal rooted in the Radarr stack directory +- Remote nodes use their node slug the same way (for example `/nodes/nas-box-42/host-console`) + +Console requires the **admin** role. The URL stays in the address bar on a phone, but Host Console is meant for a desktop browser. + ## Cockpit layout The Console page is a vertical stack of two surfaces. @@ -74,9 +82,9 @@ Environment variables whose names suggest secrets (passwords, tokens, keys, cred The Host Console is one of the most powerful features in Sencho and is treated as such: -- **Admin role required when present.** The Host Console is a limited-availability surface; only users with the **admin** role can open a console session. +- **Admin role required.** Only users with the **admin** role can open a console session. - **Browser sessions only.** Console sessions are only available from a signed-in browser session, not from API tokens. -- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry capturing the user, node, client IP, and timestamp, so shell access is fully accountable. +- **Audited.** Every console session is recorded in the audit log. Opening and closing a session each write an entry with the session principal, node, client IP, and timestamp. When the session is opened through a remote hub bridge, the principal is `console_session` and the hub operator is recorded in `acting_as`, so remote shell access stays accountable to the signed-in admin. The Host Console provides unrestricted shell access to the machine running Sencho. Do not expose Sencho on a public network without HTTPS and strong authentication. @@ -97,7 +105,7 @@ The Host Console is one of the most powerful features in Sencho and is treated a - This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec which bash` from the host. Also verify that your user has the **admin** role and that Console is available on this instance. + This typically means the shell could not be found on the host. In a Docker deployment, ensure `bash` or `sh` is available inside the container. You can check by running `docker exec which bash` from the host. Also verify that your user has the **admin** role. @@ -105,6 +113,6 @@ The Host Console is one of the most powerful features in Sencho and is treated a - Console appears only when the surface is present on the instance, and only for users with the **admin** role. Verify those conditions are met, then refresh the page. + Console appears for users with the **admin** role. If you are an admin and still do not see it, refresh the page. On a remote node, the Console tab may show a lock card when that node does not support Host Console (for example a Pilot Agent node, or a Distributed API Proxy node that does not advertise Host Console). diff --git a/docs/features/multi-node.mdx b/docs/features/multi-node.mdx index 90d138a2..3b4b719e 100644 --- a/docs/features/multi-node.mdx +++ b/docs/features/multi-node.mdx @@ -236,7 +236,7 @@ Bearer tokens grant full control over the remote Sencho instance. Treat them lik - **Rotate immediately** if a token is compromised: open the remote instance's **Settings → Infrastructure → Nodes** and click **Generate Token** to mint a new one. The previous token is invalidated instantly. - Tokens are **encrypted at rest** in the local SQLite database. -- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access always requires a real browser session on that specific instance. +- Tokens cannot be used to open interactive terminals (Host Console or container exec). Interactive shell access requires a signed-in browser session. From the control plane you can open Host Console on a compatible Distributed API Proxy remote; Pilot Agent remotes do not offer Host Console. ### Transport encryption diff --git a/docs/features/node-compatibility.mdx b/docs/features/node-compatibility.mdx index c7d345c4..388375ff 100644 --- a/docs/features/node-compatibility.mdx +++ b/docs/features/node-compatibility.mdx @@ -67,7 +67,8 @@ Every Sencho release ships with a static list of capabilities. The current list | `notifications` | Alert notifications | | `notification-routing` | Notification routing rules | | `notification-suppression` | Mute rules | -| `host-console` | Host Console | +| `host-console` | Host Console (legacy advertisement retained for mixed-version fleets) | +| `host-console-community` | Host Console without a paid-license requirement on this node | | `container-exec` | Container exec terminal | | `audit-log` | Audit log | | `scheduled-ops` | Scheduled operations | @@ -94,7 +95,7 @@ Every Sencho release ships with a static list of capabilities. The current list - A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection. + A node connected in [Pilot Agent](/features/multi-node#add-a-remote-node-pilot-agent) mode never advertises `host-console` or `host-console-community`, even after upgrading, because that feature's control-to-agent path is not yet wired through the enrollment tunnel. This is the one capability gap that upgrading the node cannot close; it only closes on a Distributed API Proxy connection. A handful of capabilities gate a smaller piece of behavior rather than a whole panel, so a missing one falls back to an older behavior instead of a lock card: diff --git a/docs/features/overview.mdx b/docs/features/overview.mdx index a7b43368..681c3f68 100644 --- a/docs/features/overview.mdx +++ b/docs/features/overview.mdx @@ -210,7 +210,7 @@ When you manage nodes running different Sencho versions, the dashboard detects e ### Host console -Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Limited-availability surface when present; admin role required. [Learn more →](/features/host-console) +Open an interactive terminal on the host OS directly in the browser with full xterm.js emulation and color support. No SSH client required. Admin role required. [Learn more →](/features/host-console) ## Security and access diff --git a/docs/features/stack-drift.mdx b/docs/features/stack-drift.mdx index ae7c5897..2297f106 100644 --- a/docs/features/stack-drift.mdx +++ b/docs/features/stack-drift.mdx @@ -28,9 +28,9 @@ The status badge at the top of the tab summarizes the comparison between the eff | Status | Meaning | |--------|---------| -| **In sync** | Running containers match the effective model: same services, images, and published ports. | -| **Drifted** | Something running differs from the model. Specific findings are listed below the badge. | -| **Not running** | The stack has no running containers. The file is present but nothing is up. | +| **In sync** | Declared services match runtime: same running services, images, and published ports, or a declared one-shot finished cleanly (`Exited (0)` with explicit `restart: "no"` or `deploy.restart_policy.condition: none`). Omitting `restart:` does not count as a one-shot. | +| **Not running** | The stack has no running containers, and no declared service is satisfied by a finished one-shot. The file is present but nothing is up. | +| **Drifted** | Something differs from the model. Specific findings are listed below the badge. | | **Unreachable** | Docker could not be reached on the active node, so drift cannot be assessed. | ## Since your last deploy @@ -62,10 +62,10 @@ When a stack is drifted, each reason is listed in the **Findings** section again |---------|---------------| | **Image** | A running container uses a different image than the Compose file declares. Expected and actual values are shown side by side. | | **Ports** | The published ports of a service differ from what the Compose file declares. | -| **Service missing** | The Compose file declares a service, but no running container matches it. | +| **Service missing** | The Compose file declares a service, but no running container matches it and no clean one-shot completion satisfies it. | | **Undeclared service** | A container is running for the stack but no matching service exists in the Compose file. | | **Network undeclared** | A running service is attached to a network that is not declared in the Compose file. | -| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. | +| **Network missing** | A network declared in the Compose file is not used by any running service or is absent from the runtime. Network usage still counts only running or restarting containers, so a finished one-shot does not keep a dedicated Compose network marked as in use. | ### Image comparison @@ -157,7 +157,7 @@ Viewing the tab requires read access to the stack; any role that can see a stack - That is expected. Drift compares the file on disk against what is actually running, so a stack with no running containers reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. + That is expected for long-running services. Drift compares the file on disk against what is actually running, so a stack with no running containers and no finished one-shot jobs reports **Not running** even when you stopped it intentionally. Deploy it to return it to **In sync**. A stack made only of one-shot jobs that already exited `0` with explicit `restart: "no"` (or `deploy.restart_policy.condition: none`) can show **In sync** (or **Drifted** for other findings such as unused networks) with no containers currently running. Click **re-check** after the deploy finishes. During a rolling update, replicas can briefly run different images, and the report captures that moment. Once every container is on the declared image, the finding clears. diff --git a/docs/features/stack-management.mdx b/docs/features/stack-management.mdx index 0774066f..08cdf81d 100644 --- a/docs/features/stack-management.mdx +++ b/docs/features/stack-management.mdx @@ -235,10 +235,10 @@ Single-service stacks keep the existing flat container layout; service headers d The logs area at the bottom of the stack view has two modes. Toggle between them with the segmented control in the top-right of the panel; your choice is remembered for next time. - Logs panel in Structured mode for the plex stack with toolbar, level filter pills, and timestamped INFO log rows + Logs panel in Structured mode with toolbar, level filter pills, and timestamped INFO log rows without service chips on a single-service stack -**Structured** (default) parses every line into a DOM row with three columns: local time, level badge, and message. Level detection is automatic: +**Structured** (default) parses every line into a DOM row with three columns: local time, level badge, and message. On multi-service or multi-container stacks, each row also shows a service chip so you can tell which service wrote the line; single-service, single-container stacks omit the chip. Level detection is automatic: - `err` rows get a red left rail and a subtle rose tint. - `warn` rows get a warm tint. @@ -368,6 +368,7 @@ The stack header groups actions by frequency of use. The most common action is t | Secondary | **Update** | `docker compose pull` + `up -d` (or build-aware rebuild when services declare `build:`) | Pulls registry images and recreates containers. When one or more services use `build:`, Update rebuilds those images from source (`compose build --pull`), pulls any remaining registry images, then recreates containers. | | Overflow | **Rollback** | Restores backup | Reverts compose and env files to the pre-deploy snapshot and redeploys. Only shown when a backup exists. | | Overflow | **Scan config** | Trivy config scan | Scans the compose file for misconfigurations (admin role). | +| Overflow | **Monitor** | Opens Monitor sheet | Opens the stack **Monitor** sheet on the Alerts tab (alert rules and Auto-heal). Same sheet as sidebar **Alerts** / **Auto-Heal**. | | Overflow | **Mute** | Creates a mute rule | Quick presets to mute notifications, deploy-success noise, or monitor alerts for this stack, plus a link to manage its mute rules in full. See [Alerts and Notifications](/features/alerts-notifications). | | Overflow | **Delete** | `down --volumes` + removes files | Stops and removes containers and volumes, then deletes the stack directory. | @@ -377,6 +378,7 @@ The stack header groups actions by frequency of use. The most common action is t |-----------|--------|---------|--------------| | Primary | **Start** | `docker compose up -d` | Starts the stack. | | Secondary | **Update** | `docker compose pull` + `up -d` (or build-aware rebuild when services declare `build:`) | Pulls registry images and recreates containers. When one or more services use `build:`, Update rebuilds those images from source (`compose build --pull`), pulls any remaining registry images, then recreates containers. | +| Overflow | **Monitor** | Opens Monitor sheet | Opens the stack **Monitor** sheet on the Alerts tab. | | Overflow | **Delete** | Removes files | Deletes the stack directory. | Use the sidebar context menu or `⌘↓` / `Ctrl+↓` for **Take down** on a stopped stack. @@ -414,8 +416,9 @@ Press `B` again or toggle the bulk mode button to leave bulk mode. ## Controlling a single service -Inside the stack detail view, each container card has a **Service actions** kebab on the right side. Use it to act on that service without touching the rest of the stack. +Inside the stack detail view, each container card has quick actions (View logs, Monitor, Open bash when available) and a **Service actions** kebab on the right side. Use them to act on that service without touching the rest of the stack. +- **Monitor**: opens the stack **Monitor** sheet with that Compose service preferred in the add forms. Shown only when the container reports a Compose service name. - **Restart service**: stops and starts all containers for that service. - **Stop service**: stops all containers for that service. Only shown when the service is running. - **Start service**: starts all containers for that service. Only shown when the service is stopped or exited. diff --git a/docs/getting-started/introduction.mdx b/docs/getting-started/introduction.mdx index d22f083e..cd1a11d8 100644 --- a/docs/getting-started/introduction.mdx +++ b/docs/getting-started/introduction.mdx @@ -34,7 +34,7 @@ The **Home** view is the default landing page. It is designed for a fast operati - The activity panel shows **Fleet Heartbeat** when remote nodes exist, or **Stack Restarts (7d)** on a local-only install. - **Recent Alerts** shows the latest notification feed and includes **Clear All Notifications** when there is anything to clear. -The top navigation starts with **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. Additional operator views (**Logs**, **Update**, **Schedules**, and **Audit**) appear based on your role and license tier. **Console** is a limited-availability operator surface documented on its own page when enabled on an instance. Fleet-wide views describe the control instance, so they are hidden while a remote node is active. Choose Classic, Smart, or Compact desktop navigation under **Settings → Appearance → Navigation**; phone navigation stays on its own layout. +The top navigation starts with **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. Additional operator views (**Logs**, **Update**, **Schedules**, and **Console**) appear for admins. **Audit** appears based on your role and license tier. Fleet-wide views describe the control instance, so they are hidden while a remote node is active. Choose Classic, Smart, or Compact desktop navigation under **Settings → Appearance → Navigation**; phone navigation stays on its own layout. ## Stack workspace diff --git a/docs/getting-started/quickstart.mdx b/docs/getting-started/quickstart.mdx index 2da8778d..fc099426 100644 --- a/docs/getting-started/quickstart.mdx +++ b/docs/getting-started/quickstart.mdx @@ -107,7 +107,7 @@ You land on **Home**, the default operational view. The health masthead reports Below the stack table, **Configuration Status** summarizes notifications, alerts, automation, security, backups, thresholds, and crash detection. The neighboring activity card shows **Fleet Heartbeat** when remote nodes exist, or **Stack Restarts (7d)** on a local-only install. **Recent Alerts** shows the latest notification feed and includes **Clear All Notifications** when there is anything to clear. -On the local node, baseline top navigation includes **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. **Logs**, **Update**, and **Schedules** appear for admins. **Console** and **Audit** depend on license, role, and whether those surfaces are enabled; hub-only views are hidden when a remote node is active. Desktop presentation (Classic bar, Smart bar, or Compact launcher) is chosen under **Settings → Appearance → Navigation**. The right side of the top bar holds global search, notifications, and the profile menu entries **Settings**, **Billing** (when a paid license is active), **Documentation**, **Open New Issue**, and **Log Out**. +On the local node, baseline top navigation includes **Home**, **Resources**, **Networking**, **Security**, and **App Store**. **Fleet** appears when your role can read nodes. **Logs**, **Update**, **Schedules**, and **Console** appear for admins. **Audit** depends on license and role; hub-only views are hidden when a remote node is active. Desktop presentation (Classic bar, Smart bar, or Compact launcher) is chosen under **Settings → Appearance → Navigation**. The right side of the top bar holds global search, notifications, and the profile menu entries **Settings**, **Billing** (when a paid license is active), **Documentation**, **Open New Issue**, and **Log Out**. The left sidebar is the stack workspace. Below the Sencho brand, it starts with the node switcher, then **Create Stack**, a bulk-mode toggle, and **Scan stacks folder** for re-indexing compose projects added outside Sencho. Use **Search stacks...** with the **All**, **Up**, **Down**, and **Updates** chips to narrow the list. On a fresh install with an empty stack list, Sencho scans your mounted compose directory automatically and shows what it found, including compose files that still need to be adopted into their own subfolder. Once stacks carry Docker Compose labels, the list groups them under those labels, with pinned stacks always floating to the top and unlabeled stacks collected at the bottom. diff --git a/docs/images/stack-view/logs-viewer.png b/docs/images/stack-view/logs-viewer.png index bf7f7482..0c64b593 100644 Binary files a/docs/images/stack-view/logs-viewer.png and b/docs/images/stack-view/logs-viewer.png differ diff --git a/docs/reference/settings.mdx b/docs/reference/settings.mdx index f9427856..42ce8d58 100644 --- a/docs/reference/settings.mdx +++ b/docs/reference/settings.mdx @@ -139,7 +139,7 @@ A live preview card shows a sample fleet-status tile so you can see a color choi | Control | What it does | |---------|--------------| | **Density** | **Comfortable** (default spacing, roomier rows and tiles for review and orientation) or **Compact** (tighter rows and tiles, fits more stacks, tasks, and audit entries on screen at once). Affects the dashboard stack table, the resource gauge strip, the Settings Hub sidebar, the Schedules and Audit Log tables, and every other data table in Sencho. Typography, color, and layout structure stay the same; only vertical padding compresses. | -| **Log chip color** | **Unified** uses the accent color for every service's log chip. **Per service** assigns each service a stable label color for faster visual scanning across a busy log stream. | +| **Log chip color** | Applies on multi-service or multi-container stacks (chips are hidden for a single service with a single container). **Unified** uses the accent color for every service's log chip. **Per service** assigns each service a stable label color for faster visual scanning across a busy log stream. | ### Navigation @@ -554,7 +554,7 @@ How long Sencho keeps historical data on this node before pruning it. | Setting | Default | Max | Description | |---------|---------|-----|-------------| | **Container metrics** | 24 hrs | 8,760 (1 year) | How long to keep per-container CPU, RAM, and network history for dashboard charts. | -| **Notification log** | 30 days | 365 | How long to keep alert and notification history. | +| **Notification log** | 30 days | 365 | Maximum how long to keep alert and notification history. Stack-associated entries are also removed when that stack is deleted. | | **Scan history per digest** | 50 scans | 1,000 | How many vulnerability scans to keep per image digest (or per image reference when no digest is stored). Older scans beyond the cap are pruned. | | **Remove scans for deleted images and stacks** | On | - | When on, scan results are deleted once their image is gone from this node or their stack is deleted, so the Security Overview stays tied to what still exists. Turn it off to keep scan history for removed images and stacks. | | **Audit log** | 90 days | 365 | How long to keep audit trail entries. Requires Admiral. | diff --git a/frontend/src/components/AuditLogView.tsx b/frontend/src/components/AuditLogView.tsx index 4c99d92b..cd8741d7 100644 --- a/frontend/src/components/AuditLogView.tsx +++ b/frontend/src/components/AuditLogView.tsx @@ -27,6 +27,7 @@ interface AuditEntry { node_id: number | null; ip_address: string; summary: string; + acting_as?: string | null; flags?: AnomalyFlag[]; } @@ -464,6 +465,11 @@ export function AuditLogView({ headerActions }: AuditLogViewProps = {}) { {entry.username} + {entry.acting_as ? ( + + acting as {entry.acting_as} + + ) : null} @@ -492,6 +498,10 @@ export function AuditLogView({ headerActions }: AuditLogViewProps = {}) { IP Address {entry.ip_address || '-'} +
+ Acting as + {entry.acting_as || '-'} +
Node ID {entry.node_id ?? 'Local'} @@ -673,6 +683,9 @@ function StreamRow({ entry, now }: StreamRowProps) {
{entry.username || 'system'} + {entry.acting_as ? ( + (acting as {entry.acting_as}) + ) : null} {verb.toLowerCase()} {target || entry.path}
diff --git a/frontend/src/components/EditorLayout.tsx b/frontend/src/components/EditorLayout.tsx index 6ceec945..be3611eb 100644 --- a/frontend/src/components/EditorLayout.tsx +++ b/frontend/src/components/EditorLayout.tsx @@ -256,6 +256,7 @@ export default function EditorLayout() { markAllRead, deleteNotification, clearAllNotifications, + removeNotificationsForStack, } = useNotifications({ nodes, onStateInvalidate: scheduleStateInvalidateRefresh, @@ -289,6 +290,7 @@ export default function EditorLayout() { }, canOfferVolumeRemoval, onDeletedOpenStack: () => onDeletedOpenStackRef.current(), + removeNotificationsForStack, }); // Wire the ref now that stackActions is available @@ -658,6 +660,10 @@ export default function EditorLayout() { changeEnvFile={stackActions.changeEnvFile} openLogViewer={stackActions.openLogViewer} openBashModal={stackActions.openBashModal} + onOpenMonitor={stackName ? () => overlayState.openAlertSheet(stackName) : undefined} + onOpenServiceMonitor={stackName + ? (serviceName) => overlayState.openAlertSheet(stackName, { serviceName }) + : undefined} serviceAction={stackActions.serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} diff --git a/frontend/src/components/EditorLayout/EditorView.tsx b/frontend/src/components/EditorLayout/EditorView.tsx index 4488f336..d2ad501b 100644 --- a/frontend/src/components/EditorLayout/EditorView.tsx +++ b/frontend/src/components/EditorLayout/EditorView.tsx @@ -177,12 +177,14 @@ export interface EditorViewProps { // Container / service actions openLogViewer: (containerId: string, containerName: string) => void; openBashModal: (containerId: string, containerName: string) => void; + onOpenMonitor?: () => void; + onOpenServiceMonitor?: (serviceName: string) => void; serviceAction: ( action: 'start' | 'stop' | 'restart', serviceName: string, ) => Promise; - // Declared-service facts for the multi-service header split (§12). Empty - // on single-service stacks and older remotes (capability-gated fetch), so + // Declared-service facts for the multi-service header layout. Empty on + // single-service stacks and older remotes (capability-gated fetch), so // ContainersHealth falls back to the flat single-service layout. Optional // so callers/tests that never deal in services can omit them. effectiveServices?: EffectiveServiceSpec[]; @@ -278,6 +280,8 @@ export function EditorView(props: EditorViewProps) { changeEnvFile, openLogViewer, openBashModal, + onOpenMonitor, + onOpenServiceMonitor, serviceAction, effectiveServices = [], serviceUpdateStatuses = [], @@ -388,9 +392,8 @@ export function EditorView(props: EditorViewProps) { }); }; - // Declared-service headers (§12) need the same expandable, scroll-wrapped - // layout as a multi-container stack even when only one container of a - // multi-service stack is currently running. + // Multi-service stacks need the same expandable, scroll-wrapped layout as a + // multi-container stack even when only one container is currently running. const isMultiContainerLayout = safeContainers.length > 1 || effectiveServices.length > 1; // Below md, render the segmented full-screen mobile detail instead of the @@ -401,6 +404,17 @@ export function EditorView(props: EditorViewProps) { return ; } + const stackLogsSection = ( + + ); + return (
@@ -437,6 +451,7 @@ export function EditorView(props: EditorViewProps) { showTakeDown={showTakeDown} isSelfStack={isSelfStack} stackMuteActions={stackMuteActions} + onOpenMonitor={onOpenMonitor} />
{recoveryResult && loadingAction == null && ( @@ -474,6 +489,7 @@ export function EditorView(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} @@ -498,6 +514,7 @@ export function EditorView(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} @@ -518,23 +535,9 @@ export function EditorView(props: EditorViewProps) { Hidden when containers are expanded to fill the column. */} {!containersExpanded && (isMultiContainerLayout ? (
- + {stackLogsSection}
- ) : ( - - ))} + ) : stackLogsSection)}
)} diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx index 1a94442f..073fae25 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.test.tsx @@ -1,16 +1,22 @@ -import { describe, it, expect, vi } from 'vitest'; +import { describe, it, expect, vi, afterEach } from 'vitest'; import { render, screen, fireEvent } from '@testing-library/react'; import { useState, type ReactNode } from 'react'; import { MobileStackDetail } from './MobileStackDetail'; import type { EditorViewProps } from './EditorView'; +import type { ContainerInfo } from './EditorView'; +import type { EffectiveServiceSpec } from '@/types/effectiveServices'; // The detail's heavy children stream logs, parse compose, and render container // stats; stub them with markers so this test focuses on segment behavior and the -// mobile editing flow. +// mobile editing flow. Capture showServiceChips for wiring assertions. +let lastShowServiceChips: boolean | undefined; vi.mock('./editor-view-blocks', () => ({ StackIdentityHeader: () =>
identity-header
, ContainersHealth: () =>
health-pane
, - StackLogsSection: () =>
logs-pane
, + StackLogsSection: ({ showServiceChips }: { showServiceChips: boolean }) => { + lastShowServiceChips = showServiceChips; + return
logs-pane
; + }, })); // Prop-aware so the edit affordance (canEdit + onEditCompose) is exercised, not // just the read-only marker. @@ -298,3 +304,65 @@ describe('MobileStackDetail mobile editing', () => { expect(setContent).not.toHaveBeenCalled(); }); }); + +function containerStub(id: string, name: string): ContainerInfo { + return { + Id: id, + Names: [name], + State: 'running', + Status: 'Up 1 minute', + }; +} + +function serviceStub(name: string): EffectiveServiceSpec { + return { + name, + declaredImage: `${name}:latest`, + hasBuild: false, + expectedReplicas: 1, + dependsOn: [], + hasHealthcheck: false, + }; +} + +describe('MobileStackDetail showServiceChips wiring', () => { + afterEach(() => { + lastShowServiceChips = undefined; + }); + + it('passes false for one container and one declared service', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(false); + }); + + it('passes true for two containers', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); + + it('passes true for one container and two declared services', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); +}); diff --git a/frontend/src/components/EditorLayout/MobileStackDetail.tsx b/frontend/src/components/EditorLayout/MobileStackDetail.tsx index fd57963b..c2d409dd 100644 --- a/frontend/src/components/EditorLayout/MobileStackDetail.tsx +++ b/frontend/src/components/EditorLayout/MobileStackDetail.tsx @@ -61,8 +61,10 @@ export function MobileStackDetail(props: EditorViewProps) { changeEnvFile, openLogViewer, openBashModal, + onOpenMonitor, + onOpenServiceMonitor, serviceAction, - effectiveServices, + effectiveServices = [], serviceUpdateStatuses, serviceUpdateInProgress, onRequestServiceUpdate, @@ -88,6 +90,7 @@ export function MobileStackDetail(props: EditorViewProps) { const [segment, setSegment] = useState('logs'); const safeContainers = containers || []; + const isMultiContainerLayout = safeContainers.length > 1 || effectiveServices.length > 1; const isRunning = safeContainers.some(c => c.State === 'running'); const canEditStack = can('stack:edit', 'stack', stackName); @@ -158,6 +161,7 @@ export function MobileStackDetail(props: EditorViewProps) { showTakeDown={showTakeDown} isSelfStack={isSelfStack} stackMuteActions={stackMuteActions} + onOpenMonitor={onOpenMonitor} />
@@ -228,6 +232,7 @@ export function MobileStackDetail(props: EditorViewProps) { activeNode={activeNode} openLogViewer={openLogViewer} openBashModal={openBashModal} + onOpenServiceMonitor={onOpenServiceMonitor} serviceAction={serviceAction} effectiveServices={effectiveServices} serviceUpdateStatuses={serviceUpdateStatuses} @@ -241,7 +246,12 @@ export function MobileStackDetail(props: EditorViewProps) { )} {segment === 'logs' && ( - + )} {segment === 'compose' && (
diff --git a/frontend/src/components/EditorLayout/ShellOverlays.tsx b/frontend/src/components/EditorLayout/ShellOverlays.tsx index b7dfb8de..e0e8cc1c 100644 --- a/frontend/src/components/EditorLayout/ShellOverlays.tsx +++ b/frontend/src/components/EditorLayout/ShellOverlays.tsx @@ -117,6 +117,7 @@ export function ShellOverlays({ onOpenChange={(open) => { if (!open) closeStackMonitor(); }} stackName={stackMonitor?.stackName ?? ''} initialTab={stackMonitor?.tab ?? 'alerts'} + initialService={stackMonitor?.serviceName} /> {/* Pre-update readiness check */} diff --git a/frontend/src/components/EditorLayout/ViewRouter.tsx b/frontend/src/components/EditorLayout/ViewRouter.tsx index a27f4602..45301ba5 100644 --- a/frontend/src/components/EditorLayout/ViewRouter.tsx +++ b/frontend/src/components/EditorLayout/ViewRouter.tsx @@ -1,8 +1,11 @@ import { Suspense, lazy, type ReactNode } from 'react'; +import { Unplug } from 'lucide-react'; import { Skeleton } from '@/components/ui/skeleton'; import { useAuth } from '@/context/AuthContext'; -import { useExperimental } from '@/hooks/useExperimental'; -import { PaidGate } from '../PaidGate'; +import { useLicense } from '@/context/LicenseContext'; +import { useNodes } from '@/context/NodeContext'; +import { resolveHostConsoleCapability } from '@/lib/routing/hostConsoleCapability'; +import { LockCard } from '../ui/LockCard'; import { CapabilityGate } from '../CapabilityGate'; import { HubOnlyGate } from '../HubOnlyGate'; import LazyBoundary from '../LazyBoundary'; @@ -17,7 +20,7 @@ import type { MuteRuleDraft } from '@/lib/muteRules'; import type { ActiveView } from './hooks/useViewNavigationState'; import type { StackUpdateInfo } from '@/types/imageUpdates'; import type { SecurityTab, FleetTab } from '@/lib/events'; -import { isStackEditorDeepLink } from '@/lib/router/readUrlRouteState'; +import { isStackEditorDeepLink, isHostConsoleStackDeepLink } from '@/lib/router/readUrlRouteState'; import type { NavDestination } from '@/lib/navigation/appNavRegistry'; // Paid-tier views are loaded on demand. Their internal PaidGate / @@ -147,7 +150,8 @@ export function ViewRouter({ quickLinkCandidates, }: ViewRouterProps): ReactNode { const { can } = useAuth(); - const { experimental, experimentalReady } = useExperimental(); + const { isPaid, licenseReady } = useLicense(); + const { activeNode, activeNodeMeta } = useNodes(); if (activeView === 'settings') { return ( ; + } + if (activeNode == null) return ; + const capState = resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: activeNode.type === 'remote', + isPaid, + licenseReady, + activeNodeMeta, + }); + if (capState === 'loading') return ; + if (capState === 'locked') { + const nodeName = activeNode.name; + const version = activeNodeMeta?.version; + let versionHint = `${nodeName} does not advertise this capability.`; + if (version && version !== 'unknown' && version !== '0.0.0-dev') { + versionHint = `${nodeName} is running v${version}.`; + } + return ( + + ); + } + const nodeId = activeNode.id; return ( - - - - - - - + + + ); } // Stack workspace: keep a loading shell while the stack URL hydrates. diff --git a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx index 4d2dd253..083e9283 100644 --- a/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/ContainersHealth.test.tsx @@ -509,4 +509,43 @@ describe('containers load states', () => { await user.click(screen.getByRole('button', { name: /retry/i })); expect(onRetry).toHaveBeenCalledTimes(1); }); + + it('shows Monitor when Service is set and calls onOpenServiceMonitor', async () => { + const onOpenServiceMonitor = vi.fn(); + const user = userEvent.setup(); + const c = { ...container([{ PrivatePort: 80, PublicPort: 8080 }]), Service: 'web' } as ContainerInfo; + render( + , + ); + + await user.click(screen.getByRole('button', { name: 'Monitor web' })); + expect(onOpenServiceMonitor).toHaveBeenCalledWith('web'); + }); + + it('hides Monitor when the container has no Service label', () => { + render( + , + ); + expect(screen.queryByRole('button', { name: /Monitor / })).toBeNull(); + }); }); diff --git a/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx b/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx index 90a95a30..0317d52f 100644 --- a/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/EditorView.test.tsx @@ -17,11 +17,15 @@ vi.mock('@/lib/monacoLoader', () => ({ }, })); -// Stub heavy children; this test only asserts the Monaco language prop. +// Capture StackLogsSection props for showServiceChips wiring tests. +let lastShowServiceChips: boolean | undefined; vi.mock('../editor-view-blocks', () => ({ StackIdentityHeader: () =>
identity-header
, ContainersHealth: () =>
health-pane
, - StackLogsSection: () =>
logs-pane
, + StackLogsSection: ({ showServiceChips }: { showServiceChips: boolean }) => { + lastShowServiceChips = showServiceChips; + return
logs-pane
; + }, })); vi.mock('../../StackAnatomyPanel', () => ({ default: () =>
anatomy-pane
, @@ -98,6 +102,7 @@ describe('EditorView Monaco language prop', () => { lastLanguage = undefined; lastValue = undefined; lastReadOnly = undefined; + lastShowServiceChips = undefined; }); it('passes language="ini" when the env tab is active', () => { @@ -149,6 +154,7 @@ describe('EditorView single edit gate', () => { lastLanguage = undefined; lastValue = undefined; lastReadOnly = undefined; + lastShowServiceChips = undefined; }); it('shows Save & Deploy immediately without an Edit button when compose editor is open', () => { @@ -191,3 +197,65 @@ describe('EditorView single edit gate', () => { expect(closeComposeEditor).toHaveBeenCalledTimes(1); }); }); + +function containerStub(id: string, name: string): EditorViewProps['containers'][number] { + return { + Id: id, + Names: [name], + State: 'running', + Status: 'Up 1 minute', + }; +} + +function serviceStub(name: string) { + return { + name, + declaredImage: `${name}:latest`, + hasBuild: false, + expectedReplicas: 1, + dependsOn: [] as string[], + hasHealthcheck: false, + }; +} + +describe('EditorView showServiceChips wiring', () => { + afterEach(() => { + lastShowServiceChips = undefined; + }); + + it('passes false for one container and one declared service', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(false); + }); + + it('passes true for two containers', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); + + it('passes true for one container and two declared services', () => { + render( + , + ); + expect(lastShowServiceChips).toBe(true); + }); +}); diff --git a/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx b/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx index f61b63f7..43e98509 100644 --- a/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/StackIdentityHeader.test.tsx @@ -103,4 +103,15 @@ describe('StackIdentityHeader', () => { expect(screen.queryByRole('menuitem', { name: /Take down/i })).toBeNull(); }); + + it('shows Monitor in More actions and calls onOpenMonitor', async () => { + const user = userEvent.setup(); + const onOpenMonitor = vi.fn(); + renderHeader({ onOpenMonitor, backupInfo: { exists: false, timestamp: null } }); + + await user.click(screen.getByRole('button', { name: 'More actions' })); + await user.click(screen.getByRole('menuitem', { name: 'Monitor' })); + + expect(onOpenMonitor).toHaveBeenCalledTimes(1); + }); }); diff --git a/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx b/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx new file mode 100644 index 00000000..44b3dda4 --- /dev/null +++ b/frontend/src/components/EditorLayout/__tests__/StackLogsSection.test.tsx @@ -0,0 +1,77 @@ +/** + * StackLogsSection forwards showServiceChips to StructuredLogViewer and leaves + * the raw-terminal contract unchanged. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, fireEvent } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { StackLogsSection } from '../editor-view-blocks'; + +let lastViewerShowServiceChips: boolean | undefined; + +vi.mock('../../StructuredLogViewer', () => ({ + default: ({ showServiceChips }: { showServiceChips?: boolean }) => { + lastViewerShowServiceChips = showServiceChips; + return
; + }, +})); + +vi.mock('../../Terminal', () => ({ + default: ({ stackName }: { stackName: string }) => ( +
{stackName}
+ ), +})); + +vi.mock('../../ErrorBoundary', () => ({ + default: ({ children }: { children: ReactNode }) => <>{children}, +})); + +describe('StackLogsSection showServiceChips forwarding', () => { + beforeEach(() => { + lastViewerShowServiceChips = undefined; + }); + + it('forwards true to StructuredLogViewer in structured mode', () => { + render( + , + ); + expect(screen.getByTestId('structured-log-viewer')).toBeInTheDocument(); + expect(lastViewerShowServiceChips).toBe(true); + }); + + it('forwards false to StructuredLogViewer in structured mode', () => { + render( + , + ); + expect(screen.getByTestId('structured-log-viewer')).toBeInTheDocument(); + expect(lastViewerShowServiceChips).toBe(false); + }); + + it('renders TerminalComponent in raw mode without requiring chip props', () => { + const setLogsMode = vi.fn(); + render( + , + ); + expect(screen.getByTestId('raw-terminal')).toHaveTextContent('web'); + expect(screen.queryByTestId('structured-log-viewer')).toBeNull(); + expect(lastViewerShowServiceChips).toBeUndefined(); + + fireEvent.click(screen.getByRole('button', { name: /Structured/i })); + expect(setLogsMode).toHaveBeenCalledWith('structured'); + }); +}); diff --git a/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx b/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx new file mode 100644 index 00000000..bbe6660a --- /dev/null +++ b/frontend/src/components/EditorLayout/__tests__/ViewRouter.test.tsx @@ -0,0 +1,179 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import * as AuthContext from '@/context/AuthContext'; +import * as LicenseContext from '@/context/LicenseContext'; +import * as NodeContext from '@/context/NodeContext'; +import { ViewRouter } from '../ViewRouter'; + +vi.mock('@/context/AuthContext'); +vi.mock('@/context/LicenseContext'); +vi.mock('@/context/NodeContext'); + +vi.mock('../../HostConsole', () => ({ + default: ({ nodeId, stackName }: { nodeId: number; stackName?: string | null }) => ( +
+ Host Console +
+ ), +})); + +vi.mock('../../LazyBoundary', () => ({ + default: ({ children }: { children: ReactNode }) => <>{children}, +})); + +const baseProps = { + activeView: 'host-console' as const, + selectedFile: null as string | null, + isLoading: false, + settingsSection: 'appearance' as const, + onSettingsSectionChange: vi.fn(), + onTemplateDeploySuccess: vi.fn(), + onHostConsoleClose: vi.fn(), + onFleetNavigateToNode: vi.fn(), + onOpenNodeNetworking: vi.fn(), + filterNodeId: null, + onClearScheduledOpsFilter: vi.fn(), + schedulePrefill: null, + onPrefillConsumed: vi.fn(), + muteRulePrefill: null, + onMutePrefillConsumed: vi.fn(), + notifications: [] as [], + onNavigateToStack: vi.fn(), + onOpenSettingsSection: vi.fn(), + onClearNotifications: vi.fn(), + securityTab: 'overview' as const, + onSecurityTabChange: vi.fn(), + renderEditor: () => null, + stackUpdates: {}, + urlHydratingStack: null as string | null, + isFileLoading: false, + quickLinkCandidates: [], +}; + +describe('ViewRouter host-console', () => { + beforeEach(() => { + window.history.replaceState({}, '', '/nodes/local/host-console'); + vi.mocked(AuthContext.useAuth).mockReturnValue({ + can: (p: string) => p === 'system:console', + } as unknown as ReturnType); + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: false, + licenseReady: true, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + activeNodeMeta: { version: '0.96.0', capabilities: ['host-console', 'host-console-community'], fetchedAt: 1 }, + } as unknown as ReturnType); + }); + + afterEach(() => { + window.history.replaceState({}, '', '/'); + }); + + it('renders Host Console for a Community admin on the local node', async () => { + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-node-id')).toBe('1'); + }); + + it('does not mount HostConsole while the active node is unresolved', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: null, + activeNodeMeta: null, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('does not mount HostConsole while a stack deep link is still hydrating', () => { + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('does not mount a root shell while the URL targets a stack-scoped Console', () => { + window.history.replaceState({}, '', '/nodes/local/host-console/radarr'); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('mounts stack-scoped Console only after selectedFile matches the route', async () => { + window.history.replaceState({}, '', '/nodes/local/host-console/radarr'); + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-stack')).toBe('radarr'); + expect(el.getAttribute('data-node-id')).toBe('1'); + }); + + it('renders nothing without system:console', () => { + vi.mocked(AuthContext.useAuth).mockReturnValue({ + can: () => false, + } as unknown as ReturnType); + const { container } = render(); + expect(container.querySelector('[data-testid="host-console"]')).toBeNull(); + }); + + it('shows a skeleton while remote metadata is loading (does not mount HostConsole)', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: null, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + }); + + it('shows a lock card for Community + legacy remote without mounting HostConsole', () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + expect(screen.getByText(/Host Console is not available on this node/i)).toBeTruthy(); + }); + + it('mounts Host Console for Admiral + legacy remote after meta resolves', async () => { + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: true, + licenseReady: true, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + const el = await screen.findByTestId('host-console'); + expect(el.getAttribute('data-node-id')).toBe('2'); + }); + + it('shows a skeleton for legacy-only remote while license is still loading', () => { + vi.mocked(LicenseContext.useLicense).mockReturnValue({ + isPaid: false, + licenseReady: false, + } as unknown as ReturnType); + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 2, name: 'Legacy', type: 'remote' }, + activeNodeMeta: { version: '0.95.0', capabilities: ['host-console'], fetchedAt: 1 }, + } as unknown as ReturnType); + render(); + expect(screen.queryByTestId('host-console')).toBeNull(); + expect(screen.queryByText(/Host Console is not available on this node/i)).toBeNull(); + }); + + it('mounts Host Console for Community + host-console-community remote', async () => { + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 3, name: 'NewPeer', type: 'remote' }, + activeNodeMeta: { + version: '0.96.0', + capabilities: ['host-console', 'host-console-community'], + fetchedAt: 1, + }, + } as unknown as ReturnType); + render(); + expect(await screen.findByTestId('host-console')).toBeTruthy(); + }); +}); diff --git a/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx b/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx index 81303274..25c21452 100644 --- a/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx +++ b/frontend/src/components/EditorLayout/__tests__/useViewNavigationState.test.tsx @@ -51,7 +51,7 @@ function mockDeployer() { function mockPaidAdmin() { vi.mocked(AuthContext.useAuth).mockReturnValue({ isAdmin: true, - can: (p: string) => p === 'system:audit' || p === 'node:read', + can: (p: string) => p === 'system:audit' || p === 'system:console' || p === 'node:read', permissionsStatus: 'ready', } as unknown as ReturnType); vi.mocked(LicenseContext.useLicense).mockReturnValue({ @@ -63,7 +63,7 @@ function mockPaidAdmin() { function mockCommunityAdmin() { vi.mocked(AuthContext.useAuth).mockReturnValue({ isAdmin: true, - can: (p: string) => p === 'node:read', + can: (p: string) => p === 'system:console' || p === 'node:read', permissionsStatus: 'ready', } as unknown as ReturnType); vi.mocked(LicenseContext.useLicense).mockReturnValue({ @@ -269,13 +269,13 @@ describe('useViewNavigationState', () => { expect(result.current.navItems.map(i => i.value)).toContain('global-observability'); }); - it('shows Update and Schedules for a community admin (now free) but hides paid Console and Audit', () => { + it('shows Update, Schedules, and Console for a community admin; Audit stays paid', () => { mockCommunityAdmin(); const { result } = renderHook(() => useViewNavigationState()); const values = result.current.navItems.map(i => i.value); expect(values).toContain('auto-updates'); expect(values).toContain('scheduled-ops'); - expect(values).not.toContain('host-console'); + expect(values).toContain('host-console'); expect(values).not.toContain('audit-log'); // The auto-updates nav item surfaces under the short label "Update". expect(result.current.navItems.find(i => i.value === 'auto-updates')?.label).toBe('Update'); @@ -429,53 +429,29 @@ describe('useViewNavigationState', () => { expect(result.current.securityTab).toBe('overview'); }); - // ── experimental discovery ───────────────────────────────────────────────── + // ── host-console discovery (no longer experimental) ──────────────────────── - it('hides Console from nav for a paid admin when experimental discovery is off', () => { + it('keeps Console in nav for a paid admin when experimental discovery is off', () => { mockPaidAdmin(); useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); const { result } = renderHook(() => useViewNavigationState()); - expect(result.current.navItems.map(i => i.value)).not.toContain('host-console'); + expect(result.current.navItems.map(i => i.value)).toContain('host-console'); }); - it('hides Console from nav while experimental metadata is still loading', () => { + it('keeps Console in nav while experimental metadata is still loading', () => { mockPaidAdmin(); useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); const { result } = renderHook(() => useViewNavigationState()); - expect(result.current.navItems.map(i => i.value)).not.toContain('host-console'); + expect(result.current.navItems.map(i => i.value)).toContain('host-console'); }); - it('does not normalize a host-console deep link before experimental readiness', () => { + it('keeps a host-console deep link selected when experimental is off', () => { mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); + useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); const onNavigateToDashboard = vi.fn(); const { result } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); act(() => result.current.setActiveView('host-console')); expect(result.current.activeView).toBe('host-console'); expect(onNavigateToDashboard).not.toHaveBeenCalled(); }); - - it('keeps host-console selected when delayed experimental resolves enabled', () => { - mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); - const onNavigateToDashboard = vi.fn(); - const { result, rerender } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); - act(() => result.current.setActiveView('host-console')); - useExperimentalMock.mockReturnValue({ experimental: true, experimentalReady: true }); - rerender(); - expect(result.current.activeView).toBe('host-console'); - expect(onNavigateToDashboard).not.toHaveBeenCalled(); - }); - - it('normalizes host-console once when experimental resolves disabled', () => { - mockPaidAdmin(); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: false }); - const onNavigateToDashboard = vi.fn(); - const { result, rerender } = renderHook(() => useViewNavigationState({ onNavigateToDashboard })); - act(() => result.current.setActiveView('host-console')); - useExperimentalMock.mockReturnValue({ experimental: false, experimentalReady: true }); - rerender(); - expect(result.current.activeView).toBe('dashboard'); - expect(onNavigateToDashboard).toHaveBeenCalled(); - }); }); diff --git a/frontend/src/components/EditorLayout/editor-view-blocks.tsx b/frontend/src/components/EditorLayout/editor-view-blocks.tsx index c70bb0b3..09735b60 100644 --- a/frontend/src/components/EditorLayout/editor-view-blocks.tsx +++ b/frontend/src/components/EditorLayout/editor-view-blocks.tsx @@ -22,7 +22,8 @@ import { Maximize2, Minimize2, AlertCircle, - RefreshCw + RefreshCw, + HeartPulse, } from 'lucide-react'; import { useCallback, useEffect, useRef, useState } from 'react'; import { Button } from '../ui/button'; @@ -144,6 +145,8 @@ export interface StackIdentityHeaderProps { /** True when this stack is the running Sencho instance on the active node. */ isSelfStack?: boolean; stackMuteActions?: ReturnType; + /** Opens the stack Monitor sheet on the Alerts tab. */ + onOpenMonitor?: () => void; } // Breadcrumb + serif title + state pill + action bar. The action buttons grow @@ -170,6 +173,7 @@ export function StackIdentityHeader({ showTakeDown, isSelfStack = false, stackMuteActions, + onOpenMonitor, }: StackIdentityHeaderProps) { const selfProtected = isSelfStack; return ( @@ -206,7 +210,7 @@ export function StackIdentityHeader({ const canScan = trivy.available && isAdmin; const canMute = stackMuteActions?.canMute ?? false; const hasOverflowExtras = canRollback || canScan; - const hasOverflow = hasOverflowExtras || canDelete || canMute; + const hasOverflow = hasOverflowExtras || canDelete || canMute || onOpenMonitor; if (!canDeploy && !hasOverflow) return null; return (
@@ -278,8 +282,14 @@ export function StackIdentityHeader({ {stackMisconfigScanning ? 'Scanning...' : 'Scan config'} )} + {onOpenMonitor && ( + + + Monitor + + )} {stackMuteActions && } - {(canRollback || canScan || stackMuteActions?.canMute) && canDelete && } + {(canRollback || canScan || onOpenMonitor || stackMuteActions?.canMute) && canDelete && } {canDelete && ( void; openBashModal: (containerId: string, containerName: string) => void; + /** Opens Monitor (Alerts tab); preselects the Compose service in add forms when listed. */ + onOpenServiceMonitor?: (serviceName: string) => void; serviceAction: (action: 'start' | 'stop' | 'restart', serviceName: string) => Promise; // Declared Compose services from the effective model. Multi-service // headers (owning Update/Rebuild + badge + Start/Stop/Restart) render only @@ -335,6 +347,7 @@ export function ContainersHealth({ activeNode, openLogViewer, openBashModal, + onOpenServiceMonitor, serviceAction, effectiveServices = [], serviceUpdateStatuses = [], @@ -346,8 +359,8 @@ export function ContainersHealth({ containersLoadError = null, onRetryContainersLoad, }: ContainersHealthProps) { - // Multi-service only (§12): a single-service stack keeps the existing flat - // layout untouched, including its per-container Start/Stop/Restart kebab. + // Multi-service only: a single-service stack keeps the existing flat layout + // untouched, including its per-container Start/Stop/Restart kebab. const isMultiService = effectiveServices.length > 1; const [copiedUrlId, setCopiedUrlId] = useState(null); const copiedUrlTimerRef = useRef(null); @@ -445,9 +458,9 @@ export function ContainersHealth({ ) : null; // One container card. `hideServiceMenu` drops the per-container - // Start/Stop/Restart kebab on multi-service stacks, where the declared- - // service header above owns that action instead (§12 point 4: child cards - // keep only logs, shell, ports, metrics). + // Start/Stop/Restart kebab on multi-service stacks; the declared-service + // header above owns lifecycle actions. Child cards keep logs, shell, ports, + // and metrics only. const renderContainerCard = (container: ContainerInfo, hideServiceMenu: boolean) => { let mainPort: number | undefined; let mainPortPrivate: number | undefined; @@ -474,6 +487,7 @@ export function ContainersHealth({ : ''; const containerName = container?.Names?.[0]?.replace(/^\//, '') || container?.Id?.slice(0, 12) || 'container'; + const composeService = container.Service; const isActive = container.State === 'running' || container.State === 'paused'; const health = container.healthStatus; const uptime = isActive ? extractUptime(container.Status) : null; @@ -565,6 +579,24 @@ export function ContainersHealth({ View logs + {onOpenServiceMonitor && composeService && ( + + + + + + Monitor {composeService} + + + )} {isAdmin && ( @@ -807,6 +839,8 @@ export interface StackLogsSectionProps { stackName: string; logsMode: 'structured' | 'raw'; setLogsMode: (mode: 'structured' | 'raw') => void; + /** True when the stack has more than one service or container; gates log chips. */ + showServiceChips: boolean; /** When set, the structured viewer shows an expand control that collapses * the Command Center to give the logs more vertical room. */ logsExpanded?: boolean; @@ -814,7 +848,7 @@ export interface StackLogsSectionProps { } // Logs pane: structured / raw-terminal toggle + the live viewer. -export function StackLogsSection({ stackName, logsMode, setLogsMode, logsExpanded, onToggleLogsExpand }: StackLogsSectionProps) { +export function StackLogsSection({ stackName, logsMode, setLogsMode, showServiceChips, logsExpanded, onToggleLogsExpand }: StackLogsSectionProps) { return (
@@ -844,7 +878,7 @@ export function StackLogsSection({ stackName, logsMode, setLogsMode, logsExpande
{logsMode === 'structured' ? ( - + ) : (
diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts index 5550164e..fe5acd96 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.test.ts @@ -22,6 +22,9 @@ const makeNotif = (overrides: Partial = {}): NotificationItem id: 1, level: 'info', message: 'test', timestamp: 1000, is_read: 0, ...overrides, }); +const nodeMessageKeys = (items: NotificationItem[]): string[] => + items.map((n) => `${n.nodeId}:${n.message}`).sort(); + class MockWS { static instances: MockWS[] = []; static readonly CONNECTING = 0; @@ -251,4 +254,362 @@ describe('useNotifications', () => { expect(MockWS.instances).toHaveLength(2); await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); }); + + it('removeNotificationsForStack drops only the matching node and stack', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const remote = makeRemoteNode('online', { id: 2, name: 'Remote-B' }); + const { result } = renderHook(() => + useNotifications({ nodes: [localNode, remote], onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'a-web' }), + }), + }); + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 2, stack_name: 'db', message: 'a-db' }), + }), + }); + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 3, stack_name: 'web', message: 'b-web' }), + }), + }); + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 4, message: 'a-unattached' }), + }), + }); + }); + + expect(result.current.notifications).toHaveLength(4); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + + expect(result.current.notifications.map((n) => `${n.nodeId}:${n.stack_name ?? ''}:${n.message}`).sort()).toEqual([ + '1::a-unattached', + '1:db:a-db', + '2:web:b-web', + ]); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + expect(result.current.notifications).toHaveLength(3); + }); + + it('refetches notifications on scope=notifications invalidate after mount fetch', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const nodes = [localNode]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + // No stack-deleted action: this case only asserts refetch delta. + nodeId: 1, + ts: 1000, + }), + }); + }); + + await waitFor(() => { + const after = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + expect(after).toBe(afterMount + 1); + }); + }); + + it('remote notifications invalidate triggers one additional remote fetch', async () => { + const remote = makeRemoteNode('online', { id: 2, name: 'sencho-sat-qa' }); + const nodes = [localNode, remote]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + (fetchForNode as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [{ id: 9, level: 'error', message: 'remote row', timestamp: 2000, is_read: 0, stack_name: 'db' }], + }); + + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 2)); + const afterMount = (fetchForNode as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications' && c[1] === 2, + ).length; + + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + nodeId: 2, + ts: 1000, + }), + }); + }); + + await waitFor(() => { + const after = (fetchForNode as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications' && c[1] === 2, + ).length; + expect(after).toBe(afterMount + 1); + }); + await waitFor(() => expect(result.current.notifications.some((n) => n.nodeId === 2 && n.nodeName === 'sencho-sat-qa')).toBe(true)); + }); + + it('unrelated state-invalidate scopes do not refetch notifications', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const nodes = [localNode]; + renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = (apiFetch as ReturnType).mock.calls.filter( + (c) => c[0] === '/notifications', + ).length; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'stack', + nodeId: 1, + stackName: 'web', + action: 'start', + ts: 1000, + }), + }); + }); + + expect( + (apiFetch as ReturnType).mock.calls.filter((c) => c[0] === '/notifications').length, + ).toBe(afterMount); + }); + + it('remote stack-deleted purge uses hub rn.id when payload nodeId collides with local', async () => { + (apiFetch as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + (fetchForNode as ReturnType).mockResolvedValue({ ok: true, json: async () => [] }); + const remote = makeRemoteNode('online', { id: 7, name: 'Remote-7' }); + const nodes = [localNode, remote]; + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(fetchForNode).toHaveBeenCalledWith('/notifications', 7)); + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'local-web' }), + }), + }); + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 2, stack_name: 'web', message: 'remote-web' }), + }), + }); + }); + expect(nodeMessageKeys(result.current.notifications)).toEqual(['1:local-web', '7:remote-web']); + + // Production remotes broadcast their own local DB id (often 1), which collides + // with the hub local node. The remote socket must purge hub rn.id=7 only. + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + expect(nodeMessageKeys(result.current.notifications)).toEqual(['1:local-web']); + }); + + it('preserves cached notifications for a failed refetch leg after invalidate', async () => { + const remote = makeRemoteNode('online', { id: 7, name: 'Remote-7' }); + // Stable identity so the nodes effect does not re-fetch on every render. + const nodes = [localNode, remote]; + let remoteFail = false; + (apiFetch as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [ + { id: 10, level: 'info', message: 'local-kept', timestamp: 5000, is_read: 0, stack_name: 'other' }, + ], + }); + (fetchForNode as ReturnType).mockImplementation(() => { + if (remoteFail) { + return Promise.resolve({ ok: false, status: 502, json: async () => [] }); + } + return Promise.resolve({ + ok: true, + json: async () => [ + { id: 20, level: 'error', message: 'remote-web', timestamp: 4000, is_read: 0, stack_name: 'web' }, + { id: 21, level: 'info', message: 'remote-db', timestamp: 3000, is_read: 0, stack_name: 'db' }, + ], + }); + }); + + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate: vi.fn(), onImageUpdatesChange: vi.fn() }), + ); + await waitFor(() => expect(result.current.notifications.some((n) => n.message === 'remote-web')).toBe(true)); + expect(nodeMessageKeys(result.current.notifications)).toEqual([ + '1:local-kept', + '7:remote-db', + '7:remote-web', + ]); + + remoteFail = true; + (apiFetch as ReturnType).mockResolvedValue({ + ok: true, + json: async () => [ + { id: 11, level: 'info', message: 'local-kept-v2', timestamp: 6000, is_read: 0, stack_name: 'other' }, + ], + }); + + act(() => { + MockWS.instances[1]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + // Wait for post-invalidate merge (local slice replaced), not only optimistic purge. + await waitFor(() => { + expect(result.current.notifications.some((n) => n.message === 'local-kept-v2')).toBe(true); + }); + expect(result.current.notifications.some((n) => n.message === 'remote-web')).toBe(false); + expect(nodeMessageKeys(result.current.notifications)).toEqual([ + '1:local-kept-v2', + '7:remote-db', + ]); + }); + + it('does not restore a deleted stack notification from a stale pre-removal fetch', async () => { + let resolveStale!: (value: { ok: boolean; status: number; json: () => Promise }) => void; + const stalePromise = new Promise<{ ok: boolean; status: number; json: () => Promise }>((resolve) => { + resolveStale = resolve; + }); + let call = 0; + let releaseStale = false; + (apiFetch as ReturnType).mockImplementation(() => { + call += 1; + if (!releaseStale && call >= 2) return stalePromise; + if (releaseStale) { + return Promise.resolve({ + ok: true, + status: 200, + json: async () => [ + { id: 2, level: 'info', message: 'fresh-db', timestamp: 4000, is_read: 0, stack_name: 'db' }, + ], + }); + } + return Promise.resolve({ ok: true, status: 200, json: async () => [] }); + }); + + // Stable identity so the nodes effect does not re-fetch on every render. + const nodes = [localNode]; + const onStateInvalidate = vi.fn(); + const onImageUpdatesChange = vi.fn(); + const { result } = renderHook(() => + useNotifications({ nodes, onStateInvalidate, onImageUpdatesChange }), + ); + await waitFor(() => expect(apiFetch).toHaveBeenCalled()); + const afterMount = call; + + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'notification', + payload: makeNotif({ id: 1, stack_name: 'web', message: 'keep-until-remove' }), + }), + }); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(true); + + // Start an in-flight fetch without act: React's act waits on the deferred promise. + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'other', + ts: Date.now(), + }), + }); + await waitFor(() => expect(call).toBeGreaterThan(afterMount)); + + act(() => { + result.current.removeNotificationsForStack(localNode.id, 'web'); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + + resolveStale({ + ok: true, + status: 200, + json: async () => [ + { id: 1, level: 'error', message: 'stale-web', timestamp: 3000, is_read: 0, stack_name: 'web' }, + ], + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + + releaseStale = true; + act(() => { + MockWS.instances[0]?.onmessage?.({ + data: JSON.stringify({ + type: 'state-invalidate', + scope: 'notifications', + action: 'stack-deleted', + nodeId: 1, + stackName: 'web', + ts: Date.now(), + }), + }); + }); + + await waitFor(() => { + expect(result.current.notifications.some((n) => n.stack_name === 'db' && n.message === 'fresh-db')).toBe(true); + }); + expect(result.current.notifications.some((n) => n.stack_name === 'web')).toBe(false); + }); }); diff --git a/frontend/src/components/EditorLayout/hooks/useNotifications.ts b/frontend/src/components/EditorLayout/hooks/useNotifications.ts index f0a8e2a0..573d1656 100644 --- a/frontend/src/components/EditorLayout/hooks/useNotifications.ts +++ b/frontend/src/components/EditorLayout/hooks/useNotifications.ts @@ -28,8 +28,12 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang // Its spans instrument only that first fetch so later polls do not pollute the // report. const notificationsReadyRef = useRef(false); + // Monotonic generation so a slow pre-removal fetch cannot overwrite state + // after removeNotificationsForStack (or a newer fetch) has already run. + const notificationFetchGeneration = useRef(0); const fetchNotifications = async () => { + const generation = ++notificationFetchGeneration.current; try { const currentNodes = nodesRef.current; const localNode = currentNodes.find(n => n.type === 'local'); @@ -41,7 +45,10 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang const localPromise = apiFetch('/notifications', { localOnly: true } as Parameters[1]); const remotePromises = remoteNodes.map(n => fetchForNode('/notifications', n.id)); - const all: NotificationItem[] = []; + // Only successful legs replace their node slice. A failed/non-OK/decode-failed + // leg is not treated as an empty list; that node's cached rows stay until a + // later successful response (including an intentional empty list). + const successfulSlices = new Map(); const instrument = !notificationsReadyRef.current; const headersSpan = instrument ? beginSpan('fetch_headers', { background: true }) : null; @@ -55,8 +62,15 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang if (bodySpan !== null) endSpan(bodySpan); bodySpan = null; const dispatchSpan = instrument ? beginSpan('state_dispatch', { background: true }) : null; - data.forEach(n => all.push({ ...n, nodeId: localNode?.id ?? -1, nodeName: localNode?.name ?? 'Local' })); + const localId = localNode?.id ?? -1; + const localName = localNode?.name ?? 'Local'; + successfulSlices.set( + localId, + data.map(n => ({ ...n, nodeId: localId, nodeName: localName })), + ); if (dispatchSpan !== null) endSpan(dispatchSpan); + } else { + console.error(`[Notifications] local fetch HTTP ${localRes.status}`); } } catch (e) { if (headersSpan !== null) endSpan(headersSpan, { outcome: 'error' }); @@ -71,17 +85,38 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang // Remotes settle in the background; they never gate the milestone above. const remoteNodeResults = await Promise.allSettled(remotePromises); - for (let i = 0; i < remoteNodes.length; i++) { + for (const [i, rn] of remoteNodes.entries()) { const result = remoteNodeResults[i]; - if (result?.status === 'fulfilled' && result.value.ok) { + if (result.status !== 'fulfilled') { + console.error(`[Notifications] remote fetch error (${rn.name}):`, result.reason); + continue; + } + if (!result.value.ok) { + console.error(`[Notifications] remote fetch HTTP ${result.value.status} (${rn.name})`); + continue; + } + try { const data = await result.value.json() as Omit[]; - const rn = remoteNodes[i]; - data.forEach(n => all.push({ ...n, nodeId: rn.id, nodeName: rn.name })); + successfulSlices.set( + rn.id, + data.map(n => ({ ...n, nodeId: rn.id, nodeName: rn.name })), + ); + } catch (e) { + console.error(`[Notifications] remote fetch decode error (${rn.name}):`, e); } } - all.sort((a, b) => b.timestamp - a.timestamp); - setNotifications(all); + if (generation !== notificationFetchGeneration.current) return; + // Keep failed/offline node slices; drop rows for nodes no longer in the roster. + const activeNodeIds = new Set(currentNodes.map(n => n.id)); + setNotifications(prev => { + const preserved = prev.filter(n => + n.nodeId == null + || (activeNodeIds.has(n.nodeId) && !successfulSlices.has(n.nodeId)), + ); + const replaced = [...successfulSlices.values()].flat(); + return [...preserved, ...replaced].sort((a, b) => b.timestamp - a.timestamp); + }); } catch (e) { console.error('[Notifications] fetch error:', e); } @@ -90,6 +125,33 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang const fetchNotificationsRef = useRef(fetchNotifications); fetchNotificationsRef.current = fetchNotifications; + const purgeStackNotifications = (nodeId: number, stackName: string) => { + notificationFetchGeneration.current += 1; + setNotifications(prev => + prev.filter(n => n.nodeId !== nodeId || n.stack_name !== stackName), + ); + }; + + const tryPurgeDeletedStackFromInvalidate = ( + msg: { action?: unknown; nodeId?: unknown; stackName?: unknown }, + fallbackNodeId?: number, + ) => { + if (msg.action !== 'stack-deleted' || typeof msg.stackName !== 'string') return; + const nodeId = typeof msg.nodeId === 'number' ? msg.nodeId : fallbackNodeId; + if (nodeId === undefined) return; + purgeStackNotifications(nodeId, msg.stackName); + }; + + const reconcileNotificationsInvalidate = ( + msg: { action?: unknown; nodeId?: unknown; stackName?: unknown }, + fallbackNodeId?: number, + ) => { + tryPurgeDeletedStackFromInvalidate(msg, fallbackNodeId); + void fetchNotificationsRef.current(); + }; + const reconcileNotificationsInvalidateRef = useRef(reconcileNotificationsInvalidate); + reconcileNotificationsInvalidateRef.current = reconcileNotificationsInvalidate; + // Local notification WebSocket with exponential-backoff reconnect. useEffect(() => { const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; @@ -125,7 +187,9 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang } else if (msg.type === 'state-invalidate') { window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: msg })); onStateInvalidateRef.current(); - if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { + if (msg.scope === 'notifications') { + reconcileNotificationsInvalidateRef.current(msg); + } else if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { onImageUpdatesChangeRef.current(); } } @@ -203,7 +267,11 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang } else if (msg.type === 'state-invalidate') { window.dispatchEvent(new CustomEvent('sencho:state-invalidate', { detail: { ...msg, nodeId: rn.id } })); onStateInvalidateRef.current(); - if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { + if (msg.scope === 'notifications') { + // Remote payloads use the remote's local DB node ID. Hub UI state + // is keyed by rn.id, so always reconcile with the hub node ID. + reconcileNotificationsInvalidateRef.current({ ...msg, nodeId: rn.id }); + } else if (msg.scope === 'image-updates' && msg.action === 'stack-updated') { onImageUpdatesChangeRef.current(); } } @@ -324,5 +392,6 @@ export function useNotifications({ nodes, onStateInvalidate, onImageUpdatesChang markAllRead, deleteNotification, clearAllNotifications, + removeNotificationsForStack: purgeStackNotifications, } as const; } diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts index 8438a881..2da64452 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.test.ts @@ -74,6 +74,19 @@ describe('useOverlayState', () => { expect(result.current.stackMonitor).toEqual({ stackName: 'web-stack', tab: 'alerts' }); }); + it.each([ + ['openAlertSheet', 'alerts' as const], + ['openAutoHeal', 'auto-heal' as const], + ])('%s can carry a serviceName for form preselect', (openFn, tab) => { + const { result } = renderHook(() => useOverlayState()); + act(() => result.current[openFn as 'openAlertSheet' | 'openAutoHeal']('web-stack', { serviceName: 'api' })); + expect(result.current.stackMonitor).toEqual({ + stackName: 'web-stack', + tab, + serviceName: 'api', + }); + }); + it('openAutoHeal opens stack monitor on the auto-heal tab', () => { const { result } = renderHook(() => useOverlayState()); act(() => result.current.openAutoHeal('web-stack')); diff --git a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts index 8ddfabd5..86aeea97 100644 --- a/frontend/src/components/EditorLayout/hooks/useOverlayState.ts +++ b/frontend/src/components/EditorLayout/hooks/useOverlayState.ts @@ -25,6 +25,12 @@ type PolicyBlock = { }; type Container = { id: string; name: string }; +type StackMonitorState = { + stackName: string; + tab: 'alerts' | 'auto-heal'; + serviceName?: string; +}; + // Kept here (not in useStackActions) so overlay state can hold load options // without a circular type import between the two hooks. export type LoadFileOptions = { @@ -109,12 +115,12 @@ export function useOverlayState() { return () => window.removeEventListener(SENCHO_OPEN_LOGS_EVENT, handler); }, [openLogViewer]); // openLogViewer is stable (useCallback with empty deps) - const [stackMonitor, setStackMonitor] = useState<{ stackName: string; tab: 'alerts' | 'auto-heal' } | null>(null); - const openAlertSheet = useCallback((stackName: string) => { - setStackMonitor({ stackName, tab: 'alerts' }); + const [stackMonitor, setStackMonitor] = useState(null); + const openAlertSheet = useCallback((stackName: string, opts?: { serviceName?: string }) => { + setStackMonitor({ stackName, tab: 'alerts', serviceName: opts?.serviceName }); }, []); - const openAutoHeal = useCallback((stackName: string) => { - setStackMonitor({ stackName, tab: 'auto-heal' }); + const openAutoHeal = useCallback((stackName: string, opts?: { serviceName?: string }) => { + setStackMonitor({ stackName, tab: 'auto-heal', serviceName: opts?.serviceName }); }, []); const closeStackMonitor = useCallback(() => setStackMonitor(null), []); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts index 56812468..50441939 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.test.ts @@ -130,6 +130,7 @@ function setup(over: { activeNode?: Parameters[0]['activeNode']; setActiveNode?: Parameters[0]['setActiveNode']; onDeletedOpenStack?: () => void; + removeNotificationsForStack?: (nodeId: number, stackName: string) => void; } = {}) { const editorState = makeEditorState(over.editorState); const stackListState = makeStackListState(over.stackList); @@ -141,6 +142,7 @@ function setup(over: { const overlayState = makeOverlay(over.overlay); const setActiveNode = over.setActiveNode ?? vi.fn(); const onDeletedOpenStack = over.onDeletedOpenStack ?? vi.fn(); + const removeNotificationsForStack = over.removeNotificationsForStack ?? vi.fn(); const { result } = renderHook(() => useStackActions({ @@ -157,9 +159,10 @@ function setup(over: { hasUpdateGuard: over.hasUpdateGuard ?? false, canEditStack: over.canEditStack ?? (() => true), onDeletedOpenStack, + removeNotificationsForStack, }), ); - return { result, editorState, stackListState, overlayState, navState, setActiveNode, onDeletedOpenStack }; + return { result, editorState, stackListState, overlayState, navState, setActiveNode, onDeletedOpenStack, removeNotificationsForStack }; } describe('useStackActions.saveFile', () => { @@ -1617,4 +1620,40 @@ describe('useStackActions.deleteStack', () => { expect(onDeletedOpenStack).not.toHaveBeenCalled(); expect(stackListState.refreshStacks).not.toHaveBeenCalled(); }); + + it('calls removeNotificationsForStack with node id and canonical name on success', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response(null, { status: 200 })); + const removeNotificationsForStack = vi.fn(); + const { result } = setup({ + overlay: { stackToDelete: 'web.yml' }, + stackList: { selectedFile: 'web.yml', files: ['web.yml'] }, + navState: { activeView: 'editor' }, + activeNode: { id: 7, type: 'local' } as Parameters[0]['activeNode'], + removeNotificationsForStack, + }); + + await act(async () => { + await result.current.deleteStack(false); + }); + + expect(removeNotificationsForStack).toHaveBeenCalledTimes(1); + expect(removeNotificationsForStack).toHaveBeenCalledWith(7, 'web'); + }); + + it('does not call removeNotificationsForStack on a non-OK delete', async () => { + vi.mocked(apiFetch).mockResolvedValue(new Response('boom', { status: 500 })); + const removeNotificationsForStack = vi.fn(); + const { result } = setup({ + overlay: { stackToDelete: 'web.yml' }, + stackList: { selectedFile: 'web.yml', files: ['web.yml'] }, + navState: { activeView: 'editor' }, + removeNotificationsForStack, + }); + + await act(async () => { + await result.current.deleteStack(false); + }); + + expect(removeNotificationsForStack).not.toHaveBeenCalled(); + }); }); diff --git a/frontend/src/components/EditorLayout/hooks/useStackActions.ts b/frontend/src/components/EditorLayout/hooks/useStackActions.ts index 18f9beff..10023e33 100644 --- a/frontend/src/components/EditorLayout/hooks/useStackActions.ts +++ b/frontend/src/components/EditorLayout/hooks/useStackActions.ts @@ -176,6 +176,12 @@ interface UseStackActionsOptions { * owns that state, and an optional callback would silently skip it. */ onDeletedOpenStack: () => void; + /** + * Drop in-memory notifications for a deleted stack on the active node. + * Caller must pass the node id and canonical stack basename (no .yml/.yaml). + * Optional so unit tests that do not exercise delete can omit it. + */ + removeNotificationsForStack?: (nodeId: number, stackName: string) => void; } const isRecord = (value: unknown): value is Record => @@ -399,6 +405,7 @@ export function useStackActions(options: UseStackActionsOptions) { canEditStack, canOfferVolumeRemoval = false, onDeletedOpenStack, + removeNotificationsForStack, } = options; const pendingStackLoadRef = useRef(null); @@ -1895,6 +1902,7 @@ export function useStackActions(options: UseStackActionsOptions) { stackListState.files.find( f => f === stackToDelete || f.replace(/\.(yml|yaml)$/, '') === stackToDelete, ) ?? stackToDelete; + const canonicalName = deleteKey.replace(/\.(yml|yaml)$/, ''); if (stackListState.isStackBusy(deleteKey)) return; stackListState.setStackAction(deleteKey, 'delete'); try { @@ -1914,11 +1922,14 @@ export function useStackActions(options: UseStackActionsOptions) { toast.success('Stack deleted successfully!'); overlayState.closeDeleteDialog(); const selected = stackListState.selectedFile; - const stripExt = (name: string) => name.replace(/\.(yml|yaml)$/, ''); + const nodeId = activeNode?.id; + if (nodeId != null && removeNotificationsForStack) { + removeNotificationsForStack(nodeId, canonicalName); + } // Always clear a deleted selection, even when another top-level view is // visible (Resources, Networking, etc.). Leaving that view is gated on // the editor being the active surface below. - if (selected != null && stripExt(selected) === stripExt(deleteKey)) { + if (selected != null && selected.replace(/\.(yml|yaml)$/, '') === canonicalName) { resetEditorState(); if (navState.activeView === 'editor') { navState.setActiveView('dashboard'); diff --git a/frontend/src/components/HostConsole.tsx b/frontend/src/components/HostConsole.tsx index 0761a337..21e62b55 100644 --- a/frontend/src/components/HostConsole.tsx +++ b/frontend/src/components/HostConsole.tsx @@ -8,6 +8,8 @@ import { useNodes } from '@/context/NodeContext'; import { copyToClipboard } from '@/lib/clipboard'; interface HostConsoleProps { + /** Resolved active node id; WebSocket must target this id, not localStorage. */ + nodeId: number; stackName?: string | null; onClose: () => void; } @@ -27,7 +29,7 @@ function formatUptime(ms: number): string { type ConnState = 'reconnecting' | 'connected' | 'disconnected'; -export default function HostConsole({ stackName, onClose }: HostConsoleProps) { +export default function HostConsole({ nodeId, stackName, onClose }: HostConsoleProps) { const { activeNode } = useNodes(); const terminalRef = useRef(null); const xtermRef = useRef(null); @@ -93,12 +95,12 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { }); const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; - const activeNodeId = localStorage.getItem('sencho-active-node') || ''; - const nodeParam = activeNodeId ? `nodeId=${activeNodeId}` : ''; - const stackParam = stackName ? `stack=${encodeURIComponent(stackName)}` : ''; - const queryString = [nodeParam, stackParam].filter(Boolean).join('&'); - const wsUrl = `${wsProtocol}//${window.location.host}/api/system/host-console${queryString ? `?${queryString}` : ''}`; - const ws = new WebSocket(wsUrl); + const qs = stackName + ? `nodeId=${nodeId}&stack=${encodeURIComponent(stackName)}` + : `nodeId=${nodeId}`; + const ws = new WebSocket( + `${wsProtocol}//${window.location.host}/api/system/host-console?${qs}`, + ); wsRef.current = ws; ws.onopen = () => { @@ -194,7 +196,7 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { fitAddonRef.current = null; serializeRef.current = null; }; - }, [stackName, reconnectNonce]); + }, [nodeId, stackName, reconnectNonce]); const handleCopy = useCallback(() => { const term = xtermRef.current; @@ -236,7 +238,10 @@ export default function HostConsole({ stackName, onClose }: HostConsoleProps) { const stateWord = connState === 'disconnected' ? 'Disconnected' : connState === 'reconnecting' ? 'Reconnecting' : 'Connected'; - const nodeLabel = activeNode ? (activeNode.type === 'local' ? 'LOCAL' : activeNode.name.toUpperCase()) : 'LOCAL'; + let nodeLabel = `NODE ${nodeId}`; + if (activeNode?.id === nodeId) { + nodeLabel = activeNode.type === 'local' ? 'LOCAL' : activeNode.name.toUpperCase(); + } const kicker = `HOST CONSOLE · ${nodeLabel}`; const uptime = mountedAt != null ? formatUptime(tick - mountedAt) : '—'; diff --git a/frontend/src/components/StackAlertSheet.test.tsx b/frontend/src/components/StackAlertSheet.test.tsx new file mode 100644 index 00000000..a598be3d --- /dev/null +++ b/frontend/src/components/StackAlertSheet.test.tsx @@ -0,0 +1,308 @@ +/** + * StackAlertSheet Alerts tab: service targeting, services-state machine, + * capability gating, and active-node reset. + */ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor, fireEvent } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; + +const { nodeState } = vi.hoisted(() => ({ + nodeState: { + activeNode: { id: 1, type: 'local', name: 'Local' } as { id: number; type: string; name: string } | null, + activeNodeMeta: { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + } as { version: string; capabilities: string[] } | null, + }, +})); + +vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); +vi.mock('@/components/ui/toast-store', () => ({ + toast: { + error: vi.fn(), + success: vi.fn(), + warning: vi.fn(), + info: vi.fn(), + loading: vi.fn(), + dismiss: vi.fn(), + }, +})); +vi.mock('@/context/NodeContext', () => ({ + useNodes: () => ({ + activeNode: nodeState.activeNode, + activeNodeMeta: nodeState.activeNodeMeta, + hasCapability: (cap: string) => nodeState.activeNodeMeta?.capabilities.includes(cap) === true, + }), +})); +vi.mock('@/context/AuthContext', () => ({ + useAuth: () => ({ isAdmin: true }), +})); + +import { apiFetch } from '@/lib/api'; +import { toast } from '@/components/ui/toast-store'; +import { StackAlertSheet } from './StackAlertSheet'; + +const mockedFetch = apiFetch as unknown as ReturnType; + +function jsonRes(body: unknown, ok = true, status = ok ? 200 : 500) { + return { + ok, + status, + json: async () => body, + text: async () => '', + } as unknown as Response; +} + +beforeEach(() => { + nodeState.activeNode = { id: 1, type: 'local', name: 'Local' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + mockedFetch.mockReset(); + vi.mocked(toast.success).mockReset(); + vi.mocked(toast.error).mockReset(); +}); + +function mockHappyPath(services: string[] = ['api', 'database'], alerts: unknown[] = []) { + mockedFetch.mockImplementation(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes('/agents')) { + return jsonRes([{ type: 'discord', enabled: true }]); + } + if (url.includes('/services')) { + return jsonRes(services); + } + if (url.startsWith('/alerts') && (!init || !init.method || init.method === 'GET')) { + return jsonRes(alerts); + } + if (url === '/alerts' && init?.method === 'POST') { + const body = JSON.parse(String(init.body)); + return jsonRes({ id: 99, ...body }, true, 201); + } + return jsonRes(null, false); + }); +} + +describe('StackAlertSheet Alerts tab', () => { + it('POSTs selected service_name when capability is present', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + await waitFor(() => { + expect(mockedFetch.mock.calls.some(([url]) => String(url).includes('/services'))).toBe(true); + }); + + // Service combobox is the first one in the Add new rule form. + await waitFor(() => { + const serviceBox = screen.getAllByRole('combobox')[0]; + expect(serviceBox).not.toBeDisabled(); + }); + await user.click(screen.getAllByRole('combobox')[0]); + await user.click(await screen.findByRole('button', { name: 'api' })); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + expect(post).toBeDefined(); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBe('api'); + expect(body.stack_name).toBe('my-stack'); + }); + }); + + it('POSTs service_name null for All services', async () => { + mockHappyPath(); + const user = userEvent.setup(); + render( {}} stackName="my-stack" />); + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('shows Not in compose only after a successful services list', async () => { + mockHappyPath(['database'], [{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + + render( {}} stackName="my-stack" />); + + await waitFor(() => { + expect(screen.getByText(/Not in compose/i)).toBeInTheDocument(); + }); + }); + + it('does not show Not in compose when services fetch fails', async () => { + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) return jsonRes(null, false, 500); + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('api')).toBeInTheDocument()); + expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument(); + }); + + it('hides service selector when capability is missing and posts null', async () => { + nodeState.activeNodeMeta = { version: '0.90.0', capabilities: [] }; + mockHappyPath(); + const user = userEvent.setup(); + + render( {}} stackName="my-stack" />); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + expect(screen.queryByText(/does not support service-scoped/i)).toBeInTheDocument(); + expect(screen.queryByRole('combobox', { name: /all services/i })).not.toBeInTheDocument(); + + fireEvent.change(screen.getByPlaceholderText('e.g. 90'), { target: { value: '80' } }); + await user.click(screen.getByText('Add Rule')); + + await waitFor(() => { + const post = mockedFetch.mock.calls.find( + ([url, init]) => String(url) === '/alerts' && (init as RequestInit | undefined)?.method === 'POST', + ); + const body = JSON.parse(String((post![1] as RequestInit).body)); + expect(body.service_name).toBeNull(); + }); + }); + + it('resets services state when active node changes', async () => { + let servicesCalls = 0; + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/agents')) return jsonRes([{ type: 'discord', enabled: true }]); + if (url.includes('/services')) { + servicesCalls += 1; + return jsonRes(servicesCalls === 1 ? ['api'] : ['worker']); + } + if (url.includes('/alerts')) { + return jsonRes([{ + id: 1, + stack_name: 'my-stack', + service_name: 'api', + metric: 'cpu_percent', + operator: '>', + threshold: 80, + duration_mins: 5, + cooldown_mins: 60, + }]); + } + return jsonRes(null, false); + }); + + const { rerender } = render( + {}} stackName="my-stack" />, + ); + + await waitFor(() => expect(servicesCalls).toBe(1)); + await waitFor(() => expect(screen.queryByText(/Not in compose/i)).not.toBeInTheDocument()); + + nodeState.activeNode = { id: 2, type: 'remote', name: 'Remote' }; + nodeState.activeNodeMeta = { + version: '1.0.0', + capabilities: ['service-scoped-stack-alert'], + }; + rerender( {}} stackName="my-stack" />); + + await waitFor(() => expect(servicesCalls).toBe(2)); + // New node's list has only worker, so the api-targeted rule is missing. + await waitFor(() => expect(screen.getByText(/Not in compose/i)).toBeInTheDocument()); + }); + + it('prefills the Service combobox from initialService when listed', async () => { + mockHappyPath(['api', 'database']); + render( + {}} + stackName="my-stack" + initialService="api" + />, + ); + + await waitFor(() => { + expect(screen.getByText('Add Rule')).toBeInTheDocument(); + expect(screen.getAllByRole('combobox')[0]).toHaveTextContent('api'); + }); + }); + + it('ignores Alerts initialService when service-scoped capability is absent', async () => { + nodeState.activeNodeMeta = { version: '0.90.0', capabilities: [] }; + mockHappyPath(['api', 'database']); + render( + {}} + stackName="my-stack" + initialService="api" + />, + ); + + await waitFor(() => expect(screen.getByText('Add Rule')).toBeInTheDocument()); + expect(screen.queryByText('All services')).toBeNull(); + expect( + mockedFetch.mock.calls.some(([url]) => String(url).includes('/services')), + ).toBe(false); + }); + + it('prefills Auto-heal Service from initialService when listed', async () => { + mockedFetch.mockImplementation(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/auto-heal/policies')) return jsonRes([]); + if (url.includes('/services')) return jsonRes(['api', 'database']); + return jsonRes(null, false); + }); + + render( + {}} + stackName="my-stack" + initialTab="auto-heal" + initialService="api" + />, + ); + + await waitFor(() => { + expect(screen.getByText('Add Policy')).toBeInTheDocument(); + expect(screen.getAllByRole('combobox')[0]).toHaveTextContent('api'); + }); + }); +}); diff --git a/frontend/src/components/StackAlertSheet.tsx b/frontend/src/components/StackAlertSheet.tsx index ee665527..bd1766d2 100644 --- a/frontend/src/components/StackAlertSheet.tsx +++ b/frontend/src/components/StackAlertSheet.tsx @@ -20,12 +20,14 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/comp import { Trash2, HelpCircle, AlertTriangle, Info, CheckCircle2, Loader2, ChevronDown, ChevronUp } from 'lucide-react'; import { toast } from '@/components/ui/toast-store'; import { apiFetch } from '@/lib/api'; +import { SERVICE_SCOPED_STACK_ALERT_CAPABILITY } from '@/lib/capabilities'; import { useNodes } from '@/context/NodeContext'; import { useAuth } from '@/context/AuthContext'; interface StackAlert { id?: number; stack_name: string; + service_name: string | null; metric: string; operator: string; threshold: number; @@ -33,6 +35,11 @@ interface StackAlert { cooldown_mins: number; } +type ServicesState = + | { status: 'idle' | 'loading' } + | { status: 'success'; options: string[] } + | { status: 'error' }; + interface AutoHealPolicy { id?: number; node_id: number; @@ -71,6 +78,8 @@ interface StackAlertSheetProps { onOpenChange: (open: boolean) => void; stackName: string; initialTab?: MonitorTab; + /** Prefill Add-form service comboboxes when opening from a service card. */ + initialService?: string; } interface AgentStatus { @@ -117,6 +126,10 @@ function actionColorClass(action: AutoHealHistoryEntry['action']): string { return 'text-muted-foreground'; } +function preselectListedService(initialService: string | undefined, names: string[]): string { + return initialService && names.includes(initialService) ? initialService : ''; +} + function actionLabel(action: AutoHealHistoryEntry['action']): string { switch (action) { case 'restarted': return 'Restarted'; @@ -129,7 +142,13 @@ function actionLabel(action: AutoHealHistoryEntry['action']): string { } } -export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'alerts' }: StackAlertSheetProps) { +export function StackAlertSheet({ + open, + onOpenChange, + stackName, + initialTab = 'alerts', + initialService, +}: StackAlertSheetProps) { const [activeTab, setActiveTab] = useState(initialTab); useEffect(() => { @@ -140,6 +159,7 @@ export function StackAlertSheet({ open, onOpenChange, stackName, initialTab = 'a { id: 'alerts', label: 'Alerts' }, { id: 'auto-heal', label: 'Auto-heal' }, ]; + const prefillService = open ? initialService : undefined; return ( } - {activeTab === 'auto-heal' && } + {activeTab === 'alerts' && ( + + )} + {activeTab === 'auto-heal' && ( + + )} ); } -function AlertsTab({ stackName }: { stackName: string }) { +function AlertsTab({ stackName, initialService }: { stackName: string; initialService?: string }) { const { isAdmin } = useAuth(); - const { activeNode } = useNodes(); + const { activeNode, activeNodeMeta } = useNodes(); const isRemote = activeNode?.type === 'remote'; + const canScopeService = activeNodeMeta?.capabilities.includes(SERVICE_SCOPED_STACK_ALERT_CAPABILITY) === true; const [alerts, setAlerts] = useState([]); const [isLoading, setIsLoading] = useState(false); @@ -172,7 +197,9 @@ function AlertsTab({ stackName }: { stackName: string }) { hasEnabled: false, enabledTypes: [], }); + const [servicesState, setServicesState] = useState({ status: 'idle' }); + const [service, setService] = useState(''); const [metric, setMetric] = useState('cpu_percent'); const [operator, setOperator] = useState('>'); const [threshold, setThreshold] = useState(''); @@ -183,7 +210,35 @@ function AlertsTab({ stackName }: { stackName: string }) { if (!stackName) return; fetchAlerts(); fetchAgentStatus(); - }, [stackName]); // eslint-disable-line react-hooks/exhaustive-deps + }, [stackName, activeNode?.id]); // eslint-disable-line react-hooks/exhaustive-deps + + useEffect(() => { + if (!stackName || !canScopeService) { + setServicesState({ status: 'idle' }); + setService(''); + return; + } + + let cancelled = false; + setServicesState({ status: 'loading' }); + setService(''); + + void (async () => { + try { + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); + if (!res.ok) throw new Error(`services ${res.status}`); + const names = await res.json() as string[]; + if (cancelled) return; + setServicesState({ status: 'success', options: names }); + setService(preselectListedService(initialService, names)); + } catch (e) { + console.error('[StackAlertSheet] Failed to fetch stack services', e); + if (!cancelled) setServicesState({ status: 'error' }); + } + })(); + + return () => { cancelled = true; }; + }, [stackName, activeNode?.id, canScopeService, initialService]); const fetchAlerts = async () => { try { @@ -224,8 +279,10 @@ function AlertsTab({ stackName }: { stackName: string }) { return; } setIsLoading(true); + const scopedServiceName = canScopeService && service !== '' ? service : null; const newAlert = { stack_name: stackName, + service_name: scopedServiceName, metric, operator, threshold: parseFloat(threshold), @@ -240,6 +297,7 @@ function AlertsTab({ stackName }: { stackName: string }) { if (res.ok) { toast.success('Alert rule added.'); setThreshold(''); + setService(''); fetchAlerts(); } else { const err = await res.json().catch(() => ({})); @@ -272,6 +330,32 @@ function AlertsTab({ stackName }: { stackName: string }) { } }; + const serviceComboOptions = [ + { value: '', label: 'All services' }, + ...(servicesState.status === 'success' + ? servicesState.options.map(n => ({ value: n, label: n })) + : []), + ]; + + const renderTargetLabel = (alert: StackAlert) => { + if (!alert.service_name) { + return All services; + } + const missing = servicesState.status === 'success' + && !servicesState.options.includes(alert.service_name); + if (missing) { + return ( + + {alert.service_name} (Not in compose) + + ); + } + return {alert.service_name}; + }; + const renderAgentStatusBanner = () => { if (agentStatus.loading) { return ( @@ -355,7 +439,8 @@ function AlertsTab({ stackName }: { stackName: string }) { {metricLabels[alert.metric] || alert.metric} {alert.operator} {alert.threshold}
- Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m + {renderTargetLabel(alert)} + {' '}• Trigger after {alert.duration_mins}m • Cooldown {alert.cooldown_mins}m
{isAdmin && ( @@ -378,6 +463,31 @@ function AlertsTab({ stackName }: { stackName: string }) { {isAdmin && (
+ {canScopeService && ( +
+ + + {servicesState.status === 'error' && ( +

+ Could not load Compose services. The rule will target all services. +

+ )} +
+ )} + {!canScopeService && activeNodeMeta && ( +

+ This node does not support service-scoped alert rules yet. Update the node to target a specific service. +

+ )} +
@@ -476,7 +586,15 @@ function AlertsTab({ stackName }: { stackName: string }) { ); } -function AutoHealTab({ stackName, open }: { stackName: string; open: boolean }) { +function AutoHealTab({ + stackName, + open, + initialService, +}: { + stackName: string; + open: boolean; + initialService?: string; +}) { const { isAdmin } = useAuth(); const [policies, setPolicies] = useState([]); const [loading, setLoading] = useState(false); @@ -493,17 +611,25 @@ function AutoHealTab({ stackName, open }: { stackName: string; open: boolean }) useEffect(() => { if (!open || !stackName) return; setLoading(true); + setService(''); apiFetch(`/auto-heal/policies?stackName=${encodeURIComponent(stackName)}`) .then(res => res.json() as Promise) .then(data => setPolicies(data)) .catch(() => toast.error('Failed to load auto-heal policies.')) .finally(() => setLoading(false)); - apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`) - .then(res => res.json() as Promise) - .then(names => setServiceOptions(names.map(n => ({ value: n, label: n })))) - .catch(() => { /* services list is optional, silently skip */ }); - }, [open, stackName]); + void (async () => { + try { + const res = await apiFetch(`/stacks/${encodeURIComponent(stackName)}/services`); + if (!res.ok) throw new Error(`services ${res.status}`); + const names = await res.json() as string[]; + setServiceOptions(names.map(n => ({ value: n, label: n }))); + setService(preselectListedService(initialService, names)); + } catch (e) { + console.error('[StackAlertSheet] Failed to fetch stack services', e); + } + })(); + }, [open, stackName, initialService]); const handleToggle = async (id: number, enabled: boolean) => { setSaving(true); diff --git a/frontend/src/components/StructuredLogViewer.tsx b/frontend/src/components/StructuredLogViewer.tsx index e9086de2..150f16d1 100644 --- a/frontend/src/components/StructuredLogViewer.tsx +++ b/frontend/src/components/StructuredLogViewer.tsx @@ -3,11 +3,17 @@ import { Button } from './ui/button'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from './ui/tooltip'; import { Download, RefreshCw, Maximize2, Minimize2 } from 'lucide-react'; import { cn } from '@/lib/utils'; -import { useLogChipColorMode } from '@/hooks/use-log-chip-color-mode'; +import { useLogChipColorMode, type LogChipColorMode } from '@/hooks/use-log-chip-color-mode'; import { hashLabel } from '@/lib/label-colors'; interface StructuredLogViewerProps { stackName: string; + /** + * When true, render a service chip on prefixed log rows. + * Pass true only for multi-service or multi-container stacks; defaults to + * false so single-service streams stay uncluttered. + */ + showServiceChips?: boolean; /** When set, renders an expand/collapse control next to the download button. */ expanded?: boolean; onToggleExpand?: () => void; @@ -20,7 +26,7 @@ interface LogRow { ts: string | null; level: LogLevel; message: string; - /** Normalized service name extracted from the log prefix, or null for synthetic / old-format rows. */ + /** Display name from the log prefix (normalized Compose service name), or null for synthetic / old-format rows. */ containerName: string | null; /** True when this row was synthesized by the client (e.g. reconnect sentinel). */ synthetic?: boolean; @@ -69,7 +75,36 @@ function formatTs(iso: string | null): string { return `${hh}:${mm}:${ss}`; } -export default function StructuredLogViewer({ stackName, expanded, onToggleExpand }: StructuredLogViewerProps) { +function stackDisplayName(stackName: string): string { + return stackName.replace(/\.(yml|yaml)$/, ''); +} + +function LogServiceChip({ name, colorMode }: { name: string; colorMode: LogChipColorMode }) { + const perService = colorMode === 'per-service'; + const labelKey = hashLabel(name); + return ( + + {name} + + ); +} + +export default function StructuredLogViewer({ stackName, showServiceChips = false, expanded, onToggleExpand }: StructuredLogViewerProps) { const [rows, setRows] = useState([]); const [filter, setFilter] = useState('all'); const [following, setFollowing] = useState(true); @@ -91,7 +126,7 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan setFollowing(true); followingRef.current = true; - const cleanStackName = stackName.replace(/\.(yml|yaml)$/, ''); + const cleanStackName = stackDisplayName(stackName); const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; const activeNodeId = localStorage.getItem('sencho-active-node') || ''; const wsUrl = `${wsProtocol}//${window.location.host}/api/stacks/${cleanStackName}/logs${activeNodeId ? `?nodeId=${activeNodeId}` : ''}`; @@ -238,12 +273,12 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; - a.download = `${stackName.replace(/\.(yml|yaml)$/, '')}-logs.txt`; + a.download = `${stackDisplayName(stackName)}-logs.txt`; a.click(); URL.revokeObjectURL(url); }; - const label = `logs · ${stackName.replace(/\.(yml|yaml)$/, '')}`; + const label = `logs · ${stackDisplayName(stackName)}`; return (
@@ -346,25 +381,8 @@ export default function StructuredLogViewer({ stackName, expanded, onToggleExpan {row.level} - {row.containerName && ( - - {row.containerName} - + {showServiceChips && row.containerName && ( + )} {row.message} diff --git a/frontend/src/components/__tests__/HostConsole.test.tsx b/frontend/src/components/__tests__/HostConsole.test.tsx new file mode 100644 index 00000000..bb883b89 --- /dev/null +++ b/frontend/src/components/__tests__/HostConsole.test.tsx @@ -0,0 +1,141 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, waitFor, act } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import * as NodeContext from '@/context/NodeContext'; +import HostConsole from '../HostConsole'; + +vi.mock('@/context/NodeContext'); + +vi.mock('@/lib/xtermLoader', () => { + class FakeTerminal { + cols = 80; + rows = 24; + open = vi.fn(); + focus = vi.fn(); + write = vi.fn(); + clear = vi.fn(); + dispose = vi.fn(); + getSelection = vi.fn(() => ''); + loadAddon = vi.fn(); + onData = vi.fn(); + } + class FakeFitAddon { + fit = vi.fn(); + } + class FakeSerializeAddon { + serialize = vi.fn(() => ''); + } + return { + loadXtermModules: async () => ({ + Terminal: FakeTerminal, + FitAddon: FakeFitAddon, + SerializeAddon: FakeSerializeAddon, + }), + }; +}); + +vi.mock('../ui/PageMasthead', () => ({ + PageMasthead: ({ children }: { children?: ReactNode }) =>
{children}
, +})); + +vi.mock('../ui/button', () => ({ + Button: ({ children, ...props }: { children?: ReactNode } & Record) => ( + + ), +})); + +vi.mock('@/lib/clipboard', () => ({ + copyToClipboard: vi.fn(async () => undefined), +})); + +type FakeWs = { + url: string; + readyState: number; + close: ReturnType; + send: ReturnType; + onopen: ((ev?: unknown) => void) | null; + onmessage: ((ev: { data: string }) => void) | null; + onerror: ((ev?: unknown) => void) | null; + onclose: ((ev?: unknown) => void) | null; +}; + +describe('HostConsole socket targeting', () => { + const sockets: FakeWs[] = []; + let OriginalWebSocket: typeof WebSocket; + + beforeEach(() => { + sockets.length = 0; + OriginalWebSocket = globalThis.WebSocket; + vi.mocked(NodeContext.useNodes).mockReturnValue({ + activeNode: { id: 1, name: 'Local', type: 'local' }, + } as unknown as ReturnType); + + globalThis.WebSocket = class { + static OPEN = 1; + static CLOSED = 3; + url: string; + readyState = 0; + close = vi.fn(() => { this.readyState = 3; }); + send = vi.fn(); + onopen: FakeWs['onopen'] = null; + onmessage: FakeWs['onmessage'] = null; + onerror: FakeWs['onerror'] = null; + onclose: FakeWs['onclose'] = null; + constructor(url: string) { + this.url = url; + sockets.push(this as unknown as FakeWs); + queueMicrotask(() => { + this.readyState = 1; + this.onopen?.(undefined); + }); + } + } as unknown as typeof WebSocket; + }); + + afterEach(() => { + globalThis.WebSocket = OriginalWebSocket; + localStorage.removeItem('sencho-active-node'); + vi.clearAllMocks(); + }); + + it('opens the WebSocket with the explicit nodeId (not localStorage)', async () => { + localStorage.setItem('sencho-active-node', '99'); + render(); + await waitFor(() => expect(sockets.length).toBe(1)); + expect(sockets[0].url).toContain('nodeId=7'); + expect(sockets[0].url).not.toContain('nodeId=99'); + }); + + it('includes the stack parameter when provided', async () => { + render(); + await waitFor(() => expect(sockets.length).toBe(1)); + expect(sockets[0].url).toContain('stack=radarr'); + }); + + it('closes the prior socket and opens a new one when nodeId changes', async () => { + const { rerender } = render(); + await waitFor(() => expect(sockets.length).toBe(1)); + const first = sockets[0]; + + await act(async () => { + rerender(); + }); + await waitFor(() => expect(sockets.length).toBe(2)); + expect(first.close).toHaveBeenCalled(); + expect(sockets[1].url).toContain('nodeId=2'); + }); + + it('reconnects when stackName changes so a root shell is not retained', async () => { + const { rerender } = render(); + await waitFor(() => expect(sockets.length).toBe(1)); + const first = sockets[0]; + expect(first.url).not.toContain('stack='); + + await act(async () => { + rerender(); + }); + await waitFor(() => expect(sockets.length).toBe(2)); + expect(first.close).toHaveBeenCalled(); + expect(sockets[1].url).toContain('stack=radarr'); + }); +}); diff --git a/frontend/src/components/__tests__/StructuredLogViewer.test.tsx b/frontend/src/components/__tests__/StructuredLogViewer.test.tsx index e335903e..716c52ad 100644 --- a/frontend/src/components/__tests__/StructuredLogViewer.test.tsx +++ b/frontend/src/components/__tests__/StructuredLogViewer.test.tsx @@ -1,7 +1,7 @@ /** * Unit tests for StructuredLogViewer's log-row lifecycle (stack switching, - * row clearing, auto-follow reset, level filter), container name chip - * rendering, and chip color mode (unified / per-service). + * row clearing, auto-follow reset, level filter), service chip rendering, + * and chip color mode (unified / per-service). */ import { render, screen, cleanup, fireEvent, act } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; @@ -182,51 +182,78 @@ describe('StructuredLogViewer', () => { expect(MockWS.instances[1].url).not.toContain('.yaml'); }); - // ── Container name chip ──────────────────────────────────────────── + // ── Service chip ─────────────────────────────────────────────────── - it('renders a container name chip when the WebSocket message includes a prefix', async () => { - const { container } = render(); + it('renders a service chip when showServiceChips is true and the line has a prefix', async () => { + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - expect(container.textContent).toContain('redis'); - expect(container.textContent).toContain('connected'); + const chip = screen.getByTitle('redis'); + expect(chip).toHaveTextContent('redis'); + expect(screen.getByText('connected')).toBeInTheDocument(); }); - it('does not render a container name chip for old-format lines with no prefix', async () => { - const { container } = render(); + it('hides the chip by default but keeps the message and download attribution', async () => { + let capturedBlob: Blob | null = null; + vi.stubGlobal('URL', { + ...URL, + createObjectURL: vi.fn((blob: Blob) => { + capturedBlob = blob; + return 'blob:fake'; + }), + revokeObjectURL: vi.fn(), + }); + + render(); + await act(async () => { + MockWS.instances[0].onopen?.(); + MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); + }); + + expect(screen.queryByTitle('redis')).toBeNull(); + expect(screen.getByText('connected')).toBeInTheDocument(); + + await userEvent.click(screen.getByLabelText('Download logs')); + expect(capturedBlob).not.toBeNull(); + const text = await capturedBlob!.text(); + expect(text).toContain('[redis]'); + expect(text).toContain('connected'); + }); + + it('does not render a service chip for old-format lines with no prefix', async () => { + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: '2025-01-01T12:00:00Z plain message\n' }); }); - expect(container.textContent).toContain('plain message'); - expect(container.querySelector('.select-none')).toBeNull(); + expect(screen.getByText('plain message')).toBeInTheDocument(); + expect(screen.queryByTitle('plain message')).toBeNull(); }); it('renders a dotted container name correctly', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'api.v1 | 2025-01-01T12:00:00Z ready\n' }); }); - expect(container.textContent).toContain('api.v1'); - expect(container.textContent).toContain('ready'); + expect(screen.getByTitle('api.v1')).toHaveTextContent('api.v1'); + expect(screen.getByText('ready')).toBeInTheDocument(); }); it('handles pipe in message body without false prefix extraction', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z value | other\n' }); }); - expect(container.textContent).toContain('redis'); - expect(container.textContent).toContain('value'); - expect(container.textContent).toContain('other'); + expect(screen.getByTitle('redis')).toHaveTextContent('redis'); + expect(screen.getByText(/value \| other/)).toBeInTheDocument(); }); // ── Download ──────────────────────────────────────────────────────── @@ -242,7 +269,7 @@ describe('StructuredLogViewer', () => { revokeObjectURL: vi.fn(), }); - render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); @@ -287,14 +314,13 @@ describe('StructuredLogViewer', () => { // ── Chip color mode ────────────────────────────────────────────────── it('in unified mode (default), chip has brand classes and no inline style', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; - expect(chip).not.toBeNull(); + const chip = screen.getByTitle('redis'); expect(chip.className).toContain('text-brand/80'); expect(chip.className).toContain('bg-brand/10'); expect(chip.getAttribute('style')).toBeNull(); @@ -302,27 +328,26 @@ describe('StructuredLogViewer', () => { it('in per-service mode, chip has inline label-token style', async () => { localStorage.setItem(LOG_CHIP_COLOR_KEY, 'per-service'); - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; - expect(chip).not.toBeNull(); + const chip = screen.getByTitle('redis'); const style = chip.getAttribute('style') ?? ''; expect(style).toContain('--label-'); expect(style).toContain('-bg'); }); it('updates chip style when setting changes from unified to per-service', async () => { - const { container } = render(); + render(); await act(async () => { MockWS.instances[0].onopen?.(); MockWS.instances[0].onmessage?.({ data: 'redis | 2025-01-01T12:00:00Z connected\n' }); }); - const chip = container.querySelector('.select-none') as HTMLElement; + const chip = screen.getByTitle('redis'); expect(chip.getAttribute('style')).toBeNull(); localStorage.setItem(LOG_CHIP_COLOR_KEY, 'per-service'); diff --git a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx index df538584..6ad8850b 100644 --- a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx +++ b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.test.tsx @@ -13,12 +13,13 @@ vi.mock('@/lib/api', () => ({ apiFetch: vi.fn() })); const toastError = vi.fn(); const toastSuccess = vi.fn(); +const toastWarning = vi.fn(); vi.mock('@/components/ui/toast-store', () => ({ toast: { error: (...a: unknown[]) => toastError(...a), success: (...a: unknown[]) => toastSuccess(...a), info: vi.fn(), - warning: vi.fn(), + warning: (...a: unknown[]) => toastWarning(...a), loading: vi.fn(() => 'toast-id'), dismiss: vi.fn(), }, @@ -112,3 +113,165 @@ it('surfaces an error toast when the prune returns non-ok', async () => { await user.click(screen.getByRole('button', { name: 'Dry run' })); await waitFor(() => expect(toastError).toHaveBeenCalledWith('prune blew up')); }); + +function estimateCallCount(): number { + return mockedFetch.mock.calls.filter(c => c[0] === '/fleet/prune/estimate').length; +} + +it('re-fetches the estimate after a partial-success real prune', async () => { + const user = userEvent.setup(); + let estimatePhase: 'pre' | 'post' = 'pre'; + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + if (estimatePhase === 'pre') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + return Promise.resolve(jsonResponse(200, { + totalBytes: 0, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 0, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + // One target succeeded before a later failure left the node unreachable. + // reachable:false must not suppress the refresh (target.success is the signal). + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: false, + error: 'transport failed after images', + targets: [ + { target: 'images', success: true, reclaimedBytes: 1024 }, + { target: 'volumes', success: false, reclaimedBytes: 0, error: 'unreachable' }, + ], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + expect(screen.getByText('~ 1 KB reclaimable')).toBeInTheDocument(); + expect(screen.getByText('1 KB')).toBeInTheDocument(); + const callsBeforePrune = estimateCallCount(); + + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + estimatePhase = 'post'; + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + + await waitFor(() => expect(estimateCallCount()).toBe(callsBeforePrune + 1)); + const postEstimateCall = [...mockedFetch.mock.calls].reverse().find(c => c[0] === '/fleet/prune/estimate'); + expect(postEstimateCall).toBeTruthy(); + expect(JSON.parse(postEstimateCall![1].body)).toEqual({ targets: ['images'], scope: 'managed' }); + await waitFor(() => expect(screen.getByText('0 reclaimable')).toBeInTheDocument()); + await waitFor(() => expect(screen.getByText('0 Bytes')).toBeInTheDocument()); +}); + +it('does not re-fetch the estimate after a dry run', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [{ target: 'images', success: true, reclaimedBytes: 4096, dryRun: true }], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + const callsBefore = estimateCallCount(); + await user.click(screen.getByRole('button', { name: 'Dry run' })); + await waitFor(() => expect(toastSuccess).toHaveBeenCalled()); + // Debounce window: a refresh would schedule another estimate within 350ms. + await new Promise(r => setTimeout(r, 500)); + expect(estimateCallCount()).toBe(callsBefore); +}); + +it('does not re-fetch the estimate when every target fails on a 2xx response', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [{ target: 'images', success: false, reclaimedBytes: 0, error: 'A prune is already running on this node' }], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + const callsBefore = estimateCallCount(); + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + await waitFor(() => expect(toastError).toHaveBeenCalled()); + await new Promise(r => setTimeout(r, 500)); + expect(estimateCallCount()).toBe(callsBefore); +}); + +it('warns instead of claiming total failure when a reachable node has mixed per-target outcomes', async () => { + const user = userEvent.setup(); + mockedFetch.mockImplementation((url: string) => { + if (url === '/fleet/prune/estimate') { + return Promise.resolve(jsonResponse(200, { + totalBytes: 1024, + perNode: [{ nodeId: 1, nodeName: 'central', reclaimableBytes: 1024, reachable: true }], + })); + } + if (url === '/fleet/labels/fleet-prune') { + // Images succeed, volumes fail on the same reachable node. Pre-fix toast + // logic treated the node as fully failed (okNodes=0) and said every node + // failed even though Docker mutated for images. + return Promise.resolve(jsonResponse(200, { + results: [{ + nodeId: 1, + nodeName: 'central', + reachable: true, + targets: [ + { target: 'images', success: true, reclaimedBytes: 1024 }, + { target: 'volumes', success: false, reclaimedBytes: 0, error: 'volume prune failed' }, + ], + }], + })); + } + return Promise.resolve(jsonResponse(404, {})); + }); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Prune fleet' })).toBeEnabled()); + await user.click(screen.getByRole('button', { name: 'Prune fleet' })); + const dialog = await screen.findByRole('alertdialog'); + await user.click(within(dialog).getByRole('button', { name: 'Prune managed' })); + + await waitFor(() => expect(toastWarning).toHaveBeenCalled()); + expect(toastWarning.mock.calls[0][0]).toMatch(/1\/1 nodes reclaimed space/); + expect(toastError).not.toHaveBeenCalledWith('Prune failed on every node. See results below.'); +}); diff --git a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx index a2f8cebb..34729407 100644 --- a/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx +++ b/frontend/src/components/fleet/FleetActions/cards/FleetPruneCard.tsx @@ -48,6 +48,9 @@ export function FleetPruneCard({ nodes }: Props) { const [running, setRunning] = useState(false); const [results, setResults] = useState([]); const [estimate, setEstimate] = useState({ kind: 'idle' }); + // Bumped after a real prune mutates at least one target so the estimate + // effect re-runs without changing targets/scope. + const [estimateEpoch, setEstimateEpoch] = useState(0); const toggleTarget = (target: PruneTarget) => { setTargets(prev => { @@ -92,7 +95,7 @@ export function FleetPruneCard({ nodes }: Props) { cancelled = true; if (estimateDebounceRef.current) clearTimeout(estimateDebounceRef.current); }; - }, [targets, scope]); + }, [targets, scope, estimateEpoch]); async function run(opts: { dryRun: boolean }) { if (targets.size === 0) return; @@ -113,6 +116,13 @@ export function FleetPruneCard({ nodes }: Props) { return; } const apiResults = (body.results as FleetPruneNodeResult[]) ?? []; + // HTTP 200 still arrives when every target fails; only a successful + // target means Docker state changed and the live estimate is stale. + // Scan targets independently of node.reachable (an early remote target + // can succeed before a later transport error marks the node unreachable). + if (!opts.dryRun && apiResults.some(node => node.targets.some(target => target.success))) { + setEstimateEpoch(e => e + 1); + } const rows: ResultRow[] = apiResults.map((node) => { const totalBytes = node.targets.reduce((sum, t) => sum + (t.reclaimedBytes ?? 0), 0); const allOk = node.reachable && node.targets.every(t => t.success); @@ -133,19 +143,24 @@ export function FleetPruneCard({ nodes }: Props) { }); setResults(rows); const totalNodes = apiResults.length; - const okNodes = apiResults.filter(n => n.reachable && n.targets.every(t => t.success)).length; + // Fully OK: every target on a reachable node succeeded. Partial: at + // least one target succeeded anywhere (mixed per-target on one node + // still counts). "Failed on every node" only when zero targets succeeded. + const fullyOkNodes = apiResults.filter(n => n.reachable && n.targets.every(t => t.success)).length; + const nodesWithAnySuccess = apiResults.filter(n => n.targets.some(t => t.success)).length; + const anyTargetSucceeded = nodesWithAnySuccess > 0; const totalReclaimed = apiResults.reduce( (sum, n) => sum + n.targets.reduce((s, t) => s + (t.reclaimedBytes ?? 0), 0), 0, ); if (opts.dryRun) { toast.success(`Dry run: ${formatBytes(totalReclaimed)} would be reclaimed across ${totalNodes} node${totalNodes === 1 ? '' : 's'}.`); - } else if (okNodes === totalNodes && totalNodes > 0) { + } else if (fullyOkNodes === totalNodes && totalNodes > 0) { toast.success(`Reclaimed ${formatBytes(totalReclaimed)} across ${totalNodes} node${totalNodes === 1 ? '' : 's'}.`); - } else if (okNodes === 0) { + } else if (!anyTargetSucceeded) { toast.error('Prune failed on every node. See results below.'); } else { - toast.warning(`${okNodes}/${totalNodes} nodes succeeded · ${formatBytes(totalReclaimed)} reclaimed. See results below.`); + toast.warning(`${nodesWithAnySuccess}/${totalNodes} nodes reclaimed space · ${formatBytes(totalReclaimed)} reclaimed. See results below.`); } } catch (err) { toast.dismiss(toastId); diff --git a/frontend/src/components/settings/AppearanceSection.tsx b/frontend/src/components/settings/AppearanceSection.tsx index 05099453..684671f7 100644 --- a/frontend/src/components/settings/AppearanceSection.tsx +++ b/frontend/src/components/settings/AppearanceSection.tsx @@ -447,7 +447,7 @@ export function AppearanceSection({ { expect(screen.getByText('Constrained graphics')).toBeTruthy(); }); + it('states that log chip color applies on multi-service or multi-container stacks', () => { + render(); + expect( + screen.getByText(/Applies to service chips on multi-service or multi-container stacks/i), + ).toBeTruthy(); + }); + it('readability locks the header + chart controls and disables the glow slider', () => { const { container } = render(); // Baseline: nothing reduced, so no slider is disabled. diff --git a/frontend/src/lib/capabilities.ts b/frontend/src/lib/capabilities.ts index 850581e5..887ccedd 100644 --- a/frontend/src/lib/capabilities.ts +++ b/frontend/src/lib/capabilities.ts @@ -16,6 +16,7 @@ export const CAPABILITIES = [ 'notification-suppression', 'notification-suppression-schedule', 'host-console', + 'host-console-community', 'container-exec', 'audit-log', 'scheduled-ops', @@ -36,10 +37,18 @@ export const CAPABILITIES = [ 'stack-down-remove-volumes', 'guided-external-network-preflight', 'service-scoped-update', + 'service-scoped-stack-alert', ] as const; export type Capability = (typeof CAPABILITIES)[number]; +/** Legacy Host Console advertisement (Admiral hubs still accept this on remotes). */ +export const HOST_CONSOLE_CAPABILITY = 'host-console' as const satisfies Capability; + +/** Host Console works without a paid license on this node. */ +export const HOST_CONSOLE_COMMUNITY_CAPABILITY = 'host-console-community' as const satisfies Capability; + export const STACK_DOWN_REMOVE_VOLUMES_CAPABILITY = 'stack-down-remove-volumes' as const satisfies Capability; export const GUIDED_EXTERNAL_NETWORK_PREFLIGHT_CAPABILITY = 'guided-external-network-preflight' as const satisfies Capability; export const SERVICE_SCOPED_UPDATE_CAPABILITY = 'service-scoped-update' as const satisfies Capability; +export const SERVICE_SCOPED_STACK_ALERT_CAPABILITY = 'service-scoped-stack-alert' as const satisfies Capability; diff --git a/frontend/src/lib/navigation/buildNavigationModel.test.ts b/frontend/src/lib/navigation/buildNavigationModel.test.ts index d6ffba00..f1a24944 100644 --- a/frontend/src/lib/navigation/buildNavigationModel.test.ts +++ b/frontend/src/lib/navigation/buildNavigationModel.test.ts @@ -77,19 +77,31 @@ describe('buildNavigationModel', () => { expect(values).not.toContain('audit-log'); }); - it('omits Console until experimental discovery is ready and enabled via reachCtx only', () => { + it('includes Console for system:console regardless of experimental discovery', () => { expect( - buildNavigationModel(makeCtx({ experimentalReady: false, experimental: false })) - .allPageItems.map((i) => i.value), - ).not.toContain('host-console'); - expect( - buildNavigationModel(makeCtx({ experimentalReady: true, experimental: false })) - .allPageItems.map((i) => i.value), - ).not.toContain('host-console'); - expect( - buildNavigationModel(makeCtx({ experimentalReady: true, experimental: true })) + buildNavigationModel(makeCtx({ + experimentalReady: true, + experimental: false, + isPaid: false, + can: (a) => a === 'system:console' || a === 'node:read', + })) .allPageItems.map((i) => i.value), ).toContain('host-console'); + expect( + buildNavigationModel(makeCtx({ + experimentalReady: false, + experimental: false, + can: (a) => a === 'system:console' || a === 'node:read', + })) + .allPageItems.map((i) => i.value), + ).toContain('host-console'); + }); + + it('omits Console without system:console', () => { + expect( + buildNavigationModel(makeCtx({ can: () => false, isAdmin: false })) + .allPageItems.map((i) => i.value), + ).not.toContain('host-console'); }); it('excludes hidden views from quick-link candidates', () => { diff --git a/frontend/src/lib/navigation/buildNavigationModel.ts b/frontend/src/lib/navigation/buildNavigationModel.ts index 82eb59d4..faf73f17 100644 --- a/frontend/src/lib/navigation/buildNavigationModel.ts +++ b/frontend/src/lib/navigation/buildNavigationModel.ts @@ -28,12 +28,6 @@ function isVisuallyDiscoverable(item: AppNavItem, reachCtx: ReachabilityContext) // Settings is always discoverable in the launcher when the operator can open Settings. return true; } - // Console: fail-closed visual discovery until /meta settles and the flag is on. - // URL normalization still uses isViewHidden cold-load deferral separately. - if (item.value === 'host-console') { - if (!reachCtx.experimentalReady || !reachCtx.experimental) return false; - return !isViewHidden(item.value, reachCtx); - } return !isViewHidden(item.value, reachCtx); } diff --git a/frontend/src/lib/router/readUrlRouteState.ts b/frontend/src/lib/router/readUrlRouteState.ts index 4d190751..6a53d161 100644 --- a/frontend/src/lib/router/readUrlRouteState.ts +++ b/frontend/src/lib/router/readUrlRouteState.ts @@ -19,11 +19,21 @@ const DEFAULT: UrlRouteState = { filterNodeId: null, }; -/** True when the current URL is a stack workspace deep link (detail or editor). */ -export function isStackEditorDeepLink(): boolean { +/** True when the URL is a stack-scoped deep link for the given view. */ +function isStackScopedDeepLink(view: ActiveView): boolean { if (typeof window === 'undefined') return false; const parsed = parsePath(window.location.pathname, window.location.search); - return parsed.view === 'editor' && parsed.stackName != null; + return parsed.view === view && parsed.stackName != null; +} + +/** True when the current URL is a stack workspace deep link (detail or editor). */ +export function isStackEditorDeepLink(): boolean { + return isStackScopedDeepLink('editor'); +} + +/** True when the URL targets Host Console rooted in a stack directory. */ +export function isHostConsoleStackDeepLink(): boolean { + return isStackScopedDeepLink('host-console'); } /** Read shell navigation fields from the current browser URL (cold-load bootstrap). */ diff --git a/frontend/src/lib/router/senchoRoute.test.ts b/frontend/src/lib/router/senchoRoute.test.ts index 42a07c6a..f744382b 100644 --- a/frontend/src/lib/router/senchoRoute.test.ts +++ b/frontend/src/lib/router/senchoRoute.test.ts @@ -165,4 +165,21 @@ describe('senchoRoute', () => { expect(parsed.view).toBe('networking'); expect(parsed.nodeSlug).toBe('local'); }); + + it('round-trips Host Console without a stack', () => { + const path = buildPath({ ...base, activeView: 'host-console', stackName: null }); + expect(path).toBe('/nodes/local/host-console'); + const parsed = parsePath(path, ''); + expect(parsed.view).toBe('host-console'); + expect(parsed.nodeSlug).toBe('local'); + expect(parsed.stackName).toBeNull(); + }); + + it('round-trips Host Console rooted in a stack directory', () => { + const path = buildPath({ ...base, activeView: 'host-console', stackName: 'radarr' }); + expect(path).toBe('/nodes/local/host-console/radarr'); + const parsed = parsePath(path, ''); + expect(parsed.view).toBe('host-console'); + expect(parsed.stackName).toBe('radarr'); + }); }); diff --git a/frontend/src/lib/routing/hostConsoleCapability.test.ts b/frontend/src/lib/routing/hostConsoleCapability.test.ts new file mode 100644 index 00000000..f43141f1 --- /dev/null +++ b/frontend/src/lib/routing/hostConsoleCapability.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect } from 'vitest'; +import { resolveHostConsoleCapability } from './hostConsoleCapability'; + +describe('resolveHostConsoleCapability', () => { + it('returns loading when the active node is unresolved', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: false, + isRemote: false, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('loading'); + }); + + it('allows local nodes without waiting for meta', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: false, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('allowed'); + }); + + it('returns loading when remote meta is absent', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: null, + })).toBe('loading'); + }); + + it('allows Community when remote advertises host-console-community', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console', 'host-console-community'] }, + })).toBe('allowed'); + }); + + it('locks Community when remote only has legacy host-console', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('locked'); + }); + + it('allows Admiral when remote only has legacy host-console', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: true, + licenseReady: true, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('allowed'); + }); + + it('returns loading for legacy-only remote while license is not ready', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: false, + activeNodeMeta: { capabilities: ['host-console'] }, + })).toBe('loading'); + }); + + it('allows community-capable remote without waiting on license', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: false, + licenseReady: false, + activeNodeMeta: { capabilities: ['host-console-community'] }, + })).toBe('allowed'); + }); + + it('locks Pilot / empty capability lists', () => { + expect(resolveHostConsoleCapability({ + nodeResolved: true, + isRemote: true, + isPaid: true, + licenseReady: true, + activeNodeMeta: { capabilities: ['stacks', 'fleet'] }, + })).toBe('locked'); + }); +}); diff --git a/frontend/src/lib/routing/hostConsoleCapability.ts b/frontend/src/lib/routing/hostConsoleCapability.ts new file mode 100644 index 00000000..71004993 --- /dev/null +++ b/frontend/src/lib/routing/hostConsoleCapability.ts @@ -0,0 +1,49 @@ +import { + HOST_CONSOLE_CAPABILITY, + HOST_CONSOLE_COMMUNITY_CAPABILITY, +} from '@/lib/capabilities'; + +export type HostConsoleCapabilityState = 'loading' | 'allowed' | 'locked'; + +export interface HostConsoleCapabilityInput { + /** False until NodeContext resolves an active node (cold load). Never treat as local. */ + nodeResolved: boolean; + /** True when the resolved active node is a remote Distributed API Proxy or Pilot node. */ + isRemote: boolean; + /** Hub license: Admiral may accept legacy `host-console` on remotes. */ + isPaid: boolean; + /** + * False while LicenseContext is still loading. Legacy-remote allowance + * must wait so a cold load does not flash LockCard as Community. + */ + licenseReady: boolean; + /** + * Cached `/api/meta` for the active node. Null means metadata has not been + * fetched yet (or the node is unresolved). Must not be confused with + * optimistic `hasCapability()` which returns true while meta is absent. + */ + activeNodeMeta: { capabilities: readonly string[] } | null; +} + +/** + * Whether Host Console content may mount for the active node. + * + * Unresolved nodes stay in `loading`. Local nodes are treated as compatible + * once RBAC passed (same build). Remote nodes wait for metadata, then require + * `host-console-community`, or (Admiral only) legacy `host-console`. + */ +export function resolveHostConsoleCapability( + input: HostConsoleCapabilityInput, +): HostConsoleCapabilityState { + const { nodeResolved, isRemote, isPaid, licenseReady, activeNodeMeta } = input; + if (!nodeResolved) return 'loading'; + if (!isRemote) return 'allowed'; + if (!activeNodeMeta) return 'loading'; + + const caps = activeNodeMeta.capabilities; + if (caps.includes(HOST_CONSOLE_COMMUNITY_CAPABILITY)) return 'allowed'; + if (!caps.includes(HOST_CONSOLE_CAPABILITY)) return 'locked'; + // Legacy host-console only: Admiral hubs may open it; wait for license first. + if (!licenseReady) return 'loading'; + return isPaid ? 'allowed' : 'locked'; +} diff --git a/frontend/src/lib/routing/reachability.test.ts b/frontend/src/lib/routing/reachability.test.ts index 24058316..11362710 100644 --- a/frontend/src/lib/routing/reachability.test.ts +++ b/frontend/src/lib/routing/reachability.test.ts @@ -49,25 +49,25 @@ describe('reachability', () => { expect(isViewHidden('fleet', noFleet)).toBe(true); }); - it('preserves paid views when license metadata failed', () => { - const licenseError = ctx({ licenseStatus: 'error', experimental: true }); + it('preserves host-console when authz is not ready', () => { + const licenseError = ctx({ licenseStatus: 'error', can: (a) => a === 'system:console' }); expect(isViewHidden('host-console', licenseError)).toBe(false); }); - it('does not apply experimental hide to host-console until experimentalReady', () => { - const loading = ctx({ experimental: false, experimentalReady: false, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', loading)).toBe(false); + it('hides host-console without system:console when ready', () => { + const noConsole = ctx({ can: () => false, isPaid: false, experimental: false }); + expect(isViewHidden('host-console', noConsole)).toBe(true); + expect(normalizeHiddenView('host-console', noConsole)).toBe('dashboard'); }); - it('hides host-console when experimental is ready and off even for paid admin', () => { - const off = ctx({ experimental: false, experimentalReady: true, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', off)).toBe(true); - expect(normalizeHiddenView('host-console', off)).toBe('dashboard'); - }); - - it('keeps host-console when experimental is on for paid admin', () => { - const on = ctx({ experimental: true, experimentalReady: true, isPaid: true, isAdmin: true }); - expect(isViewHidden('host-console', on)).toBe(false); + it('keeps host-console for system:console regardless of tier or experimental', () => { + const community = ctx({ + isPaid: false, + experimental: false, + experimentalReady: true, + can: (a) => a === 'system:console', + }); + expect(isViewHidden('host-console', community)).toBe(false); }); it('hides routing and secrets fleet tabs only after experimentalReady when off', () => { diff --git a/frontend/src/lib/routing/reachability.ts b/frontend/src/lib/routing/reachability.ts index df3836eb..0349e9f8 100644 --- a/frontend/src/lib/routing/reachability.ts +++ b/frontend/src/lib/routing/reachability.ts @@ -43,11 +43,7 @@ export function isViewHidden(view: ActiveView, ctx: ReachabilityContext): boolea if (!ctx.isAdmin && (view === 'auto-updates' || view === 'scheduled-ops')) return true; if (!ctx.can('node:read') && view === 'fleet') return true; if (view === 'host-console') { - // Defer experimental hide until ready so enabled deep links survive cold load. - if (experimentalDiscoveryReady(ctx) && !ctx.experimental) return true; - if (!ctx.isPaid) return true; - if (!ctx.isAdmin) return true; - return false; + return !ctx.can('system:console'); } if (!ctx.isPaid) { if (view === 'audit-log') return true; @@ -67,7 +63,7 @@ export function isViewCapabilityLocked(view: ActiveView, ctx: ReachabilityContex export function isFleetTabHidden(tab: FleetTab, ctx: ReachabilityContext): boolean { if (!authzReady(ctx)) return false; if (tab === 'container-labels' && !ctx.containerLabelsEnabled) return true; - // Defer experimental hide until ready (same cold-load contract as host-console). + // Defer experimental hide until ready so deep links survive cold load. if ((tab === 'routing' || tab === 'secrets') && experimentalDiscoveryReady(ctx) && !ctx.experimental) { return true; } diff --git a/package-lock.json b/package-lock.json index 26366d72..d40bb592 100644 --- a/package-lock.json +++ b/package-lock.json @@ -692,9 +692,9 @@ "license": "MIT" }, "node_modules/fast-uri": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", - "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.4.tgz", + "integrity": "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw==", "dev": true, "funding": [ {