From 448a64a10deb022a7c96c10fac7463bd83e231e6 Mon Sep 17 00:00:00 2001 From: SaelixCode Date: Mon, 9 Mar 2026 14:02:10 -0400 Subject: [PATCH] feat: implement enterprise sse global logs and developer mode --- CHANGELOG.md | 3 + backend/src/index.ts | 82 ++++++++++++++ backend/src/services/DatabaseService.ts | 2 + .../components/GlobalObservabilityView.tsx | 102 +++++++++++++++--- frontend/src/components/SettingsModal.tsx | 66 +++++++++++- 5 files changed, 236 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b7021e..cf025b6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- **Added:** Developer Mode setting to enable true Real-Time (SSE) global log streaming and infinite scroll. +- **Added:** Configurable polling rates for standard global logs monitoring. +- **Added:** React Throttle Buffer to prevent UI freezing during heavy real-time log ingestion. - **Fixed:** Global Logs aggressive auto-scrolling preventing users from reading log history. - **Fixed:** Quiet stacks missing from the Global Logs filter dropdown by fetching the definitive stack list independently. - **Fixed:** Global logs misclassifying INFO messages as errors due to naive string matching. diff --git a/backend/src/index.ts b/backend/src/index.ts index b9019dc6..05487d61 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -852,6 +852,88 @@ app.get('/api/logs/global', async (req: Request, res: Response) => { } }); +app.get('/api/logs/global/stream', async (req: Request, res: Response) => { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders(); + + const dockerController = DockerController.getInstance(); + const streams: NodeJS.ReadableStream[] = []; + + try { + const containers = await dockerController.getRunningContainers(); + + await Promise.all(containers.map(async (c) => { + const stackName = c.Labels?.['com.docker.compose.project'] || 'system'; + let rawName = c.Names?.[0]?.replace(/^\//, '') || c.Id.substring(0, 12); + let containerName = rawName; + if (rawName.startsWith(`${stackName}-`)) containerName = rawName.replace(`${stackName}-`, '').replace(/-1$/, ''); + else if (rawName.startsWith(`${stackName}_`)) containerName = rawName.replace(`${stackName}_`, '').replace(/_1$/, ''); + + try { + const container = dockerController.getDocker().getContainer(c.Id); + const inspect = await container.inspect(); + const isTty = inspect.Config.Tty; + + // Dev mode gets a larger tail + const stream = await container.logs({ follow: true, stdout: true, stderr: true, tail: 500, timestamps: true }); + streams.push(stream); + + const processLine = (line: string, source: string) => { + if (!line.trim()) return; + const timeMatch = line.match(/^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(.*)/); + let cleanMessage = line; + let timestampMs = Date.now(); + + if (timeMatch) { + timestampMs = new Date(timeMatch[1]).getTime(); + cleanMessage = timeMatch[2]; + } + + let level = source === 'STDERR' ? 'ERROR' : 'INFO'; + if (/\[\s*(info|inf|debug|dbg)\s*\]/i.test(cleanMessage) || /^(info|debug)\b/i.test(cleanMessage)) level = 'INFO'; + else if (/\b(warn|warning)\b/i.test(cleanMessage)) level = 'WARN'; + else if (/\b(error|err|fatal|exception)\b/i.test(cleanMessage)) level = 'ERROR'; + + res.write(`data: ${JSON.stringify({ stackName, containerName, source, level, message: cleanMessage, timestampMs })}\n\n`); + }; + + stream.on('data', (chunk: Buffer) => { + if (isTty) { + const payload = chunk.toString('utf-8').replace(/[\u0000-\u0008\u000B-\u001F\u007F-\u009F]/g, ""); + payload.split('\n').forEach(line => processLine(line, 'STDOUT')); + } else { + let offset = 0; + while (offset < chunk.length) { + if (offset + 8 > chunk.length) break; + const streamType = chunk[offset]; + const length = chunk.readUInt32BE(offset + 4); + offset += 8; + if (offset + length > chunk.length) break; + + const payload = chunk.slice(offset, offset + length).toString('utf-8'); + offset += length; + payload.split('\n').forEach(line => processLine(line, streamType === 2 ? 'STDERR' : 'STDOUT')); + } + } + }); + } catch (err) { /* ignore */ } + })); + + // Cleanup when client closes the tab or switches views + req.on('close', () => { + streams.forEach(s => { + try { (s as any).destroy(); } catch (e) { } + }); + }); + + } catch (error) { + res.write(`data: ${JSON.stringify({ level: 'ERROR', message: '[Sencho] Failed to attach global log stream.', timestampMs: Date.now(), stackName: 'system', containerName: 'backend', source: 'STDERR' })}\n\n`); + res.end(); + } +}); + // Get host system stats app.get('/api/system/stats', async (req: Request, res: Response) => { try { diff --git a/backend/src/services/DatabaseService.ts b/backend/src/services/DatabaseService.ts index e6b01535..cae947e9 100644 --- a/backend/src/services/DatabaseService.ts +++ b/backend/src/services/DatabaseService.ts @@ -119,6 +119,8 @@ export class DatabaseService { stmt.run('host_disk_limit', '90'); stmt.run('global_crash', '1'); stmt.run('docker_janitor_gb', '5'); + stmt.run('global_logs_refresh', '5'); // Default 5 seconds + stmt.run('developer_mode', '0'); // Default off } private migrateJsonConfig(dataDir: string) { diff --git a/frontend/src/components/GlobalObservabilityView.tsx b/frontend/src/components/GlobalObservabilityView.tsx index 45148011..3f860b4c 100644 --- a/frontend/src/components/GlobalObservabilityView.tsx +++ b/frontend/src/components/GlobalObservabilityView.tsx @@ -4,6 +4,7 @@ import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { DropdownMenu, DropdownMenuCheckboxItem, DropdownMenuContent, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'; import { RefreshCw, Download, Trash2, Search, Filter } from 'lucide-react'; +import { apiFetch } from '@/lib/api'; interface LogEntry { @@ -20,6 +21,10 @@ export function GlobalObservabilityView() { const [loading, setLoading] = useState(true); const [allStacks, setAllStacks] = useState([]); + // Settings state + const [devMode, setDevMode] = useState(false); + const [pollRate, setPollRate] = useState(5); + // Filters const [searchQuery, setSearchQuery] = useState(''); const [selectedStacks, setSelectedStacks] = useState([]); @@ -28,24 +33,24 @@ export function GlobalObservabilityView() { const bottomRef = useRef(null); const [isAutoScrollEnabled, setIsAutoScrollEnabled] = useState(true); - const fetchData = async () => { - setLoading(true); - try { - const logsRes = await fetch('/api/logs/global'); - if (logsRes.ok) { - setLogs(await logsRes.json()); - } - } catch (error) { - console.error('Failed to fetch global logs:', error); - } finally { - setLoading(false); - } - }; + // SSE throttle buffer + const bufferRef = useRef([]); + // Fetch settings on mount useEffect(() => { - fetchData(); - const interval = setInterval(fetchData, 10000); // Poll every 10s - return () => clearInterval(interval); + const fetchSettings = async () => { + try { + const res = await apiFetch('/settings'); + if (res.ok) { + const data = await res.json(); + setDevMode(data.developer_mode === '1'); + setPollRate(parseInt(data.global_logs_refresh || '5', 10)); + } + } catch (e) { + console.error('Failed to fetch settings:', e); + } + }; + fetchSettings(); }, []); // Fetch definitive stack list from the filesystem, independent of log data @@ -64,6 +69,65 @@ export function GlobalObservabilityView() { fetchStacks(); }, []); + // Data fetching: Polling (standard) vs SSE (dev mode) + useEffect(() => { + if (devMode) { + // SSE mode + const eventSource = new EventSource('/api/logs/global/stream'); + + eventSource.onmessage = (event) => { + try { + const entry: LogEntry = JSON.parse(event.data); + bufferRef.current.push(entry); + } catch (e) { /* ignore parse errors */ } + }; + + eventSource.onerror = () => { + // SSE will auto-reconnect, no action needed + }; + + // 500ms throttle: flush buffer into React state + const flushInterval = setInterval(() => { + if (bufferRef.current.length > 0) { + const batch = bufferRef.current.splice(0); + setLogs(prev => { + const merged = [...prev, ...batch]; + // Infinite scroll: no slice limit in dev mode + merged.sort((a, b) => a.timestampMs - b.timestampMs); + return merged; + }); + } + }, 500); + + setLoading(false); + + return () => { + eventSource.close(); + clearInterval(flushInterval); + bufferRef.current = []; + }; + } else { + // Standard polling mode + const fetchData = async () => { + setLoading(true); + try { + const logsRes = await fetch('/api/logs/global'); + if (logsRes.ok) { + setLogs(await logsRes.json()); + } + } catch (error) { + console.error('Failed to fetch global logs:', error); + } finally { + setLoading(false); + } + }; + + fetchData(); + const interval = setInterval(fetchData, pollRate * 1000); + return () => clearInterval(interval); + } + }, [devMode, pollRate]); + const handleStackToggle = (stack: string) => { setSelectedStacks(prev => prev.includes(stack) ? prev.filter(s => s !== stack) : [...prev, stack] @@ -168,6 +232,12 @@ export function GlobalObservabilityView() { + + {devMode && ( +
+ ● LIVE +
+ )} {loading && logs.length === 0 && ( diff --git a/frontend/src/components/SettingsModal.tsx b/frontend/src/components/SettingsModal.tsx index f4407cc3..5100d6af 100644 --- a/frontend/src/components/SettingsModal.tsx +++ b/frontend/src/components/SettingsModal.tsx @@ -11,7 +11,8 @@ import { Label } from '@/components/ui/label'; import { Slider } from '@/components/ui/slider'; import { toast } from 'sonner'; import { apiFetch } from '@/lib/api'; -import { Shield, Activity, Bell, Palette, Moon, Sun } from 'lucide-react'; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select'; +import { Shield, Activity, Bell, Palette, Moon, Sun, Code } from 'lucide-react'; interface Agent { type: 'discord' | 'slack' | 'webhook'; @@ -27,7 +28,7 @@ interface SettingsModalProps { } export function SettingsModal({ isOpen, onClose, isDarkMode, setIsDarkMode }: SettingsModalProps) { - const [activeSection, setActiveSection] = useState<'account' | 'system' | 'notifications' | 'appearance'>('account'); + const [activeSection, setActiveSection] = useState<'account' | 'system' | 'notifications' | 'appearance' | 'developer'>('account'); // Auth State const [authData, setAuthData] = useState({ oldPassword: '', newPassword: '', confirmPassword: '' }); @@ -45,7 +46,9 @@ export function SettingsModal({ isOpen, onClose, isDarkMode, setIsDarkMode }: Se host_ram_limit: '90', host_disk_limit: '90', global_crash: '1', - docker_janitor_gb: '5' + docker_janitor_gb: '5', + global_logs_refresh: '5', + developer_mode: '0' }); const [isLoading, setIsLoading] = useState(false); @@ -255,6 +258,14 @@ export function SettingsModal({ isOpen, onClose, isDarkMode, setIsDarkMode }: Se Appearance + @@ -418,6 +429,55 @@ export function SettingsModal({ isOpen, onClose, isDarkMode, setIsDarkMode }: Se )} + {activeSection === 'developer' && ( +
+
+

Developer

+

Power user settings for real-time observability and extended diagnostics.

+
+ +
+
+
+ +

Enable Real-Time Metrics & Extended Logs

+
+ handleSettingChange('developer_mode', c ? '1' : '0')} + /> +
+ +
+ + + {settings.developer_mode === '1' && ( +

SSE streaming is active — polling rate is overridden by real-time streaming.

+ )} +
+
+ +
+ +
+
+ )} +