diff --git a/backend/src/__tests__/image-updates-routes.test.ts b/backend/src/__tests__/image-updates-routes.test.ts index 572b86e5..83d85d1b 100644 --- a/backend/src/__tests__/image-updates-routes.test.ts +++ b/backend/src/__tests__/image-updates-routes.test.ts @@ -94,6 +94,54 @@ describe('GET /api/image-updates/fleet', () => { }); }); +describe('POST /api/image-updates/fleet/refresh', () => { + it('rejects unauthenticated requests with 401', async () => { + const res = await request(app).post('/api/image-updates/fleet/refresh'); + expect(res.status).toBe(401); + }); + + it('rejects non-admin users with 403', async () => { + const res = await request(app).post('/api/image-updates/fleet/refresh').set('Cookie', viewerCookie); + expect(res.status).toBe(403); + }); + + it('returns triggered/rateLimited/failed arrays for admin caller', async () => { + const res = await request(app).post('/api/image-updates/fleet/refresh').set('Cookie', adminCookie); + expect(res.status).toBe(200); + expect(Array.isArray(res.body.triggered)).toBe(true); + expect(Array.isArray(res.body.rateLimited)).toBe(true); + expect(Array.isArray(res.body.failed)).toBe(true); + // The single local node should land in either triggered (first hit) or + // rateLimited (cooldown from a prior /refresh in this suite). + const localNodeBuckets = res.body.triggered.length + res.body.rateLimited.length; + expect(localNodeBuckets).toBeGreaterThanOrEqual(1); + }); + + it('invalidates the fleet aggregation cache', async () => { + const { CacheService } = await import('../services/CacheService'); + // Prime the cache by hitting the GET endpoint, then refresh, then + // confirm the cache key was wiped. + await request(app).get('/api/image-updates/fleet').set('Cookie', adminCookie); + expect(CacheService.getInstance().get('fleet-updates')).toBeDefined(); + await request(app).post('/api/image-updates/fleet/refresh').set('Cookie', adminCookie); + expect(CacheService.getInstance().get('fleet-updates')).toBeUndefined(); + }); + + it('downgrades to 402-style upgrade response when license is community', async () => { + const { LicenseService } = await import('../services/LicenseService'); + const tierSpy = vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('community'); + try { + const res = await request(app).post('/api/image-updates/fleet/refresh').set('Cookie', adminCookie); + // requirePaid responds with a non-2xx status carrying an upgrade payload. + expect(res.status).not.toBe(200); + expect(res.status).toBeGreaterThanOrEqual(400); + } finally { + tierSpy.mockRestore(); + vi.spyOn(LicenseService.getInstance(), 'getTier').mockReturnValue('paid'); + } + }); +}); + describe('POST /api/auto-update/execute', () => { it('rejects unauthenticated requests with 401', async () => { const res = await request(app).post('/api/auto-update/execute').send({ target: '*' }); diff --git a/backend/src/routes/imageUpdates.ts b/backend/src/routes/imageUpdates.ts index a73804a4..0e70a1e5 100644 --- a/backend/src/routes/imageUpdates.ts +++ b/backend/src/routes/imageUpdates.ts @@ -10,7 +10,7 @@ import { LicenseService } from '../services/LicenseService'; import { NotificationService } from '../services/NotificationService'; import { enforcePolicyPreDeploy } from '../services/PolicyEnforcement'; import { authMiddleware } from '../middleware/auth'; -import { requireAdmin } from '../middleware/tierGates'; +import { requireAdmin, requirePaid } from '../middleware/tierGates'; import { buildPolicyGateOptions } from '../helpers/policyGate'; import { isValidStackName } from '../utils/validation'; import { getErrorMessage } from '../utils/errors'; @@ -110,6 +110,74 @@ imageUpdatesRouter.get('/fleet', authMiddleware, async (_req: Request, res: Resp } }); +imageUpdatesRouter.post('/fleet/refresh', authMiddleware, async (_req: Request, res: Response): Promise => { + if (!requireAdmin(_req, res)) return; + if (!requirePaid(_req, res)) return; + + const db = DatabaseService.getInstance(); + const nodes = db.getNodes(); + const nr = NodeRegistry.getInstance(); + const triggered: number[] = []; + const rateLimited: number[] = []; + const failed: number[] = []; + + // ImageUpdateService is a per-instance singleton, so the local node's manual + // refresh fires at most once per request regardless of how many local rows + // exist in the schema. + const localNode = nodes.find(n => n.type === 'local'); + if (localNode) { + try { + if (ImageUpdateService.getInstance().triggerManualRefresh()) { + triggered.push(localNode.id); + } else { + rateLimited.push(localNode.id); + } + } catch (e) { + console.error(`[ImageUpdates] Local fleet refresh failed for node ${localNode.id}:`, e); + failed.push(localNode.id); + } + } + + const remoteNodes = nodes.filter(n => n.type === 'remote' && n.status === 'online' && n.api_url); + const remoteResults = await Promise.allSettled( + remoteNodes.map(async (node) => { + const proxyTarget = nr.getProxyTarget(node.id); + const baseUrl = node.api_url!.replace(/\/$/, ''); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), REMOTE_NODE_FETCH_TIMEOUT_MS); + try { + const resp = await fetch(`${baseUrl}/api/image-updates/refresh`, { + method: 'POST', + headers: proxyTarget?.apiToken + ? { Authorization: `Bearer ${proxyTarget.apiToken}` } + : {}, + signal: controller.signal, + }); + clearTimeout(timeout); + return { nodeId: node.id, status: resp.status }; + } catch (e) { + clearTimeout(timeout); + return { nodeId: node.id, status: 0, error: e }; + } + }), + ); + + for (const entry of remoteResults) { + if (entry.status !== 'fulfilled') continue; + const { nodeId, status } = entry.value; + if (status >= 200 && status < 300) { + triggered.push(nodeId); + } else if (status === 429) { + rateLimited.push(nodeId); + } else { + failed.push(nodeId); + } + } + + CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); + res.json({ triggered, rateLimited, failed }); +}); + /** * Execute auto-update for a single stack (or for every stack on the local * node when target="*"). This runs on whichever Sencho instance receives @@ -233,6 +301,7 @@ autoUpdateRouter.post('/execute', authMiddleware, async (req: Request, res: Resp } } + CacheService.getInstance().invalidate(FLEET_UPDATE_CACHE_KEY); res.json({ result: results.join('\n') }); } catch (error) { const msg = getErrorMessage(error, 'Auto-update execution failed'); diff --git a/docs/features/auto-update-policies.mdx b/docs/features/auto-update-policies.mdx index 349b049e..24746b63 100644 --- a/docs/features/auto-update-policies.mdx +++ b/docs/features/auto-update-policies.mdx @@ -23,7 +23,9 @@ Each card shows: Readiness board with hero count, per-stack cards, and risk tags -The hero at the top counts pending updates fleet-wide and tells you how many of them are ready to apply without human review. Major version jumps and stacks with blocked registries are counted separately so they can be reviewed before the scheduler runs. +The hero at the top counts pending updates across every node in your fleet and tells you how many of them are ready to apply without human review. Major version jumps and stacks with blocked registries are counted separately so they can be reviewed before the scheduler runs. + +Cards are grouped by node, with a section header for each node that has at least one pending update. The local node is listed first, followed by remote nodes alphabetically. If any of your online nodes is unreachable when the page loads, a small line under the hero shows how many of your online nodes responded. ## Workflow @@ -31,7 +33,7 @@ The hero at the top counts pending updates fleet-wide and tells you how many of 2. Skim the card grid. Patch and minor bumps render with a green or amber tag; major bumps render in red and are marked as blocked. 3. For a safe update, click **Apply now** on the card to pull and recreate the stack immediately. 4. For a major bump, review the changelog preview and the rollback target before deciding. If you still want to apply it, switch to the **Schedules** view and create or edit an auto-update task for that stack. -5. Use **Recheck all** in the hero to force an immediate registry poll if you want to bypass the cached update status. +5. Use **Recheck** in the hero to force an immediate registry poll across every reachable node. Per-node cooldowns still apply, and the toast tells you how many nodes were triggered, rate-limited, or failed. ## Risk tags @@ -57,7 +59,9 @@ The task lives alongside restart, prune, snapshot, and scan tasks in the same ti ## Multi-node support -The readiness board scopes to the active node selected in the sidebar, matching the scope of the auto-update schedule attached to each card. When a remote node is selected, Sencho proxies the registry checks and the apply call to the remote Sencho instance via the Distributed API. No additional configuration is needed. +The readiness board shows pending updates from every node in your fleet in a single view, regardless of which node is selected in the sidebar. You do not need to switch nodes to inspect what is pending elsewhere. + +Each node group renders its own card grid. **Apply now** runs on the node that owns the stack, and **Recheck** fans out to every reachable node so registries get polled in parallel. Sencho handles the routing through the Distributed API; no additional configuration is needed. ## How readiness is computed @@ -83,8 +87,12 @@ The stack has a major version bump. Open the cron schedule for that stack and ap ### Card stays stuck on "Checking" -The registry call is either still pending or failed. Click **Recheck all** in the hero to retry. If the stack has private-registry credentials, confirm they are still valid in **Settings > Registries**. +The registry call is either still pending or failed. Click **Recheck** in the hero to retry. If the stack has private-registry credentials, confirm they are still valid in **Settings > Registries**. ### "Nothing to update" but I see an update on another view -Image update detection runs every six hours. The readiness board uses the same cached status. Trigger **Recheck all** to force a fresh check, or see [Image update detection](/features/image-update-detection) for details on the refresh cycle. +Image update detection runs every six hours on each node. The readiness board uses the same cached status. Trigger **Recheck** to force a fresh check across every reachable node, or see [Image update detection](/features/image-update-detection) for details on the refresh cycle. + +### Banner says "X of Y nodes reachable" + +One or more nodes that are marked online in your fleet did not respond within the request timeout. Pending updates from those nodes are not shown until they come back. Check the node's status from the Fleet view and the network path between this Sencho instance and the unreachable node. diff --git a/docs/images/auto-update/readiness-board.png b/docs/images/auto-update/readiness-board.png index 786fe4c0..87496505 100644 Binary files a/docs/images/auto-update/readiness-board.png and b/docs/images/auto-update/readiness-board.png differ diff --git a/frontend/src/components/AutoUpdateReadinessView.tsx b/frontend/src/components/AutoUpdateReadinessView.tsx index 2c65760b..e67a5771 100644 --- a/frontend/src/components/AutoUpdateReadinessView.tsx +++ b/frontend/src/components/AutoUpdateReadinessView.tsx @@ -1,9 +1,10 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { Button } from '@/components/ui/button'; import { Card } from '@/components/ui/card'; -import { RefreshCw, Shield, AlertTriangle, ShieldAlert, Clock, Play, CalendarClock } from 'lucide-react'; +import { Badge } from '@/components/ui/badge'; +import { RefreshCw, Shield, AlertTriangle, ShieldAlert, Clock, Play, CalendarClock, Monitor, Globe } from 'lucide-react'; import { toast } from '@/components/ui/toast-store'; -import { apiFetch } from '@/lib/api'; +import { apiFetch, fetchForNode } from '@/lib/api'; import { PaidGate } from '@/components/PaidGate'; import { useNodes } from '@/context/NodeContext'; import type { ScheduledTask } from '@/types/scheduling'; @@ -40,12 +41,24 @@ interface UpdatePreview { interface StackCard { stack: string; + nodeId: number; preview: UpdatePreview | null; previewLoaded: boolean; scheduledTask: ScheduledTask | null; applying: boolean; } +interface NodeGroup { + nodeId: number; + nodeName: string; + nodeType: 'local' | 'remote'; + cards: StackCard[]; +} + +interface FleetUpdateResponse { + [nodeId: string]: Record; +} + function formatRelative(ts: number | null): string { if (ts == null) return ''; const delta = ts - Date.now(); @@ -127,9 +140,9 @@ function StackReadinessCard({ onApply, }: { card: StackCard; - onApply: (stack: string) => void; + onApply: (stack: string, nodeId: number) => void; }) { - const { stack, preview, previewLoaded, scheduledTask, applying } = card; + const { stack, nodeId, preview, previewLoaded, scheduledTask, applying } = card; const loading = !previewLoaded; const failed = previewLoaded && preview === null; const blocked = preview?.summary.blocked ?? false; @@ -213,7 +226,7 @@ function StackReadinessCard({