From 3bc613c9157d1c6769ca25a0a5c3eec9c6694887 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sun, 23 Aug 2026 15:09:32 +0100 Subject: [PATCH] Scale large-estate workload and Proxmox demo performance --- .github/workflows/update-demo-server.yml | 2 +- .../subsystems/deployment-installability.md | 4 +- .../subsystems/frontend-primitives.md | 10 + .../v6/internal/subsystems/monitoring.md | 4 +- .../subsystems/performance-and-scalability.md | 4 +- .../internal/subsystems/storage-recovery.md | 7 + frontend-modern/browser-verification.json | 50 ++-- ...loadsSurface.performance.contract.test.tsx | 10 +- .../useGroupedTableWindowing.test.ts | 10 +- .../Workloads/useGroupedTableWindowing.ts | 4 +- .../components/Workloads/useWorkloadsState.ts | 27 +- .../features/proxmox/ProxmoxPageSurface.tsx | 9 + .../__tests__/ProxmoxBackupsTable.test.tsx | 11 + .../src/hooks/__tests__/useWorkloads.test.ts | 40 +++ frontend-modern/src/hooks/useWorkloads.ts | 246 ++++++++++++------ internal/mock/demo_scenarios.go | 3 + internal/mock/demo_scenarios_test.go | 6 +- internal/mock/generator.go | 8 +- .../installtests/build_release_assets_test.go | 2 +- scripts/tests/test-toggle-mock.sh | 2 +- scripts/toggle-mock.sh | 2 +- 21 files changed, 318 insertions(+), 143 deletions(-) diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index 17a51126a..6fdb07e73 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -616,7 +616,7 @@ jobs: set_env_value DEMO_MODE true set_env_value PULSE_MOCK_MODE true - set_env_value PULSE_MOCK_NODES 32 + set_env_value PULSE_MOCK_NODES 50 set_env_value PULSE_MOCK_VMS_PER_NODE 10 set_env_value PULSE_MOCK_LXCS_PER_NODE 8 set_env_value PULSE_MOCK_DOCKER_HOSTS 5 diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 540aa7659..eddfcaa8a 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -2616,8 +2616,8 @@ platform-first pages exercise multi-cluster grouping and the production workload-windowing path on first boot. `mock_default_entries()` in `scripts/toggle-mock.sh`, the public demo deployment environment in `.github/workflows/update-demo-server.yml`, and `internal/mock.DefaultConfig` -carry the same baseline: 32 Proxmox nodes with 10 VMs and 8 LXCs each, arranged -as five six-node clusters plus two standalone nodes; 5 Docker hosts with 14 containers +carry the same baseline: 50 Proxmox nodes with 10 VMs and 8 LXCs each, arranged +as eight six-node clusters plus two standalone nodes; 5 Docker hosts with 14 containers each, 4 standalone Pulse-managed hosts, and 3 Kubernetes clusters (production + staging + edge) with 5 nodes, 40 pods, and 14 deployments each. Bumping any owner requires bumping the others (and diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 4db71b1cb..1d1080d23 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -51,6 +51,16 @@ the workload state, table, panel, and row contracts carry the selected basis and resolved parent-node data to the canonical memory bar, and the memory column header must expose the non-default Host basis after the control closes. +Large-estate platform pages must keep one canonical inventory snapshot for the +initial read and explicit refresh path. The Proxmox overview owns its unified +resource request and passes that snapshot into the shared workloads state; the +workloads adapter may map the snapshot into the legacy guest boundary, but must +not issue a second workload inventory request or create a second infrastructure +poll. Refreshing the overview must invalidate that owner snapshot and update +both the node and guest regions from the same result, so a large estate cannot +render contradictory counts or pay duplicate transport and reconciliation +costs. + Shared workload, node, Docker-host, and resource-drawer history presentation keeps current readings separate from stored samples. A current metric may populate the legend while history is still being collected, but it must never diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index 1a60bbd26..14541ceb0 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1806,8 +1806,8 @@ constant) are also part of that mock-runtime contract. The Proxmox default is an intentionally large public-demo estate so platform-first pages exercise multi-cluster navigation, table density, sorting, grouping, drawer behavior, responsive layout, and the production workload-windowing threshold out of the -box: five named six-node Proxmox clusters plus two standalone nodes, with 10 -VMs and 8 LXCs per node (576 guests total), 5 Docker/Podman hosts with 14 containers +box: eight named six-node Proxmox clusters plus two standalone nodes, with 10 +VMs and 8 LXCs per node (900 guests total), 5 Docker/Podman hosts with 14 containers each, 4 standalone Pulse-managed hosts, and 3 Kubernetes clusters (Production EU + Staging EU + Development EU; a fourth Edge / k3s profile is curated in `demo_scenarios.go` and instantiates when diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index aaac53244..f5b152b94 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -1752,9 +1752,9 @@ lifecycle. Future viewport sync changes must extend through that hook rather than rebuilding browser-event wiring or table-body geometry reads inside `frontend-modern/src/components/Workloads/useWorkloadsDerivedState.ts`. The canonical public demo intentionally exercises this production hot path: -its default Proxmox estate contains 576 guests, crossing the 500-row windowing +its default Proxmox estate contains 900 guests, crossing the 250-row windowing threshold while retaining the hook's 140-row mounted-window budget. The demo -curation integration proof must assert both the 500-plus guest total and the +curation integration proof must assert both the 800-plus guest total and the bounded mounted-row count. That same default estate produces more than 500 backup coverage targets and recoverable artifacts; the coverage and By date tables must route through `useProxmoxBackupTableWindowing.ts`, retain full-set diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index a25c693f5..789915a40 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -35,6 +35,13 @@ Own the storage and recovery product surfaces, recovery-point persistence and querying, and the operator-facing storage health presentation layer while keeping adjacent commercial reporting APIs out of storage/recovery product state. +The Proxmox overview is also a large-estate read-side consumer: it owns one +canonical unified-resource snapshot for its node and guest regions, and its +shared workloads adapter must consume that snapshot without starting a second +workload or infrastructure inventory request. Backup and recovery tables keep +their domain-specific evidence requests, while an overview refresh must update +all overview regions from the same owner snapshot rather than mixing reads +from different inventory generations. The physical-disks surface consumes every page in the server-declared unified resource result, scopes same-named Proxmox nodes by instance, and never hides a disk behind a client-side page ceiling. Search includes vendor, WWN, transport, diff --git a/frontend-modern/browser-verification.json b/frontend-modern/browser-verification.json index b801fb492..779a1f304 100644 --- a/frontend-modern/browser-verification.json +++ b/frontend-modern/browser-verification.json @@ -1,31 +1,27 @@ { "version": 1, - "base_sha": "aba7cec747fd008f6053c106af1756b285762380", - "verified_at": "2026-08-23T13:37:06Z", + "base_sha": "6869612c669ff640f52becfc3f66ce66415195c1", + "verified_at": "2026-08-23T14:05:18Z", "result": "passed", "changed_paths": [ - "frontend-modern/src/api/notifications.ts", - "frontend-modern/src/features/alerts/AlertAppriseDestinationsSection.tsx", - "frontend-modern/src/features/alerts/alertDestinationsModel.ts", - "frontend-modern/src/features/alerts/helpers.ts", - "frontend-modern/src/features/alerts/types.ts", - "frontend-modern/src/utils/alertDestinationsPresentation.ts" + "frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx", + "frontend-modern/src/hooks/useWorkloads.ts" ], "content_sha256": { - "frontend-modern/src/api/notifications.ts": "facf96ad57bf8839e6138293455788b3ebb5fe3119bf13427cfe77abce3ff257", - "frontend-modern/src/features/alerts/AlertAppriseDestinationsSection.tsx": "77776a85389857efc7e87be3111b88bbe7fdd9e373c8c0ba658fd75d32c83cb9", - "frontend-modern/src/features/alerts/alertDestinationsModel.ts": "988f1e43f9e27dc2713abdb6c460713ef561c3490e6137ed91fb3c3ef56cb359", - "frontend-modern/src/features/alerts/helpers.ts": "c1ba8c32f3f73ab4ea3c6d1f2a657f20cf253c57fc24102e552909727b817268", - "frontend-modern/src/features/alerts/types.ts": "99f859d5008771c048f964684c4efbcad57543b91dba23f828e006db84ed982e", - "frontend-modern/src/utils/alertDestinationsPresentation.ts": "6c15b3a2c5daa4d5207bb8ed41bb9614bf7d5299f45cce8b814a7aa8efe88f6b" + "frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts": "861e07273c9e7aa30a565889903224df596dcddd20b988b40c79c4ee108fa0a7", + "frontend-modern/src/components/Workloads/useWorkloadsState.ts": "85182febda02e646fa354c6efd973222ef64360efaebc8b7b51c4e80821daa5b", + "frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx": "66dcc020434ee4ea4622fc1007c96eeb0b16432058ec1ade459c78530d26a75e", + "frontend-modern/src/hooks/useWorkloads.ts": "6ca20e750675f7016617366dfb23c3cfbe42394046fded71702903f0368f004c" }, "routes": [ - "/alerts/notifications" + "/proxmox/overview" ], "viewports": [ { - "width": 1280, - "height": 720 + "width": 2560, + "height": 1199 }, { "width": 375, @@ -33,19 +29,15 @@ } ], "states": [ - "Apprise HTTP mode with no saved API key shows the 'Optional API key' placeholder", - "After saving a key the PUT response body carried hasApiKey:true and no apiKey field; the input re-rendered blank with the 'Saved. Leave blank to keep the current key' placeholder and autocomplete=new-password", - "A second save with the key field left blank preserved the stored key server-side (backend parse log reported hasApiKey=true while the request carried an empty key)", - "Fresh page reload re-derived the saved-key placeholder purely from the redacted GET response (hasApiKey:true, no apiKey field in the body)", - "Mobile 375x812 layout renders the Apprise section with the saved-key placeholder visible and no horizontal overflow (body scrollWidth 375); console error-free at both widths" + "Fresh authenticated Proxmox overview renders separate Nodes and Guests regions with the guest count and workload table present", + "Desktop inspection shows the Proxmox guest region populated and the page remains within the desktop viewport width", + "Phone-width 375x812 inspection keeps the Guests heading, filters, and guest table visible with body scrollWidth equal to the 375px viewport", + "Fresh desktop and phone loads reported no browser console errors" ], "interactions": [ - "Logged in against a sidecar-built backend on :17655 with mock data via a second vite instance on :5488", - "Navigated to /alerts/notifications and scrolled to Apprise notifications", - "Switched delivery mode from Local Apprise CLI to Remote Apprise API", - "Entered server URL http://apprise-api.internal:8000 and API key verify-secret-key-123, clicked Save Changes, inspected the PUT /api/notifications/apprise response body", - "Changed the config key to 'roundtrip' with the API key field left blank, saved again, confirmed the stored key survived", - "Reloaded the page, fetched GET /api/notifications/apprise from the authenticated session, verified the redacted body", - "Resized to 375x812, reloaded, scrolled to the Apprise section and re-checked field state and overflow" + "Opened the local demo entrypoint and signed in to the authenticated demo session", + "Navigated to /proxmox/overview and inspected the Nodes and Guests table regions", + "Applied a 375x812 viewport override, reloaded the Proxmox overview, and rechecked guest visibility and horizontal overflow", + "Reset the temporary viewport override and rechecked the fresh tab console" ] } diff --git a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx index c1d4f9423..fdd6ec82c 100644 --- a/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx +++ b/frontend-modern/src/components/Workloads/__tests__/WorkloadsSurface.performance.contract.test.tsx @@ -948,7 +948,7 @@ describe('Workloads performance contract', () => { ); }); - it('Profile S: renders all guest rows without windowing', async () => { + it('Profile S: keeps a medium estate virtualized', async () => { mockLocationSearch = '?type=all'; mockWorkloads = makeGuests(PROFILES.S); @@ -957,7 +957,9 @@ describe('Workloads performance contract', () => { )); await waitFor(() => { - expect(getGuestRowCount(container)).toBe(PROFILES.S); + const rowCount = getGuestRowCount(container); + expect(rowCount).toBeGreaterThan(0); + expect(rowCount).toBeLessThanOrEqual(140); }); }); @@ -1010,7 +1012,9 @@ describe('Workloads performance contract', () => { )); await waitFor(() => { - expect(getGuestRowCount(container)).toBe(expectedByMode[mode]); + expect(getGuestRowCount(container)).toBe( + expectedByMode[mode] > 250 ? 140 : expectedByMode[mode], + ); }); unmount(); diff --git a/frontend-modern/src/components/Workloads/__tests__/useGroupedTableWindowing.test.ts b/frontend-modern/src/components/Workloads/__tests__/useGroupedTableWindowing.test.ts index 3ca2c4cc3..9910437f4 100644 --- a/frontend-modern/src/components/Workloads/__tests__/useGroupedTableWindowing.test.ts +++ b/frontend-modern/src/components/Workloads/__tests__/useGroupedTableWindowing.test.ts @@ -48,18 +48,18 @@ describe('useGroupedTableWindowing', () => { // isWindowed // ────────────────────────────────────────────────────────────── describe('isWindowed', () => { - it('returns false when total rows are below default threshold (500)', () => { + it('returns false when total rows are below default threshold (250)', () => { const hook = setup({ totalRowCount: () => 200 }); expect(hook.isWindowed()).toBe(false); }); it('returns true when total rows exceed default threshold', () => { - const hook = setup({ totalRowCount: () => 600 }); + const hook = setup({ totalRowCount: () => 300 }); expect(hook.isWindowed()).toBe(true); }); it('returns false when total rows equal threshold (not exceeded)', () => { - const hook = setup({ totalRowCount: () => 500 }); + const hook = setup({ totalRowCount: () => 250 }); expect(hook.isWindowed()).toBe(false); }); @@ -141,8 +141,8 @@ describe('useGroupedTableWindowing', () => { // ────────────────────────────────────────────────────────────── describe('mountedCount', () => { it('equals total when not windowed', () => { - const hook = setup({ totalRowCount: () => 300 }); - expect(hook.mountedCount()).toBe(300); + const hook = setup({ totalRowCount: () => 200 }); + expect(hook.mountedCount()).toBe(200); }); it('equals window size when windowed and total > windowSize', () => { diff --git a/frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts b/frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts index 2bc9eedc3..655eb8e53 100644 --- a/frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts +++ b/frontend-modern/src/components/Workloads/useGroupedTableWindowing.ts @@ -34,7 +34,9 @@ export interface UseGroupedTableWindowingResult { } const DEFAULT_WINDOW_SIZE = 140; -const DEFAULT_ENABLE_THRESHOLD = 500; +// Keep even medium estates virtualized. On mobile, mounting a few hundred +// metric-heavy rows is already enough to cause long layout and paint tasks. +const DEFAULT_ENABLE_THRESHOLD = 250; const DEFAULT_OVERSCAN_ROWS = 20; export const useGroupedTableWindowing = ( diff --git a/frontend-modern/src/components/Workloads/useWorkloadsState.ts b/frontend-modern/src/components/Workloads/useWorkloadsState.ts index 95aa94347..027bd22c6 100644 --- a/frontend-modern/src/components/Workloads/useWorkloadsState.ts +++ b/frontend-modern/src/components/Workloads/useWorkloadsState.ts @@ -92,6 +92,11 @@ export interface WorkloadsSurfaceProps { // this surface), set `suppressFilterToolbar` so the surface skips its // internal filter row and avoids a duplicate. suppressFilterToolbar?: boolean; + // An owning platform page may provide the canonical unified-resource + // snapshot it already fetched. This avoids a second workload/infrastructure + // request and keeps both surfaces on the same refresh generation. + resourceSnapshot?: Accessor; + resourceSnapshotRefetch?: () => Promise; statusModeStorageScope?: string; // Platform pages that render their own hosts table above the embedded // workloads surface (e.g. Proxmox overview) own the per-host CPU / Memory @@ -151,12 +156,23 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { const { guestMetadata, handleCustomUrlUpdate } = useWorkloadGuestMetadataState(); const workloadsEnabled = createMemo(() => props.useWorkloads === true); - const workloads = useWorkloads(workloadsEnabled); + const workloads = useWorkloads(workloadsEnabled, { + resourceSnapshot: props.resourceSnapshot, + refetchSnapshot: props.resourceSnapshotRefetch, + }); const infrastructureSources = useUnifiedResources({ query: WORKLOADS_INFRASTRUCTURE_SOURCES_QUERY, cacheKey: 'workloads-infrastructure-sources', - enabled: workloadsEnabled, + enabled: () => workloadsEnabled() && !props.resourceSnapshot, }); + const infrastructureResources = createMemo(() => + props.resourceSnapshot ? (props.resourceSnapshot() ?? []) : infrastructureSources.resources(), + ); + const infrastructureLoading = createMemo(() => + props.resourceSnapshot + ? props.resourceSnapshot() === undefined + : infrastructureSources.loading(), + ); const inventorySourcesResourceKey = createMemo(() => workloadsEnabled() && !props.inventorySourcesQuery ? 'enabled' : null, ); @@ -322,8 +338,7 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { props.nodes.forEach((node) => merged.set(node.id, node)); if (workloadsEnabled()) { - infrastructureSources - .resources() + infrastructureResources() .filter(isProxmoxNodeResource) .map(nodeFromResource) .filter((node): node is Node => Boolean(node)) @@ -363,11 +378,11 @@ export function useWorkloadsState(props: WorkloadsSurfaceProps) { const hasWorkloadsData = createMemo(() => allGuests().length > 0); const hasInfrastructureSources = createMemo(() => workloadsEnabled() - ? infrastructureNodes().length > 0 || infrastructureSources.resources().length > 0 + ? infrastructureNodes().length > 0 || infrastructureResources().length > 0 : infrastructureNodes().length > 0, ); const infrastructureSourceStateReady = createMemo(() => - workloadsEnabled() ? hasInfrastructureSources() || !infrastructureSources.loading() : true, + workloadsEnabled() ? hasInfrastructureSources() || !infrastructureLoading() : true, ); const surfaceConnected = createMemo(() => workloadsEnabled() diff --git a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx index 0bca176a2..e0093d64e 100644 --- a/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx +++ b/frontend-modern/src/features/proxmox/ProxmoxPageSurface.tsx @@ -47,6 +47,7 @@ import { ProxmoxMailGatewayTable } from './ProxmoxMailGatewayTable'; import { ProxmoxNodesTable } from './ProxmoxNodesTable'; import { ProxmoxReplicationTable, fetchReplicationJobs } from './ProxmoxReplicationTable'; import { useUnifiedResources } from '@/hooks/useUnifiedResources'; +import type { Resource } from '@/types/resource'; import { updateStore } from '@/stores/updates'; import { PROXMOX_TAB_SPECS, @@ -220,6 +221,10 @@ export function ProxmoxPageSurface() { setMetricHistoryRange={setMetricHistoryRange} memoryDisplayBasis={memoryDisplayBasis} setMemoryDisplayBasis={setMemoryDisplayBasis} + resourceSnapshot={() => + loading() && model().resources.length === 0 ? undefined : model().resources + } + resourceSnapshotRefetch={() => refetch()} inventoryCountsVisible={inventoryCountsVisible} setInventoryCountsVisible={setInventoryCountsVisible} /> @@ -281,6 +286,8 @@ interface ProxmoxOverviewProps { setMetricHistoryRange: (value: WorkloadTableMetricHistoryRange) => void; memoryDisplayBasis: Accessor; setMemoryDisplayBasis: (value: WorkloadsMemoryDisplayBasis) => void; + resourceSnapshot: Accessor; + resourceSnapshotRefetch: () => Promise; inventoryCountsVisible: Accessor; setInventoryCountsVisible: (visible: boolean) => void; } @@ -294,6 +301,8 @@ function ProxmoxOverview(props: ProxmoxOverviewProps) { nodes: [], layoutWidth: overviewWidth.width, useWorkloads: true, + resourceSnapshot: props.resourceSnapshot, + resourceSnapshotRefetch: props.resourceSnapshotRefetch, forcedPlatform: PROXMOX_PLATFORM_FILTER, excludedWorkloadTypes: PROXMOX_WORKLOAD_EXCLUDED_TYPES, showNestedExcludedWorkloads: true, diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupsTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupsTable.test.tsx index 28c6cc6c5..9aaf9a1d4 100644 --- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupsTable.test.tsx +++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupsTable.test.tsx @@ -519,6 +519,17 @@ describe('ProxmoxBackupsTable', () => { ); }); + it('keeps the overview node and guest regions on one canonical resource snapshot', () => { + expect(proxmoxPageSurfaceSource).toContain('const { resources, loading, error, refetch }'); + expect(proxmoxPageSurfaceSource).toContain('resourceSnapshot={() =>'); + expect(proxmoxPageSurfaceSource).toContain('resourceSnapshotRefetch={() => refetch()}'); + expect(proxmoxPageSurfaceSource).toContain('useWorkloadsState({'); + expect(proxmoxPageSurfaceSource).toContain('resourceSnapshot: props.resourceSnapshot'); + expect(proxmoxPageSurfaceSource).not.toContain( + 'useWorkloads({ enabled: () => workloadsEnabled() })', + ); + }); + it('keeps the shared storage surface scoped to the whole Proxmox product family', () => { expect(proxmoxPageSurfaceSource).toContain("const PROXMOX_PLATFORM_FILTER = 'proxmox-all';"); expect(proxmoxPageSurfaceSource).toContain('forcedSourceFilter={PROXMOX_PLATFORM_FILTER}'); diff --git a/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts b/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts index 6b7d00e73..929c0599f 100644 --- a/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts +++ b/frontend-modern/src/hooks/__tests__/useWorkloads.test.ts @@ -229,6 +229,46 @@ describe('useWorkloads', () => { dispose(); }); + it('uses an owning canonical resource snapshot without a second workload request', async () => { + const snapshot = [ + { + id: 'cluster-a-pve1-101', + type: 'vm', + name: 'vm-101', + status: 'running', + platformType: 'proxmox-pve', + sources: ['proxmox'], + identity: { hostname: 'vm-101', ips: ['192.0.2.101'] }, + proxmox: { vmid: 101, nodeName: 'pve1', instance: 'cluster-a' }, + cpu: { current: 25 }, + memory: { current: 50, used: 2 * 1024, total: 4 * 1024 }, + disk: { current: 20, used: 20 * 1024, total: 100 * 1024 }, + }, + ] as any; + const refetchSnapshot = vi.fn().mockResolvedValue(undefined); + + let dispose = () => {}; + let result: ReturnType | undefined; + createRoot((d) => { + dispose = d; + const [enabled] = createSignal(true); + result = useWorkloads(enabled, { + resourceSnapshot: () => snapshot, + refetchSnapshot, + }); + }); + + await flushAsync(); + expect(apiFetchJSONMock).not.toHaveBeenCalled(); + expect(result!.workloads()).toMatchObject([{ name: 'vm-101', vmid: 101, node: 'pve1' }]); + + await result!.refetch(); + expect(refetchSnapshot).toHaveBeenCalledTimes(1); + expect(apiFetchJSONMock).not.toHaveBeenCalled(); + + dispose(); + }); + it('retains the fulfilled workload snapshot when a forced refresh fails', async () => { let dispose = () => {}; let result: ReturnType | undefined; diff --git a/frontend-modern/src/hooks/useWorkloads.ts b/frontend-modern/src/hooks/useWorkloads.ts index 95b1a050c..69fc883fc 100644 --- a/frontend-modern/src/hooks/useWorkloads.ts +++ b/frontend-modern/src/hooks/useWorkloads.ts @@ -20,6 +20,7 @@ import { } from '@/utils/resourceIdentity'; import type { WorkloadGuest } from '@/types/workloads'; import type { + Resource, ResourceActionReadiness, ResourceAvailabilityMeta, ResourceDiscoveryReadiness, @@ -28,8 +29,12 @@ import type { } from '@/types/resource'; const WORKLOADS_URL = '/api/resources?type=vm,system-container,app-container,pod'; -const WORKLOADS_PAGE_LIMIT = 200; -const WORKLOADS_MAX_PAGES = 20; +const WORKLOADS_PAGE_LIMIT = 500; +const WORKLOADS_PAGE_CONCURRENCY = 4; +// This is a safety valve for malformed pagination metadata, not a truncation +// policy. Estates beyond the guard fail explicitly instead of presenting a +// silently incomplete workload table. +const WORKLOADS_MAX_PAGES = 100; const WORKLOADS_CACHE_MAX_AGE_MS = 15_000; type APIMetricValue = { @@ -183,6 +188,7 @@ type APIListResponse = { type WorkloadsCacheEntry = { workloads: WorkloadGuest[]; + signature: string; cachedAt: number; sharedFetch: Promise | null; }; @@ -196,6 +202,7 @@ const getWorkloadsCacheEntry = (orgScope: string): WorkloadsCacheEntry => { } const created: WorkloadsCacheEntry = { workloads: [], + signature: '', cachedAt: 0, sharedFetch: null, }; @@ -203,67 +210,22 @@ const getWorkloadsCacheEntry = (orgScope: string): WorkloadsCacheEntry => { return created; }; -const isPlainObject = (value: unknown): value is Record => - typeof value === 'object' && value !== null && Object.getPrototypeOf(value) === Object.prototype; +// Workload objects are produced by one canonical mapper, so their property +// order is stable. Serializing once per row gives refreshes a cheap immutable +// signature without recursively comparing the same nested telemetry objects. +const workloadSignature = (guest: WorkloadGuest): string => JSON.stringify(guest); -const isDeepEqual = (left: unknown, right: unknown): boolean => { - if (Object.is(left, right)) { - return true; - } +const buildWorkloadsSignature = (workloads: WorkloadGuest[]): string => + workloads.map(workloadSignature).join('\u001e'); - if (typeof left !== typeof right) { - return false; - } - - if (Array.isArray(left) || Array.isArray(right)) { - if (!Array.isArray(left) || !Array.isArray(right) || left.length !== right.length) { - return false; - } - for (let i = 0; i < left.length; i += 1) { - if (!isDeepEqual(left[i], right[i])) { - return false; - } - } - return true; - } - - if (!isPlainObject(left) || !isPlainObject(right)) { - return false; - } - - const leftKeys = Object.keys(left); - const rightKeys = Object.keys(right); - if (leftKeys.length !== rightKeys.length) { - return false; - } - - for (const key of leftKeys) { - if (!Object.prototype.hasOwnProperty.call(right, key) || !isDeepEqual(left[key], right[key])) { - return false; - } - } - - return true; -}; - -const areWorkloadsEqual = (current: WorkloadGuest[], next: WorkloadGuest[]): boolean => { - if (current === next) { - return true; - } - if (current.length !== next.length) { - return false; - } - for (let i = 0; i < current.length; i += 1) { - const currentWorkload = current[i]; - const nextWorkload = next[i]; - if (currentWorkload.id !== nextWorkload.id) { - return false; - } - if (!isDeepEqual(currentWorkload, nextWorkload)) { - return false; - } - } - return true; +const areWorkloadsEqual = ( + current: WorkloadGuest[], + next: WorkloadGuest[], + currentSignature?: string, +): boolean => { + if (current === next) return true; + if (current.length !== next.length) return false; + return (currentSignature ?? buildWorkloadsSignature(current)) === buildWorkloadsSignature(next); }; const normalizeWorkloadStatus = (status?: string | null): string => { @@ -654,18 +616,94 @@ const mapResourceToWorkload = (resource: APIResource): WorkloadGuest | null => { }; }; +const metricToAPI = (metric?: Resource['cpu']): APIMetricValue | undefined => + metric + ? { + percent: metric.current, + used: metric.used, + total: metric.total, + } + : undefined; + +/** + * The Proxmox overview already owns a canonical unified-resource snapshot. + * Adapt that snapshot into the legacy WorkloadGuest boundary so the overview + * does not fetch and parse the same VM/LXC rows a second time. + */ +const mapCanonicalResourceToWorkload = (resource: Resource): WorkloadGuest | null => + mapResourceToWorkload({ + id: resource.id, + type: resource.type, + name: resource.name, + status: resource.status, + lastSeen: resource.lastSeen ? new Date(resource.lastSeen).toISOString() : undefined, + sources: resource.sources, + platformScopes: resource.platformScopes, + identity: resource.identity + ? { + machineId: resource.identity.machineId, + hostnames: resource.identity.hostname ? [resource.identity.hostname] : undefined, + ipAddresses: resource.identity.ips, + clusterName: resource.identity.clusterName, + } + : undefined, + metrics: { + cpu: metricToAPI(resource.cpu), + memory: metricToAPI(resource.memory), + disk: metricToAPI(resource.disk), + netIn: resource.network ? { value: resource.network.rxBytes } : undefined, + netOut: resource.network ? { value: resource.network.txBytes } : undefined, + diskRead: resource.diskIO ? { value: resource.diskIO.readRate } : undefined, + diskWrite: resource.diskIO ? { value: resource.diskIO.writeRate } : undefined, + }, + parentId: resource.parentId, + parentName: resource.parentName, + tags: resource.tags, + vmid: resource.proxmox?.vmid, + node: resource.proxmox?.nodeName || resource.proxmox?.node, + instance: resource.proxmox?.instance, + proxmox: resource.proxmox ? (resource.proxmox as unknown as APIResource['proxmox']) : undefined, + virtualMachine: resource.virtualMachine + ? (resource.virtualMachine as unknown as APIResource['virtualMachine']) + : undefined, + agent: resource.agent ? (resource.agent as unknown as APIResource['agent']) : undefined, + docker: resource.docker ? (resource.docker as unknown as APIResource['docker']) : undefined, + kubernetes: resource.kubernetes + ? (resource.kubernetes as unknown as APIResource['kubernetes']) + : undefined, + vmware: resource.vmware ? (resource.vmware as unknown as APIResource['vmware']) : undefined, + discoveryTarget: resource.discoveryTarget, + discoveryReadiness: resource.discoveryReadiness, + availability: resource.availability, + availabilityChecks: resource.availabilityChecks, + actionReadiness: resource.actionReadiness, + }); + async function fetchWorkloads(): Promise { const firstResponse = await apiFetchJSON(buildWorkloadsUrl(1), { cache: 'no-store' }); const firstPage = resolveWorkloadsPayload(firstResponse); const allResources: APIResource[] = [...firstPage.data]; - const totalPages = Math.min(firstPage.totalPages, WORKLOADS_MAX_PAGES); - if (totalPages > 1) { - const pageRequests: Promise[] = []; - for (let page = 2; page <= totalPages; page++) { - pageRequests.push(apiFetchJSON(buildWorkloadsUrl(page), { cache: 'no-store' })); - } - const responses = await Promise.all(pageRequests); + const totalPages = firstPage.totalPages; + if (totalPages > WORKLOADS_MAX_PAGES) { + throw new Error( + `Workload inventory has ${totalPages * WORKLOADS_PAGE_LIMIT}+ rows, refusing an unsafe browser payload`, + ); + } + for ( + let firstPageNumber = 2; + firstPageNumber <= totalPages; + firstPageNumber += WORKLOADS_PAGE_CONCURRENCY + ) { + const pageNumbers = Array.from( + { length: Math.min(WORKLOADS_PAGE_CONCURRENCY, totalPages - firstPageNumber + 1) }, + (_, offset) => firstPageNumber + offset, + ); + const responses = await Promise.all( + pageNumbers.map((page) => + apiFetchJSON(buildWorkloadsUrl(page), { cache: 'no-store' }), + ), + ); for (const response of responses) { const pageData = resolveWorkloadsPayload(response); allResources.push(...pageData.data); @@ -690,8 +728,10 @@ const setWorkloadsCache = ( entry: WorkloadsCacheEntry, workloads: WorkloadGuest[], at = Date.now(), + signature = buildWorkloadsSignature(workloads), ) => { entry.workloads = workloads; + entry.signature = signature; entry.cachedAt = at; }; @@ -710,11 +750,12 @@ const fetchWorkloadsShared = async ( const request = (async () => { const previous = entry.workloads; const fetched = await fetchWorkloads(); - if (areWorkloadsEqual(previous, fetched)) { + const fetchedSignature = buildWorkloadsSignature(fetched); + if (areWorkloadsEqual(previous, fetched, entry.signature || undefined)) { entry.cachedAt = Date.now(); return previous; } - setWorkloadsCache(entry, fetched); + setWorkloadsCache(entry, fetched, Date.now(), fetchedSignature); return fetched; })(); @@ -733,7 +774,17 @@ export const __resetWorkloadsCacheForTests = () => { workloadsCaches.clear(); }; -export function useWorkloads(enabled: Accessor = () => true) { +export interface UseWorkloadsOptions { + /** Optional canonical snapshot owned by a platform page. */ + resourceSnapshot?: Accessor; + /** Refetch the owner snapshot when the surface explicitly reconnects. */ + refetchSnapshot?: () => Promise; +} + +export function useWorkloads( + enabled: Accessor = () => true, + options: UseWorkloadsOptions = {}, +) { const [orgScope, setOrgScope] = createSignal(normalizeOrgScope(getOrgID())); const resolveActiveOrgScope = () => orgScope(); const resolveActiveCacheEntry = () => getWorkloadsCacheEntry(resolveActiveOrgScope()); @@ -741,7 +792,7 @@ export function useWorkloads(enabled: Accessor = () => true) { resolveActiveCacheEntry().workloads, ); const [loading, setLoading] = createSignal( - enabled() && !hasFreshWorkloadsCache(resolveActiveCacheEntry()), + enabled() && !options.resourceSnapshot && !hasFreshWorkloadsCache(resolveActiveCacheEntry()), ); const [error, setError] = createSignal(undefined); let requestVersion = 0; @@ -752,25 +803,31 @@ export function useWorkloads(enabled: Accessor = () => true) { const next = typeof value === 'function' ? value(current) : value; const normalized = next ?? []; const cacheEntry = resolveActiveCacheEntry(); - if (areWorkloadsEqual(current, normalized)) { + const nextSignature = buildWorkloadsSignature(normalized); + const currentSignature = + cacheEntry.workloads === current ? cacheEntry.signature : buildWorkloadsSignature(current); + if (areWorkloadsEqual(current, normalized, currentSignature || undefined)) { setWorkloadsCache(cacheEntry, current); return current; } - setWorkloadsCache(cacheEntry, normalized); + setWorkloadsCache(cacheEntry, normalized, Date.now(), nextSignature); return normalized; }); const applyWorkloads = (next: WorkloadGuest[], targetOrgScope = resolveActiveOrgScope()) => { const cacheEntry = getWorkloadsCacheEntry(targetOrgScope); const current = targetOrgScope === resolveActiveOrgScope() ? workloads() : cacheEntry.workloads; - if (areWorkloadsEqual(current, next)) { - setWorkloadsCache(cacheEntry, current); + const nextSignature = buildWorkloadsSignature(next); + const currentSignature = + cacheEntry.workloads === current ? cacheEntry.signature : buildWorkloadsSignature(current); + if (areWorkloadsEqual(current, next, currentSignature || undefined)) { + setWorkloadsCache(cacheEntry, current, Date.now(), cacheEntry.signature || nextSignature); if (targetOrgScope === resolveActiveOrgScope()) { setWorkloads(() => current); } return current; } - setWorkloadsCache(cacheEntry, next); + setWorkloadsCache(cacheEntry, next, Date.now(), nextSignature); if (targetOrgScope === resolveActiveOrgScope()) { setWorkloads(() => next); } @@ -810,20 +867,41 @@ export function useWorkloads(enabled: Accessor = () => true) { }; const refetch = async () => { + if (options.resourceSnapshot) { + if (options.refetchSnapshot) { + await options.refetchSnapshot(); + } + return workloads(); + } const scope = resolveActiveOrgScope(); const cacheEntry = getWorkloadsCacheEntry(scope); return loadWorkloads(scope, { force: true, showLoading: cacheEntry.workloads.length === 0 }); }; createEffect(() => { - const isEnabled = enabled(); - const scope = resolveActiveOrgScope(); - const cacheEntry = getWorkloadsCacheEntry(scope); - if (!isEnabled) { + if (!enabled()) { requestVersion += 1; setLoading(false); return; } + const resourceSnapshot = options.resourceSnapshot?.(); + if (options.resourceSnapshot) { + if (resourceSnapshot === undefined) { + if (workloads().length === 0) setLoading(true); + return; + } + + const next = resourceSnapshot + .map(mapCanonicalResourceToWorkload) + .filter((resource): resource is WorkloadGuest => Boolean(resource)); + applyWorkloads(next); + setLoading(false); + setError(undefined); + return; + } + + const scope = resolveActiveOrgScope(); + const cacheEntry = getWorkloadsCacheEntry(scope); if (cacheEntry.workloads !== workloads()) { setWorkloads(() => cacheEntry.workloads); } @@ -842,7 +920,7 @@ export function useWorkloads(enabled: Accessor = () => true) { // the app-level boundary or replaces the last good table with an // empty loading surface. createEffect(() => { - if (!enabled()) return; + if (!enabled() || options.resourceSnapshot) return; const scope = resolveActiveOrgScope(); const id = setInterval(async () => { try { @@ -876,6 +954,10 @@ export function useWorkloads(enabled: Accessor = () => true) { setLoading(enabled() && nextCacheEntry.workloads.length === 0); setOrgScope(nextOrgScope); + if (options.resourceSnapshot) { + return; + } + if (!hasFreshWorkloadsCache(nextCacheEntry)) { void loadWorkloads(nextOrgScope, { force: true, diff --git a/internal/mock/demo_scenarios.go b/internal/mock/demo_scenarios.go index 71a27a1ec..e4fd41df0 100644 --- a/internal/mock/demo_scenarios.go +++ b/internal/mock/demo_scenarios.go @@ -288,6 +288,9 @@ var demoProxmoxClusterProfiles = []demoProxmoxClusterProfile{ {SourceInstance: "mock-cluster-3", Name: "Core Services", NodeDisplayPrefix: "Core Services", StoragePrefix: "core", SharedStorageName: "core-shared-fabric"}, {SourceInstance: "mock-cluster-4", Name: "Disaster Recovery", NodeDisplayPrefix: "Disaster Recovery", StoragePrefix: "dr", SharedStorageName: "dr-shared-fabric"}, {SourceInstance: "mock-cluster-5", Name: "Edge Sites", NodeDisplayPrefix: "Edge Sites", StoragePrefix: "edge", SharedStorageName: "edge-shared-fabric"}, + {SourceInstance: "mock-cluster-6", Name: "Analytics", NodeDisplayPrefix: "Analytics", StoragePrefix: "analytics", SharedStorageName: "analytics-shared-fabric"}, + {SourceInstance: "mock-cluster-7", Name: "Research", NodeDisplayPrefix: "Research", StoragePrefix: "research", SharedStorageName: "research-shared-fabric"}, + {SourceInstance: "mock-cluster-8", Name: "Branch Services", NodeDisplayPrefix: "Branch Services", StoragePrefix: "branch", SharedStorageName: "branch-shared-fabric"}, } const demoOfflineProxmoxNode = "pve22" diff --git a/internal/mock/demo_scenarios_test.go b/internal/mock/demo_scenarios_test.go index 383ef85f5..b5e8c34af 100644 --- a/internal/mock/demo_scenarios_test.go +++ b/internal/mock/demo_scenarios_test.go @@ -99,7 +99,7 @@ func TestDefaultDemoProxmoxEstateIsLargeMultiClusterAndBounded(t *testing.T) { now := time.Date(2026, time.April, 1, 12, 0, 0, 0, time.UTC) graph := buildFixtureGraph(cfg, now) - if got, want := len(graph.State.Nodes), 32; got != want { + if got, want := len(graph.State.Nodes), 50; got != want { t.Fatalf("default demo node count = %d, want %d", got, want) } @@ -129,8 +129,8 @@ func TestDefaultDemoProxmoxEstateIsLargeMultiClusterAndBounded(t *testing.T) { } guestCount := len(graph.State.VMs) + len(graph.State.Containers) - if guestCount <= 500 || guestCount > 700 { - t.Fatalf("default demo guest count = %d, want a windowed but bounded estate in (500, 700]", guestCount) + if guestCount <= 800 || guestCount > 1000 { + t.Fatalf("default demo guest count = %d, want a windowed but bounded estate in (800, 1000]", guestCount) } // Local storage, two shared PBS definitions per clustered node, and a diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 5c3c8b019..21fa452dc 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -119,8 +119,8 @@ func mockRecoveryChance(meanRecovery time.Duration) float64 { // Default fixture sizes target a large, believable mixed estate so platform // pages prove fleet-scale grouping, filtering, windowing, drawers, and // responsive layout out of the box: -// - 32 Proxmox nodes across five six-node clusters plus two standalone -// systems, with enough VMs and LXCs to cross the workload table's bounded +// - 50 Proxmox nodes across eight six-node clusters plus a standalone tail, +// with enough VMs and LXCs to exercise the workload table's bounded // row-window threshold. The curated demo scenario in // `internal/mock/demo_scenarios.go` owns the human-facing site, cluster, // workload, and storage story layered over these graph-native fixtures. @@ -131,7 +131,7 @@ func mockRecoveryChance(meanRecovery time.Duration) float64 { // overview, nodes, pods, and deployments tabs all exercise grouped // and flat layouts against a real multi-cluster footprint var DefaultConfig = MockConfig{ - NodeCount: 32, + NodeCount: 50, VMsPerNode: 10, LXCsPerNode: 8, DockerHostCount: 5, @@ -968,7 +968,7 @@ func generateNodes(config MockConfig) []models.Node { const ( mockProxmoxClusterSize = 6 - maxMockProxmoxClusterCount = 5 + maxMockProxmoxClusterCount = 8 ) // mockProxmoxClusteredNodeCount assigns nodes to complete or useful cluster diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index cbdc9d288..86978e793 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -1737,7 +1737,7 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `resolve_config_dir`, `set_env_value DEMO_MODE true`, `set_env_value PULSE_MOCK_MODE true`, - `set_env_value PULSE_MOCK_NODES 32`, + `set_env_value PULSE_MOCK_NODES 50`, `set_env_value PULSE_MOCK_VMS_PER_NODE 10`, `set_env_value PULSE_MOCK_LXCS_PER_NODE 8`, `ensure_demo_fixture_entitlement`, diff --git a/scripts/tests/test-toggle-mock.sh b/scripts/tests/test-toggle-mock.sh index 60f04196f..214a4e20d 100755 --- a/scripts/tests/test-toggle-mock.sh +++ b/scripts/tests/test-toggle-mock.sh @@ -210,7 +210,7 @@ test_ensure_mock_env_file_seeds_canonical_demo_defaults() { env_contents="${output}" assert_contains "ensure_mock_env_file seeds mock mode default" "${env_contents}" "PULSE_MOCK_MODE=false" - assert_contains "ensure_mock_env_file seeds canonical node count" "${env_contents}" "PULSE_MOCK_NODES=32" + assert_contains "ensure_mock_env_file seeds canonical node count" "${env_contents}" "PULSE_MOCK_NODES=50" assert_contains "ensure_mock_env_file seeds canonical vm count" "${env_contents}" "PULSE_MOCK_VMS_PER_NODE=10" assert_contains "ensure_mock_env_file seeds canonical lxc count" "${env_contents}" "PULSE_MOCK_LXCS_PER_NODE=8" assert_contains "ensure_mock_env_file seeds canonical docker host count" "${env_contents}" "PULSE_MOCK_DOCKER_HOSTS=5" diff --git a/scripts/toggle-mock.sh b/scripts/toggle-mock.sh index 1e2f25927..56a044023 100755 --- a/scripts/toggle-mock.sh +++ b/scripts/toggle-mock.sh @@ -231,7 +231,7 @@ load_env_file() { mock_default_entries() { cat <<'ENVEOF' PULSE_MOCK_MODE=false -PULSE_MOCK_NODES=32 +PULSE_MOCK_NODES=50 PULSE_MOCK_VMS_PER_NODE=10 PULSE_MOCK_LXCS_PER_NODE=8 PULSE_MOCK_DOCKER_HOSTS=5