From 08827bb8871d74f5cd400e306fcbb53245d57e16 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 21 Aug 2026 17:33:15 +0100 Subject: [PATCH] Extract chart and resource query services --- docs/release-control/v6/internal/status.json | 12 +- .../v6/internal/subsystems/agent-lifecycle.md | 14 +- .../v6/internal/subsystems/api-contracts.md | 42 +- .../v6/internal/subsystems/registry.json | 10 +- .../internal/subsystems/storage-recovery.md | 10 +- .../internal/subsystems/unified-resources.md | 8 +- internal/api/actions.go | 9 +- internal/api/agent_resource_context.go | 9 +- internal/api/{ => chartapi}/charts_test.go | 2 +- internal/api/chartapi/service.go | 3265 +++++++++++++++++ .../api/chartapi/service_handlers_test.go | 1280 +++++++ internal/api/chartapi/types.go | 330 ++ internal/api/charts_compat.go | 51 + internal/api/contract_test.go | 36 +- internal/api/filters_branchcov0722pm_test.go | 10 +- internal/api/load_test.go | 125 - internal/api/patrol_action_broker_test.go | 4 +- .../api/pure_helpers_branchcov0724pm_test.go | 21 +- internal/api/resource_test_support_test.go | 64 + .../k8s_namespaces.go} | 9 +- internal/api/resourceapi/load_test.go | 179 + internal/api/resourceapi/norace_test.go | 5 + internal/api/resourceapi/race_test.go | 5 + internal/api/{ => resourceapi}/resources.go | 330 +- .../resources_frontend_types_test.go | 6 +- .../resources_pool_health_contract_test.go | 4 +- .../resources_shared_cache_test.go | 7 +- .../resources_store_close_test.go | 32 +- .../resources_store_isolation_test.go | 4 +- .../resources_tenant_security_test.go | 23 +- .../api/{ => resourceapi}/resources_test.go | 243 +- internal/api/resources_compat.go | 148 + .../api/resources_state_integration_test.go | 117 + internal/api/router.go | 3185 +--------------- internal/api/router_misc_additional_test.go | 1208 ------ internal/api/types.go | 342 +- .../unifiedresources/code_standards_test.go | 49 +- 37 files changed, 5931 insertions(+), 5267 deletions(-) rename internal/api/{ => chartapi}/charts_test.go (99%) create mode 100644 internal/api/chartapi/service.go create mode 100644 internal/api/chartapi/service_handlers_test.go create mode 100644 internal/api/chartapi/types.go create mode 100644 internal/api/charts_compat.go create mode 100644 internal/api/resource_test_support_test.go rename internal/api/{resources_k8s_namespaces.go => resourceapi/k8s_namespaces.go} (92%) create mode 100644 internal/api/resourceapi/load_test.go create mode 100644 internal/api/resourceapi/norace_test.go create mode 100644 internal/api/resourceapi/race_test.go rename internal/api/{ => resourceapi}/resources.go (87%) rename internal/api/{ => resourceapi}/resources_frontend_types_test.go (99%) rename internal/api/{ => resourceapi}/resources_pool_health_contract_test.go (98%) rename internal/api/{ => resourceapi}/resources_shared_cache_test.go (94%) rename internal/api/{ => resourceapi}/resources_store_close_test.go (67%) rename internal/api/{ => resourceapi}/resources_store_isolation_test.go (93%) rename internal/api/{ => resourceapi}/resources_tenant_security_test.go (85%) rename internal/api/{ => resourceapi}/resources_test.go (96%) create mode 100644 internal/api/resources_compat.go create mode 100644 internal/api/resources_state_integration_test.go diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 749711dcf..bd34e6d8e 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -1232,7 +1232,7 @@ }, { "repo": "pulse", - "path": "internal/api/resources_tenant_security_test.go", + "path": "internal/api/resourceapi/resources_tenant_security_test.go", "kind": "file" }, { @@ -5459,7 +5459,7 @@ }, { "repo": "pulse", - "path": "internal/api/resources.go", + "path": "internal/api/resourceapi/resources.go", "kind": "file" }, { @@ -5899,7 +5899,7 @@ }, { "repo": "pulse", - "path": "internal/api/resources.go", + "path": "internal/api/resourceapi/resources.go", "kind": "file" }, { @@ -6643,12 +6643,12 @@ }, { "repo": "pulse", - "path": "internal/api/resources.go", + "path": "internal/api/resourceapi/resources.go", "kind": "file" }, { "repo": "pulse", - "path": "internal/api/resources_test.go", + "path": "internal/api/resourceapi/resources_test.go", "kind": "file" }, { @@ -7424,7 +7424,7 @@ }, { "repo": "pulse", - "path": "internal/api/resources.go", + "path": "internal/api/resourceapi/resources.go", "kind": "file" }, { diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 534fc5b15..a381ac580 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -1560,7 +1560,7 @@ the intentionally sparse public response. They must not bake vendor model IDs or provider-model fallback rules into install or activation flows just because those routes share the backend API tree. - Lifecycle-adjacent resource reads that traverse `internal/api/resources.go` + Lifecycle-adjacent resource reads that traverse `internal/api/resourceapi/resources.go` must also preserve the canonical unified-resource `name -> type -> id` order instead of inheriting map order or page-local re-sorts, so install and runtime hydration do not present one resource ordering at first load and @@ -1584,7 +1584,7 @@ the intentionally sparse public response. snapshot must carry the same canonical resource types and display names as `/api/resources` instead of briefly showing legacy host aliases before the first websocket-backed refresh lands. - Chart-adjacent lifecycle reads in shared `internal/api/router.go` must obey + Chart-adjacent lifecycle reads in shared `internal/api/chartapi/service.go` must obey that same mock-aware unified snapshot boundary: demo `/api/charts` and `/api/charts/infrastructure` payloads may not bypass `GetUnifiedReadStateOrSnapshot()` and silently drop VMware-backed host rows @@ -2039,7 +2039,7 @@ the intentionally sparse public response. user identity passed through that helper, but they must not treat a missing configured role header as administrator proof. 13. Preserve shipped security-doc guidance in shared lifecycle setup helpers so `internal/api/configapi/config_setup_handlers.go` and adjacent install/setup runtime paths point operators at the running build's local security documentation route rather than GitHub `main` links. -14. Keep shared `internal/api/router.go` workload-chart downsampling presentation-only: when that router caps mixed-cadence workload history into equal-time buckets for operator-facing cards, lifecycle-adjacent setup and fleet surfaces must not reuse the shaped chart samples as heartbeat, enrollment, or last-seen authority. +14. Keep shared `internal/api/chartapi/service.go` workload-chart downsampling presentation-only: when that service caps mixed-cadence workload history into equal-time buckets for operator-facing cards, lifecycle-adjacent setup and fleet surfaces must not reuse the shaped chart samples as heartbeat, enrollment, or last-seen authority. That same presentation-only boundary must preserve canonical millisecond timestamps when it serializes chart points, so lifecycle-adjacent first-host and fleet surfaces do not misread rounded chart samples as duplicate or restarted heartbeat evidence. The same rule now applies to storage summary interaction. Shared sticky-card or row-hover focus behavior on infrastructure, workloads, and storage may reuse the canonical chart transport, but lifecycle-adjacent install, enrollment, and fleet surfaces must not treat highlighted summary series or sticky-shell state as agent freshness or setup progress. The same rule now applies to infrastructure-summary metric filters. Shared @@ -2049,7 +2049,7 @@ the intentionally sparse public response. omitted disk or network series as missing lifecycle telemetry, missing agent capabilities, or reduced fleet freshness truth. The same rule now applies to retired compact dashboard summary payloads. - Shared `internal/api/resources.go` routes must not restore + Shared `internal/api/resourceapi/resources.go` routes must not restore `/api/resources/dashboard-summary` as a compatibility read; lifecycle surfaces must continue to use install inventory, enrollment proof, and fleet freshness truth from their owning contracts. @@ -3701,7 +3701,7 @@ credential reaches only the explicit backend-owned route inventory, so lifecycle-adjacent setup and install flows cannot accidentally widen the paired-device credential just by touching neighboring `internal/api/` routes. The same shared API runtime now also exposes dedicated unified-resource -timeline reads through `internal/api/resources.go` plus the bundled facet +timeline reads through `internal/api/resourceapi/resources.go` plus the bundled facet history read used by the drawer, but those query surfaces remain owned by the API and unified-resource contracts rather than by lifecycle continuity. Those timeline reads also accept governed filters for change kind, source @@ -3710,7 +3710,7 @@ agent lifecycle routing still stays on canonical fleet-continuity ownership instead of re-deriving resource history locally. Those dedicated resource timeline and facet reads are also relationship-aware at the API boundary: lifecycle-adjacent fleet views may consume the direct plus -`relatedResources` history returned by `internal/api/resources.go`, but they +`relatedResources` history returned by `internal/api/resourceapi/resources.go`, but they must not rebuild cross-resource timeline joins inside lifecycle-owned routes or change the direct-only store default used by other callers. The bundled facet read may also expose the selected resource's canonical @@ -3719,7 +3719,7 @@ surfaces must treat those fields as API/unified-resource facts rather than agent-lifecycle-owned install, approval, or topology state. Agent-host, Kubernetes, and runtime parentage exposed through `ParentID` must therefore enter shared drawers as facet relationships from -`internal/api/resources.go`; lifecycle surfaces must not rederive those edges +`internal/api/resourceapi/resources.go`; lifecycle surfaces must not rederive those edges from agent install state, cluster names, or local fleet table grouping. That same shared `internal/api/` boundary now also exposes a dedicated VM inventory export route for reporting. Fleet and install surfaces may coexist diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 504c10ecc..53827172a 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -35,6 +35,15 @@ binding policy are shared through `internal/api/agenttokens/` and `internal/api/agentbinding/`, so config enrollment and root Router admission consume one security contract without importing each other. +Unified resource registry construction, tenant stores, list/detail/facet +queries, discovery projections, and storage/resource response contracts live +with their tests in `internal/api/resourceapi/`. Chart aggregation, monitor +history queries, response contracts, downsampling, per-tenant caches, and all +six chart handlers live with their tests in `internal/api/chartapi/`. Root +`ResourceHandlers` and Router methods remain source-compatible composition +facades for action/operator-state mutations, extensions, and route wiring; they +must not regain resource-query or chart computation behavior. + Browser WebSocket shutdown distinguishes ordinary client lifecycle from transport failure. Normal closure, navigation/going-away, and abnormal closure codes used by disconnected clients are informational; policy violations and @@ -139,7 +148,10 @@ continues to mint and quote the enrollment token. 1a. `internal/api/platform_connection_shared.go` 1b. `internal/api/metadata_handlers_shared.go` 1c. `internal/config/persistence_metadata_accessors.go` -2. `internal/api/resources.go` +2. `internal/api/resourceapi/resources.go` + 2a. `internal/api/resources_compat.go` + 2b. `internal/api/chartapi/service.go` + 2c. `internal/api/chartapi/types.go` 3. `internal/api/discovery_handlers.go` 4. `internal/api/alerting/alerts.go` 4a. `internal/api/attention_handlers.go` @@ -836,7 +848,7 @@ token-gated PowerShell payload until its installer owns the same saved-state update contract. Summary-chart response caching is a shared API boundary: -`internal/api/router.go` may serve a short cached JSON payload for repeated +`internal/api/chartapi/service.go` may serve a short cached JSON payload for repeated infrastructure-summary and workloads-summary requests with the same organization, range, metric set, and workload scope, but that cache is transport-only. It may amortize polling and remount cost, but it must not @@ -1897,7 +1909,7 @@ payload shape change when the portal presents compact client rows. true server failures remain explicit. 77. `internal/api/relay_mobile_capability.go` shared with `relay-runtime`: the backend-owned Pulse Mobile relay capability inventory is both a relay runtime boundary and a canonical API payload contract surface. 78. `internal/api/relay_mobile_capability_generated.go` shared with `relay-runtime`: the generated Pulse Mobile route inventory is both a relay runtime allowlist and the backend projection of the canonical mobile API contract. -78. `internal/api/resources.go` shared with `unified-resources`: the unified resource endpoint is both a backend payload contract surface and a unified-resource runtime boundary. +78. `internal/api/resourceapi/resources.go` shared with `unified-resources`: the unified resource endpoint is both a backend payload contract surface and a unified-resource runtime boundary. 79. `internal/api/security.go` shared with `security-privacy`: the security handlers are both a security/privacy control surface and a canonical API payload contract boundary. That same shared security/API boundary owns CSRF replacement-token concurrency. When parallel browser mutations arrive with stale or missing @@ -2129,7 +2141,7 @@ a new API state machine, queue contract, or verification-accounting field. than CSS-derived state. 3. Add dedicated contract tests for new stable payloads Unified resource type-filter and organization-share resource type additions - must route through `internal/api/resources.go`, `internal/api/org_handlers.go`, + must route through `internal/api/resourceapi/resources.go`, `internal/api/org_handlers.go`, frontend resource typing, and `internal/api/contract_test.go` together. Native provider projections such as TrueNAS `network-share` may be accepted by `/api/resources` filters and cross-organization share normalization only @@ -2227,7 +2239,7 @@ a new API state machine, queue contract, or verification-accounting field. metadata; that token may bind once to the first command agent ID and hostname that registers, and a later different agent or hostname must be rejected. Generic unbound `agent:exec` tokens remain fail-closed. -4. Route unified resource sensitivity, routing, and `aiSafeSummary` payload changes through `internal/api/resources.go`, `internal/api/contract_test.go`, and the canonical frontend resource consumer proofs together; resource governance metadata must not ship as an API-only or frontend-only heuristic +4. Route unified resource sensitivity, routing, and `aiSafeSummary` payload changes through `internal/api/resourceapi/resources.go`, `internal/api/contract_test.go`, and the canonical frontend resource consumer proofs together; resource governance metadata must not ship as an API-only or frontend-only heuristic That same resource payload contract owns `aggregations.policyPosture` on `/api/resources` and `/api/resources/stats`. The aggregation must be derived from canonical unified-resource policy metadata, normalized as camelCase @@ -2521,7 +2533,7 @@ a new API state machine, queue contract, or verification-accounting field. `GET /api/audit/actions` and `GET /api/audit/actions/{id}/events` as verification adapters, but they must remain read-only views of the canonical action audit and lifecycle trail rather than a second audit store. -6. Route dedicated unified-resource timeline and facet-bundle reads through `frontend-modern/src/api/resources.ts`, `internal/api/resources.go`, and `internal/api/contract_test.go` together so the backend facet contract and the frontend client stay aligned on one timeline-first surface, while capability and relationship detail stays backend-owned for AI correlation and change detection. +6. Route dedicated unified-resource timeline and facet-bundle reads through `frontend-modern/src/api/resources.ts`, `internal/api/resourceapi/resources.go`, and `internal/api/contract_test.go` together so the backend facet contract and the frontend client stay aligned on one timeline-first surface, while capability and relationship detail stays backend-owned for AI correlation and change detection. `/api/resources/{id}/timeline` and `/api/resources/{id}/facets` must keep resource timelines relationship-aware by opting into the canonical `ResourceChangeFilters.IncludeRelated` store path, so a resource timeline @@ -2539,7 +2551,7 @@ a new API state machine, queue contract, or verification-accounting field. the monitoring-read scope. Platform pages may use it for API-authored provider activity such as vSphere tasks and events, but they must not create page-local activity stores or a second query vocabulary. -7. Route unified-resource list ordering through `internal/api/resources.go`, `internal/api/contract_test.go`, and the owned unified-resource registry helpers together; list payloads must stay deterministic for equal-name resources by carrying one canonical `name -> type -> id` tie-break across cold seed, REST pagination, and websocket-backed refreshes instead of inheriting map order or page-local re-sorts +7. Route unified-resource list ordering through `internal/api/resourceapi/resources.go`, `internal/api/contract_test.go`, and the owned unified-resource registry helpers together; list payloads must stay deterministic for equal-name resources by carrying one canonical `name -> type -> id` tie-break across cold seed, REST pagination, and websocket-backed refreshes instead of inheriting map order or page-local re-sorts That same shared API contract also owns the external resource `type`, canonical display name, and cluster identity published through `/api/resources` and `/api/state`; the websocket/state hydrate path must not emit legacy aliases or raw store labels once the unified resource contract has normalized them. Lightweight integration polling uses the same canonical read-state boundary: `GET /api/state/summary` is an authenticated `monitoring:read` route that @@ -3044,9 +3056,9 @@ a new API state machine, queue contract, or verification-accounting field. 22. Keep hosted billing-state quickstart grants retired from new shared API flows: `internal/api/hosted_entitlement_refresh.go`, hosted signup, and trial-state construction must not auto-grant or refresh quickstart inventory for new workspaces, while low-level billing-state readers may still preserve historical fields that already exist on disk. 23. Keep hosted AI settings bootstrap on the shared API contract as a retired path: `internal/api/ai_hosted_runtime.go`, `internal/api/ai_handlers.go`, `internal/api/ai_handler.go`, and `internal/api/contract_test.go` must treat a missing `ai.enc` in hosted mode as an unconfigured BYOK/local-provider state, not as a machine-owned `quickstart:pulse-hosted` bootstrap condition. Hosted tenant reads may inherit billing state for commercial authorization, but they must not create quickstart-backed AI config or call the quickstart bootstrap upstream route. 24. Keep post-boot AI enablement contract-backed on the shared AI/mobile approval surface: `internal/api/ai_handler.go`, `internal/api/ai_handlers.go`, `internal/api/router.go`, `internal/api/router_routes_ai_relay.go`, and `internal/api/contract_test.go` must turn the governed approvals-list API into the canonical empty-list payload as soon as settings-driven AI enablement succeeds, rather than leaving that surface on `503 Approval store not initialized` until some separate startup-only side effect happens. The same post-boot lifecycle owns the investigation surface: a successful in-process chat start or restart must reapply the same live dependency wiring as cold startup, including the Enterprise investigation orchestrator and Patrol circuit breaker, so enabling Intelligence after boot cannot leave finding investigations on `503 Investigation orchestrator not initialized` until the server restarts. -25. Keep infrastructure summary chart transport contract-backed on the shared API surface: `internal/api/router.go`, `internal/api/contract_test.go`, and frontend infrastructure summary consumers must normalize long-range mixed-cadence history into equal-time summary buckets before shipping the infrastructure charts API payload, so 7-day and 30-day summary cards do not expose compressed right-edge tails just because recent samples arrive at a finer storage resolution. -26. Keep long-range workload chart transport time-proportional on the shared API surface: `internal/api/router.go`, `internal/api/contract_test.go`, and workload chart consumers must cap mixed-cadence workload history by equal-time buckets rather than raw point index for the per-workload and aggregate workload chart APIs, so 7-day and 30-day workload cards do not bunch recent samples at the right edge just because recent telemetry is stored more densely. -27. Keep chart timestamp precision canonical on that same shared API surface: when `internal/api/router.go` serializes monitoring history into infrastructure or workload chart payloads, it must preserve canonical millisecond timestamps from the shared monitoring timeline instead of rounding through whole-second conversion, so seeded mock history and live appends collapse onto one operator-visible timeline instead of appearing as duplicated tail samples. +25. Keep infrastructure summary chart transport contract-backed on the shared API surface: `internal/api/chartapi/service.go`, `internal/api/contract_test.go`, and frontend infrastructure summary consumers must normalize long-range mixed-cadence history into equal-time summary buckets before shipping the infrastructure charts API payload, so 7-day and 30-day summary cards do not expose compressed right-edge tails just because recent samples arrive at a finer storage resolution. +26. Keep long-range workload chart transport time-proportional on the shared API surface: `internal/api/chartapi/service.go`, `internal/api/contract_test.go`, and workload chart consumers must cap mixed-cadence workload history by equal-time buckets rather than raw point index for the per-workload and aggregate workload chart APIs, so 7-day and 30-day workload cards do not bunch recent samples at the right edge just because recent telemetry is stored more densely. +27. Keep chart timestamp precision canonical on that same shared API surface: when `internal/api/chartapi/service.go` serializes monitoring history into infrastructure or workload chart payloads, it must preserve canonical millisecond timestamps from the shared monitoring timeline instead of rounding through whole-second conversion, so seeded mock history and live appends collapse onto one operator-visible timeline instead of appearing as duplicated tail samples. 28. Keep Patrol remediation payload naming backward-compatible without leaking legacy automation-first wording into product copy. `frontend-modern/src/api/patrol.ts`, `internal/api/ai_handlers.go`, and `internal/api/router_routes_ai_relay.go` @@ -3056,7 +3068,7 @@ a new API state machine, queue contract, or verification-accounting field. contract must describe the operator-visible capability as remediation or safe remediation workflows. 29. Keep storage chart identity canonical on that same shared API surface: the shared storage charts endpoint must key pool and physical-disk series by the resolved unified-resource `MetricsTarget.ResourceID`, not by canonical resource IDs or page-local aliases, so storage rows, focused summary cards, sticky summary shells, and detail charts all address the same history series in live and mock mode. -30. Keep synthetic summary-chart fallback identity canonical on that same shared API surface: when `internal/api/router.go` has to synthesize mock summary history for infrastructure, workloads, or storage cards, it must derive the fallback from canonical `resourceType`, `resourceID`, and `metricType` ownership instead of raw min/max seed-prefix helpers, so range changes and runtime mock updates stay on one governed timeline. +30. Keep synthetic summary-chart fallback identity canonical on that same shared API surface: when `internal/api/chartapi/service.go` has to synthesize mock summary history for infrastructure, workloads, or storage cards, it must derive the fallback from canonical `resourceType`, `resourceID`, and `metricType` ownership instead of raw min/max seed-prefix helpers, so range changes and runtime mock updates stay on one governed timeline. The same compact chart boundary also owns aggregate-only storage summary transport. `/api/charts/storage-summary` may batch only the canonical `used` and `avail` storage series required for the aggregate capacity @@ -3066,7 +3078,7 @@ a new API state machine, queue contract, or verification-accounting field. When mock mode is active, that same endpoint must come from the monitor-owned aggregate summary cache rather than rehydrating each pool chart on request. -31. Keep workload-chart response identity canonical on that same shared API surface: `internal/api/router.go`, `internal/api/contract_test.go`, and workload summary consumers must emit provider-backed VM and system-container series under the same canonical workload IDs that workloads page rows use, while resolving history through the unified `MetricsTarget.ResourceID`, so hover and focus selection do not fall off for provider-backed rows. +31. Keep workload-chart response identity canonical on that same shared API surface: `internal/api/chartapi/service.go`, `internal/api/contract_test.go`, and workload summary consumers must emit provider-backed VM and system-container series under the same canonical workload IDs that workloads page rows use, while resolving history through the unified `MetricsTarget.ResourceID`, so hover and focus selection do not fall off for provider-backed rows. Kubernetes pod workload rows follow that same contract through their metrics target. `/api/resources` may expose pod history only through the unified `MetricsTarget.ResourceID`, but that target must be the canonical @@ -3151,7 +3163,7 @@ a new API state machine, queue contract, or verification-accounting field. `internal/monitoring/monitor_metrics.go` instead of fetching the full guest metric set and trimming after the API payload is already assembled. 39. Keep the retired compact dashboard overview route absent from that same - shared API surface. `internal/api/resources.go`, + shared API surface. `internal/api/resourceapi/resources.go`, `internal/api/router_routes_monitoring.go`, and `frontend-modern/src/api/resources.ts` must not restore `/api/resources/dashboard-summary`, `useDashboardOverview`, or frontend @@ -6320,7 +6332,7 @@ Action-plan stale-plan protection on those audit records now uses the canonical response contract stays deterministic without extra version baggage. The same API contract now also owns the dedicated frontend resource facet client in `frontend-modern/src/api/resources.ts`, which fetches the governed -capability, relationship, and timeline surfaces from `internal/api/resources.go` +capability, relationship, and timeline surfaces from `internal/api/resourceapi/resources.go` instead of teaching the drawer or list views to reconstruct them inline. Those facet reads now explicitly include the selected resource's canonical `capabilities` and `relationships`, so action affordances and relationship-map @@ -6385,7 +6397,7 @@ kinds, source types, and source adapters through the shared unified-resource change-filter parser, so API validation stays owned by the change model rather than being re-parsed separately in the HTTP handler. The tenant-scoped unified resource API now also stays on canonical -unified-resource seeds end to end: `internal/api/resources.go`, +unified-resource seeds end to end: `internal/api/resourceapi/resources.go`, `internal/api/router_helpers.go`, and `internal/api/state_provider.go` no longer treat raw tenant `StateSnapshot` data as a live registry-seeding owner once `UnifiedResourceSnapshotForTenant` is available. diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index a20467a86..ca8dbfa59 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -919,7 +919,7 @@ ] }, { - "path": "internal/api/resources.go", + "path": "internal/api/resourceapi/resources.go", "rationale": "the unified resource endpoint is both a backend payload contract surface and a unified-resource runtime boundary", "subsystems": [ "api-contracts", @@ -5510,7 +5510,7 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ - "internal/api/resources_test.go", + "internal/api/resourceapi/resources_test.go", "internal/monitoring/monitor_polling_test.go", "internal/truenas/client_api_shapes_test.go", "internal/truenas/client_test.go", @@ -5563,7 +5563,7 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ - "internal/api/resources_test.go", + "internal/api/resourceapi/resources_test.go", "internal/storagehealth/risk_test.go", "internal/storagehealth/topology_test.go", "internal/storagehealth/zfs_pool_health_contract_test.go" @@ -7447,7 +7447,7 @@ "frontend-modern/src/utils/sourceTypePresentation.ts", "frontend-modern/src/utils/workloads.ts", "frontend-modern/src/utils/workloadTypePresentation.ts", - "internal/api/resources.go", + "internal/api/resourceapi/resources.go", "internal/platformsupport/manifest_generated.go" ], "verification": { @@ -7622,7 +7622,7 @@ "frontend-modern/src/types/resource.ts", "frontend-modern/src/utils/resourceIdentity.ts", "frontend-modern/src/utils/workloads.ts", - "internal/api/resources.go" + "internal/api/resourceapi/resources.go" ], "allow_same_subsystem_tests": false, "test_prefixes": [], diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index f467a3df2..68ffcaa56 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -41,7 +41,7 @@ disk behind a client-side page ceiling. Search includes vendor, WWN, transport, instance, controller, and member target. Explicit `0%` life remaining is a known critical value for SSD/NVMe media; absent or negative wearout remains the neutral unknown state. -The shared `internal/api/resources.go` registry builder may repair +The shared `internal/api/resourceapi/resources.go` registry builder may repair availability-check identity by replaying authoritative supplemental records. That availability composition remains owned by API contracts and unified resources; it does not make availability rows recovery points, storage health, @@ -1017,7 +1017,7 @@ recovery scope, or a storage/recovery-owned secret source. reinterpret that gap as a new counted system or invent a storage-local grace rule when the shared API and monitoring boundary already carry recent host continuity. - Any adjacent list surfaces that reuse `internal/api/resources.go` must also + Any adjacent list surfaces that reuse `internal/api/resourceapi/resources.go` must also preserve the canonical unified-resource `name -> type -> id` order so duplicate-name storage and recovery resources do not reshuffle between cold hydrate, paginated reads, and later live runtime updates. @@ -1068,7 +1068,7 @@ recovery scope, or a storage/recovery-owned secret source. boundary for storage/recovery consumers: org-scope or enabled-state changes must invalidate stale in-flight REST refreshes before their errors or request-guard cleanup can leak into the active resource snapshot. - Shared chart transports in `internal/api/router.go` must follow the same + Shared chart transports in `internal/api/chartapi/service.go` must follow the same rule in mock mode: `/api/storage-charts` and adjacent infrastructure chart payloads must read through `GetUnifiedReadStateOrSnapshot()` so storage and recovery consumers stay aligned with the canonical mock unified snapshot @@ -1573,13 +1573,13 @@ recovery scope, or a storage/recovery-owned secret source. `certificate_*` incident on those availability facets remain monitoring and alerts context. They do not prove storage encryption, backup integrity, repository authenticity, protection coverage, or restore readiness. -33. Keep infrastructure summary chart bucketing and short response caching presentation-only on the adjacent shared API boundary. When `internal/api/router.go` normalizes mixed-cadence infrastructure history into equal-time summary buckets or serves a cached summary payload for repeated operator-facing summary-card requests, storage and recovery may consume the resulting visual context only; they must not reinterpret those normalized chart samples, cached timestamps, or cache hits as recovery freshness windows, backup cadence, or restore evidence. +33. Keep infrastructure summary chart bucketing and short response caching presentation-only on the adjacent shared API boundary. When `internal/api/chartapi/service.go` normalizes mixed-cadence infrastructure history into equal-time summary buckets or serves a cached summary payload for repeated operator-facing summary-card requests, storage and recovery may consume the resulting visual context only; they must not reinterpret those normalized chart samples, cached timestamps, or cache hits as recovery freshness windows, backup cadence, or restore evidence. The same router may wire the adjacent server-owned agent command authorization verifier, but storage/recovery must not treat an approval id, signed command grant, or command result as restore authorization or recovery evidence; that authority remains action-governance and agent-lifecycle owned. -34. Keep workload chart downsampling and short response caching presentation-only on that same adjacent shared API boundary. When `internal/api/router.go` caps mixed-cadence workload history into equal-time buckets or serves a cached workload-summary payload for repeated operator-facing workload-card requests, storage and recovery may consume the resulting visual context only; they must not reinterpret those shaped chart samples, cached timestamps, or cache hits as recovery freshness windows, backup cadence, or restore evidence. +34. Keep workload chart downsampling and short response caching presentation-only on that same adjacent shared API boundary. When `internal/api/chartapi/service.go` caps mixed-cadence workload history into equal-time buckets or serves a cached workload-summary payload for repeated operator-facing workload-card requests, storage and recovery may consume the resulting visual context only; they must not reinterpret those shaped chart samples, cached timestamps, or cache hits as recovery freshness windows, backup cadence, or restore evidence. The same adjacent chart boundary now covers compact storage capacity transport. `internal/api/router.go` may batch only the canonical `used` and `avail` storage series for `/api/charts/storage-summary`, but storage diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 3794c0c78..dff41f2c8 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -818,7 +818,7 @@ container inventory table. runtime workload belongs to both Docker and an owning infrastructure platform. 18. `frontend-modern/src/utils/workloads.ts` shared with `performance-and-scalability`: the stable workload metadata identity helper is both a unified-resource persistence boundary and a workloads hot-path lookup boundary. -19. `internal/api/resources.go` shared with `api-contracts`: the unified resource endpoint is both a backend payload contract surface and a unified-resource runtime boundary. +19. `internal/api/resourceapi/resources.go` shared with `api-contracts`: the unified resource endpoint is both a backend payload contract surface and a unified-resource runtime boundary. `/api/resources` type filters must accept URL-encoded comma-separated lists from browser query builders exactly like literal comma separators, so Docker / Podman runtime pages do not lose `docker-host` inventory while requesting @@ -1137,7 +1137,7 @@ AI-only summary payloads, or page-local heuristics. `internal/unifiedresources/pbs_rollups.go`, `internal/unifiedresources/monitored_systems.go`, `internal/unifiedresources/monitored_system_projection.go`, and - the shared list-order helpers consumed by `internal/api/resources.go`; + the shared list-order helpers consumed by `internal/api/resourceapi/resources.go`; canonical unified-resource lists must preserve one deterministic `name -> type -> id` order across registry reads, REST pagination, and websocket-backed refreshes so equal-name resources do not silently reshuffle @@ -3319,7 +3319,7 @@ large historical tables during startup. Timeline reads must resolve legacy `timestamp` and nullable `observed_at` values through read-time fallback expressions, while writes preserve the legacy timestamp on target databases that still require it. -`internal/api/resources.go` now exposes that same history through dedicated +`internal/api/resourceapi/resources.go` now exposes that same history through dedicated `/api/resources/{id}/timeline` reads, while the bundled `/api/resources/{id}/facets` surface keeps the facet summary and recent-change history available without forcing consumers to parse the full resource payload. @@ -3888,7 +3888,7 @@ The unified resource core is strong and canonical, but monitoring and some frontend/API consumers are still being tightened around it. Tenant-scoped API resource seeding now also stays on unified-resource ownership -end to end: `internal/api/resources.go` consumes +end to end: `internal/api/resourceapi/resources.go` consumes `UnifiedResourceSnapshotForTenant` as the canonical tenant registry seed, and no longer falls back to raw tenant `StateSnapshot` seeding on the live request path when that unified seed is empty. diff --git a/internal/api/actions.go b/internal/api/actions.go index 86d44a896..335c4936b 100644 --- a/internal/api/actions.go +++ b/internal/api/actions.go @@ -13,6 +13,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/actionlifecycle" "github.com/rcourtman/pulse-go-rewrite/internal/actionplanner" "github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" "github.com/rcourtman/pulse-go-rewrite/internal/mock" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -816,12 +817,12 @@ func projectActionAudits(records []unified.ActionAuditRecord, registry *unified. func projectActionAudit(record unified.ActionAuditRecord, registry *unified.ResourceRegistry) actionAuditProjection { projection := actionAuditProjection{ActionAuditRecord: record} - resource, ok := presentationResourceByID(registry, record.Request.ResourceID) + resource, ok := resourceapi.PresentationResourceByID(registry, record.Request.ResourceID) if ok && resource != nil && strings.TrimSpace(resource.Name) != "" { projection.Resource = &actionResourcePresentation{ ID: unified.CanonicalResourceID(resource.ID), Name: strings.TrimSpace(resource.Name), - Type: resourceContractType(*resource), + Type: resourceapi.ContractType(*resource), } for _, capability := range resource.Capabilities { if capability.Name == record.Request.CapabilityName { @@ -834,10 +835,10 @@ func projectActionAudit(record unified.ActionAuditRecord, registry *unified.Reso projection.BlastRadius = make([]actionResourcePresentation, 0, len(record.Plan.PredictedBlastRadius)) for _, affectedID := range record.Plan.PredictedBlastRadius { entry := actionResourcePresentation{ID: affectedID} - if affected, found := presentationResourceByID(registry, affectedID); found && affected != nil { + if affected, found := resourceapi.PresentationResourceByID(registry, affectedID); found && affected != nil { entry.ID = unified.CanonicalResourceID(affected.ID) entry.Name = strings.TrimSpace(affected.Name) - entry.Type = resourceContractType(*affected) + entry.Type = resourceapi.ContractType(*affected) } projection.BlastRadius = append(projection.BlastRadius, entry) } diff --git a/internal/api/agent_resource_context.go b/internal/api/agent_resource_context.go index c69d680e6..baaf4039b 100644 --- a/internal/api/agent_resource_context.go +++ b/internal/api/agent_resource_context.go @@ -11,6 +11,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities" "github.com/rcourtman/pulse-go-rewrite/internal/agentcontext" "github.com/rcourtman/pulse-go-rewrite/internal/ai/approval" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/telemetry" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -492,7 +493,7 @@ func (h *AgentContextHandler) HandleResourceContext(w http.ResponseWriter, r *ht http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) return } - resource, resourceID, ok := presentationResourceByReference(registry, resourceID) + resource, resourceID, ok := resourceapi.PresentationResourceByReference(registry, resourceID) if !ok { writeJSONError(w, http.StatusNotFound, agentcapabilities.AgentErrCodeResourceNotFound, "No resource is registered with this canonical id.") @@ -507,8 +508,8 @@ func (h *AgentContextHandler) HandleResourceContext(w http.ResponseWriter, r *ht generatedAt := time.Now().UTC() resourceCopy := *resource - attachDiscoveryTarget(&resourceCopy) - h.resources.attachDiscoveryReadiness(&resourceCopy, generatedAt) + resourceapi.AttachDiscoveryTarget(&resourceCopy) + h.resources.AttachDiscoveryReadiness(&resourceCopy, generatedAt) bundle := AgentResourceContext{ CanonicalID: resourceID, @@ -617,7 +618,7 @@ func (h *AgentContextHandler) HandleResourceCapabilities(w http.ResponseWriter, http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) return } - resource, resourceID, ok := presentationResourceByReference(registry, resourceID) + resource, resourceID, ok := resourceapi.PresentationResourceByReference(registry, resourceID) if !ok { writeJSONError(w, http.StatusNotFound, agentcapabilities.AgentErrCodeResourceNotFound, "No resource is registered with this canonical id.") diff --git a/internal/api/charts_test.go b/internal/api/chartapi/charts_test.go similarity index 99% rename from internal/api/charts_test.go rename to internal/api/chartapi/charts_test.go index 5fad04298..6ec7b1fc1 100644 --- a/internal/api/charts_test.go +++ b/internal/api/chartapi/charts_test.go @@ -1,4 +1,4 @@ -package api +package chartapi import ( "encoding/json" diff --git a/internal/api/chartapi/service.go b/internal/api/chartapi/service.go new file mode 100644 index 000000000..3cde9594e --- /dev/null +++ b/internal/api/chartapi/service.go @@ -0,0 +1,3265 @@ +package chartapi + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" + "github.com/rcourtman/pulse-go-rewrite/internal/mock" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" + unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rs/zerolog/log" + "golang.org/x/sync/singleflight" +) + +const workloadChartsCacheTTL = 3 * time.Second +const summaryChartsCacheTTL = 5 * time.Second + +type cachedWorkloadChartsEntry struct { + body []byte + cachedAt time.Time +} + +type summaryChartsCacheEntry struct { + payload []byte + expiresAt time.Time +} + +// MonitorResolver supplies the authenticated tenant monitor selected by the +// router context. Chart computation remains entirely owned by Service. +type MonitorResolver interface { + MonitorForContext(context.Context) *monitoring.Monitor +} + +// Service owns chart queries, aggregation, serialization, caching, and +// singleflight coordination independently of the HTTP router package. +type Service struct { + resolver MonitorResolver + + infrastructureChartsMu sync.Mutex + infrastructureCharts map[string]summaryChartsCacheEntry + workloadsSummaryChartsMu sync.Mutex + workloadsSummaryCharts map[string]summaryChartsCacheEntry + workloadChartsComputeGroup singleflight.Group + workloadChartsCacheMu sync.RWMutex + workloadChartsCache map[string]cachedWorkloadChartsEntry +} + +func NewService(resolver MonitorResolver) *Service { + return &Service{resolver: resolver} +} + +func (r *Service) getTenantMonitor(ctx context.Context) *monitoring.Monitor { + if r == nil || r.resolver == nil { + return nil + } + return r.resolver.MonitorForContext(ctx) +} + +func storageChartsSelectedNodeName(resource unifiedresources.Resource) string { + if name := strings.TrimSpace(resource.Name); name != "" { + return name + } + if resource.TrueNAS != nil { + if hostname := strings.TrimSpace(resource.TrueNAS.Hostname); hostname != "" { + return hostname + } + } + for _, hostname := range resource.Identity.Hostnames { + if hostname = strings.TrimSpace(hostname); hostname != "" { + return hostname + } + } + return "" +} + +func storageChartsSelectedNodeInstance(resource unifiedresources.Resource) string { + if resource.Proxmox == nil { + return "" + } + return strings.TrimSpace(resource.Proxmox.Instance) +} + +// handleCharts handles chart data requests +func (r *Service) HandleCharts(w http.ResponseWriter, req *http.Request) { + log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Charts endpoint hit") + const inMemoryChartThreshold = 2 * time.Hour + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Get time range from query parameters + query := req.URL.Query() + timeRange := query.Get("range") + if timeRange == "" { + timeRange = "1h" + } + + // Convert time range to duration. + duration := parseChartsRangeDuration(timeRange) + + // Get tenant-specific monitor and current state + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) + return + } + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + http.Error(w, "State unavailable", http.StatusInternalServerError) + return + } + metricsStoreEnabled := monitor.GetMetricsStore() != nil + primarySourceHint := "memory" + if metricsStoreEnabled && duration > inMemoryChartThreshold { + primarySourceHint = "store_or_memory_fallback" + } + + // Create chart data structure that matches frontend expectations + chartData := make(map[string]VMChartData) + nodeData := make(map[string]NodeChartData) + + currentTime := time.Now().UnixMilli() // JavaScript timestamp format + oldestTimestamp := currentTime + + // Process VMs and Containers - batch-load historical data (1-2 SQL calls + // per family instead of N). + oldestTimestamp = collectGuestChartData(monitor, "vm", readState.VMs(), duration, chartData, currentTime, oldestTimestamp) + oldestTimestamp = collectGuestChartData(monitor, "container", readState.Containers(), duration, chartData, currentTime, oldestTimestamp) + + // Process Storage - batch-load historical data (1-2 SQL calls instead of N). + storageData := make(map[string]StorageChartData) + spList := readState.StoragePools() + storageIDs := make([]string, 0, len(spList)) + for _, sp := range spList { + if sp == nil { + continue + } + if sid := sp.SourceID(); sid != "" { + storageIDs = append(storageIDs, sid) + } + } + storageBatchMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration) + for _, sp := range spList { + if sp == nil { + continue + } + sid := sp.SourceID() + if sid == "" { + continue + } + storageData[sid] = make(StorageChartData) + if batchMetrics, ok := storageBatchMetrics[sid]; ok { + if usagePoints, found := batchMetrics["usage"]; found && len(usagePoints) > 0 { + storageData[sid]["disk"] = make([]MetricPoint, len(usagePoints)) + for i, point := range usagePoints { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + storageData[sid]["disk"][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + } + if len(storageData[sid]["disk"]) == 0 { + storageData[sid]["disk"] = []MetricPoint{ + {Timestamp: currentTime, Value: sp.DiskPercent()}, + } + } + } + + // Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5). + nodeMetricTypes := []string{"cpu", "memory", "disk", "netin", "netout"} + nodeList := readState.Nodes() + nodeIDs := make([]string, 0, len(nodeList)) + for _, node := range nodeList { + if node == nil { + continue + } + if nid := node.SourceID(); nid != "" { + nodeIDs = append(nodeIDs, nid) + } + } + nodeBatchMetrics := monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration) + for _, node := range nodeList { + if node == nil { + continue + } + nid := node.SourceID() + if nid == "" { + continue + } + nodeData[nid] = make(NodeChartData) + if batchMetrics, ok := nodeBatchMetrics[nid]; ok { + for _, metricType := range nodeMetricTypes { + points, found := batchMetrics[metricType] + if !found { + continue + } + nodeData[nid][metricType] = make([]MetricPoint, len(points)) + for i, point := range points { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + nodeData[nid][metricType][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + } + for _, metricType := range nodeMetricTypes { + if len(nodeData[nid][metricType]) == 0 { + var value float64 + hasFallbackValue := true + switch metricType { + case "cpu": + value = node.CPUPercent() + case "memory": + value = node.MemoryPercent() + case "disk": + value = node.DiskPercent() + default: + hasFallbackValue = false + } + if hasFallbackValue { + nodeData[nid][metricType] = []MetricPoint{ + {Timestamp: currentTime, Value: value}, + } + } + } + } + } + + // Build guest type map with canonical v6 names. + guestTypes := make(map[string]string) + for _, vm := range readState.VMs() { + if vm == nil { + continue + } + if sid := vm.SourceID(); sid != "" { + guestTypes[sid] = "vm" + } + } + for _, ct := range readState.Containers() { + if ct == nil { + continue + } + if sid := ct.SourceID(); sid != "" { + guestTypes[sid] = "system-container" + } + } + for _, dc := range readState.DockerContainers() { + if dc == nil { + continue + } + if key := strings.TrimSpace(dc.ID()); key != "" { + guestTypes[key] = "app-container" + } + } + + // Process Docker containers - batch-load historical data (1-2 SQL calls instead of N). + dockerData := make(map[string]VMChartData) + dcList := readState.DockerContainers() + dcRequests := make([]monitoring.GuestChartRequest, 0, len(dcList)) + for _, dc := range dcList { + _, request, ok := appContainerChartRequest(dc) + if !ok { + continue + } + dcRequests = append(dcRequests, request) + } + dcBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerContainer", dcRequests, duration, infrastructureSummaryMetricOrder...) + for _, dc := range dcList { + responseKey, request, ok := appContainerChartRequest(dc) + if !ok { + continue + } + dockerData[responseKey] = make(VMChartData) + if batchMetrics, ok := dcBatchMetrics[request.SQLResourceID]; ok { + oldestTimestamp = fillChartSeriesFromBatch(dockerData[responseKey], batchMetrics, oldestTimestamp) + } + if len(dockerData[responseKey]["cpu"]) == 0 { + dockerData[responseKey]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dc.CPUPercent()}} + dockerData[responseKey]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dc.MemoryPercent()}} + dockerData[responseKey]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: dc.DiskPercent()}} + } + } + + // Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N). + dockerHostData := make(map[string]VMChartData) + dhList := readState.DockerHosts() + dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList)) + for _, dh := range dhList { + if dh == nil { + continue + } + if dhID := dh.HostSourceID(); dhID != "" { + dhRequests = append(dhRequests, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID), + SQLResourceID: dhID, + }) + } + } + dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, infrastructureSummaryMetricOrder...) + for _, dh := range dhList { + if dh == nil { + continue + } + dhID := dh.HostSourceID() + if dhID == "" { + continue + } + dockerHostData[dhID] = make(VMChartData) + if batchMetrics, ok := dhBatchMetrics[dhID]; ok { + oldestTimestamp = fillChartSeriesFromBatch(dockerHostData[dhID], batchMetrics, oldestTimestamp) + } + if len(dockerHostData[dhID]["cpu"]) == 0 { + dockerHostData[dhID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dh.CPUPercent()}} + dockerHostData[dhID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dh.MemoryPercent()}} + var diskPercent float64 + if disks := dh.Disks(); len(disks) > 0 { + diskPercent = disks[0].Usage + } + dockerHostData[dhID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: diskPercent}} + } + } + + // Process unified agents - batch-load historical data (1-2 SQL calls instead of N). + agentData := make(map[string]VMChartData) + hostList := readState.Hosts() + agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList)) + for _, h := range hostList { + _, request, ok := hostAgentChartRequest(h) + if !ok { + continue + } + agentRequests = append(agentRequests, request) + } + agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, infrastructureSummaryMetricOrder...) + for _, h := range hostList { + hID, request, ok := hostAgentChartRequest(h) + if !ok { + continue + } + agentData[hID] = make(VMChartData) + if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok { + oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp) + } + if len(agentData[hID]["cpu"]) == 0 { + agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}} + agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}} + agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: h.DiskPercent()}} + } + } + + countChartPoints := func(metricsMap map[string]VMChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + + countNodePoints := func(metricsMap map[string]NodeChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + + countStoragePoints := func(metricsMap map[string]StorageChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + + guestPoints := countChartPoints(chartData) + nodePoints := countNodePoints(nodeData) + storagePoints := countStoragePoints(storageData) + dockerContainerPoints := countChartPoints(dockerData) + dockerHostPoints := countChartPoints(dockerHostData) + agentPoints := countChartPoints(agentData) + + response := ChartResponse{ + ChartData: chartData, + NodeData: nodeData, + StorageData: storageData, + DockerData: dockerData, + DockerHostData: dockerHostData, + AgentData: agentData, + GuestTypes: guestTypes, + Timestamp: currentTime, + Stats: ChartStats{ + OldestDataTimestamp: oldestTimestamp, + Range: timeRange, + RangeSeconds: int64(duration / time.Second), + MetricsStoreEnabled: metricsStoreEnabled, + PrimarySourceHint: primarySourceHint, + InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), + PointCounts: ChartPointCounts{ + Total: guestPoints + nodePoints + storagePoints + dockerContainerPoints + dockerHostPoints + agentPoints, + Guests: guestPoints, + Nodes: nodePoints, + Storage: storagePoints, + DockerContainers: dockerContainerPoints, + DockerHosts: dockerHostPoints, + Agents: agentPoints, + }, + }, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + log.Error().Err(err).Msg("Failed to encode chart data response") + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + log.Debug(). + Int("guests", len(chartData)). + Int("nodes", len(nodeData)). + Int("storage", len(storageData)). + Int("dockerContainers", len(dockerData)). + Int("agents", len(agentData)). + Str("range", timeRange). + Msg("Chart data response sent") +} + +func parseWorkloadMaxPoints(raw string) int { + const ( + defaultMaxPoints = 180 + minMaxPoints = 30 + maxMaxPoints = 500 + ) + + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return defaultMaxPoints + } + + value, err := strconv.Atoi(trimmed) + if err != nil { + return defaultMaxPoints + } + if value < minMaxPoints { + return minMaxPoints + } + if value > maxMaxPoints { + return maxMaxPoints + } + return value +} + +func ParseWorkloadMaxPoints(raw string) int { return parseWorkloadMaxPoints(raw) } + +func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) { + if host == nil { + return "", monitoring.GuestChartRequest{}, false + } + + if agentID := strings.TrimSpace(host.AgentID()); agentID != "" { + return agentID, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("agent:%s", agentID), + SQLResourceID: agentID, + }, true + } + + target := host.MetricsTarget() + if target == nil { + return "", monitoring.GuestChartRequest{}, false + } + + metricID := strings.TrimSpace(target.ResourceID) + if metricID == "" { + return "", monitoring.GuestChartRequest{}, false + } + + return metricID, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("agent:%s", metricID), + SQLResourceID: metricID, + }, true +} + +func appContainerChartMetricID(container *unifiedresources.DockerContainerView) string { + if container == nil { + return "" + } + + if target := container.MetricsTarget(); target != nil { + if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { + return metricID + } + } + + return strings.TrimSpace(container.ContainerID()) +} + +func appContainerChartRequest(container *unifiedresources.DockerContainerView) (string, monitoring.GuestChartRequest, bool) { + if container == nil { + return "", monitoring.GuestChartRequest{}, false + } + + responseKey := strings.TrimSpace(container.ID()) + if responseKey == "" { + responseKey = strings.TrimSpace(container.ContainerID()) + } + metricID := appContainerChartMetricID(container) + if responseKey == "" || metricID == "" { + return "", monitoring.GuestChartRequest{}, false + } + + return responseKey, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("docker:%s", metricID), + SQLResourceID: metricID, + }, true +} + +func canonicalGuestResponseKey(resourceID, instance, node string, vmid int) string { + trimmedInstance := strings.TrimSpace(instance) + trimmedNode := strings.TrimSpace(node) + if trimmedInstance != "" && trimmedNode != "" && vmid > 0 { + return fmt.Sprintf("%s:%s:%d", trimmedInstance, trimmedNode, vmid) + } + return strings.TrimSpace(resourceID) +} + +func vmChartMetricID(vm *unifiedresources.VMView) string { + if vm == nil { + return "" + } + + if target := vm.MetricsTarget(); target != nil { + if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { + return metricID + } + } + + return strings.TrimSpace(vm.SourceID()) +} + +func vmChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) { + if vm == nil { + return "", monitoring.GuestChartRequest{}, false + } + + responseKey := canonicalGuestResponseKey(vm.ID(), vm.Instance(), vm.Node(), vm.VMID()) + metricID := vmChartMetricID(vm) + if responseKey == "" || metricID == "" { + return "", monitoring.GuestChartRequest{}, false + } + + return responseKey, monitoring.GuestChartRequest{ + InMemoryKey: metricID, + SQLResourceID: metricID, + }, true +} + +func VMChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) { + return vmChartRequest(vm) +} + +func systemContainerChartMetricID(container *unifiedresources.ContainerView) string { + if container == nil { + return "" + } + + if target := container.MetricsTarget(); target != nil { + if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { + return metricID + } + } + + return strings.TrimSpace(container.SourceID()) +} + +func systemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) { + if container == nil { + return "", monitoring.GuestChartRequest{}, false + } + + responseKey := canonicalGuestResponseKey(container.ID(), container.Instance(), container.Node(), container.VMID()) + metricID := systemContainerChartMetricID(container) + if responseKey == "" || metricID == "" { + return "", monitoring.GuestChartRequest{}, false + } + + return responseKey, monitoring.GuestChartRequest{ + InMemoryKey: metricID, + SQLResourceID: metricID, + }, true +} + +func SystemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) { + return systemContainerChartRequest(container) +} + +func capMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint { + if len(points) <= maxPoints || maxPoints <= 0 { + return points + } + if maxPoints == 1 { + return []MetricPoint{points[len(points)-1]} + } + + result := make([]MetricPoint, 0, maxPoints) + step := float64(len(points)-1) / float64(maxPoints-1) + prevIndex := -1 + + for i := 0; i < maxPoints; i++ { + index := int(float64(i)*step + 0.5) + if index <= prevIndex { + index = prevIndex + 1 + } + if index >= len(points) { + index = len(points) - 1 + } + result = append(result, points[index]) + prevIndex = index + } + + if result[len(result)-1].Timestamp != points[len(points)-1].Timestamp { + result[len(result)-1] = points[len(points)-1] + } + return result +} + +func CapMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint { + return capMetricPointSeriesByIndex(points, maxPoints) +} + +const ( + infrastructureSummaryMinSeriesPoints = 24 + infrastructureSummaryMaxSeriesPoints = 96 + workloadsSummaryMinSeriesPoints = 24 + workloadsSummaryMaxSeriesPoints = 96 +) + +const InfrastructureSummaryMaxSeriesPoints = infrastructureSummaryMaxSeriesPoints +const WorkloadsSummaryMaxSeriesPoints = workloadsSummaryMaxSeriesPoints + +// capMetricPointSeries keeps mixed-cadence series visually proportional across +// the selected time window. Index-based capping over-selects recent dense +// samples, which bunches the right edge on long ranges. +func capMetricPointSeries(points []MetricPoint, maxPoints int) []MetricPoint { + if len(points) <= maxPoints || maxPoints <= 0 { + return points + } + if maxPoints == 1 { + return []MetricPoint{points[len(points)-1]} + } + + startTimestamp := points[0].Timestamp + endTimestamp := points[len(points)-1].Timestamp + if endTimestamp <= startTimestamp { + return capMetricPointSeriesByIndex(points, maxPoints) + } + + bucketSpan := float64(endTimestamp-startTimestamp) / float64(maxPoints-1) + if bucketSpan < 1 { + return capMetricPointSeriesByIndex(points, maxPoints) + } + + type timeBucketRepresentative struct { + point MetricPoint + distance float64 + ok bool + } + + buckets := make([]timeBucketRepresentative, maxPoints) + for _, point := range points { + index := int(math.Round(float64(point.Timestamp-startTimestamp) / bucketSpan)) + if index < 0 { + index = 0 + } + if index >= maxPoints { + index = maxPoints - 1 + } + + targetTimestamp := float64(startTimestamp) + bucketSpan*float64(index) + distance := math.Abs(float64(point.Timestamp) - targetTimestamp) + current := buckets[index] + if !current.ok || + distance < current.distance || + (distance == current.distance && point.Timestamp > current.point.Timestamp) { + buckets[index] = timeBucketRepresentative{ + point: point, + distance: distance, + ok: true, + } + } + } + + result := make([]MetricPoint, 0, maxPoints) + result = append(result, points[0]) + lastAddedTimestamp := points[0].Timestamp + for index := 1; index < maxPoints-1; index++ { + bucket := buckets[index] + if !bucket.ok { + continue + } + if bucket.point.Timestamp <= lastAddedTimestamp { + continue + } + result = append(result, bucket.point) + lastAddedTimestamp = bucket.point.Timestamp + } + + lastPoint := points[len(points)-1] + if lastPoint.Timestamp <= lastAddedTimestamp { + result[len(result)-1] = lastPoint + return result + } + + result = append(result, lastPoint) + return result +} + +func targetBoundedSummarySeriesPoints(duration time.Duration, minPoints, maxPoints int) int { + if duration <= 0 { + return minPoints + } + + target := int(duration / time.Minute) + if target < minPoints { + target = minPoints + } + if target > maxPoints { + target = maxPoints + } + if target < 2 { + target = 2 + } + return target +} + +type infrastructureSummaryBucket struct { + count int + sum float64 + max float64 + firstTimestamp int64 + lastTimestamp int64 + lastValue float64 +} + +func targetInfrastructureSummarySeriesPoints(duration time.Duration) int { + return targetBoundedSummarySeriesPoints( + duration, + infrastructureSummaryMinSeriesPoints, + infrastructureSummaryMaxSeriesPoints, + ) +} + +func infrastructureChartsCacheKey(req *http.Request, timeRange string, requestedMetricNames []string) string { + orgID := strings.TrimSpace(apicontext.OrgID(req.Context())) + if orgID == "" { + orgID = "default" + } + return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.Join(requestedMetricNames, ",") +} + +func (r *Service) cachedInfrastructureChartsPayload(key string, now time.Time) ([]byte, bool) { + if r == nil || key == "" { + return nil, false + } + r.infrastructureChartsMu.Lock() + defer r.infrastructureChartsMu.Unlock() + + entry, ok := r.infrastructureCharts[key] + if !ok { + return nil, false + } + if !now.Before(entry.expiresAt) { + delete(r.infrastructureCharts, key) + return nil, false + } + return entry.payload, true +} + +func (r *Service) cacheInfrastructureChartsPayload(key string, payload []byte, now time.Time) { + if r == nil || key == "" || len(payload) == 0 { + return + } + r.infrastructureChartsMu.Lock() + defer r.infrastructureChartsMu.Unlock() + if r.infrastructureCharts == nil { + r.infrastructureCharts = make(map[string]summaryChartsCacheEntry, 8) + } + r.infrastructureCharts[key] = summaryChartsCacheEntry{ + payload: payload, + expiresAt: now.Add(summaryChartsCacheTTL), + } +} + +func targetWorkloadsSummarySeriesPoints(duration time.Duration) int { + return targetBoundedSummarySeriesPoints( + duration, + workloadsSummaryMinSeriesPoints, + workloadsSummaryMaxSeriesPoints, + ) +} + +func workloadsSummaryChartsCacheKey(req *http.Request, timeRange, selectedNodeID string) string { + orgID := strings.TrimSpace(apicontext.OrgID(req.Context())) + if orgID == "" { + orgID = "default" + } + return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.TrimSpace(selectedNodeID) +} + +func (r *Service) cachedWorkloadsSummaryChartsPayload(key string, now time.Time) ([]byte, bool) { + if r == nil || key == "" { + return nil, false + } + r.workloadsSummaryChartsMu.Lock() + defer r.workloadsSummaryChartsMu.Unlock() + + entry, ok := r.workloadsSummaryCharts[key] + if !ok { + return nil, false + } + if !now.Before(entry.expiresAt) { + delete(r.workloadsSummaryCharts, key) + return nil, false + } + return entry.payload, true +} + +func (r *Service) cacheWorkloadsSummaryChartsPayload(key string, payload []byte, now time.Time) { + if r == nil || key == "" || len(payload) == 0 { + return + } + r.workloadsSummaryChartsMu.Lock() + defer r.workloadsSummaryChartsMu.Unlock() + if r.workloadsSummaryCharts == nil { + r.workloadsSummaryCharts = make(map[string]summaryChartsCacheEntry, 8) + } + r.workloadsSummaryCharts[key] = summaryChartsCacheEntry{ + payload: payload, + expiresAt: now.Add(summaryChartsCacheTTL), + } +} + +func aggregateInfrastructureSummaryBucketValue( + metricType string, + bucket infrastructureSummaryBucket, + isLastBucket bool, +) float64 { + if bucket.count == 0 { + return 0 + } + if isLastBucket { + return bucket.lastValue + } + + switch metricType { + case "memory", "disk": + return bucket.sum / float64(bucket.count) + default: + return bucket.max + } +} + +// normalizeInfrastructureSummaryMetricPointSeries folds mixed-cadence history +// into equal-time buckets for the infrastructure summary endpoint so long-range +// sparklines do not bunch recent higher-resolution samples at the right edge. +func normalizeInfrastructureSummaryMetricPointSeries( + points []MetricPoint, + metricType string, + duration time.Duration, + windowEndMillis int64, +) []MetricPoint { + targetPoints := targetInfrastructureSummarySeriesPoints(duration) + if len(points) <= targetPoints || targetPoints < 2 || duration <= 0 { + return points + } + + durationMillis := int64(duration / time.Millisecond) + if durationMillis <= 0 { + return points + } + + windowStartMillis := windowEndMillis - durationMillis + bucketCount := targetPoints + buckets := make([]infrastructureSummaryBucket, bucketCount) + firstNonEmpty := -1 + lastNonEmpty := -1 + + for _, point := range points { + if point.Timestamp < windowStartMillis || point.Timestamp > windowEndMillis { + continue + } + bucketIndex := int(((point.Timestamp - windowStartMillis) * int64(bucketCount)) / durationMillis) + if bucketIndex < 0 { + bucketIndex = 0 + } + if bucketIndex >= bucketCount { + bucketIndex = bucketCount - 1 + } + + bucket := &buckets[bucketIndex] + if bucket.count == 0 { + bucket.max = point.Value + bucket.firstTimestamp = point.Timestamp + if firstNonEmpty == -1 { + firstNonEmpty = bucketIndex + } + } else if point.Value > bucket.max { + bucket.max = point.Value + } + bucket.count++ + bucket.sum += point.Value + bucket.lastTimestamp = point.Timestamp + bucket.lastValue = point.Value + lastNonEmpty = bucketIndex + } + + if firstNonEmpty == -1 || lastNonEmpty == -1 { + return points + } + + result := make([]MetricPoint, 0, targetPoints) + for bucketIndex := 0; bucketIndex < bucketCount; bucketIndex++ { + bucket := buckets[bucketIndex] + if bucket.count == 0 { + continue + } + + bucketStartMillis := windowStartMillis + (int64(bucketIndex)*durationMillis)/int64(bucketCount) + bucketEndMillis := windowStartMillis + (int64(bucketIndex+1)*durationMillis)/int64(bucketCount) + timestamp := bucketStartMillis + (bucketEndMillis-bucketStartMillis)/2 + switch bucketIndex { + case firstNonEmpty: + timestamp = bucket.firstTimestamp + case lastNonEmpty: + timestamp = bucket.lastTimestamp + } + + result = append(result, MetricPoint{ + Timestamp: timestamp, + Value: aggregateInfrastructureSummaryBucketValue( + metricType, + bucket, + bucketIndex == lastNonEmpty, + ), + }) + } + + if len(result) == 0 { + return points + } + return result +} + +func normalizeInfrastructureSummaryChartSeries( + metrics map[string][]MetricPoint, + duration time.Duration, + windowEndMillis int64, +) { + for metricType, points := range metrics { + metrics[metricType] = normalizeInfrastructureSummaryMetricPointSeries( + points, + metricType, + duration, + windowEndMillis, + ) + } +} + +// sparklineMetrics lists the metric types consumed by summary sparklines +// and density maps. Metrics not in this set are omitted to keep payloads small. +// guestChartSourceView is the guest view subset the infrastructure summary +// chart builder consumes from VMs and LXC containers. +type guestChartSourceView interface { + comparable + SourceID() string + CPUPercent() float64 + MemoryPercent() float64 + MemoryUsed() int64 + DiskPercent() float64 + NetIn() float64 + NetOut() float64 +} + +// collectGuestChartData batch-loads sparkline history for one proxmox guest +// family into chartData (1-2 SQL calls instead of N) and returns the updated +// oldest chart timestamp. Guests without history fall back to a single +// current-value point per metric. +func collectGuestChartData[V guestChartSourceView]( + monitor *monitoring.Monitor, + storeType string, + guests []V, + duration time.Duration, + chartData map[string]VMChartData, + currentTime, oldestTimestamp int64, +) int64 { + var zero V + requests := make([]monitoring.GuestChartRequest, 0, len(guests)) + for _, g := range guests { + if g == zero { + continue + } + if id := g.SourceID(); id != "" { + requests = append(requests, monitoring.GuestChartRequest{InMemoryKey: id, SQLResourceID: id}) + } + } + batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, guestSparklineMetricOrder...) + for _, g := range guests { + if g == zero { + continue + } + id := g.SourceID() + if id == "" { + continue + } + chartData[id] = make(VMChartData) + if batchMetrics, ok := batch[id]; ok { + oldestTimestamp = fillChartSeriesFromBatch(chartData[id], batchMetrics, oldestTimestamp) + } + if len(chartData[id]["cpu"]) == 0 { + chartData[id]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: g.CPUPercent()}} + chartData[id]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: g.MemoryPercent()}} + chartData[id]["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(g.MemoryUsed())}} + chartData[id]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: g.DiskPercent()}} + chartData[id]["netin"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetIn()}} + chartData[id]["netout"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetOut()}} + } + } + return oldestTimestamp +} + +// fillChartSeriesFromBatch copies sparkline-eligible batch metric points +// into dst and returns the updated oldest chart timestamp. Shared by the +// per-family infrastructure summary chart loops. +func fillChartSeriesFromBatch(dst VMChartData, batchMetrics map[string][]monitoring.MetricPoint, oldestTimestamp int64) int64 { + for metricType, points := range batchMetrics { + if !sparklineMetrics[metricType] { + continue + } + dst[metricType] = make([]MetricPoint, len(points)) + for i, point := range points { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + dst[metricType][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + return oldestTimestamp +} + +var sparklineMetrics = map[string]bool{ + "cpu": true, + "memory": true, + "memoryused": true, + "disk": true, + "diskread": true, + "diskwrite": true, + "netin": true, + "netout": true, +} + +var infrastructureSummaryMetricOrder = []string{ + "cpu", + "memory", + "disk", + "diskread", + "diskwrite", + "netin", + "netout", +} + +var guestSparklineMetricOrder = []string{ + "cpu", + "memory", + "memoryused", + "disk", + "diskread", + "diskwrite", + "netin", + "netout", +} + +var workloadSummaryMetricOrder = []string{ + "cpu", + "memory", + "disk", + "netin", + "netout", +} + +func parseInfrastructureSummaryRequestedMetrics( + query url.Values, +) ([]string, map[string]bool, error) { + rawValues, ok := query["metrics"] + if !ok || len(rawValues) == 0 { + requested := make(map[string]bool, len(infrastructureSummaryMetricOrder)) + for _, metricType := range infrastructureSummaryMetricOrder { + requested[metricType] = true + } + return append([]string(nil), infrastructureSummaryMetricOrder...), requested, nil + } + + requestedList := make([]string, 0, len(infrastructureSummaryMetricOrder)) + requestedSet := make(map[string]bool, len(infrastructureSummaryMetricOrder)) + invalid := make([]string, 0) + + for _, rawValue := range rawValues { + for _, part := range strings.Split(rawValue, ",") { + metricType := strings.TrimSpace(strings.ToLower(part)) + if metricType == "" { + continue + } + if !sparklineMetrics[metricType] { + invalid = append(invalid, metricType) + continue + } + if requestedSet[metricType] { + continue + } + requestedSet[metricType] = true + requestedList = append(requestedList, metricType) + } + } + + if len(invalid) > 0 { + return nil, nil, fmt.Errorf("invalid infrastructure metrics filter: %s", strings.Join(invalid, ", ")) + } + if len(requestedList) == 0 { + return nil, nil, fmt.Errorf("infrastructure metrics filter must include at least one valid metric") + } + return requestedList, requestedSet, nil +} + +func convertMetricsForChart( + metrics map[string][]monitoring.MetricPoint, + oldestTimestamp *int64, + maxPoints int, +) VMChartData { + converted := make(VMChartData, len(metrics)) + for metricType, metricPoints := range metrics { + if !sparklineMetrics[metricType] { + continue + } + points := make([]MetricPoint, len(metricPoints)) + for i, point := range metricPoints { + ts := point.Timestamp.UnixMilli() + if ts < *oldestTimestamp { + *oldestTimestamp = ts + } + points[i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + converted[metricType] = capMetricPointSeries(points, maxPoints) + } + return converted +} + +// guestLiveMetricsView is the slice of the unified workload view API needed +// to seed a chart from live values; VM and container views both satisfy it. +type guestLiveMetricsView interface { + CPUPercent() float64 + MemoryPercent() float64 + MemoryUsed() int64 + DiskPercent() float64 + NetIn() float64 + NetOut() float64 +} + +// guestChartSeriesWithLiveFallback converts a guest's batched metric history +// into chart series, substituting single live-value points when no history +// exists yet so freshly added guests still chart. +func guestChartSeriesWithLiveFallback( + metrics map[string][]monitoring.MetricPoint, + guest guestLiveMetricsView, + oldestTimestamp *int64, + maxPoints int, + currentTime int64, +) VMChartData { + series := convertMetricsForChart(metrics, oldestTimestamp, maxPoints) + if len(series["cpu"]) == 0 { + series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: guest.CPUPercent()}} + series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: guest.MemoryPercent()}} + series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(guest.MemoryUsed())}} + series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: guest.DiskPercent()}} + series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetIn()}} + series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetOut()}} + } + return series +} + +const ( + mockWorkloadMinSeriesPoints = 24 + mockWorkloadMaxSeriesPoints = 180 +) + +func targetMockSeriesPoints(duration time.Duration, maxPoints int) int { + target := int(duration / (2 * time.Minute)) + if target < mockWorkloadMinSeriesPoints { + target = mockWorkloadMinSeriesPoints + } + if maxPoints > 0 && target > maxPoints { + target = maxPoints + } + if target > mockWorkloadMaxSeriesPoints { + target = mockWorkloadMaxSeriesPoints + } + if target < 2 { + target = 2 + } + return target +} + +func TargetMockSeriesPoints(duration time.Duration, maxPoints int) int { + return targetMockSeriesPoints(duration, maxPoints) +} + +// mockMetricStyle returns the series style for a given metric type. +func mockMetricStyle(metricType string) monitoring.SeriesStyle { + switch metricType { + case "cpu", "diskread", "diskwrite", "netin", "netout": + return monitoring.StyleSpiky + case "memory": + return monitoring.StylePlateau + default: + return monitoring.StyleFlat + } +} + +// generateStyledMockSeries produces a MetricPoint slice using the style-based +// generator from the monitoring package. +func generateStyledMockSeries( + nowMillis int64, + duration time.Duration, + numPoints int, + current float64, + resourceType string, + resourceID string, + metricType string, +) []MetricPoint { + style := mockMetricStyle(metricType) + + durationMillis := int64(duration / time.Millisecond) + if durationMillis <= 0 { + durationMillis = int64(time.Minute / time.Millisecond) + } + step := durationMillis / int64(numPoints-1) + if step <= 0 { + step = 1 + } + startMillis := nowMillis - durationMillis + timestamps := make([]time.Time, numPoints) + for i := 0; i < numPoints; i++ { + timestamps[i] = time.UnixMilli(startMillis + int64(i)*step) + } + values := monitoring.GenerateSeededResourceMetricSeriesForTimestamps( + current, + timestamps, + resourceType, + resourceID, + metricType, + style, + ) + points := make([]MetricPoint, numPoints) + for i := 0; i < numPoints; i++ { + points[i] = MetricPoint{ + Timestamp: startMillis + int64(i)*step, + Value: values[i], + } + } + return points +} + +func GenerateStyledMockSeries( + nowMillis int64, + duration time.Duration, + numPoints int, + current float64, + resourceType string, + resourceID string, + metricType string, +) []MetricPoint { + return generateStyledMockSeries(nowMillis, duration, numPoints, current, resourceType, resourceID, metricType) +} + +func buildSyntheticMetricHistorySeries( + now time.Time, + duration time.Duration, + maxPoints int, + resourceType string, + resourceID string, + metricType string, + current float64, +) []monitoring.MetricPoint { + switch metricType { + case "disk", "diskread", "diskwrite": + case "smart_temp": + if current <= 0 { + return nil + } + default: + return nil + } + + numPoints := targetMockSeriesPoints(duration, maxPoints) + series := generateStyledMockSeries( + now.UnixMilli(), duration, numPoints, + current, resourceType, resourceID, metricType, + ) + + converted := make([]monitoring.MetricPoint, len(series)) + for i, point := range series { + converted[i] = monitoring.MetricPoint{ + Timestamp: time.UnixMilli(point.Timestamp), + Value: point.Value, + } + } + + return converted +} + +func BuildSyntheticMetricHistorySeries( + now time.Time, + duration time.Duration, + maxPoints int, + resourceType string, + resourceID string, + metricType string, + current float64, +) []monitoring.MetricPoint { + return buildSyntheticMetricHistorySeries(now, duration, maxPoints, resourceType, resourceID, metricType, current) +} + +func buildMockWorkloadMetricHistorySeries( + now time.Time, + duration time.Duration, + maxPoints int, + resourceType string, + resourceID string, + metricType string, + current float64, +) []monitoring.MetricPoint { + switch metricType { + case "cpu", "memory", "disk": + case "diskread", "diskwrite", "netin", "netout": + default: + return nil + } + + numPoints := targetMockSeriesPoints(duration, maxPoints) + series := generateStyledMockSeries( + now.UnixMilli(), duration, numPoints, + current, resourceType, resourceID, metricType, + ) + + converted := make([]monitoring.MetricPoint, len(series)) + for i, point := range series { + converted[i] = monitoring.MetricPoint{ + Timestamp: time.UnixMilli(point.Timestamp), + Value: point.Value, + } + } + + return converted +} + +func BuildMockWorkloadMetricHistorySeries( + now time.Time, + duration time.Duration, + maxPoints int, + resourceType string, + resourceID string, + metricType string, + current float64, +) []monitoring.MetricPoint { + return buildMockWorkloadMetricHistorySeries(now, duration, maxPoints, resourceType, resourceID, metricType, current) +} + +// handleWorkloadCharts serves workload-only chart data used by workloads +// sparklines. It intentionally excludes infrastructure/storage chart payloads +// to keep requests small and stable for large fleets. +func (r *Service) HandleWorkloadCharts(w http.ResponseWriter, req *http.Request) { + log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workload charts endpoint hit") + const inMemoryChartThreshold = 2 * time.Hour + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + query := req.URL.Query() + timeRange := query.Get("range") + if timeRange == "" { + timeRange = "1h" + } + selectedNodeID := strings.TrimSpace(query.Get("node")) + maxPointsRaw := query.Get("maxPoints") + maxPoints := parseWorkloadMaxPoints(maxPointsRaw) + duration := parseChartsRangeDuration(timeRange) + + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) + return + } + + orgID := apicontext.OrgID(req.Context()) + if orgID == "" { + orgID = "default" + } + cacheKey := orgID + "|" + timeRange + "|" + selectedNodeID + "|" + maxPointsRaw + + r.workloadChartsCacheMu.RLock() + if entry, ok := r.workloadChartsCache[cacheKey]; ok && time.Since(entry.cachedAt) <= workloadChartsCacheTTL { + body := entry.body + r.workloadChartsCacheMu.RUnlock() + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(body); err != nil { + log.Error().Err(err).Msg("Failed to write cached workload chart data response") + } + return + } + r.workloadChartsCacheMu.RUnlock() + + v, err, _ := r.workloadChartsComputeGroup.Do(cacheKey, func() (any, error) { + // Re-check cache inside the singleflight barrier in case an earlier + // caller already populated it while we were queued. + r.workloadChartsCacheMu.RLock() + if entry, ok := r.workloadChartsCache[cacheKey]; ok && time.Since(entry.cachedAt) <= workloadChartsCacheTTL { + r.workloadChartsCacheMu.RUnlock() + return entry.body, nil + } + r.workloadChartsCacheMu.RUnlock() + + body, err := r.buildWorkloadChartsResponse(req.Context(), monitor, timeRange, selectedNodeID, maxPoints, duration, inMemoryChartThreshold) + if err != nil { + return nil, err + } + r.workloadChartsCacheMu.Lock() + if r.workloadChartsCache == nil { + r.workloadChartsCache = map[string]cachedWorkloadChartsEntry{} + } + r.workloadChartsCache[cacheKey] = cachedWorkloadChartsEntry{body: body, cachedAt: time.Now()} + r.workloadChartsCacheMu.Unlock() + return body, nil + }) + if err != nil { + log.Error().Err(err).Msg("Failed to build workload chart data response") + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(v.([]byte)); err != nil { + log.Error().Err(err).Msg("Failed to write workload chart data response") + } +} + +// buildWorkloadChartsResponse runs the heavy compute path for handleWorkloadCharts +// and returns the marshaled JSON body. Extracted so the handler can wrap it +// with caching + singleflight. +func (r *Service) buildWorkloadChartsResponse( + ctx context.Context, + monitor *monitoring.Monitor, + timeRange string, + selectedNodeID string, + maxPoints int, + duration time.Duration, + inMemoryChartThreshold time.Duration, +) ([]byte, error) { + _ = ctx + nodes := monitor.NodesSnapshot() + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + return nil, fmt.Errorf("state unavailable") + } + metricsStoreEnabled := monitor.GetMetricsStore() != nil + primarySourceHint := "memory" + if metricsStoreEnabled && duration > inMemoryChartThreshold { + primarySourceHint = "store_or_memory_fallback" + } + + currentTime := time.Now().UnixMilli() + oldestTimestamp := currentTime + + var selectedNode *models.Node + if selectedNodeID != "" { + for idx := range nodes { + if nodes[idx].ID == selectedNodeID { + selectedNode = &nodes[idx] + break + } + } + if selectedNode == nil { + log.Debug(). + Str("selectedNodeID", selectedNodeID). + Msg("Workload charts node filter not found in current state; falling back to global scope") + } + } + + matchesSelectedNode := func(instance, nodeName string) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) && + strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name)) + } + + matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + if host == nil { + return false + } + nodeName := strings.TrimSpace(selectedNode.Name) + if nodeName == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || + strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) + } + + matchesSelectedAgentHostView := func(host *unifiedresources.HostView) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + if host == nil { + return false + } + nodeName := strings.TrimSpace(selectedNode.Name) + if nodeName == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || + strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) || + strings.EqualFold(strings.TrimSpace(host.AgentID()), nodeName) || + strings.EqualFold(strings.TrimSpace(host.ID()), nodeName) + } + + matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + if pod == nil { + return false + } + nodeName := strings.TrimSpace(selectedNode.Name) + if nodeName == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName) + } + + chartData := make(map[string]VMChartData) + dockerData := make(map[string]VMChartData) + + guestTypes := make(map[string]string) + + vmList := make([]*unifiedresources.VMView, 0) + vmResponseKeys := make([]string, 0) + vmRequests := make([]monitoring.GuestChartRequest, 0) + for _, vm := range readState.VMs() { + if vm == nil { + continue + } + if !matchesSelectedNode(vm.Instance(), vm.Node()) { + continue + } + + responseKey, request, ok := vmChartRequest(vm) + if !ok { + continue + } + + vmList = append(vmList, vm) + vmResponseKeys = append(vmResponseKeys, responseKey) + vmRequests = append(vmRequests, request) + } + containerList := make([]*unifiedresources.ContainerView, 0) + containerResponseKeys := make([]string, 0) + containerRequests := make([]monitoring.GuestChartRequest, 0) + for _, ct := range readState.Containers() { + if ct == nil { + continue + } + if !matchesSelectedNode(ct.Instance(), ct.Node()) { + continue + } + + responseKey, request, ok := systemContainerChartRequest(ct) + if !ok { + continue + } + + containerList = append(containerList, ct) + containerResponseKeys = append(containerResponseKeys, responseKey) + containerRequests = append(containerRequests, request) + } + podList := make([]*unifiedresources.PodView, 0) + podRequests := make([]monitoring.GuestChartRequest, 0) + for _, pod := range readState.Pods() { + if pod == nil { + continue + } + if !matchesSelectedKubernetesPodView(pod) { + continue + } + + metricKey := kubernetesPodMetricIDFromView(pod) + if metricKey == "" { + continue + } + + podList = append(podList, pod) + podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey}) + } + dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts())) + for _, host := range readState.DockerHosts() { + if host == nil { + continue + } + dockerHostsByID[host.ID()] = host + } + agentHostsByID := make(map[string]*unifiedresources.HostView, len(readState.Hosts())) + for _, host := range readState.Hosts() { + if host == nil { + continue + } + agentHostsByID[host.ID()] = host + } + + dockerContainerList := make([]*unifiedresources.DockerContainerView, 0) + dockerContainerRequests := make([]monitoring.GuestChartRequest, 0) + dockerContainerKeys := make([]string, 0) + for _, container := range readState.DockerContainers() { + if container == nil { + continue + } + + if selectedNodeID != "" && selectedNode != nil { + host := dockerHostsByID[container.ParentID()] + if host != nil { + if !matchesSelectedDockerHostView(host) { + continue + } + } else { + agentHost := agentHostsByID[container.ParentID()] + if agentHost == nil || !matchesSelectedAgentHostView(agentHost) { + continue + } + } + } + + responseKey, request, ok := appContainerChartRequest(container) + if !ok { + continue + } + dockerContainerList = append(dockerContainerList, container) + dockerContainerKeys = append(dockerContainerKeys, responseKey) + dockerContainerRequests = append(dockerContainerRequests, request) + } + var ( + vmBatchMetrics map[string]map[string][]monitoring.MetricPoint + containerBatchMetrics map[string]map[string][]monitoring.MetricPoint + podBatchMetrics map[string]map[string][]monitoring.MetricPoint + dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint + ) + var workloadChartsBatchWG sync.WaitGroup + workloadChartsBatchWG.Add(4) + go func() { + defer workloadChartsBatchWG.Done() + vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, guestSparklineMetricOrder...) + }() + go func() { + defer workloadChartsBatchWG.Done() + containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, guestSparklineMetricOrder...) + }() + go func() { + defer workloadChartsBatchWG.Done() + podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...) + }() + go func() { + defer workloadChartsBatchWG.Done() + dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, infrastructureSummaryMetricOrder...) + }() + workloadChartsBatchWG.Wait() + + for idx, vm := range vmList { + responseKey := vmResponseKeys[idx] + metricID := vmRequests[idx].SQLResourceID + guestTypes[responseKey] = "vm" + chartData[responseKey] = guestChartSeriesWithLiveFallback(vmBatchMetrics[metricID], vm, &oldestTimestamp, maxPoints, currentTime) + } + + for idx, ct := range containerList { + responseKey := containerResponseKeys[idx] + metricID := containerRequests[idx].SQLResourceID + guestTypes[responseKey] = "system-container" + chartData[responseKey] = guestChartSeriesWithLiveFallback(containerBatchMetrics[metricID], ct, &oldestTimestamp, maxPoints, currentTime) + } + + for _, pod := range podList { + metricKey := kubernetesPodMetricIDFromView(pod) + series := convertMetricsForChart(podBatchMetrics[metricKey], &oldestTimestamp, maxPoints) + guestTypes[metricKey] = "k8s" + + if len(series["cpu"]) == 0 { + series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: pod.CPUPercent()}} + series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: pod.MemoryPercent()}} + series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: pod.DiskPercent()}} + series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetInRate()}} + series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetOutRate()}} + } + chartData[metricKey] = series + } + + for idx, container := range dockerContainerList { + responseKey := dockerContainerKeys[idx] + metricID := dockerContainerRequests[idx].SQLResourceID + series := convertMetricsForChart(dockerContainerBatchMetrics[metricID], &oldestTimestamp, maxPoints) + guestTypes[responseKey] = "app-container" + + if len(series["cpu"]) == 0 { + series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: container.CPUPercent()}} + series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: container.MemoryPercent()}} + series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: container.DiskPercent()}} + series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetInRate()}} + series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetOutRate()}} + } + dockerData[responseKey] = series + } + + countChartPoints := func(metricsMap map[string]VMChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + + guestPoints := countChartPoints(chartData) + dockerContainerPoints := countChartPoints(dockerData) + + response := EmptyWorkloadChartsResponse() + response.ChartData = chartData + response.DockerData = dockerData + response.GuestTypes = guestTypes + response.Timestamp = currentTime + response.Stats = ChartStats{ + OldestDataTimestamp: oldestTimestamp, + Range: timeRange, + RangeSeconds: int64(duration / time.Second), + MetricsStoreEnabled: metricsStoreEnabled, + PrimarySourceHint: primarySourceHint, + InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), + PointCounts: ChartPointCounts{ + Total: guestPoints + dockerContainerPoints, + Guests: guestPoints, + DockerContainers: dockerContainerPoints, + }, + } + + body, err := json.Marshal(response.NormalizeCollections()) + if err != nil { + return nil, fmt.Errorf("marshal workload chart response: %w", err) + } + return body, nil +} + +// parseChartsRangeDuration converts the UI chart range query (e.g. "5m", "1h") +// into a duration. This is shared by /api/charts and /api/charts/infrastructure +// to prevent drift. +func parseChartsRangeDuration(rangeStr string) time.Duration { + switch rangeStr { + case "5m": + return 5 * time.Minute + case "15m": + return 15 * time.Minute + case "30m": + return 30 * time.Minute + case "1h": + return time.Hour + case "4h": + return 4 * time.Hour + case "8h": + return 8 * time.Hour + case "12h": + return 12 * time.Hour + case "24h": + return 24 * time.Hour + case "7d": + return 7 * 24 * time.Hour + case "30d": + return 30 * 24 * time.Hour + default: + return time.Hour + } +} + +// handleInfrastructureCharts serves infrastructure-only chart data. +// This is intentionally narrower than /api/charts to reduce payload size and server-side compute +// for the Infrastructure page summary cards. +func (r *Service) HandleInfrastructureCharts(w http.ResponseWriter, req *http.Request) { + log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Infrastructure charts endpoint hit") + const inMemoryChartThreshold = 2 * time.Hour + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Get time range from query parameters + query := req.URL.Query() + timeRange := query.Get("range") + if timeRange == "" { + timeRange = "1h" + } + requestedMetricNames, requestedMetrics, err := parseInfrastructureSummaryRequestedMetrics(query) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Convert time range to duration. + duration := parseChartsRangeDuration(timeRange) + + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) + return + } + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + http.Error(w, "State unavailable", http.StatusInternalServerError) + return + } + metricsStoreEnabled := monitor.GetMetricsStore() != nil + primarySourceHint := "memory" + if metricsStoreEnabled && duration > inMemoryChartThreshold { + primarySourceHint = "store_or_memory_fallback" + } + + now := time.Now() + cacheKey := infrastructureChartsCacheKey(req, timeRange, requestedMetricNames) + if payload, ok := r.cachedInfrastructureChartsPayload(cacheKey, now); ok { + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(payload); err != nil { + log.Error().Err(err).Msg("Failed to write cached infrastructure chart data response") + } + return + } + + currentTime := now.UnixMilli() + oldestTimestamp := currentTime + + // Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5). + nodeMetricTypes := make([]string, 0, 5) + for _, metricType := range []string{"cpu", "memory", "disk", "netin", "netout"} { + if requestedMetrics[metricType] { + nodeMetricTypes = append(nodeMetricTypes, metricType) + } + } + nodeData := make(map[string]NodeChartData) + nodeList := readState.Nodes() + nodeIDs := make([]string, 0, len(nodeList)) + for _, node := range nodeList { + if node == nil { + continue + } + if nid := node.SourceID(); nid != "" { + nodeIDs = append(nodeIDs, nid) + } + } + nodeBatchMetrics := map[string]map[string][]monitoring.MetricPoint{} + if len(nodeMetricTypes) > 0 { + nodeBatchMetrics = monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration) + } + for _, node := range nodeList { + if node == nil { + continue + } + nid := node.SourceID() + if nid == "" { + continue + } + nodeData[nid] = make(NodeChartData) + if batchMetrics, ok := nodeBatchMetrics[nid]; ok { + for _, metricType := range nodeMetricTypes { + points, found := batchMetrics[metricType] + if !found { + continue + } + nodeData[nid][metricType] = make([]MetricPoint, len(points)) + for i, point := range points { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + nodeData[nid][metricType][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + } + for _, metricType := range nodeMetricTypes { + if len(nodeData[nid][metricType]) > 0 { + continue + } + var value float64 + hasFallbackValue := true + switch metricType { + case "cpu": + value = node.CPUPercent() + case "memory": + value = node.MemoryPercent() + case "disk": + value = node.DiskPercent() + default: + hasFallbackValue = false + } + if hasFallbackValue { + nodeData[nid][metricType] = []MetricPoint{ + {Timestamp: currentTime, Value: value}, + } + } + } + normalizeInfrastructureSummaryChartSeries(nodeData[nid], duration, currentTime) + } + + // Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N). + dockerHostData := make(map[string]VMChartData) + dhList := readState.DockerHosts() + dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList)) + for _, dh := range dhList { + if dh == nil { + continue + } + if dhID := dh.HostSourceID(); dhID != "" { + dhRequests = append(dhRequests, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID), + SQLResourceID: dhID, + }) + } + } + dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, requestedMetricNames...) + for _, dh := range dhList { + if dh == nil { + continue + } + dhID := dh.HostSourceID() + if dhID == "" { + continue + } + dockerHostData[dhID] = make(VMChartData) + if batchMetrics, ok := dhBatchMetrics[dhID]; ok { + for metricType, points := range batchMetrics { + if !requestedMetrics[metricType] { + continue + } + dockerHostData[dhID][metricType] = make([]MetricPoint, len(points)) + for i, point := range points { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + dockerHostData[dhID][metricType][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + } + for _, metricType := range requestedMetricNames { + if len(dockerHostData[dhID][metricType]) > 0 { + continue + } + var value float64 + hasFallbackValue := true + switch metricType { + case "cpu": + value = dh.CPUPercent() + case "memory": + value = dh.MemoryPercent() + case "disk": + if disks := dh.Disks(); len(disks) > 0 { + value = disks[0].Usage + } + default: + hasFallbackValue = false + } + if hasFallbackValue { + dockerHostData[dhID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}} + } + } + normalizeInfrastructureSummaryChartSeries(dockerHostData[dhID], duration, currentTime) + } + + // Process unified agents - batch-load historical data (1-2 SQL calls instead of N). + agentData := make(map[string]VMChartData) + hostList := readState.Hosts() + agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList)) + for _, h := range hostList { + _, request, ok := hostAgentChartRequest(h) + if !ok { + continue + } + agentRequests = append(agentRequests, request) + } + agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, requestedMetricNames...) + for _, h := range hostList { + hID, request, ok := hostAgentChartRequest(h) + if !ok { + continue + } + agentData[hID] = make(VMChartData) + if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok { + for metricType, points := range batchMetrics { + if !requestedMetrics[metricType] { + continue + } + agentData[hID][metricType] = make([]MetricPoint, len(points)) + for i, point := range points { + ts := point.Timestamp.UnixMilli() + if ts < oldestTimestamp { + oldestTimestamp = ts + } + agentData[hID][metricType][i] = MetricPoint{ + Timestamp: ts, + Value: point.Value, + } + } + } + } + for _, metricType := range requestedMetricNames { + if len(agentData[hID][metricType]) > 0 { + continue + } + var value float64 + hasFallbackValue := true + switch metricType { + case "cpu": + value = h.CPUPercent() + case "memory": + value = h.MemoryPercent() + case "disk": + value = h.DiskPercent() + default: + hasFallbackValue = false + } + if hasFallbackValue { + agentData[hID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}} + } + } + normalizeInfrastructureSummaryChartSeries(agentData[hID], duration, currentTime) + } + + countNodePoints := func(metricsMap map[string]NodeChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + countChartPoints := func(metricsMap map[string]VMChartData) int { + total := 0 + for _, metricSeries := range metricsMap { + for _, points := range metricSeries { + total += len(points) + } + } + return total + } + + nodePoints := countNodePoints(nodeData) + dockerHostPoints := countChartPoints(dockerHostData) + agentPoints := countChartPoints(agentData) + + response := EmptyInfrastructureChartsResponse() + response.NodeData = nodeData + response.DockerHostData = dockerHostData + response.AgentData = agentData + response.Timestamp = currentTime + response.Stats = ChartStats{ + OldestDataTimestamp: oldestTimestamp, + Range: timeRange, + RangeSeconds: int64(duration / time.Second), + MetricsStoreEnabled: metricsStoreEnabled, + PrimarySourceHint: primarySourceHint, + InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), + PointCounts: ChartPointCounts{ + Total: nodePoints + dockerHostPoints + agentPoints, + Nodes: nodePoints, + DockerHosts: dockerHostPoints, + Agents: agentPoints, + }, + } + + payload, err := json.Marshal(response.NormalizeCollections()) + if err != nil { + log.Error().Err(err).Msg("Failed to encode infrastructure chart data response") + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + payload = append(payload, '\n') + r.cacheInfrastructureChartsPayload(cacheKey, payload, now) + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(payload); err != nil { + log.Error().Err(err).Msg("Failed to write infrastructure chart data response") + } +} + +type workloadSummaryBuckets struct { + cpu workloadSummaryMetricBucket + memory workloadSummaryMetricBucket + disk workloadSummaryMetricBucket + network workloadSummaryMetricBucket +} + +type workloadSummaryMetricBucket struct { + sum float64 + max float64 + count int +} + +func (bucket *workloadSummaryMetricBucket) add(value float64) { + if bucket == nil { + return + } + if bucket.count == 0 || value > bucket.max { + bucket.max = value + } + bucket.sum += value + bucket.count++ +} + +func (bucket workloadSummaryMetricBucket) average() float64 { + if bucket.count == 0 { + return 0 + } + return bucket.sum / float64(bucket.count) +} + +type workloadsSummarySnapshot struct { + id string + name string + cpu float64 + memory float64 + disk float64 + network float64 +} + +func workloadSummaryBucketTimestamp(timestampMs int64) int64 { + const bucketSizeMs = int64(30_000) + return (timestampMs / bucketSizeMs) * bucketSizeMs +} + +func clampWorkloadPercent(value float64) float64 { + if value != value { + return 0 + } + if value < 0 { + return 0 + } + if value > 100 { + return 100 + } + return value +} + +func proxmoxModelCPURatioPercent(value float64) float64 { + return clampWorkloadPercent(value * 100) +} + +func ProxmoxModelCPURatioPercent(value float64) float64 { + return proxmoxModelCPURatioPercent(value) +} + +func clampNonNegativeWorkloadValue(value float64) float64 { + if value != value { + return 0 + } + if value < 0 { + return 0 + } + return value +} + +func kubernetesPodMetricIDFromView(pod *unifiedresources.PodView) string { + if pod == nil { + return "" + } + clusterKey := strings.TrimSpace(pod.ClusterID()) + if clusterKey == "" { + clusterKey = strings.TrimSpace(pod.ClusterName()) + } + podKey := strings.TrimSpace(pod.PodUID()) + if podKey == "" { + namespace := strings.TrimSpace(pod.Namespace()) + name := strings.TrimSpace(pod.Name()) + if namespace != "" || name != "" { + podKey = fmt.Sprintf("%s/%s", namespace, name) + } + } + if clusterKey == "" || podKey == "" { + return "" + } + return fmt.Sprintf("k8s:%s:pod:%s", clusterKey, podKey) +} + +func getOrCreateWorkloadBucket(buckets map[int64]*workloadSummaryBuckets, bucketTs int64) *workloadSummaryBuckets { + if bucket, ok := buckets[bucketTs]; ok { + return bucket + } + bucket := &workloadSummaryBuckets{} + buckets[bucketTs] = bucket + return bucket +} + +func appendWorkloadMetricPoints( + buckets map[int64]*workloadSummaryBuckets, + points []monitoring.MetricPoint, + target string, + oldestTimestamp *int64, +) int { + added := 0 + for _, point := range points { + ts := point.Timestamp.UnixMilli() + if ts <= 0 { + continue + } + if ts < *oldestTimestamp { + *oldestTimestamp = ts + } + bucketTs := workloadSummaryBucketTimestamp(ts) + bucket := getOrCreateWorkloadBucket(buckets, bucketTs) + value := clampNonNegativeWorkloadValue(point.Value) + switch target { + case "cpu", "memory", "disk": + value = clampWorkloadPercent(value) + } + switch target { + case "cpu": + bucket.cpu.add(value) + case "memory": + bucket.memory.add(value) + case "disk": + bucket.disk.add(value) + case "network": + bucket.network.add(value) + } + added++ + } + return added +} + +func mergeWorkloadNetworkPoints( + netIn []monitoring.MetricPoint, + netOut []monitoring.MetricPoint, +) []monitoring.MetricPoint { + totals := make(map[int64]float64) + for _, point := range netIn { + ts := point.Timestamp.UnixMilli() + if ts <= 0 { + continue + } + totals[ts] += clampNonNegativeWorkloadValue(point.Value) + } + for _, point := range netOut { + ts := point.Timestamp.UnixMilli() + if ts <= 0 { + continue + } + totals[ts] += clampNonNegativeWorkloadValue(point.Value) + } + if len(totals) == 0 { + return nil + } + keys := make([]int64, 0, len(totals)) + for ts := range totals { + keys = append(keys, ts) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + points := make([]monitoring.MetricPoint, 0, len(keys)) + for _, ts := range keys { + points = append(points, monitoring.MetricPoint{ + Timestamp: time.UnixMilli(ts), + Value: totals[ts], + }) + } + return points +} + +func buildWorkloadsSummaryMetric( + buckets map[int64]*workloadSummaryBuckets, + selector func(*workloadSummaryBuckets) workloadSummaryMetricBucket, +) WorkloadsSummaryMetricData { + keys := make([]int64, 0, len(buckets)) + for ts := range buckets { + keys = append(keys, ts) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + + data := WorkloadsSummaryMetricData{ + P50: make([]MetricPoint, 0, len(keys)), + P95: make([]MetricPoint, 0, len(keys)), + } + for _, ts := range keys { + bucket := selector(buckets[ts]) + if bucket.count == 0 { + continue + } + data.P50 = append(data.P50, MetricPoint{ + Timestamp: ts, + Value: bucket.average(), + }) + data.P95 = append(data.P95, MetricPoint{ + Timestamp: ts, + Value: bucket.max, + }) + } + return data +} + +func summaryMetricPointCount(metric WorkloadsSummaryMetricData) int { + return len(metric.P50) + len(metric.P95) +} + +func normalizeWorkloadsSummaryMetricPointSeries( + metric WorkloadsSummaryMetricData, + duration time.Duration, +) WorkloadsSummaryMetricData { + targetPoints := targetWorkloadsSummarySeriesPoints(duration) + metric.P50 = capMetricPointSeries(metric.P50, targetPoints) + metric.P95 = capMetricPointSeries(metric.P95, targetPoints) + return metric +} + +func latestSummaryMetricValue(points []monitoring.MetricPoint, fallback float64, clamp func(float64) float64) float64 { + if len(points) == 0 { + return clamp(fallback) + } + + latest := points[0] + for i := 1; i < len(points); i++ { + if points[i].Timestamp.After(latest.Timestamp) { + latest = points[i] + } + } + return clamp(latest.Value) +} + +func buildWorkloadsTopContributors( + snapshots []workloadsSummarySnapshot, + selector func(workloadsSummarySnapshot) float64, +) []WorkloadsSummaryContributor { + contributors := make([]WorkloadsSummaryContributor, 0, len(snapshots)) + for _, snapshot := range snapshots { + value := selector(snapshot) + if value <= 0 { + continue + } + contributors = append(contributors, WorkloadsSummaryContributor{ + ID: snapshot.id, + Name: snapshot.name, + Value: value, + }) + } + + sort.Slice(contributors, func(i, j int) bool { + if contributors[i].Value == contributors[j].Value { + if contributors[i].Name == contributors[j].Name { + return contributors[i].ID < contributors[j].ID + } + return contributors[i].Name < contributors[j].Name + } + return contributors[i].Value > contributors[j].Value + }) + + if len(contributors) > 3 { + contributors = contributors[:3] + } + return contributors +} + +func buildWorkloadsBlastRadius( + snapshots []workloadsSummarySnapshot, + selector func(workloadsSummarySnapshot) float64, +) WorkloadsSummaryBlastRadius { + values := make([]float64, 0, len(snapshots)) + for _, snapshot := range snapshots { + value := selector(snapshot) + if value <= 0 { + continue + } + values = append(values, value) + } + + if len(values) == 0 { + return WorkloadsSummaryBlastRadius{ + Scope: "idle", + Top3Share: 0, + ActiveWorkloads: 0, + } + } + + sort.Slice(values, func(i, j int) bool { return values[i] > values[j] }) + total := 0.0 + for _, value := range values { + total += value + } + + topCount := 3 + if len(values) < topCount { + topCount = len(values) + } + top3 := 0.0 + for i := 0; i < topCount; i++ { + top3 += values[i] + } + + share := 0.0 + if total > 0 { + share = (top3 / total) * 100 + } + + scope := "distributed" + switch { + case share >= 80: + scope = "concentrated" + case share >= 55: + scope = "mixed" + } + + return WorkloadsSummaryBlastRadius{ + Scope: scope, + Top3Share: share, + ActiveWorkloads: len(values), + } +} + +// handleWorkloadsSummaryCharts serves compact, aggregate workload sparklines +// for the Workloads top cards. It intentionally avoids returning per-workload +// time series to keep payloads bounded for large fleets. +func (r *Service) HandleWorkloadsSummaryCharts(w http.ResponseWriter, req *http.Request) { + log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workloads summary charts endpoint hit") + const inMemoryChartThreshold = 2 * time.Hour + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + query := req.URL.Query() + timeRange := query.Get("range") + if timeRange == "" { + timeRange = "1h" + } + selectedNodeID := strings.TrimSpace(query.Get("node")) + duration := parseChartsRangeDuration(timeRange) + + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) + return + } + nodes := monitor.NodesSnapshot() + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + http.Error(w, "State unavailable", http.StatusInternalServerError) + return + } + + now := time.Now() + cacheKey := workloadsSummaryChartsCacheKey(req, timeRange, selectedNodeID) + if payload, ok := r.cachedWorkloadsSummaryChartsPayload(cacheKey, now); ok { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(payload) + return + } + + mockModeEnabled := mock.IsMockEnabled() + metricsStoreEnabled := monitor.GetMetricsStore() != nil + primarySourceHint := "memory" + if metricsStoreEnabled && duration > inMemoryChartThreshold { + primarySourceHint = "store_or_memory_fallback" + } + + currentTime := now.UnixMilli() + currentTimeTime := now + oldestTimestamp := currentTime + buckets := make(map[int64]*workloadSummaryBuckets) + guestPointCount := 0 + guestCounts := WorkloadsGuestCounts{} + snapshots := make([]workloadsSummarySnapshot, 0, len(readState.VMs())+len(readState.Containers())) + + var selectedNode *models.Node + if selectedNodeID != "" { + for idx := range nodes { + if nodes[idx].ID == selectedNodeID { + selectedNode = &nodes[idx] + break + } + } + if selectedNode == nil { + log.Debug(). + Str("selectedNodeID", selectedNodeID). + Msg("Workloads summary node filter not found in current state; falling back to global scope") + } + } + + matchesSelectedNode := func(instance, nodeName string) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) && + strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name)) + } + + matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + if host == nil { + return false + } + nodeName := strings.TrimSpace(selectedNode.Name) + if nodeName == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || + strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) + } + + matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool { + if selectedNodeID == "" { + return true + } + if selectedNode == nil { + return true + } + if pod == nil { + return false + } + nodeName := strings.TrimSpace(selectedNode.Name) + if nodeName == "" { + return false + } + return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName) + } + + vmList := make([]*unifiedresources.VMView, 0) + vmResponseKeys := make([]string, 0) + vmRequests := make([]monitoring.GuestChartRequest, 0) + for _, vm := range readState.VMs() { + if vm == nil { + continue + } + if !matchesSelectedNode(vm.Instance(), vm.Node()) { + continue + } + responseKey, request, ok := vmChartRequest(vm) + if !ok { + continue + } + vmList = append(vmList, vm) + vmResponseKeys = append(vmResponseKeys, responseKey) + vmRequests = append(vmRequests, request) + } + containerList := make([]*unifiedresources.ContainerView, 0) + containerResponseKeys := make([]string, 0) + containerRequests := make([]monitoring.GuestChartRequest, 0) + for _, ct := range readState.Containers() { + if ct == nil { + continue + } + if !matchesSelectedNode(ct.Instance(), ct.Node()) { + continue + } + responseKey, request, ok := systemContainerChartRequest(ct) + if !ok { + continue + } + containerList = append(containerList, ct) + containerResponseKeys = append(containerResponseKeys, responseKey) + containerRequests = append(containerRequests, request) + } + podList := make([]*unifiedresources.PodView, 0) + podRequests := make([]monitoring.GuestChartRequest, 0) + for _, pod := range readState.Pods() { + if pod == nil { + continue + } + if !matchesSelectedKubernetesPodView(pod) { + continue + } + + metricKey := kubernetesPodMetricIDFromView(pod) + if metricKey == "" { + continue + } + podList = append(podList, pod) + podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey}) + } + dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts())) + for _, host := range readState.DockerHosts() { + if host == nil { + continue + } + dockerHostsByID[host.ID()] = host + } + + dockerContainerList := make([]*unifiedresources.DockerContainerView, 0) + dockerContainerRequests := make([]monitoring.GuestChartRequest, 0) + for _, container := range readState.DockerContainers() { + if container == nil { + continue + } + if selectedNodeID != "" && selectedNode != nil { + host := dockerHostsByID[container.ParentID()] + if host == nil || !matchesSelectedDockerHostView(host) { + continue + } + } + containerID := strings.TrimSpace(container.ContainerID()) + if containerID == "" { + continue + } + dockerContainerList = append(dockerContainerList, container) + dockerContainerRequests = append(dockerContainerRequests, monitoring.GuestChartRequest{ + InMemoryKey: fmt.Sprintf("docker:%s", containerID), + SQLResourceID: containerID, + }) + } + var ( + vmBatchMetrics map[string]map[string][]monitoring.MetricPoint + containerBatchMetrics map[string]map[string][]monitoring.MetricPoint + podBatchMetrics map[string]map[string][]monitoring.MetricPoint + dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint + ) + var workloadsSummaryBatchWG sync.WaitGroup + workloadsSummaryBatchWG.Add(4) + go func() { + defer workloadsSummaryBatchWG.Done() + vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, workloadSummaryMetricOrder...) + }() + go func() { + defer workloadsSummaryBatchWG.Done() + containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, workloadSummaryMetricOrder...) + }() + go func() { + defer workloadsSummaryBatchWG.Done() + podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...) + }() + go func() { + defer workloadsSummaryBatchWG.Done() + dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, workloadSummaryMetricOrder...) + }() + workloadsSummaryBatchWG.Wait() + + var guestSummaryPoints int + snapshots, guestSummaryPoints = appendGuestWorkloadSummaries(vmList, vmResponseKeys, vmRequests, vmBatchMetrics, currentTimeTime, &guestCounts, buckets, snapshots, &oldestTimestamp) + guestPointCount += guestSummaryPoints + + snapshots, guestSummaryPoints = appendGuestWorkloadSummaries(containerList, containerResponseKeys, containerRequests, containerBatchMetrics, currentTimeTime, &guestCounts, buckets, snapshots, &oldestTimestamp) + guestPointCount += guestSummaryPoints + + for _, pod := range podList { + metricKey := kubernetesPodMetricIDFromView(pod) + + guestCounts.Total++ + if strings.EqualFold(pod.PodPhase(), "running") { + guestCounts.Running++ + } else { + guestCounts.Stopped++ + } + + snapshot := workloadsSummarySnapshot{ + id: metricKey, + name: strings.TrimSpace(pod.Namespace()), + cpu: clampWorkloadPercent(pod.CPUPercent()), + memory: clampWorkloadPercent(pod.MemoryPercent()), + disk: clampWorkloadPercent(pod.DiskPercent()), + network: clampNonNegativeWorkloadValue(pod.NetInRate() + pod.NetOutRate()), + } + if name := strings.TrimSpace(pod.Name()); name != "" { + if snapshot.name == "" { + snapshot.name = name + } else { + snapshot.name = fmt.Sprintf("%s/%s", snapshot.name, name) + } + } + if snapshot.name == "" { + snapshot.name = metricKey + } + + metrics := podBatchMetrics[metricKey] + cpuPoints := metrics["cpu"] + if len(cpuPoints) == 0 { + cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.CPUPercent()}} + } + memoryPoints := metrics["memory"] + if len(memoryPoints) == 0 { + memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.MemoryPercent()}} + } + diskPoints := metrics["disk"] + if len(diskPoints) == 0 { + diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.DiskPercent()}} + } + netInPoints := metrics["netin"] + if len(netInPoints) == 0 { + netInPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.NetInRate()}} + } + netOutPoints := metrics["netout"] + if len(netOutPoints) == 0 { + netOutPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.NetOutRate()}} + } + + if mockModeEnabled { + if len(cpuPoints) < mockWorkloadMinSeriesPoints { + cpuPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "cpu", snapshot.cpu) + } + if len(memoryPoints) < mockWorkloadMinSeriesPoints { + memoryPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "memory", snapshot.memory) + } + if len(diskPoints) < mockWorkloadMinSeriesPoints { + diskPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "disk", snapshot.disk) + } + if len(netInPoints) < mockWorkloadMinSeriesPoints { + netInPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "netin", pod.NetInRate()) + } + if len(netOutPoints) < mockWorkloadMinSeriesPoints { + netOutPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "netout", pod.NetOutRate()) + } + } + + networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) + + snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) + snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) + snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) + snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) + + guestPointCount += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, diskPoints, "disk", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, networkPoints, "network", &oldestTimestamp) + snapshots = append(snapshots, snapshot) + } + + for _, container := range dockerContainerList { + containerID := strings.TrimSpace(container.ContainerID()) + guestCounts.Total++ + containerState := strings.TrimSpace(container.ContainerState()) + isRunning := workloadSummaryStatusIsRunning(containerState, container.Status()) + if !isRunning && containerState == "" { + isRunning = container.CPUPercent() > 0 || + container.MemoryPercent() > 0 || + container.NetInRate() > 0 || + container.NetOutRate() > 0 + } + if isRunning { + guestCounts.Running++ + } else { + guestCounts.Stopped++ + } + + snapshot := workloadsSummarySnapshot{ + id: containerID, + name: strings.TrimSpace(container.Name()), + cpu: clampWorkloadPercent(container.CPUPercent()), + memory: clampWorkloadPercent(container.MemoryPercent()), + disk: clampWorkloadPercent(container.DiskPercent()), + network: 0, + } + if snapshot.name == "" { + snapshot.name = containerID + } + + metrics := dockerContainerBatchMetrics[containerID] + cpuPoints := metrics["cpu"] + if len(cpuPoints) == 0 { + cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.CPUPercent()}} + } + memoryPoints := metrics["memory"] + if len(memoryPoints) == 0 { + memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.MemoryPercent()}} + } + diskPoints := metrics["disk"] + if len(diskPoints) == 0 { + diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.DiskPercent()}} + } + netInPoints := metrics["netin"] + netOutPoints := metrics["netout"] + + networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) + + snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) + snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) + snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) + snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) + + guestPointCount += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, diskPoints, "disk", &oldestTimestamp) + guestPointCount += appendWorkloadMetricPoints(buckets, networkPoints, "network", &oldestTimestamp) + snapshots = append(snapshots, snapshot) + } + + cpuMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { + return bucket.cpu + }) + memoryMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { + return bucket.memory + }) + diskMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { + return bucket.disk + }) + networkMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { + return bucket.network + }) + cpuMetric = normalizeWorkloadsSummaryMetricPointSeries(cpuMetric, duration) + memoryMetric = normalizeWorkloadsSummaryMetricPointSeries(memoryMetric, duration) + diskMetric = normalizeWorkloadsSummaryMetricPointSeries(diskMetric, duration) + networkMetric = normalizeWorkloadsSummaryMetricPointSeries(networkMetric, duration) + + summaryPointCount := summaryMetricPointCount(cpuMetric) + + summaryMetricPointCount(memoryMetric) + + summaryMetricPointCount(diskMetric) + + summaryMetricPointCount(networkMetric) + + topContributors := WorkloadsSummaryContributors{ + CPU: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.cpu + }), + Memory: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.memory + }), + Disk: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.disk + }), + Network: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.network + }), + } + + blastRadius := WorkloadsSummaryBlastRadiusGroup{ + CPU: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.cpu + }), + Memory: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.memory + }), + Disk: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.disk + }), + Network: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { + return snapshot.network + }), + } + + response := EmptyWorkloadsSummaryChartsResponse() + response.CPU = cpuMetric + response.Memory = memoryMetric + response.Disk = diskMetric + response.Network = networkMetric + response.GuestCounts = guestCounts + response.TopContributors = topContributors + response.BlastRadius = blastRadius + response.Timestamp = currentTime + response.Stats = ChartStats{ + OldestDataTimestamp: oldestTimestamp, + Range: timeRange, + RangeSeconds: int64(duration / time.Second), + MetricsStoreEnabled: metricsStoreEnabled, + PrimarySourceHint: primarySourceHint, + InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), + PointCounts: ChartPointCounts{ + Total: summaryPointCount, + Guests: guestPointCount, + }, + } + + payload, err := json.Marshal(response.NormalizeCollections()) + if err != nil { + log.Error().Err(err).Msg("Failed to encode workloads summary chart data response") + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + r.cacheWorkloadsSummaryChartsPayload(cacheKey, payload, now) + + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(payload); err != nil { + log.Error().Err(err).Msg("Failed to encode workloads summary chart data response") + return + } +} + +// guestWorkloadSummaryView is the guest view subset the workloads summary +// loop consumes from VMs and LXC containers. +type guestWorkloadSummaryView interface { + Status() unifiedresources.ResourceStatus + Name() string + CPUPercent() float64 + MemoryPercent() float64 + DiskPercent() float64 + NetIn() float64 + NetOut() float64 +} + +// appendGuestWorkloadSummaries accumulates workload-summary snapshots and +// chart points for one proxmox guest family (VMs or LXC containers), +// returning the extended snapshot slice and the number of points added. +func appendGuestWorkloadSummaries[V guestWorkloadSummaryView]( + guests []V, + responseKeys []string, + requests []monitoring.GuestChartRequest, + batchMetrics map[string]map[string][]monitoring.MetricPoint, + currentTimeTime time.Time, + guestCounts *WorkloadsGuestCounts, + buckets map[int64]*workloadSummaryBuckets, + snapshots []workloadsSummarySnapshot, + oldestTimestamp *int64, +) ([]workloadsSummarySnapshot, int) { + added := 0 + for idx, g := range guests { + responseKey := responseKeys[idx] + metricID := requests[idx].SQLResourceID + guestCounts.Total++ + if workloadSummaryStatusIsRunning("", g.Status()) { + guestCounts.Running++ + } else { + guestCounts.Stopped++ + } + + snapshot := workloadsSummarySnapshot{ + id: responseKey, + name: strings.TrimSpace(g.Name()), + cpu: clampWorkloadPercent(g.CPUPercent()), + memory: clampWorkloadPercent(g.MemoryPercent()), + disk: clampWorkloadPercent(g.DiskPercent()), + network: clampNonNegativeWorkloadValue(g.NetIn() + g.NetOut()), + } + if snapshot.name == "" { + snapshot.name = responseKey + } + + metrics := batchMetrics[metricID] + cpuPoints := metrics["cpu"] + if len(cpuPoints) == 0 { + cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.CPUPercent()}} + } + memoryPoints := metrics["memory"] + if len(memoryPoints) == 0 { + memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.MemoryPercent()}} + } + diskPoints := metrics["disk"] + if len(diskPoints) == 0 { + diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.DiskPercent()}} + } + netInPoints := metrics["netin"] + netOutPoints := metrics["netout"] + if len(netInPoints) == 0 && len(netOutPoints) == 0 { + netInPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.NetIn()}} + netOutPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.NetOut()}} + } + + networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) + + snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) + snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) + snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) + snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) + + added += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", oldestTimestamp) + added += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", oldestTimestamp) + added += appendWorkloadMetricPoints(buckets, diskPoints, "disk", oldestTimestamp) + added += appendWorkloadMetricPoints(buckets, networkPoints, "network", oldestTimestamp) + snapshots = append(snapshots, snapshot) + } + return snapshots, added +} + +func workloadSummaryStatusIsRunning(runtimeState string, status unifiedresources.ResourceStatus) bool { + switch strings.ToLower(strings.TrimSpace(runtimeState)) { + case "running", "online", "ok": + return true + case "stopped", "offline", "paused", "created", "dead", "exited": + return false + } + + switch status { + case unifiedresources.StatusOnline: + return true + case unifiedresources.StatusWarning: + // Warning is an attention state on a running workload (degraded + // guest state, stale source data); power-off maps to StatusOffline. + return true + case unifiedresources.StatusOffline: + return false + } + + return false +} + +// handleStorageCharts returns pool capacity and physical disk temperature +// time-series for the storage summary sparklines. +func (r *Service) HandleStorageCharts(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + // Parse query parameters + query := req.URL.Query() + rangeMinutes := 60 // default 1 hour + if rangeStr := query.Get("range"); rangeStr != "" { + if _, err := fmt.Sscanf(rangeStr, "%d", &rangeMinutes); err != nil { + log.Warn().Err(err).Str("range", rangeStr).Msg("Invalid range parameter; using default") + } + } + + duration := time.Duration(rangeMinutes) * time.Minute + selectedNodeID := strings.TrimSpace(query.Get("node")) + + // Use tenant-aware monitor + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Monitor not available", http.StatusInternalServerError) + return + } + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + http.Error(w, "State unavailable", http.StatusInternalServerError) + return + } + + // Resolve node filter from canonical unified resources so storage charts use + // the same node identity model as the frontend storage page. + var selectedNodeName, selectedNodeInstance string + if selectedNodeID != "" { + found := false + for _, resource := range monitor.GetUnifiedResources() { + if strings.TrimSpace(resource.ID) != selectedNodeID { + continue + } + selectedNodeName = storageChartsSelectedNodeName(resource) + selectedNodeInstance = storageChartsSelectedNodeInstance(resource) + if selectedNodeName != "" || selectedNodeInstance != "" { + found = true + break + } + } + if !found { + for _, n := range monitor.NodesSnapshot() { + if n.ID == selectedNodeID { + selectedNodeName = n.Name + selectedNodeInstance = n.Instance + found = true + break + } + } + } + if !found { + log.Debug(). + Str("selectedNodeID", selectedNodeID). + Msg("Storage charts node filter not found in current state; falling back to global scope") + } + } + matchesNode := func(nodeName, instance string) bool { + if selectedNodeName == "" { + return true + } + if !strings.EqualFold(strings.TrimSpace(nodeName), selectedNodeName) { + return false + } + if selectedNodeInstance != "" && instance != "" { + return strings.EqualFold(strings.TrimSpace(instance), selectedNodeInstance) + } + return true + } + + // Build pool chart data from the canonical storage summary batch path so + // the dashboard and storage page share one efficient history retrieval model. + poolNames := make(map[string]string, len(readState.StoragePools())) + storageIDs := make([]string, 0, len(readState.StoragePools())) + for _, sp := range readState.StoragePools() { + if sp == nil { + continue + } + if !matchesNode(sp.Node(), sp.Instance()) { + continue + } + sid := sp.SourceID() + if sid == "" { + continue + } + poolNames[sid] = sp.Name() + storageIDs = append(storageIDs, sid) + } + + poolMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration) + pools := make(map[string]StoragePoolChartData, len(storageIDs)) + for _, sid := range storageIDs { + metrics := poolMetrics[sid] + pools[sid] = StoragePoolChartData{ + Name: poolNames[sid], + Usage: monitorPointsToAPI(metrics["usage"]), + Used: monitorPointsToAPI(metrics["used"]), + Avail: monitorPointsToAPI(metrics["avail"]), + } + } + + // Build disk temperature chart data + diskEntries := monitor.GetPhysicalDiskTemperatureCharts(duration) + disks := make(map[string]StorageDiskChartData, len(diskEntries)) + for id, entry := range diskEntries { + if !matchesNode(entry.Node, entry.Instance) { + continue + } + disks[id] = StorageDiskChartData{ + Name: entry.Name, + Node: entry.Node, + Temperature: monitorPointsToAPI(entry.Temperature), + } + } + + resp := EmptyStorageChartsResponse() + resp.Pools = pools + resp.Disks = disks + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp.NormalizeCollections()); err != nil { + log.Error().Err(err).Msg("Failed to encode storage chart data") + http.Error(w, "Internal server error", http.StatusInternalServerError) + } +} + +// handleStorageSummaryCharts serves a compact aggregate capacity trend for the +// dashboard storage card. It intentionally avoids returning per-pool and +// per-disk series so the dashboard does not overfetch the full storage page +// payload. +func (r *Service) HandleStorageSummaryCharts(w http.ResponseWriter, req *http.Request) { + const inMemoryChartThreshold = 2 * time.Hour + + if req.Method != http.MethodGet && req.Method != http.MethodHead { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + query := req.URL.Query() + timeRange := query.Get("range") + if timeRange == "" { + timeRange = "24h" + } + duration := parseChartsRangeDuration(timeRange) + + monitor := r.getTenantMonitor(req.Context()) + if monitor == nil { + http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) + return + } + + currentTime := time.Now().UnixMilli() + capacity, oldestTimestamp := monitor.GetStorageSummaryCapacityTrend(duration) + if oldestTimestamp == 0 { + oldestTimestamp = currentTime + } + + metricsStoreEnabled := monitor.GetMetricsStore() != nil + primarySourceHint := "memory" + if metricsStoreEnabled && duration > inMemoryChartThreshold { + primarySourceHint = "store_or_memory_fallback" + } + + resp := EmptyStorageSummaryTrendResponse() + resp.Capacity = monitorPointsToAPI(capacity) + resp.Timestamp = currentTime + resp.Stats = ChartStats{ + OldestDataTimestamp: oldestTimestamp, + Range: timeRange, + RangeSeconds: int64(duration / time.Second), + MetricsStoreEnabled: metricsStoreEnabled, + PrimarySourceHint: primarySourceHint, + InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), + PointCounts: ChartPointCounts{ + Total: len(resp.Capacity), + Storage: len(resp.Capacity), + }, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(resp.NormalizeCollections()); err != nil { + log.Error().Err(err).Msg("Failed to encode storage summary chart data") + http.Error(w, "Internal server error", http.StatusInternalServerError) + } +} + +// monitorPointsToAPI converts monitoring MetricPoints (time.Time timestamps) +// to API MetricPoints (Unix millisecond timestamps) for JSON serialization. +func monitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint { + if len(points) == 0 { + return nil + } + out := make([]MetricPoint, len(points)) + for i, p := range points { + out[i] = MetricPoint{Timestamp: p.Timestamp.UnixMilli(), Value: p.Value} + } + return out +} + +func MonitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint { + return monitorPointsToAPI(points) +} diff --git a/internal/api/chartapi/service_handlers_test.go b/internal/api/chartapi/service_handlers_test.go new file mode 100644 index 000000000..d9e10d206 --- /dev/null +++ b/internal/api/chartapi/service_handlers_test.go @@ -0,0 +1,1280 @@ +package chartapi + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "testing" + "time" + "unsafe" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" + "github.com/rcourtman/pulse-go-rewrite/internal/testutil" + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/internal/vmware" +) + +type fixedMonitorResolver struct { + monitor *monitoring.Monitor +} + +func (r fixedMonitorResolver) MonitorForContext(context.Context) *monitoring.Monitor { + return r.monitor +} + +func setUnexportedField(t *testing.T, target any, fieldName string, value any) { + t.Helper() + field := reflect.ValueOf(target).Elem().FieldByName(fieldName) + if !field.IsValid() { + t.Fatalf("field %q not found", fieldName) + } + reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Set(reflect.ValueOf(value)) +} + +func setMockModeForTest(t *testing.T, enabled bool) { + t.Helper() + testutil.SetMockMode(t, enabled) +} + +func newTestMonitor(t *testing.T) (*monitoring.Monitor, *models.State, *monitoring.MetricsHistory) { + t.Helper() + + monitor := &monitoring.Monitor{} + state := models.NewState() + metricsHistory := monitoring.NewMetricsHistory(10, time.Hour) + + setUnexportedField(t, monitor, "state", state) + setUnexportedField(t, monitor, "metricsHistory", metricsHistory) + + return monitor, state, metricsHistory +} + +// syncTestResourceStore populates a MonitorAdapter (ResourceRegistry) from the +// legacy state so that GetUnifiedReadState() returns a valid ReadState. +// Call this after setting state.VMs, state.Nodes, etc. +func syncTestResourceStore(t *testing.T, monitor *monitoring.Monitor, state *models.State) { + t.Helper() + adapter := unifiedresources.NewMonitorAdapter(nil) + adapter.PopulateFromSnapshot(state.GetSnapshot()) + setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) +} + +func mapKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func TestHandleCharts_Success(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}} + syncTestResourceStore(t, monitor, state) + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } +} + +func TestHandleCharts_StatsDebugMetadata(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}} + state.Nodes = []models.Node{{ID: "node-1", Name: "node-one", CPU: 0.1}} + state.Storage = []models.Storage{{ID: "store-1", Name: "Store One", Used: 50, Total: 100}} + syncTestResourceStore(t, monitor, state) + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + body := rec.Body.Bytes() + + var decoded ChartResponse + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("unmarshal ChartResponse: %v", err) + } + + if decoded.Stats.Range != "5m" { + t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) + } + if decoded.Stats.RangeSeconds != 300 { + t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) + } + if decoded.Stats.MetricsStoreEnabled { + t.Fatalf("expected stats.metricsStoreEnabled=false in test monitor, got true") + } + if decoded.Stats.PrimarySourceHint != "memory" { + t.Fatalf("expected stats.primarySourceHint=memory, got %q", decoded.Stats.PrimarySourceHint) + } + if decoded.Stats.InMemoryThresholdSecs != 7200 { + t.Fatalf("expected stats.inMemoryThresholdSecs=7200, got %d", decoded.Stats.InMemoryThresholdSecs) + } + if decoded.Stats.OldestDataTimestamp <= 0 { + t.Fatalf("expected stats.oldestDataTimestamp to be set, got %d", decoded.Stats.OldestDataTimestamp) + } + if decoded.Stats.OldestDataTimestamp > decoded.Timestamp { + t.Fatalf( + "expected stats.oldestDataTimestamp <= timestamp, got oldest=%d timestamp=%d", + decoded.Stats.OldestDataTimestamp, + decoded.Timestamp, + ) + } + + // With no history in the test monitor, handleCharts falls back to synthetic points: + // guests: cpu/memory/memoryused/disk/netin/netout (6 — diskread/diskwrite excluded from sparkline payloads) + // nodes: cpu/memory/disk (3) + // storage: disk (1) + if decoded.Stats.PointCounts.Guests != 6 { + t.Fatalf("expected stats.pointCounts.guests=6, got %d", decoded.Stats.PointCounts.Guests) + } + if points := decoded.ChartData["vm-1"]["memoryused"]; len(points) != 1 || points[0].Value != 0 { + t.Fatalf("expected additive raw memory-used fallback point, got %+v", points) + } + if decoded.Stats.PointCounts.Nodes != 3 { + t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes) + } + if decoded.Stats.PointCounts.Storage != 1 { + t.Fatalf("expected stats.pointCounts.storage=1, got %d", decoded.Stats.PointCounts.Storage) + } + if decoded.Stats.PointCounts.DockerContainers != 0 || decoded.Stats.PointCounts.DockerHosts != 0 || decoded.Stats.PointCounts.Agents != 0 { + t.Fatalf( + "expected dockerContainers/dockerHosts/agents all 0, got dc=%d dh=%d agents=%d", + decoded.Stats.PointCounts.DockerContainers, + decoded.Stats.PointCounts.DockerHosts, + decoded.Stats.PointCounts.Agents, + ) + } + + sum := decoded.Stats.PointCounts.Guests + + decoded.Stats.PointCounts.Nodes + + decoded.Stats.PointCounts.Storage + + decoded.Stats.PointCounts.DockerContainers + + decoded.Stats.PointCounts.DockerHosts + + decoded.Stats.PointCounts.Agents + if decoded.Stats.PointCounts.Total != sum { + t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total) + } + + var raw map[string]interface{} + if err := json.Unmarshal(body, &raw); err != nil { + t.Fatalf("unmarshal raw JSON: %v", err) + } + stats, ok := raw["stats"].(map[string]interface{}) + if !ok { + t.Fatalf("expected stats object in JSON response") + } + if _, ok := stats["pointCounts"]; !ok { + t.Fatalf("expected stats.pointCounts to be present in JSON response") + } +} + +func TestHandleCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) { + setMockModeForTest(t, true) + + fixtures := vmware.DefaultFixtures() + if len(fixtures.Hosts) == 0 { + t.Fatal("expected default VMware fixtures to include at least one host") + } + expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host) + + monitor, state, _ := newTestMonitor(t) + state.Hosts = []models.Host{{ + ID: "live-store-host-1", + Hostname: "live-store-host-1", + CPUUsage: 11.0, + Memory: models.Memory{Usage: 22.0}, + Disks: []models.Disk{{Usage: 33.0}}, + Status: "online", + }} + syncTestResourceStore(t, monitor, state) + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + var decoded ChartResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal ChartResponse: %v", err) + } + + vmwareSeries, ok := decoded.AgentData[expectedHostID] + if !ok { + t.Fatalf("expected VMware host %q in full charts payload, got keys=%v", expectedHostID, mapKeys(decoded.AgentData)) + } + if len(vmwareSeries["cpu"]) == 0 { + t.Fatalf("expected VMware host %q cpu series", expectedHostID) + } + if _, ok := decoded.AgentData["live-store-host-1"]; ok { + t.Fatalf("expected mock-aware full charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData)) + } +} + +func TestHandleInfrastructureCharts_Lightweight(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-1", + Name: "node-one", + Status: "online", + CPU: 0.1, + Memory: models.Memory{Usage: 12.0}, + Disk: models.Disk{Usage: 34.0}, + }} + state.DockerHosts = []models.DockerHost{{ + ID: "docker-host-1", + Runtime: "docker", + CPUUsage: 23.0, + Memory: models.Memory{Usage: 45.0}, + Disks: []models.Disk{{Usage: 67.0}}, + Status: "online", + }} + state.Hosts = []models.Host{{ + ID: "host-1", + Hostname: "host-one", + CPUUsage: 11.0, + Memory: models.Memory{Usage: 22.0}, + Disks: []models.Disk{{Usage: 33.0}}, + Status: "online", + }} + syncTestResourceStore(t, monitor, state) + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleInfrastructureCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } + + body := rec.Body.Bytes() + + var decoded InfrastructureChartsResponse + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) + } + if decoded.Stats.Range != "5m" { + t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) + } + if decoded.Stats.RangeSeconds != 300 { + t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) + } + + // With no history in the test monitor, handler falls back to synthetic points: + // nodes: cpu/memory/disk (3) + // dockerHosts: cpu/memory/disk (3) + // agents: cpu/memory/disk (3) + if decoded.Stats.PointCounts.Nodes != 3 { + t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes) + } + if decoded.Stats.PointCounts.DockerHosts != 3 { + t.Fatalf("expected stats.pointCounts.dockerHosts=3, got %d", decoded.Stats.PointCounts.DockerHosts) + } + if decoded.Stats.PointCounts.Agents != 3 { + t.Fatalf("expected stats.pointCounts.agents=3, got %d", decoded.Stats.PointCounts.Agents) + } + sum := decoded.Stats.PointCounts.Nodes + decoded.Stats.PointCounts.DockerHosts + decoded.Stats.PointCounts.Agents + if decoded.Stats.PointCounts.Total != sum { + t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total) + } + + var raw map[string]interface{} + if err := json.Unmarshal(body, &raw); err != nil { + t.Fatalf("unmarshal raw JSON: %v", err) + } + for _, forbidden := range []string{"data", "storageData", "dockerData", "guestTypes"} { + if _, ok := raw[forbidden]; ok { + t.Fatalf("expected %q to be absent from infra summary response", forbidden) + } + } +} + +func TestHandleInfrastructureCharts_MetricFilter(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-1", + Name: "node-one", + Status: "online", + CPU: 0.1, + Memory: models.Memory{Usage: 12.0}, + Disk: models.Disk{Usage: 34.0}, + }} + state.DockerHosts = []models.DockerHost{{ + ID: "docker-host-1", + Runtime: "docker", + CPUUsage: 23.0, + Memory: models.Memory{Usage: 45.0}, + Disks: []models.Disk{{Usage: 67.0}}, + Status: "online", + }} + state.Hosts = []models.Host{{ + ID: "host-1", + Hostname: "host-one", + CPUUsage: 11.0, + Memory: models.Memory{Usage: 22.0}, + Disks: []models.Disk{{Usage: 33.0}}, + Status: "online", + }} + syncTestResourceStore(t, monitor, state) + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m&metrics=cpu,memory", nil) + rec := httptest.NewRecorder() + + service.HandleInfrastructureCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded InfrastructureChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) + } + + if decoded.Stats.PointCounts.Nodes != 2 { + t.Fatalf("expected stats.pointCounts.nodes=2, got %d", decoded.Stats.PointCounts.Nodes) + } + if decoded.Stats.PointCounts.DockerHosts != 2 { + t.Fatalf("expected stats.pointCounts.dockerHosts=2, got %d", decoded.Stats.PointCounts.DockerHosts) + } + if decoded.Stats.PointCounts.Agents != 2 { + t.Fatalf("expected stats.pointCounts.agents=2, got %d", decoded.Stats.PointCounts.Agents) + } + if _, ok := decoded.NodeData["node-1"]["disk"]; ok { + t.Fatalf("expected disk series to be filtered out of node payload") + } + if _, ok := decoded.DockerHostData["docker-host-1"]["disk"]; ok { + t.Fatalf("expected disk series to be filtered out of docker host payload") + } + if _, ok := decoded.AgentData["host-1"]["disk"]; ok { + t.Fatalf("expected disk series to be filtered out of agent payload") + } +} + +func TestHandleInfrastructureCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) { + setMockModeForTest(t, true) + + fixtures := vmware.DefaultFixtures() + if len(fixtures.Hosts) == 0 { + t.Fatal("expected default VMware fixtures to include at least one host") + } + expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host) + + monitor, state, _ := newTestMonitor(t) + state.Hosts = []models.Host{{ + ID: "live-store-host-1", + Hostname: "live-store-host-1", + CPUUsage: 11.0, + Memory: models.Memory{Usage: 22.0}, + Disks: []models.Disk{{Usage: 33.0}}, + Status: "online", + }} + syncTestResourceStore(t, monitor, state) + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleInfrastructureCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + var decoded InfrastructureChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) + } + + vmwareSeries, ok := decoded.AgentData[expectedHostID] + if !ok { + t.Fatalf("expected VMware host %q in infrastructure charts, got keys=%v", expectedHostID, mapKeys(decoded.AgentData)) + } + if len(vmwareSeries["cpu"]) == 0 { + t.Fatalf("expected VMware host %q cpu series", expectedHostID) + } + if _, ok := decoded.AgentData["live-store-host-1"]; ok { + t.Fatalf("expected mock-aware infrastructure charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData)) + } +} + +func TestHandleWorkloadsSummaryCharts_AggregatesAndCounts(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-pve-1", + Name: "pve-1", + Instance: "pve", + }} + state.VMs = []models.VM{{ + ID: "vm-101", + Name: "vm-101", + Node: "pve-1", + Instance: "pve", + Status: "running", + CPU: 0.25, + Memory: models.Memory{Usage: 40.0}, + Disk: models.Disk{Usage: 55.0}, + NetworkIn: 1200, + NetworkOut: 800, + }} + state.Containers = []models.Container{{ + ID: "ct-201", + Name: "ct-201", + Node: "pve-1", + Instance: "pve", + Status: "stopped", + CPU: 0.10, + Memory: models.Memory{Usage: 30.0}, + Disk: models.Disk{Usage: 45.0}, + NetworkIn: 400, + NetworkOut: 600, + }} + state.DockerHosts = []models.DockerHost{{ + ID: "docker-host-1", + Runtime: "docker", + Containers: []models.DockerContainer{{ + ID: "docker-1", + Name: "docker-1", + State: "running", + CPUPercent: 35.0, + MemoryPercent: 60.0, + WritableLayerBytes: 10, + RootFilesystemBytes: 100, + }}, + }} + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m", nil) + rec := httptest.NewRecorder() + + service.HandleWorkloadsSummaryCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } + + body := rec.Body.Bytes() + + var decoded WorkloadsSummaryChartsResponse + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) + } + if decoded.Stats.Range != "5m" { + t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) + } + if decoded.Stats.RangeSeconds != 300 { + t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) + } + + if decoded.GuestCounts.Total != 3 { + t.Fatalf("expected guestCounts.total=3, got %d", decoded.GuestCounts.Total) + } + if decoded.GuestCounts.Running != 2 { + t.Fatalf("expected guestCounts.running=2, got %d", decoded.GuestCounts.Running) + } + if decoded.GuestCounts.Stopped != 1 { + t.Fatalf("expected guestCounts.stopped=1, got %d", decoded.GuestCounts.Stopped) + } + + for metricName, metric := range map[string]WorkloadsSummaryMetricData{ + "cpu": decoded.CPU, + "memory": decoded.Memory, + "disk": decoded.Disk, + "network": decoded.Network, + } { + if len(metric.P50) == 0 { + t.Fatalf("expected %s p50 points to be present", metricName) + } + if len(metric.P95) == 0 { + t.Fatalf("expected %s p95 points to be present", metricName) + } + } + + if decoded.Stats.PointCounts.Total <= 0 { + t.Fatalf("expected stats.pointCounts.total to be > 0, got %d", decoded.Stats.PointCounts.Total) + } + if decoded.Stats.PointCounts.Guests <= 0 { + t.Fatalf("expected stats.pointCounts.guests to be > 0, got %d", decoded.Stats.PointCounts.Guests) + } + + if decoded.BlastRadius.CPU.Scope != "concentrated" { + t.Fatalf("expected cpu blast radius concentrated, got %q", decoded.BlastRadius.CPU.Scope) + } + if decoded.BlastRadius.Network.Scope != "concentrated" { + t.Fatalf("expected network blast radius concentrated, got %q", decoded.BlastRadius.Network.Scope) + } + + // Node-scoped request should only include workloads that match the selected node. + nodeReq := httptest.NewRequest( + http.MethodGet, + "/api/charts/workloads-summary?range=5m&node=node-pve-1", + nil, + ) + nodeRec := httptest.NewRecorder() + + service.HandleWorkloadsSummaryCharts(nodeRec, nodeReq) + + if nodeRec.Code != http.StatusOK { + t.Fatalf("node-scoped expected status %d, got %d", http.StatusOK, nodeRec.Code) + } + + var nodeScoped WorkloadsSummaryChartsResponse + if err := json.Unmarshal(nodeRec.Body.Bytes(), &nodeScoped); err != nil { + t.Fatalf("unmarshal node-scoped WorkloadsSummaryChartsResponse: %v", err) + } + if nodeScoped.GuestCounts.Total != 2 { + t.Fatalf("expected node-scoped guestCounts.total=2, got %d", nodeScoped.GuestCounts.Total) + } + if nodeScoped.GuestCounts.Running != 1 { + t.Fatalf("expected node-scoped guestCounts.running=1, got %d", nodeScoped.GuestCounts.Running) + } + if nodeScoped.GuestCounts.Stopped != 1 { + t.Fatalf("expected node-scoped guestCounts.stopped=1, got %d", nodeScoped.GuestCounts.Stopped) + } + if nodeScoped.BlastRadius.CPU.Scope != "concentrated" { + t.Fatalf("expected node-scoped cpu blast radius concentrated, got %q", nodeScoped.BlastRadius.CPU.Scope) + } +} + +func TestHandleWorkloadCharts_WorkloadOnlyPayloadAndNodeFilter(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-pve-1", + Name: "pve-1", + Instance: "pve", + }} + state.VMs = []models.VM{ + { + ID: "vm-101", + Name: "vm-101", + Node: "pve-1", + Instance: "pve", + CPU: 0.2, + Memory: models.Memory{Usage: 55}, + Disk: models.Disk{Usage: 40}, + }, + { + ID: "vm-202", + Name: "vm-202", + Node: "other", + Instance: "other", + CPU: 0.4, + Memory: models.Memory{Usage: 65}, + Disk: models.Disk{Usage: 70}, + }, + } + state.Containers = []models.Container{ + { + ID: "ct-301", + Name: "ct-301", + Node: "pve-1", + Instance: "pve", + CPU: 0.3, + Memory: models.Memory{Usage: 40}, + Disk: models.Disk{Usage: 25}, + }, + } + state.DockerHosts = []models.DockerHost{{ + ID: "docker-host-1", + Hostname: "pve-1", + DisplayName: "pve-1", + Containers: []models.DockerContainer{{ + ID: "docker-401", + Name: "docker-401", + CPUPercent: 22, + MemoryPercent: 30, + }}, + }} + + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil { + t.Fatal("expected unified read state") + } + + vmResponseKey := "" + containerResponseKey := "" + excludedResponseKey := "" + for _, vm := range readState.VMs() { + if vm == nil { + continue + } + switch vm.Name() { + case "vm-101": + vmResponseKey, _, _ = VMChartRequest(vm) + case "vm-202": + excludedResponseKey, _, _ = VMChartRequest(vm) + } + } + for _, ct := range readState.Containers() { + if ct == nil { + continue + } + if ct.Name() == "ct-301" { + containerResponseKey, _, _ = SystemContainerChartRequest(ct) + } + } + if vmResponseKey == "" || containerResponseKey == "" || excludedResponseKey == "" { + t.Fatalf( + "expected canonical response keys for vm/container test fixtures, got vm=%q ct=%q excluded=%q", + vmResponseKey, + containerResponseKey, + excludedResponseKey, + ) + } + + service := NewService(fixedMonitorResolver{monitor: monitor}) + + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil) + rec := httptest.NewRecorder() + service.HandleWorkloadCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } + + var decoded WorkloadChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) + } + + if decoded.Stats.Range != "5m" { + t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) + } + if decoded.Stats.RangeSeconds != 300 { + t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) + } + if len(decoded.ChartData) != 3 { + t.Fatalf("expected 3 workload chart entries, got %d", len(decoded.ChartData)) + } + if len(decoded.DockerData) != 1 { + t.Fatalf("expected 1 docker chart entry, got %d", len(decoded.DockerData)) + } + if decoded.GuestTypes[vmResponseKey] != "vm" { + t.Fatalf("expected vm guest type for %s, got %q", vmResponseKey, decoded.GuestTypes[vmResponseKey]) + } + if decoded.GuestTypes[containerResponseKey] != "system-container" { + t.Fatalf( + "expected system-container guest type for %s, got %q", + containerResponseKey, + decoded.GuestTypes[containerResponseKey], + ) + } + if decoded.Stats.PointCounts.Total <= 0 { + t.Fatalf("expected stats.pointCounts.total > 0, got %d", decoded.Stats.PointCounts.Total) + } + + // Node-scoped request should only include workloads linked to node-pve-1 + // plus docker containers running on matching docker hosts. + nodeReq := httptest.NewRequest( + http.MethodGet, + "/api/charts/workloads?range=5m&node=node-pve-1", + nil, + ) + nodeRec := httptest.NewRecorder() + service.HandleWorkloadCharts(nodeRec, nodeReq) + + if nodeRec.Code != http.StatusOK { + t.Fatalf("node-scoped expected status %d, got %d", http.StatusOK, nodeRec.Code) + } + + var scoped WorkloadChartsResponse + if err := json.Unmarshal(nodeRec.Body.Bytes(), &scoped); err != nil { + t.Fatalf("unmarshal node-scoped WorkloadChartsResponse: %v", err) + } + if len(scoped.ChartData) != 2 { + t.Fatalf("expected 2 scoped workload chart entries, got %d", len(scoped.ChartData)) + } + if _, ok := scoped.ChartData[excludedResponseKey]; ok { + t.Fatalf("expected %s to be excluded by node scope", excludedResponseKey) + } + if len(scoped.DockerData) != 1 { + t.Fatalf("expected 1 scoped docker chart entry, got %d", len(scoped.DockerData)) + } +} + +func TestHandleWorkloadCharts_UsesCanonicalWorkloadIDsForVMwareVMs(t *testing.T) { + monitor, state, history := newTestMonitor(t) + now := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Minute) + metricID := "vc-1:vm:vm-201" + resourceID := "vm-vmware-1" + + history.AddGuestMetric(metricID, "cpu", 37, now.Add(-10*time.Minute)) + history.AddGuestMetric(metricID, "memory", 62, now.Add(-5*time.Minute)) + + adapter := unifiedresources.NewMonitorAdapter(nil) + adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ + unifiedresources.SourceVMware: { + { + SourceID: metricID, + Resource: unifiedresources.Resource{ + ID: resourceID, + Type: unifiedresources.ResourceTypeVM, + Name: "warehouse-api-01", + Status: unifiedresources.StatusOnline, + LastSeen: now, + MetricsTarget: &unifiedresources.MetricsTarget{ + ResourceType: "vm", + ResourceID: metricID, + }, + VMware: &unifiedresources.VMwareData{ + ConnectionID: "vc-1", + EntityType: "vm", + ManagedObjectID: "vm-201", + }, + }, + }, + }, + }) + setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) + + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil { + t.Fatalf("expected one VMware VM in unified read state, got %+v", readState) + } + resourceID, _, ok := VMChartRequest(readState.VMs()[0]) + if !ok { + t.Fatal("expected canonical VMware vm chart request") + } + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=1h", nil) + rec := httptest.NewRecorder() + service.HandleWorkloadCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + var decoded WorkloadChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) + } + + series, ok := decoded.ChartData[resourceID] + if !ok { + t.Fatalf("expected VMware workload chart keyed by canonical workload id %q, got %v", resourceID, decoded.ChartData) + } + if _, ok := decoded.ChartData[metricID]; ok { + t.Fatalf("expected VMware metrics target id %q to stay out of workload chart response keys", metricID) + } + if decoded.GuestTypes[resourceID] != "vm" { + t.Fatalf("expected guest type vm for %q, got %q", resourceID, decoded.GuestTypes[resourceID]) + } + if len(series["cpu"]) == 0 { + t.Fatalf("expected VMware cpu series for %q", resourceID) + } + if got := series["cpu"][len(series["cpu"])-1].Value; got != 37 { + t.Fatalf("expected latest VMware cpu value 37, got %v", got) + } +} + +func TestHandleWorkloadsSummaryCharts_UsesCanonicalWorkloadIDsForVMwareVMs(t *testing.T) { + monitor, state, history := newTestMonitor(t) + now := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Minute) + metricID := "vc-1:vm:vm-201" + resourceID := "vm-vmware-1" + + history.AddGuestMetric(metricID, "cpu", 51, now.Add(-10*time.Minute)) + history.AddGuestMetric(metricID, "memory", 64, now.Add(-5*time.Minute)) + history.AddGuestMetric(metricID, "disk", 43, now.Add(-3*time.Minute)) + history.AddGuestMetric(metricID, "netin", 1200, now.Add(-2*time.Minute)) + history.AddGuestMetric(metricID, "netout", 800, now.Add(-2*time.Minute)) + + adapter := unifiedresources.NewMonitorAdapter(nil) + adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ + unifiedresources.SourceVMware: { + { + SourceID: metricID, + Resource: unifiedresources.Resource{ + ID: resourceID, + Type: unifiedresources.ResourceTypeVM, + Name: "warehouse-api-01", + Status: unifiedresources.StatusOnline, + LastSeen: now, + MetricsTarget: &unifiedresources.MetricsTarget{ + ResourceType: "vm", + ResourceID: metricID, + }, + VMware: &unifiedresources.VMwareData{ + ConnectionID: "vc-1", + EntityType: "vm", + ManagedObjectID: "vm-201", + }, + }, + }, + }, + }) + setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) + + readState := monitor.GetUnifiedReadStateOrSnapshot() + if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil { + t.Fatalf("expected one VMware VM in unified read state, got %+v", readState) + } + resourceID, _, ok := VMChartRequest(readState.VMs()[0]) + if !ok { + t.Fatal("expected canonical VMware vm chart request") + } + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=1h", nil) + rec := httptest.NewRecorder() + service.HandleWorkloadsSummaryCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) + } + + var decoded WorkloadsSummaryChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) + } + + if decoded.GuestCounts.Total != 1 || decoded.GuestCounts.Running != 1 { + t.Fatalf("expected guestCounts total/running = 1/1, got %+v", decoded.GuestCounts) + } + if len(decoded.TopContributors.CPU) == 0 { + t.Fatal("expected at least one cpu top contributor") + } + if decoded.TopContributors.CPU[0].ID != resourceID { + t.Fatalf("expected cpu top contributor id %q, got %+v", resourceID, decoded.TopContributors.CPU[0]) + } + if decoded.TopContributors.CPU[0].ID == metricID { + t.Fatalf("expected workloads summary contributor id to avoid raw metrics target %q", metricID) + } + if got := decoded.TopContributors.CPU[0].Name; got != "warehouse-api-01" { + t.Fatalf("expected cpu top contributor name warehouse-api-01, got %q", got) + } +} + +func TestHandleWorkloadCharts_IncludesKubernetesPods(t *testing.T) { + setMockModeForTest(t, false) + + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-pve-1", + Name: "pve-1", + Instance: "pve", + }} + state.KubernetesClusters = []models.KubernetesCluster{{ + ID: "cluster-k8s-summary-test", + Name: "cluster-k8s-summary-test", + Status: "online", + Pods: []models.KubernetesPod{ + { + UID: "pod-001", + Name: "api-0", + Namespace: "default", + NodeName: "pve-1", + Phase: "Running", + Containers: []models.KubernetesPodContainer{ + {Name: "api", Ready: true}, + }, + }, + }, + }} + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil) + rec := httptest.NewRecorder() + service.HandleWorkloadCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded WorkloadChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) + } + + var metricID string + for id, typ := range decoded.GuestTypes { + if typ == "k8s" { + metricID = id + break + } + } + if metricID == "" { + t.Fatal("expected at least one kubernetes pod series") + } + + series, ok := decoded.ChartData[metricID] + if !ok { + t.Fatalf("expected kubernetes pod series for %s", metricID) + } + if len(series["cpu"]) == 0 { + t.Fatalf("expected kubernetes cpu points for %s", metricID) + } + if got := len(series["disk"]); got != 1 || series["disk"][0].Value != 0 { + t.Fatalf("expected unsupported kubernetes pod disk metric fallback to single zero point; got len=%d value=%v", got, series["disk"]) + } + if got := len(series["netin"]); got != 1 || series["netin"][0].Value != 0 { + t.Fatalf("expected unsupported kubernetes pod network metric fallback to single zero point; got len=%d value=%v", got, series["netin"]) + } +} + +func TestHandleWorkloadsSummaryCharts_IncludesKubernetesPods(t *testing.T) { + setMockModeForTest(t, false) + + monitor, state, _ := newTestMonitor(t) + state.KubernetesClusters = []models.KubernetesCluster{{ + ID: "cluster-alpha", + Name: "cluster-alpha", + Status: "online", + Pods: []models.KubernetesPod{ + { + UID: "pod-001", + Name: "api-0", + Namespace: "default", + Phase: "Running", + Containers: []models.KubernetesPodContainer{ + {Name: "api", Ready: true}, + }, + }, + { + UID: "pod-002", + Name: "batch-0", + Namespace: "ops", + Phase: "Failed", + Containers: []models.KubernetesPodContainer{ + {Name: "worker", Ready: false}, + }, + }, + }, + }} + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m", nil) + rec := httptest.NewRecorder() + service.HandleWorkloadsSummaryCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded WorkloadsSummaryChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) + } + + if decoded.GuestCounts.Total != 2 { + t.Fatalf("expected guestCounts.total=2, got %d", decoded.GuestCounts.Total) + } + if decoded.GuestCounts.Running != 1 { + t.Fatalf("expected guestCounts.running=1, got %d", decoded.GuestCounts.Running) + } + if decoded.GuestCounts.Stopped != 1 { + t.Fatalf("expected guestCounts.stopped=1, got %d", decoded.GuestCounts.Stopped) + } + if len(decoded.CPU.P50) == 0 { + t.Fatal("expected cpu summary points for kubernetes workloads") + } +} + +func TestHandleWorkloadCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-pve-1", + Name: "pve-1", + Instance: "pve", + }} + state.VMs = []models.VM{{ + ID: "vm-101", + Name: "vm-101", + Node: "pve-1", + Instance: "pve", + CPU: 0.3, + Memory: models.Memory{Total: 100, Used: 42, Free: 58, Usage: 42}, + Disk: models.Disk{Usage: 55}, + }} + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m&node=missing-node-id", nil) + rec := httptest.NewRecorder() + + service.HandleWorkloadCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded WorkloadChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) + } + if len(decoded.ChartData) == 0 { + t.Fatalf("expected fallback to global scope when node filter is stale") + } + for _, series := range decoded.ChartData { + if points := series["memoryused"]; len(points) != 1 || points[0].Value != 42 { + t.Fatalf("expected raw memory-used workload fallback point, got %+v", points) + } + } +} + +func TestHandleWorkloadsSummaryCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) { + monitor, state, _ := newTestMonitor(t) + state.Nodes = []models.Node{{ + ID: "node-pve-1", + Name: "pve-1", + Instance: "pve", + }} + state.VMs = []models.VM{{ + ID: "vm-101", + Name: "vm-101", + Node: "pve-1", + Instance: "pve", + CPU: 0.3, + Memory: models.Memory{Usage: 42}, + Disk: models.Disk{Usage: 55}, + Status: "running", + }} + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m&node=missing-node-id", nil) + rec := httptest.NewRecorder() + + service.HandleWorkloadsSummaryCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded WorkloadsSummaryChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) + } + if decoded.GuestCounts.Total == 0 { + t.Fatalf("expected fallback to global scope when summary node filter is stale") + } +} + +func TestHandleStorageCharts_Success(t *testing.T) { + monitor, state, metricsHistory := newTestMonitor(t) + state.Storage = []models.Storage{{ID: "store-1", Name: "Store One"}} + metricsHistory.AddStorageMetric("store-1", "usage", 0.4, time.Now()) + syncTestResourceStore(t, monitor, state) + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/storage/charts?range=30", nil) + rec := httptest.NewRecorder() + + service.HandleStorageCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } +} + +func TestHandleStorageCharts_IncludesSupplementalStorageAndResolvesUnifiedNodeFilter(t *testing.T) { + monitor, state, metricsHistory := newTestMonitor(t) + now := time.Now() + metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "usage", 0.25, now) + metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "used", 3.57*1024*1024*1024*1024, now) + metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "avail", 11.03*1024*1024*1024*1024, now) + metricsHistory.AddStorageMetric("pool:archive", "usage", 0.36, now) + + adapter := unifiedresources.NewMonitorAdapter(nil) + adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ + unifiedresources.SourceVMware: { + { + SourceID: "vc-1:host:host-101", + Resource: unifiedresources.Resource{ + ID: "agent-vmware-1", + Type: unifiedresources.ResourceTypeAgent, + Name: "esxi-01.lab.local", + Identity: unifiedresources.ResourceIdentity{ + Hostnames: []string{"esxi-01.lab.local"}, + }, + MetricsTarget: &unifiedresources.MetricsTarget{ + ResourceType: "agent", + ResourceID: "vc-1:host:host-101", + }, + VMware: &unifiedresources.VMwareData{ + ConnectionID: "vc-1", + EntityType: "host", + ManagedObjectID: "host-101", + }, + }, + }, + { + SourceID: "vc-1:datastore:datastore-202", + ParentSourceID: "vc-1:host:host-101", + Resource: unifiedresources.Resource{ + ID: "storage-vmware-1", + Type: unifiedresources.ResourceTypeStorage, + Name: "archive-tier", + ParentName: "esxi-01.lab.local", + MetricsTarget: &unifiedresources.MetricsTarget{ + ResourceType: "storage", + ResourceID: "vc-1:datastore:datastore-202", + }, + Storage: &unifiedresources.StorageMeta{ + Type: "datastore", + Platform: "vmware", + Nodes: []string{"esxi-01.lab.local", "esxi-02.lab.local"}, + }, + VMware: &unifiedresources.VMwareData{ + ConnectionID: "vc-1", + EntityType: "datastore", + ManagedObjectID: "datastore-202", + RuntimeHostName: "esxi-01.lab.local", + }, + }, + }, + }, + unifiedresources.SourceTrueNAS: { + { + SourceID: "system:truenas-main", + Resource: unifiedresources.Resource{ + ID: "agent-truenas-1", + Type: unifiedresources.ResourceTypeAgent, + Name: "truenas-main", + Identity: unifiedresources.ResourceIdentity{ + Hostnames: []string{"truenas-main"}, + }, + TrueNAS: &unifiedresources.TrueNASData{ + Hostname: "truenas-main", + }, + }, + }, + { + SourceID: "pool:archive", + ParentSourceID: "system:truenas-main", + Resource: unifiedresources.Resource{ + ID: "storage-truenas-1", + Type: unifiedresources.ResourceTypeStorage, + Name: "archive", + ParentName: "truenas-main", + MetricsTarget: &unifiedresources.MetricsTarget{ + ResourceType: "storage", + ResourceID: "pool:archive", + }, + Storage: &unifiedresources.StorageMeta{ + Type: "zfs-pool", + Platform: "truenas", + }, + TrueNAS: &unifiedresources.TrueNASData{ + Hostname: "truenas-main", + }, + }, + }, + }, + }) + selectedNodeID := "" + for _, resource := range adapter.GetAll() { + if resource.Type == unifiedresources.ResourceTypeAgent && resource.Name == "esxi-01.lab.local" { + selectedNodeID = resource.ID + break + } + } + if selectedNodeID == "" { + t.Fatal("expected canonical VMware host resource id in adapter") + } + setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/storage-charts?range=30&node="+selectedNodeID, nil) + rec := httptest.NewRecorder() + + service.HandleStorageCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + + var decoded StorageChartsResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal StorageChartsResponse: %v", err) + } + if len(decoded.Pools) != 1 { + t.Fatalf("expected unified node filter to keep 1 VMware pool, got %d (%v)", len(decoded.Pools), decoded.Pools) + } + if _, ok := decoded.Pools["vc-1:datastore:datastore-202"]; !ok { + t.Fatalf("expected VMware datastore chart keyed by canonical metrics target, got %v", decoded.Pools) + } + if _, ok := decoded.Pools["pool:archive"]; ok { + t.Fatalf("expected TrueNAS pool to be filtered out by VMware host selection, got %v", decoded.Pools) + } +} + +func TestHandleStorageSummaryCharts_AggregatesCapacityAcrossPools(t *testing.T) { + monitor, state, metricsHistory := newTestMonitor(t) + now := time.Now() + state.Storage = []models.Storage{ + {ID: "store-1", Name: "Store One"}, + {ID: "store-2", Name: "Store Two"}, + } + metricsHistory.AddStorageMetric("store-1", "used", 400, now) + metricsHistory.AddStorageMetric("store-1", "avail", 600, now) + metricsHistory.AddStorageMetric("store-2", "used", 100, now) + metricsHistory.AddStorageMetric("store-2", "avail", 900, now) + syncTestResourceStore(t, monitor, state) + + service := NewService(fixedMonitorResolver{monitor: monitor}) + req := httptest.NewRequest(http.MethodGet, "/api/charts/storage-summary?range=1h", nil) + rec := httptest.NewRecorder() + + service.HandleStorageSummaryCharts(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected application/json, got %q", ct) + } + + var decoded StorageSummaryTrendResponse + if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { + t.Fatalf("unmarshal StorageSummaryTrendResponse: %v", err) + } + if len(decoded.Capacity) != 1 { + t.Fatalf("expected 1 aggregate capacity point, got %d (%+v)", len(decoded.Capacity), decoded.Capacity) + } + if decoded.Capacity[0].Value != 25 { + t.Fatalf("expected aggregate capacity of 25%%, got %+v", decoded.Capacity[0]) + } + if decoded.Stats.PointCounts.Total != 1 { + t.Fatalf("expected summary point count 1, got %+v", decoded.Stats.PointCounts) + } +} diff --git a/internal/api/chartapi/types.go b/internal/api/chartapi/types.go new file mode 100644 index 000000000..42646e942 --- /dev/null +++ b/internal/api/chartapi/types.go @@ -0,0 +1,330 @@ +package chartapi + +// ChartResponse represents chart data +type ChartResponse struct { + ChartData map[string]VMChartData `json:"data"` + NodeData map[string]NodeChartData `json:"nodeData"` + StorageData map[string]StorageChartData `json:"storageData"` + DockerData map[string]VMChartData `json:"dockerData"` // Docker container metrics (keyed by container ID) + DockerHostData map[string]VMChartData `json:"dockerHostData"` // Docker host metrics (keyed by host ID) + AgentData map[string]VMChartData `json:"agentData"` // Unified agent metrics (keyed by agent ID) + GuestTypes map[string]string `json:"guestTypes"` // Maps guest ID to type ("vm", "system-container", "k8s") + Timestamp int64 `json:"timestamp"` + Stats ChartStats `json:"stats"` +} + +func EmptyChartResponse() ChartResponse { + return ChartResponse{}.NormalizeCollections() +} + +func (r ChartResponse) NormalizeCollections() ChartResponse { + if r.ChartData == nil { + r.ChartData = map[string]VMChartData{} + } + if r.NodeData == nil { + r.NodeData = map[string]NodeChartData{} + } + if r.StorageData == nil { + r.StorageData = map[string]StorageChartData{} + } + if r.DockerData == nil { + r.DockerData = map[string]VMChartData{} + } + if r.DockerHostData == nil { + r.DockerHostData = map[string]VMChartData{} + } + if r.AgentData == nil { + r.AgentData = map[string]VMChartData{} + } + if r.GuestTypes == nil { + r.GuestTypes = map[string]string{} + } + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// InfrastructureChartsResponse is a lightweight variant of ChartResponse used by +// infra-only clients (Infrastructure summary sparklines, prewarm caches). +// It avoids the heavy guest/storage chart payload and associated compute. +type InfrastructureChartsResponse struct { + NodeData map[string]NodeChartData `json:"nodeData"` + DockerHostData map[string]VMChartData `json:"dockerHostData"` // Docker host metrics (keyed by host ID) + AgentData map[string]VMChartData `json:"agentData"` // Unified agent metrics (keyed by agent ID) + Timestamp int64 `json:"timestamp"` + Stats ChartStats `json:"stats"` +} + +func EmptyInfrastructureChartsResponse() InfrastructureChartsResponse { + return InfrastructureChartsResponse{}.NormalizeCollections() +} + +func (r InfrastructureChartsResponse) NormalizeCollections() InfrastructureChartsResponse { + if r.NodeData == nil { + r.NodeData = map[string]NodeChartData{} + } + if r.DockerHostData == nil { + r.DockerHostData = map[string]VMChartData{} + } + if r.AgentData == nil { + r.AgentData = map[string]VMChartData{} + } + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// WorkloadChartsResponse is a lightweight chart payload used by Workloads +// summary sparklines. It intentionally excludes infrastructure/storage series. +type WorkloadChartsResponse struct { + ChartData map[string]VMChartData `json:"data"` // Workload metrics keyed by workload ID + DockerData map[string]VMChartData `json:"dockerData"` // Docker container metrics keyed by container ID + GuestTypes map[string]string `json:"guestTypes"` // Maps guest ID to type ("vm", "system-container", "k8s") + Timestamp int64 `json:"timestamp"` // Unix timestamp in milliseconds + Stats ChartStats `json:"stats"` // Includes pointCounts + source hints +} + +func EmptyWorkloadChartsResponse() WorkloadChartsResponse { + return WorkloadChartsResponse{}.NormalizeCollections() +} + +func (r WorkloadChartsResponse) NormalizeCollections() WorkloadChartsResponse { + if r.ChartData == nil { + r.ChartData = map[string]VMChartData{} + } + if r.DockerData == nil { + r.DockerData = map[string]VMChartData{} + } + if r.GuestTypes == nil { + r.GuestTypes = map[string]string{} + } + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// WorkloadsSummaryMetricData captures aggregate workload trend lines for a +// single metric (median and p95 across workloads). +type WorkloadsSummaryMetricData struct { + P50 []MetricPoint `json:"p50"` + P95 []MetricPoint `json:"p95"` +} + +func (m WorkloadsSummaryMetricData) NormalizeCollections() WorkloadsSummaryMetricData { + if m.P50 == nil { + m.P50 = []MetricPoint{} + } + if m.P95 == nil { + m.P95 = []MetricPoint{} + } + return m +} + +// WorkloadsGuestCounts captures workload counts used by the workloads summary +// cards for quick context. +type WorkloadsGuestCounts struct { + Total int `json:"total"` + Running int `json:"running"` + Stopped int `json:"stopped"` +} + +// WorkloadsSummaryContributor identifies a high-impact workload for a metric. +type WorkloadsSummaryContributor struct { + ID string `json:"id"` + Name string `json:"name"` + Value float64 `json:"value"` +} + +// WorkloadsSummaryContributors groups top contributors by metric. +type WorkloadsSummaryContributors struct { + CPU []WorkloadsSummaryContributor `json:"cpu"` + Memory []WorkloadsSummaryContributor `json:"memory"` + Disk []WorkloadsSummaryContributor `json:"disk"` + Network []WorkloadsSummaryContributor `json:"network"` +} + +func (c WorkloadsSummaryContributors) NormalizeCollections() WorkloadsSummaryContributors { + if c.CPU == nil { + c.CPU = []WorkloadsSummaryContributor{} + } + if c.Memory == nil { + c.Memory = []WorkloadsSummaryContributor{} + } + if c.Disk == nil { + c.Disk = []WorkloadsSummaryContributor{} + } + if c.Network == nil { + c.Network = []WorkloadsSummaryContributor{} + } + return c +} + +// WorkloadsSummaryBlastRadius describes how concentrated each metric pressure is. +type WorkloadsSummaryBlastRadius struct { + Scope string `json:"scope"` // idle, concentrated, mixed, distributed + Top3Share float64 `json:"top3Share"` + ActiveWorkloads int `json:"activeWorkloads"` +} + +// WorkloadsSummaryBlastRadiusGroup groups blast-radius insights by metric. +type WorkloadsSummaryBlastRadiusGroup struct { + CPU WorkloadsSummaryBlastRadius `json:"cpu"` + Memory WorkloadsSummaryBlastRadius `json:"memory"` + Disk WorkloadsSummaryBlastRadius `json:"disk"` + Network WorkloadsSummaryBlastRadius `json:"network"` +} + +// WorkloadsSummaryChartsResponse is a compact response for workloads top-card +// sparklines. It avoids returning per-workload time series. +type WorkloadsSummaryChartsResponse struct { + CPU WorkloadsSummaryMetricData `json:"cpu"` + Memory WorkloadsSummaryMetricData `json:"memory"` + Disk WorkloadsSummaryMetricData `json:"disk"` + Network WorkloadsSummaryMetricData `json:"network"` + GuestCounts WorkloadsGuestCounts `json:"guestCounts"` + TopContributors WorkloadsSummaryContributors `json:"topContributors"` + BlastRadius WorkloadsSummaryBlastRadiusGroup `json:"blastRadius"` + Timestamp int64 `json:"timestamp"` + Stats ChartStats `json:"stats"` +} + +func EmptyWorkloadsSummaryChartsResponse() WorkloadsSummaryChartsResponse { + return WorkloadsSummaryChartsResponse{}.NormalizeCollections() +} + +func (r WorkloadsSummaryChartsResponse) NormalizeCollections() WorkloadsSummaryChartsResponse { + r.CPU = r.CPU.NormalizeCollections() + r.Memory = r.Memory.NormalizeCollections() + r.Disk = r.Disk.NormalizeCollections() + r.Network = r.Network.NormalizeCollections() + r.TopContributors = r.TopContributors.NormalizeCollections() + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// StorageSummaryTrendResponse is a compact response for the dashboard storage +// card. It intentionally avoids returning per-pool and per-disk series. +type StorageSummaryTrendResponse struct { + Capacity []MetricPoint `json:"capacity"` + Timestamp int64 `json:"timestamp"` + Stats ChartStats `json:"stats"` +} + +func EmptyStorageSummaryTrendResponse() StorageSummaryTrendResponse { + return StorageSummaryTrendResponse{}.NormalizeCollections() +} + +func (r StorageSummaryTrendResponse) NormalizeCollections() StorageSummaryTrendResponse { + if r.Capacity == nil { + r.Capacity = []MetricPoint{} + } + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// ChartStats represents chart statistics +type ChartStats struct { + OldestDataTimestamp int64 `json:"oldestDataTimestamp"` + Range string `json:"range,omitempty"` + RangeSeconds int64 `json:"rangeSeconds,omitempty"` + MetricsStoreEnabled bool `json:"metricsStoreEnabled"` + PrimarySourceHint string `json:"primarySourceHint,omitempty"` + InMemoryThresholdSecs int64 `json:"inMemoryThresholdSecs,omitempty"` + PointCounts ChartPointCounts `json:"pointCounts,omitempty"` +} + +func (s ChartStats) NormalizeCollections() ChartStats { + return s +} + +// ChartPointCounts summarizes how many points were returned in /api/charts. +type ChartPointCounts struct { + Total int `json:"total,omitempty"` + Guests int `json:"guests,omitempty"` + Nodes int `json:"nodes,omitempty"` + Storage int `json:"storage,omitempty"` + DockerContainers int `json:"dockerContainers,omitempty"` + DockerHosts int `json:"dockerHosts,omitempty"` + Agents int `json:"agents,omitempty"` +} + +// VMChartData represents chart data for a VM +type VMChartData map[string][]MetricPoint + +// NodeChartData represents chart data for a node +type NodeChartData map[string][]MetricPoint + +// StorageChartData represents chart data for storage +type StorageChartData map[string][]MetricPoint + +// StorageChartsResponse represents storage charts API response. +// It includes both pool-level capacity metrics and physical disk metrics. +type StorageChartsResponse struct { + Pools map[string]StoragePoolChartData `json:"pools"` + Disks map[string]StorageDiskChartData `json:"disks"` + Stats ChartStats `json:"stats"` +} + +func EmptyStorageChartsResponse() StorageChartsResponse { + return StorageChartsResponse{}.NormalizeCollections() +} + +func (r StorageChartsResponse) NormalizeCollections() StorageChartsResponse { + if r.Pools == nil { + r.Pools = map[string]StoragePoolChartData{} + } + if r.Disks == nil { + r.Disks = map[string]StorageDiskChartData{} + } + for key, pool := range r.Pools { + r.Pools[key] = pool.NormalizeCollections() + } + for key, disk := range r.Disks { + r.Disks[key] = disk.NormalizeCollections() + } + r.Stats = r.Stats.NormalizeCollections() + return r +} + +// StoragePoolChartData holds per-pool capacity time-series. +type StoragePoolChartData struct { + Name string `json:"name"` + Usage []MetricPoint `json:"usage"` + Used []MetricPoint `json:"used"` + Avail []MetricPoint `json:"avail"` +} + +func (d StoragePoolChartData) NormalizeCollections() StoragePoolChartData { + if d.Usage == nil { + d.Usage = []MetricPoint{} + } + if d.Used == nil { + d.Used = []MetricPoint{} + } + if d.Avail == nil { + d.Avail = []MetricPoint{} + } + return d +} + +// StorageDiskChartData holds per-disk temperature time-series. +type StorageDiskChartData struct { + Name string `json:"name"` + Node string `json:"node"` + Temperature []MetricPoint `json:"temperature"` +} + +func (d StorageDiskChartData) NormalizeCollections() StorageDiskChartData { + if d.Temperature == nil { + d.Temperature = []MetricPoint{} + } + return d +} + +// MetricPoint represents a single metric data point +type MetricPoint struct { + Timestamp int64 `json:"timestamp"` + Value float64 `json:"value"` +} + +// AgentVersionResponse represents Docker / Podman module version information. +type AgentVersionResponse struct { + Version string `json:"version"` +} diff --git a/internal/api/charts_compat.go b/internal/api/charts_compat.go new file mode 100644 index 000000000..74e539f3a --- /dev/null +++ b/internal/api/charts_compat.go @@ -0,0 +1,51 @@ +package api + +import ( + "context" + "net/http" + + "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" +) + +type routerChartMonitorResolver struct { + router *Router +} + +func (r routerChartMonitorResolver) MonitorForContext(ctx context.Context) *monitoring.Monitor { + if r.router == nil { + return nil + } + return r.router.getTenantMonitor(ctx) +} + +func (r *Router) ensureChartService() *chartapi.Service { + if r.chartService == nil { + r.chartService = chartapi.NewService(routerChartMonitorResolver{router: r}) + } + return r.chartService +} + +func (r *Router) handleCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleCharts(w, req) +} + +func (r *Router) handleWorkloadCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleWorkloadCharts(w, req) +} + +func (r *Router) handleInfrastructureCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleInfrastructureCharts(w, req) +} + +func (r *Router) handleWorkloadsSummaryCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleWorkloadsSummaryCharts(w, req) +} + +func (r *Router) handleStorageCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleStorageCharts(w, req) +} + +func (r *Router) handleStorageSummaryCharts(w http.ResponseWriter, req *http.Request) { + r.ensureChartService().HandleStorageSummaryCharts(w, req) +} diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 5d7162b46..e4ed6e0b2 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -37,7 +37,9 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" "github.com/rcourtman/pulse-go-rewrite/internal/ai/unified" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" "github.com/rcourtman/pulse-go-rewrite/internal/api/configapi" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/license/entitlements" "github.com/rcourtman/pulse-go-rewrite/internal/mock" @@ -480,7 +482,7 @@ func TestContract_UnifiedSeedSourcesUsesSourceOwnedAvailabilityEndpoint(t *testi TargetID: "ops-api", }}, } - sources := unifiedSeedSources([]unifiedresources.Resource{hostProjection}) + sources := resourceapi.UnifiedSeedSources([]unifiedresources.Resource{hostProjection}) if _, ok := sources[unifiedresources.SourceAvailability]; ok { t.Fatalf("seed sources = %v, host projection must not claim provider ownership", sources) } @@ -492,7 +494,7 @@ func TestContract_UnifiedSeedSourcesUsesSourceOwnedAvailabilityEndpoint(t *testi TargetID: "ops-api", }}, } - sources = unifiedSeedSources([]unifiedresources.Resource{hostProjection, check}) + sources = resourceapi.UnifiedSeedSources([]unifiedresources.Resource{hostProjection, check}) if _, ok := sources[unifiedresources.SourceAvailability]; !ok { t.Fatalf("seed sources = %v, want availability from source-owned endpoint", sources) } @@ -2501,7 +2503,7 @@ func TestContract_AISettingsProviderRegistryMetadata(t *testing.T) { func TestContract_ChartMetricPointsPreserveMillisecondPrecision(t *testing.T) { pointTime := time.Date(2026, time.March, 31, 12, 0, 0, 987_000_000, time.UTC) - converted := monitorPointsToAPI([]monitoring.MetricPoint{{ + converted := chartapi.MonitorPointsToAPI([]monitoring.MetricPoint{{ Timestamp: pointTime, Value: 42, }}) @@ -2915,8 +2917,8 @@ func TestContract_InfrastructureChartsNormalizeLongRangeMixedCadence(t *testing. if len(cpuSeries) == 0 { t.Fatal("expected normalized cpu series") } - if len(cpuSeries) > infrastructureSummaryMaxSeriesPoints { - t.Fatalf("expected cpu series <= %d points, got %d", infrastructureSummaryMaxSeriesPoints, len(cpuSeries)) + if len(cpuSeries) > chartapi.InfrastructureSummaryMaxSeriesPoints { + t.Fatalf("expected cpu series <= %d points, got %d", chartapi.InfrastructureSummaryMaxSeriesPoints, len(cpuSeries)) } if cpuSeries[len(cpuSeries)-1].Timestamp != now.UnixMilli() { t.Fatalf("expected latest cpu timestamp %d, got %d", now.UnixMilli(), cpuSeries[len(cpuSeries)-1].Timestamp) @@ -3235,7 +3237,7 @@ func TestContract_WorkloadChartsUseCanonicalWorkloadIDsForProviderBackedVMs(t *t if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil { t.Fatalf("expected one provider-backed VM in read state, got %+v", readState) } - resourceID, _, ok := vmChartRequest(readState.VMs()[0]) + resourceID, _, ok := chartapi.VMChartRequest(readState.VMs()[0]) if !ok { t.Fatal("expected canonical vm chart request") } @@ -3373,11 +3375,11 @@ func TestContract_WorkloadsSummaryChartsNormalizeLongRangeMixedCadence(t *testin if len(decoded.CPU.P50) == 0 { t.Fatal("expected normalized workload summary p50 series") } - if len(decoded.CPU.P50) > workloadsSummaryMaxSeriesPoints { - t.Fatalf("expected workload summary p50 <= %d points, got %d", workloadsSummaryMaxSeriesPoints, len(decoded.CPU.P50)) + if len(decoded.CPU.P50) > chartapi.WorkloadsSummaryMaxSeriesPoints { + t.Fatalf("expected workload summary p50 <= %d points, got %d", chartapi.WorkloadsSummaryMaxSeriesPoints, len(decoded.CPU.P50)) } - if len(decoded.CPU.P95) > workloadsSummaryMaxSeriesPoints { - t.Fatalf("expected workload summary p95 <= %d points, got %d", workloadsSummaryMaxSeriesPoints, len(decoded.CPU.P95)) + if len(decoded.CPU.P95) > chartapi.WorkloadsSummaryMaxSeriesPoints { + t.Fatalf("expected workload summary p95 <= %d points, got %d", chartapi.WorkloadsSummaryMaxSeriesPoints, len(decoded.CPU.P95)) } if decoded.CPU.P50[len(decoded.CPU.P50)-1].Timestamp != now.UnixMilli() { t.Fatalf("expected latest workload summary timestamp %d, got %d", now.UnixMilli(), decoded.CPU.P50[len(decoded.CPU.P50)-1].Timestamp) @@ -3396,9 +3398,9 @@ func TestContract_WorkloadsSummaryChartsNormalizeLongRangeMixedCadence(t *testin } func TestContract_WorkloadChartMetricBudgetGuardrailsRemainCanonical(t *testing.T) { - data, err := os.ReadFile("router.go") + data, err := os.ReadFile("chartapi/service.go") if err != nil { - t.Fatalf("failed to read router.go: %v", err) + t.Fatalf("failed to read chartapi/service.go: %v", err) } source := string(data) @@ -3428,7 +3430,7 @@ func TestContract_WorkloadChartMetricBudgetGuardrailsRemainCanonical(t *testing. } for _, snippet := range requiredSnippets { if !strings.Contains(source, snippet) { - t.Fatalf("router.go must contain %q", snippet) + t.Fatalf("chartapi/service.go must contain %q", snippet) } } } @@ -3436,7 +3438,7 @@ func TestContract_WorkloadChartMetricBudgetGuardrailsRemainCanonical(t *testing. func TestContract_GenerateStyledMockSeries_UsesTimestampBasedCurve(t *testing.T) { now := time.Date(2026, time.March, 31, 12, 0, 0, 0, time.UTC).UnixMilli() - coarse := generateStyledMockSeries( + coarse := chartapi.GenerateStyledMockSeries( now, time.Hour, 7, @@ -3445,7 +3447,7 @@ func TestContract_GenerateStyledMockSeries_UsesTimestampBasedCurve(t *testing.T) "orion-2-f54579833f9c", "memory", ) - fine := generateStyledMockSeries( + fine := chartapi.GenerateStyledMockSeries( now, time.Hour, 13, @@ -13331,11 +13333,11 @@ func TestContractResourceFiltersAcceptNativeDockerAndKubernetesInventory(t *test "k8s-event", }, ",") - if unsupported := unsupportedResourceTypeFilterTokens(raw); len(unsupported) != 0 { + if unsupported := resourceapi.UnsupportedResourceTypeFilterTokens(raw); len(unsupported) != 0 { t.Fatalf("native platform inventory filters should be supported, got unsupported=%v", unsupported) } - got := parseResourceTypes(raw) + got := resourceapi.ParseResourceTypes(raw) for _, resourceType := range []unifiedresources.ResourceType{ unifiedresources.ResourceTypeDockerImage, unifiedresources.ResourceTypeDockerVolume, diff --git a/internal/api/filters_branchcov0722pm_test.go b/internal/api/filters_branchcov0722pm_test.go index 21257a91b..92acac805 100644 --- a/internal/api/filters_branchcov0722pm_test.go +++ b/internal/api/filters_branchcov0722pm_test.go @@ -5,6 +5,8 @@ import ( "net/http/httptest" "testing" + "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" "github.com/rcourtman/pulse-go-rewrite/internal/deploy" "github.com/rcourtman/pulse-go-rewrite/internal/models" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -15,7 +17,7 @@ import ( // - filterPVEBackups (pve_backups.go) // - filterReplicationJobs (replication.go) // - cephDiscoveryTarget (resources.go) -// - capMetricPointSeriesByIndex (router.go) +// - chartapi.CapMetricPointSeriesByIndex (router.go) // - deriveInstallJobStatus (deploy_handlers.go) // // Conventions mirror internal/api/deploy_handlers_test.go (package api, @@ -257,7 +259,7 @@ func TestBranchcov0722PM_CephDiscoveryTarget(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := cephDiscoveryTarget(tc.resource) + got := resourceapi.CephDiscoveryTarget(tc.resource) if tc.wantNil { if got != nil { t.Fatalf("expected nil target, got %+v", got) @@ -283,7 +285,7 @@ func TestBranchcov0722PM_CephDiscoveryTarget(t *testing.T) { } } -// --- capMetricPointSeriesByIndex -------------------------------------------- +// --- chartapi.CapMetricPointSeriesByIndex -------------------------------------------- func TestBranchcov0722PM_CapMetricPointSeriesByIndex(t *testing.T) { // 10 distinct points: timestamp == index, value == index. @@ -352,7 +354,7 @@ func TestBranchcov0722PM_CapMetricPointSeriesByIndex(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - got := capMetricPointSeriesByIndex(tc.points, tc.maxPoints) + got := chartapi.CapMetricPointSeriesByIndex(tc.points, tc.maxPoints) if len(got) != tc.wantLen { t.Fatalf("len: got %d, want %d", len(got), tc.wantLen) } diff --git a/internal/api/load_test.go b/internal/api/load_test.go index 84e299c2d..7cbcbb628 100644 --- a/internal/api/load_test.go +++ b/internal/api/load_test.go @@ -18,131 +18,6 @@ import ( "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" ) -// TestLoad_500Node_ConcurrentResources validates that the /api/resources -// endpoint sustains acceptable latency under concurrent load with 500 nodes -// and ~2500 VMs in state, simulating a large self-hosted deployment. -// -// The test launches 50 concurrent goroutines each making serial requests over -// a 2-second window. It measures p50/p95/p99 and total throughput. -func TestLoad_500Node_ConcurrentResources(t *testing.T) { - skipUnderRace(t) - suppressTestLogs(t) - - state := buildLargeDeploymentState(t, 500) - - cfg := &config.Config{DataPath: t.TempDir()} - handlers := NewResourceHandlers(cfg) - handlers.SetStateProvider(&loadTestStateProvider{state: state}) - - // Warm the cache with an initial request. - req := httptest.NewRequest(http.MethodGet, "/api/resources", nil) - rec := httptest.NewRecorder() - handlers.HandleListResources(rec, req) - if rec.Code != http.StatusOK { - t.Fatalf("warmup failed: status %d, body: %s", rec.Code, rec.Body.String()) - } - var warmupResp map[string]interface{} - if err := json.Unmarshal(rec.Body.Bytes(), &warmupResp); err != nil { - t.Fatalf("warmup unmarshal: %v", err) - } - meta, _ := warmupResp["meta"].(map[string]interface{}) - total, _ := meta["total"].(float64) - // 500 nodes + 2500 VMs = 3000 resources exactly. - if int(total) != 3000 { - t.Fatalf("warmup: expected 3000 total resources, got %v", total) - } - t.Logf("state populated: %.0f total resources", total) - - const ( - concurrency = 50 - duration = 2 * time.Second - ) - - var ( - latencies = make([][]time.Duration, concurrency) - errors int64 - totalCount int64 - ) - - // Use a barrier to start all goroutines simultaneously, ensuring wall-clock - // timing starts after goroutine launch overhead. - var ready sync.WaitGroup - ready.Add(concurrency) - var wg sync.WaitGroup - - for g := 0; g < concurrency; g++ { - wg.Add(1) - g := g - latencies[g] = make([]time.Duration, 0, 200) - go func() { - defer wg.Done() - ready.Done() - ready.Wait() // All goroutines wait here until everyone is launched - deadline := time.Now().Add(duration) - for time.Now().Before(deadline) { - reqStart := time.Now() - req := httptest.NewRequest(http.MethodGet, "/api/resources?limit=50", nil) - rec := httptest.NewRecorder() - handlers.HandleListResources(rec, req) - elapsed := time.Since(reqStart) - if rec.Code != http.StatusOK { - atomic.AddInt64(&errors, 1) - continue - } - latencies[g] = append(latencies[g], elapsed) - atomic.AddInt64(&totalCount, 1) - } - }() - } - ready.Wait() - start := time.Now() - wg.Wait() - wallTime := time.Since(start) - - if errors > 0 { - t.Errorf("got %d error responses", errors) - } - - all := mergeLatencies(latencies) - if len(all) == 0 { - t.Fatal("no successful requests recorded") - } - - p50 := percentile(all, 0.50) - p95 := percentile(all, 0.95) - p99 := percentile(all, 0.99) - rps := float64(totalCount) / wallTime.Seconds() - - t.Logf("resources 500-node load: %d requests in %v (%.1f rps)", totalCount, wallTime, rps) - t.Logf(" p50=%v p95=%v p99=%v", p50, p95, p99) - - // At 500-node scale (3000 resources) with 50 concurrent goroutines, the - // resource handler processes snapshot comparison, filtering, sorting, - // pagination, and JSON serialization under mutex contention. The p95 budget - // accounts for this in-process overhead — it catches gross regressions - // (e.g., O(n²) algorithms, lock contention bugs) without being flaky. The - // hosted-runner budget reflects the April 9, 2026 RC dry run (~3.23s p95). - target := effectiveLoadP95Budget("resources", 3*time.Second) - if p95 > target { - failOrSkipLoadOverrun(t, "p95 latency %v exceeds %v budget for 500-node concurrent resources load", p95, target) - } - // Use completed request count rather than wall-clock RPS so tail-overrun - // doesn't get double-counted by both the latency budget and a throughput - // floor. The hosted-runner floor was recalibrated on 2026-05-11 down from - // 65 to 40 after the post-rc.4 operator-state, agent-substrate context, - // and action-audit plumbing added enough per-request CPU on the 2-core - // GitHub runner to push consistent throughput to 50-55 requests in the - // 2s test window (p95 latency stays inside the 4s envelope). Local - // hardware (more cores) still sees 300+ requests; the local floor of 100 - // is unchanged. The test still catches gross regressions (e.g. drops to - // single-digit rps from algorithmic blowup) while accommodating the - // observed CI ceiling. - minCount := effectiveLoadMinCount(100, 40) - if totalCount < minCount { - failOrSkipLoadOverrun(t, "completed only %d requests, expected at least %d for 500-node concurrent resources load", totalCount, minCount) - } -} - // TestLoad_500Node_ConcurrentMetricsHistory validates that /api/metrics-store/history // sustains acceptable latency when 50 concurrent goroutines query chart data for // resources across a 500-node deployment. diff --git a/internal/api/patrol_action_broker_test.go b/internal/api/patrol_action_broker_test.go index d853067cc..3f0501746 100644 --- a/internal/api/patrol_action_broker_test.go +++ b/internal/api/patrol_action_broker_test.go @@ -205,9 +205,7 @@ func TestPatrolRefreshRebindsCurrentPolicyAndTrustedOrigin(t *testing.T) { snapshot: models.StateSnapshot{LastUpdate: changed.UpdatedAt}, resources: []unified.Resource{changed}, }) - h.cacheMu.Lock() - h.registryCache = make(map[string]registryCacheEntry) - h.cacheMu.Unlock() + h.invalidateCache("default") policyVersion = "tenant-v2" h.SetActionRefreshPlanner(NewActionRefreshPlanner(h, policy)) diff --git a/internal/api/pure_helpers_branchcov0724pm_test.go b/internal/api/pure_helpers_branchcov0724pm_test.go index 0a41b0159..0dfb2ad00 100644 --- a/internal/api/pure_helpers_branchcov0724pm_test.go +++ b/internal/api/pure_helpers_branchcov0724pm_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" @@ -22,7 +23,7 @@ import ( // - restoreAgentExecMetadata (agent_exec_token_binding.go) // - buildAlertConnectionSnapshotsWithRuntimeSources (connections_alerts.go) // - mockProtectionPostures (recovery_handlers.go) -// - buildMockWorkloadMetricHistorySeries (router.go) +// - chartapi.BuildMockWorkloadMetricHistorySeries (router.go) // --------------------------------------------------------------------------- // restoreAgentExecMetadata @@ -394,7 +395,7 @@ func TestBranchcov0724pmMockProtectionPostures(t *testing.T) { } // --------------------------------------------------------------------------- -// buildMockWorkloadMetricHistorySeries +// chartapi.BuildMockWorkloadMetricHistorySeries // --------------------------------------------------------------------------- func TestBranchcov0724pmBuildMockWorkloadMetricHistorySeries(t *testing.T) { @@ -402,7 +403,7 @@ func TestBranchcov0724pmBuildMockWorkloadMetricHistorySeries(t *testing.T) { t.Run("unsupported_metric_type_returns_nil", func(t *testing.T) { for _, metricType := range []string{"smart_temp", "bogus", ""} { - got := buildMockWorkloadMetricHistorySeries(now, time.Hour, 100, "vm", "res-1", metricType, 50.0) + got := chartapi.BuildMockWorkloadMetricHistorySeries(now, time.Hour, 100, "vm", "res-1", metricType, 50.0) if got != nil { t.Fatalf("metricType %q: expected nil, got %d points", metricType, len(got)) } @@ -413,11 +414,11 @@ func TestBranchcov0724pmBuildMockWorkloadMetricHistorySeries(t *testing.T) { supported := []string{"cpu", "memory", "disk", "diskread", "diskwrite", "netin", "netout"} duration := 2 * time.Hour maxPoints := 0 - expectedLen := targetMockSeriesPoints(duration, maxPoints) + expectedLen := chartapi.TargetMockSeriesPoints(duration, maxPoints) for _, metricType := range supported { t.Run(metricType, func(t *testing.T) { - got := buildMockWorkloadMetricHistorySeries(now, duration, maxPoints, "vm", "res-1", metricType, 50.0) + got := chartapi.BuildMockWorkloadMetricHistorySeries(now, duration, maxPoints, "vm", "res-1", metricType, 50.0) if len(got) != expectedLen { t.Fatalf("expected %d points, got %d", expectedLen, len(got)) } @@ -444,23 +445,23 @@ func TestBranchcov0724pmBuildMockWorkloadMetricHistorySeries(t *testing.T) { t.Run("max_points_caps_series_length", func(t *testing.T) { duration := 24 * time.Hour maxPoints := 50 - got := buildMockWorkloadMetricHistorySeries(now, duration, maxPoints, "vm", "res-1", "cpu", 50.0) + got := chartapi.BuildMockWorkloadMetricHistorySeries(now, duration, maxPoints, "vm", "res-1", "cpu", 50.0) if len(got) != maxPoints { t.Fatalf("expected %d points (capped by maxPoints), got %d", maxPoints, len(got)) } }) t.Run("deterministic_for_same_inputs", func(t *testing.T) { - first := buildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) - second := buildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) + first := chartapi.BuildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) + second := chartapi.BuildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) if !reflect.DeepEqual(first, second) { t.Fatalf("series not deterministic for identical inputs") } }) t.Run("different_resource_ids_produce_different_series", func(t *testing.T) { - a := buildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) - b := buildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-2", "cpu", 50.0) + a := chartapi.BuildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-1", "cpu", 50.0) + b := chartapi.BuildMockWorkloadMetricHistorySeries(now, 2*time.Hour, 0, "vm", "res-2", "cpu", 50.0) if reflect.DeepEqual(a, b) { t.Fatalf("expected different series for different resource IDs") } diff --git a/internal/api/resource_test_support_test.go b/internal/api/resource_test_support_test.go new file mode 100644 index 000000000..a352221c9 --- /dev/null +++ b/internal/api/resource_test_support_test.go @@ -0,0 +1,64 @@ +package api + +import ( + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +// These fixtures remain in the API integration-test package because action, +// agent-context, and router tests compose the public ResourceHandlers facade. +type resourceUnifiedSeedProvider struct { + snapshot models.StateSnapshot + resources []unified.Resource +} + +type resourceStateProvider struct { + snapshot models.StateSnapshot +} + +func (p resourceStateProvider) ReadSnapshot() models.StateSnapshot { return p.snapshot } + +type tenantResourceStateProvider struct { + snapshots map[string]models.StateSnapshot +} + +func (p tenantResourceStateProvider) GetStateForTenant(orgID string) models.StateSnapshot { + return p.snapshots[orgID] +} + +func (p tenantResourceStateProvider) UnifiedReadStateForTenant(orgID string) unified.ReadState { + return SnapshotReadState(p.GetStateForTenant(orgID)) +} + +func (p tenantResourceStateProvider) UnifiedResourceSnapshotForTenant(orgID string) ([]unified.Resource, time.Time) { + snapshot := p.GetStateForTenant(orgID) + if snapshot.LastUpdate.IsZero() { + return nil, time.Time{} + } + return []unified.Resource{{ + ID: "agent-tenant-seeded", Type: unified.ResourceTypeAgent, Name: "tenant-seeded", + Status: unified.StatusOnline, LastSeen: snapshot.LastUpdate, UpdatedAt: snapshot.LastUpdate, + Sources: []unified.DataSource{unified.SourceAgent}, + Identity: unified.ResourceIdentity{Hostnames: []string{"tenant-seeded"}}, + }}, snapshot.LastUpdate +} + +func (p resourceUnifiedSeedProvider) ReadSnapshot() models.StateSnapshot { return p.snapshot } + +func (p resourceUnifiedSeedProvider) UnifiedResourceSnapshot() ([]unified.Resource, time.Time) { + return append([]unified.Resource(nil), p.resources...), p.snapshot.LastUpdate +} + +type mutableResourceUnifiedSeedProvider struct { + snapshot models.StateSnapshot + resources []unified.Resource + freshness time.Time +} + +func (p *mutableResourceUnifiedSeedProvider) ReadSnapshot() models.StateSnapshot { return p.snapshot } + +func (p *mutableResourceUnifiedSeedProvider) UnifiedResourceSnapshot() ([]unified.Resource, time.Time) { + return append([]unified.Resource(nil), p.resources...), p.freshness +} diff --git a/internal/api/resources_k8s_namespaces.go b/internal/api/resourceapi/k8s_namespaces.go similarity index 92% rename from internal/api/resources_k8s_namespaces.go rename to internal/api/resourceapi/k8s_namespaces.go index 234dedee3..bfc04923c 100644 --- a/internal/api/resources_k8s_namespaces.go +++ b/internal/api/resourceapi/k8s_namespaces.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "encoding/json" @@ -6,6 +6,7 @@ import ( "sort" "strings" + "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -41,7 +42,7 @@ func (r k8sNamespacesResponse) normalizeCollections() k8sNamespacesResponse { // HandleK8sNamespaces handles GET /api/resources/k8s/namespaces?cluster= // and returns namespace-level counts for Pods and Deployments. -func (h *ResourceHandlers) HandleK8sNamespaces(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleK8sNamespaces(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return @@ -53,10 +54,10 @@ func (h *ResourceHandlers) HandleK8sNamespaces(w http.ResponseWriter, r *http.Re return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) resources, _, err := h.sharedRawResources(orgID) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } diff --git a/internal/api/resourceapi/load_test.go b/internal/api/resourceapi/load_test.go new file mode 100644 index 000000000..f3a3ff52e --- /dev/null +++ b/internal/api/resourceapi/load_test.go @@ -0,0 +1,179 @@ +package resourceapi + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/models" +) + +// TestLoad_500Node_ConcurrentResources validates the resource query service +// with 500 nodes and 2,500 VMs under 50-way concurrent request load. +func TestLoad_500Node_ConcurrentResources(t *testing.T) { + if raceEnabled { + t.Skip("skipping latency test under -race") + } + + state := buildResourceLoadState(t, 500) + handlers := NewQueryService(&config.Config{DataPath: t.TempDir()}) + handlers.SetStateProvider(&resourceLoadStateProvider{state: state}) + + req := httptest.NewRequest(http.MethodGet, "/api/resources", nil) + rec := httptest.NewRecorder() + handlers.HandleListResources(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("warmup failed: status %d, body: %s", rec.Code, rec.Body.String()) + } + var warmupResp map[string]interface{} + if err := json.Unmarshal(rec.Body.Bytes(), &warmupResp); err != nil { + t.Fatalf("warmup unmarshal: %v", err) + } + meta, _ := warmupResp["meta"].(map[string]interface{}) + total, _ := meta["total"].(float64) + if int(total) != 3000 { + t.Fatalf("warmup: expected 3000 total resources, got %v", total) + } + + const concurrency = 50 + const duration = 2 * time.Second + latencies := make([][]time.Duration, concurrency) + var errors, totalCount int64 + var ready, workers sync.WaitGroup + ready.Add(concurrency) + + for index := 0; index < concurrency; index++ { + workers.Add(1) + index := index + latencies[index] = make([]time.Duration, 0, 200) + go func() { + defer workers.Done() + ready.Done() + ready.Wait() + deadline := time.Now().Add(duration) + for time.Now().Before(deadline) { + started := time.Now() + req := httptest.NewRequest(http.MethodGet, "/api/resources?limit=50", nil) + rec := httptest.NewRecorder() + handlers.HandleListResources(rec, req) + if rec.Code != http.StatusOK { + atomic.AddInt64(&errors, 1) + continue + } + latencies[index] = append(latencies[index], time.Since(started)) + atomic.AddInt64(&totalCount, 1) + } + }() + } + ready.Wait() + started := time.Now() + workers.Wait() + wallTime := time.Since(started) + if errors > 0 { + t.Errorf("got %d error responses", errors) + } + + all := mergeResourceLoadLatencies(latencies) + if len(all) == 0 { + t.Fatal("no successful requests recorded") + } + p50 := resourceLoadPercentile(all, 0.50) + p95 := resourceLoadPercentile(all, 0.95) + p99 := resourceLoadPercentile(all, 0.99) + t.Logf("resources 500-node load: %d requests in %v (%.1f rps)", totalCount, wallTime, float64(totalCount)/wallTime.Seconds()) + t.Logf("p50=%v p95=%v p99=%v", p50, p95, p99) + + target := 3 * time.Second + if os.Getenv("GITHUB_ACTIONS") == "true" { + target = 4 * time.Second + } + if p95 > target { + resourceLoadOverrun(t, "p95 latency %v exceeds %v budget for 500-node concurrent resources load", p95, target) + } + minimum := int64(100) + if os.Getenv("GITHUB_ACTIONS") == "true" { + minimum = 40 + } + if totalCount < minimum { + resourceLoadOverrun(t, "completed only %d requests, expected at least %d for 500-node concurrent resources load", totalCount, minimum) + } +} + +func buildResourceLoadState(t *testing.T, numNodes int) *models.State { + t.Helper() + state := models.NewState() + const numInstances = 10 + basePerInstance, remainder := numNodes/numInstances, numNodes%numInstances + nodesSoFar := 0 + for instance := 0; instance < numInstances; instance++ { + instanceName := fmt.Sprintf("pve%d", instance) + count := basePerInstance + if instance < remainder { + count++ + } + if count == 0 { + continue + } + nodes := make([]models.Node, count) + for node := range nodes { + globalIndex := nodesSoFar + node + nodes[node] = models.Node{ + ID: fmt.Sprintf("%s:node%d", instanceName, node), Name: fmt.Sprintf("node-%d", globalIndex), Instance: instanceName, + Status: "online", CPU: float64(globalIndex%80+10) / 100, + Memory: models.Memory{Usage: float64(globalIndex%60 + 20), Total: 64 << 30, Used: 32 << 30}, + Disk: models.Disk{Usage: float64(globalIndex%40 + 30), Total: 500 << 30, Used: 250 << 30}, + } + } + state.UpdateNodesForInstance(instanceName, nodes) + vms := make([]models.VM, count*5) + for vm := range vms { + nodeIndex := vm / 5 + globalIndex := (nodesSoFar+nodeIndex)*5 + vm%5 + vms[vm] = models.VM{ + ID: fmt.Sprintf("%s:node%d:%d", instanceName, nodeIndex, 1000+globalIndex), VMID: 1000 + globalIndex, + Name: fmt.Sprintf("vm-%d", globalIndex), Node: fmt.Sprintf("node%d", nodeIndex), Instance: instanceName, + Status: "running", Type: "qemu", CPU: float64(globalIndex%80+10) / 100, + Memory: models.Memory{Usage: float64(globalIndex%60 + 20), Total: 4 << 30, Used: 2 << 30}, + Disk: models.Disk{Usage: float64(globalIndex%40 + 30), Total: 50 << 30, Used: 25 << 30}, + } + } + state.UpdateVMsForInstance(instanceName, vms) + nodesSoFar += count + } + return state +} + +func resourceLoadOverrun(t *testing.T, format string, args ...interface{}) { + t.Helper() + if os.Getenv("GITHUB_ACTIONS") == "true" { + t.Errorf(format, args...) + return + } + t.Skipf("%s (host CPU contention can cause this locally; CI enforces the budget)", fmt.Sprintf(format, args...)) +} + +func mergeResourceLoadLatencies(groups [][]time.Duration) []time.Duration { + var merged []time.Duration + for _, group := range groups { + merged = append(merged, group...) + } + return merged +} + +func resourceLoadPercentile(durations []time.Duration, percentile float64) time.Duration { + sorted := append([]time.Duration(nil), durations...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + return sorted[int(float64(len(sorted)-1)*percentile)] +} + +type resourceLoadStateProvider struct{ state *models.State } + +func (p *resourceLoadStateProvider) ReadSnapshot() models.StateSnapshot { return p.state.GetSnapshot() } diff --git a/internal/api/resourceapi/norace_test.go b/internal/api/resourceapi/norace_test.go new file mode 100644 index 000000000..7d63256d2 --- /dev/null +++ b/internal/api/resourceapi/norace_test.go @@ -0,0 +1,5 @@ +//go:build !race + +package resourceapi + +const raceEnabled = false diff --git a/internal/api/resourceapi/race_test.go b/internal/api/resourceapi/race_test.go new file mode 100644 index 000000000..231fa6430 --- /dev/null +++ b/internal/api/resourceapi/race_test.go @@ -0,0 +1,5 @@ +//go:build race + +package resourceapi + +const raceEnabled = true diff --git a/internal/api/resources.go b/internal/api/resourceapi/resources.go similarity index 87% rename from internal/api/resources.go rename to internal/api/resourceapi/resources.go index b08e6b0be..3297ae78a 100644 --- a/internal/api/resources.go +++ b/internal/api/resourceapi/resources.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "context" @@ -14,6 +14,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/actionlifecycle" + "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" @@ -22,33 +23,35 @@ import ( "github.com/rs/zerolog/log" ) -// ResourceHandlers provides HTTP handlers for the unified resource API. -type ResourceHandlers struct { - cfg *config.Config - storeMu sync.Mutex - stores map[string]unified.ResourceStore - cacheMu sync.Mutex - registryCache map[string]registryCacheEntry - supplementalMu sync.RWMutex - supplementalRecords map[unified.DataSource]SupplementalRecordsProvider - stateProvider SnapshotProvider - tenantStateProvider TenantStateProvider - actionExecutor ActionExecutor - actionCompleted func(unified.ActionAuditRecord) - actionTransition func(orgID string, record unified.ActionAuditRecord) - policyAdmission *actionlifecycle.PolicyAdmissionCoordinator - actionEmergencyStop func(orgID string) (bool, error) - actionDecisionAuthorizer actionlifecycle.DecisionAuthorizer - actionExecutionAuthorizer actionlifecycle.ExecutionAuthorizer - actionRefreshPlanner actionlifecycle.RefreshPlanner - discoveryReadiness ResourceDiscoveryReadinessProvider - operatorStateChanged func(orgID, resourceID string) +// QueryService provides HTTP handlers for the unified resource API. +type QueryService struct { + cfg *config.Config + storeMu sync.Mutex + stores map[string]unified.ResourceStore + cacheMu sync.Mutex + registryCache map[string]registryCacheEntry + supplementalMu sync.RWMutex + supplementalRecords map[unified.DataSource]SupplementalRecordsProvider + stateProvider SnapshotProvider + tenantStateProvider TenantStateProvider + actionAvailability actionlifecycle.AvailabilityChecker + discoveryReadiness ResourceDiscoveryReadinessProvider } -// SetOperatorStateChanged installs the runtime reconciliation hook invoked -// after a canonical operator-state mutation commits successfully. -func (h *ResourceHandlers) SetOperatorStateChanged(callback func(orgID, resourceID string)) { - h.operatorStateChanged = callback +// SnapshotProvider supplies the legacy monitoring snapshot used while callers +// migrate to the canonical unified-resource snapshot contract. +type SnapshotProvider = models.SnapshotProvider + +// TenantStateProvider is the tenant-scoped resource read contract. +type TenantStateProvider interface { + UnifiedReadStateForTenant(orgID string) unified.ReadState + UnifiedResourceSnapshotForTenant(orgID string) ([]unified.Resource, time.Time) +} + +// UnifiedResourceSnapshotProvider supplies a canonical unified-resource seed +// and freshness marker for the default tenant. +type UnifiedResourceSnapshotProvider interface { + UnifiedResourceSnapshot() ([]unified.Resource, time.Time) } // ResourceDiscoveryReadinessProvider projects service-discovery state onto a @@ -92,72 +95,41 @@ type TenantSupplementalSnapshotSourceOwner interface { SnapshotOwnedSourcesForOrg(orgID string) []unified.DataSource } -// NewResourceHandlers creates a new ResourceHandlers. -func NewResourceHandlers(cfg *config.Config) *ResourceHandlers { - return &ResourceHandlers{ +// NewQueryService creates a unified resource query and storage service. +func NewQueryService(cfg *config.Config) *QueryService { + return &QueryService{ cfg: cfg, stores: make(map[string]unified.ResourceStore), registryCache: make(map[string]registryCacheEntry), supplementalRecords: make(map[unified.DataSource]SupplementalRecordsProvider), - policyAdmission: &actionlifecycle.PolicyAdmissionCoordinator{}, } } // SetStateProvider sets the state provider for on-demand population. -func (h *ResourceHandlers) SetStateProvider(provider SnapshotProvider) { +func (h *QueryService) SetStateProvider(provider SnapshotProvider) { h.stateProvider = provider } // SetTenantStateProvider sets the tenant-aware provider. -func (h *ResourceHandlers) SetTenantStateProvider(provider TenantStateProvider) { +func (h *QueryService) SetTenantStateProvider(provider TenantStateProvider) { h.tenantStateProvider = provider } -// SetActionExecutor configures the API-owned action execution driver. -func (h *ResourceHandlers) SetActionExecutor(executor ActionExecutor) { - h.actionExecutor = executor -} - -func (h *ResourceHandlers) SetActionEmergencyStopChecker(checker func(orgID string) (bool, error)) { - h.actionEmergencyStop = checker -} - -func (h *ResourceHandlers) SetActionAuthorizers(decision actionlifecycle.DecisionAuthorizer, execution actionlifecycle.ExecutionAuthorizer) { - h.actionDecisionAuthorizer = decision - h.actionExecutionAuthorizer = execution -} - -// SetActionRefreshPlanner installs the trusted reconstruction hook used when -// a broker-originated immutable plan must be replaced. -func (h *ResourceHandlers) SetActionRefreshPlanner(planner actionlifecycle.RefreshPlanner) { - h.actionRefreshPlanner = planner -} - -// SetActionCompletedPublisher configures the terminal action notification hook -// used by the agent SSE bridge. It is intentionally outside the execution -// driver so refused-before-dispatch failures and future executor -// implementations all publish from the API-owned lifecycle boundary. -func (h *ResourceHandlers) SetActionCompletedPublisher(publisher func(unified.ActionAuditRecord)) { - h.actionCompleted = publisher -} - -// SetActionTransitionPublisher configures the persisted-state transition -// hook on the shared action lifecycle: plan creation, approval decisions, -// and terminal execution outcomes, published only after the corresponding -// store write succeeds. The org ID keys per-tenant reconciliation (e.g. -// mapping a Patrol-origin action's decision back onto its finding). -func (h *ResourceHandlers) SetActionTransitionPublisher(publisher func(orgID string, record unified.ActionAuditRecord)) { - h.actionTransition = publisher +// SetActionAvailabilityChecker configures the read-only projection of action +// readiness onto resource responses. Mutation lifecycle ownership remains in +// the API composition layer. +func (h *QueryService) SetActionAvailabilityChecker(checker actionlifecycle.AvailabilityChecker) { + h.actionAvailability = checker } // SetDiscoveryReadinessProvider wires the canonical discovery-readiness // projection onto resource responses and Assistant context packs. -func (h *ResourceHandlers) SetDiscoveryReadinessProvider(provider ResourceDiscoveryReadinessProvider) { +func (h *QueryService) SetDiscoveryReadinessProvider(provider ResourceDiscoveryReadinessProvider) { h.discoveryReadiness = provider } // SetSupplementalRecordsProvider configures additional records for a source. -func (h *ResourceHandlers) SetSupplementalRecordsProvider(source unified.DataSource, provider SupplementalRecordsProvider) { +func (h *QueryService) SetSupplementalRecordsProvider(source unified.DataSource, provider SupplementalRecordsProvider) { h.supplementalMu.Lock() if h.supplementalRecords == nil { h.supplementalRecords = make(map[unified.DataSource]SupplementalRecordsProvider) @@ -176,16 +148,16 @@ func (h *ResourceHandlers) SetSupplementalRecordsProvider(source unified.DataSou } // HandleListResources handles GET /api/resources. -func (h *ResourceHandlers) HandleListResources(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleListResources(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) sharedResources, registry, err := h.sharedPresentationResources(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -227,16 +199,16 @@ func (h *ResourceHandlers) HandleListResources(w http.ResponseWriter, r *http.Re } // HandleStorageSummary handles GET /api/resources/storage-summary. -func (h *ResourceHandlers) HandleStorageSummary(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleStorageSummary(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) resources, _, err := h.sharedRawResources(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -257,16 +229,16 @@ func (h *ResourceHandlers) HandleStorageSummary(w http.ResponseWriter, r *http.R } // HandleStorageIncidents handles GET /api/resources/storage-incidents. -func (h *ResourceHandlers) HandleStorageIncidents(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleStorageIncidents(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) resources, _, err := h.sharedRawResources(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -313,16 +285,16 @@ func pruneResourceForListResponse(resource *unified.Resource) { } // HandleGetResource handles GET /api/resources/{id}. -func (h *ResourceHandlers) HandleGetResource(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleGetResource(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) registry, err := h.buildRegistry(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -354,9 +326,9 @@ func (h *ResourceHandlers) HandleGetResource(w http.ResponseWriter, r *http.Requ json.NewEncoder(w).Encode(resourceCopy) } -func (h *ResourceHandlers) applyActionAvailability(ctx context.Context, resources []unified.Resource) { - checker, ok := h.actionExecutor.(ActionAvailabilityChecker) - if !ok || checker == nil { +func (h *QueryService) applyActionAvailability(ctx context.Context, resources []unified.Resource) { + checker := h.actionAvailability + if checker == nil { return } for i := range resources { @@ -404,6 +376,12 @@ func presentationResourceByID(registry *unified.ResourceRegistry, resourceID str return resource, ok } +// PresentationResourceByID resolves a resource through the canonical +// presentation identity map used by list and detail handlers. +func PresentationResourceByID(registry *unified.ResourceRegistry, resourceID string) (*unified.Resource, bool) { + return presentationResourceByID(registry, resourceID) +} + func presentationResourceByReference(registry *unified.ResourceRegistry, ref string) (*unified.Resource, string, bool) { ref = unified.CanonicalResourceID(ref) if ref == "" || registry == nil { @@ -424,6 +402,12 @@ func presentationResourceByReference(registry *unified.ResourceRegistry, ref str return rawResource, resolvedID, true } +// PresentationResourceByReference resolves canonical IDs and accepted aliases +// through the same presentation identity map as the resource API. +func PresentationResourceByReference(registry *unified.ResourceRegistry, ref string) (*unified.Resource, string, bool) { + return presentationResourceByReference(registry, ref) +} + type resourceFacetCountsResponse = unified.ResourceFacetCounts type resourceFacetBundleResponse struct { @@ -472,7 +456,7 @@ func parseResourceTimelineQuery(r *http.Request, defaultLimit int, includeRelate } // HandleResourceRoutes dispatches nested resource routes. -func (h *ResourceHandlers) HandleResourceRoutes(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleResourceRoutes(w http.ResponseWriter, r *http.Request) { if strings.TrimSuffix(r.URL.Path, "/") == "/api/resources/timeline" { h.HandleListResourceTimeline(w, r) return @@ -509,21 +493,21 @@ func (h *ResourceHandlers) HandleResourceRoutes(w http.ResponseWriter, r *http.R } // HandleGetResourceFacets handles GET /api/resources/{id}/facets. -func (h *ResourceHandlers) HandleGetResourceFacets(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleGetResourceFacets(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) registry, err := h.buildRegistry(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -553,27 +537,27 @@ func (h *ResourceHandlers) HandleGetResourceFacets(w http.ResponseWriter, r *htt recentChanges, err := store.GetRecentChangesFiltered(resourceID, since, limit, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } changeCount, err := store.CountRecentChangesFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } changeKindCounts, err := store.CountRecentChangesByKindFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } sourceTypeCounts, err := store.CountRecentChangesBySourceTypeFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } sourceAdapterCounts, err := store.CountRecentChangesBySourceAdapterFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -593,16 +577,16 @@ func (h *ResourceHandlers) HandleGetResourceFacets(w http.ResponseWriter, r *htt } // HandleGetChildren handles GET /api/resources/{id}/children. -func (h *ResourceHandlers) HandleGetChildren(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleGetChildren(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) registry, err := h.buildRegistry(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -628,16 +612,16 @@ func (h *ResourceHandlers) HandleGetChildren(w http.ResponseWriter, r *http.Requ } // HandleGetMetrics handles GET /api/resources/{id}/metrics. -func (h *ResourceHandlers) HandleGetMetrics(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleGetMetrics(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) registry, err := h.buildRegistry(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -661,16 +645,16 @@ func (h *ResourceHandlers) HandleGetMetrics(w http.ResponseWriter, r *http.Reque } // HandleListResourceTimeline handles GET /api/resources/timeline. -func (h *ResourceHandlers) HandleListResourceTimeline(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleListResourceTimeline(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -686,12 +670,12 @@ func (h *ResourceHandlers) HandleListResourceTimeline(w http.ResponseWriter, r * changes, err := store.GetRecentChangesFiltered(resourceID, since, limit, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } changeCount, err := store.CountRecentChangesFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -704,16 +688,16 @@ func (h *ResourceHandlers) HandleListResourceTimeline(w http.ResponseWriter, r * } // HandleGetResourceTimeline handles GET /api/resources/{id}/timeline. -func (h *ResourceHandlers) HandleGetResourceTimeline(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleGetResourceTimeline(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -737,12 +721,12 @@ func (h *ResourceHandlers) HandleGetResourceTimeline(w http.ResponseWriter, r *h changes, err := store.GetRecentChangesFiltered(resourceID, since, limit, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } changeCount, err := store.CountRecentChangesFiltered(resourceID, since, filters) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -755,16 +739,16 @@ func (h *ResourceHandlers) HandleGetResourceTimeline(w http.ResponseWriter, r *h } // HandleStats handles GET /api/resources/stats. -func (h *ResourceHandlers) HandleStats(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleStats(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) allResources, _, err := h.sharedPresentationResources(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -777,16 +761,16 @@ func (h *ResourceHandlers) HandleStats(w http.ResponseWriter, r *http.Request) { } // HandleLink handles POST /api/resources/{id}/link. -func (h *ResourceHandlers) HandleLink(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleLink(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -823,7 +807,7 @@ func (h *ResourceHandlers) HandleLink(w http.ResponseWriter, r *http.Request) { } if err := store.AddLink(link); err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } h.invalidateCache(orgID) @@ -836,16 +820,16 @@ func (h *ResourceHandlers) HandleLink(w http.ResponseWriter, r *http.Request) { } // HandleUnlink handles POST /api/resources/{id}/unlink. -func (h *ResourceHandlers) HandleUnlink(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleUnlink(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -881,7 +865,7 @@ func (h *ResourceHandlers) HandleUnlink(w http.ResponseWriter, r *http.Request) } if err := store.AddExclusion(exclusion); err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } h.invalidateCache(orgID) @@ -894,16 +878,16 @@ func (h *ResourceHandlers) HandleUnlink(w http.ResponseWriter, r *http.Request) } // HandleReportMerge handles POST /api/resources/{id}/report-merge. -func (h *ResourceHandlers) HandleReportMerge(w http.ResponseWriter, r *http.Request) { +func (h *QueryService) HandleReportMerge(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) return } - orgID := GetOrgID(r.Context()) + orgID := apicontext.OrgID(r.Context()) store, err := h.getStore(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -927,7 +911,7 @@ func (h *ResourceHandlers) HandleReportMerge(w http.ResponseWriter, r *http.Requ registry, err := h.buildRegistry(orgID) if err != nil { - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } @@ -988,7 +972,7 @@ func (h *ResourceHandlers) HandleReportMerge(w http.ResponseWriter, r *http.Requ Str("resourceID", path). Str("candidateID", target.CandidateID). Msg("Failed to add resource merge exclusion") - http.Error(w, sanitizeErrorForClient(err, "Internal server error"), http.StatusInternalServerError) + http.Error(w, sanitizeError(err, "Internal server error"), http.StatusInternalServerError) return } exclusionsAdded += 1 @@ -1017,7 +1001,7 @@ func (h *ResourceHandlers) HandleReportMerge(w http.ResponseWriter, r *http.Requ } // buildRegistry constructs a registry for the current tenant. -func (h *ResourceHandlers) buildRegistry(orgID string) (*unified.ResourceRegistry, error) { +func (h *QueryService) buildRegistry(orgID string) (*unified.ResourceRegistry, error) { store, err := h.getStore(orgID) if err != nil { return nil, err @@ -1084,12 +1068,22 @@ func (h *ResourceHandlers) buildRegistry(orgID string) (*unified.ResourceRegistr } h.cacheMu.Lock() + if h.registryCache == nil { + h.registryCache = make(map[string]registryCacheEntry) + } h.registryCache[key] = registryCacheEntry{registry: registry, lastUpdate: seed.lastUpdate} h.cacheMu.Unlock() return registry, nil } +// BuildRegistry returns the canonical tenant resource registry. It is exposed +// for API composition services that coordinate resource mutations and action +// lifecycle decisions against the same durable store. +func (h *QueryService) BuildRegistry(orgID string) (*unified.ResourceRegistry, error) { + return h.buildRegistry(orgID) +} + func sortedSupplementalSources( providers map[unified.DataSource]SupplementalRecordsProvider, ) []unified.DataSource { @@ -1109,7 +1103,7 @@ func sortedSupplementalSources( return sources } -func (h *ResourceHandlers) registrySeed(orgID string) (registrySeed, error) { +func (h *QueryService) registrySeed(orgID string) (registrySeed, error) { seed := registrySeed{} if orgID != "" && orgID != "default" { @@ -1241,6 +1235,11 @@ func unifiedSeedSources(resources []unified.Resource) map[unified.DataSource]str return sources } +// UnifiedSeedSources reports source-owned records present in a canonical seed. +func UnifiedSeedSources(resources []unified.Resource) map[unified.DataSource]struct{} { + return unifiedSeedSources(resources) +} + func unifiedSeedIncludesSource(seedSources map[unified.DataSource]struct{}, source unified.DataSource) bool { if len(seedSources) == 0 { return false @@ -1275,9 +1274,18 @@ func normalizeDataSourceAlias(source unified.DataSource) unified.DataSource { } } -func (h *ResourceHandlers) getStore(orgID string) (unified.ResourceStore, error) { +// NormalizeDataSourceAlias canonicalizes legacy source identifiers accepted at +// the router and query boundaries. +func NormalizeDataSourceAlias(source unified.DataSource) unified.DataSource { + return normalizeDataSourceAlias(source) +} + +func (h *QueryService) getStore(orgID string) (unified.ResourceStore, error) { h.storeMu.Lock() defer h.storeMu.Unlock() + if h.stores == nil { + h.stores = make(map[string]unified.ResourceStore) + } key := cacheKey(orgID) if store, ok := h.stores[key]; ok { return store, nil @@ -1294,13 +1302,25 @@ func (h *ResourceHandlers) getStore(orgID string) (unified.ResourceStore, error) return store, nil } +// InvalidateCache clears a tenant's resource registry and presentation cache +// after an external mutation coordinated by another API domain. +func (h *QueryService) InvalidateCache(orgID string) { + h.invalidateCache(orgID) +} + +// Store returns the tenant-scoped durable resource store shared by resource +// queries and API mutation services. +func (h *QueryService) Store(orgID string) (unified.ResourceStore, error) { + return h.getStore(orgID) +} + // CloseTenantStore closes and evicts one org's cached resource store. // // getStore opens a SQLite handle per org and caches it for the process // lifetime. Without this, offboarding a tenant left its handle open: the // -wal and -shm files stay on disk, the file descriptors are never returned, // and the tenant directory cannot be fully removed. -func (h *ResourceHandlers) CloseTenantStore(orgID string) error { +func (h *QueryService) CloseTenantStore(orgID string) error { if h == nil { return nil } @@ -1323,7 +1343,7 @@ func (h *ResourceHandlers) CloseTenantStore(orgID string) error { // CloseStores closes and evicts every cached resource store. Used on shutdown // so SQLite handles are released rather than left to process exit. -func (h *ResourceHandlers) CloseStores() error { +func (h *QueryService) CloseStores() error { if h == nil { return nil } @@ -1349,7 +1369,7 @@ func (h *ResourceHandlers) CloseStores() error { return errors.Join(errs...) } -func (h *ResourceHandlers) invalidateCache(orgID string) { +func (h *QueryService) invalidateCache(orgID string) { key := cacheKey(orgID) h.cacheMu.Lock() delete(h.registryCache, key) @@ -1533,7 +1553,7 @@ func flatCopyResources(shared []unified.Resource) []unified.Resource { // sharedPresentationResources returns the org's cached presentation-shape // resource list for the current registry generation, building it at most // once per generation instead of deep-cloning the registry on every request. -func (h *ResourceHandlers) sharedPresentationResources(orgID string) ([]unified.Resource, *unified.ResourceRegistry, error) { +func (h *QueryService) sharedPresentationResources(orgID string) ([]unified.Resource, *unified.ResourceRegistry, error) { registry, err := h.buildRegistry(orgID) if err != nil { return nil, nil, err @@ -1561,7 +1581,7 @@ func (h *ResourceHandlers) sharedPresentationResources(orgID string) ([]unified. // sharedRawResources is sharedPresentationResources for the raw (uncoalesced) // registry list. -func (h *ResourceHandlers) sharedRawResources(orgID string) ([]unified.Resource, *unified.ResourceRegistry, error) { +func (h *QueryService) sharedRawResources(orgID string) ([]unified.Resource, *unified.ResourceRegistry, error) { registry, err := h.buildRegistry(orgID) if err != nil { return nil, nil, err @@ -2114,6 +2134,11 @@ func parseResourceTypes(raw string) map[unified.ResourceType]struct{} { return result } +// ParseResourceTypes parses the public resource type filter grammar. +func ParseResourceTypes(raw string) map[unified.ResourceType]struct{} { + return parseResourceTypes(raw) +} + func unsupportedResourceTypeFilterTokens(raw string) []string { if strings.TrimSpace(raw) == "" { return nil @@ -2127,6 +2152,11 @@ func unsupportedResourceTypeFilterTokens(raw string) []string { return unsupported } +// UnsupportedResourceTypeFilterTokens returns rejected public filter tokens. +func UnsupportedResourceTypeFilterTokens(raw string) []string { + return unsupportedResourceTypeFilterTokens(raw) +} + func isSupportedResourceTypeFilterToken(token string) bool { return len(resourceTypeFilterAdapter(token)) > 0 } @@ -2308,7 +2338,7 @@ func attachDiscoveryTarget(resource *unified.Resource) { } } -func (h *ResourceHandlers) attachDiscoveryReadinesses(resources []unified.Resource, now time.Time) { +func (h *QueryService) attachDiscoveryReadinesses(resources []unified.Resource, now time.Time) { if h == nil || h.discoveryReadiness == nil { return } @@ -2317,7 +2347,7 @@ func (h *ResourceHandlers) attachDiscoveryReadinesses(resources []unified.Resour } } -func (h *ResourceHandlers) attachDiscoveryReadiness(resource *unified.Resource, now time.Time) { +func (h *QueryService) attachDiscoveryReadiness(resource *unified.Resource, now time.Time) { if h == nil || h.discoveryReadiness == nil || resource == nil { return } @@ -2328,6 +2358,12 @@ func (h *ResourceHandlers) attachDiscoveryReadiness(resource *unified.Resource, resource.DiscoveryReadiness = &readiness } +// AttachDiscoveryReadiness projects current discovery state onto a resource +// used by another API domain. +func (h *QueryService) AttachDiscoveryReadiness(resource *unified.Resource, now time.Time) { + h.attachDiscoveryReadiness(resource, now) +} + func attachMetricsTargets(resources []unified.Resource, registry *unified.ResourceRegistry) { for i := range resources { attachMetricsTarget(&resources[i], registry) @@ -2347,6 +2383,16 @@ func resourceContractType(r unified.Resource) unified.ResourceType { return unified.ContractResourceType(r) } +// ContractType maps an internal resource shape onto the public API contract. +func ContractType(resource unified.Resource) unified.ResourceType { + return resourceContractType(resource) +} + +// AttachDiscoveryTarget projects the canonical discovery target metadata. +func AttachDiscoveryTarget(resource *unified.Resource) { + attachDiscoveryTarget(resource) +} + // applyResourceContractTypes rewrites Type fields on the response slice so the // REST API exposes the canonical resource-type contract. func applyResourceContractTypes(resources []unified.Resource) { @@ -2440,6 +2486,11 @@ func cephDiscoveryTarget(resource unified.Resource) *unified.DiscoveryTarget { } } +// CephDiscoveryTarget constructs the canonical discovery target for a Ceph resource. +func CephDiscoveryTarget(resource unified.Resource) *unified.DiscoveryTarget { + return cephDiscoveryTarget(resource) +} + func hostDiscoveryTarget(resource unified.Resource) *unified.DiscoveryTarget { linkedAgentID := "" agentBacked := hasSource(resource.Sources, unified.SourceAgent) || resource.Agent != nil @@ -2739,3 +2790,10 @@ func parseIntDefault(raw string, def int) int { func getUserID(r *http.Request) string { return auth.GetUser(r.Context()) } + +func sanitizeError(err error, genericMessage string) string { + if err != nil { + log.Error().Err(err).Msg(genericMessage) + } + return genericMessage +} diff --git a/internal/api/resources_frontend_types_test.go b/internal/api/resourceapi/resources_frontend_types_test.go similarity index 99% rename from internal/api/resources_frontend_types_test.go rename to internal/api/resourceapi/resources_frontend_types_test.go index 2d809b218..8bc365dd1 100644 --- a/internal/api/resources_frontend_types_test.go +++ b/internal/api/resourceapi/resources_frontend_types_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "encoding/json" @@ -371,7 +371,7 @@ func TestResourceListUsesCanonicalContractTypes(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) // Test 1: legacy ?type=node is accepted at the request boundary but resolves @@ -578,7 +578,7 @@ func TestResourceGetUsesCanonicalContractType(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) // First get the resource ID from a list query. diff --git a/internal/api/resources_pool_health_contract_test.go b/internal/api/resourceapi/resources_pool_health_contract_test.go similarity index 98% rename from internal/api/resources_pool_health_contract_test.go rename to internal/api/resourceapi/resources_pool_health_contract_test.go index cdfa76362..a0ab24372 100644 --- a/internal/api/resources_pool_health_contract_test.go +++ b/internal/api/resourceapi/resources_pool_health_contract_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "encoding/json" @@ -16,7 +16,7 @@ import ( func TestResourceListPreservesCanonicalPoolHealthEvidence(t *testing.T) { observedAt := time.Date(2026, 7, 24, 10, 0, 0, 0, time.UTC) cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: observedAt}, resources: []unified.Resource{{ diff --git a/internal/api/resources_shared_cache_test.go b/internal/api/resourceapi/resources_shared_cache_test.go similarity index 94% rename from internal/api/resources_shared_cache_test.go rename to internal/api/resourceapi/resources_shared_cache_test.go index e08149750..8ca7ddb56 100644 --- a/internal/api/resources_shared_cache_test.go +++ b/internal/api/resourceapi/resources_shared_cache_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "context" @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -35,7 +36,7 @@ func TestPruneResourceForListResponseDoesNotMutateSharedPMG(t *testing.T) { func TestSharedResourceListsCachedPerGenerationAndImmuneToRequestDecoration(t *testing.T) { now := time.Date(2026, 8, 5, 12, 0, 0, 0, time.UTC) - h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unifiedresources.Resource{ @@ -81,7 +82,7 @@ func TestSharedResourceListsCachedPerGenerationAndImmuneToRequestDecoration(t *t // A full list request prunes PMG detail in its response; the shared cache // must keep the untouched data for later consumers. req := httptest.NewRequest(http.MethodGet, "/api/resources", nil) - req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, orgID)) + req = req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, orgID)) rec := httptest.NewRecorder() h.HandleListResources(rec, req) if rec.Code != http.StatusOK { diff --git a/internal/api/resources_store_close_test.go b/internal/api/resourceapi/resources_store_close_test.go similarity index 67% rename from internal/api/resources_store_close_test.go rename to internal/api/resourceapi/resources_store_close_test.go index f899c353b..683ba7fe2 100644 --- a/internal/api/resources_store_close_test.go +++ b/internal/api/resourceapi/resources_store_close_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "os" @@ -27,7 +27,7 @@ func sqliteSidecarsPresent(t *testing.T, dataDir, orgID string) bool { // descriptors, and its -wal/-shm files outlive the tenant. func TestResourceHandlers_CloseTenantStoreReleasesTheHandle(t *testing.T) { dataDir := t.TempDir() - handlers := NewResourceHandlers(&config.Config{DataPath: dataDir}) + handlers := NewQueryService(&config.Config{DataPath: dataDir}) t.Cleanup(func() { _ = handlers.CloseStores() }) if _, err := handlers.getStore("client-x"); err != nil { @@ -54,7 +54,7 @@ func TestResourceHandlers_CloseTenantStoreReleasesTheHandle(t *testing.T) { func TestResourceHandlers_CloseStoresReleasesEveryTenant(t *testing.T) { dataDir := t.TempDir() - handlers := NewResourceHandlers(&config.Config{DataPath: dataDir}) + handlers := NewQueryService(&config.Config{DataPath: dataDir}) for _, orgID := range []string{"client-a", "client-b"} { if _, err := handlers.getStore(orgID); err != nil { @@ -77,29 +77,3 @@ func TestResourceHandlers_CloseStoresReleasesEveryTenant(t *testing.T) { t.Fatalf("%d store(s) still cached after CloseStores", remaining) } } - -// Closing must be safe to repeat and safe on a nil handler: the shutdown path -// runs before every dependency is guaranteed to be wired. -func TestResourceHandlers_CloseIsIdempotentAndNilSafe(t *testing.T) { - var nilHandlers *ResourceHandlers - if err := nilHandlers.CloseStores(); err != nil { - t.Fatalf("nil handler CloseStores: %v", err) - } - if err := nilHandlers.CloseTenantStore("client-a"); err != nil { - t.Fatalf("nil handler CloseTenantStore: %v", err) - } - - handlers := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) - if _, err := handlers.getStore("client-a"); err != nil { - t.Fatalf("getStore: %v", err) - } - if err := handlers.CloseStores(); err != nil { - t.Fatalf("first CloseStores: %v", err) - } - if err := handlers.CloseStores(); err != nil { - t.Fatalf("second CloseStores must be a no-op: %v", err) - } - if err := handlers.CloseTenantStore("client-a"); err != nil { - t.Fatalf("CloseTenantStore on an evicted org must be a no-op: %v", err) - } -} diff --git a/internal/api/resources_store_isolation_test.go b/internal/api/resourceapi/resources_store_isolation_test.go similarity index 93% rename from internal/api/resources_store_isolation_test.go rename to internal/api/resourceapi/resources_store_isolation_test.go index 1d8a6c133..4615e8b77 100644 --- a/internal/api/resources_store_isolation_test.go +++ b/internal/api/resourceapi/resources_store_isolation_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "testing" @@ -8,7 +8,7 @@ import ( ) func TestResourceHandlers_GetStore_DotAndUnderscoreOrgIDsRemainIsolated(t *testing.T) { - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) dotStore, err := h.getStore("org.a") if err != nil { diff --git a/internal/api/resources_tenant_security_test.go b/internal/api/resourceapi/resources_tenant_security_test.go similarity index 85% rename from internal/api/resources_tenant_security_test.go rename to internal/api/resourceapi/resources_tenant_security_test.go index 41385519c..d19d70c38 100644 --- a/internal/api/resources_tenant_security_test.go +++ b/internal/api/resourceapi/resources_tenant_security_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "context" @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" @@ -19,7 +20,9 @@ type tenantResourceStateProvider struct { } func (p tenantResourceStateProvider) UnifiedReadStateForTenant(orgID string) unified.ReadState { - return SnapshotReadState(p.GetStateForTenant(orgID)) + registry := unified.NewRegistry(nil) + registry.IngestSnapshot(p.GetStateForTenant(orgID)) + return unified.NewMonitorAdapter(registry) } func (p tenantResourceStateProvider) GetStateForTenant(orgID string) models.StateSnapshot { @@ -53,13 +56,13 @@ func (p tenantResourceStateProvider) UnifiedResourceSnapshotForTenant(orgID stri func TestResourceHandlers_NonDefaultOrgRequiresTenantStateProvider(t *testing.T) { now := time.Now().UTC() - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{ Hosts: []models.Host{{ID: "host-default", Hostname: "default", Status: "online", LastSeen: now}}, }}) req := httptest.NewRequest(http.MethodGet, "/api/resources", nil) - req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "acme")) + req = req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, "acme")) rec := httptest.NewRecorder() h.HandleListResources(rec, req) @@ -74,7 +77,7 @@ func TestResourceHandlers_NonDefaultOrgRequiresTenantStateProvider(t *testing.T) func TestResourceHandlers_NonDefaultOrgUsesTenantStateProvider(t *testing.T) { now := time.Now().UTC() - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{ Hosts: []models.Host{{ID: "host-default", Hostname: "default", Status: "online", LastSeen: now}}, }}) @@ -86,7 +89,7 @@ func TestResourceHandlers_NonDefaultOrgUsesTenantStateProvider(t *testing.T) { }}) req := httptest.NewRequest(http.MethodGet, "/api/resources?type=agent", nil) - req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "acme")) + req = req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, "acme")) rec := httptest.NewRecorder() h.HandleListResources(rec, req) @@ -109,13 +112,13 @@ func TestResourceHandlers_NonDefaultOrgUsesTenantStateProvider(t *testing.T) { func TestResourceHandlers_NonDefaultOrgUsesTenantUnifiedSeedProvider(t *testing.T) { now := time.Now().UTC() - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetTenantStateProvider(tenantResourceStateProvider{snapshots: map[string]models.StateSnapshot{ "acme": {LastUpdate: now}, }}) req := httptest.NewRequest(http.MethodGet, "/api/resources?type=agent", nil) - req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "acme")) + req = req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, "acme")) rec := httptest.NewRecorder() h.HandleListResources(rec, req) @@ -138,7 +141,7 @@ func TestResourceHandlers_NonDefaultOrgUsesTenantUnifiedSeedProvider(t *testing. func TestResourceHandlers_NonDefaultOrgDoesNotFallbackToRawSnapshotSeeding(t *testing.T) { now := time.Now().UTC() - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{ Hosts: []models.Host{{ID: "host-default", Hostname: "default", Status: "online", LastSeen: now}}, }}) @@ -150,7 +153,7 @@ func TestResourceHandlers_NonDefaultOrgDoesNotFallbackToRawSnapshotSeeding(t *te }}) req := httptest.NewRequest(http.MethodGet, "/api/resources?type=agent", nil) - req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "acme")) + req = req.WithContext(context.WithValue(req.Context(), apicontext.OrgIDContextKey, "acme")) rec := httptest.NewRecorder() h.HandleListResources(rec, req) diff --git a/internal/api/resources_test.go b/internal/api/resourceapi/resources_test.go similarity index 96% rename from internal/api/resources_test.go rename to internal/api/resourceapi/resources_test.go index 09d8dae4b..8d04a8b71 100644 --- a/internal/api/resources_test.go +++ b/internal/api/resourceapi/resources_test.go @@ -1,4 +1,4 @@ -package api +package resourceapi import ( "bytes" @@ -17,7 +17,6 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/storagehealth" "github.com/rcourtman/pulse-go-rewrite/internal/truenas" unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" - authpkg "github.com/rcourtman/pulse-go-rewrite/pkg/auth" ) type resourceStateProvider struct { @@ -64,6 +63,47 @@ type mockSupplementalRecordsProvider struct { ownedSources []unified.DataSource } +type ownedSupplementalProvider struct { + source unified.DataSource +} + +func (p ownedSupplementalProvider) GetCurrentRecords() []unified.IngestRecord { return nil } + +func (p ownedSupplementalProvider) SnapshotOwnedSources() []unified.DataSource { + return []unified.DataSource{p.source} +} + +type staticResourceDiscoveryReadinessProvider struct { + byResource map[string]unified.ResourceDiscoveryReadiness +} + +func (p staticResourceDiscoveryReadinessProvider) DiscoveryReadinessForResource(resource unified.Resource, now time.Time) unified.ResourceDiscoveryReadiness { + if readiness, ok := p.byResource[unified.CanonicalResourceID(resource.ID)]; ok { + if readiness.GeneratedAt.IsZero() { + readiness.GeneratedAt = now + } + if resource.DiscoveryTarget != nil { + if readiness.ResourceType == "" { + readiness.ResourceType = resource.DiscoveryTarget.ResourceType + } + if readiness.TargetID == "" { + readiness.TargetID = resource.DiscoveryTarget.AgentID + } + if readiness.ResourceID == "" { + readiness.ResourceID = resource.DiscoveryTarget.ResourceID + } + } + return readiness + } + readiness := unified.ResourceDiscoveryReadiness{State: unified.ResourceDiscoveryReadinessMissing, Source: "service-discovery", GeneratedAt: now} + if resource.DiscoveryTarget != nil { + readiness.ResourceType = resource.DiscoveryTarget.ResourceType + readiness.TargetID = resource.DiscoveryTarget.AgentID + readiness.ResourceID = resource.DiscoveryTarget.ResourceID + } + return readiness +} + func TestResourceListRepairsPreFixAvailabilitySnapshotFromConfiguredTargets(t *testing.T) { now := time.Date(2026, 7, 23, 22, 0, 0, 0, time.UTC) hostID := "agent-core2026" @@ -126,7 +166,7 @@ func TestResourceListRepairsPreFixAvailabilitySnapshotFromConfiguredTargets(t *t } } - h := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + h := NewQueryService(&config.Config{DataPath: t.TempDir()}) h.SetStateProvider(&mutableResourceUnifiedSeedProvider{ resources: seed, freshness: now, @@ -200,7 +240,7 @@ func (m mockSupplementalRecordsProvider) SupplementalRecords(*monitoring.Monitor func TestResourceListRejectsLegacyHostTypeFilter(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{}}) rec := httptest.NewRecorder() @@ -244,7 +284,7 @@ func TestResourceListMergesLinkedHost(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -293,7 +333,7 @@ func TestResourceListMergesLinkedHost(t *testing.T) { func TestResourceListUsesUnifiedSeedProvider(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -381,7 +421,7 @@ func TestResourceListAttachesDiscoveryReadiness(t *testing.T) { now := time.Date(2026, 6, 4, 11, 0, 0, 0, time.UTC) observedAt := now.Add(-5 * time.Minute) cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -441,7 +481,7 @@ func TestResourceListAttachesDiscoveryReadiness(t *testing.T) { func TestResourceListUsesDeterministicNameTieBreakers(t *testing.T) { now := time.Date(2026, 4, 11, 0, 0, 0, 0, time.UTC) cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -503,7 +543,7 @@ func TestResourceListInvalidatesUnifiedSeedCacheOnFreshnessChange(t *testing.T) } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(provider) firstRec := httptest.NewRecorder() @@ -582,7 +622,7 @@ func TestResourceListMergesOneSidedLinkedHostWhenHostnameCorroborates(t *testing } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -645,7 +685,7 @@ func TestResourceListDoesNotMergeOneSidedLinkedHostWithoutHostnameCorroboration( } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -713,7 +753,7 @@ func TestResourceListCollapsesClusterAndStandaloneNodeViewsByEndpoint(t *testing } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -744,7 +784,7 @@ func TestResourceListCollapsesClusterAndStandaloneNodeViewsByEndpoint(t *testing func TestResourceListDerivesProxmoxWorkloadParentFromUnifiedSeed(t *testing.T) { now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC) cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -821,107 +861,6 @@ func TestResourceListDerivesProxmoxWorkloadParentFromUnifiedSeed(t *testing.T) { } } -func TestStateEndpointDerivesProxmoxWorkloadParentFromSupplementalRecords(t *testing.T) { - now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC) - hashedPassword, err := authpkg.HashPassword("password") - if err != nil { - t.Fatalf("hash password: %v", err) - } - dataPath := t.TempDir() - InitSessionStore(dataPath) - InitCSRFStore(dataPath) - cfg := &config.Config{ - DataPath: dataPath, - AuthUser: "admin", - AuthPass: hashedPassword, - } - monitor, err := monitoring.New(cfg) - if err != nil { - t.Fatalf("new monitor: %v", err) - } - t.Cleanup(func() { monitor.Stop() }) - - monitor.SetResourceStore(unified.NewMonitorAdapter(nil)) - monitor.SetSupplementalRecordsProvider(unified.SourceProxmox, mockSupplementalRecordsProvider{ - records: []unified.IngestRecord{ - { - SourceID: "homelab-delly", - Resource: unified.Resource{ - Type: unified.ResourceTypeAgent, - Name: "delly", - Status: unified.StatusOnline, - LastSeen: now, - Proxmox: &unified.ProxmoxData{ - SourceID: "homelab-delly", - NodeName: "delly", - ClusterName: "homelab", - Instance: "delly", - }, - }, - Identity: unified.ResourceIdentity{ - MachineID: "machine-delly", - Hostnames: []string{"delly"}, - }, - }, - { - SourceID: "delly:delly:104", - Resource: unified.Resource{ - Type: unified.ResourceTypeSystemContainer, - Name: "cloudflared", - Status: unified.StatusOnline, - LastSeen: now, - Proxmox: &unified.ProxmoxData{ - SourceID: "delly:delly:104", - NodeName: "delly", - ClusterName: "homelab", - Instance: "delly", - VMID: 104, - }, - }, - Identity: unified.ResourceIdentity{Hostnames: []string{"cloudflared"}}, - }, - }, - }) - - router := &Router{config: cfg, monitor: monitor} - rec := httptest.NewRecorder() - req := httptest.NewRequest(http.MethodGet, "/api/state", nil) - req.SetBasicAuth("admin", "password") - router.handleState(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("/api/state status = %d, body=%s", rec.Code, rec.Body.String()) - } - - var state models.StateFrontend - if err := json.NewDecoder(rec.Body).Decode(&state); err != nil { - t.Fatalf("decode /api/state: %v", err) - } - - dellyCount := 0 - dellyID := "" - cloudflaredParentID := "" - for _, resource := range state.Resources { - switch { - case resource.Type == string(unified.ResourceTypeAgent) && resource.Name == "delly": - dellyCount++ - dellyID = resource.ID - case resource.Type == string(unified.ResourceTypeSystemContainer) && resource.Name == "cloudflared": - cloudflaredParentID = resource.ParentID - } - } - - if dellyCount != 1 { - t.Fatalf("expected exactly one delly resource in /api/state, got %d: %#v", dellyCount, state.Resources) - } - if dellyID == "" { - t.Fatalf("expected delly resource id in /api/state: %#v", state.Resources) - } - if cloudflaredParentID != dellyID { - t.Fatalf("expected cloudflared parent %q, got %q: %#v", dellyID, cloudflaredParentID, state.Resources) - } -} - func TestResourceListCollapsesAsymmetricLinkedClusterNodeViews(t *testing.T) { now := time.Now().UTC() snapshot := models.StateSnapshot{ @@ -966,7 +905,7 @@ func TestResourceListCollapsesAsymmetricLinkedClusterNodeViews(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -1040,7 +979,7 @@ func TestResourceListCollapsesHostLinkedClusterNodeViews(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -1110,7 +1049,7 @@ func TestResourceListCollapsesHostLinkedNodeViewsAcrossEndpointForms(t *testing. } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -1173,7 +1112,7 @@ func TestResourceListIncludesHostSMARTPhysicalDisks(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -1224,7 +1163,7 @@ func TestResourceListUsesCanonicalMetricIDForProxmoxPhysicalDisks(t *testing.T) } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -1261,7 +1200,7 @@ func TestResourceGetResource(t *testing.T) { snapshot := models.StateSnapshot{Hosts: []models.Host{host}} cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) listRec := httptest.NewRecorder() @@ -1344,7 +1283,7 @@ func TestResourceGetFacetsAndTimeline(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource, node}, @@ -1739,7 +1678,7 @@ func TestResourceLinkMergesResources(t *testing.T) { snapshot := models.StateSnapshot{Hosts: []models.Host{host}, DockerHosts: []models.DockerHost{dockerHost}} cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) listRec := httptest.NewRecorder() @@ -1863,7 +1802,7 @@ func TestResourceLinkFromAgentKeepsHypervisorGuestAuthorityInAPIState(t *testing for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: tc.snapshot}) list := func() ResourcesResponse { @@ -1965,7 +1904,7 @@ func TestResourceReportMergeCreatesExclusions(t *testing.T) { snapshot := models.StateSnapshot{Hosts: []models.Host{host}, DockerHosts: []models.DockerHost{dockerHost}} cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) listRec := httptest.NewRecorder() @@ -2044,7 +1983,7 @@ func TestResourceListIncludesKubernetesPods(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2122,7 +2061,7 @@ func TestResourceListProjectsKubernetesClusterAgentVersionOntoNode(t *testing.T) } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2185,7 +2124,7 @@ func TestResourceListFiltersCanonicalKubernetesNamespace(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2411,7 +2350,7 @@ func TestK8sNamespacesEndpointAggregatesPodsAndDeployments(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2626,7 +2565,7 @@ func TestResourceListRejectsLegacyKubernetesTypeAlias(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2680,7 +2619,7 @@ func TestResourceListReturnsCanonicalKubernetesMetricsTargets(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2784,7 +2723,7 @@ func TestResourceListIncludesDockerSwarmServicesAndFiltersByCluster(t *testing.T } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) // Unfiltered-by-cluster: expect the service to show up exactly once. @@ -2879,7 +2818,7 @@ func TestResourceListIncludesPBSAndPMG(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -2982,7 +2921,7 @@ func TestResourceListIncludesStorageMetadata(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3086,7 +3025,7 @@ func TestResourceListIncludesStorageConsumerImpact(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3206,7 +3145,7 @@ func TestResourceListIncludesPBSStorageConsumerImpact(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3283,7 +3222,7 @@ func TestResourceListIncludesPBSPrimaryIncidentRollup(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3419,7 +3358,7 @@ func TestPBSResourceListIncludesProtectedWorkloadRollup(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3500,7 +3439,7 @@ func TestResourceListIncludesHostUnraidStorage(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3556,7 +3495,7 @@ func TestResourceListReturnsCanonicalStorageMetricsTargets(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) rec := httptest.NewRecorder() @@ -3583,7 +3522,7 @@ func TestResourceListReturnsCanonicalStorageMetricsTargets(t *testing.T) { t.Run("truenas pool", func(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: now}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: []unified.IngestRecord{ @@ -3636,7 +3575,7 @@ func TestResourceListReturnsCanonicalStorageMetricsTargets(t *testing.T) { func TestResourceStorageSummaryRollsUpIncidents(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -3772,7 +3711,7 @@ func TestResourceListIncludesTrueNASPhysicalDiskTemperature(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: now}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: truenas.NewProvider(truenas.DefaultFixtures()).Records(), @@ -3851,7 +3790,7 @@ func TestResourceListIncludesTrueNASAppsAsAppContainers(t *testing.T) { fixtures := truenas.DefaultFixtures() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: time.Now().UTC()}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: truenas.NewProvider(fixtures).Records(), @@ -3917,7 +3856,7 @@ func TestResourceListIncludesTrueNASVMsAsCanonicalWorkloads(t *testing.T) { fixtures := truenas.DefaultFixtures() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: time.Now().UTC()}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: truenas.NewProvider(fixtures).Records(), @@ -3976,7 +3915,7 @@ func TestResourceListIncludesTrueNASSystemAsCanonicalHost(t *testing.T) { }) cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: time.Now().UTC()}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: truenas.NewProvider(truenas.DefaultFixtures()).Records(), @@ -4026,7 +3965,7 @@ func TestResourceListIncludesTrueNASSystemAsCanonicalHost(t *testing.T) { func TestResourceStorageIncidentsGroupsCanonicalSections(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -4209,7 +4148,7 @@ func TestResourceListIncludesTrueNASFromSupplementalProvider(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: models.StateSnapshot{LastUpdate: now}}) h.SetSupplementalRecordsProvider(unified.SourceTrueNAS, mockSupplementalRecordsProvider{ records: []unified.IngestRecord{ @@ -4259,7 +4198,7 @@ func TestResourceListUnifiedSeedSkipsSupplementalReingest(t *testing.T) { now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -4321,7 +4260,7 @@ func TestResourceListUnifiedSeedIngestsOwnedSupplementalWhenSourceMissing(t *tes now := time.Now().UTC() cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceUnifiedSeedProvider{ snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{ @@ -4398,7 +4337,7 @@ func TestResourceListSupplementalOwnerSuppressesSnapshotSource(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) h.SetSupplementalRecordsProvider(unified.SourceAgent, mockSupplementalRecordsProvider{ ownedSources: []unified.DataSource{unified.SourceAgent}, @@ -4455,7 +4394,7 @@ func TestResourceListWithoutSupplementalProvider(t *testing.T) { } cfg := &config.Config{DataPath: t.TempDir()} - h := NewResourceHandlers(cfg) + h := NewQueryService(cfg) h.SetStateProvider(resourceStateProvider{snapshot: snapshot}) truenasRec := httptest.NewRecorder() @@ -4493,7 +4432,7 @@ func TestResourceListWithoutSupplementalProvider(t *testing.T) { func TestSupplementalSnapshotOwnedSources_TrueNASProviders(t *testing.T) { sources := supplementalSnapshotOwnedSources(map[unified.DataSource]SupplementalRecordsProvider{ - unified.SourceTrueNAS: mockSupplementalRecordsAdapter{source: unified.SourceTrueNAS}, + unified.SourceTrueNAS: ownedSupplementalProvider{source: unified.SourceTrueNAS}, }, "default") if len(sources) != 1 || sources[0] != unified.SourceTrueNAS { diff --git a/internal/api/resources_compat.go b/internal/api/resources_compat.go new file mode 100644 index 000000000..f06cc2a3a --- /dev/null +++ b/internal/api/resources_compat.go @@ -0,0 +1,148 @@ +package api + +import ( + "net/http" + "strings" + + "github.com/rcourtman/pulse-go-rewrite/internal/actionlifecycle" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" + "github.com/rcourtman/pulse-go-rewrite/internal/config" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/auth" +) + +// ResourceHandlers preserves the established router and extension surface +// while composing the resource-query service with API-owned action mutations. +// QueryService owns resource registry construction, durable resource storage, +// filtering, projections, and read handlers. The fields below remain here +// because action lifecycle and operator-state mutations are separate domains. +type ResourceHandlers struct { + resourceapi.QueryService + + cfg *config.Config + tenantStateProvider TenantStateProvider + actionExecutor ActionExecutor + actionCompleted func(unified.ActionAuditRecord) + actionTransition func(orgID string, record unified.ActionAuditRecord) + policyAdmission *actionlifecycle.PolicyAdmissionCoordinator + actionEmergencyStop func(orgID string) (bool, error) + actionDecisionAuthorizer actionlifecycle.DecisionAuthorizer + actionExecutionAuthorizer actionlifecycle.ExecutionAuthorizer + actionRefreshPlanner actionlifecycle.RefreshPlanner + operatorStateChanged func(orgID, resourceID string) +} + +type ResourceDiscoveryReadinessProvider = resourceapi.ResourceDiscoveryReadinessProvider +type SupplementalRecordsProvider = resourceapi.SupplementalRecordsProvider +type TenantSupplementalRecordsProvider = resourceapi.TenantSupplementalRecordsProvider +type SupplementalSnapshotSourceOwner = resourceapi.SupplementalSnapshotSourceOwner +type TenantSupplementalSnapshotSourceOwner = resourceapi.TenantSupplementalSnapshotSourceOwner +type ResourcesResponse = resourceapi.ResourcesResponse +type ResourcesMeta = resourceapi.ResourcesMeta +type StorageSummaryResponse = resourceapi.StorageSummaryResponse +type StorageSummaryIncident = resourceapi.StorageSummaryIncident +type StorageIncidentsResponse = resourceapi.StorageIncidentsResponse +type StorageIncidentSection = resourceapi.StorageIncidentSection + +func EmptyResourcesResponse() ResourcesResponse { return resourceapi.EmptyResourcesResponse() } + +func EmptyStorageSummaryResponse() StorageSummaryResponse { + return resourceapi.EmptyStorageSummaryResponse() +} + +func EmptyStorageIncidentsResponse() StorageIncidentsResponse { + return resourceapi.EmptyStorageIncidentsResponse() +} + +// NewResourceHandlers builds the compatibility composition used by Router. +func NewResourceHandlers(cfg *config.Config) *ResourceHandlers { + return &ResourceHandlers{ + QueryService: *resourceapi.NewQueryService(cfg), + cfg: cfg, + policyAdmission: &actionlifecycle.PolicyAdmissionCoordinator{}, + } +} + +func (h *ResourceHandlers) ensureQueryService() *resourceapi.QueryService { + return &h.QueryService +} + +func (h *ResourceHandlers) SetStateProvider(provider SnapshotProvider) { + h.ensureQueryService().SetStateProvider(provider) +} + +func (h *ResourceHandlers) SetTenantStateProvider(provider TenantStateProvider) { + h.tenantStateProvider = provider + h.ensureQueryService().SetTenantStateProvider(provider) +} + +func (h *ResourceHandlers) SetActionExecutor(executor ActionExecutor) { + h.actionExecutor = executor + checker, _ := executor.(actionlifecycle.AvailabilityChecker) + h.ensureQueryService().SetActionAvailabilityChecker(checker) +} + +func (h *ResourceHandlers) SetActionEmergencyStopChecker(checker func(orgID string) (bool, error)) { + h.actionEmergencyStop = checker +} + +func (h *ResourceHandlers) SetActionAuthorizers(decision actionlifecycle.DecisionAuthorizer, execution actionlifecycle.ExecutionAuthorizer) { + h.actionDecisionAuthorizer = decision + h.actionExecutionAuthorizer = execution +} + +func (h *ResourceHandlers) SetActionRefreshPlanner(planner actionlifecycle.RefreshPlanner) { + h.actionRefreshPlanner = planner +} + +func (h *ResourceHandlers) SetActionCompletedPublisher(publisher func(unified.ActionAuditRecord)) { + h.actionCompleted = publisher +} + +func (h *ResourceHandlers) SetActionTransitionPublisher(publisher func(orgID string, record unified.ActionAuditRecord)) { + h.actionTransition = publisher +} + +func (h *ResourceHandlers) SetOperatorStateChanged(callback func(orgID, resourceID string)) { + h.operatorStateChanged = callback +} + +func (h *ResourceHandlers) buildRegistry(orgID string) (*unified.ResourceRegistry, error) { + return h.ensureQueryService().BuildRegistry(orgID) +} + +func (h *ResourceHandlers) getStore(orgID string) (unified.ResourceStore, error) { + return h.ensureQueryService().Store(orgID) +} + +func (h *ResourceHandlers) invalidateCache(orgID string) { + h.ensureQueryService().InvalidateCache(orgID) +} + +func (h *ResourceHandlers) CloseTenantStore(orgID string) error { + if h == nil { + return nil + } + return h.QueryService.CloseTenantStore(orgID) +} + +func (h *ResourceHandlers) CloseStores() error { + if h == nil { + return nil + } + return h.QueryService.CloseStores() +} + +// firstNonEmptyTrimmed is retained for non-resource connection projections. +func firstNonEmptyTrimmed(values ...string) string { + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + return trimmed + } + } + return "" +} + +func getUserID(r *http.Request) string { + return auth.GetUser(r.Context()) +} diff --git a/internal/api/resources_state_integration_test.go b/internal/api/resources_state_integration_test.go new file mode 100644 index 000000000..af9722ffe --- /dev/null +++ b/internal/api/resources_state_integration_test.go @@ -0,0 +1,117 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + authpkg "github.com/rcourtman/pulse-go-rewrite/pkg/auth" +) + +type stateSupplementalRecordsProvider struct { + records []unified.IngestRecord +} + +func (p stateSupplementalRecordsProvider) GetCurrentRecords() []unified.IngestRecord { + return append([]unified.IngestRecord(nil), p.records...) +} + +func (p stateSupplementalRecordsProvider) SupplementalRecords(*monitoring.Monitor, string) []unified.IngestRecord { + return p.GetCurrentRecords() +} + +func TestStateEndpointDerivesProxmoxWorkloadParentFromSupplementalRecords(t *testing.T) { + now := time.Date(2026, 5, 14, 10, 0, 0, 0, time.UTC) + hashedPassword, err := authpkg.HashPassword("password") + if err != nil { + t.Fatalf("hash password: %v", err) + } + dataPath := t.TempDir() + InitSessionStore(dataPath) + InitCSRFStore(dataPath) + cfg := &config.Config{DataPath: dataPath, AuthUser: "admin", AuthPass: hashedPassword} + monitor, err := monitoring.New(cfg) + if err != nil { + t.Fatalf("new monitor: %v", err) + } + t.Cleanup(func() { monitor.Stop() }) + + monitor.SetResourceStore(unified.NewMonitorAdapter(nil)) + monitor.SetSupplementalRecordsProvider(unified.SourceProxmox, stateSupplementalRecordsProvider{records: []unified.IngestRecord{ + { + SourceID: "homelab-delly", + Resource: unified.Resource{ + Type: unified.ResourceTypeAgent, Name: "delly", Status: unified.StatusOnline, LastSeen: now, + Proxmox: &unified.ProxmoxData{SourceID: "homelab-delly", NodeName: "delly", ClusterName: "homelab", Instance: "delly"}, + }, + Identity: unified.ResourceIdentity{MachineID: "machine-delly", Hostnames: []string{"delly"}}, + }, + { + SourceID: "delly:delly:104", + Resource: unified.Resource{ + Type: unified.ResourceTypeSystemContainer, Name: "cloudflared", Status: unified.StatusOnline, LastSeen: now, + Proxmox: &unified.ProxmoxData{SourceID: "delly:delly:104", NodeName: "delly", ClusterName: "homelab", Instance: "delly", VMID: 104}, + }, + Identity: unified.ResourceIdentity{Hostnames: []string{"cloudflared"}}, + }, + }}) + + router := &Router{config: cfg, monitor: monitor} + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/state", nil) + req.SetBasicAuth("admin", "password") + router.handleState(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("/api/state status = %d, body=%s", rec.Code, rec.Body.String()) + } + + var state models.StateFrontend + if err := json.NewDecoder(rec.Body).Decode(&state); err != nil { + t.Fatalf("decode /api/state: %v", err) + } + dellyCount, dellyID, cloudflaredParentID := 0, "", "" + for _, resource := range state.Resources { + switch { + case resource.Type == string(unified.ResourceTypeAgent) && resource.Name == "delly": + dellyCount++ + dellyID = resource.ID + case resource.Type == string(unified.ResourceTypeSystemContainer) && resource.Name == "cloudflared": + cloudflaredParentID = resource.ParentID + } + } + if dellyCount != 1 || dellyID == "" || cloudflaredParentID != dellyID { + t.Fatalf("unexpected /api/state resource projection: dellyCount=%d dellyID=%q cloudflaredParentID=%q resources=%#v", dellyCount, dellyID, cloudflaredParentID, state.Resources) + } +} + +// Closing must be safe to repeat and safe on a nil compatibility facade: the +// shutdown path runs before every dependency is guaranteed to be wired. +func TestResourceHandlers_CloseIsIdempotentAndNilSafe(t *testing.T) { + var nilHandlers *ResourceHandlers + if err := nilHandlers.CloseStores(); err != nil { + t.Fatalf("nil handler CloseStores: %v", err) + } + if err := nilHandlers.CloseTenantStore("client-a"); err != nil { + t.Fatalf("nil handler CloseTenantStore: %v", err) + } + + handlers := NewResourceHandlers(&config.Config{DataPath: t.TempDir()}) + if _, err := handlers.getStore("client-a"); err != nil { + t.Fatalf("getStore: %v", err) + } + if err := handlers.CloseStores(); err != nil { + t.Fatalf("first CloseStores: %v", err) + } + if err := handlers.CloseStores(); err != nil { + t.Fatalf("second CloseStores must be a no-op: %v", err) + } + if err := handlers.CloseTenantStore("client-a"); err != nil { + t.Fatalf("CloseTenantStore on an evicted org must be a no-op: %v", err) + } +} diff --git a/internal/api/router.go b/internal/api/router.go index c7f09eb77..9012be2b2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "io" - "math" "net" "net/http" "net/url" @@ -41,6 +40,8 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools" "github.com/rcourtman/pulse-go-rewrite/internal/ai/unified" "github.com/rcourtman/pulse-go-rewrite/internal/alerts" + "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" + "github.com/rcourtman/pulse-go-rewrite/internal/api/resourceapi" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/deploy" "github.com/rcourtman/pulse-go-rewrite/internal/maintenancesentinel" @@ -70,19 +71,6 @@ import ( "golang.org/x/sync/singleflight" ) -// workloadChartsCacheTTL absorbs repeated polls for the same sparkline data. -// Workload chart data refreshes on the metrics polling cadence (5-30 s); a -// short TTL is well below that and invisible for trend sparklines while -// dropping the cost of repeated polls from many clients. The cache itself -// lives on the Router so tests don't pollute each other and tenants don't -// cross-contaminate. -const workloadChartsCacheTTL = 3 * time.Second - -type cachedWorkloadChartsEntry struct { - body []byte - cachedAt time.Time -} - type relayRuntimeClient interface { Status() relay.ClientStatus Close() @@ -96,6 +84,7 @@ type Router struct { eventLogger routerEventLogger monitor *monitoring.Monitor // Legacy/Default support mtMonitor *monitoring.MultiTenantMonitor // Multi-tenant manager + chartService *chartapi.Service alertHandlers *AlertHandlers configHandlers *ConfigHandlers trueNASHandlers *TrueNASHandlers @@ -168,10 +157,6 @@ type Router struct { bootstrapTokenPath string checksumMu sync.RWMutex checksumCache map[string]checksumCacheEntry - infrastructureChartsMu sync.Mutex - infrastructureCharts map[string]summaryChartsCacheEntry - workloadsSummaryChartsMu sync.Mutex - workloadsSummaryCharts map[string]summaryChartsCacheEntry installScriptClient *http.Client relayMu sync.RWMutex relayClient relayRuntimeClient @@ -190,17 +175,7 @@ type Router struct { // Per-router perf state: keeps caches and singleflight groups isolated // between tests and prevents cross-tenant cache pollution. - stateComputeGroup singleflight.Group - workloadChartsComputeGroup singleflight.Group - workloadChartsCacheMu sync.RWMutex - workloadChartsCache map[string]cachedWorkloadChartsEntry -} - -const summaryChartsCacheTTL = 5 * time.Second - -type summaryChartsCacheEntry struct { - payload []byte - expiresAt time.Time + stateComputeGroup singleflight.Group } func pulseBinDir() string { @@ -210,30 +185,6 @@ func pulseBinDir() string { return "/opt/pulse/bin" } -func storageChartsSelectedNodeName(resource unifiedresources.Resource) string { - if name := strings.TrimSpace(resource.Name); name != "" { - return name - } - if resource.TrueNAS != nil { - if hostname := strings.TrimSpace(resource.TrueNAS.Hostname); hostname != "" { - return hostname - } - } - for _, hostname := range resource.Identity.Hostnames { - if hostname = strings.TrimSpace(hostname); hostname != "" { - return hostname - } - } - return "" -} - -func storageChartsSelectedNodeInstance(resource unifiedresources.Resource) string { - if resource.Proxmox == nil { - return "" - } - return strings.TrimSpace(resource.Proxmox.Instance) -} - func isDirectLoopbackRequest(req *http.Request) bool { if req == nil { return false @@ -303,6 +254,7 @@ func NewRouter(cfg *config.Config, monitor *monitoring.Monitor, mtMonitor *monit monitorSupplementalRecords: make(map[unifiedresources.DataSource]monitoring.MonitorSupplementalRecordsProvider), startedPatrolOrgs: make(map[string]bool), } + r.chartService = chartapi.NewService(routerChartMonitorResolver{router: r}) if r.wsHub != nil { r.wsHub.SetTrustedProxyChecker(isTrustedProxyIP) } @@ -6154,3115 +6106,6 @@ func (r *Router) handleStorage(w http.ResponseWriter, req *http.Request) { } } -// handleCharts handles chart data requests -func (r *Router) handleCharts(w http.ResponseWriter, req *http.Request) { - log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Charts endpoint hit") - const inMemoryChartThreshold = 2 * time.Hour - - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Get time range from query parameters - query := req.URL.Query() - timeRange := query.Get("range") - if timeRange == "" { - timeRange = "1h" - } - - // Convert time range to duration. - duration := parseChartsRangeDuration(timeRange) - - // Get tenant-specific monitor and current state - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) - return - } - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - http.Error(w, "State unavailable", http.StatusInternalServerError) - return - } - metricsStoreEnabled := monitor.GetMetricsStore() != nil - primarySourceHint := "memory" - if metricsStoreEnabled && duration > inMemoryChartThreshold { - primarySourceHint = "store_or_memory_fallback" - } - - // Create chart data structure that matches frontend expectations - chartData := make(map[string]VMChartData) - nodeData := make(map[string]NodeChartData) - - currentTime := time.Now().UnixMilli() // JavaScript timestamp format - oldestTimestamp := currentTime - - // Process VMs and Containers - batch-load historical data (1-2 SQL calls - // per family instead of N). - oldestTimestamp = collectGuestChartData(monitor, "vm", readState.VMs(), duration, chartData, currentTime, oldestTimestamp) - oldestTimestamp = collectGuestChartData(monitor, "container", readState.Containers(), duration, chartData, currentTime, oldestTimestamp) - - // Process Storage - batch-load historical data (1-2 SQL calls instead of N). - storageData := make(map[string]StorageChartData) - spList := readState.StoragePools() - storageIDs := make([]string, 0, len(spList)) - for _, sp := range spList { - if sp == nil { - continue - } - if sid := sp.SourceID(); sid != "" { - storageIDs = append(storageIDs, sid) - } - } - storageBatchMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration) - for _, sp := range spList { - if sp == nil { - continue - } - sid := sp.SourceID() - if sid == "" { - continue - } - storageData[sid] = make(StorageChartData) - if batchMetrics, ok := storageBatchMetrics[sid]; ok { - if usagePoints, found := batchMetrics["usage"]; found && len(usagePoints) > 0 { - storageData[sid]["disk"] = make([]MetricPoint, len(usagePoints)) - for i, point := range usagePoints { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - storageData[sid]["disk"][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - } - if len(storageData[sid]["disk"]) == 0 { - storageData[sid]["disk"] = []MetricPoint{ - {Timestamp: currentTime, Value: sp.DiskPercent()}, - } - } - } - - // Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5). - nodeMetricTypes := []string{"cpu", "memory", "disk", "netin", "netout"} - nodeList := readState.Nodes() - nodeIDs := make([]string, 0, len(nodeList)) - for _, node := range nodeList { - if node == nil { - continue - } - if nid := node.SourceID(); nid != "" { - nodeIDs = append(nodeIDs, nid) - } - } - nodeBatchMetrics := monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration) - for _, node := range nodeList { - if node == nil { - continue - } - nid := node.SourceID() - if nid == "" { - continue - } - nodeData[nid] = make(NodeChartData) - if batchMetrics, ok := nodeBatchMetrics[nid]; ok { - for _, metricType := range nodeMetricTypes { - points, found := batchMetrics[metricType] - if !found { - continue - } - nodeData[nid][metricType] = make([]MetricPoint, len(points)) - for i, point := range points { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - nodeData[nid][metricType][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - } - for _, metricType := range nodeMetricTypes { - if len(nodeData[nid][metricType]) == 0 { - var value float64 - hasFallbackValue := true - switch metricType { - case "cpu": - value = node.CPUPercent() - case "memory": - value = node.MemoryPercent() - case "disk": - value = node.DiskPercent() - default: - hasFallbackValue = false - } - if hasFallbackValue { - nodeData[nid][metricType] = []MetricPoint{ - {Timestamp: currentTime, Value: value}, - } - } - } - } - } - - // Build guest type map with canonical v6 names. - guestTypes := make(map[string]string) - for _, vm := range readState.VMs() { - if vm == nil { - continue - } - if sid := vm.SourceID(); sid != "" { - guestTypes[sid] = "vm" - } - } - for _, ct := range readState.Containers() { - if ct == nil { - continue - } - if sid := ct.SourceID(); sid != "" { - guestTypes[sid] = "system-container" - } - } - for _, dc := range readState.DockerContainers() { - if dc == nil { - continue - } - if key := strings.TrimSpace(dc.ID()); key != "" { - guestTypes[key] = "app-container" - } - } - - // Process Docker containers - batch-load historical data (1-2 SQL calls instead of N). - dockerData := make(map[string]VMChartData) - dcList := readState.DockerContainers() - dcRequests := make([]monitoring.GuestChartRequest, 0, len(dcList)) - for _, dc := range dcList { - _, request, ok := appContainerChartRequest(dc) - if !ok { - continue - } - dcRequests = append(dcRequests, request) - } - dcBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerContainer", dcRequests, duration, infrastructureSummaryMetricOrder...) - for _, dc := range dcList { - responseKey, request, ok := appContainerChartRequest(dc) - if !ok { - continue - } - dockerData[responseKey] = make(VMChartData) - if batchMetrics, ok := dcBatchMetrics[request.SQLResourceID]; ok { - oldestTimestamp = fillChartSeriesFromBatch(dockerData[responseKey], batchMetrics, oldestTimestamp) - } - if len(dockerData[responseKey]["cpu"]) == 0 { - dockerData[responseKey]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dc.CPUPercent()}} - dockerData[responseKey]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dc.MemoryPercent()}} - dockerData[responseKey]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: dc.DiskPercent()}} - } - } - - // Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N). - dockerHostData := make(map[string]VMChartData) - dhList := readState.DockerHosts() - dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList)) - for _, dh := range dhList { - if dh == nil { - continue - } - if dhID := dh.HostSourceID(); dhID != "" { - dhRequests = append(dhRequests, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID), - SQLResourceID: dhID, - }) - } - } - dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, infrastructureSummaryMetricOrder...) - for _, dh := range dhList { - if dh == nil { - continue - } - dhID := dh.HostSourceID() - if dhID == "" { - continue - } - dockerHostData[dhID] = make(VMChartData) - if batchMetrics, ok := dhBatchMetrics[dhID]; ok { - oldestTimestamp = fillChartSeriesFromBatch(dockerHostData[dhID], batchMetrics, oldestTimestamp) - } - if len(dockerHostData[dhID]["cpu"]) == 0 { - dockerHostData[dhID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dh.CPUPercent()}} - dockerHostData[dhID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dh.MemoryPercent()}} - var diskPercent float64 - if disks := dh.Disks(); len(disks) > 0 { - diskPercent = disks[0].Usage - } - dockerHostData[dhID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: diskPercent}} - } - } - - // Process unified agents - batch-load historical data (1-2 SQL calls instead of N). - agentData := make(map[string]VMChartData) - hostList := readState.Hosts() - agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList)) - for _, h := range hostList { - _, request, ok := hostAgentChartRequest(h) - if !ok { - continue - } - agentRequests = append(agentRequests, request) - } - agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, infrastructureSummaryMetricOrder...) - for _, h := range hostList { - hID, request, ok := hostAgentChartRequest(h) - if !ok { - continue - } - agentData[hID] = make(VMChartData) - if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok { - oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp) - } - if len(agentData[hID]["cpu"]) == 0 { - agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}} - agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}} - agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: h.DiskPercent()}} - } - } - - countChartPoints := func(metricsMap map[string]VMChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - - countNodePoints := func(metricsMap map[string]NodeChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - - countStoragePoints := func(metricsMap map[string]StorageChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - - guestPoints := countChartPoints(chartData) - nodePoints := countNodePoints(nodeData) - storagePoints := countStoragePoints(storageData) - dockerContainerPoints := countChartPoints(dockerData) - dockerHostPoints := countChartPoints(dockerHostData) - agentPoints := countChartPoints(agentData) - - response := ChartResponse{ - ChartData: chartData, - NodeData: nodeData, - StorageData: storageData, - DockerData: dockerData, - DockerHostData: dockerHostData, - AgentData: agentData, - GuestTypes: guestTypes, - Timestamp: currentTime, - Stats: ChartStats{ - OldestDataTimestamp: oldestTimestamp, - Range: timeRange, - RangeSeconds: int64(duration / time.Second), - MetricsStoreEnabled: metricsStoreEnabled, - PrimarySourceHint: primarySourceHint, - InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), - PointCounts: ChartPointCounts{ - Total: guestPoints + nodePoints + storagePoints + dockerContainerPoints + dockerHostPoints + agentPoints, - Guests: guestPoints, - Nodes: nodePoints, - Storage: storagePoints, - DockerContainers: dockerContainerPoints, - DockerHosts: dockerHostPoints, - Agents: agentPoints, - }, - }, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(response); err != nil { - log.Error().Err(err).Msg("Failed to encode chart data response") - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - - log.Debug(). - Int("guests", len(chartData)). - Int("nodes", len(nodeData)). - Int("storage", len(storageData)). - Int("dockerContainers", len(dockerData)). - Int("agents", len(agentData)). - Str("range", timeRange). - Msg("Chart data response sent") -} - -func parseWorkloadMaxPoints(raw string) int { - const ( - defaultMaxPoints = 180 - minMaxPoints = 30 - maxMaxPoints = 500 - ) - - trimmed := strings.TrimSpace(raw) - if trimmed == "" { - return defaultMaxPoints - } - - value, err := strconv.Atoi(trimmed) - if err != nil { - return defaultMaxPoints - } - if value < minMaxPoints { - return minMaxPoints - } - if value > maxMaxPoints { - return maxMaxPoints - } - return value -} - -func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) { - if host == nil { - return "", monitoring.GuestChartRequest{}, false - } - - if agentID := strings.TrimSpace(host.AgentID()); agentID != "" { - return agentID, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("agent:%s", agentID), - SQLResourceID: agentID, - }, true - } - - target := host.MetricsTarget() - if target == nil { - return "", monitoring.GuestChartRequest{}, false - } - - metricID := strings.TrimSpace(target.ResourceID) - if metricID == "" { - return "", monitoring.GuestChartRequest{}, false - } - - return metricID, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("agent:%s", metricID), - SQLResourceID: metricID, - }, true -} - -func appContainerChartMetricID(container *unifiedresources.DockerContainerView) string { - if container == nil { - return "" - } - - if target := container.MetricsTarget(); target != nil { - if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { - return metricID - } - } - - return strings.TrimSpace(container.ContainerID()) -} - -func appContainerChartRequest(container *unifiedresources.DockerContainerView) (string, monitoring.GuestChartRequest, bool) { - if container == nil { - return "", monitoring.GuestChartRequest{}, false - } - - responseKey := strings.TrimSpace(container.ID()) - if responseKey == "" { - responseKey = strings.TrimSpace(container.ContainerID()) - } - metricID := appContainerChartMetricID(container) - if responseKey == "" || metricID == "" { - return "", monitoring.GuestChartRequest{}, false - } - - return responseKey, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("docker:%s", metricID), - SQLResourceID: metricID, - }, true -} - -func canonicalGuestResponseKey(resourceID, instance, node string, vmid int) string { - trimmedInstance := strings.TrimSpace(instance) - trimmedNode := strings.TrimSpace(node) - if trimmedInstance != "" && trimmedNode != "" && vmid > 0 { - return fmt.Sprintf("%s:%s:%d", trimmedInstance, trimmedNode, vmid) - } - return strings.TrimSpace(resourceID) -} - -func vmChartMetricID(vm *unifiedresources.VMView) string { - if vm == nil { - return "" - } - - if target := vm.MetricsTarget(); target != nil { - if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { - return metricID - } - } - - return strings.TrimSpace(vm.SourceID()) -} - -func vmChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) { - if vm == nil { - return "", monitoring.GuestChartRequest{}, false - } - - responseKey := canonicalGuestResponseKey(vm.ID(), vm.Instance(), vm.Node(), vm.VMID()) - metricID := vmChartMetricID(vm) - if responseKey == "" || metricID == "" { - return "", monitoring.GuestChartRequest{}, false - } - - return responseKey, monitoring.GuestChartRequest{ - InMemoryKey: metricID, - SQLResourceID: metricID, - }, true -} - -func systemContainerChartMetricID(container *unifiedresources.ContainerView) string { - if container == nil { - return "" - } - - if target := container.MetricsTarget(); target != nil { - if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { - return metricID - } - } - - return strings.TrimSpace(container.SourceID()) -} - -func systemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) { - if container == nil { - return "", monitoring.GuestChartRequest{}, false - } - - responseKey := canonicalGuestResponseKey(container.ID(), container.Instance(), container.Node(), container.VMID()) - metricID := systemContainerChartMetricID(container) - if responseKey == "" || metricID == "" { - return "", monitoring.GuestChartRequest{}, false - } - - return responseKey, monitoring.GuestChartRequest{ - InMemoryKey: metricID, - SQLResourceID: metricID, - }, true -} - -func capMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint { - if len(points) <= maxPoints || maxPoints <= 0 { - return points - } - if maxPoints == 1 { - return []MetricPoint{points[len(points)-1]} - } - - result := make([]MetricPoint, 0, maxPoints) - step := float64(len(points)-1) / float64(maxPoints-1) - prevIndex := -1 - - for i := 0; i < maxPoints; i++ { - index := int(float64(i)*step + 0.5) - if index <= prevIndex { - index = prevIndex + 1 - } - if index >= len(points) { - index = len(points) - 1 - } - result = append(result, points[index]) - prevIndex = index - } - - if result[len(result)-1].Timestamp != points[len(points)-1].Timestamp { - result[len(result)-1] = points[len(points)-1] - } - return result -} - -const ( - infrastructureSummaryMinSeriesPoints = 24 - infrastructureSummaryMaxSeriesPoints = 96 - workloadsSummaryMinSeriesPoints = 24 - workloadsSummaryMaxSeriesPoints = 96 -) - -// capMetricPointSeries keeps mixed-cadence series visually proportional across -// the selected time window. Index-based capping over-selects recent dense -// samples, which bunches the right edge on long ranges. -func capMetricPointSeries(points []MetricPoint, maxPoints int) []MetricPoint { - if len(points) <= maxPoints || maxPoints <= 0 { - return points - } - if maxPoints == 1 { - return []MetricPoint{points[len(points)-1]} - } - - startTimestamp := points[0].Timestamp - endTimestamp := points[len(points)-1].Timestamp - if endTimestamp <= startTimestamp { - return capMetricPointSeriesByIndex(points, maxPoints) - } - - bucketSpan := float64(endTimestamp-startTimestamp) / float64(maxPoints-1) - if bucketSpan < 1 { - return capMetricPointSeriesByIndex(points, maxPoints) - } - - type timeBucketRepresentative struct { - point MetricPoint - distance float64 - ok bool - } - - buckets := make([]timeBucketRepresentative, maxPoints) - for _, point := range points { - index := int(math.Round(float64(point.Timestamp-startTimestamp) / bucketSpan)) - if index < 0 { - index = 0 - } - if index >= maxPoints { - index = maxPoints - 1 - } - - targetTimestamp := float64(startTimestamp) + bucketSpan*float64(index) - distance := math.Abs(float64(point.Timestamp) - targetTimestamp) - current := buckets[index] - if !current.ok || - distance < current.distance || - (distance == current.distance && point.Timestamp > current.point.Timestamp) { - buckets[index] = timeBucketRepresentative{ - point: point, - distance: distance, - ok: true, - } - } - } - - result := make([]MetricPoint, 0, maxPoints) - result = append(result, points[0]) - lastAddedTimestamp := points[0].Timestamp - for index := 1; index < maxPoints-1; index++ { - bucket := buckets[index] - if !bucket.ok { - continue - } - if bucket.point.Timestamp <= lastAddedTimestamp { - continue - } - result = append(result, bucket.point) - lastAddedTimestamp = bucket.point.Timestamp - } - - lastPoint := points[len(points)-1] - if lastPoint.Timestamp <= lastAddedTimestamp { - result[len(result)-1] = lastPoint - return result - } - - result = append(result, lastPoint) - return result -} - -func targetBoundedSummarySeriesPoints(duration time.Duration, minPoints, maxPoints int) int { - if duration <= 0 { - return minPoints - } - - target := int(duration / time.Minute) - if target < minPoints { - target = minPoints - } - if target > maxPoints { - target = maxPoints - } - if target < 2 { - target = 2 - } - return target -} - -type infrastructureSummaryBucket struct { - count int - sum float64 - max float64 - firstTimestamp int64 - lastTimestamp int64 - lastValue float64 -} - -func targetInfrastructureSummarySeriesPoints(duration time.Duration) int { - return targetBoundedSummarySeriesPoints( - duration, - infrastructureSummaryMinSeriesPoints, - infrastructureSummaryMaxSeriesPoints, - ) -} - -func infrastructureChartsCacheKey(req *http.Request, timeRange string, requestedMetricNames []string) string { - orgID := strings.TrimSpace(GetOrgID(req.Context())) - if orgID == "" { - orgID = "default" - } - return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.Join(requestedMetricNames, ",") -} - -func (r *Router) cachedInfrastructureChartsPayload(key string, now time.Time) ([]byte, bool) { - if r == nil || key == "" { - return nil, false - } - r.infrastructureChartsMu.Lock() - defer r.infrastructureChartsMu.Unlock() - - entry, ok := r.infrastructureCharts[key] - if !ok { - return nil, false - } - if !now.Before(entry.expiresAt) { - delete(r.infrastructureCharts, key) - return nil, false - } - return entry.payload, true -} - -func (r *Router) cacheInfrastructureChartsPayload(key string, payload []byte, now time.Time) { - if r == nil || key == "" || len(payload) == 0 { - return - } - r.infrastructureChartsMu.Lock() - defer r.infrastructureChartsMu.Unlock() - if r.infrastructureCharts == nil { - r.infrastructureCharts = make(map[string]summaryChartsCacheEntry, 8) - } - r.infrastructureCharts[key] = summaryChartsCacheEntry{ - payload: payload, - expiresAt: now.Add(summaryChartsCacheTTL), - } -} - -func targetWorkloadsSummarySeriesPoints(duration time.Duration) int { - return targetBoundedSummarySeriesPoints( - duration, - workloadsSummaryMinSeriesPoints, - workloadsSummaryMaxSeriesPoints, - ) -} - -func workloadsSummaryChartsCacheKey(req *http.Request, timeRange, selectedNodeID string) string { - orgID := strings.TrimSpace(GetOrgID(req.Context())) - if orgID == "" { - orgID = "default" - } - return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.TrimSpace(selectedNodeID) -} - -func (r *Router) cachedWorkloadsSummaryChartsPayload(key string, now time.Time) ([]byte, bool) { - if r == nil || key == "" { - return nil, false - } - r.workloadsSummaryChartsMu.Lock() - defer r.workloadsSummaryChartsMu.Unlock() - - entry, ok := r.workloadsSummaryCharts[key] - if !ok { - return nil, false - } - if !now.Before(entry.expiresAt) { - delete(r.workloadsSummaryCharts, key) - return nil, false - } - return entry.payload, true -} - -func (r *Router) cacheWorkloadsSummaryChartsPayload(key string, payload []byte, now time.Time) { - if r == nil || key == "" || len(payload) == 0 { - return - } - r.workloadsSummaryChartsMu.Lock() - defer r.workloadsSummaryChartsMu.Unlock() - if r.workloadsSummaryCharts == nil { - r.workloadsSummaryCharts = make(map[string]summaryChartsCacheEntry, 8) - } - r.workloadsSummaryCharts[key] = summaryChartsCacheEntry{ - payload: payload, - expiresAt: now.Add(summaryChartsCacheTTL), - } -} - -func aggregateInfrastructureSummaryBucketValue( - metricType string, - bucket infrastructureSummaryBucket, - isLastBucket bool, -) float64 { - if bucket.count == 0 { - return 0 - } - if isLastBucket { - return bucket.lastValue - } - - switch metricType { - case "memory", "disk": - return bucket.sum / float64(bucket.count) - default: - return bucket.max - } -} - -// normalizeInfrastructureSummaryMetricPointSeries folds mixed-cadence history -// into equal-time buckets for the infrastructure summary endpoint so long-range -// sparklines do not bunch recent higher-resolution samples at the right edge. -func normalizeInfrastructureSummaryMetricPointSeries( - points []MetricPoint, - metricType string, - duration time.Duration, - windowEndMillis int64, -) []MetricPoint { - targetPoints := targetInfrastructureSummarySeriesPoints(duration) - if len(points) <= targetPoints || targetPoints < 2 || duration <= 0 { - return points - } - - durationMillis := int64(duration / time.Millisecond) - if durationMillis <= 0 { - return points - } - - windowStartMillis := windowEndMillis - durationMillis - bucketCount := targetPoints - buckets := make([]infrastructureSummaryBucket, bucketCount) - firstNonEmpty := -1 - lastNonEmpty := -1 - - for _, point := range points { - if point.Timestamp < windowStartMillis || point.Timestamp > windowEndMillis { - continue - } - bucketIndex := int(((point.Timestamp - windowStartMillis) * int64(bucketCount)) / durationMillis) - if bucketIndex < 0 { - bucketIndex = 0 - } - if bucketIndex >= bucketCount { - bucketIndex = bucketCount - 1 - } - - bucket := &buckets[bucketIndex] - if bucket.count == 0 { - bucket.max = point.Value - bucket.firstTimestamp = point.Timestamp - if firstNonEmpty == -1 { - firstNonEmpty = bucketIndex - } - } else if point.Value > bucket.max { - bucket.max = point.Value - } - bucket.count++ - bucket.sum += point.Value - bucket.lastTimestamp = point.Timestamp - bucket.lastValue = point.Value - lastNonEmpty = bucketIndex - } - - if firstNonEmpty == -1 || lastNonEmpty == -1 { - return points - } - - result := make([]MetricPoint, 0, targetPoints) - for bucketIndex := 0; bucketIndex < bucketCount; bucketIndex++ { - bucket := buckets[bucketIndex] - if bucket.count == 0 { - continue - } - - bucketStartMillis := windowStartMillis + (int64(bucketIndex)*durationMillis)/int64(bucketCount) - bucketEndMillis := windowStartMillis + (int64(bucketIndex+1)*durationMillis)/int64(bucketCount) - timestamp := bucketStartMillis + (bucketEndMillis-bucketStartMillis)/2 - switch bucketIndex { - case firstNonEmpty: - timestamp = bucket.firstTimestamp - case lastNonEmpty: - timestamp = bucket.lastTimestamp - } - - result = append(result, MetricPoint{ - Timestamp: timestamp, - Value: aggregateInfrastructureSummaryBucketValue( - metricType, - bucket, - bucketIndex == lastNonEmpty, - ), - }) - } - - if len(result) == 0 { - return points - } - return result -} - -func normalizeInfrastructureSummaryChartSeries( - metrics map[string][]MetricPoint, - duration time.Duration, - windowEndMillis int64, -) { - for metricType, points := range metrics { - metrics[metricType] = normalizeInfrastructureSummaryMetricPointSeries( - points, - metricType, - duration, - windowEndMillis, - ) - } -} - -// sparklineMetrics lists the metric types consumed by summary sparklines -// and density maps. Metrics not in this set are omitted to keep payloads small. -// guestChartSourceView is the guest view subset the infrastructure summary -// chart builder consumes from VMs and LXC containers. -type guestChartSourceView interface { - comparable - SourceID() string - CPUPercent() float64 - MemoryPercent() float64 - MemoryUsed() int64 - DiskPercent() float64 - NetIn() float64 - NetOut() float64 -} - -// collectGuestChartData batch-loads sparkline history for one proxmox guest -// family into chartData (1-2 SQL calls instead of N) and returns the updated -// oldest chart timestamp. Guests without history fall back to a single -// current-value point per metric. -func collectGuestChartData[V guestChartSourceView]( - monitor *monitoring.Monitor, - storeType string, - guests []V, - duration time.Duration, - chartData map[string]VMChartData, - currentTime, oldestTimestamp int64, -) int64 { - var zero V - requests := make([]monitoring.GuestChartRequest, 0, len(guests)) - for _, g := range guests { - if g == zero { - continue - } - if id := g.SourceID(); id != "" { - requests = append(requests, monitoring.GuestChartRequest{InMemoryKey: id, SQLResourceID: id}) - } - } - batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, guestSparklineMetricOrder...) - for _, g := range guests { - if g == zero { - continue - } - id := g.SourceID() - if id == "" { - continue - } - chartData[id] = make(VMChartData) - if batchMetrics, ok := batch[id]; ok { - oldestTimestamp = fillChartSeriesFromBatch(chartData[id], batchMetrics, oldestTimestamp) - } - if len(chartData[id]["cpu"]) == 0 { - chartData[id]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: g.CPUPercent()}} - chartData[id]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: g.MemoryPercent()}} - chartData[id]["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(g.MemoryUsed())}} - chartData[id]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: g.DiskPercent()}} - chartData[id]["netin"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetIn()}} - chartData[id]["netout"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetOut()}} - } - } - return oldestTimestamp -} - -// fillChartSeriesFromBatch copies sparkline-eligible batch metric points -// into dst and returns the updated oldest chart timestamp. Shared by the -// per-family infrastructure summary chart loops. -func fillChartSeriesFromBatch(dst VMChartData, batchMetrics map[string][]monitoring.MetricPoint, oldestTimestamp int64) int64 { - for metricType, points := range batchMetrics { - if !sparklineMetrics[metricType] { - continue - } - dst[metricType] = make([]MetricPoint, len(points)) - for i, point := range points { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - dst[metricType][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - return oldestTimestamp -} - -var sparklineMetrics = map[string]bool{ - "cpu": true, - "memory": true, - "memoryused": true, - "disk": true, - "diskread": true, - "diskwrite": true, - "netin": true, - "netout": true, -} - -var infrastructureSummaryMetricOrder = []string{ - "cpu", - "memory", - "disk", - "diskread", - "diskwrite", - "netin", - "netout", -} - -var guestSparklineMetricOrder = []string{ - "cpu", - "memory", - "memoryused", - "disk", - "diskread", - "diskwrite", - "netin", - "netout", -} - -var workloadSummaryMetricOrder = []string{ - "cpu", - "memory", - "disk", - "netin", - "netout", -} - -func parseInfrastructureSummaryRequestedMetrics( - query url.Values, -) ([]string, map[string]bool, error) { - rawValues, ok := query["metrics"] - if !ok || len(rawValues) == 0 { - requested := make(map[string]bool, len(infrastructureSummaryMetricOrder)) - for _, metricType := range infrastructureSummaryMetricOrder { - requested[metricType] = true - } - return append([]string(nil), infrastructureSummaryMetricOrder...), requested, nil - } - - requestedList := make([]string, 0, len(infrastructureSummaryMetricOrder)) - requestedSet := make(map[string]bool, len(infrastructureSummaryMetricOrder)) - invalid := make([]string, 0) - - for _, rawValue := range rawValues { - for _, part := range strings.Split(rawValue, ",") { - metricType := strings.TrimSpace(strings.ToLower(part)) - if metricType == "" { - continue - } - if !sparklineMetrics[metricType] { - invalid = append(invalid, metricType) - continue - } - if requestedSet[metricType] { - continue - } - requestedSet[metricType] = true - requestedList = append(requestedList, metricType) - } - } - - if len(invalid) > 0 { - return nil, nil, fmt.Errorf("invalid infrastructure metrics filter: %s", strings.Join(invalid, ", ")) - } - if len(requestedList) == 0 { - return nil, nil, fmt.Errorf("infrastructure metrics filter must include at least one valid metric") - } - return requestedList, requestedSet, nil -} - -func convertMetricsForChart( - metrics map[string][]monitoring.MetricPoint, - oldestTimestamp *int64, - maxPoints int, -) VMChartData { - converted := make(VMChartData, len(metrics)) - for metricType, metricPoints := range metrics { - if !sparklineMetrics[metricType] { - continue - } - points := make([]MetricPoint, len(metricPoints)) - for i, point := range metricPoints { - ts := point.Timestamp.UnixMilli() - if ts < *oldestTimestamp { - *oldestTimestamp = ts - } - points[i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - converted[metricType] = capMetricPointSeries(points, maxPoints) - } - return converted -} - -// guestLiveMetricsView is the slice of the unified workload view API needed -// to seed a chart from live values; VM and container views both satisfy it. -type guestLiveMetricsView interface { - CPUPercent() float64 - MemoryPercent() float64 - MemoryUsed() int64 - DiskPercent() float64 - NetIn() float64 - NetOut() float64 -} - -// guestChartSeriesWithLiveFallback converts a guest's batched metric history -// into chart series, substituting single live-value points when no history -// exists yet so freshly added guests still chart. -func guestChartSeriesWithLiveFallback( - metrics map[string][]monitoring.MetricPoint, - guest guestLiveMetricsView, - oldestTimestamp *int64, - maxPoints int, - currentTime int64, -) VMChartData { - series := convertMetricsForChart(metrics, oldestTimestamp, maxPoints) - if len(series["cpu"]) == 0 { - series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: guest.CPUPercent()}} - series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: guest.MemoryPercent()}} - series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(guest.MemoryUsed())}} - series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: guest.DiskPercent()}} - series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetIn()}} - series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetOut()}} - } - return series -} - -const ( - mockWorkloadMinSeriesPoints = 24 - mockWorkloadMaxSeriesPoints = 180 -) - -func targetMockSeriesPoints(duration time.Duration, maxPoints int) int { - target := int(duration / (2 * time.Minute)) - if target < mockWorkloadMinSeriesPoints { - target = mockWorkloadMinSeriesPoints - } - if maxPoints > 0 && target > maxPoints { - target = maxPoints - } - if target > mockWorkloadMaxSeriesPoints { - target = mockWorkloadMaxSeriesPoints - } - if target < 2 { - target = 2 - } - return target -} - -// mockMetricStyle returns the series style for a given metric type. -func mockMetricStyle(metricType string) monitoring.SeriesStyle { - switch metricType { - case "cpu", "diskread", "diskwrite", "netin", "netout": - return monitoring.StyleSpiky - case "memory": - return monitoring.StylePlateau - default: - return monitoring.StyleFlat - } -} - -// generateStyledMockSeries produces a MetricPoint slice using the style-based -// generator from the monitoring package. -func generateStyledMockSeries( - nowMillis int64, - duration time.Duration, - numPoints int, - current float64, - resourceType string, - resourceID string, - metricType string, -) []MetricPoint { - style := mockMetricStyle(metricType) - - durationMillis := int64(duration / time.Millisecond) - if durationMillis <= 0 { - durationMillis = int64(time.Minute / time.Millisecond) - } - step := durationMillis / int64(numPoints-1) - if step <= 0 { - step = 1 - } - startMillis := nowMillis - durationMillis - timestamps := make([]time.Time, numPoints) - for i := 0; i < numPoints; i++ { - timestamps[i] = time.UnixMilli(startMillis + int64(i)*step) - } - values := monitoring.GenerateSeededResourceMetricSeriesForTimestamps( - current, - timestamps, - resourceType, - resourceID, - metricType, - style, - ) - points := make([]MetricPoint, numPoints) - for i := 0; i < numPoints; i++ { - points[i] = MetricPoint{ - Timestamp: startMillis + int64(i)*step, - Value: values[i], - } - } - return points -} - -func buildSyntheticMetricHistorySeries( - now time.Time, - duration time.Duration, - maxPoints int, - resourceType string, - resourceID string, - metricType string, - current float64, -) []monitoring.MetricPoint { - switch metricType { - case "disk", "diskread", "diskwrite": - case "smart_temp": - if current <= 0 { - return nil - } - default: - return nil - } - - numPoints := targetMockSeriesPoints(duration, maxPoints) - series := generateStyledMockSeries( - now.UnixMilli(), duration, numPoints, - current, resourceType, resourceID, metricType, - ) - - converted := make([]monitoring.MetricPoint, len(series)) - for i, point := range series { - converted[i] = monitoring.MetricPoint{ - Timestamp: time.UnixMilli(point.Timestamp), - Value: point.Value, - } - } - - return converted -} - -func buildMockWorkloadMetricHistorySeries( - now time.Time, - duration time.Duration, - maxPoints int, - resourceType string, - resourceID string, - metricType string, - current float64, -) []monitoring.MetricPoint { - switch metricType { - case "cpu", "memory", "disk": - case "diskread", "diskwrite", "netin", "netout": - default: - return nil - } - - numPoints := targetMockSeriesPoints(duration, maxPoints) - series := generateStyledMockSeries( - now.UnixMilli(), duration, numPoints, - current, resourceType, resourceID, metricType, - ) - - converted := make([]monitoring.MetricPoint, len(series)) - for i, point := range series { - converted[i] = monitoring.MetricPoint{ - Timestamp: time.UnixMilli(point.Timestamp), - Value: point.Value, - } - } - - return converted -} - -// handleWorkloadCharts serves workload-only chart data used by workloads -// sparklines. It intentionally excludes infrastructure/storage chart payloads -// to keep requests small and stable for large fleets. -func (r *Router) handleWorkloadCharts(w http.ResponseWriter, req *http.Request) { - log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workload charts endpoint hit") - const inMemoryChartThreshold = 2 * time.Hour - - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - query := req.URL.Query() - timeRange := query.Get("range") - if timeRange == "" { - timeRange = "1h" - } - selectedNodeID := strings.TrimSpace(query.Get("node")) - maxPointsRaw := query.Get("maxPoints") - maxPoints := parseWorkloadMaxPoints(maxPointsRaw) - duration := parseChartsRangeDuration(timeRange) - - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) - return - } - - orgID := GetOrgID(req.Context()) - if orgID == "" { - orgID = "default" - } - cacheKey := orgID + "|" + timeRange + "|" + selectedNodeID + "|" + maxPointsRaw - - r.workloadChartsCacheMu.RLock() - if entry, ok := r.workloadChartsCache[cacheKey]; ok && time.Since(entry.cachedAt) <= workloadChartsCacheTTL { - body := entry.body - r.workloadChartsCacheMu.RUnlock() - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(body); err != nil { - log.Error().Err(err).Msg("Failed to write cached workload chart data response") - } - return - } - r.workloadChartsCacheMu.RUnlock() - - v, err, _ := r.workloadChartsComputeGroup.Do(cacheKey, func() (any, error) { - // Re-check cache inside the singleflight barrier in case an earlier - // caller already populated it while we were queued. - r.workloadChartsCacheMu.RLock() - if entry, ok := r.workloadChartsCache[cacheKey]; ok && time.Since(entry.cachedAt) <= workloadChartsCacheTTL { - r.workloadChartsCacheMu.RUnlock() - return entry.body, nil - } - r.workloadChartsCacheMu.RUnlock() - - body, err := r.buildWorkloadChartsResponse(req.Context(), monitor, timeRange, selectedNodeID, maxPoints, duration, inMemoryChartThreshold) - if err != nil { - return nil, err - } - r.workloadChartsCacheMu.Lock() - if r.workloadChartsCache == nil { - r.workloadChartsCache = map[string]cachedWorkloadChartsEntry{} - } - r.workloadChartsCache[cacheKey] = cachedWorkloadChartsEntry{body: body, cachedAt: time.Now()} - r.workloadChartsCacheMu.Unlock() - return body, nil - }) - if err != nil { - log.Error().Err(err).Msg("Failed to build workload chart data response") - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(v.([]byte)); err != nil { - log.Error().Err(err).Msg("Failed to write workload chart data response") - } -} - -// buildWorkloadChartsResponse runs the heavy compute path for handleWorkloadCharts -// and returns the marshaled JSON body. Extracted so the handler can wrap it -// with caching + singleflight. -func (r *Router) buildWorkloadChartsResponse( - ctx context.Context, - monitor *monitoring.Monitor, - timeRange string, - selectedNodeID string, - maxPoints int, - duration time.Duration, - inMemoryChartThreshold time.Duration, -) ([]byte, error) { - _ = ctx - nodes := monitor.NodesSnapshot() - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - return nil, fmt.Errorf("state unavailable") - } - metricsStoreEnabled := monitor.GetMetricsStore() != nil - primarySourceHint := "memory" - if metricsStoreEnabled && duration > inMemoryChartThreshold { - primarySourceHint = "store_or_memory_fallback" - } - - currentTime := time.Now().UnixMilli() - oldestTimestamp := currentTime - - var selectedNode *models.Node - if selectedNodeID != "" { - for idx := range nodes { - if nodes[idx].ID == selectedNodeID { - selectedNode = &nodes[idx] - break - } - } - if selectedNode == nil { - log.Debug(). - Str("selectedNodeID", selectedNodeID). - Msg("Workload charts node filter not found in current state; falling back to global scope") - } - } - - matchesSelectedNode := func(instance, nodeName string) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) && - strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name)) - } - - matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - if host == nil { - return false - } - nodeName := strings.TrimSpace(selectedNode.Name) - if nodeName == "" { - return false - } - return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || - strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) - } - - matchesSelectedAgentHostView := func(host *unifiedresources.HostView) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - if host == nil { - return false - } - nodeName := strings.TrimSpace(selectedNode.Name) - if nodeName == "" { - return false - } - return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || - strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) || - strings.EqualFold(strings.TrimSpace(host.AgentID()), nodeName) || - strings.EqualFold(strings.TrimSpace(host.ID()), nodeName) - } - - matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - if pod == nil { - return false - } - nodeName := strings.TrimSpace(selectedNode.Name) - if nodeName == "" { - return false - } - return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName) - } - - chartData := make(map[string]VMChartData) - dockerData := make(map[string]VMChartData) - - guestTypes := make(map[string]string) - - vmList := make([]*unifiedresources.VMView, 0) - vmResponseKeys := make([]string, 0) - vmRequests := make([]monitoring.GuestChartRequest, 0) - for _, vm := range readState.VMs() { - if vm == nil { - continue - } - if !matchesSelectedNode(vm.Instance(), vm.Node()) { - continue - } - - responseKey, request, ok := vmChartRequest(vm) - if !ok { - continue - } - - vmList = append(vmList, vm) - vmResponseKeys = append(vmResponseKeys, responseKey) - vmRequests = append(vmRequests, request) - } - containerList := make([]*unifiedresources.ContainerView, 0) - containerResponseKeys := make([]string, 0) - containerRequests := make([]monitoring.GuestChartRequest, 0) - for _, ct := range readState.Containers() { - if ct == nil { - continue - } - if !matchesSelectedNode(ct.Instance(), ct.Node()) { - continue - } - - responseKey, request, ok := systemContainerChartRequest(ct) - if !ok { - continue - } - - containerList = append(containerList, ct) - containerResponseKeys = append(containerResponseKeys, responseKey) - containerRequests = append(containerRequests, request) - } - podList := make([]*unifiedresources.PodView, 0) - podRequests := make([]monitoring.GuestChartRequest, 0) - for _, pod := range readState.Pods() { - if pod == nil { - continue - } - if !matchesSelectedKubernetesPodView(pod) { - continue - } - - metricKey := kubernetesPodMetricIDFromView(pod) - if metricKey == "" { - continue - } - - podList = append(podList, pod) - podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey}) - } - dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts())) - for _, host := range readState.DockerHosts() { - if host == nil { - continue - } - dockerHostsByID[host.ID()] = host - } - agentHostsByID := make(map[string]*unifiedresources.HostView, len(readState.Hosts())) - for _, host := range readState.Hosts() { - if host == nil { - continue - } - agentHostsByID[host.ID()] = host - } - - dockerContainerList := make([]*unifiedresources.DockerContainerView, 0) - dockerContainerRequests := make([]monitoring.GuestChartRequest, 0) - dockerContainerKeys := make([]string, 0) - for _, container := range readState.DockerContainers() { - if container == nil { - continue - } - - if selectedNodeID != "" && selectedNode != nil { - host := dockerHostsByID[container.ParentID()] - if host != nil { - if !matchesSelectedDockerHostView(host) { - continue - } - } else { - agentHost := agentHostsByID[container.ParentID()] - if agentHost == nil || !matchesSelectedAgentHostView(agentHost) { - continue - } - } - } - - responseKey, request, ok := appContainerChartRequest(container) - if !ok { - continue - } - dockerContainerList = append(dockerContainerList, container) - dockerContainerKeys = append(dockerContainerKeys, responseKey) - dockerContainerRequests = append(dockerContainerRequests, request) - } - var ( - vmBatchMetrics map[string]map[string][]monitoring.MetricPoint - containerBatchMetrics map[string]map[string][]monitoring.MetricPoint - podBatchMetrics map[string]map[string][]monitoring.MetricPoint - dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint - ) - var workloadChartsBatchWG sync.WaitGroup - workloadChartsBatchWG.Add(4) - go func() { - defer workloadChartsBatchWG.Done() - vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, guestSparklineMetricOrder...) - }() - go func() { - defer workloadChartsBatchWG.Done() - containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, guestSparklineMetricOrder...) - }() - go func() { - defer workloadChartsBatchWG.Done() - podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...) - }() - go func() { - defer workloadChartsBatchWG.Done() - dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, infrastructureSummaryMetricOrder...) - }() - workloadChartsBatchWG.Wait() - - for idx, vm := range vmList { - responseKey := vmResponseKeys[idx] - metricID := vmRequests[idx].SQLResourceID - guestTypes[responseKey] = "vm" - chartData[responseKey] = guestChartSeriesWithLiveFallback(vmBatchMetrics[metricID], vm, &oldestTimestamp, maxPoints, currentTime) - } - - for idx, ct := range containerList { - responseKey := containerResponseKeys[idx] - metricID := containerRequests[idx].SQLResourceID - guestTypes[responseKey] = "system-container" - chartData[responseKey] = guestChartSeriesWithLiveFallback(containerBatchMetrics[metricID], ct, &oldestTimestamp, maxPoints, currentTime) - } - - for _, pod := range podList { - metricKey := kubernetesPodMetricIDFromView(pod) - series := convertMetricsForChart(podBatchMetrics[metricKey], &oldestTimestamp, maxPoints) - guestTypes[metricKey] = "k8s" - - if len(series["cpu"]) == 0 { - series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: pod.CPUPercent()}} - series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: pod.MemoryPercent()}} - series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: pod.DiskPercent()}} - series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetInRate()}} - series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetOutRate()}} - } - chartData[metricKey] = series - } - - for idx, container := range dockerContainerList { - responseKey := dockerContainerKeys[idx] - metricID := dockerContainerRequests[idx].SQLResourceID - series := convertMetricsForChart(dockerContainerBatchMetrics[metricID], &oldestTimestamp, maxPoints) - guestTypes[responseKey] = "app-container" - - if len(series["cpu"]) == 0 { - series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: container.CPUPercent()}} - series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: container.MemoryPercent()}} - series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: container.DiskPercent()}} - series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetInRate()}} - series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetOutRate()}} - } - dockerData[responseKey] = series - } - - countChartPoints := func(metricsMap map[string]VMChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - - guestPoints := countChartPoints(chartData) - dockerContainerPoints := countChartPoints(dockerData) - - response := EmptyWorkloadChartsResponse() - response.ChartData = chartData - response.DockerData = dockerData - response.GuestTypes = guestTypes - response.Timestamp = currentTime - response.Stats = ChartStats{ - OldestDataTimestamp: oldestTimestamp, - Range: timeRange, - RangeSeconds: int64(duration / time.Second), - MetricsStoreEnabled: metricsStoreEnabled, - PrimarySourceHint: primarySourceHint, - InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), - PointCounts: ChartPointCounts{ - Total: guestPoints + dockerContainerPoints, - Guests: guestPoints, - DockerContainers: dockerContainerPoints, - }, - } - - body, err := json.Marshal(response.NormalizeCollections()) - if err != nil { - return nil, fmt.Errorf("marshal workload chart response: %w", err) - } - return body, nil -} - -// parseChartsRangeDuration converts the UI chart range query (e.g. "5m", "1h") -// into a duration. This is shared by /api/charts and /api/charts/infrastructure -// to prevent drift. -func parseChartsRangeDuration(rangeStr string) time.Duration { - switch rangeStr { - case "5m": - return 5 * time.Minute - case "15m": - return 15 * time.Minute - case "30m": - return 30 * time.Minute - case "1h": - return time.Hour - case "4h": - return 4 * time.Hour - case "8h": - return 8 * time.Hour - case "12h": - return 12 * time.Hour - case "24h": - return 24 * time.Hour - case "7d": - return 7 * 24 * time.Hour - case "30d": - return 30 * 24 * time.Hour - default: - return time.Hour - } -} - -// handleInfrastructureCharts serves infrastructure-only chart data. -// This is intentionally narrower than /api/charts to reduce payload size and server-side compute -// for the Infrastructure page summary cards. -func (r *Router) handleInfrastructureCharts(w http.ResponseWriter, req *http.Request) { - log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Infrastructure charts endpoint hit") - const inMemoryChartThreshold = 2 * time.Hour - - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Get time range from query parameters - query := req.URL.Query() - timeRange := query.Get("range") - if timeRange == "" { - timeRange = "1h" - } - requestedMetricNames, requestedMetrics, err := parseInfrastructureSummaryRequestedMetrics(query) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - // Convert time range to duration. - duration := parseChartsRangeDuration(timeRange) - - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) - return - } - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - http.Error(w, "State unavailable", http.StatusInternalServerError) - return - } - metricsStoreEnabled := monitor.GetMetricsStore() != nil - primarySourceHint := "memory" - if metricsStoreEnabled && duration > inMemoryChartThreshold { - primarySourceHint = "store_or_memory_fallback" - } - - now := time.Now() - cacheKey := infrastructureChartsCacheKey(req, timeRange, requestedMetricNames) - if payload, ok := r.cachedInfrastructureChartsPayload(cacheKey, now); ok { - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(payload); err != nil { - log.Error().Err(err).Msg("Failed to write cached infrastructure chart data response") - } - return - } - - currentTime := now.UnixMilli() - oldestTimestamp := currentTime - - // Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5). - nodeMetricTypes := make([]string, 0, 5) - for _, metricType := range []string{"cpu", "memory", "disk", "netin", "netout"} { - if requestedMetrics[metricType] { - nodeMetricTypes = append(nodeMetricTypes, metricType) - } - } - nodeData := make(map[string]NodeChartData) - nodeList := readState.Nodes() - nodeIDs := make([]string, 0, len(nodeList)) - for _, node := range nodeList { - if node == nil { - continue - } - if nid := node.SourceID(); nid != "" { - nodeIDs = append(nodeIDs, nid) - } - } - nodeBatchMetrics := map[string]map[string][]monitoring.MetricPoint{} - if len(nodeMetricTypes) > 0 { - nodeBatchMetrics = monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration) - } - for _, node := range nodeList { - if node == nil { - continue - } - nid := node.SourceID() - if nid == "" { - continue - } - nodeData[nid] = make(NodeChartData) - if batchMetrics, ok := nodeBatchMetrics[nid]; ok { - for _, metricType := range nodeMetricTypes { - points, found := batchMetrics[metricType] - if !found { - continue - } - nodeData[nid][metricType] = make([]MetricPoint, len(points)) - for i, point := range points { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - nodeData[nid][metricType][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - } - for _, metricType := range nodeMetricTypes { - if len(nodeData[nid][metricType]) > 0 { - continue - } - var value float64 - hasFallbackValue := true - switch metricType { - case "cpu": - value = node.CPUPercent() - case "memory": - value = node.MemoryPercent() - case "disk": - value = node.DiskPercent() - default: - hasFallbackValue = false - } - if hasFallbackValue { - nodeData[nid][metricType] = []MetricPoint{ - {Timestamp: currentTime, Value: value}, - } - } - } - normalizeInfrastructureSummaryChartSeries(nodeData[nid], duration, currentTime) - } - - // Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N). - dockerHostData := make(map[string]VMChartData) - dhList := readState.DockerHosts() - dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList)) - for _, dh := range dhList { - if dh == nil { - continue - } - if dhID := dh.HostSourceID(); dhID != "" { - dhRequests = append(dhRequests, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID), - SQLResourceID: dhID, - }) - } - } - dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, requestedMetricNames...) - for _, dh := range dhList { - if dh == nil { - continue - } - dhID := dh.HostSourceID() - if dhID == "" { - continue - } - dockerHostData[dhID] = make(VMChartData) - if batchMetrics, ok := dhBatchMetrics[dhID]; ok { - for metricType, points := range batchMetrics { - if !requestedMetrics[metricType] { - continue - } - dockerHostData[dhID][metricType] = make([]MetricPoint, len(points)) - for i, point := range points { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - dockerHostData[dhID][metricType][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - } - for _, metricType := range requestedMetricNames { - if len(dockerHostData[dhID][metricType]) > 0 { - continue - } - var value float64 - hasFallbackValue := true - switch metricType { - case "cpu": - value = dh.CPUPercent() - case "memory": - value = dh.MemoryPercent() - case "disk": - if disks := dh.Disks(); len(disks) > 0 { - value = disks[0].Usage - } - default: - hasFallbackValue = false - } - if hasFallbackValue { - dockerHostData[dhID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}} - } - } - normalizeInfrastructureSummaryChartSeries(dockerHostData[dhID], duration, currentTime) - } - - // Process unified agents - batch-load historical data (1-2 SQL calls instead of N). - agentData := make(map[string]VMChartData) - hostList := readState.Hosts() - agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList)) - for _, h := range hostList { - _, request, ok := hostAgentChartRequest(h) - if !ok { - continue - } - agentRequests = append(agentRequests, request) - } - agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, requestedMetricNames...) - for _, h := range hostList { - hID, request, ok := hostAgentChartRequest(h) - if !ok { - continue - } - agentData[hID] = make(VMChartData) - if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok { - for metricType, points := range batchMetrics { - if !requestedMetrics[metricType] { - continue - } - agentData[hID][metricType] = make([]MetricPoint, len(points)) - for i, point := range points { - ts := point.Timestamp.UnixMilli() - if ts < oldestTimestamp { - oldestTimestamp = ts - } - agentData[hID][metricType][i] = MetricPoint{ - Timestamp: ts, - Value: point.Value, - } - } - } - } - for _, metricType := range requestedMetricNames { - if len(agentData[hID][metricType]) > 0 { - continue - } - var value float64 - hasFallbackValue := true - switch metricType { - case "cpu": - value = h.CPUPercent() - case "memory": - value = h.MemoryPercent() - case "disk": - value = h.DiskPercent() - default: - hasFallbackValue = false - } - if hasFallbackValue { - agentData[hID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}} - } - } - normalizeInfrastructureSummaryChartSeries(agentData[hID], duration, currentTime) - } - - countNodePoints := func(metricsMap map[string]NodeChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - countChartPoints := func(metricsMap map[string]VMChartData) int { - total := 0 - for _, metricSeries := range metricsMap { - for _, points := range metricSeries { - total += len(points) - } - } - return total - } - - nodePoints := countNodePoints(nodeData) - dockerHostPoints := countChartPoints(dockerHostData) - agentPoints := countChartPoints(agentData) - - response := EmptyInfrastructureChartsResponse() - response.NodeData = nodeData - response.DockerHostData = dockerHostData - response.AgentData = agentData - response.Timestamp = currentTime - response.Stats = ChartStats{ - OldestDataTimestamp: oldestTimestamp, - Range: timeRange, - RangeSeconds: int64(duration / time.Second), - MetricsStoreEnabled: metricsStoreEnabled, - PrimarySourceHint: primarySourceHint, - InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), - PointCounts: ChartPointCounts{ - Total: nodePoints + dockerHostPoints + agentPoints, - Nodes: nodePoints, - DockerHosts: dockerHostPoints, - Agents: agentPoints, - }, - } - - payload, err := json.Marshal(response.NormalizeCollections()) - if err != nil { - log.Error().Err(err).Msg("Failed to encode infrastructure chart data response") - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - payload = append(payload, '\n') - r.cacheInfrastructureChartsPayload(cacheKey, payload, now) - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(payload); err != nil { - log.Error().Err(err).Msg("Failed to write infrastructure chart data response") - } -} - -type workloadSummaryBuckets struct { - cpu workloadSummaryMetricBucket - memory workloadSummaryMetricBucket - disk workloadSummaryMetricBucket - network workloadSummaryMetricBucket -} - -type workloadSummaryMetricBucket struct { - sum float64 - max float64 - count int -} - -func (bucket *workloadSummaryMetricBucket) add(value float64) { - if bucket == nil { - return - } - if bucket.count == 0 || value > bucket.max { - bucket.max = value - } - bucket.sum += value - bucket.count++ -} - -func (bucket workloadSummaryMetricBucket) average() float64 { - if bucket.count == 0 { - return 0 - } - return bucket.sum / float64(bucket.count) -} - -type workloadsSummarySnapshot struct { - id string - name string - cpu float64 - memory float64 - disk float64 - network float64 -} - -func workloadSummaryBucketTimestamp(timestampMs int64) int64 { - const bucketSizeMs = int64(30_000) - return (timestampMs / bucketSizeMs) * bucketSizeMs -} - -func clampWorkloadPercent(value float64) float64 { - if value != value { - return 0 - } - if value < 0 { - return 0 - } - if value > 100 { - return 100 - } - return value -} - -func proxmoxModelCPURatioPercent(value float64) float64 { - return clampWorkloadPercent(value * 100) -} - -func clampNonNegativeWorkloadValue(value float64) float64 { - if value != value { - return 0 - } - if value < 0 { - return 0 - } - return value -} - -func kubernetesPodMetricIDFromView(pod *unifiedresources.PodView) string { - if pod == nil { - return "" - } - clusterKey := strings.TrimSpace(pod.ClusterID()) - if clusterKey == "" { - clusterKey = strings.TrimSpace(pod.ClusterName()) - } - podKey := strings.TrimSpace(pod.PodUID()) - if podKey == "" { - namespace := strings.TrimSpace(pod.Namespace()) - name := strings.TrimSpace(pod.Name()) - if namespace != "" || name != "" { - podKey = fmt.Sprintf("%s/%s", namespace, name) - } - } - if clusterKey == "" || podKey == "" { - return "" - } - return fmt.Sprintf("k8s:%s:pod:%s", clusterKey, podKey) -} - -func getOrCreateWorkloadBucket(buckets map[int64]*workloadSummaryBuckets, bucketTs int64) *workloadSummaryBuckets { - if bucket, ok := buckets[bucketTs]; ok { - return bucket - } - bucket := &workloadSummaryBuckets{} - buckets[bucketTs] = bucket - return bucket -} - -func appendWorkloadMetricPoints( - buckets map[int64]*workloadSummaryBuckets, - points []monitoring.MetricPoint, - target string, - oldestTimestamp *int64, -) int { - added := 0 - for _, point := range points { - ts := point.Timestamp.UnixMilli() - if ts <= 0 { - continue - } - if ts < *oldestTimestamp { - *oldestTimestamp = ts - } - bucketTs := workloadSummaryBucketTimestamp(ts) - bucket := getOrCreateWorkloadBucket(buckets, bucketTs) - value := clampNonNegativeWorkloadValue(point.Value) - switch target { - case "cpu", "memory", "disk": - value = clampWorkloadPercent(value) - } - switch target { - case "cpu": - bucket.cpu.add(value) - case "memory": - bucket.memory.add(value) - case "disk": - bucket.disk.add(value) - case "network": - bucket.network.add(value) - } - added++ - } - return added -} - -func mergeWorkloadNetworkPoints( - netIn []monitoring.MetricPoint, - netOut []monitoring.MetricPoint, -) []monitoring.MetricPoint { - totals := make(map[int64]float64) - for _, point := range netIn { - ts := point.Timestamp.UnixMilli() - if ts <= 0 { - continue - } - totals[ts] += clampNonNegativeWorkloadValue(point.Value) - } - for _, point := range netOut { - ts := point.Timestamp.UnixMilli() - if ts <= 0 { - continue - } - totals[ts] += clampNonNegativeWorkloadValue(point.Value) - } - if len(totals) == 0 { - return nil - } - keys := make([]int64, 0, len(totals)) - for ts := range totals { - keys = append(keys, ts) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - points := make([]monitoring.MetricPoint, 0, len(keys)) - for _, ts := range keys { - points = append(points, monitoring.MetricPoint{ - Timestamp: time.UnixMilli(ts), - Value: totals[ts], - }) - } - return points -} - -func buildWorkloadsSummaryMetric( - buckets map[int64]*workloadSummaryBuckets, - selector func(*workloadSummaryBuckets) workloadSummaryMetricBucket, -) WorkloadsSummaryMetricData { - keys := make([]int64, 0, len(buckets)) - for ts := range buckets { - keys = append(keys, ts) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - - data := WorkloadsSummaryMetricData{ - P50: make([]MetricPoint, 0, len(keys)), - P95: make([]MetricPoint, 0, len(keys)), - } - for _, ts := range keys { - bucket := selector(buckets[ts]) - if bucket.count == 0 { - continue - } - data.P50 = append(data.P50, MetricPoint{ - Timestamp: ts, - Value: bucket.average(), - }) - data.P95 = append(data.P95, MetricPoint{ - Timestamp: ts, - Value: bucket.max, - }) - } - return data -} - -func summaryMetricPointCount(metric WorkloadsSummaryMetricData) int { - return len(metric.P50) + len(metric.P95) -} - -func normalizeWorkloadsSummaryMetricPointSeries( - metric WorkloadsSummaryMetricData, - duration time.Duration, -) WorkloadsSummaryMetricData { - targetPoints := targetWorkloadsSummarySeriesPoints(duration) - metric.P50 = capMetricPointSeries(metric.P50, targetPoints) - metric.P95 = capMetricPointSeries(metric.P95, targetPoints) - return metric -} - -func latestSummaryMetricValue(points []monitoring.MetricPoint, fallback float64, clamp func(float64) float64) float64 { - if len(points) == 0 { - return clamp(fallback) - } - - latest := points[0] - for i := 1; i < len(points); i++ { - if points[i].Timestamp.After(latest.Timestamp) { - latest = points[i] - } - } - return clamp(latest.Value) -} - -func buildWorkloadsTopContributors( - snapshots []workloadsSummarySnapshot, - selector func(workloadsSummarySnapshot) float64, -) []WorkloadsSummaryContributor { - contributors := make([]WorkloadsSummaryContributor, 0, len(snapshots)) - for _, snapshot := range snapshots { - value := selector(snapshot) - if value <= 0 { - continue - } - contributors = append(contributors, WorkloadsSummaryContributor{ - ID: snapshot.id, - Name: snapshot.name, - Value: value, - }) - } - - sort.Slice(contributors, func(i, j int) bool { - if contributors[i].Value == contributors[j].Value { - if contributors[i].Name == contributors[j].Name { - return contributors[i].ID < contributors[j].ID - } - return contributors[i].Name < contributors[j].Name - } - return contributors[i].Value > contributors[j].Value - }) - - if len(contributors) > 3 { - contributors = contributors[:3] - } - return contributors -} - -func buildWorkloadsBlastRadius( - snapshots []workloadsSummarySnapshot, - selector func(workloadsSummarySnapshot) float64, -) WorkloadsSummaryBlastRadius { - values := make([]float64, 0, len(snapshots)) - for _, snapshot := range snapshots { - value := selector(snapshot) - if value <= 0 { - continue - } - values = append(values, value) - } - - if len(values) == 0 { - return WorkloadsSummaryBlastRadius{ - Scope: "idle", - Top3Share: 0, - ActiveWorkloads: 0, - } - } - - sort.Slice(values, func(i, j int) bool { return values[i] > values[j] }) - total := 0.0 - for _, value := range values { - total += value - } - - topCount := 3 - if len(values) < topCount { - topCount = len(values) - } - top3 := 0.0 - for i := 0; i < topCount; i++ { - top3 += values[i] - } - - share := 0.0 - if total > 0 { - share = (top3 / total) * 100 - } - - scope := "distributed" - switch { - case share >= 80: - scope = "concentrated" - case share >= 55: - scope = "mixed" - } - - return WorkloadsSummaryBlastRadius{ - Scope: scope, - Top3Share: share, - ActiveWorkloads: len(values), - } -} - -// handleWorkloadsSummaryCharts serves compact, aggregate workload sparklines -// for the Workloads top cards. It intentionally avoids returning per-workload -// time series to keep payloads bounded for large fleets. -func (r *Router) handleWorkloadsSummaryCharts(w http.ResponseWriter, req *http.Request) { - log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workloads summary charts endpoint hit") - const inMemoryChartThreshold = 2 * time.Hour - - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - query := req.URL.Query() - timeRange := query.Get("range") - if timeRange == "" { - timeRange = "1h" - } - selectedNodeID := strings.TrimSpace(query.Get("node")) - duration := parseChartsRangeDuration(timeRange) - - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) - return - } - nodes := monitor.NodesSnapshot() - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - http.Error(w, "State unavailable", http.StatusInternalServerError) - return - } - - now := time.Now() - cacheKey := workloadsSummaryChartsCacheKey(req, timeRange, selectedNodeID) - if payload, ok := r.cachedWorkloadsSummaryChartsPayload(cacheKey, now); ok { - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(payload) - return - } - - mockModeEnabled := mock.IsMockEnabled() - metricsStoreEnabled := monitor.GetMetricsStore() != nil - primarySourceHint := "memory" - if metricsStoreEnabled && duration > inMemoryChartThreshold { - primarySourceHint = "store_or_memory_fallback" - } - - currentTime := now.UnixMilli() - currentTimeTime := now - oldestTimestamp := currentTime - buckets := make(map[int64]*workloadSummaryBuckets) - guestPointCount := 0 - guestCounts := WorkloadsGuestCounts{} - snapshots := make([]workloadsSummarySnapshot, 0, len(readState.VMs())+len(readState.Containers())) - - var selectedNode *models.Node - if selectedNodeID != "" { - for idx := range nodes { - if nodes[idx].ID == selectedNodeID { - selectedNode = &nodes[idx] - break - } - } - if selectedNode == nil { - log.Debug(). - Str("selectedNodeID", selectedNodeID). - Msg("Workloads summary node filter not found in current state; falling back to global scope") - } - } - - matchesSelectedNode := func(instance, nodeName string) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) && - strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name)) - } - - matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - if host == nil { - return false - } - nodeName := strings.TrimSpace(selectedNode.Name) - if nodeName == "" { - return false - } - return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) || - strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) - } - - matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool { - if selectedNodeID == "" { - return true - } - if selectedNode == nil { - return true - } - if pod == nil { - return false - } - nodeName := strings.TrimSpace(selectedNode.Name) - if nodeName == "" { - return false - } - return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName) - } - - vmList := make([]*unifiedresources.VMView, 0) - vmResponseKeys := make([]string, 0) - vmRequests := make([]monitoring.GuestChartRequest, 0) - for _, vm := range readState.VMs() { - if vm == nil { - continue - } - if !matchesSelectedNode(vm.Instance(), vm.Node()) { - continue - } - responseKey, request, ok := vmChartRequest(vm) - if !ok { - continue - } - vmList = append(vmList, vm) - vmResponseKeys = append(vmResponseKeys, responseKey) - vmRequests = append(vmRequests, request) - } - containerList := make([]*unifiedresources.ContainerView, 0) - containerResponseKeys := make([]string, 0) - containerRequests := make([]monitoring.GuestChartRequest, 0) - for _, ct := range readState.Containers() { - if ct == nil { - continue - } - if !matchesSelectedNode(ct.Instance(), ct.Node()) { - continue - } - responseKey, request, ok := systemContainerChartRequest(ct) - if !ok { - continue - } - containerList = append(containerList, ct) - containerResponseKeys = append(containerResponseKeys, responseKey) - containerRequests = append(containerRequests, request) - } - podList := make([]*unifiedresources.PodView, 0) - podRequests := make([]monitoring.GuestChartRequest, 0) - for _, pod := range readState.Pods() { - if pod == nil { - continue - } - if !matchesSelectedKubernetesPodView(pod) { - continue - } - - metricKey := kubernetesPodMetricIDFromView(pod) - if metricKey == "" { - continue - } - podList = append(podList, pod) - podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey}) - } - dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts())) - for _, host := range readState.DockerHosts() { - if host == nil { - continue - } - dockerHostsByID[host.ID()] = host - } - - dockerContainerList := make([]*unifiedresources.DockerContainerView, 0) - dockerContainerRequests := make([]monitoring.GuestChartRequest, 0) - for _, container := range readState.DockerContainers() { - if container == nil { - continue - } - if selectedNodeID != "" && selectedNode != nil { - host := dockerHostsByID[container.ParentID()] - if host == nil || !matchesSelectedDockerHostView(host) { - continue - } - } - containerID := strings.TrimSpace(container.ContainerID()) - if containerID == "" { - continue - } - dockerContainerList = append(dockerContainerList, container) - dockerContainerRequests = append(dockerContainerRequests, monitoring.GuestChartRequest{ - InMemoryKey: fmt.Sprintf("docker:%s", containerID), - SQLResourceID: containerID, - }) - } - var ( - vmBatchMetrics map[string]map[string][]monitoring.MetricPoint - containerBatchMetrics map[string]map[string][]monitoring.MetricPoint - podBatchMetrics map[string]map[string][]monitoring.MetricPoint - dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint - ) - var workloadsSummaryBatchWG sync.WaitGroup - workloadsSummaryBatchWG.Add(4) - go func() { - defer workloadsSummaryBatchWG.Done() - vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, workloadSummaryMetricOrder...) - }() - go func() { - defer workloadsSummaryBatchWG.Done() - containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, workloadSummaryMetricOrder...) - }() - go func() { - defer workloadsSummaryBatchWG.Done() - podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...) - }() - go func() { - defer workloadsSummaryBatchWG.Done() - dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, workloadSummaryMetricOrder...) - }() - workloadsSummaryBatchWG.Wait() - - var guestSummaryPoints int - snapshots, guestSummaryPoints = appendGuestWorkloadSummaries(vmList, vmResponseKeys, vmRequests, vmBatchMetrics, currentTimeTime, &guestCounts, buckets, snapshots, &oldestTimestamp) - guestPointCount += guestSummaryPoints - - snapshots, guestSummaryPoints = appendGuestWorkloadSummaries(containerList, containerResponseKeys, containerRequests, containerBatchMetrics, currentTimeTime, &guestCounts, buckets, snapshots, &oldestTimestamp) - guestPointCount += guestSummaryPoints - - for _, pod := range podList { - metricKey := kubernetesPodMetricIDFromView(pod) - - guestCounts.Total++ - if strings.EqualFold(pod.PodPhase(), "running") { - guestCounts.Running++ - } else { - guestCounts.Stopped++ - } - - snapshot := workloadsSummarySnapshot{ - id: metricKey, - name: strings.TrimSpace(pod.Namespace()), - cpu: clampWorkloadPercent(pod.CPUPercent()), - memory: clampWorkloadPercent(pod.MemoryPercent()), - disk: clampWorkloadPercent(pod.DiskPercent()), - network: clampNonNegativeWorkloadValue(pod.NetInRate() + pod.NetOutRate()), - } - if name := strings.TrimSpace(pod.Name()); name != "" { - if snapshot.name == "" { - snapshot.name = name - } else { - snapshot.name = fmt.Sprintf("%s/%s", snapshot.name, name) - } - } - if snapshot.name == "" { - snapshot.name = metricKey - } - - metrics := podBatchMetrics[metricKey] - cpuPoints := metrics["cpu"] - if len(cpuPoints) == 0 { - cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.CPUPercent()}} - } - memoryPoints := metrics["memory"] - if len(memoryPoints) == 0 { - memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.MemoryPercent()}} - } - diskPoints := metrics["disk"] - if len(diskPoints) == 0 { - diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.DiskPercent()}} - } - netInPoints := metrics["netin"] - if len(netInPoints) == 0 { - netInPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.NetInRate()}} - } - netOutPoints := metrics["netout"] - if len(netOutPoints) == 0 { - netOutPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: pod.NetOutRate()}} - } - - if mockModeEnabled { - if len(cpuPoints) < mockWorkloadMinSeriesPoints { - cpuPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "cpu", snapshot.cpu) - } - if len(memoryPoints) < mockWorkloadMinSeriesPoints { - memoryPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "memory", snapshot.memory) - } - if len(diskPoints) < mockWorkloadMinSeriesPoints { - diskPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "disk", snapshot.disk) - } - if len(netInPoints) < mockWorkloadMinSeriesPoints { - netInPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "netin", pod.NetInRate()) - } - if len(netOutPoints) < mockWorkloadMinSeriesPoints { - netOutPoints = buildMockWorkloadMetricHistorySeries(currentTimeTime, duration, 0, "k8s", metricKey, "netout", pod.NetOutRate()) - } - } - - networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) - - snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) - snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) - snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) - snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) - - guestPointCount += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, diskPoints, "disk", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, networkPoints, "network", &oldestTimestamp) - snapshots = append(snapshots, snapshot) - } - - for _, container := range dockerContainerList { - containerID := strings.TrimSpace(container.ContainerID()) - guestCounts.Total++ - containerState := strings.TrimSpace(container.ContainerState()) - isRunning := workloadSummaryStatusIsRunning(containerState, container.Status()) - if !isRunning && containerState == "" { - isRunning = container.CPUPercent() > 0 || - container.MemoryPercent() > 0 || - container.NetInRate() > 0 || - container.NetOutRate() > 0 - } - if isRunning { - guestCounts.Running++ - } else { - guestCounts.Stopped++ - } - - snapshot := workloadsSummarySnapshot{ - id: containerID, - name: strings.TrimSpace(container.Name()), - cpu: clampWorkloadPercent(container.CPUPercent()), - memory: clampWorkloadPercent(container.MemoryPercent()), - disk: clampWorkloadPercent(container.DiskPercent()), - network: 0, - } - if snapshot.name == "" { - snapshot.name = containerID - } - - metrics := dockerContainerBatchMetrics[containerID] - cpuPoints := metrics["cpu"] - if len(cpuPoints) == 0 { - cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.CPUPercent()}} - } - memoryPoints := metrics["memory"] - if len(memoryPoints) == 0 { - memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.MemoryPercent()}} - } - diskPoints := metrics["disk"] - if len(diskPoints) == 0 { - diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: container.DiskPercent()}} - } - netInPoints := metrics["netin"] - netOutPoints := metrics["netout"] - - networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) - - snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) - snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) - snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) - snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) - - guestPointCount += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, diskPoints, "disk", &oldestTimestamp) - guestPointCount += appendWorkloadMetricPoints(buckets, networkPoints, "network", &oldestTimestamp) - snapshots = append(snapshots, snapshot) - } - - cpuMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { - return bucket.cpu - }) - memoryMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { - return bucket.memory - }) - diskMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { - return bucket.disk - }) - networkMetric := buildWorkloadsSummaryMetric(buckets, func(bucket *workloadSummaryBuckets) workloadSummaryMetricBucket { - return bucket.network - }) - cpuMetric = normalizeWorkloadsSummaryMetricPointSeries(cpuMetric, duration) - memoryMetric = normalizeWorkloadsSummaryMetricPointSeries(memoryMetric, duration) - diskMetric = normalizeWorkloadsSummaryMetricPointSeries(diskMetric, duration) - networkMetric = normalizeWorkloadsSummaryMetricPointSeries(networkMetric, duration) - - summaryPointCount := summaryMetricPointCount(cpuMetric) + - summaryMetricPointCount(memoryMetric) + - summaryMetricPointCount(diskMetric) + - summaryMetricPointCount(networkMetric) - - topContributors := WorkloadsSummaryContributors{ - CPU: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.cpu - }), - Memory: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.memory - }), - Disk: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.disk - }), - Network: buildWorkloadsTopContributors(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.network - }), - } - - blastRadius := WorkloadsSummaryBlastRadiusGroup{ - CPU: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.cpu - }), - Memory: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.memory - }), - Disk: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.disk - }), - Network: buildWorkloadsBlastRadius(snapshots, func(snapshot workloadsSummarySnapshot) float64 { - return snapshot.network - }), - } - - response := EmptyWorkloadsSummaryChartsResponse() - response.CPU = cpuMetric - response.Memory = memoryMetric - response.Disk = diskMetric - response.Network = networkMetric - response.GuestCounts = guestCounts - response.TopContributors = topContributors - response.BlastRadius = blastRadius - response.Timestamp = currentTime - response.Stats = ChartStats{ - OldestDataTimestamp: oldestTimestamp, - Range: timeRange, - RangeSeconds: int64(duration / time.Second), - MetricsStoreEnabled: metricsStoreEnabled, - PrimarySourceHint: primarySourceHint, - InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), - PointCounts: ChartPointCounts{ - Total: summaryPointCount, - Guests: guestPointCount, - }, - } - - payload, err := json.Marshal(response.NormalizeCollections()) - if err != nil { - log.Error().Err(err).Msg("Failed to encode workloads summary chart data response") - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - r.cacheWorkloadsSummaryChartsPayload(cacheKey, payload, now) - - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(payload); err != nil { - log.Error().Err(err).Msg("Failed to encode workloads summary chart data response") - return - } -} - -// guestWorkloadSummaryView is the guest view subset the workloads summary -// loop consumes from VMs and LXC containers. -type guestWorkloadSummaryView interface { - Status() unifiedresources.ResourceStatus - Name() string - CPUPercent() float64 - MemoryPercent() float64 - DiskPercent() float64 - NetIn() float64 - NetOut() float64 -} - -// appendGuestWorkloadSummaries accumulates workload-summary snapshots and -// chart points for one proxmox guest family (VMs or LXC containers), -// returning the extended snapshot slice and the number of points added. -func appendGuestWorkloadSummaries[V guestWorkloadSummaryView]( - guests []V, - responseKeys []string, - requests []monitoring.GuestChartRequest, - batchMetrics map[string]map[string][]monitoring.MetricPoint, - currentTimeTime time.Time, - guestCounts *WorkloadsGuestCounts, - buckets map[int64]*workloadSummaryBuckets, - snapshots []workloadsSummarySnapshot, - oldestTimestamp *int64, -) ([]workloadsSummarySnapshot, int) { - added := 0 - for idx, g := range guests { - responseKey := responseKeys[idx] - metricID := requests[idx].SQLResourceID - guestCounts.Total++ - if workloadSummaryStatusIsRunning("", g.Status()) { - guestCounts.Running++ - } else { - guestCounts.Stopped++ - } - - snapshot := workloadsSummarySnapshot{ - id: responseKey, - name: strings.TrimSpace(g.Name()), - cpu: clampWorkloadPercent(g.CPUPercent()), - memory: clampWorkloadPercent(g.MemoryPercent()), - disk: clampWorkloadPercent(g.DiskPercent()), - network: clampNonNegativeWorkloadValue(g.NetIn() + g.NetOut()), - } - if snapshot.name == "" { - snapshot.name = responseKey - } - - metrics := batchMetrics[metricID] - cpuPoints := metrics["cpu"] - if len(cpuPoints) == 0 { - cpuPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.CPUPercent()}} - } - memoryPoints := metrics["memory"] - if len(memoryPoints) == 0 { - memoryPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.MemoryPercent()}} - } - diskPoints := metrics["disk"] - if len(diskPoints) == 0 { - diskPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.DiskPercent()}} - } - netInPoints := metrics["netin"] - netOutPoints := metrics["netout"] - if len(netInPoints) == 0 && len(netOutPoints) == 0 { - netInPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.NetIn()}} - netOutPoints = []monitoring.MetricPoint{{Timestamp: currentTimeTime, Value: g.NetOut()}} - } - - networkPoints := mergeWorkloadNetworkPoints(netInPoints, netOutPoints) - - snapshot.cpu = latestSummaryMetricValue(cpuPoints, snapshot.cpu, clampWorkloadPercent) - snapshot.memory = latestSummaryMetricValue(memoryPoints, snapshot.memory, clampWorkloadPercent) - snapshot.disk = latestSummaryMetricValue(diskPoints, snapshot.disk, clampWorkloadPercent) - snapshot.network = latestSummaryMetricValue(networkPoints, snapshot.network, clampNonNegativeWorkloadValue) - - added += appendWorkloadMetricPoints(buckets, cpuPoints, "cpu", oldestTimestamp) - added += appendWorkloadMetricPoints(buckets, memoryPoints, "memory", oldestTimestamp) - added += appendWorkloadMetricPoints(buckets, diskPoints, "disk", oldestTimestamp) - added += appendWorkloadMetricPoints(buckets, networkPoints, "network", oldestTimestamp) - snapshots = append(snapshots, snapshot) - } - return snapshots, added -} - -func workloadSummaryStatusIsRunning(runtimeState string, status unifiedresources.ResourceStatus) bool { - switch strings.ToLower(strings.TrimSpace(runtimeState)) { - case "running", "online", "ok": - return true - case "stopped", "offline", "paused", "created", "dead", "exited": - return false - } - - switch status { - case unifiedresources.StatusOnline: - return true - case unifiedresources.StatusWarning: - // Warning is an attention state on a running workload (degraded - // guest state, stale source data); power-off maps to StatusOffline. - return true - case unifiedresources.StatusOffline: - return false - } - - return false -} - -// handleStorageCharts returns pool capacity and physical disk temperature -// time-series for the storage summary sparklines. -func (r *Router) handleStorageCharts(w http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse query parameters - query := req.URL.Query() - rangeMinutes := 60 // default 1 hour - if rangeStr := query.Get("range"); rangeStr != "" { - if _, err := fmt.Sscanf(rangeStr, "%d", &rangeMinutes); err != nil { - log.Warn().Err(err).Str("range", rangeStr).Msg("Invalid range parameter; using default") - } - } - - duration := time.Duration(rangeMinutes) * time.Minute - selectedNodeID := strings.TrimSpace(query.Get("node")) - - // Use tenant-aware monitor - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Monitor not available", http.StatusInternalServerError) - return - } - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - http.Error(w, "State unavailable", http.StatusInternalServerError) - return - } - - // Resolve node filter from canonical unified resources so storage charts use - // the same node identity model as the frontend storage page. - var selectedNodeName, selectedNodeInstance string - if selectedNodeID != "" { - found := false - for _, resource := range monitor.GetUnifiedResources() { - if strings.TrimSpace(resource.ID) != selectedNodeID { - continue - } - selectedNodeName = storageChartsSelectedNodeName(resource) - selectedNodeInstance = storageChartsSelectedNodeInstance(resource) - if selectedNodeName != "" || selectedNodeInstance != "" { - found = true - break - } - } - if !found { - for _, n := range monitor.NodesSnapshot() { - if n.ID == selectedNodeID { - selectedNodeName = n.Name - selectedNodeInstance = n.Instance - found = true - break - } - } - } - if !found { - log.Debug(). - Str("selectedNodeID", selectedNodeID). - Msg("Storage charts node filter not found in current state; falling back to global scope") - } - } - matchesNode := func(nodeName, instance string) bool { - if selectedNodeName == "" { - return true - } - if !strings.EqualFold(strings.TrimSpace(nodeName), selectedNodeName) { - return false - } - if selectedNodeInstance != "" && instance != "" { - return strings.EqualFold(strings.TrimSpace(instance), selectedNodeInstance) - } - return true - } - - // Build pool chart data from the canonical storage summary batch path so - // the dashboard and storage page share one efficient history retrieval model. - poolNames := make(map[string]string, len(readState.StoragePools())) - storageIDs := make([]string, 0, len(readState.StoragePools())) - for _, sp := range readState.StoragePools() { - if sp == nil { - continue - } - if !matchesNode(sp.Node(), sp.Instance()) { - continue - } - sid := sp.SourceID() - if sid == "" { - continue - } - poolNames[sid] = sp.Name() - storageIDs = append(storageIDs, sid) - } - - poolMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration) - pools := make(map[string]StoragePoolChartData, len(storageIDs)) - for _, sid := range storageIDs { - metrics := poolMetrics[sid] - pools[sid] = StoragePoolChartData{ - Name: poolNames[sid], - Usage: monitorPointsToAPI(metrics["usage"]), - Used: monitorPointsToAPI(metrics["used"]), - Avail: monitorPointsToAPI(metrics["avail"]), - } - } - - // Build disk temperature chart data - diskEntries := monitor.GetPhysicalDiskTemperatureCharts(duration) - disks := make(map[string]StorageDiskChartData, len(diskEntries)) - for id, entry := range diskEntries { - if !matchesNode(entry.Node, entry.Instance) { - continue - } - disks[id] = StorageDiskChartData{ - Name: entry.Name, - Node: entry.Node, - Temperature: monitorPointsToAPI(entry.Temperature), - } - } - - resp := EmptyStorageChartsResponse() - resp.Pools = pools - resp.Disks = disks - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(resp.NormalizeCollections()); err != nil { - log.Error().Err(err).Msg("Failed to encode storage chart data") - http.Error(w, "Internal server error", http.StatusInternalServerError) - } -} - -// handleStorageSummaryCharts serves a compact aggregate capacity trend for the -// dashboard storage card. It intentionally avoids returning per-pool and -// per-disk series so the dashboard does not overfetch the full storage page -// payload. -func (r *Router) handleStorageSummaryCharts(w http.ResponseWriter, req *http.Request) { - const inMemoryChartThreshold = 2 * time.Hour - - if req.Method != http.MethodGet && req.Method != http.MethodHead { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - query := req.URL.Query() - timeRange := query.Get("range") - if timeRange == "" { - timeRange = "24h" - } - duration := parseChartsRangeDuration(timeRange) - - monitor := r.getTenantMonitor(req.Context()) - if monitor == nil { - http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError) - return - } - - currentTime := time.Now().UnixMilli() - capacity, oldestTimestamp := monitor.GetStorageSummaryCapacityTrend(duration) - if oldestTimestamp == 0 { - oldestTimestamp = currentTime - } - - metricsStoreEnabled := monitor.GetMetricsStore() != nil - primarySourceHint := "memory" - if metricsStoreEnabled && duration > inMemoryChartThreshold { - primarySourceHint = "store_or_memory_fallback" - } - - resp := EmptyStorageSummaryTrendResponse() - resp.Capacity = monitorPointsToAPI(capacity) - resp.Timestamp = currentTime - resp.Stats = ChartStats{ - OldestDataTimestamp: oldestTimestamp, - Range: timeRange, - RangeSeconds: int64(duration / time.Second), - MetricsStoreEnabled: metricsStoreEnabled, - PrimarySourceHint: primarySourceHint, - InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second), - PointCounts: ChartPointCounts{ - Total: len(resp.Capacity), - Storage: len(resp.Capacity), - }, - } - - w.Header().Set("Content-Type", "application/json") - if err := json.NewEncoder(w).Encode(resp.NormalizeCollections()); err != nil { - log.Error().Err(err).Msg("Failed to encode storage summary chart data") - http.Error(w, "Internal server error", http.StatusInternalServerError) - } -} - -// monitorPointsToAPI converts monitoring MetricPoints (time.Time timestamps) -// to API MetricPoints (Unix millisecond timestamps) for JSON serialization. -func monitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint { - if len(points) == 0 { - return nil - } - out := make([]MetricPoint, len(points)) - for i, p := range points { - out[i] = MetricPoint{Timestamp: p.Timestamp.UnixMilli(), Value: p.Value} - } - return out -} - // handleMetricsStoreStats returns statistics about the persistent metrics store func (r *Router) handleMetricsStoreStats(w http.ResponseWriter, req *http.Request) { if req.Method != http.MethodGet { @@ -9447,7 +6290,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) // The in-memory history enforces its own retention limits, so it will naturally return // whatever data is available (better than showing "Collecting data..." indefinitely). fallbackAllowed := true - historyMaxPoints := parseWorkloadMaxPoints(query.Get("maxPoints")) + historyMaxPoints := chartapi.ParseWorkloadMaxPoints(query.Get("maxPoints")) buildHistoryPoints := func(points []monitoring.MetricPoint, bucketSecs int64) []map[string]interface{} { if len(points) == 0 { return []map[string]interface{}{} @@ -9717,7 +6560,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if vm == nil { return points } - points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: proxmoxModelCPURatioPercent(vm.CPU)} + points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: chartapi.ProxmoxModelCPURatioPercent(vm.CPU)} points["memory"] = monitoring.MetricPoint{Timestamp: now, Value: vm.Memory.Usage} if vm.Disk.Usage >= 0 { points["disk"] = monitoring.MetricPoint{Timestamp: now, Value: vm.Disk.Usage} @@ -9731,7 +6574,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if ct == nil { return points } - points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: proxmoxModelCPURatioPercent(ct.CPU)} + points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: chartapi.ProxmoxModelCPURatioPercent(ct.CPU)} points["memory"] = monitoring.MetricPoint{Timestamp: now, Value: ct.Memory.Usage} if ct.Disk.Usage >= 0 { points["disk"] = monitoring.MetricPoint{Timestamp: now, Value: ct.Disk.Usage} @@ -9745,7 +6588,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if node == nil { return points } - points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: proxmoxModelCPURatioPercent(node.CPU)} + points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: chartapi.ProxmoxModelCPURatioPercent(node.CPU)} points["memory"] = monitoring.MetricPoint{Timestamp: now, Value: node.Memory.Usage} points["disk"] = monitoring.MetricPoint{Timestamp: now, Value: node.Disk.Usage} if temperature := primaryNodeTemperatureCelsius(node.Temperature); temperature != nil { @@ -9800,7 +6643,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if node == nil { return points } - points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: proxmoxModelCPURatioPercent(node.CPU)} + points["cpu"] = monitoring.MetricPoint{Timestamp: now, Value: chartapi.ProxmoxModelCPURatioPercent(node.CPU)} points["memory"] = monitoring.MetricPoint{Timestamp: now, Value: node.Memory.Usage} points["disk"] = monitoring.MetricPoint{Timestamp: now, Value: node.Disk.Usage} if temperature := primaryNodeTemperatureCelsius(node.Temperature); temperature != nil { @@ -9878,7 +6721,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) current = float64(disk.PhysicalDisk.Temperature) } if current > 0 || metricType == "disk" || metricType == "diskread" || metricType == "diskwrite" { - series := buildSyntheticMetricHistorySeries( + series := chartapi.BuildSyntheticMetricHistorySeries( end, duration, historyMaxPoints, @@ -10183,7 +7026,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) if response == nil && mock.IsMockEnabled() && runtimeResourceType == "disk" && (metricType == "smart_temp" || metricType == "disk" || metricType == "diskread" || metricType == "diskwrite") { - targetPoints := targetMockSeriesPoints(duration, historyMaxPoints) + targetPoints := chartapi.TargetMockSeriesPoints(duration, historyMaxPoints) if len(points) > 0 && len(points) < targetPoints { current := points[len(points)-1].Value if metricType == "smart_temp" { @@ -10192,7 +7035,7 @@ func (r *Router) handleMetricsHistory(w http.ResponseWriter, req *http.Request) } } if metricType != "smart_temp" || current > 0 { - series := buildSyntheticMetricHistorySeries( + series := chartapi.BuildSyntheticMetricHistorySeries( end, duration, historyMaxPoints, @@ -11202,7 +8045,7 @@ func (a mockSupplementalRecordsAdapter) SupplementalChanges(_ *monitoring.Monito } func (a mockSupplementalRecordsAdapter) SnapshotOwnedSources() []unifiedresources.DataSource { - normalized := normalizeDataSourceAlias(a.source) + normalized := resourceapi.NormalizeDataSourceAlias(a.source) if normalized == "" { return nil } diff --git a/internal/api/router_misc_additional_test.go b/internal/api/router_misc_additional_test.go index 3acf17227..34dd145eb 100644 --- a/internal/api/router_misc_additional_test.go +++ b/internal/api/router_misc_additional_test.go @@ -14,7 +14,6 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" - "github.com/rcourtman/pulse-go-rewrite/internal/vmware" ) func newTestMonitor(t *testing.T) (*monitoring.Monitor, *models.State, *monitoring.MetricsHistory) { @@ -396,1213 +395,6 @@ func TestHandleStorage_Success(t *testing.T) { } } -func TestHandleCharts_Success(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}} - syncTestResourceStore(t, monitor, state) - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } -} - -func TestHandleCharts_StatsDebugMetadata(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.VMs = []models.VM{{ID: "vm-1", Name: "vm-one", CPU: 0.2}} - state.Nodes = []models.Node{{ID: "node-1", Name: "node-one", CPU: 0.1}} - state.Storage = []models.Storage{{ID: "store-1", Name: "Store One", Used: 50, Total: 100}} - syncTestResourceStore(t, monitor, state) - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - body := rec.Body.Bytes() - - var decoded ChartResponse - if err := json.Unmarshal(body, &decoded); err != nil { - t.Fatalf("unmarshal ChartResponse: %v", err) - } - - if decoded.Stats.Range != "5m" { - t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) - } - if decoded.Stats.RangeSeconds != 300 { - t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) - } - if decoded.Stats.MetricsStoreEnabled { - t.Fatalf("expected stats.metricsStoreEnabled=false in test monitor, got true") - } - if decoded.Stats.PrimarySourceHint != "memory" { - t.Fatalf("expected stats.primarySourceHint=memory, got %q", decoded.Stats.PrimarySourceHint) - } - if decoded.Stats.InMemoryThresholdSecs != 7200 { - t.Fatalf("expected stats.inMemoryThresholdSecs=7200, got %d", decoded.Stats.InMemoryThresholdSecs) - } - if decoded.Stats.OldestDataTimestamp <= 0 { - t.Fatalf("expected stats.oldestDataTimestamp to be set, got %d", decoded.Stats.OldestDataTimestamp) - } - if decoded.Stats.OldestDataTimestamp > decoded.Timestamp { - t.Fatalf( - "expected stats.oldestDataTimestamp <= timestamp, got oldest=%d timestamp=%d", - decoded.Stats.OldestDataTimestamp, - decoded.Timestamp, - ) - } - - // With no history in the test monitor, handleCharts falls back to synthetic points: - // guests: cpu/memory/memoryused/disk/netin/netout (6 — diskread/diskwrite excluded from sparkline payloads) - // nodes: cpu/memory/disk (3) - // storage: disk (1) - if decoded.Stats.PointCounts.Guests != 6 { - t.Fatalf("expected stats.pointCounts.guests=6, got %d", decoded.Stats.PointCounts.Guests) - } - if points := decoded.ChartData["vm-1"]["memoryused"]; len(points) != 1 || points[0].Value != 0 { - t.Fatalf("expected additive raw memory-used fallback point, got %+v", points) - } - if decoded.Stats.PointCounts.Nodes != 3 { - t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes) - } - if decoded.Stats.PointCounts.Storage != 1 { - t.Fatalf("expected stats.pointCounts.storage=1, got %d", decoded.Stats.PointCounts.Storage) - } - if decoded.Stats.PointCounts.DockerContainers != 0 || decoded.Stats.PointCounts.DockerHosts != 0 || decoded.Stats.PointCounts.Agents != 0 { - t.Fatalf( - "expected dockerContainers/dockerHosts/agents all 0, got dc=%d dh=%d agents=%d", - decoded.Stats.PointCounts.DockerContainers, - decoded.Stats.PointCounts.DockerHosts, - decoded.Stats.PointCounts.Agents, - ) - } - - sum := decoded.Stats.PointCounts.Guests + - decoded.Stats.PointCounts.Nodes + - decoded.Stats.PointCounts.Storage + - decoded.Stats.PointCounts.DockerContainers + - decoded.Stats.PointCounts.DockerHosts + - decoded.Stats.PointCounts.Agents - if decoded.Stats.PointCounts.Total != sum { - t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total) - } - - var raw map[string]interface{} - if err := json.Unmarshal(body, &raw); err != nil { - t.Fatalf("unmarshal raw JSON: %v", err) - } - stats, ok := raw["stats"].(map[string]interface{}) - if !ok { - t.Fatalf("expected stats object in JSON response") - } - if _, ok := stats["pointCounts"]; !ok { - t.Fatalf("expected stats.pointCounts to be present in JSON response") - } -} - -func TestHandleCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) { - setMockModeForTest(t, true) - - fixtures := vmware.DefaultFixtures() - if len(fixtures.Hosts) == 0 { - t.Fatal("expected default VMware fixtures to include at least one host") - } - expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host) - - monitor, state, _ := newTestMonitor(t) - state.Hosts = []models.Host{{ - ID: "live-store-host-1", - Hostname: "live-store-host-1", - CPUUsage: 11.0, - Memory: models.Memory{Usage: 22.0}, - Disks: []models.Disk{{Usage: 33.0}}, - Status: "online", - }} - syncTestResourceStore(t, monitor, state) - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) - } - - var decoded ChartResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal ChartResponse: %v", err) - } - - vmwareSeries, ok := decoded.AgentData[expectedHostID] - if !ok { - t.Fatalf("expected VMware host %q in full charts payload, got keys=%v", expectedHostID, mapKeys(decoded.AgentData)) - } - if len(vmwareSeries["cpu"]) == 0 { - t.Fatalf("expected VMware host %q cpu series", expectedHostID) - } - if _, ok := decoded.AgentData["live-store-host-1"]; ok { - t.Fatalf("expected mock-aware full charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData)) - } -} - -func TestHandleInfrastructureCharts_Lightweight(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-1", - Name: "node-one", - Status: "online", - CPU: 0.1, - Memory: models.Memory{Usage: 12.0}, - Disk: models.Disk{Usage: 34.0}, - }} - state.DockerHosts = []models.DockerHost{{ - ID: "docker-host-1", - Runtime: "docker", - CPUUsage: 23.0, - Memory: models.Memory{Usage: 45.0}, - Disks: []models.Disk{{Usage: 67.0}}, - Status: "online", - }} - state.Hosts = []models.Host{{ - ID: "host-1", - Hostname: "host-one", - CPUUsage: 11.0, - Memory: models.Memory{Usage: 22.0}, - Disks: []models.Disk{{Usage: 33.0}}, - Status: "online", - }} - syncTestResourceStore(t, monitor, state) - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleInfrastructureCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } - - body := rec.Body.Bytes() - - var decoded InfrastructureChartsResponse - if err := json.Unmarshal(body, &decoded); err != nil { - t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) - } - if decoded.Stats.Range != "5m" { - t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) - } - if decoded.Stats.RangeSeconds != 300 { - t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) - } - - // With no history in the test monitor, handler falls back to synthetic points: - // nodes: cpu/memory/disk (3) - // dockerHosts: cpu/memory/disk (3) - // agents: cpu/memory/disk (3) - if decoded.Stats.PointCounts.Nodes != 3 { - t.Fatalf("expected stats.pointCounts.nodes=3, got %d", decoded.Stats.PointCounts.Nodes) - } - if decoded.Stats.PointCounts.DockerHosts != 3 { - t.Fatalf("expected stats.pointCounts.dockerHosts=3, got %d", decoded.Stats.PointCounts.DockerHosts) - } - if decoded.Stats.PointCounts.Agents != 3 { - t.Fatalf("expected stats.pointCounts.agents=3, got %d", decoded.Stats.PointCounts.Agents) - } - sum := decoded.Stats.PointCounts.Nodes + decoded.Stats.PointCounts.DockerHosts + decoded.Stats.PointCounts.Agents - if decoded.Stats.PointCounts.Total != sum { - t.Fatalf("expected stats.pointCounts.total=%d, got %d", sum, decoded.Stats.PointCounts.Total) - } - - var raw map[string]interface{} - if err := json.Unmarshal(body, &raw); err != nil { - t.Fatalf("unmarshal raw JSON: %v", err) - } - for _, forbidden := range []string{"data", "storageData", "dockerData", "guestTypes"} { - if _, ok := raw[forbidden]; ok { - t.Fatalf("expected %q to be absent from infra summary response", forbidden) - } - } -} - -func TestHandleInfrastructureCharts_MetricFilter(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-1", - Name: "node-one", - Status: "online", - CPU: 0.1, - Memory: models.Memory{Usage: 12.0}, - Disk: models.Disk{Usage: 34.0}, - }} - state.DockerHosts = []models.DockerHost{{ - ID: "docker-host-1", - Runtime: "docker", - CPUUsage: 23.0, - Memory: models.Memory{Usage: 45.0}, - Disks: []models.Disk{{Usage: 67.0}}, - Status: "online", - }} - state.Hosts = []models.Host{{ - ID: "host-1", - Hostname: "host-one", - CPUUsage: 11.0, - Memory: models.Memory{Usage: 22.0}, - Disks: []models.Disk{{Usage: 33.0}}, - Status: "online", - }} - syncTestResourceStore(t, monitor, state) - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m&metrics=cpu,memory", nil) - rec := httptest.NewRecorder() - - router.handleInfrastructureCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded InfrastructureChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) - } - - if decoded.Stats.PointCounts.Nodes != 2 { - t.Fatalf("expected stats.pointCounts.nodes=2, got %d", decoded.Stats.PointCounts.Nodes) - } - if decoded.Stats.PointCounts.DockerHosts != 2 { - t.Fatalf("expected stats.pointCounts.dockerHosts=2, got %d", decoded.Stats.PointCounts.DockerHosts) - } - if decoded.Stats.PointCounts.Agents != 2 { - t.Fatalf("expected stats.pointCounts.agents=2, got %d", decoded.Stats.PointCounts.Agents) - } - if _, ok := decoded.NodeData["node-1"]["disk"]; ok { - t.Fatalf("expected disk series to be filtered out of node payload") - } - if _, ok := decoded.DockerHostData["docker-host-1"]["disk"]; ok { - t.Fatalf("expected disk series to be filtered out of docker host payload") - } - if _, ok := decoded.AgentData["host-1"]["disk"]; ok { - t.Fatalf("expected disk series to be filtered out of agent payload") - } -} - -func TestHandleInfrastructureCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing.T) { - setMockModeForTest(t, true) - - fixtures := vmware.DefaultFixtures() - if len(fixtures.Hosts) == 0 { - t.Fatal("expected default VMware fixtures to include at least one host") - } - expectedHostID := vmware.SourceID(fixtures.ConnectionID, "host", fixtures.Hosts[0].Host) - - monitor, state, _ := newTestMonitor(t) - state.Hosts = []models.Host{{ - ID: "live-store-host-1", - Hostname: "live-store-host-1", - CPUUsage: 11.0, - Memory: models.Memory{Usage: 22.0}, - Disks: []models.Disk{{Usage: 33.0}}, - Status: "online", - }} - syncTestResourceStore(t, monitor, state) - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/infrastructure?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleInfrastructureCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) - } - - var decoded InfrastructureChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal InfrastructureChartsResponse: %v", err) - } - - vmwareSeries, ok := decoded.AgentData[expectedHostID] - if !ok { - t.Fatalf("expected VMware host %q in infrastructure charts, got keys=%v", expectedHostID, mapKeys(decoded.AgentData)) - } - if len(vmwareSeries["cpu"]) == 0 { - t.Fatalf("expected VMware host %q cpu series", expectedHostID) - } - if _, ok := decoded.AgentData["live-store-host-1"]; ok { - t.Fatalf("expected mock-aware infrastructure charts to ignore live store-only host, got keys=%v", mapKeys(decoded.AgentData)) - } -} - -func TestHandleWorkloadsSummaryCharts_AggregatesAndCounts(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-pve-1", - Name: "pve-1", - Instance: "pve", - }} - state.VMs = []models.VM{{ - ID: "vm-101", - Name: "vm-101", - Node: "pve-1", - Instance: "pve", - Status: "running", - CPU: 0.25, - Memory: models.Memory{Usage: 40.0}, - Disk: models.Disk{Usage: 55.0}, - NetworkIn: 1200, - NetworkOut: 800, - }} - state.Containers = []models.Container{{ - ID: "ct-201", - Name: "ct-201", - Node: "pve-1", - Instance: "pve", - Status: "stopped", - CPU: 0.10, - Memory: models.Memory{Usage: 30.0}, - Disk: models.Disk{Usage: 45.0}, - NetworkIn: 400, - NetworkOut: 600, - }} - state.DockerHosts = []models.DockerHost{{ - ID: "docker-host-1", - Runtime: "docker", - Containers: []models.DockerContainer{{ - ID: "docker-1", - Name: "docker-1", - State: "running", - CPUPercent: 35.0, - MemoryPercent: 60.0, - WritableLayerBytes: 10, - RootFilesystemBytes: 100, - }}, - }} - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m", nil) - rec := httptest.NewRecorder() - - router.handleWorkloadsSummaryCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } - - body := rec.Body.Bytes() - - var decoded WorkloadsSummaryChartsResponse - if err := json.Unmarshal(body, &decoded); err != nil { - t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) - } - if decoded.Stats.Range != "5m" { - t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) - } - if decoded.Stats.RangeSeconds != 300 { - t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) - } - - if decoded.GuestCounts.Total != 3 { - t.Fatalf("expected guestCounts.total=3, got %d", decoded.GuestCounts.Total) - } - if decoded.GuestCounts.Running != 2 { - t.Fatalf("expected guestCounts.running=2, got %d", decoded.GuestCounts.Running) - } - if decoded.GuestCounts.Stopped != 1 { - t.Fatalf("expected guestCounts.stopped=1, got %d", decoded.GuestCounts.Stopped) - } - - for metricName, metric := range map[string]WorkloadsSummaryMetricData{ - "cpu": decoded.CPU, - "memory": decoded.Memory, - "disk": decoded.Disk, - "network": decoded.Network, - } { - if len(metric.P50) == 0 { - t.Fatalf("expected %s p50 points to be present", metricName) - } - if len(metric.P95) == 0 { - t.Fatalf("expected %s p95 points to be present", metricName) - } - } - - if decoded.Stats.PointCounts.Total <= 0 { - t.Fatalf("expected stats.pointCounts.total to be > 0, got %d", decoded.Stats.PointCounts.Total) - } - if decoded.Stats.PointCounts.Guests <= 0 { - t.Fatalf("expected stats.pointCounts.guests to be > 0, got %d", decoded.Stats.PointCounts.Guests) - } - - if decoded.BlastRadius.CPU.Scope != "concentrated" { - t.Fatalf("expected cpu blast radius concentrated, got %q", decoded.BlastRadius.CPU.Scope) - } - if decoded.BlastRadius.Network.Scope != "concentrated" { - t.Fatalf("expected network blast radius concentrated, got %q", decoded.BlastRadius.Network.Scope) - } - - // Node-scoped request should only include workloads that match the selected node. - nodeReq := httptest.NewRequest( - http.MethodGet, - "/api/charts/workloads-summary?range=5m&node=node-pve-1", - nil, - ) - nodeRec := httptest.NewRecorder() - - router.handleWorkloadsSummaryCharts(nodeRec, nodeReq) - - if nodeRec.Code != http.StatusOK { - t.Fatalf("node-scoped expected status %d, got %d", http.StatusOK, nodeRec.Code) - } - - var nodeScoped WorkloadsSummaryChartsResponse - if err := json.Unmarshal(nodeRec.Body.Bytes(), &nodeScoped); err != nil { - t.Fatalf("unmarshal node-scoped WorkloadsSummaryChartsResponse: %v", err) - } - if nodeScoped.GuestCounts.Total != 2 { - t.Fatalf("expected node-scoped guestCounts.total=2, got %d", nodeScoped.GuestCounts.Total) - } - if nodeScoped.GuestCounts.Running != 1 { - t.Fatalf("expected node-scoped guestCounts.running=1, got %d", nodeScoped.GuestCounts.Running) - } - if nodeScoped.GuestCounts.Stopped != 1 { - t.Fatalf("expected node-scoped guestCounts.stopped=1, got %d", nodeScoped.GuestCounts.Stopped) - } - if nodeScoped.BlastRadius.CPU.Scope != "concentrated" { - t.Fatalf("expected node-scoped cpu blast radius concentrated, got %q", nodeScoped.BlastRadius.CPU.Scope) - } -} - -func TestHandleWorkloadCharts_WorkloadOnlyPayloadAndNodeFilter(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-pve-1", - Name: "pve-1", - Instance: "pve", - }} - state.VMs = []models.VM{ - { - ID: "vm-101", - Name: "vm-101", - Node: "pve-1", - Instance: "pve", - CPU: 0.2, - Memory: models.Memory{Usage: 55}, - Disk: models.Disk{Usage: 40}, - }, - { - ID: "vm-202", - Name: "vm-202", - Node: "other", - Instance: "other", - CPU: 0.4, - Memory: models.Memory{Usage: 65}, - Disk: models.Disk{Usage: 70}, - }, - } - state.Containers = []models.Container{ - { - ID: "ct-301", - Name: "ct-301", - Node: "pve-1", - Instance: "pve", - CPU: 0.3, - Memory: models.Memory{Usage: 40}, - Disk: models.Disk{Usage: 25}, - }, - } - state.DockerHosts = []models.DockerHost{{ - ID: "docker-host-1", - Hostname: "pve-1", - DisplayName: "pve-1", - Containers: []models.DockerContainer{{ - ID: "docker-401", - Name: "docker-401", - CPUPercent: 22, - MemoryPercent: 30, - }}, - }} - - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil { - t.Fatal("expected unified read state") - } - - vmResponseKey := "" - containerResponseKey := "" - excludedResponseKey := "" - for _, vm := range readState.VMs() { - if vm == nil { - continue - } - switch vm.Name() { - case "vm-101": - vmResponseKey, _, _ = vmChartRequest(vm) - case "vm-202": - excludedResponseKey, _, _ = vmChartRequest(vm) - } - } - for _, ct := range readState.Containers() { - if ct == nil { - continue - } - if ct.Name() == "ct-301" { - containerResponseKey, _, _ = systemContainerChartRequest(ct) - } - } - if vmResponseKey == "" || containerResponseKey == "" || excludedResponseKey == "" { - t.Fatalf( - "expected canonical response keys for vm/container test fixtures, got vm=%q ct=%q excluded=%q", - vmResponseKey, - containerResponseKey, - excludedResponseKey, - ) - } - - router := &Router{monitor: monitor} - - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil) - rec := httptest.NewRecorder() - router.handleWorkloadCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } - - var decoded WorkloadChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) - } - - if decoded.Stats.Range != "5m" { - t.Fatalf("expected stats.range=5m, got %q", decoded.Stats.Range) - } - if decoded.Stats.RangeSeconds != 300 { - t.Fatalf("expected stats.rangeSeconds=300, got %d", decoded.Stats.RangeSeconds) - } - if len(decoded.ChartData) != 3 { - t.Fatalf("expected 3 workload chart entries, got %d", len(decoded.ChartData)) - } - if len(decoded.DockerData) != 1 { - t.Fatalf("expected 1 docker chart entry, got %d", len(decoded.DockerData)) - } - if decoded.GuestTypes[vmResponseKey] != "vm" { - t.Fatalf("expected vm guest type for %s, got %q", vmResponseKey, decoded.GuestTypes[vmResponseKey]) - } - if decoded.GuestTypes[containerResponseKey] != "system-container" { - t.Fatalf( - "expected system-container guest type for %s, got %q", - containerResponseKey, - decoded.GuestTypes[containerResponseKey], - ) - } - if decoded.Stats.PointCounts.Total <= 0 { - t.Fatalf("expected stats.pointCounts.total > 0, got %d", decoded.Stats.PointCounts.Total) - } - - // Node-scoped request should only include workloads linked to node-pve-1 - // plus docker containers running on matching docker hosts. - nodeReq := httptest.NewRequest( - http.MethodGet, - "/api/charts/workloads?range=5m&node=node-pve-1", - nil, - ) - nodeRec := httptest.NewRecorder() - router.handleWorkloadCharts(nodeRec, nodeReq) - - if nodeRec.Code != http.StatusOK { - t.Fatalf("node-scoped expected status %d, got %d", http.StatusOK, nodeRec.Code) - } - - var scoped WorkloadChartsResponse - if err := json.Unmarshal(nodeRec.Body.Bytes(), &scoped); err != nil { - t.Fatalf("unmarshal node-scoped WorkloadChartsResponse: %v", err) - } - if len(scoped.ChartData) != 2 { - t.Fatalf("expected 2 scoped workload chart entries, got %d", len(scoped.ChartData)) - } - if _, ok := scoped.ChartData[excludedResponseKey]; ok { - t.Fatalf("expected %s to be excluded by node scope", excludedResponseKey) - } - if len(scoped.DockerData) != 1 { - t.Fatalf("expected 1 scoped docker chart entry, got %d", len(scoped.DockerData)) - } -} - -func TestHandleWorkloadCharts_UsesCanonicalWorkloadIDsForVMwareVMs(t *testing.T) { - monitor, state, history := newTestMonitor(t) - now := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Minute) - metricID := "vc-1:vm:vm-201" - resourceID := "vm-vmware-1" - - history.AddGuestMetric(metricID, "cpu", 37, now.Add(-10*time.Minute)) - history.AddGuestMetric(metricID, "memory", 62, now.Add(-5*time.Minute)) - - adapter := unifiedresources.NewMonitorAdapter(nil) - adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ - unifiedresources.SourceVMware: { - { - SourceID: metricID, - Resource: unifiedresources.Resource{ - ID: resourceID, - Type: unifiedresources.ResourceTypeVM, - Name: "warehouse-api-01", - Status: unifiedresources.StatusOnline, - LastSeen: now, - MetricsTarget: &unifiedresources.MetricsTarget{ - ResourceType: "vm", - ResourceID: metricID, - }, - VMware: &unifiedresources.VMwareData{ - ConnectionID: "vc-1", - EntityType: "vm", - ManagedObjectID: "vm-201", - }, - }, - }, - }, - }) - setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) - - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil { - t.Fatalf("expected one VMware VM in unified read state, got %+v", readState) - } - resourceID, _, ok := vmChartRequest(readState.VMs()[0]) - if !ok { - t.Fatal("expected canonical VMware vm chart request") - } - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=1h", nil) - rec := httptest.NewRecorder() - router.handleWorkloadCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) - } - - var decoded WorkloadChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) - } - - series, ok := decoded.ChartData[resourceID] - if !ok { - t.Fatalf("expected VMware workload chart keyed by canonical workload id %q, got %v", resourceID, decoded.ChartData) - } - if _, ok := decoded.ChartData[metricID]; ok { - t.Fatalf("expected VMware metrics target id %q to stay out of workload chart response keys", metricID) - } - if decoded.GuestTypes[resourceID] != "vm" { - t.Fatalf("expected guest type vm for %q, got %q", resourceID, decoded.GuestTypes[resourceID]) - } - if len(series["cpu"]) == 0 { - t.Fatalf("expected VMware cpu series for %q", resourceID) - } - if got := series["cpu"][len(series["cpu"])-1].Value; got != 37 { - t.Fatalf("expected latest VMware cpu value 37, got %v", got) - } -} - -func TestHandleWorkloadsSummaryCharts_UsesCanonicalWorkloadIDsForVMwareVMs(t *testing.T) { - monitor, state, history := newTestMonitor(t) - now := time.Now().UTC().Add(-10 * time.Minute).Truncate(time.Minute) - metricID := "vc-1:vm:vm-201" - resourceID := "vm-vmware-1" - - history.AddGuestMetric(metricID, "cpu", 51, now.Add(-10*time.Minute)) - history.AddGuestMetric(metricID, "memory", 64, now.Add(-5*time.Minute)) - history.AddGuestMetric(metricID, "disk", 43, now.Add(-3*time.Minute)) - history.AddGuestMetric(metricID, "netin", 1200, now.Add(-2*time.Minute)) - history.AddGuestMetric(metricID, "netout", 800, now.Add(-2*time.Minute)) - - adapter := unifiedresources.NewMonitorAdapter(nil) - adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ - unifiedresources.SourceVMware: { - { - SourceID: metricID, - Resource: unifiedresources.Resource{ - ID: resourceID, - Type: unifiedresources.ResourceTypeVM, - Name: "warehouse-api-01", - Status: unifiedresources.StatusOnline, - LastSeen: now, - MetricsTarget: &unifiedresources.MetricsTarget{ - ResourceType: "vm", - ResourceID: metricID, - }, - VMware: &unifiedresources.VMwareData{ - ConnectionID: "vc-1", - EntityType: "vm", - ManagedObjectID: "vm-201", - }, - }, - }, - }, - }) - setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) - - readState := monitor.GetUnifiedReadStateOrSnapshot() - if readState == nil || len(readState.VMs()) != 1 || readState.VMs()[0] == nil { - t.Fatalf("expected one VMware VM in unified read state, got %+v", readState) - } - resourceID, _, ok := vmChartRequest(readState.VMs()[0]) - if !ok { - t.Fatal("expected canonical VMware vm chart request") - } - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=1h", nil) - rec := httptest.NewRecorder() - router.handleWorkloadsSummaryCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d: %s", http.StatusOK, rec.Code, rec.Body.String()) - } - - var decoded WorkloadsSummaryChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) - } - - if decoded.GuestCounts.Total != 1 || decoded.GuestCounts.Running != 1 { - t.Fatalf("expected guestCounts total/running = 1/1, got %+v", decoded.GuestCounts) - } - if len(decoded.TopContributors.CPU) == 0 { - t.Fatal("expected at least one cpu top contributor") - } - if decoded.TopContributors.CPU[0].ID != resourceID { - t.Fatalf("expected cpu top contributor id %q, got %+v", resourceID, decoded.TopContributors.CPU[0]) - } - if decoded.TopContributors.CPU[0].ID == metricID { - t.Fatalf("expected workloads summary contributor id to avoid raw metrics target %q", metricID) - } - if got := decoded.TopContributors.CPU[0].Name; got != "warehouse-api-01" { - t.Fatalf("expected cpu top contributor name warehouse-api-01, got %q", got) - } -} - -func TestHandleWorkloadCharts_IncludesKubernetesPods(t *testing.T) { - setMockModeForTest(t, false) - - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-pve-1", - Name: "pve-1", - Instance: "pve", - }} - state.KubernetesClusters = []models.KubernetesCluster{{ - ID: "cluster-k8s-summary-test", - Name: "cluster-k8s-summary-test", - Status: "online", - Pods: []models.KubernetesPod{ - { - UID: "pod-001", - Name: "api-0", - Namespace: "default", - NodeName: "pve-1", - Phase: "Running", - Containers: []models.KubernetesPodContainer{ - {Name: "api", Ready: true}, - }, - }, - }, - }} - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m", nil) - rec := httptest.NewRecorder() - router.handleWorkloadCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded WorkloadChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) - } - - var metricID string - for id, typ := range decoded.GuestTypes { - if typ == "k8s" { - metricID = id - break - } - } - if metricID == "" { - t.Fatal("expected at least one kubernetes pod series") - } - - series, ok := decoded.ChartData[metricID] - if !ok { - t.Fatalf("expected kubernetes pod series for %s", metricID) - } - if len(series["cpu"]) == 0 { - t.Fatalf("expected kubernetes cpu points for %s", metricID) - } - if got := len(series["disk"]); got != 1 || series["disk"][0].Value != 0 { - t.Fatalf("expected unsupported kubernetes pod disk metric fallback to single zero point; got len=%d value=%v", got, series["disk"]) - } - if got := len(series["netin"]); got != 1 || series["netin"][0].Value != 0 { - t.Fatalf("expected unsupported kubernetes pod network metric fallback to single zero point; got len=%d value=%v", got, series["netin"]) - } -} - -func TestHandleWorkloadsSummaryCharts_IncludesKubernetesPods(t *testing.T) { - setMockModeForTest(t, false) - - monitor, state, _ := newTestMonitor(t) - state.KubernetesClusters = []models.KubernetesCluster{{ - ID: "cluster-alpha", - Name: "cluster-alpha", - Status: "online", - Pods: []models.KubernetesPod{ - { - UID: "pod-001", - Name: "api-0", - Namespace: "default", - Phase: "Running", - Containers: []models.KubernetesPodContainer{ - {Name: "api", Ready: true}, - }, - }, - { - UID: "pod-002", - Name: "batch-0", - Namespace: "ops", - Phase: "Failed", - Containers: []models.KubernetesPodContainer{ - {Name: "worker", Ready: false}, - }, - }, - }, - }} - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m", nil) - rec := httptest.NewRecorder() - router.handleWorkloadsSummaryCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded WorkloadsSummaryChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) - } - - if decoded.GuestCounts.Total != 2 { - t.Fatalf("expected guestCounts.total=2, got %d", decoded.GuestCounts.Total) - } - if decoded.GuestCounts.Running != 1 { - t.Fatalf("expected guestCounts.running=1, got %d", decoded.GuestCounts.Running) - } - if decoded.GuestCounts.Stopped != 1 { - t.Fatalf("expected guestCounts.stopped=1, got %d", decoded.GuestCounts.Stopped) - } - if len(decoded.CPU.P50) == 0 { - t.Fatal("expected cpu summary points for kubernetes workloads") - } -} - -func TestHandleWorkloadCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-pve-1", - Name: "pve-1", - Instance: "pve", - }} - state.VMs = []models.VM{{ - ID: "vm-101", - Name: "vm-101", - Node: "pve-1", - Instance: "pve", - CPU: 0.3, - Memory: models.Memory{Total: 100, Used: 42, Free: 58, Usage: 42}, - Disk: models.Disk{Usage: 55}, - }} - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads?range=5m&node=missing-node-id", nil) - rec := httptest.NewRecorder() - - router.handleWorkloadCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded WorkloadChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadChartsResponse: %v", err) - } - if len(decoded.ChartData) == 0 { - t.Fatalf("expected fallback to global scope when node filter is stale") - } - for _, series := range decoded.ChartData { - if points := series["memoryused"]; len(points) != 1 || points[0].Value != 42 { - t.Fatalf("expected raw memory-used workload fallback point, got %+v", points) - } - } -} - -func TestHandleWorkloadsSummaryCharts_UnknownNodeFilterFallsBackToGlobalScope(t *testing.T) { - monitor, state, _ := newTestMonitor(t) - state.Nodes = []models.Node{{ - ID: "node-pve-1", - Name: "pve-1", - Instance: "pve", - }} - state.VMs = []models.VM{{ - ID: "vm-101", - Name: "vm-101", - Node: "pve-1", - Instance: "pve", - CPU: 0.3, - Memory: models.Memory{Usage: 42}, - Disk: models.Disk{Usage: 55}, - Status: "running", - }} - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/workloads-summary?range=5m&node=missing-node-id", nil) - rec := httptest.NewRecorder() - - router.handleWorkloadsSummaryCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded WorkloadsSummaryChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal WorkloadsSummaryChartsResponse: %v", err) - } - if decoded.GuestCounts.Total == 0 { - t.Fatalf("expected fallback to global scope when summary node filter is stale") - } -} - -func TestHandleStorageCharts_Success(t *testing.T) { - monitor, state, metricsHistory := newTestMonitor(t) - state.Storage = []models.Storage{{ID: "store-1", Name: "Store One"}} - metricsHistory.AddStorageMetric("store-1", "usage", 0.4, time.Now()) - syncTestResourceStore(t, monitor, state) - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/storage/charts?range=30", nil) - rec := httptest.NewRecorder() - - router.handleStorageCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } -} - -func TestHandleStorageCharts_IncludesSupplementalStorageAndResolvesUnifiedNodeFilter(t *testing.T) { - monitor, state, metricsHistory := newTestMonitor(t) - now := time.Now() - metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "usage", 0.25, now) - metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "used", 3.57*1024*1024*1024*1024, now) - metricsHistory.AddStorageMetric("vc-1:datastore:datastore-202", "avail", 11.03*1024*1024*1024*1024, now) - metricsHistory.AddStorageMetric("pool:archive", "usage", 0.36, now) - - adapter := unifiedresources.NewMonitorAdapter(nil) - adapter.PopulateSnapshotAndSupplemental(state.GetSnapshot(), map[unifiedresources.DataSource][]unifiedresources.IngestRecord{ - unifiedresources.SourceVMware: { - { - SourceID: "vc-1:host:host-101", - Resource: unifiedresources.Resource{ - ID: "agent-vmware-1", - Type: unifiedresources.ResourceTypeAgent, - Name: "esxi-01.lab.local", - Identity: unifiedresources.ResourceIdentity{ - Hostnames: []string{"esxi-01.lab.local"}, - }, - MetricsTarget: &unifiedresources.MetricsTarget{ - ResourceType: "agent", - ResourceID: "vc-1:host:host-101", - }, - VMware: &unifiedresources.VMwareData{ - ConnectionID: "vc-1", - EntityType: "host", - ManagedObjectID: "host-101", - }, - }, - }, - { - SourceID: "vc-1:datastore:datastore-202", - ParentSourceID: "vc-1:host:host-101", - Resource: unifiedresources.Resource{ - ID: "storage-vmware-1", - Type: unifiedresources.ResourceTypeStorage, - Name: "archive-tier", - ParentName: "esxi-01.lab.local", - MetricsTarget: &unifiedresources.MetricsTarget{ - ResourceType: "storage", - ResourceID: "vc-1:datastore:datastore-202", - }, - Storage: &unifiedresources.StorageMeta{ - Type: "datastore", - Platform: "vmware", - Nodes: []string{"esxi-01.lab.local", "esxi-02.lab.local"}, - }, - VMware: &unifiedresources.VMwareData{ - ConnectionID: "vc-1", - EntityType: "datastore", - ManagedObjectID: "datastore-202", - RuntimeHostName: "esxi-01.lab.local", - }, - }, - }, - }, - unifiedresources.SourceTrueNAS: { - { - SourceID: "system:truenas-main", - Resource: unifiedresources.Resource{ - ID: "agent-truenas-1", - Type: unifiedresources.ResourceTypeAgent, - Name: "truenas-main", - Identity: unifiedresources.ResourceIdentity{ - Hostnames: []string{"truenas-main"}, - }, - TrueNAS: &unifiedresources.TrueNASData{ - Hostname: "truenas-main", - }, - }, - }, - { - SourceID: "pool:archive", - ParentSourceID: "system:truenas-main", - Resource: unifiedresources.Resource{ - ID: "storage-truenas-1", - Type: unifiedresources.ResourceTypeStorage, - Name: "archive", - ParentName: "truenas-main", - MetricsTarget: &unifiedresources.MetricsTarget{ - ResourceType: "storage", - ResourceID: "pool:archive", - }, - Storage: &unifiedresources.StorageMeta{ - Type: "zfs-pool", - Platform: "truenas", - }, - TrueNAS: &unifiedresources.TrueNASData{ - Hostname: "truenas-main", - }, - }, - }, - }, - }) - selectedNodeID := "" - for _, resource := range adapter.GetAll() { - if resource.Type == unifiedresources.ResourceTypeAgent && resource.Name == "esxi-01.lab.local" { - selectedNodeID = resource.ID - break - } - } - if selectedNodeID == "" { - t.Fatal("expected canonical VMware host resource id in adapter") - } - setUnexportedField(t, monitor, "resourceStore", monitoring.ResourceStoreInterface(adapter)) - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/storage-charts?range=30&node="+selectedNodeID, nil) - rec := httptest.NewRecorder() - - router.handleStorageCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - - var decoded StorageChartsResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal StorageChartsResponse: %v", err) - } - if len(decoded.Pools) != 1 { - t.Fatalf("expected unified node filter to keep 1 VMware pool, got %d (%v)", len(decoded.Pools), decoded.Pools) - } - if _, ok := decoded.Pools["vc-1:datastore:datastore-202"]; !ok { - t.Fatalf("expected VMware datastore chart keyed by canonical metrics target, got %v", decoded.Pools) - } - if _, ok := decoded.Pools["pool:archive"]; ok { - t.Fatalf("expected TrueNAS pool to be filtered out by VMware host selection, got %v", decoded.Pools) - } -} - -func TestHandleStorageSummaryCharts_AggregatesCapacityAcrossPools(t *testing.T) { - monitor, state, metricsHistory := newTestMonitor(t) - now := time.Now() - state.Storage = []models.Storage{ - {ID: "store-1", Name: "Store One"}, - {ID: "store-2", Name: "Store Two"}, - } - metricsHistory.AddStorageMetric("store-1", "used", 400, now) - metricsHistory.AddStorageMetric("store-1", "avail", 600, now) - metricsHistory.AddStorageMetric("store-2", "used", 100, now) - metricsHistory.AddStorageMetric("store-2", "avail", 900, now) - syncTestResourceStore(t, monitor, state) - - router := &Router{monitor: monitor} - req := httptest.NewRequest(http.MethodGet, "/api/charts/storage-summary?range=1h", nil) - rec := httptest.NewRecorder() - - router.handleStorageSummaryCharts(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("expected status %d, got %d", http.StatusOK, rec.Code) - } - if ct := rec.Header().Get("Content-Type"); ct != "application/json" { - t.Fatalf("expected application/json, got %q", ct) - } - - var decoded StorageSummaryTrendResponse - if err := json.Unmarshal(rec.Body.Bytes(), &decoded); err != nil { - t.Fatalf("unmarshal StorageSummaryTrendResponse: %v", err) - } - if len(decoded.Capacity) != 1 { - t.Fatalf("expected 1 aggregate capacity point, got %d (%+v)", len(decoded.Capacity), decoded.Capacity) - } - if decoded.Capacity[0].Value != 25 { - t.Fatalf("expected aggregate capacity of 25%%, got %+v", decoded.Capacity[0]) - } - if decoded.Stats.PointCounts.Total != 1 { - t.Fatalf("expected summary point count 1, got %+v", decoded.Stats.PointCounts) - } -} - func TestEstablishSession(t *testing.T) { InitPersistentAuthStores(t.TempDir()) router := &Router{} diff --git a/internal/api/types.go b/internal/api/types.go index 6e3177b40..0fd4420d3 100644 --- a/internal/api/types.go +++ b/internal/api/types.go @@ -1,5 +1,7 @@ package api +import "github.com/rcourtman/pulse-go-rewrite/internal/api/chartapi" + // Common response types for API endpoints // HealthResponse represents the health check response @@ -42,329 +44,41 @@ type VersionResponse struct { ContainerID string `json:"containerId,omitempty"` } -// ChartResponse represents chart data -type ChartResponse struct { - ChartData map[string]VMChartData `json:"data"` - NodeData map[string]NodeChartData `json:"nodeData"` - StorageData map[string]StorageChartData `json:"storageData"` - DockerData map[string]VMChartData `json:"dockerData"` // Docker container metrics (keyed by container ID) - DockerHostData map[string]VMChartData `json:"dockerHostData"` // Docker host metrics (keyed by host ID) - AgentData map[string]VMChartData `json:"agentData"` // Unified agent metrics (keyed by agent ID) - GuestTypes map[string]string `json:"guestTypes"` // Maps guest ID to type ("vm", "system-container", "k8s") - Timestamp int64 `json:"timestamp"` - Stats ChartStats `json:"stats"` -} - -func EmptyChartResponse() ChartResponse { - return ChartResponse{}.NormalizeCollections() -} - -func (r ChartResponse) NormalizeCollections() ChartResponse { - if r.ChartData == nil { - r.ChartData = map[string]VMChartData{} - } - if r.NodeData == nil { - r.NodeData = map[string]NodeChartData{} - } - if r.StorageData == nil { - r.StorageData = map[string]StorageChartData{} - } - if r.DockerData == nil { - r.DockerData = map[string]VMChartData{} - } - if r.DockerHostData == nil { - r.DockerHostData = map[string]VMChartData{} - } - if r.AgentData == nil { - r.AgentData = map[string]VMChartData{} - } - if r.GuestTypes == nil { - r.GuestTypes = map[string]string{} - } - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// InfrastructureChartsResponse is a lightweight variant of ChartResponse used by -// infra-only clients (Infrastructure summary sparklines, prewarm caches). -// It avoids the heavy guest/storage chart payload and associated compute. -type InfrastructureChartsResponse struct { - NodeData map[string]NodeChartData `json:"nodeData"` - DockerHostData map[string]VMChartData `json:"dockerHostData"` // Docker host metrics (keyed by host ID) - AgentData map[string]VMChartData `json:"agentData"` // Unified agent metrics (keyed by agent ID) - Timestamp int64 `json:"timestamp"` - Stats ChartStats `json:"stats"` -} +type ChartResponse = chartapi.ChartResponse +type InfrastructureChartsResponse = chartapi.InfrastructureChartsResponse +type WorkloadChartsResponse = chartapi.WorkloadChartsResponse +type WorkloadsSummaryMetricData = chartapi.WorkloadsSummaryMetricData +type WorkloadsGuestCounts = chartapi.WorkloadsGuestCounts +type WorkloadsSummaryContributor = chartapi.WorkloadsSummaryContributor +type WorkloadsSummaryContributors = chartapi.WorkloadsSummaryContributors +type WorkloadsSummaryBlastRadius = chartapi.WorkloadsSummaryBlastRadius +type WorkloadsSummaryBlastRadiusGroup = chartapi.WorkloadsSummaryBlastRadiusGroup +type WorkloadsSummaryChartsResponse = chartapi.WorkloadsSummaryChartsResponse +type StorageSummaryTrendResponse = chartapi.StorageSummaryTrendResponse +type ChartStats = chartapi.ChartStats +type ChartPointCounts = chartapi.ChartPointCounts +type VMChartData = chartapi.VMChartData +type NodeChartData = chartapi.NodeChartData +type StorageChartData = chartapi.StorageChartData +type StorageChartsResponse = chartapi.StorageChartsResponse +type StoragePoolChartData = chartapi.StoragePoolChartData +type StorageDiskChartData = chartapi.StorageDiskChartData +type MetricPoint = chartapi.MetricPoint +func EmptyChartResponse() ChartResponse { return chartapi.EmptyChartResponse() } func EmptyInfrastructureChartsResponse() InfrastructureChartsResponse { - return InfrastructureChartsResponse{}.NormalizeCollections() + return chartapi.EmptyInfrastructureChartsResponse() } - -func (r InfrastructureChartsResponse) NormalizeCollections() InfrastructureChartsResponse { - if r.NodeData == nil { - r.NodeData = map[string]NodeChartData{} - } - if r.DockerHostData == nil { - r.DockerHostData = map[string]VMChartData{} - } - if r.AgentData == nil { - r.AgentData = map[string]VMChartData{} - } - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// WorkloadChartsResponse is a lightweight chart payload used by Workloads -// summary sparklines. It intentionally excludes infrastructure/storage series. -type WorkloadChartsResponse struct { - ChartData map[string]VMChartData `json:"data"` // Workload metrics keyed by workload ID - DockerData map[string]VMChartData `json:"dockerData"` // Docker container metrics keyed by container ID - GuestTypes map[string]string `json:"guestTypes"` // Maps guest ID to type ("vm", "system-container", "k8s") - Timestamp int64 `json:"timestamp"` // Unix timestamp in milliseconds - Stats ChartStats `json:"stats"` // Includes pointCounts + source hints -} - func EmptyWorkloadChartsResponse() WorkloadChartsResponse { - return WorkloadChartsResponse{}.NormalizeCollections() + return chartapi.EmptyWorkloadChartsResponse() } - -func (r WorkloadChartsResponse) NormalizeCollections() WorkloadChartsResponse { - if r.ChartData == nil { - r.ChartData = map[string]VMChartData{} - } - if r.DockerData == nil { - r.DockerData = map[string]VMChartData{} - } - if r.GuestTypes == nil { - r.GuestTypes = map[string]string{} - } - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// WorkloadsSummaryMetricData captures aggregate workload trend lines for a -// single metric (median and p95 across workloads). -type WorkloadsSummaryMetricData struct { - P50 []MetricPoint `json:"p50"` - P95 []MetricPoint `json:"p95"` -} - -func (m WorkloadsSummaryMetricData) NormalizeCollections() WorkloadsSummaryMetricData { - if m.P50 == nil { - m.P50 = []MetricPoint{} - } - if m.P95 == nil { - m.P95 = []MetricPoint{} - } - return m -} - -// WorkloadsGuestCounts captures workload counts used by the workloads summary -// cards for quick context. -type WorkloadsGuestCounts struct { - Total int `json:"total"` - Running int `json:"running"` - Stopped int `json:"stopped"` -} - -// WorkloadsSummaryContributor identifies a high-impact workload for a metric. -type WorkloadsSummaryContributor struct { - ID string `json:"id"` - Name string `json:"name"` - Value float64 `json:"value"` -} - -// WorkloadsSummaryContributors groups top contributors by metric. -type WorkloadsSummaryContributors struct { - CPU []WorkloadsSummaryContributor `json:"cpu"` - Memory []WorkloadsSummaryContributor `json:"memory"` - Disk []WorkloadsSummaryContributor `json:"disk"` - Network []WorkloadsSummaryContributor `json:"network"` -} - -func (c WorkloadsSummaryContributors) NormalizeCollections() WorkloadsSummaryContributors { - if c.CPU == nil { - c.CPU = []WorkloadsSummaryContributor{} - } - if c.Memory == nil { - c.Memory = []WorkloadsSummaryContributor{} - } - if c.Disk == nil { - c.Disk = []WorkloadsSummaryContributor{} - } - if c.Network == nil { - c.Network = []WorkloadsSummaryContributor{} - } - return c -} - -// WorkloadsSummaryBlastRadius describes how concentrated each metric pressure is. -type WorkloadsSummaryBlastRadius struct { - Scope string `json:"scope"` // idle, concentrated, mixed, distributed - Top3Share float64 `json:"top3Share"` - ActiveWorkloads int `json:"activeWorkloads"` -} - -// WorkloadsSummaryBlastRadiusGroup groups blast-radius insights by metric. -type WorkloadsSummaryBlastRadiusGroup struct { - CPU WorkloadsSummaryBlastRadius `json:"cpu"` - Memory WorkloadsSummaryBlastRadius `json:"memory"` - Disk WorkloadsSummaryBlastRadius `json:"disk"` - Network WorkloadsSummaryBlastRadius `json:"network"` -} - -// WorkloadsSummaryChartsResponse is a compact response for workloads top-card -// sparklines. It avoids returning per-workload time series. -type WorkloadsSummaryChartsResponse struct { - CPU WorkloadsSummaryMetricData `json:"cpu"` - Memory WorkloadsSummaryMetricData `json:"memory"` - Disk WorkloadsSummaryMetricData `json:"disk"` - Network WorkloadsSummaryMetricData `json:"network"` - GuestCounts WorkloadsGuestCounts `json:"guestCounts"` - TopContributors WorkloadsSummaryContributors `json:"topContributors"` - BlastRadius WorkloadsSummaryBlastRadiusGroup `json:"blastRadius"` - Timestamp int64 `json:"timestamp"` - Stats ChartStats `json:"stats"` -} - func EmptyWorkloadsSummaryChartsResponse() WorkloadsSummaryChartsResponse { - return WorkloadsSummaryChartsResponse{}.NormalizeCollections() + return chartapi.EmptyWorkloadsSummaryChartsResponse() } - -func (r WorkloadsSummaryChartsResponse) NormalizeCollections() WorkloadsSummaryChartsResponse { - r.CPU = r.CPU.NormalizeCollections() - r.Memory = r.Memory.NormalizeCollections() - r.Disk = r.Disk.NormalizeCollections() - r.Network = r.Network.NormalizeCollections() - r.TopContributors = r.TopContributors.NormalizeCollections() - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// StorageSummaryTrendResponse is a compact response for the dashboard storage -// card. It intentionally avoids returning per-pool and per-disk series. -type StorageSummaryTrendResponse struct { - Capacity []MetricPoint `json:"capacity"` - Timestamp int64 `json:"timestamp"` - Stats ChartStats `json:"stats"` -} - func EmptyStorageSummaryTrendResponse() StorageSummaryTrendResponse { - return StorageSummaryTrendResponse{}.NormalizeCollections() -} - -func (r StorageSummaryTrendResponse) NormalizeCollections() StorageSummaryTrendResponse { - if r.Capacity == nil { - r.Capacity = []MetricPoint{} - } - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// ChartStats represents chart statistics -type ChartStats struct { - OldestDataTimestamp int64 `json:"oldestDataTimestamp"` - Range string `json:"range,omitempty"` - RangeSeconds int64 `json:"rangeSeconds,omitempty"` - MetricsStoreEnabled bool `json:"metricsStoreEnabled"` - PrimarySourceHint string `json:"primarySourceHint,omitempty"` - InMemoryThresholdSecs int64 `json:"inMemoryThresholdSecs,omitempty"` - PointCounts ChartPointCounts `json:"pointCounts,omitempty"` -} - -func (s ChartStats) NormalizeCollections() ChartStats { - return s -} - -// ChartPointCounts summarizes how many points were returned in /api/charts. -type ChartPointCounts struct { - Total int `json:"total,omitempty"` - Guests int `json:"guests,omitempty"` - Nodes int `json:"nodes,omitempty"` - Storage int `json:"storage,omitempty"` - DockerContainers int `json:"dockerContainers,omitempty"` - DockerHosts int `json:"dockerHosts,omitempty"` - Agents int `json:"agents,omitempty"` -} - -// VMChartData represents chart data for a VM -type VMChartData map[string][]MetricPoint - -// NodeChartData represents chart data for a node -type NodeChartData map[string][]MetricPoint - -// StorageChartData represents chart data for storage -type StorageChartData map[string][]MetricPoint - -// StorageChartsResponse represents storage charts API response. -// It includes both pool-level capacity metrics and physical disk metrics. -type StorageChartsResponse struct { - Pools map[string]StoragePoolChartData `json:"pools"` - Disks map[string]StorageDiskChartData `json:"disks"` - Stats ChartStats `json:"stats"` -} - -func EmptyStorageChartsResponse() StorageChartsResponse { - return StorageChartsResponse{}.NormalizeCollections() -} - -func (r StorageChartsResponse) NormalizeCollections() StorageChartsResponse { - if r.Pools == nil { - r.Pools = map[string]StoragePoolChartData{} - } - if r.Disks == nil { - r.Disks = map[string]StorageDiskChartData{} - } - for key, pool := range r.Pools { - r.Pools[key] = pool.NormalizeCollections() - } - for key, disk := range r.Disks { - r.Disks[key] = disk.NormalizeCollections() - } - r.Stats = r.Stats.NormalizeCollections() - return r -} - -// StoragePoolChartData holds per-pool capacity time-series. -type StoragePoolChartData struct { - Name string `json:"name"` - Usage []MetricPoint `json:"usage"` - Used []MetricPoint `json:"used"` - Avail []MetricPoint `json:"avail"` -} - -func (d StoragePoolChartData) NormalizeCollections() StoragePoolChartData { - if d.Usage == nil { - d.Usage = []MetricPoint{} - } - if d.Used == nil { - d.Used = []MetricPoint{} - } - if d.Avail == nil { - d.Avail = []MetricPoint{} - } - return d -} - -// StorageDiskChartData holds per-disk temperature time-series. -type StorageDiskChartData struct { - Name string `json:"name"` - Node string `json:"node"` - Temperature []MetricPoint `json:"temperature"` -} - -func (d StorageDiskChartData) NormalizeCollections() StorageDiskChartData { - if d.Temperature == nil { - d.Temperature = []MetricPoint{} - } - return d -} - -// MetricPoint represents a single metric data point -type MetricPoint struct { - Timestamp int64 `json:"timestamp"` - Value float64 `json:"value"` + return chartapi.EmptyStorageSummaryTrendResponse() } +func EmptyStorageChartsResponse() StorageChartsResponse { return chartapi.EmptyStorageChartsResponse() } // AgentVersionResponse represents Docker / Podman module version information. type AgentVersionResponse struct { diff --git a/internal/unifiedresources/code_standards_test.go b/internal/unifiedresources/code_standards_test.go index 9ef974fd8..f4481d60c 100644 --- a/internal/unifiedresources/code_standards_test.go +++ b/internal/unifiedresources/code_standards_test.go @@ -650,7 +650,7 @@ func TestCloneResourceCopiesHostThermalState(t *testing.T) { } func TestProxmoxWorkloadActionTargetsStayBackendAuthored(t *testing.T) { - apiSource, err := os.ReadFile(filepath.Join("..", "api", "resources.go")) + apiSource, err := os.ReadFile(filepath.Join("..", "api", "resourceapi", "resources.go")) if err != nil { t.Fatalf("read api resources source: %v", err) } @@ -695,9 +695,9 @@ func TestDockerSwarmEvidenceGuardStaysInAdapter(t *testing.T) { } func TestAPIResourcesKeepsOwnedSupplementalGapFillAndVMwareAlias(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "api", "resources.go")) + data, err := os.ReadFile(filepath.Join("..", "api", "resourceapi", "resources.go")) if err != nil { - t.Fatalf("failed to read ../api/resources.go: %v", err) + t.Fatalf("failed to read ../api/resourceapi/resources.go: %v", err) } source := string(data) @@ -709,23 +709,23 @@ func TestAPIResourcesKeepsOwnedSupplementalGapFillAndVMwareAlias(t *testing.T) { } for _, snippet := range requiredSnippets { if !strings.Contains(source, snippet) { - t.Fatalf("../api/resources.go must contain %q", snippet) + t.Fatalf("../api/resourceapi/resources.go must contain %q", snippet) } } } func TestResourceAPIUsesCanonicalTenantUnifiedSeed(t *testing.T) { - data, err := os.ReadFile(filepath.Join("..", "api", "resources.go")) + data, err := os.ReadFile(filepath.Join("..", "api", "resourceapi", "resources.go")) if err != nil { t.Fatalf("failed to read resources.go: %v", err) } source := string(data) if strings.Contains(source, "GetStateForTenant(") { - t.Fatalf("internal/api/resources.go must not fall back to tenant StateSnapshot seeding") + t.Fatalf("internal/api/resourceapi/resources.go must not fall back to tenant StateSnapshot seeding") } if !strings.Contains(source, "UnifiedResourceSnapshotForTenant(orgID)") { - t.Fatalf("internal/api/resources.go must use tenant unified resource snapshots as the canonical seed") + t.Fatalf("internal/api/resourceapi/resources.go must use tenant unified resource snapshots as the canonical seed") } } @@ -887,7 +887,7 @@ func TestCephPoolsProjectThroughCanonicalStoragePath(t *testing.T) { func TestResourceAPIExposesDedicatedFacetReads(t *testing.T) { requiredSnippets := map[string][]string{ - filepath.Join("..", "api", "resources.go"): { + filepath.Join("..", "api", "resourceapi", "resources.go"): { "HandleGetResourceFacets", "HandleGetResourceTimeline", "HandleListResourceTimeline", @@ -1027,7 +1027,7 @@ func TestActionExecutionContractStaysAPIOwned(t *testing.T) { "agentcapabilities.AgentErrCodeActionPlanDrift", "agentcapabilities.AgentErrCodeActionExecutorUnavailable", }, - filepath.Join("..", "api", "resources.go"): { + filepath.Join("..", "api", "resources_compat.go"): { "actionExecutor ActionExecutor", "actionCompleted func(unified.ActionAuditRecord)", "actionTransition func(orgID string, record unified.ActionAuditRecord)", @@ -1035,7 +1035,10 @@ func TestActionExecutionContractStaysAPIOwned(t *testing.T) { "func (h *ResourceHandlers) SetActionCompletedPublisher(", "func (h *ResourceHandlers) SetActionTransitionPublisher(", "policyAdmission *actionlifecycle.PolicyAdmissionCoordinator", - "func (h *ResourceHandlers) applyActionAvailability(ctx context.Context, resources []unified.Resource)", + }, + filepath.Join("..", "api", "resourceapi", "resources.go"): { + "actionAvailability actionlifecycle.AvailabilityChecker", + "func (h *QueryService) applyActionAvailability(ctx context.Context, resources []unified.Resource)", "resources[i].ActionReadiness = readinesses", }, filepath.Join("..", "api", "agent_events.go"): { @@ -1093,7 +1096,7 @@ func TestResourceChangeFilterParsingIsOwnedByUnifiedResources(t *testing.T) { "func parseResourceChangeSourceTypes(values []string) ([]ChangeSourceType, error)", "func parseResourceChangeSourceAdapters(values []string) ([]ChangeSourceAdapter, error)", }, - filepath.Join("..", "api", "resources.go"): { + filepath.Join("..", "api", "resourceapi", "resources.go"): { "unified.ParseResourceChangeFilters(r.URL.Query()[\"kind\"], r.URL.Query()[\"sourceType\"], r.URL.Query()[\"sourceAdapter\"])", }, } @@ -1252,7 +1255,7 @@ func TestCanonicalMetadataRefreshHelperUsedByConsumers(t *testing.T) { filepath.Join(".", "clone.go"): { "RefreshCanonicalMetadata(&out)", }, - filepath.Join("..", "api", "resources.go"): { + filepath.Join("..", "api", "resourceapi", "resources.go"): { "unified.RefreshCanonicalMetadata(&resourceCopy)", "unified.RefreshCanonicalMetadataSlice(paged)", "unified.RefreshCanonicalMetadataSlice(children)", @@ -1380,7 +1383,7 @@ func TestPolicyPostureSummaryIsOwnedByUnifiedResources(t *testing.T) { filepath.Join("..", "ai", "resource_context.go"): { "unifiedresources.SummarizePolicyPosture(allResources)", }, - filepath.Join("..", "api", "resources.go"): { + filepath.Join("..", "api", "resourceapi", "resources.go"): { "resourcePolicyPostureAggregation(allResources)", "unified.ResourcePolicyPostureContract(unified.SummarizePolicyPosture(canonicalResources))", }, @@ -2304,7 +2307,7 @@ func TestV6ReleaseFacingAPITestsCoverLegacyHostRejection(t *testing.T) { }, }, { - path: filepath.Join(repoRoot, "internal", "api", "resources_test.go"), + path: filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources_test.go"), requiredSnippets: []string{ `/api/resources?type=host`, `unsupported type filter token(s): host`, @@ -2544,14 +2547,14 @@ func TestV6DirectHostAliasValidatorCoverage(t *testing.T) { }, }, { - path: filepath.Join(repoRoot, "internal", "api", "resources_test.go"), + path: filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources_test.go"), requiredSnippets: []string{ `/api/resources?type=host`, `unsupported type filter token(s): host`, }, }, { - path: filepath.Join(repoRoot, "internal", "api", "resources_frontend_types_test.go"), + path: filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources_frontend_types_test.go"), requiredSnippets: []string{ `unsupported host ignored by parser`, `TestUnsupportedResourceTypeFilterTokensRejectsLegacyAliases`, @@ -2608,7 +2611,7 @@ func TestV6DirectHostAliasValidatorCoverage(t *testing.T) { func TestResourceAPIHotPathUsesSharedPresentationSnapshot(t *testing.T) { repoRoot := filepath.Join("..", "..") - path := filepath.Join(repoRoot, "internal", "api", "resources.go") + path := filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources.go") data, err := os.ReadFile(path) if err != nil { @@ -2637,7 +2640,7 @@ func TestResourceAPIHotPathUsesSharedPresentationSnapshot(t *testing.T) { func TestCanonicalResourceOrderingContractsStayShared(t *testing.T) { repoRoot := filepath.Join("..", "..") - resourcesPath := filepath.Join(repoRoot, "internal", "api", "resources.go") + resourcesPath := filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources.go") registryPath := filepath.Join(repoRoot, "internal", "unifiedresources", "registry.go") resourcesSource, err := os.ReadFile(resourcesPath) @@ -2663,7 +2666,7 @@ func TestCanonicalResourceOrderingContractsStayShared(t *testing.T) { func TestBroadcastStateUsesSharedCanonicalResourceContract(t *testing.T) { repoRoot := filepath.Join("..", "..") typesPath := filepath.Join(repoRoot, "internal", "unifiedresources", "types.go") - resourcesPath := filepath.Join(repoRoot, "internal", "api", "resources.go") + resourcesPath := filepath.Join(repoRoot, "internal", "api", "resourceapi", "resources.go") monitorPath := filepath.Join(repoRoot, "internal", "monitoring", "monitor.go") typesSource, err := os.ReadFile(typesPath) @@ -2826,16 +2829,16 @@ func TestGuestRRDPointCarriesOnlyRecordedGuestColumns(t *testing.T) { // unremovable. This is a source-shape guard because the leak is invisible at // runtime until a tenant is deleted or a data directory is torn down. func TestCachedResourceStoresHaveATenantReleasePath(t *testing.T) { - resources, err := os.ReadFile("../api/resources.go") + resources, err := os.ReadFile("../api/resourceapi/resources.go") if err != nil { t.Fatalf("read resources.go: %v", err) } for _, fragment := range []string{ - "func (h *ResourceHandlers) CloseTenantStore(orgID string) error", - "func (h *ResourceHandlers) CloseStores() error", + "func (h *QueryService) CloseTenantStore(orgID string) error", + "func (h *QueryService) CloseStores() error", } { if !strings.Contains(string(resources), fragment) { - t.Errorf("internal/api/resources.go must expose %q so cached per-tenant stores can be released", fragment) + t.Errorf("internal/api/resourceapi/resources.go must expose %q so cached per-tenant stores can be released", fragment) } }