mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
rcourtman
4a0a05858d
GET /api/charts/workloads-summary has had no first-party caller since the v6 import: ChartsAPI.getWorkloadsSummaryCharts arrived with it and nothing ever called it. GET /api/charts/storage-summary lost its last reader when storageSummaryTrendCache.ts was deleted on 2026-05-26. Both still cost something. The workloads route carried a payload cache, an SLO test in the release latency gate and a benchmark, and mock mode rebuilt the 24-hour storage aggregate on every sampler tick for a cache nothing read. No stable release ever requested either route from the browser: v6.0.0 and later carry no caller, and the storage trend reader shipped only in v6.0.0-rc.1 to rc.5. Neither route is in docs/API.md (a list of common endpoints, not a complete one), and pulse-pro, pulse-mobile and pulse-dev-infra never call them. A monitoring:read token could reach them, so a private script may notice, but nothing known depends on them. pulse-enterprise names the SLO test only as fixture data, and its latency gate passes a test that is absent from the candidate as removed. So the routes go with their clients: handlers, response types, the monitor trend helpers, the mock aggregate and its prewarm, the SLO budget, the benchmark, and the ChartsAPI methods and types. Requests to either path now get 404. buildMockWorkloadMetricHistorySeries goes too, because the workloads-summary handler was its only production caller. A TypeScript reference graph from src/index.tsx finds no production reference to the removed methods or types. The production build loses only the two static methods from the ChartsAPI chunk (326 B raw, 49 B gzip); every other file differs only by chunk hashes and SRI values. deadcode ./cmd/pulse and golangci unused report nothing newly unreachable. On private mock stacks the Proxmox overview in Trends, a guest History drawer and the storage page make the same chart requests before and after at 1440 and 390 px, and neither build requested the removed routes. Contract prose that still described either route as live now records the retirement, and the workload chart guard pins the per-type metric sets the surviving /api/charts/workloads route requests.
2230 lines
64 KiB
Go
2230 lines
64 KiB
Go
package chartapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"math"
|
||
"net/http"
|
||
"net/url"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
||
unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||
"github.com/rs/zerolog/log"
|
||
"golang.org/x/sync/singleflight"
|
||
)
|
||
|
||
const workloadChartsCacheTTL = 3 * time.Second
|
||
const summaryChartsCacheTTL = 5 * time.Second
|
||
|
||
const (
|
||
// Both cached chart routes share one retention budget. This stays modest for
|
||
// low-memory container deployments while still holding several ordinary
|
||
// large-estate responses.
|
||
chartPayloadCacheMaxEntries = 64
|
||
chartPayloadCacheMaxBytes = 16 << 20
|
||
|
||
infrastructureChartsCachePrefix = "infrastructure|"
|
||
workloadChartsCachePrefix = "workload-charts|"
|
||
)
|
||
|
||
// MonitorResolver supplies the authenticated tenant monitor selected by the
|
||
// router context. Chart computation remains entirely owned by Service.
|
||
type MonitorResolver interface {
|
||
MonitorForContext(context.Context) *monitoring.Monitor
|
||
}
|
||
|
||
// Service owns chart queries, aggregation, serialization, caching, and
|
||
// singleflight coordination independently of the HTTP router package.
|
||
type Service struct {
|
||
resolver MonitorResolver
|
||
|
||
chartPayloads boundedChartPayloadCache
|
||
workloadChartsComputeGroup singleflight.Group
|
||
}
|
||
|
||
func NewService(resolver MonitorResolver) *Service {
|
||
return &Service{
|
||
resolver: resolver,
|
||
chartPayloads: newBoundedChartPayloadCache(
|
||
chartPayloadCacheMaxEntries,
|
||
chartPayloadCacheMaxBytes,
|
||
),
|
||
}
|
||
}
|
||
|
||
func (r *Service) getTenantMonitor(ctx context.Context) *monitoring.Monitor {
|
||
if r == nil || r.resolver == nil {
|
||
return nil
|
||
}
|
||
return r.resolver.MonitorForContext(ctx)
|
||
}
|
||
|
||
func storageChartsSelectedNodeName(resource unifiedresources.Resource) string {
|
||
if name := strings.TrimSpace(resource.Name); name != "" {
|
||
return name
|
||
}
|
||
if resource.TrueNAS != nil {
|
||
if hostname := strings.TrimSpace(resource.TrueNAS.Hostname); hostname != "" {
|
||
return hostname
|
||
}
|
||
}
|
||
for _, hostname := range resource.Identity.Hostnames {
|
||
if hostname = strings.TrimSpace(hostname); hostname != "" {
|
||
return hostname
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func storageChartsSelectedNodeInstance(resource unifiedresources.Resource) string {
|
||
if resource.Proxmox == nil {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(resource.Proxmox.Instance)
|
||
}
|
||
|
||
// handleCharts handles chart data requests
|
||
func (r *Service) HandleCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Get time range from query parameters
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
|
||
// Convert time range to duration.
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
// Get tenant-specific monitor and current state
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
// Create chart data structure that matches frontend expectations
|
||
chartData := make(map[string]VMChartData)
|
||
nodeData := make(map[string]NodeChartData)
|
||
|
||
currentTime := time.Now().UnixMilli() // JavaScript timestamp format
|
||
oldestTimestamp := currentTime
|
||
|
||
// Process VMs and Containers - batch-load historical data (1-2 SQL calls
|
||
// per family instead of N).
|
||
oldestTimestamp = collectGuestChartData(monitor, "vm", readState.VMs(), duration, chartData, currentTime, oldestTimestamp)
|
||
oldestTimestamp = collectGuestChartData(monitor, "container", readState.Containers(), duration, chartData, currentTime, oldestTimestamp)
|
||
|
||
// Process Storage - batch-load historical data (1-2 SQL calls instead of N).
|
||
storageData := make(map[string]StorageChartData)
|
||
spList := readState.StoragePools()
|
||
storageIDs := make([]string, 0, len(spList))
|
||
for _, sp := range spList {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
if sid := sp.SourceID(); sid != "" {
|
||
storageIDs = append(storageIDs, sid)
|
||
}
|
||
}
|
||
storageBatchMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration)
|
||
for _, sp := range spList {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
sid := sp.SourceID()
|
||
if sid == "" {
|
||
continue
|
||
}
|
||
storageData[sid] = make(StorageChartData)
|
||
if batchMetrics, ok := storageBatchMetrics[sid]; ok {
|
||
if usagePoints, found := batchMetrics["usage"]; found && len(usagePoints) > 0 {
|
||
storageData[sid]["disk"] = make([]MetricPoint, len(usagePoints))
|
||
for i, point := range usagePoints {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
storageData[sid]["disk"][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if len(storageData[sid]["disk"]) == 0 {
|
||
storageData[sid]["disk"] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: sp.DiskPercent()},
|
||
}
|
||
}
|
||
}
|
||
|
||
// Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5).
|
||
nodeMetricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
|
||
nodeList := readState.Nodes()
|
||
nodeIDs := make([]string, 0, len(nodeList))
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
if nid := node.SourceID(); nid != "" {
|
||
nodeIDs = append(nodeIDs, nid)
|
||
}
|
||
}
|
||
nodeBatchMetrics := monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration)
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
nid := node.SourceID()
|
||
if nid == "" {
|
||
continue
|
||
}
|
||
nodeData[nid] = make(NodeChartData)
|
||
if batchMetrics, ok := nodeBatchMetrics[nid]; ok {
|
||
for _, metricType := range nodeMetricTypes {
|
||
points, found := batchMetrics[metricType]
|
||
if !found {
|
||
continue
|
||
}
|
||
nodeData[nid][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
nodeData[nid][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range nodeMetricTypes {
|
||
if len(nodeData[nid][metricType]) == 0 {
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = node.CPUPercent()
|
||
case "memory":
|
||
value = node.MemoryPercent()
|
||
case "disk":
|
||
value = node.DiskPercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
nodeData[nid][metricType] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: value},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Build guest type map with canonical v6 names.
|
||
guestTypes := make(map[string]string)
|
||
for _, vm := range readState.VMs() {
|
||
if vm == nil {
|
||
continue
|
||
}
|
||
if sid := vm.SourceID(); sid != "" {
|
||
guestTypes[sid] = "vm"
|
||
}
|
||
}
|
||
for _, ct := range readState.Containers() {
|
||
if ct == nil {
|
||
continue
|
||
}
|
||
if sid := ct.SourceID(); sid != "" {
|
||
guestTypes[sid] = "system-container"
|
||
}
|
||
}
|
||
for _, dc := range readState.DockerContainers() {
|
||
if dc == nil {
|
||
continue
|
||
}
|
||
if key := strings.TrimSpace(dc.ID()); key != "" {
|
||
guestTypes[key] = "app-container"
|
||
}
|
||
}
|
||
|
||
// Process Docker containers - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerData := make(map[string]VMChartData)
|
||
dcList := readState.DockerContainers()
|
||
dcRequests := make([]monitoring.GuestChartRequest, 0, len(dcList))
|
||
for _, dc := range dcList {
|
||
_, request, ok := appContainerChartRequest(dc)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dcRequests = append(dcRequests, request)
|
||
}
|
||
dcBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerContainer", dcRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, dc := range dcList {
|
||
responseKey, request, ok := appContainerChartRequest(dc)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dockerData[responseKey] = make(VMChartData)
|
||
if batchMetrics, ok := dcBatchMetrics[request.SQLResourceID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(dockerData[responseKey], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(dockerData[responseKey]["cpu"]) == 0 {
|
||
dockerData[responseKey]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dc.CPUPercent()}}
|
||
dockerData[responseKey]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dc.MemoryPercent()}}
|
||
dockerData[responseKey]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: dc.DiskPercent()}}
|
||
}
|
||
}
|
||
|
||
// Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerHostData := make(map[string]VMChartData)
|
||
dhList := readState.DockerHosts()
|
||
dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList))
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
if dhID := dh.HostSourceID(); dhID != "" {
|
||
dhRequests = append(dhRequests, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID),
|
||
SQLResourceID: dhID,
|
||
})
|
||
}
|
||
}
|
||
dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
dhID := dh.HostSourceID()
|
||
if dhID == "" {
|
||
continue
|
||
}
|
||
dockerHostData[dhID] = make(VMChartData)
|
||
if batchMetrics, ok := dhBatchMetrics[dhID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(dockerHostData[dhID], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(dockerHostData[dhID]["cpu"]) == 0 {
|
||
dockerHostData[dhID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dh.CPUPercent()}}
|
||
dockerHostData[dhID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dh.MemoryPercent()}}
|
||
var diskPercent float64
|
||
if disks := dh.Disks(); len(disks) > 0 {
|
||
diskPercent = disks[0].Usage
|
||
}
|
||
dockerHostData[dhID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: diskPercent}}
|
||
}
|
||
}
|
||
|
||
// Process unified agents - batch-load historical data (1-2 SQL calls instead of N).
|
||
agentData := make(map[string]VMChartData)
|
||
hostList := readState.Hosts()
|
||
agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList))
|
||
for _, h := range hostList {
|
||
_, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentRequests = append(agentRequests, request)
|
||
}
|
||
agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, h := range hostList {
|
||
hID, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentData[hID] = make(VMChartData)
|
||
if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(agentData[hID]["cpu"]) == 0 {
|
||
agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}}
|
||
agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}}
|
||
agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: h.DiskPercent()}}
|
||
}
|
||
}
|
||
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
countNodePoints := func(metricsMap map[string]NodeChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
countStoragePoints := func(metricsMap map[string]StorageChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
guestPoints := countChartPoints(chartData)
|
||
nodePoints := countNodePoints(nodeData)
|
||
storagePoints := countStoragePoints(storageData)
|
||
dockerContainerPoints := countChartPoints(dockerData)
|
||
dockerHostPoints := countChartPoints(dockerHostData)
|
||
agentPoints := countChartPoints(agentData)
|
||
|
||
response := ChartResponse{
|
||
ChartData: chartData,
|
||
NodeData: nodeData,
|
||
StorageData: storageData,
|
||
DockerData: dockerData,
|
||
DockerHostData: dockerHostData,
|
||
AgentData: agentData,
|
||
GuestTypes: guestTypes,
|
||
Timestamp: currentTime,
|
||
Stats: ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: guestPoints + nodePoints + storagePoints + dockerContainerPoints + dockerHostPoints + agentPoints,
|
||
Guests: guestPoints,
|
||
Nodes: nodePoints,
|
||
Storage: storagePoints,
|
||
DockerContainers: dockerContainerPoints,
|
||
DockerHosts: dockerHostPoints,
|
||
Agents: agentPoints,
|
||
},
|
||
},
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := json.NewEncoder(w).Encode(response); err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
log.Debug().
|
||
Int("guests", len(chartData)).
|
||
Int("nodes", len(nodeData)).
|
||
Int("storage", len(storageData)).
|
||
Int("dockerContainers", len(dockerData)).
|
||
Int("agents", len(agentData)).
|
||
Str("range", timeRange).
|
||
Msg("Chart data response sent")
|
||
}
|
||
|
||
func parseWorkloadMaxPoints(raw string) int {
|
||
const (
|
||
defaultMaxPoints = 180
|
||
minMaxPoints = 30
|
||
maxMaxPoints = 500
|
||
)
|
||
|
||
trimmed := strings.TrimSpace(raw)
|
||
if trimmed == "" {
|
||
return defaultMaxPoints
|
||
}
|
||
|
||
value, err := strconv.Atoi(trimmed)
|
||
if err != nil {
|
||
return defaultMaxPoints
|
||
}
|
||
if value < minMaxPoints {
|
||
return minMaxPoints
|
||
}
|
||
if value > maxMaxPoints {
|
||
return maxMaxPoints
|
||
}
|
||
return value
|
||
}
|
||
|
||
func ParseWorkloadMaxPoints(raw string) int { return parseWorkloadMaxPoints(raw) }
|
||
|
||
func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) {
|
||
if host == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
if agentID := strings.TrimSpace(host.AgentID()); agentID != "" {
|
||
return agentID, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("agent:%s", agentID),
|
||
SQLResourceID: agentID,
|
||
}, true
|
||
}
|
||
|
||
target := host.MetricsTarget()
|
||
if target == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
metricID := strings.TrimSpace(target.ResourceID)
|
||
if metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return metricID, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("agent:%s", metricID),
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func appContainerChartMetricID(container *unifiedresources.DockerContainerView) string {
|
||
if container == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := container.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(container.ContainerID())
|
||
}
|
||
|
||
func appContainerChartRequest(container *unifiedresources.DockerContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
if container == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := strings.TrimSpace(container.ID())
|
||
if responseKey == "" {
|
||
responseKey = strings.TrimSpace(container.ContainerID())
|
||
}
|
||
metricID := appContainerChartMetricID(container)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("docker:%s", metricID),
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func canonicalGuestResponseKey(resourceID, instance, node string, vmid int) string {
|
||
trimmedInstance := strings.TrimSpace(instance)
|
||
trimmedNode := strings.TrimSpace(node)
|
||
if trimmedInstance != "" && trimmedNode != "" && vmid > 0 {
|
||
return fmt.Sprintf("%s:%s:%d", trimmedInstance, trimmedNode, vmid)
|
||
}
|
||
return strings.TrimSpace(resourceID)
|
||
}
|
||
|
||
func vmChartMetricID(vm *unifiedresources.VMView) string {
|
||
if vm == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := vm.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(vm.SourceID())
|
||
}
|
||
|
||
func vmChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) {
|
||
if vm == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := canonicalGuestResponseKey(vm.ID(), vm.Instance(), vm.Node(), vm.VMID())
|
||
metricID := vmChartMetricID(vm)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: metricID,
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func VMChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) {
|
||
return vmChartRequest(vm)
|
||
}
|
||
|
||
func systemContainerChartMetricID(container *unifiedresources.ContainerView) string {
|
||
if container == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := container.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(container.SourceID())
|
||
}
|
||
|
||
func systemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
if container == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := canonicalGuestResponseKey(container.ID(), container.Instance(), container.Node(), container.VMID())
|
||
metricID := systemContainerChartMetricID(container)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: metricID,
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func SystemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
return systemContainerChartRequest(container)
|
||
}
|
||
|
||
func capMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
if len(points) <= maxPoints || maxPoints <= 0 {
|
||
return points
|
||
}
|
||
if maxPoints == 1 {
|
||
return []MetricPoint{points[len(points)-1]}
|
||
}
|
||
|
||
result := make([]MetricPoint, 0, maxPoints)
|
||
step := float64(len(points)-1) / float64(maxPoints-1)
|
||
prevIndex := -1
|
||
|
||
for i := 0; i < maxPoints; i++ {
|
||
index := int(float64(i)*step + 0.5)
|
||
if index <= prevIndex {
|
||
index = prevIndex + 1
|
||
}
|
||
if index >= len(points) {
|
||
index = len(points) - 1
|
||
}
|
||
result = append(result, points[index])
|
||
prevIndex = index
|
||
}
|
||
|
||
if result[len(result)-1].Timestamp != points[len(points)-1].Timestamp {
|
||
result[len(result)-1] = points[len(points)-1]
|
||
}
|
||
return result
|
||
}
|
||
|
||
func CapMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
return capMetricPointSeriesByIndex(points, maxPoints)
|
||
}
|
||
|
||
const (
|
||
infrastructureSummaryMinSeriesPoints = 24
|
||
infrastructureSummaryMaxSeriesPoints = 96
|
||
)
|
||
|
||
const InfrastructureSummaryMaxSeriesPoints = infrastructureSummaryMaxSeriesPoints
|
||
|
||
// capMetricPointSeries keeps mixed-cadence series visually proportional across
|
||
// the selected time window. Index-based capping over-selects recent dense
|
||
// samples, which bunches the right edge on long ranges.
|
||
func capMetricPointSeries(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
if len(points) <= maxPoints || maxPoints <= 0 {
|
||
return points
|
||
}
|
||
if maxPoints == 1 {
|
||
return []MetricPoint{points[len(points)-1]}
|
||
}
|
||
|
||
startTimestamp := points[0].Timestamp
|
||
endTimestamp := points[len(points)-1].Timestamp
|
||
if endTimestamp <= startTimestamp {
|
||
return capMetricPointSeriesByIndex(points, maxPoints)
|
||
}
|
||
|
||
bucketSpan := float64(endTimestamp-startTimestamp) / float64(maxPoints-1)
|
||
if bucketSpan < 1 {
|
||
return capMetricPointSeriesByIndex(points, maxPoints)
|
||
}
|
||
|
||
type timeBucketRepresentative struct {
|
||
point MetricPoint
|
||
distance float64
|
||
ok bool
|
||
}
|
||
|
||
buckets := make([]timeBucketRepresentative, maxPoints)
|
||
for _, point := range points {
|
||
index := int(math.Round(float64(point.Timestamp-startTimestamp) / bucketSpan))
|
||
if index < 0 {
|
||
index = 0
|
||
}
|
||
if index >= maxPoints {
|
||
index = maxPoints - 1
|
||
}
|
||
|
||
targetTimestamp := float64(startTimestamp) + bucketSpan*float64(index)
|
||
distance := math.Abs(float64(point.Timestamp) - targetTimestamp)
|
||
current := buckets[index]
|
||
if !current.ok ||
|
||
distance < current.distance ||
|
||
(distance == current.distance && point.Timestamp > current.point.Timestamp) {
|
||
buckets[index] = timeBucketRepresentative{
|
||
point: point,
|
||
distance: distance,
|
||
ok: true,
|
||
}
|
||
}
|
||
}
|
||
|
||
result := make([]MetricPoint, 0, maxPoints)
|
||
result = append(result, points[0])
|
||
lastAddedTimestamp := points[0].Timestamp
|
||
for index := 1; index < maxPoints-1; index++ {
|
||
bucket := buckets[index]
|
||
if !bucket.ok {
|
||
continue
|
||
}
|
||
if bucket.point.Timestamp <= lastAddedTimestamp {
|
||
continue
|
||
}
|
||
result = append(result, bucket.point)
|
||
lastAddedTimestamp = bucket.point.Timestamp
|
||
}
|
||
|
||
lastPoint := points[len(points)-1]
|
||
if lastPoint.Timestamp <= lastAddedTimestamp {
|
||
result[len(result)-1] = lastPoint
|
||
return result
|
||
}
|
||
|
||
result = append(result, lastPoint)
|
||
return result
|
||
}
|
||
|
||
func targetBoundedSummarySeriesPoints(duration time.Duration, minPoints, maxPoints int) int {
|
||
if duration <= 0 {
|
||
return minPoints
|
||
}
|
||
|
||
target := int(duration / time.Minute)
|
||
if target < minPoints {
|
||
target = minPoints
|
||
}
|
||
if target > maxPoints {
|
||
target = maxPoints
|
||
}
|
||
if target < 2 {
|
||
target = 2
|
||
}
|
||
return target
|
||
}
|
||
|
||
type infrastructureSummaryBucket struct {
|
||
count int
|
||
sum float64
|
||
max float64
|
||
firstTimestamp int64
|
||
lastTimestamp int64
|
||
lastValue float64
|
||
}
|
||
|
||
func targetInfrastructureSummarySeriesPoints(duration time.Duration) int {
|
||
return targetBoundedSummarySeriesPoints(
|
||
duration,
|
||
infrastructureSummaryMinSeriesPoints,
|
||
infrastructureSummaryMaxSeriesPoints,
|
||
)
|
||
}
|
||
|
||
func infrastructureChartsCacheKey(req *http.Request, timeRange string, requestedMetricNames []string) string {
|
||
orgID := strings.TrimSpace(apicontext.OrgID(req.Context()))
|
||
if orgID == "" {
|
||
orgID = "default"
|
||
}
|
||
return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.Join(requestedMetricNames, ",")
|
||
}
|
||
|
||
func (r *Service) cachedInfrastructureChartsPayload(key string, now time.Time) ([]byte, bool) {
|
||
if r == nil || key == "" {
|
||
return nil, false
|
||
}
|
||
return r.chartPayloads.get(infrastructureChartsCachePrefix+key, now)
|
||
}
|
||
|
||
func (r *Service) cacheInfrastructureChartsPayload(key string, payload []byte, now time.Time) {
|
||
if r == nil || key == "" || len(payload) == 0 {
|
||
return
|
||
}
|
||
r.chartPayloads.put(
|
||
infrastructureChartsCachePrefix+key,
|
||
payload,
|
||
now.Add(summaryChartsCacheTTL),
|
||
now,
|
||
)
|
||
}
|
||
|
||
func aggregateInfrastructureSummaryBucketValue(
|
||
metricType string,
|
||
bucket infrastructureSummaryBucket,
|
||
isLastBucket bool,
|
||
) float64 {
|
||
if bucket.count == 0 {
|
||
return 0
|
||
}
|
||
if isLastBucket {
|
||
return bucket.lastValue
|
||
}
|
||
|
||
switch metricType {
|
||
case "memory", "disk":
|
||
return bucket.sum / float64(bucket.count)
|
||
default:
|
||
return bucket.max
|
||
}
|
||
}
|
||
|
||
// normalizeInfrastructureSummaryMetricPointSeries folds mixed-cadence history
|
||
// into equal-time buckets for the infrastructure summary endpoint so long-range
|
||
// sparklines do not bunch recent higher-resolution samples at the right edge.
|
||
func normalizeInfrastructureSummaryMetricPointSeries(
|
||
points []MetricPoint,
|
||
metricType string,
|
||
duration time.Duration,
|
||
windowEndMillis int64,
|
||
) []MetricPoint {
|
||
targetPoints := targetInfrastructureSummarySeriesPoints(duration)
|
||
if len(points) <= targetPoints || targetPoints < 2 || duration <= 0 {
|
||
return points
|
||
}
|
||
|
||
durationMillis := int64(duration / time.Millisecond)
|
||
if durationMillis <= 0 {
|
||
return points
|
||
}
|
||
|
||
windowStartMillis := windowEndMillis - durationMillis
|
||
bucketCount := targetPoints
|
||
buckets := make([]infrastructureSummaryBucket, bucketCount)
|
||
firstNonEmpty := -1
|
||
lastNonEmpty := -1
|
||
|
||
for _, point := range points {
|
||
if point.Timestamp < windowStartMillis || point.Timestamp > windowEndMillis {
|
||
continue
|
||
}
|
||
bucketIndex := int(((point.Timestamp - windowStartMillis) * int64(bucketCount)) / durationMillis)
|
||
if bucketIndex < 0 {
|
||
bucketIndex = 0
|
||
}
|
||
if bucketIndex >= bucketCount {
|
||
bucketIndex = bucketCount - 1
|
||
}
|
||
|
||
bucket := &buckets[bucketIndex]
|
||
if bucket.count == 0 {
|
||
bucket.max = point.Value
|
||
bucket.firstTimestamp = point.Timestamp
|
||
if firstNonEmpty == -1 {
|
||
firstNonEmpty = bucketIndex
|
||
}
|
||
} else if point.Value > bucket.max {
|
||
bucket.max = point.Value
|
||
}
|
||
bucket.count++
|
||
bucket.sum += point.Value
|
||
bucket.lastTimestamp = point.Timestamp
|
||
bucket.lastValue = point.Value
|
||
lastNonEmpty = bucketIndex
|
||
}
|
||
|
||
if firstNonEmpty == -1 || lastNonEmpty == -1 {
|
||
return points
|
||
}
|
||
|
||
result := make([]MetricPoint, 0, targetPoints)
|
||
for bucketIndex := 0; bucketIndex < bucketCount; bucketIndex++ {
|
||
bucket := buckets[bucketIndex]
|
||
if bucket.count == 0 {
|
||
continue
|
||
}
|
||
|
||
bucketStartMillis := windowStartMillis + (int64(bucketIndex)*durationMillis)/int64(bucketCount)
|
||
bucketEndMillis := windowStartMillis + (int64(bucketIndex+1)*durationMillis)/int64(bucketCount)
|
||
timestamp := bucketStartMillis + (bucketEndMillis-bucketStartMillis)/2
|
||
switch bucketIndex {
|
||
case firstNonEmpty:
|
||
timestamp = bucket.firstTimestamp
|
||
case lastNonEmpty:
|
||
timestamp = bucket.lastTimestamp
|
||
}
|
||
|
||
result = append(result, MetricPoint{
|
||
Timestamp: timestamp,
|
||
Value: aggregateInfrastructureSummaryBucketValue(
|
||
metricType,
|
||
bucket,
|
||
bucketIndex == lastNonEmpty,
|
||
),
|
||
})
|
||
}
|
||
|
||
if len(result) == 0 {
|
||
return points
|
||
}
|
||
return result
|
||
}
|
||
|
||
func normalizeInfrastructureSummaryChartSeries(
|
||
metrics map[string][]MetricPoint,
|
||
duration time.Duration,
|
||
windowEndMillis int64,
|
||
) {
|
||
for metricType, points := range metrics {
|
||
metrics[metricType] = normalizeInfrastructureSummaryMetricPointSeries(
|
||
points,
|
||
metricType,
|
||
duration,
|
||
windowEndMillis,
|
||
)
|
||
}
|
||
}
|
||
|
||
// sparklineMetrics lists the metric types consumed by summary sparklines
|
||
// and density maps. Metrics not in this set are omitted to keep payloads small.
|
||
// guestChartSourceView is the guest view subset the infrastructure summary
|
||
// chart builder consumes from VMs and LXC containers.
|
||
type guestChartSourceView interface {
|
||
comparable
|
||
SourceID() string
|
||
CPUPercent() float64
|
||
MemoryPercent() float64
|
||
MemoryUsed() int64
|
||
DiskPercent() float64
|
||
NetIn() float64
|
||
NetOut() float64
|
||
}
|
||
|
||
// collectGuestChartData batch-loads sparkline history for one proxmox guest
|
||
// family into chartData (1-2 SQL calls instead of N) and returns the updated
|
||
// oldest chart timestamp. Guests without history fall back to a single
|
||
// current-value point per metric.
|
||
func collectGuestChartData[V guestChartSourceView](
|
||
monitor *monitoring.Monitor,
|
||
storeType string,
|
||
guests []V,
|
||
duration time.Duration,
|
||
chartData map[string]VMChartData,
|
||
currentTime, oldestTimestamp int64,
|
||
) int64 {
|
||
var zero V
|
||
requests := make([]monitoring.GuestChartRequest, 0, len(guests))
|
||
for _, g := range guests {
|
||
if g == zero {
|
||
continue
|
||
}
|
||
if id := g.SourceID(); id != "" {
|
||
requests = append(requests, monitoring.GuestChartRequest{InMemoryKey: id, SQLResourceID: id})
|
||
}
|
||
}
|
||
batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, guestSparklineMetricOrder...)
|
||
for _, g := range guests {
|
||
if g == zero {
|
||
continue
|
||
}
|
||
id := g.SourceID()
|
||
if id == "" {
|
||
continue
|
||
}
|
||
chartData[id] = make(VMChartData)
|
||
if batchMetrics, ok := batch[id]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(chartData[id], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(chartData[id]["cpu"]) == 0 {
|
||
chartData[id]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: g.CPUPercent()}}
|
||
chartData[id]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: g.MemoryPercent()}}
|
||
chartData[id]["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(g.MemoryUsed())}}
|
||
chartData[id]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: g.DiskPercent()}}
|
||
chartData[id]["netin"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetIn()}}
|
||
chartData[id]["netout"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetOut()}}
|
||
}
|
||
}
|
||
return oldestTimestamp
|
||
}
|
||
|
||
// fillChartSeriesFromBatch copies sparkline-eligible batch metric points
|
||
// into dst and returns the updated oldest chart timestamp. Shared by the
|
||
// per-family infrastructure summary chart loops.
|
||
func fillChartSeriesFromBatch(dst VMChartData, batchMetrics map[string][]monitoring.MetricPoint, oldestTimestamp int64) int64 {
|
||
for metricType, points := range batchMetrics {
|
||
if !sparklineMetrics[metricType] {
|
||
continue
|
||
}
|
||
dst[metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
dst[metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
return oldestTimestamp
|
||
}
|
||
|
||
var sparklineMetrics = map[string]bool{
|
||
"cpu": true,
|
||
"memory": true,
|
||
"memoryused": true,
|
||
"disk": true,
|
||
"diskread": true,
|
||
"diskwrite": true,
|
||
"netin": true,
|
||
"netout": true,
|
||
}
|
||
|
||
var infrastructureSummaryMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"disk",
|
||
"diskread",
|
||
"diskwrite",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
var guestSparklineMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"memoryused",
|
||
"disk",
|
||
"diskread",
|
||
"diskwrite",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
var workloadSummaryMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"disk",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
func parseInfrastructureSummaryRequestedMetrics(
|
||
query url.Values,
|
||
) ([]string, map[string]bool, error) {
|
||
rawValues, ok := query["metrics"]
|
||
if !ok || len(rawValues) == 0 {
|
||
requested := make(map[string]bool, len(infrastructureSummaryMetricOrder))
|
||
for _, metricType := range infrastructureSummaryMetricOrder {
|
||
requested[metricType] = true
|
||
}
|
||
return append([]string(nil), infrastructureSummaryMetricOrder...), requested, nil
|
||
}
|
||
|
||
requestedList := make([]string, 0, len(infrastructureSummaryMetricOrder))
|
||
requestedSet := make(map[string]bool, len(infrastructureSummaryMetricOrder))
|
||
invalid := make([]string, 0)
|
||
|
||
for _, rawValue := range rawValues {
|
||
for _, part := range strings.Split(rawValue, ",") {
|
||
metricType := strings.TrimSpace(strings.ToLower(part))
|
||
if metricType == "" {
|
||
continue
|
||
}
|
||
if !sparklineMetrics[metricType] {
|
||
invalid = append(invalid, metricType)
|
||
continue
|
||
}
|
||
if requestedSet[metricType] {
|
||
continue
|
||
}
|
||
requestedSet[metricType] = true
|
||
requestedList = append(requestedList, metricType)
|
||
}
|
||
}
|
||
|
||
if len(invalid) > 0 {
|
||
return nil, nil, fmt.Errorf("invalid infrastructure metrics filter: %s", strings.Join(invalid, ", "))
|
||
}
|
||
if len(requestedList) == 0 {
|
||
return nil, nil, fmt.Errorf("infrastructure metrics filter must include at least one valid metric")
|
||
}
|
||
return requestedList, requestedSet, nil
|
||
}
|
||
|
||
func convertMetricsForChart(
|
||
metrics map[string][]monitoring.MetricPoint,
|
||
oldestTimestamp *int64,
|
||
maxPoints int,
|
||
) VMChartData {
|
||
converted := make(VMChartData, len(metrics))
|
||
for metricType, metricPoints := range metrics {
|
||
if !sparklineMetrics[metricType] {
|
||
continue
|
||
}
|
||
points := make([]MetricPoint, len(metricPoints))
|
||
for i, point := range metricPoints {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < *oldestTimestamp {
|
||
*oldestTimestamp = ts
|
||
}
|
||
points[i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
converted[metricType] = capMetricPointSeries(points, maxPoints)
|
||
}
|
||
return converted
|
||
}
|
||
|
||
// guestLiveMetricsView is the slice of the unified workload view API needed
|
||
// to seed a chart from live values; VM and container views both satisfy it.
|
||
type guestLiveMetricsView interface {
|
||
CPUPercent() float64
|
||
MemoryPercent() float64
|
||
MemoryUsed() int64
|
||
DiskPercent() float64
|
||
NetIn() float64
|
||
NetOut() float64
|
||
}
|
||
|
||
// guestChartSeriesWithLiveFallback converts a guest's batched metric history
|
||
// into chart series, substituting single live-value points when no history
|
||
// exists yet so freshly added guests still chart.
|
||
func guestChartSeriesWithLiveFallback(
|
||
metrics map[string][]monitoring.MetricPoint,
|
||
guest guestLiveMetricsView,
|
||
oldestTimestamp *int64,
|
||
maxPoints int,
|
||
currentTime int64,
|
||
) VMChartData {
|
||
series := convertMetricsForChart(metrics, oldestTimestamp, maxPoints)
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: guest.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: guest.MemoryPercent()}}
|
||
series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(guest.MemoryUsed())}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: guest.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetIn()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetOut()}}
|
||
}
|
||
return series
|
||
}
|
||
|
||
const (
|
||
mockWorkloadMinSeriesPoints = 24
|
||
mockWorkloadMaxSeriesPoints = 180
|
||
)
|
||
|
||
func targetMockSeriesPoints(duration time.Duration, maxPoints int) int {
|
||
target := int(duration / (2 * time.Minute))
|
||
if target < mockWorkloadMinSeriesPoints {
|
||
target = mockWorkloadMinSeriesPoints
|
||
}
|
||
if maxPoints > 0 && target > maxPoints {
|
||
target = maxPoints
|
||
}
|
||
if target > mockWorkloadMaxSeriesPoints {
|
||
target = mockWorkloadMaxSeriesPoints
|
||
}
|
||
if target < 2 {
|
||
target = 2
|
||
}
|
||
return target
|
||
}
|
||
|
||
func TargetMockSeriesPoints(duration time.Duration, maxPoints int) int {
|
||
return targetMockSeriesPoints(duration, maxPoints)
|
||
}
|
||
|
||
// mockMetricStyle returns the series style for a given metric type.
|
||
func mockMetricStyle(metricType string) monitoring.SeriesStyle {
|
||
switch metricType {
|
||
case "cpu", "diskread", "diskwrite", "netin", "netout":
|
||
return monitoring.StyleSpiky
|
||
case "memory":
|
||
return monitoring.StylePlateau
|
||
default:
|
||
return monitoring.StyleFlat
|
||
}
|
||
}
|
||
|
||
// generateStyledMockSeries produces a MetricPoint slice using the style-based
|
||
// generator from the monitoring package.
|
||
func generateStyledMockSeries(
|
||
nowMillis int64,
|
||
duration time.Duration,
|
||
numPoints int,
|
||
current float64,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
) []MetricPoint {
|
||
style := mockMetricStyle(metricType)
|
||
|
||
durationMillis := int64(duration / time.Millisecond)
|
||
if durationMillis <= 0 {
|
||
durationMillis = int64(time.Minute / time.Millisecond)
|
||
}
|
||
step := durationMillis / int64(numPoints-1)
|
||
if step <= 0 {
|
||
step = 1
|
||
}
|
||
startMillis := nowMillis - durationMillis
|
||
timestamps := make([]time.Time, numPoints)
|
||
for i := 0; i < numPoints; i++ {
|
||
timestamps[i] = time.UnixMilli(startMillis + int64(i)*step)
|
||
}
|
||
values := monitoring.GenerateSeededResourceMetricSeriesForTimestamps(
|
||
current,
|
||
timestamps,
|
||
resourceType,
|
||
resourceID,
|
||
metricType,
|
||
style,
|
||
)
|
||
points := make([]MetricPoint, numPoints)
|
||
for i := 0; i < numPoints; i++ {
|
||
points[i] = MetricPoint{
|
||
Timestamp: startMillis + int64(i)*step,
|
||
Value: values[i],
|
||
}
|
||
}
|
||
return points
|
||
}
|
||
|
||
func GenerateStyledMockSeries(
|
||
nowMillis int64,
|
||
duration time.Duration,
|
||
numPoints int,
|
||
current float64,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
) []MetricPoint {
|
||
return generateStyledMockSeries(nowMillis, duration, numPoints, current, resourceType, resourceID, metricType)
|
||
}
|
||
|
||
func buildSyntheticMetricHistorySeries(
|
||
now time.Time,
|
||
duration time.Duration,
|
||
maxPoints int,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
current float64,
|
||
) []monitoring.MetricPoint {
|
||
switch metricType {
|
||
case "disk", "diskread", "diskwrite", "usage":
|
||
case "smart_temp":
|
||
if current <= 0 {
|
||
return nil
|
||
}
|
||
default:
|
||
return nil
|
||
}
|
||
|
||
numPoints := targetMockSeriesPoints(duration, maxPoints)
|
||
series := generateStyledMockSeries(
|
||
now.UnixMilli(), duration, numPoints,
|
||
current, resourceType, resourceID, metricType,
|
||
)
|
||
|
||
converted := make([]monitoring.MetricPoint, len(series))
|
||
for i, point := range series {
|
||
converted[i] = monitoring.MetricPoint{
|
||
Timestamp: time.UnixMilli(point.Timestamp),
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
|
||
return converted
|
||
}
|
||
|
||
func BuildSyntheticMetricHistorySeries(
|
||
now time.Time,
|
||
duration time.Duration,
|
||
maxPoints int,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
current float64,
|
||
) []monitoring.MetricPoint {
|
||
return buildSyntheticMetricHistorySeries(now, duration, maxPoints, resourceType, resourceID, metricType, current)
|
||
}
|
||
|
||
// handleWorkloadCharts serves workload-only chart data used by workloads
|
||
// sparklines. It intentionally excludes infrastructure/storage chart payloads
|
||
// to keep requests small and stable for large fleets.
|
||
func (r *Service) HandleWorkloadCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workload charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
selectedNodeID := strings.TrimSpace(query.Get("node"))
|
||
maxPointsRaw := query.Get("maxPoints")
|
||
maxPoints := parseWorkloadMaxPoints(maxPointsRaw)
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
orgID := apicontext.OrgID(req.Context())
|
||
if orgID == "" {
|
||
orgID = "default"
|
||
}
|
||
cacheKey := orgID + "|" + timeRange + "|" + selectedNodeID + "|" + maxPointsRaw
|
||
|
||
cacheKey = workloadChartsCachePrefix + cacheKey
|
||
now := time.Now()
|
||
if body, ok := r.chartPayloads.get(cacheKey, now); ok {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if _, err := w.Write(body); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write cached workload chart data response")
|
||
}
|
||
return
|
||
}
|
||
|
||
v, err, _ := r.workloadChartsComputeGroup.Do(cacheKey, func() (any, error) {
|
||
// Re-check cache inside the singleflight barrier in case an earlier
|
||
// caller already populated it while we were queued.
|
||
if body, ok := r.chartPayloads.get(cacheKey, time.Now()); ok {
|
||
return body, nil
|
||
}
|
||
|
||
body, err := r.buildWorkloadChartsResponse(req.Context(), monitor, timeRange, selectedNodeID, maxPoints, duration, inMemoryChartThreshold)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
cachedAt := time.Now()
|
||
r.chartPayloads.put(cacheKey, body, cachedAt.Add(workloadChartsCacheTTL), cachedAt)
|
||
return body, nil
|
||
})
|
||
if err != nil {
|
||
log.Error().Err(err).Msg("Failed to build workload chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if _, err := w.Write(v.([]byte)); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write workload chart data response")
|
||
}
|
||
}
|
||
|
||
// buildWorkloadChartsResponse runs the heavy compute path for handleWorkloadCharts
|
||
// and returns the marshaled JSON body. Extracted so the handler can wrap it
|
||
// with caching + singleflight.
|
||
func (r *Service) buildWorkloadChartsResponse(
|
||
ctx context.Context,
|
||
monitor *monitoring.Monitor,
|
||
timeRange string,
|
||
selectedNodeID string,
|
||
maxPoints int,
|
||
duration time.Duration,
|
||
inMemoryChartThreshold time.Duration,
|
||
) ([]byte, error) {
|
||
_ = ctx
|
||
nodes := monitor.NodesSnapshot()
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
return nil, fmt.Errorf("state unavailable")
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
currentTime := time.Now().UnixMilli()
|
||
oldestTimestamp := currentTime
|
||
|
||
var selectedNode *models.Node
|
||
if selectedNodeID != "" {
|
||
for idx := range nodes {
|
||
if nodes[idx].ID == selectedNodeID {
|
||
selectedNode = &nodes[idx]
|
||
break
|
||
}
|
||
}
|
||
if selectedNode == nil {
|
||
log.Debug().
|
||
Str("selectedNodeID", selectedNodeID).
|
||
Msg("Workload charts node filter not found in current state; falling back to global scope")
|
||
}
|
||
}
|
||
|
||
matchesSelectedNode := func(instance, nodeName string) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) &&
|
||
strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name))
|
||
}
|
||
|
||
matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if host == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.Name()), nodeName)
|
||
}
|
||
|
||
matchesSelectedAgentHostView := func(host *unifiedresources.HostView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if host == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.AgentID()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.ID()), nodeName)
|
||
}
|
||
|
||
matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if pod == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName)
|
||
}
|
||
|
||
chartData := make(map[string]VMChartData)
|
||
dockerData := make(map[string]VMChartData)
|
||
|
||
guestTypes := make(map[string]string)
|
||
|
||
vmList := make([]*unifiedresources.VMView, 0)
|
||
vmResponseKeys := make([]string, 0)
|
||
vmRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, vm := range readState.VMs() {
|
||
if vm == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedNode(vm.Instance(), vm.Node()) {
|
||
continue
|
||
}
|
||
|
||
responseKey, request, ok := vmChartRequest(vm)
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
vmList = append(vmList, vm)
|
||
vmResponseKeys = append(vmResponseKeys, responseKey)
|
||
vmRequests = append(vmRequests, request)
|
||
}
|
||
containerList := make([]*unifiedresources.ContainerView, 0)
|
||
containerResponseKeys := make([]string, 0)
|
||
containerRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, ct := range readState.Containers() {
|
||
if ct == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedNode(ct.Instance(), ct.Node()) {
|
||
continue
|
||
}
|
||
|
||
responseKey, request, ok := systemContainerChartRequest(ct)
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
containerList = append(containerList, ct)
|
||
containerResponseKeys = append(containerResponseKeys, responseKey)
|
||
containerRequests = append(containerRequests, request)
|
||
}
|
||
podList := make([]*unifiedresources.PodView, 0)
|
||
podRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, pod := range readState.Pods() {
|
||
if pod == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedKubernetesPodView(pod) {
|
||
continue
|
||
}
|
||
|
||
metricKey := kubernetesPodMetricIDFromView(pod)
|
||
if metricKey == "" {
|
||
continue
|
||
}
|
||
|
||
podList = append(podList, pod)
|
||
podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey})
|
||
}
|
||
dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts()))
|
||
for _, host := range readState.DockerHosts() {
|
||
if host == nil {
|
||
continue
|
||
}
|
||
dockerHostsByID[host.ID()] = host
|
||
}
|
||
agentHostsByID := make(map[string]*unifiedresources.HostView, len(readState.Hosts()))
|
||
for _, host := range readState.Hosts() {
|
||
if host == nil {
|
||
continue
|
||
}
|
||
agentHostsByID[host.ID()] = host
|
||
}
|
||
|
||
dockerContainerList := make([]*unifiedresources.DockerContainerView, 0)
|
||
dockerContainerRequests := make([]monitoring.GuestChartRequest, 0)
|
||
dockerContainerKeys := make([]string, 0)
|
||
for _, container := range readState.DockerContainers() {
|
||
if container == nil {
|
||
continue
|
||
}
|
||
|
||
if selectedNodeID != "" && selectedNode != nil {
|
||
host := dockerHostsByID[container.ParentID()]
|
||
if host != nil {
|
||
if !matchesSelectedDockerHostView(host) {
|
||
continue
|
||
}
|
||
} else {
|
||
agentHost := agentHostsByID[container.ParentID()]
|
||
if agentHost == nil || !matchesSelectedAgentHostView(agentHost) {
|
||
continue
|
||
}
|
||
}
|
||
}
|
||
|
||
responseKey, request, ok := appContainerChartRequest(container)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dockerContainerList = append(dockerContainerList, container)
|
||
dockerContainerKeys = append(dockerContainerKeys, responseKey)
|
||
dockerContainerRequests = append(dockerContainerRequests, request)
|
||
}
|
||
var (
|
||
vmBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
containerBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
podBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
)
|
||
var workloadChartsBatchWG sync.WaitGroup
|
||
workloadChartsBatchWG.Add(4)
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, guestSparklineMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, guestSparklineMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, infrastructureSummaryMetricOrder...)
|
||
}()
|
||
workloadChartsBatchWG.Wait()
|
||
|
||
for idx, vm := range vmList {
|
||
responseKey := vmResponseKeys[idx]
|
||
metricID := vmRequests[idx].SQLResourceID
|
||
guestTypes[responseKey] = "vm"
|
||
chartData[responseKey] = guestChartSeriesWithLiveFallback(vmBatchMetrics[metricID], vm, &oldestTimestamp, maxPoints, currentTime)
|
||
}
|
||
|
||
for idx, ct := range containerList {
|
||
responseKey := containerResponseKeys[idx]
|
||
metricID := containerRequests[idx].SQLResourceID
|
||
guestTypes[responseKey] = "system-container"
|
||
chartData[responseKey] = guestChartSeriesWithLiveFallback(containerBatchMetrics[metricID], ct, &oldestTimestamp, maxPoints, currentTime)
|
||
}
|
||
|
||
for _, pod := range podList {
|
||
metricKey := kubernetesPodMetricIDFromView(pod)
|
||
series := convertMetricsForChart(podBatchMetrics[metricKey], &oldestTimestamp, maxPoints)
|
||
guestTypes[metricKey] = "k8s"
|
||
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: pod.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: pod.MemoryPercent()}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: pod.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetInRate()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetOutRate()}}
|
||
}
|
||
chartData[metricKey] = series
|
||
}
|
||
|
||
for idx, container := range dockerContainerList {
|
||
responseKey := dockerContainerKeys[idx]
|
||
metricID := dockerContainerRequests[idx].SQLResourceID
|
||
series := convertMetricsForChart(dockerContainerBatchMetrics[metricID], &oldestTimestamp, maxPoints)
|
||
guestTypes[responseKey] = "app-container"
|
||
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: container.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: container.MemoryPercent()}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: container.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetInRate()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetOutRate()}}
|
||
}
|
||
dockerData[responseKey] = series
|
||
}
|
||
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
guestPoints := countChartPoints(chartData)
|
||
dockerContainerPoints := countChartPoints(dockerData)
|
||
|
||
response := EmptyWorkloadChartsResponse()
|
||
response.ChartData = chartData
|
||
response.DockerData = dockerData
|
||
response.GuestTypes = guestTypes
|
||
response.Timestamp = currentTime
|
||
response.Stats = ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: guestPoints + dockerContainerPoints,
|
||
Guests: guestPoints,
|
||
DockerContainers: dockerContainerPoints,
|
||
},
|
||
}
|
||
|
||
body, err := json.Marshal(response.NormalizeCollections())
|
||
if err != nil {
|
||
return nil, fmt.Errorf("marshal workload chart response: %w", err)
|
||
}
|
||
return body, nil
|
||
}
|
||
|
||
// parseChartsRangeDuration converts the UI chart range query (e.g. "5m", "1h")
|
||
// into a duration. This is shared by /api/charts and /api/charts/infrastructure
|
||
// to prevent drift.
|
||
func parseChartsRangeDuration(rangeStr string) time.Duration {
|
||
switch rangeStr {
|
||
case "5m":
|
||
return 5 * time.Minute
|
||
case "15m":
|
||
return 15 * time.Minute
|
||
case "30m":
|
||
return 30 * time.Minute
|
||
case "1h":
|
||
return time.Hour
|
||
case "4h":
|
||
return 4 * time.Hour
|
||
case "8h":
|
||
return 8 * time.Hour
|
||
case "12h":
|
||
return 12 * time.Hour
|
||
case "24h":
|
||
return 24 * time.Hour
|
||
case "7d":
|
||
return 7 * 24 * time.Hour
|
||
case "30d":
|
||
return 30 * 24 * time.Hour
|
||
default:
|
||
return time.Hour
|
||
}
|
||
}
|
||
|
||
// handleInfrastructureCharts serves infrastructure-only chart data.
|
||
// This is intentionally narrower than /api/charts to reduce payload size and server-side compute
|
||
// for the Infrastructure page summary cards.
|
||
func (r *Service) HandleInfrastructureCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Infrastructure charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Get time range from query parameters
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
requestedMetricNames, requestedMetrics, err := parseInfrastructureSummaryRequestedMetrics(query)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||
return
|
||
}
|
||
// Convert time range to duration.
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
now := time.Now()
|
||
cacheKey := infrastructureChartsCacheKey(req, timeRange, requestedMetricNames)
|
||
if payload, ok := r.cachedInfrastructureChartsPayload(cacheKey, now); ok {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if _, err := w.Write(payload); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write cached infrastructure chart data response")
|
||
}
|
||
return
|
||
}
|
||
|
||
currentTime := now.UnixMilli()
|
||
oldestTimestamp := currentTime
|
||
|
||
// Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5).
|
||
nodeMetricTypes := make([]string, 0, 5)
|
||
for _, metricType := range []string{"cpu", "memory", "disk", "netin", "netout"} {
|
||
if requestedMetrics[metricType] {
|
||
nodeMetricTypes = append(nodeMetricTypes, metricType)
|
||
}
|
||
}
|
||
nodeData := make(map[string]NodeChartData)
|
||
nodeList := readState.Nodes()
|
||
nodeIDs := make([]string, 0, len(nodeList))
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
if nid := node.SourceID(); nid != "" {
|
||
nodeIDs = append(nodeIDs, nid)
|
||
}
|
||
}
|
||
nodeBatchMetrics := map[string]map[string][]monitoring.MetricPoint{}
|
||
if len(nodeMetricTypes) > 0 {
|
||
nodeBatchMetrics = monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration)
|
||
}
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
nid := node.SourceID()
|
||
if nid == "" {
|
||
continue
|
||
}
|
||
nodeData[nid] = make(NodeChartData)
|
||
if batchMetrics, ok := nodeBatchMetrics[nid]; ok {
|
||
for _, metricType := range nodeMetricTypes {
|
||
points, found := batchMetrics[metricType]
|
||
if !found {
|
||
continue
|
||
}
|
||
nodeData[nid][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
nodeData[nid][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range nodeMetricTypes {
|
||
if len(nodeData[nid][metricType]) > 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = node.CPUPercent()
|
||
case "memory":
|
||
value = node.MemoryPercent()
|
||
case "disk":
|
||
value = node.DiskPercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
nodeData[nid][metricType] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: value},
|
||
}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(nodeData[nid], duration, currentTime)
|
||
}
|
||
|
||
// Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerHostData := make(map[string]VMChartData)
|
||
dhList := readState.DockerHosts()
|
||
dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList))
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
if dhID := dh.HostSourceID(); dhID != "" {
|
||
dhRequests = append(dhRequests, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID),
|
||
SQLResourceID: dhID,
|
||
})
|
||
}
|
||
}
|
||
dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, requestedMetricNames...)
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
dhID := dh.HostSourceID()
|
||
if dhID == "" {
|
||
continue
|
||
}
|
||
dockerHostData[dhID] = make(VMChartData)
|
||
if batchMetrics, ok := dhBatchMetrics[dhID]; ok {
|
||
for metricType, points := range batchMetrics {
|
||
if !requestedMetrics[metricType] {
|
||
continue
|
||
}
|
||
dockerHostData[dhID][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
dockerHostData[dhID][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range requestedMetricNames {
|
||
if len(dockerHostData[dhID][metricType]) > 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = dh.CPUPercent()
|
||
case "memory":
|
||
value = dh.MemoryPercent()
|
||
case "disk":
|
||
if disks := dh.Disks(); len(disks) > 0 {
|
||
value = disks[0].Usage
|
||
}
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
dockerHostData[dhID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(dockerHostData[dhID], duration, currentTime)
|
||
}
|
||
|
||
// Process unified agents - batch-load historical data (1-2 SQL calls instead of N).
|
||
agentData := make(map[string]VMChartData)
|
||
hostList := readState.Hosts()
|
||
agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList))
|
||
for _, h := range hostList {
|
||
_, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentRequests = append(agentRequests, request)
|
||
}
|
||
agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, requestedMetricNames...)
|
||
for _, h := range hostList {
|
||
hID, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentData[hID] = make(VMChartData)
|
||
if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {
|
||
for metricType, points := range batchMetrics {
|
||
if !requestedMetrics[metricType] {
|
||
continue
|
||
}
|
||
agentData[hID][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
agentData[hID][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range requestedMetricNames {
|
||
if len(agentData[hID][metricType]) > 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = h.CPUPercent()
|
||
case "memory":
|
||
value = h.MemoryPercent()
|
||
case "disk":
|
||
value = h.DiskPercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
agentData[hID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(agentData[hID], duration, currentTime)
|
||
}
|
||
|
||
countNodePoints := func(metricsMap map[string]NodeChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
nodePoints := countNodePoints(nodeData)
|
||
dockerHostPoints := countChartPoints(dockerHostData)
|
||
agentPoints := countChartPoints(agentData)
|
||
|
||
response := EmptyInfrastructureChartsResponse()
|
||
response.NodeData = nodeData
|
||
response.DockerHostData = dockerHostData
|
||
response.AgentData = agentData
|
||
response.Timestamp = currentTime
|
||
response.Stats = ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: nodePoints + dockerHostPoints + agentPoints,
|
||
Nodes: nodePoints,
|
||
DockerHosts: dockerHostPoints,
|
||
Agents: agentPoints,
|
||
},
|
||
}
|
||
|
||
payload, err := json.Marshal(response.NormalizeCollections())
|
||
if err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode infrastructure chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
payload = append(payload, '\n')
|
||
r.cacheInfrastructureChartsPayload(cacheKey, payload, now)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if _, err := w.Write(payload); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write infrastructure chart data response")
|
||
}
|
||
}
|
||
|
||
func clampWorkloadPercent(value float64) float64 {
|
||
if value != value {
|
||
return 0
|
||
}
|
||
if value < 0 {
|
||
return 0
|
||
}
|
||
if value > 100 {
|
||
return 100
|
||
}
|
||
return value
|
||
}
|
||
|
||
func proxmoxModelCPURatioPercent(value float64) float64 {
|
||
return clampWorkloadPercent(value * 100)
|
||
}
|
||
|
||
func ProxmoxModelCPURatioPercent(value float64) float64 {
|
||
return proxmoxModelCPURatioPercent(value)
|
||
}
|
||
|
||
func kubernetesPodMetricIDFromView(pod *unifiedresources.PodView) string {
|
||
if pod == nil {
|
||
return ""
|
||
}
|
||
clusterKey := strings.TrimSpace(pod.ClusterID())
|
||
if clusterKey == "" {
|
||
clusterKey = strings.TrimSpace(pod.ClusterName())
|
||
}
|
||
podKey := strings.TrimSpace(pod.PodUID())
|
||
if podKey == "" {
|
||
namespace := strings.TrimSpace(pod.Namespace())
|
||
name := strings.TrimSpace(pod.Name())
|
||
if namespace != "" || name != "" {
|
||
podKey = fmt.Sprintf("%s/%s", namespace, name)
|
||
}
|
||
}
|
||
if clusterKey == "" || podKey == "" {
|
||
return ""
|
||
}
|
||
return fmt.Sprintf("k8s:%s:pod:%s", clusterKey, podKey)
|
||
}
|
||
|
||
// handleStorageCharts returns pool capacity and physical disk temperature
|
||
// time-series for the storage summary sparklines.
|
||
func (r *Service) HandleStorageCharts(w http.ResponseWriter, req *http.Request) {
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Parse query parameters
|
||
query := req.URL.Query()
|
||
rangeMinutes := 60 // default 1 hour
|
||
if rangeStr := query.Get("range"); rangeStr != "" {
|
||
if _, err := fmt.Sscanf(rangeStr, "%d", &rangeMinutes); err != nil {
|
||
log.Warn().Err(err).Str("range", rangeStr).Msg("Invalid range parameter; using default")
|
||
}
|
||
}
|
||
|
||
duration := time.Duration(rangeMinutes) * time.Minute
|
||
selectedNodeID := strings.TrimSpace(query.Get("node"))
|
||
|
||
// Use tenant-aware monitor
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Monitor not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
// Resolve node filter from canonical unified resources so storage charts use
|
||
// the same node identity model as the frontend storage page.
|
||
var selectedNodeName, selectedNodeInstance string
|
||
if selectedNodeID != "" {
|
||
found := false
|
||
for _, resource := range monitor.GetUnifiedResources() {
|
||
if strings.TrimSpace(resource.ID) != selectedNodeID {
|
||
continue
|
||
}
|
||
selectedNodeName = storageChartsSelectedNodeName(resource)
|
||
selectedNodeInstance = storageChartsSelectedNodeInstance(resource)
|
||
if selectedNodeName != "" || selectedNodeInstance != "" {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
for _, n := range monitor.NodesSnapshot() {
|
||
if n.ID == selectedNodeID {
|
||
selectedNodeName = n.Name
|
||
selectedNodeInstance = n.Instance
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if !found {
|
||
log.Debug().
|
||
Str("selectedNodeID", selectedNodeID).
|
||
Msg("Storage charts node filter not found in current state; falling back to global scope")
|
||
}
|
||
}
|
||
matchesNode := func(nodeName, instance string) bool {
|
||
if selectedNodeName == "" {
|
||
return true
|
||
}
|
||
if !strings.EqualFold(strings.TrimSpace(nodeName), selectedNodeName) {
|
||
return false
|
||
}
|
||
if selectedNodeInstance != "" && instance != "" {
|
||
return strings.EqualFold(strings.TrimSpace(instance), selectedNodeInstance)
|
||
}
|
||
return true
|
||
}
|
||
|
||
// Build pool chart data from the canonical storage summary batch path so
|
||
// the dashboard and storage page share one efficient history retrieval model.
|
||
poolNames := make(map[string]string, len(readState.StoragePools()))
|
||
storageIDs := make([]string, 0, len(readState.StoragePools()))
|
||
for _, sp := range readState.StoragePools() {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
if !matchesNode(sp.Node(), sp.Instance()) {
|
||
continue
|
||
}
|
||
sid := sp.SourceID()
|
||
if sid == "" {
|
||
continue
|
||
}
|
||
poolNames[sid] = sp.Name()
|
||
storageIDs = append(storageIDs, sid)
|
||
}
|
||
|
||
poolMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration)
|
||
pools := make(map[string]StoragePoolChartData, len(storageIDs))
|
||
for _, sid := range storageIDs {
|
||
metrics := poolMetrics[sid]
|
||
pools[sid] = StoragePoolChartData{
|
||
Name: poolNames[sid],
|
||
Usage: monitorPointsToAPI(metrics["usage"]),
|
||
Used: monitorPointsToAPI(metrics["used"]),
|
||
Avail: monitorPointsToAPI(metrics["avail"]),
|
||
}
|
||
}
|
||
|
||
// Build disk temperature chart data
|
||
diskEntries := monitor.GetPhysicalDiskTemperatureCharts(duration)
|
||
disks := make(map[string]StorageDiskChartData, len(diskEntries))
|
||
for id, entry := range diskEntries {
|
||
if !matchesNode(entry.Node, entry.Instance) {
|
||
continue
|
||
}
|
||
disks[id] = StorageDiskChartData{
|
||
Name: entry.Name,
|
||
Node: entry.Node,
|
||
Temperature: monitorPointsToAPI(entry.Temperature),
|
||
}
|
||
}
|
||
|
||
resp := EmptyStorageChartsResponse()
|
||
resp.Pools = pools
|
||
resp.Disks = disks
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := json.NewEncoder(w).Encode(resp.NormalizeCollections()); err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode storage chart data")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
}
|
||
}
|
||
|
||
// monitorPointsToAPI converts monitoring MetricPoints (time.Time timestamps)
|
||
// to API MetricPoints (Unix millisecond timestamps) for JSON serialization.
|
||
func monitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint {
|
||
if len(points) == 0 {
|
||
return nil
|
||
}
|
||
out := make([]MetricPoint, len(points))
|
||
for i, p := range points {
|
||
out[i] = MetricPoint{Timestamp: p.Timestamp.UnixMilli(), Value: p.Value}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func MonitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint {
|
||
return monitorPointsToAPI(points)
|
||
}
|