mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 13:39:47 +00:00
pulse-triage[bot]
c93757fb4b
Share canonical disk admission with host-agent recording and fallback, preserve measured zero and recorded observations, and check both History and summary routes. Change-source: pulse-maintainer Contract-Neutral: Restore existing canonical disk-presence semantics; no new API, authentication, identity, persistence schema or retention contract.
2268 lines
66 KiB
Go
2268 lines
66 KiB
Go
package chartapi
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"math"
|
||
"net/http"
|
||
"net/url"
|
||
"strconv"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/api/apicontext"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
||
unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||
"github.com/rs/zerolog/log"
|
||
"golang.org/x/sync/singleflight"
|
||
)
|
||
|
||
const workloadChartsCacheTTL = 3 * time.Second
|
||
const summaryChartsCacheTTL = 5 * time.Second
|
||
|
||
const (
|
||
// Both cached chart routes share one retention budget. This stays modest for
|
||
// low-memory container deployments while still holding several ordinary
|
||
// large-estate responses.
|
||
chartPayloadCacheMaxEntries = 64
|
||
chartPayloadCacheMaxBytes = 16 << 20
|
||
|
||
infrastructureChartsCachePrefix = "infrastructure|"
|
||
workloadChartsCachePrefix = "workload-charts|"
|
||
)
|
||
|
||
// MonitorResolver supplies the authenticated tenant monitor selected by the
|
||
// router context. Chart computation remains entirely owned by Service.
|
||
type MonitorResolver interface {
|
||
MonitorForContext(context.Context) *monitoring.Monitor
|
||
}
|
||
|
||
// Service owns chart queries, aggregation, serialization, caching, and
|
||
// singleflight coordination independently of the HTTP router package.
|
||
type Service struct {
|
||
resolver MonitorResolver
|
||
|
||
chartPayloads boundedChartPayloadCache
|
||
workloadChartsComputeGroup singleflight.Group
|
||
}
|
||
|
||
func NewService(resolver MonitorResolver) *Service {
|
||
return &Service{
|
||
resolver: resolver,
|
||
chartPayloads: newBoundedChartPayloadCache(
|
||
chartPayloadCacheMaxEntries,
|
||
chartPayloadCacheMaxBytes,
|
||
),
|
||
}
|
||
}
|
||
|
||
func (r *Service) getTenantMonitor(ctx context.Context) *monitoring.Monitor {
|
||
if r == nil || r.resolver == nil {
|
||
return nil
|
||
}
|
||
return r.resolver.MonitorForContext(ctx)
|
||
}
|
||
|
||
func storageChartsSelectedNodeName(resource unifiedresources.Resource) string {
|
||
if name := strings.TrimSpace(resource.Name); name != "" {
|
||
return name
|
||
}
|
||
if resource.TrueNAS != nil {
|
||
if hostname := strings.TrimSpace(resource.TrueNAS.Hostname); hostname != "" {
|
||
return hostname
|
||
}
|
||
}
|
||
for _, hostname := range resource.Identity.Hostnames {
|
||
if hostname = strings.TrimSpace(hostname); hostname != "" {
|
||
return hostname
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
func storageChartsSelectedNodeInstance(resource unifiedresources.Resource) string {
|
||
if resource.Proxmox == nil {
|
||
return ""
|
||
}
|
||
return strings.TrimSpace(resource.Proxmox.Instance)
|
||
}
|
||
|
||
// handleCharts handles chart data requests
|
||
func (r *Service) HandleCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Get time range from query parameters
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
|
||
// Convert time range to duration.
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
// Get tenant-specific monitor and current state
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
// Create chart data structure that matches frontend expectations
|
||
chartData := make(map[string]VMChartData)
|
||
nodeData := make(map[string]NodeChartData)
|
||
|
||
currentTime := time.Now().UnixMilli() // JavaScript timestamp format
|
||
oldestTimestamp := currentTime
|
||
|
||
// Process VMs and Containers - batch-load historical data (1-2 SQL calls
|
||
// per family instead of N).
|
||
oldestTimestamp = collectGuestChartData(monitor, "vm", readState.VMs(), duration, chartData, currentTime, oldestTimestamp)
|
||
oldestTimestamp = collectGuestChartData(monitor, "container", readState.Containers(), duration, chartData, currentTime, oldestTimestamp)
|
||
|
||
// Process Storage - batch-load historical data (1-2 SQL calls instead of N).
|
||
storageData := make(map[string]StorageChartData)
|
||
spList := readState.StoragePools()
|
||
storageIDs := make([]string, 0, len(spList))
|
||
for _, sp := range spList {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
if sid := sp.SourceID(); sid != "" {
|
||
storageIDs = append(storageIDs, sid)
|
||
}
|
||
}
|
||
storageBatchMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration)
|
||
for _, sp := range spList {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
sid := sp.SourceID()
|
||
if sid == "" {
|
||
continue
|
||
}
|
||
storageData[sid] = make(StorageChartData)
|
||
if batchMetrics, ok := storageBatchMetrics[sid]; ok {
|
||
if usagePoints, found := batchMetrics["usage"]; found && len(usagePoints) > 0 {
|
||
storageData[sid]["disk"] = make([]MetricPoint, len(usagePoints))
|
||
for i, point := range usagePoints {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
storageData[sid]["disk"][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if len(storageData[sid]["disk"]) == 0 {
|
||
storageData[sid]["disk"] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: sp.DiskPercent()},
|
||
}
|
||
}
|
||
}
|
||
|
||
// Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5).
|
||
nodeMetricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
|
||
nodeList := readState.Nodes()
|
||
nodeIDs := make([]string, 0, len(nodeList))
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
if nid := node.SourceID(); nid != "" {
|
||
nodeIDs = append(nodeIDs, nid)
|
||
}
|
||
}
|
||
nodeBatchMetrics := monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration)
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
nid := node.SourceID()
|
||
if nid == "" {
|
||
continue
|
||
}
|
||
nodeData[nid] = make(NodeChartData)
|
||
if batchMetrics, ok := nodeBatchMetrics[nid]; ok {
|
||
for _, metricType := range nodeMetricTypes {
|
||
points, found := batchMetrics[metricType]
|
||
if !found {
|
||
continue
|
||
}
|
||
nodeData[nid][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
nodeData[nid][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range nodeMetricTypes {
|
||
if len(nodeData[nid][metricType]) == 0 {
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = node.CPUPercent()
|
||
case "memory":
|
||
value = node.MemoryPercent()
|
||
case "disk":
|
||
value = node.DiskPercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
nodeData[nid][metricType] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: value},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Build guest type map with canonical v6 names.
|
||
guestTypes := make(map[string]string)
|
||
for _, vm := range readState.VMs() {
|
||
if vm == nil {
|
||
continue
|
||
}
|
||
if sid := vm.SourceID(); sid != "" {
|
||
guestTypes[sid] = "vm"
|
||
}
|
||
}
|
||
for _, ct := range readState.Containers() {
|
||
if ct == nil {
|
||
continue
|
||
}
|
||
if sid := ct.SourceID(); sid != "" {
|
||
guestTypes[sid] = "system-container"
|
||
}
|
||
}
|
||
for _, dc := range readState.DockerContainers() {
|
||
if dc == nil {
|
||
continue
|
||
}
|
||
if key := strings.TrimSpace(dc.ID()); key != "" {
|
||
guestTypes[key] = "app-container"
|
||
}
|
||
}
|
||
|
||
// Process Docker containers - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerData := make(map[string]VMChartData)
|
||
dcList := readState.DockerContainers()
|
||
dcRequests := make([]monitoring.GuestChartRequest, 0, len(dcList))
|
||
for _, dc := range dcList {
|
||
_, request, ok := appContainerChartRequest(dc)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dcRequests = append(dcRequests, request)
|
||
}
|
||
dcBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerContainer", dcRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, dc := range dcList {
|
||
responseKey, request, ok := appContainerChartRequest(dc)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dockerData[responseKey] = make(VMChartData)
|
||
if batchMetrics, ok := dcBatchMetrics[request.SQLResourceID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(dockerData[responseKey], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(dockerData[responseKey]["cpu"]) == 0 {
|
||
dockerData[responseKey]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dc.CPUPercent()}}
|
||
dockerData[responseKey]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dc.MemoryPercent()}}
|
||
dockerData[responseKey]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: dc.DiskPercent()}}
|
||
}
|
||
}
|
||
|
||
// Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerHostData := make(map[string]VMChartData)
|
||
dhList := readState.DockerHosts()
|
||
dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList))
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
if dhID := dh.HostSourceID(); dhID != "" {
|
||
dhRequests = append(dhRequests, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID),
|
||
SQLResourceID: dhID,
|
||
})
|
||
}
|
||
}
|
||
dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
dhID := dh.HostSourceID()
|
||
if dhID == "" {
|
||
continue
|
||
}
|
||
dockerHostData[dhID] = make(VMChartData)
|
||
if batchMetrics, ok := dhBatchMetrics[dhID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(dockerHostData[dhID], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(dockerHostData[dhID]["cpu"]) == 0 {
|
||
dockerHostData[dhID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: dh.CPUPercent()}}
|
||
dockerHostData[dhID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: dh.MemoryPercent()}}
|
||
var diskPercent float64
|
||
if disks := dh.Disks(); len(disks) > 0 {
|
||
diskPercent = disks[0].Usage
|
||
}
|
||
dockerHostData[dhID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: diskPercent}}
|
||
}
|
||
}
|
||
|
||
// Process unified agents - batch-load historical data (1-2 SQL calls instead of N).
|
||
agentData := make(map[string]VMChartData)
|
||
hostList := readState.Hosts()
|
||
agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList))
|
||
for _, h := range hostList {
|
||
_, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentRequests = append(agentRequests, request)
|
||
}
|
||
agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, infrastructureSummaryMetricOrder...)
|
||
for _, h := range hostList {
|
||
hID, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentData[hID] = make(VMChartData)
|
||
if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp)
|
||
}
|
||
if hostAgentCanSupplyLiveChartFallback(h) {
|
||
if len(agentData[hID]["cpu"]) == 0 {
|
||
agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}}
|
||
}
|
||
if len(agentData[hID]["memory"]) == 0 {
|
||
agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}}
|
||
}
|
||
if len(agentData[hID]["disk"]) == 0 {
|
||
if value, ok := h.DiskUsagePercent(); ok {
|
||
agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: value}}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
countNodePoints := func(metricsMap map[string]NodeChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
countStoragePoints := func(metricsMap map[string]StorageChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
guestPoints := countChartPoints(chartData)
|
||
nodePoints := countNodePoints(nodeData)
|
||
storagePoints := countStoragePoints(storageData)
|
||
dockerContainerPoints := countChartPoints(dockerData)
|
||
dockerHostPoints := countChartPoints(dockerHostData)
|
||
agentPoints := countChartPoints(agentData)
|
||
|
||
response := ChartResponse{
|
||
ChartData: chartData,
|
||
NodeData: nodeData,
|
||
StorageData: storageData,
|
||
DockerData: dockerData,
|
||
DockerHostData: dockerHostData,
|
||
AgentData: agentData,
|
||
GuestTypes: guestTypes,
|
||
Timestamp: currentTime,
|
||
Stats: ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: guestPoints + nodePoints + storagePoints + dockerContainerPoints + dockerHostPoints + agentPoints,
|
||
Guests: guestPoints,
|
||
Nodes: nodePoints,
|
||
Storage: storagePoints,
|
||
DockerContainers: dockerContainerPoints,
|
||
DockerHosts: dockerHostPoints,
|
||
Agents: agentPoints,
|
||
},
|
||
},
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryJSON(w, req, response, duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
log.Debug().
|
||
Int("guests", len(chartData)).
|
||
Int("nodes", len(nodeData)).
|
||
Int("storage", len(storageData)).
|
||
Int("dockerContainers", len(dockerData)).
|
||
Int("agents", len(agentData)).
|
||
Str("range", timeRange).
|
||
Msg("Chart data response sent")
|
||
}
|
||
|
||
func parseWorkloadMaxPoints(raw string) int {
|
||
const (
|
||
defaultMaxPoints = 180
|
||
minMaxPoints = 30
|
||
maxMaxPoints = 500
|
||
)
|
||
|
||
trimmed := strings.TrimSpace(raw)
|
||
if trimmed == "" {
|
||
return defaultMaxPoints
|
||
}
|
||
|
||
value, err := strconv.Atoi(trimmed)
|
||
if err != nil {
|
||
return defaultMaxPoints
|
||
}
|
||
if value < minMaxPoints {
|
||
return minMaxPoints
|
||
}
|
||
if value > maxMaxPoints {
|
||
return maxMaxPoints
|
||
}
|
||
return value
|
||
}
|
||
|
||
func ParseWorkloadMaxPoints(raw string) int { return parseWorkloadMaxPoints(raw) }
|
||
|
||
func hostAgentCanSupplyLiveChartFallback(host *unifiedresources.HostView) bool {
|
||
if host.Status() == unifiedresources.StatusOffline {
|
||
return false
|
||
}
|
||
status, ok := host.SourceStatus(unifiedresources.SourceAgent)
|
||
return !ok || status.Status != "pending"
|
||
}
|
||
|
||
func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) {
|
||
if host == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
if agentID := strings.TrimSpace(host.AgentID()); agentID != "" {
|
||
return agentID, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("agent:%s", agentID),
|
||
SQLResourceID: agentID,
|
||
}, true
|
||
}
|
||
|
||
target := host.MetricsTarget()
|
||
if target == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
metricID := strings.TrimSpace(target.ResourceID)
|
||
if metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return metricID, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("agent:%s", metricID),
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func appContainerChartMetricID(container *unifiedresources.DockerContainerView) string {
|
||
if container == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := container.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(container.ContainerID())
|
||
}
|
||
|
||
func appContainerChartRequest(container *unifiedresources.DockerContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
if container == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := strings.TrimSpace(container.ID())
|
||
if responseKey == "" {
|
||
responseKey = strings.TrimSpace(container.ContainerID())
|
||
}
|
||
metricID := appContainerChartMetricID(container)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("docker:%s", metricID),
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func canonicalGuestResponseKey(resourceID, instance, node string, vmid int) string {
|
||
trimmedInstance := strings.TrimSpace(instance)
|
||
trimmedNode := strings.TrimSpace(node)
|
||
if trimmedInstance != "" && trimmedNode != "" && vmid > 0 {
|
||
return fmt.Sprintf("%s:%s:%d", trimmedInstance, trimmedNode, vmid)
|
||
}
|
||
return strings.TrimSpace(resourceID)
|
||
}
|
||
|
||
func vmChartMetricID(vm *unifiedresources.VMView) string {
|
||
if vm == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := vm.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(vm.SourceID())
|
||
}
|
||
|
||
func vmChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) {
|
||
if vm == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := canonicalGuestResponseKey(vm.ID(), vm.Instance(), vm.Node(), vm.VMID())
|
||
metricID := vmChartMetricID(vm)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: metricID,
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func VMChartRequest(vm *unifiedresources.VMView) (string, monitoring.GuestChartRequest, bool) {
|
||
return vmChartRequest(vm)
|
||
}
|
||
|
||
func systemContainerChartMetricID(container *unifiedresources.ContainerView) string {
|
||
if container == nil {
|
||
return ""
|
||
}
|
||
|
||
if target := container.MetricsTarget(); target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
|
||
return strings.TrimSpace(container.SourceID())
|
||
}
|
||
|
||
func systemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
if container == nil {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
responseKey := canonicalGuestResponseKey(container.ID(), container.Instance(), container.Node(), container.VMID())
|
||
metricID := systemContainerChartMetricID(container)
|
||
if responseKey == "" || metricID == "" {
|
||
return "", monitoring.GuestChartRequest{}, false
|
||
}
|
||
|
||
return responseKey, monitoring.GuestChartRequest{
|
||
InMemoryKey: metricID,
|
||
SQLResourceID: metricID,
|
||
}, true
|
||
}
|
||
|
||
func SystemContainerChartRequest(container *unifiedresources.ContainerView) (string, monitoring.GuestChartRequest, bool) {
|
||
return systemContainerChartRequest(container)
|
||
}
|
||
|
||
func capMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
if len(points) <= maxPoints || maxPoints <= 0 {
|
||
return points
|
||
}
|
||
indices := chartPointIndicesByIndex(len(points), maxPoints)
|
||
result := make([]MetricPoint, len(indices))
|
||
for i, index := range indices {
|
||
result[i] = points[index]
|
||
}
|
||
return result
|
||
}
|
||
|
||
func CapMetricPointSeriesByIndex(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
return capMetricPointSeriesByIndex(points, maxPoints)
|
||
}
|
||
|
||
const (
|
||
infrastructureSummaryMinSeriesPoints = 24
|
||
infrastructureSummaryMaxSeriesPoints = 96
|
||
)
|
||
|
||
const InfrastructureSummaryMaxSeriesPoints = infrastructureSummaryMaxSeriesPoints
|
||
|
||
func chartPointIndicesByIndex(count, maxPoints int) []int {
|
||
if maxPoints == 1 {
|
||
return []int{count - 1}
|
||
}
|
||
indices := make([]int, 0, maxPoints)
|
||
step := float64(count-1) / float64(maxPoints-1)
|
||
prevIndex := -1
|
||
for i := 0; i < maxPoints; i++ {
|
||
index := int(float64(i)*step + 0.5)
|
||
if index <= prevIndex {
|
||
index = prevIndex + 1
|
||
}
|
||
if index >= count {
|
||
index = count - 1
|
||
}
|
||
indices = append(indices, index)
|
||
prevIndex = index
|
||
}
|
||
return indices
|
||
}
|
||
|
||
// chartPointIndices selects original observations, not averages or synthetic
|
||
// timestamps. Call only when count > maxPoints > 0. Keeping indices lets the
|
||
// retained reader apply the same cap before allocating wire-format points.
|
||
func chartPointIndices(count, maxPoints int, timestampAt func(int) int64) []int {
|
||
if maxPoints == 1 {
|
||
return []int{count - 1}
|
||
}
|
||
startTimestamp, endTimestamp := timestampAt(0), timestampAt(count-1)
|
||
if endTimestamp <= startTimestamp {
|
||
return chartPointIndicesByIndex(count, maxPoints)
|
||
}
|
||
bucketSpan := float64(endTimestamp-startTimestamp) / float64(maxPoints-1)
|
||
if bucketSpan < 1 {
|
||
return chartPointIndicesByIndex(count, maxPoints)
|
||
}
|
||
|
||
type representative struct {
|
||
indexPlusOne int // zero means no observation in this time bucket
|
||
distance float64
|
||
}
|
||
buckets := make([]representative, maxPoints)
|
||
for pointIndex := 0; pointIndex < count; pointIndex++ {
|
||
timestamp := timestampAt(pointIndex)
|
||
index := int(math.Round(float64(timestamp-startTimestamp) / bucketSpan))
|
||
if index < 0 {
|
||
index = 0
|
||
}
|
||
if index >= maxPoints {
|
||
index = maxPoints - 1
|
||
}
|
||
targetTimestamp := float64(startTimestamp) + bucketSpan*float64(index)
|
||
distance := math.Abs(float64(timestamp) - targetTimestamp)
|
||
current := buckets[index]
|
||
if current.indexPlusOne == 0 || distance < current.distance ||
|
||
(distance == current.distance && timestamp > timestampAt(current.indexPlusOne-1)) {
|
||
buckets[index] = representative{indexPlusOne: pointIndex + 1, distance: distance}
|
||
}
|
||
}
|
||
|
||
result := make([]int, 0, maxPoints)
|
||
result = append(result, 0)
|
||
lastAddedTimestamp := startTimestamp
|
||
for index := 1; index < maxPoints-1; index++ {
|
||
bucket := buckets[index]
|
||
if bucket.indexPlusOne == 0 {
|
||
continue
|
||
}
|
||
pointIndex := bucket.indexPlusOne - 1
|
||
timestamp := timestampAt(pointIndex)
|
||
if timestamp <= lastAddedTimestamp {
|
||
continue
|
||
}
|
||
result = append(result, pointIndex)
|
||
lastAddedTimestamp = timestamp
|
||
}
|
||
if endTimestamp <= lastAddedTimestamp {
|
||
result[len(result)-1] = count - 1
|
||
return result
|
||
}
|
||
return append(result, count-1)
|
||
}
|
||
|
||
// capMetricPointSeries keeps mixed-cadence series visually proportional across
|
||
// the selected time window. Index-based capping over-selects recent dense
|
||
// samples, which bunches the right edge on long ranges.
|
||
func capMetricPointSeries(points []MetricPoint, maxPoints int) []MetricPoint {
|
||
if len(points) <= maxPoints || maxPoints <= 0 {
|
||
return points
|
||
}
|
||
indices := chartPointIndices(len(points), maxPoints, func(i int) int64 { return points[i].Timestamp })
|
||
result := make([]MetricPoint, len(indices))
|
||
for i, index := range indices {
|
||
result[i] = points[index]
|
||
}
|
||
return result
|
||
}
|
||
|
||
func targetBoundedSummarySeriesPoints(duration time.Duration, minPoints, maxPoints int) int {
|
||
if duration <= 0 {
|
||
return minPoints
|
||
}
|
||
|
||
target := int(duration / time.Minute)
|
||
if target < minPoints {
|
||
target = minPoints
|
||
}
|
||
if target > maxPoints {
|
||
target = maxPoints
|
||
}
|
||
if target < 2 {
|
||
target = 2
|
||
}
|
||
return target
|
||
}
|
||
|
||
type infrastructureSummaryBucket struct {
|
||
count int
|
||
sum float64
|
||
max float64
|
||
firstTimestamp int64
|
||
lastTimestamp int64
|
||
lastValue float64
|
||
}
|
||
|
||
func targetInfrastructureSummarySeriesPoints(duration time.Duration) int {
|
||
return targetBoundedSummarySeriesPoints(
|
||
duration,
|
||
infrastructureSummaryMinSeriesPoints,
|
||
infrastructureSummaryMaxSeriesPoints,
|
||
)
|
||
}
|
||
|
||
func infrastructureChartsCacheKey(req *http.Request, timeRange string, requestedMetricNames []string) string {
|
||
orgID := strings.TrimSpace(apicontext.OrgID(req.Context()))
|
||
if orgID == "" {
|
||
orgID = "default"
|
||
}
|
||
return orgID + "|" + strings.TrimSpace(timeRange) + "|" + strings.Join(requestedMetricNames, ",")
|
||
}
|
||
|
||
func (r *Service) cachedInfrastructureChartsPayload(key string, now time.Time) ([]byte, bool) {
|
||
if r == nil || key == "" {
|
||
return nil, false
|
||
}
|
||
return r.chartPayloads.get(infrastructureChartsCachePrefix+key, now)
|
||
}
|
||
|
||
func (r *Service) cacheInfrastructureChartsPayload(key string, payload []byte, now time.Time) {
|
||
if r == nil || key == "" || len(payload) == 0 {
|
||
return
|
||
}
|
||
r.chartPayloads.put(
|
||
infrastructureChartsCachePrefix+key,
|
||
payload,
|
||
now.Add(summaryChartsCacheTTL),
|
||
now,
|
||
)
|
||
}
|
||
|
||
func aggregateInfrastructureSummaryBucketValue(
|
||
metricType string,
|
||
bucket infrastructureSummaryBucket,
|
||
isLastBucket bool,
|
||
) float64 {
|
||
if bucket.count == 0 {
|
||
return 0
|
||
}
|
||
if isLastBucket {
|
||
return bucket.lastValue
|
||
}
|
||
|
||
switch metricType {
|
||
case "memory", "disk":
|
||
return bucket.sum / float64(bucket.count)
|
||
default:
|
||
return bucket.max
|
||
}
|
||
}
|
||
|
||
// normalizeInfrastructureSummaryMetricPointSeries folds mixed-cadence history
|
||
// into equal-time buckets for the infrastructure summary endpoint so long-range
|
||
// sparklines do not bunch recent higher-resolution samples at the right edge.
|
||
func normalizeInfrastructureSummaryMetricPointSeries(
|
||
points []MetricPoint,
|
||
metricType string,
|
||
duration time.Duration,
|
||
windowEndMillis int64,
|
||
) []MetricPoint {
|
||
targetPoints := targetInfrastructureSummarySeriesPoints(duration)
|
||
if len(points) <= targetPoints || targetPoints < 2 || duration <= 0 {
|
||
return points
|
||
}
|
||
|
||
durationMillis := int64(duration / time.Millisecond)
|
||
if durationMillis <= 0 {
|
||
return points
|
||
}
|
||
|
||
windowStartMillis := windowEndMillis - durationMillis
|
||
bucketCount := targetPoints
|
||
buckets := make([]infrastructureSummaryBucket, bucketCount)
|
||
firstNonEmpty := -1
|
||
lastNonEmpty := -1
|
||
|
||
for _, point := range points {
|
||
if point.Timestamp < windowStartMillis || point.Timestamp > windowEndMillis {
|
||
continue
|
||
}
|
||
bucketIndex := int(((point.Timestamp - windowStartMillis) * int64(bucketCount)) / durationMillis)
|
||
if bucketIndex < 0 {
|
||
bucketIndex = 0
|
||
}
|
||
if bucketIndex >= bucketCount {
|
||
bucketIndex = bucketCount - 1
|
||
}
|
||
|
||
bucket := &buckets[bucketIndex]
|
||
if bucket.count == 0 {
|
||
bucket.max = point.Value
|
||
bucket.firstTimestamp = point.Timestamp
|
||
if firstNonEmpty == -1 {
|
||
firstNonEmpty = bucketIndex
|
||
}
|
||
} else if point.Value > bucket.max {
|
||
bucket.max = point.Value
|
||
}
|
||
bucket.count++
|
||
bucket.sum += point.Value
|
||
bucket.lastTimestamp = point.Timestamp
|
||
bucket.lastValue = point.Value
|
||
lastNonEmpty = bucketIndex
|
||
}
|
||
|
||
if firstNonEmpty == -1 || lastNonEmpty == -1 {
|
||
return points
|
||
}
|
||
|
||
result := make([]MetricPoint, 0, targetPoints)
|
||
for bucketIndex := 0; bucketIndex < bucketCount; bucketIndex++ {
|
||
bucket := buckets[bucketIndex]
|
||
if bucket.count == 0 {
|
||
continue
|
||
}
|
||
|
||
bucketStartMillis := windowStartMillis + (int64(bucketIndex)*durationMillis)/int64(bucketCount)
|
||
bucketEndMillis := windowStartMillis + (int64(bucketIndex+1)*durationMillis)/int64(bucketCount)
|
||
timestamp := bucketStartMillis + (bucketEndMillis-bucketStartMillis)/2
|
||
switch bucketIndex {
|
||
case firstNonEmpty:
|
||
timestamp = bucket.firstTimestamp
|
||
case lastNonEmpty:
|
||
timestamp = bucket.lastTimestamp
|
||
}
|
||
|
||
result = append(result, MetricPoint{
|
||
Timestamp: timestamp,
|
||
Value: aggregateInfrastructureSummaryBucketValue(
|
||
metricType,
|
||
bucket,
|
||
bucketIndex == lastNonEmpty,
|
||
),
|
||
})
|
||
}
|
||
|
||
if len(result) == 0 {
|
||
return points
|
||
}
|
||
return result
|
||
}
|
||
|
||
func normalizeInfrastructureSummaryChartSeries(
|
||
metrics map[string][]MetricPoint,
|
||
duration time.Duration,
|
||
windowEndMillis int64,
|
||
) {
|
||
for metricType, points := range metrics {
|
||
metrics[metricType] = normalizeInfrastructureSummaryMetricPointSeries(
|
||
points,
|
||
metricType,
|
||
duration,
|
||
windowEndMillis,
|
||
)
|
||
}
|
||
}
|
||
|
||
// sparklineMetrics lists the metric types consumed by summary sparklines
|
||
// and density maps. Metrics not in this set are omitted to keep payloads small.
|
||
// guestChartSourceView is the guest view subset the infrastructure summary
|
||
// chart builder consumes from VMs and LXC containers.
|
||
type guestChartSourceView interface {
|
||
comparable
|
||
SourceID() string
|
||
CPUPercent() float64
|
||
MemoryPercent() float64
|
||
MemoryUsed() int64
|
||
DiskPercent() float64
|
||
NetIn() float64
|
||
NetOut() float64
|
||
}
|
||
|
||
// collectGuestChartData batch-loads sparkline history for one proxmox guest
|
||
// family into chartData (1-2 SQL calls instead of N) and returns the updated
|
||
// oldest chart timestamp. Guests without history fall back to a single
|
||
// current-value point per metric.
|
||
func collectGuestChartData[V guestChartSourceView](
|
||
monitor *monitoring.Monitor,
|
||
storeType string,
|
||
guests []V,
|
||
duration time.Duration,
|
||
chartData map[string]VMChartData,
|
||
currentTime, oldestTimestamp int64,
|
||
) int64 {
|
||
var zero V
|
||
requests := make([]monitoring.GuestChartRequest, 0, len(guests))
|
||
for _, g := range guests {
|
||
if g == zero {
|
||
continue
|
||
}
|
||
if id := g.SourceID(); id != "" {
|
||
requests = append(requests, monitoring.GuestChartRequest{InMemoryKey: id, SQLResourceID: id})
|
||
}
|
||
}
|
||
batch := monitor.GetGuestMetricsForChartBatch(storeType, requests, duration, guestSparklineMetricOrder...)
|
||
for _, g := range guests {
|
||
if g == zero {
|
||
continue
|
||
}
|
||
id := g.SourceID()
|
||
if id == "" {
|
||
continue
|
||
}
|
||
chartData[id] = make(VMChartData)
|
||
if batchMetrics, ok := batch[id]; ok {
|
||
oldestTimestamp = fillChartSeriesFromBatch(chartData[id], batchMetrics, oldestTimestamp)
|
||
}
|
||
if len(chartData[id]["cpu"]) == 0 {
|
||
chartData[id]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: g.CPUPercent()}}
|
||
chartData[id]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: g.MemoryPercent()}}
|
||
chartData[id]["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(g.MemoryUsed())}}
|
||
chartData[id]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: g.DiskPercent()}}
|
||
chartData[id]["netin"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetIn()}}
|
||
chartData[id]["netout"] = []MetricPoint{{Timestamp: currentTime, Value: g.NetOut()}}
|
||
}
|
||
}
|
||
return oldestTimestamp
|
||
}
|
||
|
||
// fillChartSeriesFromBatch copies sparkline-eligible batch metric points
|
||
// into dst and returns the updated oldest chart timestamp. Shared by the
|
||
// per-family infrastructure summary chart loops.
|
||
func fillChartSeriesFromBatch(dst VMChartData, batchMetrics map[string][]monitoring.MetricPoint, oldestTimestamp int64) int64 {
|
||
for metricType, points := range batchMetrics {
|
||
if !sparklineMetrics[metricType] {
|
||
continue
|
||
}
|
||
dst[metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
dst[metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
return oldestTimestamp
|
||
}
|
||
|
||
var sparklineMetrics = map[string]bool{
|
||
"cpu": true,
|
||
"memory": true,
|
||
"memoryused": true,
|
||
"disk": true,
|
||
"diskread": true,
|
||
"diskwrite": true,
|
||
"netin": true,
|
||
"netout": true,
|
||
}
|
||
|
||
var infrastructureSummaryMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"disk",
|
||
"diskread",
|
||
"diskwrite",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
var guestSparklineMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"memoryused",
|
||
"disk",
|
||
"diskread",
|
||
"diskwrite",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
var workloadSummaryMetricOrder = []string{
|
||
"cpu",
|
||
"memory",
|
||
"disk",
|
||
"netin",
|
||
"netout",
|
||
}
|
||
|
||
func parseInfrastructureSummaryRequestedMetrics(
|
||
query url.Values,
|
||
) ([]string, map[string]bool, error) {
|
||
rawValues, ok := query["metrics"]
|
||
if !ok || len(rawValues) == 0 {
|
||
requested := make(map[string]bool, len(infrastructureSummaryMetricOrder))
|
||
for _, metricType := range infrastructureSummaryMetricOrder {
|
||
requested[metricType] = true
|
||
}
|
||
return append([]string(nil), infrastructureSummaryMetricOrder...), requested, nil
|
||
}
|
||
|
||
requestedList := make([]string, 0, len(infrastructureSummaryMetricOrder))
|
||
requestedSet := make(map[string]bool, len(infrastructureSummaryMetricOrder))
|
||
invalid := make([]string, 0)
|
||
|
||
for _, rawValue := range rawValues {
|
||
for _, part := range strings.Split(rawValue, ",") {
|
||
metricType := strings.TrimSpace(strings.ToLower(part))
|
||
if metricType == "" {
|
||
continue
|
||
}
|
||
if !sparklineMetrics[metricType] {
|
||
invalid = append(invalid, metricType)
|
||
continue
|
||
}
|
||
if requestedSet[metricType] {
|
||
continue
|
||
}
|
||
requestedSet[metricType] = true
|
||
requestedList = append(requestedList, metricType)
|
||
}
|
||
}
|
||
|
||
if len(invalid) > 0 {
|
||
return nil, nil, fmt.Errorf("invalid infrastructure metrics filter: %s", strings.Join(invalid, ", "))
|
||
}
|
||
if len(requestedList) == 0 {
|
||
return nil, nil, fmt.Errorf("infrastructure metrics filter must include at least one valid metric")
|
||
}
|
||
return requestedList, requestedSet, nil
|
||
}
|
||
|
||
func convertMetricsForChart(
|
||
metrics map[string][]monitoring.MetricPoint,
|
||
oldestTimestamp *int64,
|
||
maxPoints int,
|
||
) VMChartData {
|
||
converted := make(VMChartData, len(metrics))
|
||
for metricType, metricPoints := range metrics {
|
||
if !sparklineMetrics[metricType] {
|
||
continue
|
||
}
|
||
if len(metricPoints) <= maxPoints || maxPoints <= 0 {
|
||
points := make([]MetricPoint, len(metricPoints))
|
||
for i, point := range metricPoints {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < *oldestTimestamp {
|
||
*oldestTimestamp = ts
|
||
}
|
||
points[i] = MetricPoint{Timestamp: ts, Value: point.Value}
|
||
}
|
||
converted[metricType] = points
|
||
continue
|
||
}
|
||
// Oldest-data metadata describes the source observations, not just
|
||
// those selected for display (also for degenerate/unsorted inputs).
|
||
for _, point := range metricPoints {
|
||
if ts := point.Timestamp.UnixMilli(); ts < *oldestTimestamp {
|
||
*oldestTimestamp = ts
|
||
}
|
||
}
|
||
indices := chartPointIndices(len(metricPoints), maxPoints, func(i int) int64 {
|
||
return metricPoints[i].Timestamp.UnixMilli()
|
||
})
|
||
points := make([]MetricPoint, len(indices))
|
||
for i, index := range indices {
|
||
point := metricPoints[index]
|
||
points[i] = MetricPoint{Timestamp: point.Timestamp.UnixMilli(), Value: point.Value}
|
||
}
|
||
converted[metricType] = points
|
||
}
|
||
return converted
|
||
}
|
||
|
||
// guestLiveMetricsView is the slice of the unified workload view API needed
|
||
// to seed a chart from live values; VM and container views both satisfy it.
|
||
type guestLiveMetricsView interface {
|
||
CPUPercent() float64
|
||
MemoryPercent() float64
|
||
MemoryUsed() int64
|
||
DiskPercent() float64
|
||
NetIn() float64
|
||
NetOut() float64
|
||
}
|
||
|
||
// guestChartSeriesWithLiveFallback converts a guest's batched metric history
|
||
// into chart series, substituting single live-value points when no history
|
||
// exists yet so freshly added guests still chart.
|
||
func guestChartSeriesWithLiveFallback(
|
||
metrics map[string][]monitoring.MetricPoint,
|
||
guest guestLiveMetricsView,
|
||
oldestTimestamp *int64,
|
||
maxPoints int,
|
||
currentTime int64,
|
||
) VMChartData {
|
||
series := convertMetricsForChart(metrics, oldestTimestamp, maxPoints)
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: guest.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: guest.MemoryPercent()}}
|
||
series["memoryused"] = []MetricPoint{{Timestamp: currentTime, Value: float64(guest.MemoryUsed())}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: guest.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetIn()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: guest.NetOut()}}
|
||
}
|
||
return series
|
||
}
|
||
|
||
const (
|
||
mockWorkloadMinSeriesPoints = 24
|
||
mockWorkloadMaxSeriesPoints = 180
|
||
)
|
||
|
||
func targetMockSeriesPoints(duration time.Duration, maxPoints int) int {
|
||
target := int(duration / (2 * time.Minute))
|
||
if target < mockWorkloadMinSeriesPoints {
|
||
target = mockWorkloadMinSeriesPoints
|
||
}
|
||
if maxPoints > 0 && target > maxPoints {
|
||
target = maxPoints
|
||
}
|
||
if target > mockWorkloadMaxSeriesPoints {
|
||
target = mockWorkloadMaxSeriesPoints
|
||
}
|
||
if target < 2 {
|
||
target = 2
|
||
}
|
||
return target
|
||
}
|
||
|
||
func TargetMockSeriesPoints(duration time.Duration, maxPoints int) int {
|
||
return targetMockSeriesPoints(duration, maxPoints)
|
||
}
|
||
|
||
// mockMetricStyle returns the series style for a given metric type.
|
||
func mockMetricStyle(metricType string) monitoring.SeriesStyle {
|
||
switch metricType {
|
||
case "cpu", "diskread", "diskwrite", "netin", "netout":
|
||
return monitoring.StyleSpiky
|
||
case "memory":
|
||
return monitoring.StylePlateau
|
||
default:
|
||
return monitoring.StyleFlat
|
||
}
|
||
}
|
||
|
||
// generateStyledMockSeries produces a MetricPoint slice using the style-based
|
||
// generator from the monitoring package.
|
||
func generateStyledMockSeries(
|
||
nowMillis int64,
|
||
duration time.Duration,
|
||
numPoints int,
|
||
current float64,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
) []MetricPoint {
|
||
style := mockMetricStyle(metricType)
|
||
|
||
durationMillis := int64(duration / time.Millisecond)
|
||
if durationMillis <= 0 {
|
||
durationMillis = int64(time.Minute / time.Millisecond)
|
||
}
|
||
step := durationMillis / int64(numPoints-1)
|
||
if step <= 0 {
|
||
step = 1
|
||
}
|
||
startMillis := nowMillis - durationMillis
|
||
timestamps := make([]time.Time, numPoints)
|
||
for i := 0; i < numPoints; i++ {
|
||
timestamps[i] = time.UnixMilli(startMillis + int64(i)*step)
|
||
}
|
||
values := monitoring.GenerateSeededResourceMetricSeriesForTimestamps(
|
||
current,
|
||
timestamps,
|
||
resourceType,
|
||
resourceID,
|
||
metricType,
|
||
style,
|
||
)
|
||
points := make([]MetricPoint, numPoints)
|
||
for i := 0; i < numPoints; i++ {
|
||
points[i] = MetricPoint{
|
||
Timestamp: startMillis + int64(i)*step,
|
||
Value: values[i],
|
||
}
|
||
}
|
||
return points
|
||
}
|
||
|
||
func GenerateStyledMockSeries(
|
||
nowMillis int64,
|
||
duration time.Duration,
|
||
numPoints int,
|
||
current float64,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
) []MetricPoint {
|
||
return generateStyledMockSeries(nowMillis, duration, numPoints, current, resourceType, resourceID, metricType)
|
||
}
|
||
|
||
func buildSyntheticMetricHistorySeries(
|
||
now time.Time,
|
||
duration time.Duration,
|
||
maxPoints int,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
current float64,
|
||
) []monitoring.MetricPoint {
|
||
switch metricType {
|
||
case "disk", "diskread", "diskwrite", "usage":
|
||
case "smart_temp":
|
||
if current <= 0 {
|
||
return nil
|
||
}
|
||
default:
|
||
return nil
|
||
}
|
||
|
||
numPoints := targetMockSeriesPoints(duration, maxPoints)
|
||
series := generateStyledMockSeries(
|
||
now.UnixMilli(), duration, numPoints,
|
||
current, resourceType, resourceID, metricType,
|
||
)
|
||
|
||
converted := make([]monitoring.MetricPoint, len(series))
|
||
for i, point := range series {
|
||
converted[i] = monitoring.MetricPoint{
|
||
Timestamp: time.UnixMilli(point.Timestamp),
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
|
||
return converted
|
||
}
|
||
|
||
func BuildSyntheticMetricHistorySeries(
|
||
now time.Time,
|
||
duration time.Duration,
|
||
maxPoints int,
|
||
resourceType string,
|
||
resourceID string,
|
||
metricType string,
|
||
current float64,
|
||
) []monitoring.MetricPoint {
|
||
return buildSyntheticMetricHistorySeries(now, duration, maxPoints, resourceType, resourceID, metricType, current)
|
||
}
|
||
|
||
// handleWorkloadCharts serves workload-only chart data used by workloads
|
||
// sparklines. It intentionally excludes infrastructure/storage chart payloads
|
||
// to keep requests small and stable for large fleets.
|
||
func (r *Service) HandleWorkloadCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Workload charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
selectedNodeID := strings.TrimSpace(query.Get("node"))
|
||
maxPointsRaw := query.Get("maxPoints")
|
||
maxPoints := parseWorkloadMaxPoints(maxPointsRaw)
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
orgID := apicontext.OrgID(req.Context())
|
||
if orgID == "" {
|
||
orgID = "default"
|
||
}
|
||
cacheKey := orgID + "|" + timeRange + "|" + selectedNodeID + "|" + maxPointsRaw
|
||
|
||
cacheKey = historyDataModeCacheKey(workloadChartsCachePrefix + cacheKey)
|
||
now := time.Now()
|
||
if body, ok := r.chartPayloads.get(cacheKey, now); ok {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryPayload(w, req, body, duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write cached workload chart data response")
|
||
}
|
||
return
|
||
}
|
||
|
||
v, err, _ := r.workloadChartsComputeGroup.Do(cacheKey, func() (any, error) {
|
||
// Re-check cache inside the singleflight barrier in case an earlier
|
||
// caller already populated it while we were queued.
|
||
if body, ok := r.chartPayloads.get(cacheKey, time.Now()); ok {
|
||
return body, nil
|
||
}
|
||
|
||
body, err := r.buildWorkloadChartsResponse(req.Context(), monitor, timeRange, selectedNodeID, maxPoints, duration, inMemoryChartThreshold)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
cachedAt := time.Now()
|
||
r.chartPayloads.put(cacheKey, body, cachedAt.Add(workloadChartsCacheTTL), cachedAt)
|
||
return body, nil
|
||
})
|
||
if err != nil {
|
||
log.Error().Err(err).Msg("Failed to build workload chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryPayload(w, req, v.([]byte), duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write workload chart data response")
|
||
}
|
||
}
|
||
|
||
// buildWorkloadChartsResponse runs the heavy compute path for handleWorkloadCharts
|
||
// and returns the marshaled JSON body. Extracted so the handler can wrap it
|
||
// with caching + singleflight.
|
||
func (r *Service) buildWorkloadChartsResponse(
|
||
ctx context.Context,
|
||
monitor *monitoring.Monitor,
|
||
timeRange string,
|
||
selectedNodeID string,
|
||
maxPoints int,
|
||
duration time.Duration,
|
||
inMemoryChartThreshold time.Duration,
|
||
) ([]byte, error) {
|
||
_ = ctx
|
||
nodes := monitor.NodesSnapshot()
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
return nil, fmt.Errorf("state unavailable")
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
currentTime := time.Now().UnixMilli()
|
||
oldestTimestamp := currentTime
|
||
|
||
var selectedNode *models.Node
|
||
if selectedNodeID != "" {
|
||
for idx := range nodes {
|
||
if nodes[idx].ID == selectedNodeID {
|
||
selectedNode = &nodes[idx]
|
||
break
|
||
}
|
||
}
|
||
if selectedNode == nil {
|
||
log.Debug().
|
||
Str("selectedNodeID", selectedNodeID).
|
||
Msg("Workload charts node filter not found in current state; falling back to global scope")
|
||
}
|
||
}
|
||
|
||
matchesSelectedNode := func(instance, nodeName string) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(instance), strings.TrimSpace(selectedNode.Instance)) &&
|
||
strings.EqualFold(strings.TrimSpace(nodeName), strings.TrimSpace(selectedNode.Name))
|
||
}
|
||
|
||
matchesSelectedDockerHostView := func(host *unifiedresources.DockerHostView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if host == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.Name()), nodeName)
|
||
}
|
||
|
||
matchesSelectedAgentHostView := func(host *unifiedresources.HostView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if host == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(host.Hostname()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.Name()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.AgentID()), nodeName) ||
|
||
strings.EqualFold(strings.TrimSpace(host.ID()), nodeName)
|
||
}
|
||
|
||
matchesSelectedKubernetesPodView := func(pod *unifiedresources.PodView) bool {
|
||
if selectedNodeID == "" {
|
||
return true
|
||
}
|
||
if selectedNode == nil {
|
||
return true
|
||
}
|
||
if pod == nil {
|
||
return false
|
||
}
|
||
nodeName := strings.TrimSpace(selectedNode.Name)
|
||
if nodeName == "" {
|
||
return false
|
||
}
|
||
return strings.EqualFold(strings.TrimSpace(pod.NodeName()), nodeName)
|
||
}
|
||
|
||
chartData := make(map[string]VMChartData)
|
||
dockerData := make(map[string]VMChartData)
|
||
|
||
guestTypes := make(map[string]string)
|
||
|
||
vmList := make([]*unifiedresources.VMView, 0)
|
||
vmResponseKeys := make([]string, 0)
|
||
vmRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, vm := range readState.VMs() {
|
||
if vm == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedNode(vm.Instance(), vm.Node()) {
|
||
continue
|
||
}
|
||
|
||
responseKey, request, ok := vmChartRequest(vm)
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
vmList = append(vmList, vm)
|
||
vmResponseKeys = append(vmResponseKeys, responseKey)
|
||
vmRequests = append(vmRequests, request)
|
||
}
|
||
containerList := make([]*unifiedresources.ContainerView, 0)
|
||
containerResponseKeys := make([]string, 0)
|
||
containerRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, ct := range readState.Containers() {
|
||
if ct == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedNode(ct.Instance(), ct.Node()) {
|
||
continue
|
||
}
|
||
|
||
responseKey, request, ok := systemContainerChartRequest(ct)
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
containerList = append(containerList, ct)
|
||
containerResponseKeys = append(containerResponseKeys, responseKey)
|
||
containerRequests = append(containerRequests, request)
|
||
}
|
||
podList := make([]*unifiedresources.PodView, 0)
|
||
podRequests := make([]monitoring.GuestChartRequest, 0)
|
||
for _, pod := range readState.Pods() {
|
||
if pod == nil {
|
||
continue
|
||
}
|
||
if !matchesSelectedKubernetesPodView(pod) {
|
||
continue
|
||
}
|
||
|
||
metricKey := kubernetesPodMetricIDFromView(pod)
|
||
if metricKey == "" {
|
||
continue
|
||
}
|
||
|
||
podList = append(podList, pod)
|
||
podRequests = append(podRequests, monitoring.GuestChartRequest{InMemoryKey: metricKey, SQLResourceID: metricKey})
|
||
}
|
||
dockerHostsByID := make(map[string]*unifiedresources.DockerHostView, len(readState.DockerHosts()))
|
||
for _, host := range readState.DockerHosts() {
|
||
if host == nil {
|
||
continue
|
||
}
|
||
dockerHostsByID[host.ID()] = host
|
||
}
|
||
agentHostsByID := make(map[string]*unifiedresources.HostView, len(readState.Hosts()))
|
||
for _, host := range readState.Hosts() {
|
||
if host == nil {
|
||
continue
|
||
}
|
||
agentHostsByID[host.ID()] = host
|
||
}
|
||
|
||
dockerContainerList := make([]*unifiedresources.DockerContainerView, 0)
|
||
dockerContainerRequests := make([]monitoring.GuestChartRequest, 0)
|
||
dockerContainerKeys := make([]string, 0)
|
||
for _, container := range readState.DockerContainers() {
|
||
if container == nil {
|
||
continue
|
||
}
|
||
|
||
if selectedNodeID != "" && selectedNode != nil {
|
||
host := dockerHostsByID[container.ParentID()]
|
||
if host != nil {
|
||
if !matchesSelectedDockerHostView(host) {
|
||
continue
|
||
}
|
||
} else {
|
||
agentHost := agentHostsByID[container.ParentID()]
|
||
if agentHost == nil || !matchesSelectedAgentHostView(agentHost) {
|
||
continue
|
||
}
|
||
}
|
||
}
|
||
|
||
responseKey, request, ok := appContainerChartRequest(container)
|
||
if !ok {
|
||
continue
|
||
}
|
||
dockerContainerList = append(dockerContainerList, container)
|
||
dockerContainerKeys = append(dockerContainerKeys, responseKey)
|
||
dockerContainerRequests = append(dockerContainerRequests, request)
|
||
}
|
||
var (
|
||
vmBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
containerBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
podBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
dockerContainerBatchMetrics map[string]map[string][]monitoring.MetricPoint
|
||
)
|
||
var workloadChartsBatchWG sync.WaitGroup
|
||
workloadChartsBatchWG.Add(4)
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
vmBatchMetrics = monitor.GetGuestMetricsForChartBatch("vm", vmRequests, duration, guestSparklineMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
containerBatchMetrics = monitor.GetGuestMetricsForChartBatch("container", containerRequests, duration, guestSparklineMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
podBatchMetrics = monitor.GetGuestMetricsForChartBatch("k8s", podRequests, duration, workloadSummaryMetricOrder...)
|
||
}()
|
||
go func() {
|
||
defer workloadChartsBatchWG.Done()
|
||
dockerContainerBatchMetrics = monitor.GetGuestMetricsForChartBatch("dockerContainer", dockerContainerRequests, duration, infrastructureSummaryMetricOrder...)
|
||
}()
|
||
workloadChartsBatchWG.Wait()
|
||
|
||
for idx, vm := range vmList {
|
||
responseKey := vmResponseKeys[idx]
|
||
metricID := vmRequests[idx].SQLResourceID
|
||
guestTypes[responseKey] = "vm"
|
||
chartData[responseKey] = guestChartSeriesWithLiveFallback(vmBatchMetrics[metricID], vm, &oldestTimestamp, maxPoints, currentTime)
|
||
}
|
||
|
||
for idx, ct := range containerList {
|
||
responseKey := containerResponseKeys[idx]
|
||
metricID := containerRequests[idx].SQLResourceID
|
||
guestTypes[responseKey] = "system-container"
|
||
chartData[responseKey] = guestChartSeriesWithLiveFallback(containerBatchMetrics[metricID], ct, &oldestTimestamp, maxPoints, currentTime)
|
||
}
|
||
|
||
for _, pod := range podList {
|
||
metricKey := kubernetesPodMetricIDFromView(pod)
|
||
series := convertMetricsForChart(podBatchMetrics[metricKey], &oldestTimestamp, maxPoints)
|
||
guestTypes[metricKey] = "k8s"
|
||
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: pod.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: pod.MemoryPercent()}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: pod.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetInRate()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: pod.NetOutRate()}}
|
||
}
|
||
chartData[metricKey] = series
|
||
}
|
||
|
||
for idx, container := range dockerContainerList {
|
||
responseKey := dockerContainerKeys[idx]
|
||
metricID := dockerContainerRequests[idx].SQLResourceID
|
||
series := convertMetricsForChart(dockerContainerBatchMetrics[metricID], &oldestTimestamp, maxPoints)
|
||
guestTypes[responseKey] = "app-container"
|
||
|
||
if len(series["cpu"]) == 0 {
|
||
series["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: container.CPUPercent()}}
|
||
series["memory"] = []MetricPoint{{Timestamp: currentTime, Value: container.MemoryPercent()}}
|
||
series["disk"] = []MetricPoint{{Timestamp: currentTime, Value: container.DiskPercent()}}
|
||
series["netin"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetInRate()}}
|
||
series["netout"] = []MetricPoint{{Timestamp: currentTime, Value: container.NetOutRate()}}
|
||
}
|
||
dockerData[responseKey] = series
|
||
}
|
||
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
guestPoints := countChartPoints(chartData)
|
||
dockerContainerPoints := countChartPoints(dockerData)
|
||
|
||
response := EmptyWorkloadChartsResponse()
|
||
response.ChartData = chartData
|
||
response.DockerData = dockerData
|
||
response.GuestTypes = guestTypes
|
||
response.Timestamp = currentTime
|
||
response.Stats = ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: guestPoints + dockerContainerPoints,
|
||
Guests: guestPoints,
|
||
DockerContainers: dockerContainerPoints,
|
||
},
|
||
}
|
||
|
||
body, err := json.Marshal(response.NormalizeCollections())
|
||
if err != nil {
|
||
return nil, fmt.Errorf("marshal workload chart response: %w", err)
|
||
}
|
||
return body, nil
|
||
}
|
||
|
||
// parseChartsRangeDuration converts the UI chart range query (e.g. "5m", "1h")
|
||
// into a duration. This is shared by /api/charts and /api/charts/infrastructure
|
||
// to prevent drift.
|
||
func parseChartsRangeDuration(rangeStr string) time.Duration {
|
||
switch rangeStr {
|
||
case "5m":
|
||
return 5 * time.Minute
|
||
case "15m":
|
||
return 15 * time.Minute
|
||
case "30m":
|
||
return 30 * time.Minute
|
||
case "1h":
|
||
return time.Hour
|
||
case "4h":
|
||
return 4 * time.Hour
|
||
case "8h":
|
||
return 8 * time.Hour
|
||
case "12h":
|
||
return 12 * time.Hour
|
||
case "24h":
|
||
return 24 * time.Hour
|
||
case "7d":
|
||
return 7 * 24 * time.Hour
|
||
case "30d":
|
||
return 30 * 24 * time.Hour
|
||
default:
|
||
return time.Hour
|
||
}
|
||
}
|
||
|
||
// handleInfrastructureCharts serves infrastructure-only chart data.
|
||
// This is intentionally narrower than /api/charts to reduce payload size and server-side compute
|
||
// for the Infrastructure page summary cards.
|
||
func (r *Service) HandleInfrastructureCharts(w http.ResponseWriter, req *http.Request) {
|
||
log.Debug().Str("method", req.Method).Str("url", req.URL.String()).Msg("Infrastructure charts endpoint hit")
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Get time range from query parameters
|
||
query := req.URL.Query()
|
||
timeRange := query.Get("range")
|
||
if timeRange == "" {
|
||
timeRange = "1h"
|
||
}
|
||
requestedMetricNames, requestedMetrics, err := parseInfrastructureSummaryRequestedMetrics(query)
|
||
if err != nil {
|
||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||
return
|
||
}
|
||
// Convert time range to duration.
|
||
duration := parseChartsRangeDuration(timeRange)
|
||
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Tenant monitor is not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
metricsStoreEnabled := monitor.GetMetricsStore() != nil
|
||
primarySourceHint := "memory"
|
||
if metricsStoreEnabled && duration > inMemoryChartThreshold {
|
||
primarySourceHint = "store_or_memory_fallback"
|
||
}
|
||
|
||
now := time.Now()
|
||
cacheKey := historyDataModeCacheKey(infrastructureChartsCacheKey(req, timeRange, requestedMetricNames))
|
||
if payload, ok := r.cachedInfrastructureChartsPayload(cacheKey, now); ok {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryPayload(w, req, payload, duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write cached infrastructure chart data response")
|
||
}
|
||
return
|
||
}
|
||
|
||
currentTime := now.UnixMilli()
|
||
oldestTimestamp := currentTime
|
||
|
||
// Process Nodes - batch-load historical data (1-2 SQL calls instead of N×5).
|
||
nodeMetricTypes := make([]string, 0, 5)
|
||
for _, metricType := range []string{"cpu", "memory", "disk", "netin", "netout"} {
|
||
if requestedMetrics[metricType] {
|
||
nodeMetricTypes = append(nodeMetricTypes, metricType)
|
||
}
|
||
}
|
||
nodeData := make(map[string]NodeChartData)
|
||
nodeList := readState.Nodes()
|
||
nodeIDs := make([]string, 0, len(nodeList))
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
if nid := node.SourceID(); nid != "" {
|
||
nodeIDs = append(nodeIDs, nid)
|
||
}
|
||
}
|
||
nodeBatchMetrics := map[string]map[string][]monitoring.MetricPoint{}
|
||
if len(nodeMetricTypes) > 0 {
|
||
nodeBatchMetrics = monitor.GetNodeMetricsForChartBatch(nodeIDs, nodeMetricTypes, duration)
|
||
}
|
||
for _, node := range nodeList {
|
||
if node == nil {
|
||
continue
|
||
}
|
||
nid := node.SourceID()
|
||
if nid == "" {
|
||
continue
|
||
}
|
||
nodeData[nid] = make(NodeChartData)
|
||
if batchMetrics, ok := nodeBatchMetrics[nid]; ok {
|
||
for _, metricType := range nodeMetricTypes {
|
||
points, found := batchMetrics[metricType]
|
||
if !found {
|
||
continue
|
||
}
|
||
nodeData[nid][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
nodeData[nid][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range nodeMetricTypes {
|
||
if len(nodeData[nid][metricType]) > 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = node.CPUPercent()
|
||
case "memory":
|
||
value = node.MemoryPercent()
|
||
case "disk":
|
||
value = node.DiskPercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
nodeData[nid][metricType] = []MetricPoint{
|
||
{Timestamp: currentTime, Value: value},
|
||
}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(nodeData[nid], duration, currentTime)
|
||
}
|
||
|
||
// Process Docker hosts - batch-load historical data (1-2 SQL calls instead of N).
|
||
dockerHostData := make(map[string]VMChartData)
|
||
dhList := readState.DockerHosts()
|
||
dhRequests := make([]monitoring.GuestChartRequest, 0, len(dhList))
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
if dhID := dh.HostSourceID(); dhID != "" {
|
||
dhRequests = append(dhRequests, monitoring.GuestChartRequest{
|
||
InMemoryKey: fmt.Sprintf("dockerHost:%s", dhID),
|
||
SQLResourceID: dhID,
|
||
})
|
||
}
|
||
}
|
||
dhBatchMetrics := monitor.GetGuestMetricsForChartBatch("dockerHost", dhRequests, duration, requestedMetricNames...)
|
||
for _, dh := range dhList {
|
||
if dh == nil {
|
||
continue
|
||
}
|
||
dhID := dh.HostSourceID()
|
||
if dhID == "" {
|
||
continue
|
||
}
|
||
dockerHostData[dhID] = make(VMChartData)
|
||
if batchMetrics, ok := dhBatchMetrics[dhID]; ok {
|
||
for metricType, points := range batchMetrics {
|
||
if !requestedMetrics[metricType] {
|
||
continue
|
||
}
|
||
dockerHostData[dhID][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
dockerHostData[dhID][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range requestedMetricNames {
|
||
if len(dockerHostData[dhID][metricType]) > 0 {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = dh.CPUPercent()
|
||
case "memory":
|
||
value = dh.MemoryPercent()
|
||
case "disk":
|
||
if disks := dh.Disks(); len(disks) > 0 {
|
||
value = disks[0].Usage
|
||
}
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
dockerHostData[dhID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(dockerHostData[dhID], duration, currentTime)
|
||
}
|
||
|
||
// Process unified agents - batch-load historical data (1-2 SQL calls instead of N).
|
||
agentData := make(map[string]VMChartData)
|
||
hostList := readState.Hosts()
|
||
agentRequests := make([]monitoring.GuestChartRequest, 0, len(hostList))
|
||
for _, h := range hostList {
|
||
_, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentRequests = append(agentRequests, request)
|
||
}
|
||
agentBatchMetrics := monitor.GetGuestMetricsForChartBatch("agent", agentRequests, duration, requestedMetricNames...)
|
||
for _, h := range hostList {
|
||
hID, request, ok := hostAgentChartRequest(h)
|
||
if !ok {
|
||
continue
|
||
}
|
||
agentData[hID] = make(VMChartData)
|
||
if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {
|
||
for metricType, points := range batchMetrics {
|
||
if !requestedMetrics[metricType] {
|
||
continue
|
||
}
|
||
agentData[hID][metricType] = make([]MetricPoint, len(points))
|
||
for i, point := range points {
|
||
ts := point.Timestamp.UnixMilli()
|
||
if ts < oldestTimestamp {
|
||
oldestTimestamp = ts
|
||
}
|
||
agentData[hID][metricType][i] = MetricPoint{
|
||
Timestamp: ts,
|
||
Value: point.Value,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
for _, metricType := range requestedMetricNames {
|
||
if len(agentData[hID][metricType]) > 0 {
|
||
continue
|
||
}
|
||
if !hostAgentCanSupplyLiveChartFallback(h) {
|
||
continue
|
||
}
|
||
var value float64
|
||
hasFallbackValue := true
|
||
switch metricType {
|
||
case "cpu":
|
||
value = h.CPUPercent()
|
||
case "memory":
|
||
value = h.MemoryPercent()
|
||
case "disk":
|
||
value, hasFallbackValue = h.DiskUsagePercent()
|
||
default:
|
||
hasFallbackValue = false
|
||
}
|
||
if hasFallbackValue {
|
||
agentData[hID][metricType] = []MetricPoint{{Timestamp: currentTime, Value: value}}
|
||
}
|
||
}
|
||
normalizeInfrastructureSummaryChartSeries(agentData[hID], duration, currentTime)
|
||
}
|
||
|
||
countNodePoints := func(metricsMap map[string]NodeChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
countChartPoints := func(metricsMap map[string]VMChartData) int {
|
||
total := 0
|
||
for _, metricSeries := range metricsMap {
|
||
for _, points := range metricSeries {
|
||
total += len(points)
|
||
}
|
||
}
|
||
return total
|
||
}
|
||
|
||
nodePoints := countNodePoints(nodeData)
|
||
dockerHostPoints := countChartPoints(dockerHostData)
|
||
agentPoints := countChartPoints(agentData)
|
||
|
||
response := EmptyInfrastructureChartsResponse()
|
||
response.NodeData = nodeData
|
||
response.DockerHostData = dockerHostData
|
||
response.AgentData = agentData
|
||
response.Timestamp = currentTime
|
||
response.Stats = ChartStats{
|
||
OldestDataTimestamp: oldestTimestamp,
|
||
Range: timeRange,
|
||
RangeSeconds: int64(duration / time.Second),
|
||
MetricsStoreEnabled: metricsStoreEnabled,
|
||
PrimarySourceHint: primarySourceHint,
|
||
InMemoryThresholdSecs: int64(inMemoryChartThreshold / time.Second),
|
||
PointCounts: ChartPointCounts{
|
||
Total: nodePoints + dockerHostPoints + agentPoints,
|
||
Nodes: nodePoints,
|
||
DockerHosts: dockerHostPoints,
|
||
Agents: agentPoints,
|
||
},
|
||
}
|
||
|
||
payload, err := json.Marshal(response.NormalizeCollections())
|
||
if err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode infrastructure chart data response")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
payload = append(payload, '\n')
|
||
r.cacheInfrastructureChartsPayload(cacheKey, payload, now)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryPayload(w, req, payload, duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to write infrastructure chart data response")
|
||
}
|
||
}
|
||
|
||
func clampWorkloadPercent(value float64) float64 {
|
||
if value != value {
|
||
return 0
|
||
}
|
||
if value < 0 {
|
||
return 0
|
||
}
|
||
if value > 100 {
|
||
return 100
|
||
}
|
||
return value
|
||
}
|
||
|
||
func proxmoxModelCPURatioPercent(value float64) float64 {
|
||
return clampWorkloadPercent(value * 100)
|
||
}
|
||
|
||
func ProxmoxModelCPURatioPercent(value float64) float64 {
|
||
return proxmoxModelCPURatioPercent(value)
|
||
}
|
||
|
||
func kubernetesPodMetricIDFromView(pod *unifiedresources.PodView) string {
|
||
if pod == nil {
|
||
return ""
|
||
}
|
||
clusterKey := strings.TrimSpace(pod.ClusterID())
|
||
if clusterKey == "" {
|
||
clusterKey = strings.TrimSpace(pod.ClusterName())
|
||
}
|
||
podKey := strings.TrimSpace(pod.PodUID())
|
||
if podKey == "" {
|
||
namespace := strings.TrimSpace(pod.Namespace())
|
||
name := strings.TrimSpace(pod.Name())
|
||
if namespace != "" || name != "" {
|
||
podKey = fmt.Sprintf("%s/%s", namespace, name)
|
||
}
|
||
}
|
||
if clusterKey == "" || podKey == "" {
|
||
return ""
|
||
}
|
||
return fmt.Sprintf("k8s:%s:pod:%s", clusterKey, podKey)
|
||
}
|
||
|
||
// handleStorageCharts returns pool capacity and physical disk temperature
|
||
// time-series for the storage summary sparklines.
|
||
func (r *Service) HandleStorageCharts(w http.ResponseWriter, req *http.Request) {
|
||
if req.Method != http.MethodGet && req.Method != http.MethodHead {
|
||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
|
||
// Parse query parameters
|
||
query := req.URL.Query()
|
||
rangeMinutes := 60 // default 1 hour
|
||
if rangeStr := query.Get("range"); rangeStr != "" {
|
||
if _, err := fmt.Sscanf(rangeStr, "%d", &rangeMinutes); err != nil {
|
||
log.Warn().Err(err).Str("range", rangeStr).Msg("Invalid range parameter; using default")
|
||
}
|
||
}
|
||
|
||
duration := time.Duration(rangeMinutes) * time.Minute
|
||
selectedNodeID := strings.TrimSpace(query.Get("node"))
|
||
|
||
// Use tenant-aware monitor
|
||
monitor := r.getTenantMonitor(req.Context())
|
||
if monitor == nil {
|
||
http.Error(w, "Monitor not available", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
readState := monitor.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
http.Error(w, "State unavailable", http.StatusInternalServerError)
|
||
return
|
||
}
|
||
|
||
// Resolve node filter from canonical unified resources so storage charts use
|
||
// the same node identity model as the frontend storage page.
|
||
var selectedNodeName, selectedNodeInstance string
|
||
if selectedNodeID != "" {
|
||
found := false
|
||
for _, resource := range monitor.GetUnifiedResources() {
|
||
if strings.TrimSpace(resource.ID) != selectedNodeID {
|
||
continue
|
||
}
|
||
selectedNodeName = storageChartsSelectedNodeName(resource)
|
||
selectedNodeInstance = storageChartsSelectedNodeInstance(resource)
|
||
if selectedNodeName != "" || selectedNodeInstance != "" {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
for _, n := range monitor.NodesSnapshot() {
|
||
if n.ID == selectedNodeID {
|
||
selectedNodeName = n.Name
|
||
selectedNodeInstance = n.Instance
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
if !found {
|
||
log.Debug().
|
||
Str("selectedNodeID", selectedNodeID).
|
||
Msg("Storage charts node filter not found in current state; falling back to global scope")
|
||
}
|
||
}
|
||
matchesNode := func(nodeName, instance string) bool {
|
||
if selectedNodeName == "" {
|
||
return true
|
||
}
|
||
if !strings.EqualFold(strings.TrimSpace(nodeName), selectedNodeName) {
|
||
return false
|
||
}
|
||
if selectedNodeInstance != "" && instance != "" {
|
||
return strings.EqualFold(strings.TrimSpace(instance), selectedNodeInstance)
|
||
}
|
||
return true
|
||
}
|
||
|
||
// Build pool chart data from the canonical storage summary batch path so
|
||
// the dashboard and storage page share one efficient history retrieval model.
|
||
poolNames := make(map[string]string, len(readState.StoragePools()))
|
||
storageIDs := make([]string, 0, len(readState.StoragePools()))
|
||
for _, sp := range readState.StoragePools() {
|
||
if sp == nil {
|
||
continue
|
||
}
|
||
if !matchesNode(sp.Node(), sp.Instance()) {
|
||
continue
|
||
}
|
||
sid := sp.SourceID()
|
||
if sid == "" {
|
||
continue
|
||
}
|
||
poolNames[sid] = sp.Name()
|
||
storageIDs = append(storageIDs, sid)
|
||
}
|
||
|
||
poolMetrics := monitor.GetStorageMetricsForChartBatch(storageIDs, duration)
|
||
pools := make(map[string]StoragePoolChartData, len(storageIDs))
|
||
for _, sid := range storageIDs {
|
||
metrics := poolMetrics[sid]
|
||
pools[sid] = StoragePoolChartData{
|
||
Name: poolNames[sid],
|
||
Usage: monitorPointsToAPI(metrics["usage"]),
|
||
Used: monitorPointsToAPI(metrics["used"]),
|
||
Avail: monitorPointsToAPI(metrics["avail"]),
|
||
}
|
||
}
|
||
|
||
// Build disk temperature chart data
|
||
diskEntries := monitor.GetPhysicalDiskTemperatureCharts(duration)
|
||
disks := make(map[string]StorageDiskChartData, len(diskEntries))
|
||
for id, entry := range diskEntries {
|
||
if !matchesNode(entry.Node, entry.Instance) {
|
||
continue
|
||
}
|
||
disks[id] = StorageDiskChartData{
|
||
Name: entry.Name,
|
||
Node: entry.Node,
|
||
Temperature: monitorPointsToAPI(entry.Temperature),
|
||
}
|
||
}
|
||
|
||
resp := EmptyStorageChartsResponse()
|
||
resp.Pools = pools
|
||
resp.Disks = disks
|
||
|
||
w.Header().Set("Content-Type", "application/json")
|
||
if err := r.WriteHistoryJSON(w, req, resp.NormalizeCollections(), duration); err != nil {
|
||
log.Error().Err(err).Msg("Failed to encode storage chart data")
|
||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||
}
|
||
}
|
||
|
||
// monitorPointsToAPI converts monitoring MetricPoints (time.Time timestamps)
|
||
// to API MetricPoints (Unix millisecond timestamps) for JSON serialization.
|
||
func monitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint {
|
||
if len(points) == 0 {
|
||
return nil
|
||
}
|
||
out := make([]MetricPoint, len(points))
|
||
for i, p := range points {
|
||
out[i] = MetricPoint{Timestamp: p.Timestamp.UnixMilli(), Value: p.Value}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func MonitorPointsToAPI(points []monitoring.MetricPoint) []MetricPoint {
|
||
return monitorPointsToAPI(points)
|
||
}
|