package monitoring import ( "math" "sort" "strings" "time" "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/pkg/metrics" ) // inMemoryChartThreshold is the max duration reliably covered by the in-memory // metrics buffer. With 1000 data-points at a 10 s polling interval the buffer // holds ~2.8 h of data; 2 h is a safe conservative cut-off. const inMemoryChartThreshold = 2 * time.Hour // chartDownsampleTarget is the number of points returned per metric when // falling back to the persistent store. 500 is more than enough for any // sparkline or thumbnail chart. const chartDownsampleTarget = 500 const ( // When the requested chart window has no data (for example after running in // mock mode for a while), query a broader historical window so sparklines // can render immediately instead of waiting for enough fresh samples. chartGapFillLookbackMultiplier = 12 chartGapFillLookbackMin = 6 * time.Hour chartGapFillLookbackMax = 7 * 24 * time.Hour ) // For short chart ranges (<= inMemoryChartThreshold), we prefer in-memory data // for freshness unless coverage is too shallow (for example after a restart). // In that case we fall back to SQLite history to avoid mostly-empty charts. const ( shortRangeCoverageRatioNum = 3 shortRangeCoverageRatioDen = 4 shortRangeCoverageMaxSlack = 2 * time.Minute ) const ( mockChartPointTargetShortRange = 20 mockChartPointTargetMedium = 32 mockChartPointTargetLong = 48 mockChartPointTargetExtended = 64 ) const mockDashboardWorkloadPrewarmDuration = time.Hour var storageChartMetricTypes = []string{"usage", "used", "avail", "total"} var storageSummaryChartMetricTypes = []string{"used", "avail"} type mockChartMetricMapCacheKey struct { kind string resourceType string resourceID string aux string duration time.Duration } // MonitorGuestMetricHistoryProvider optionally exposes source-native guest // metric history through the canonical chart boundary when local Pulse history // is shallow. type MonitorGuestMetricHistoryProvider interface { GuestMetricHistory(m *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint } // GetGuestMetrics returns historical metrics for a guest func (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint { if m == nil || m.metricsHistory == nil { return map[string][]MetricPoint{} } candidates := monitorGuestMetricKeyCandidates(guestID) best := map[string][]MetricPoint{} bestSpan := time.Duration(0) bestPoints := 0 selected := false for _, candidate := range candidates { result := m.metricsHistory.GetAllGuestMetrics(candidate, duration) span := chartMapCoverageSpan(result) points := monitorMetricMapPointCount(result) if !selected || span > bestSpan || (span == bestSpan && points > bestPoints) { best = result bestSpan = span bestPoints = points selected = true } } return best } // GetGuestMetricsForChart returns guest metrics optimised for chart display. // Short ranges are served from the in-memory ring buffer; longer ranges fall // back to the persistent SQLite store with LTTB downsampling. // // inMemoryKey is the key used in the in-memory buffer (e.g. "docker:abc123"). // sqlResourceType/sqlResourceID are the type/id used in the SQLite store // (e.g. "dockerContainer"/"abc123"). func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration) map[string][]MetricPoint { if mock.IsMockEnabled() { return m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration)) } inMemoryResult := map[string][]MetricPoint{} if m.shouldLoadGuestChartInMemory(inMemoryKey, nil, duration) { inMemoryResult = m.GetGuestMetrics(inMemoryKey, duration) } if hasSufficientChartMapCoverage(inMemoryResult, duration) { return inMemoryResult } best := cloneMetricPointMap(inMemoryResult) if m.metricsStore != nil { if converted, ok := m.queryStoreMetricMapWithGapFillAliases(sqlResourceType, sqlResourceID, duration); ok { best = mergeGuestMetricHistory(best, converted, duration) } } if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration); len(nativeHistory) > 0 { if nativeMetrics, ok := nativeHistory[sqlResourceID]; ok { best = mergeGuestMetricHistory(best, nativeMetrics, duration) } } return best } // GetNodeMetrics returns historical metrics for a node func (m *Monitor) GetNodeMetrics(nodeID string, metricType string, duration time.Duration) []MetricPoint { return m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration) } // GetNodeMetricsForChart returns node metrics for a single metric type, // falling back to SQLite + LTTB for longer ranges. func (m *Monitor) GetNodeMetricsForChart(nodeID, metricType string, duration time.Duration) []MetricPoint { var inMemoryPoints []MetricPoint if mock.IsMockEnabled() { if m != nil && m.metricsHistory != nil { inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration) } return m.mockNodeMetricsForChart(nodeID, metricType, duration, inMemoryPoints) } if m != nil && m.shouldLoadNodeChartInMemory(nodeID, []string{metricType}, duration) { inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration) } if m.metricsStore == nil { return inMemoryPoints } if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) { return inMemoryPoints } downsampled, ok := m.queryStoreMetricSeriesWithGapFill("node", nodeID, metricType, duration) if !ok { return inMemoryPoints } if chartSeriesCoverageSpan(downsampled) <= chartSeriesCoverageSpan(inMemoryPoints) { return inMemoryPoints } return downsampled } // GetStorageMetrics returns historical metrics for storage func (m *Monitor) GetStorageMetrics(storageID string, duration time.Duration) map[string][]MetricPoint { if m == nil || m.metricsHistory == nil { return map[string][]MetricPoint{} } return m.metricsHistory.GetAllStorageMetrics(storageID, duration) } // GetDiskMetrics returns historical metrics for a physical disk. func (m *Monitor) GetDiskMetrics(resourceID string, metricType string, duration time.Duration) []MetricPoint { if m == nil || m.metricsHistory == nil { return nil } return m.metricsHistory.GetDiskMetrics(resourceID, metricType, duration) } // GetDiskMetricsForChart returns physical-disk metrics optimized for chart // display, preferring in-memory data for freshness and falling back to the // persistent store when coverage is shallow. func (m *Monitor) GetDiskMetricsForChart(resourceID string, metricType string, duration time.Duration) []MetricPoint { inMemoryPoints := m.GetDiskMetrics(resourceID, metricType, duration) if mock.IsMockEnabled() { return m.mockDiskMetricsForChart(resourceID, metricType, duration, inMemoryPoints) } if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) { return inMemoryPoints } best := cloneMetricSeries(inMemoryPoints) if converted, ok := m.queryStoreMetricMapWithGapFill("disk", resourceID, duration); ok { if candidate := converted[metricType]; shouldPreferMetricSeries(best, candidate, duration) { best = cloneMetricSeries(candidate) } } return best } // GetStorageMetricsForChart returns storage metrics, falling back to SQLite + // LTTB for longer ranges. func (m *Monitor) GetStorageMetricsForChart(storageID string, duration time.Duration) map[string][]MetricPoint { inMemoryResult := map[string][]MetricPoint{} if m != nil && m.metricsHistory != nil { inMemoryResult = m.metricsHistory.GetAllStorageMetrics(storageID, duration) } if mock.IsMockEnabled() { return m.mockStorageMetricsForChartCached(storageID, duration, inMemoryResult) } if m.metricsStore == nil { return inMemoryResult } if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, storageChartMetricTypes) { return inMemoryResult } best := cloneMetricPointMap(inMemoryResult) if converted, ok := m.queryStoreMetricMapWithGapFill("storage", storageID, duration); ok { best = mergeMetricHistory(best, converted, duration) } return best } // DiskChartEntry holds temperature chart data and display metadata for a // single physical disk. type DiskChartEntry struct { Name string `json:"name"` Node string `json:"node"` Instance string `json:"instance"` Temperature []MetricPoint `json:"temperature"` } // GetPhysicalDiskTemperatureCharts returns temperature time-series for all // physical disks that have temperature data. The Monitor owns state access // (monitoring is exempt from the ReadState-only rule), so this keeps state // reads out of API handler code. // // Uses batch queries to load all disk metrics in 1-2 SQL calls instead of N // (one per disk), avoiding N+1 query patterns on systems with many disks. func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[string]DiskChartEntry { if m == nil { return nil } mockEnabled := mock.IsMockEnabled() readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil } metricsTargetStore := m.currentMetricsTargetStore() // Phase 1: Collect disk metadata and resource IDs. type diskMeta struct { resourceID string name string node string instance string temperature int } var disks []diskMeta for _, disk := range readState.PhysicalDisks() { if disk == nil || disk.Temperature() <= 0 { continue } resourceID := "" if metricsTargetStore != nil { if target := metricsTargetStore.MetricsTargetForResource(disk.ID()); target != nil { resourceID = strings.TrimSpace(target.ResourceID) } } if resourceID == "" { resourceID = strings.TrimSpace(disk.MetricResourceID()) } if resourceID == "" { continue } name := strings.TrimSpace(disk.Model()) if name == "" { name = strings.TrimSpace(disk.DevPath()) } disks = append(disks, diskMeta{ resourceID: resourceID, name: name, node: strings.TrimSpace(disk.Node()), instance: strings.TrimSpace(disk.Instance()), temperature: disk.Temperature(), }) } if len(disks) == 0 { return make(map[string]DiskChartEntry) } // Phase 2: Batch-load metrics for all disks (1-2 queries instead of N). resourceIDs := make([]string, len(disks)) for i, d := range disks { resourceIDs[i] = d.resourceID } inMemoryHistory := make(map[string][]MetricPoint, len(disks)) if m.metricsHistory != nil { for _, d := range disks { inMemoryHistory[d.resourceID] = m.metricsHistory.GetDiskMetrics(d.resourceID, "smart_temp", duration) } } var batchMetrics map[string]map[string][]MetricPoint var nativeHistory map[string][]MetricPoint if !mockEnabled { batchMetrics = m.queryStoreBatchMetricMapWithGapFill("disk", resourceIDs, duration, []string{"smart_temp"}) nativeHistory = m.nativePhysicalDiskTemperatureHistory(duration) } // Phase 3: Build result entries. result := make(map[string]DiskChartEntry, len(disks)) for _, d := range disks { tempPoints := cloneMetricSeries(inMemoryHistory[d.resourceID]) if mockEnabled { tempPoints = m.mockDiskMetricsForChart(d.resourceID, "smart_temp", duration, tempPoints) } else { if resMetrics, ok := batchMetrics[d.resourceID]; ok { if pts, found := resMetrics["smart_temp"]; found { if shouldPreferMetricSeries(tempPoints, pts, duration) { tempPoints = cloneMetricSeries(pts) } } } if nativePoints, ok := nativeHistory[d.resourceID]; ok { nativePoints = lttb(nativePoints, chartDownsampleTarget) if shouldPreferMetricSeries(tempPoints, nativePoints, duration) { tempPoints = cloneMetricSeries(nativePoints) } } } // Sparklines require >= 2 points. If the store returned 0 or 1 points // but the disk has a live temperature reading, pad to 2 points so the // chart can render (flat line at current temperature). if len(tempPoints) < 2 { now := time.Now() tempPoints = []MetricPoint{ {Timestamp: now.Add(-60 * time.Second), Value: float64(d.temperature)}, {Timestamp: now, Value: float64(d.temperature)}, } } result[d.resourceID] = DiskChartEntry{ Name: d.name, Node: d.node, Instance: d.instance, Temperature: tempPoints, } } return result } func (m *Monitor) nativePhysicalDiskTemperatureHistory(duration time.Duration) map[string][]MetricPoint { if mock.IsMockEnabled() { return nil } providers := m.supplementalProviderSnapshot() if len(providers) == 0 { return nil } orgID := "default" if m != nil { if trimmed := strings.TrimSpace(m.GetOrgID()); trimmed != "" { orgID = trimmed } } history := make(map[string][]MetricPoint) for _, provider := range providers { historyProvider, ok := provider.(MonitorPhysicalDiskTemperatureHistoryProvider) if !ok { continue } nativeHistory := historyProvider.PhysicalDiskTemperatureHistory(m, orgID, duration) for resourceID, points := range nativeHistory { if strings.TrimSpace(resourceID) == "" || len(points) == 0 { continue } if existing, ok := history[resourceID]; !ok || chartSeriesCoverageSpan(points) > chartSeriesCoverageSpan(existing) { history[resourceID] = points } } } if len(history) == 0 { return nil } return history } func (m *Monitor) currentMetricsTargetStore() MetricsTargetResourceStore { if m == nil { return nil } m.mu.RLock() defer m.mu.RUnlock() store, ok := m.resourceStore.(MetricsTargetResourceStore) if !ok { return nil } return store } // queryStoreBatchMetricMapWithGapFill queries metrics for multiple resource IDs // of the same type in a single batch. If some resources have no data in the // primary window, re-queries those with a broader lookback window. // Returns map[resourceID]map[metricType][]MetricPoint (already downsampled). func (m *Monitor) queryStoreBatchMetricMapWithGapFill(resourceType string, resourceIDs []string, duration time.Duration, metricTypes []string) map[string]map[string][]MetricPoint { if m == nil || m.metricsStore == nil || len(resourceIDs) == 0 { return nil } end := time.Now() start := end.Add(-duration) // queryBatch fetches and downsamples metrics for the given IDs. queryBatch := func(ids []string, from time.Time) map[string]map[string][]MetricPoint { sqlResult, err := m.metricsStore.QueryMetricTypesBatch(resourceType, ids, metricTypes, from, end, 0) if err != nil || len(sqlResult) == 0 { return nil } // Convert metrics.MetricPoint → monitoring.MetricPoint and downsample. converted := make(map[string]map[string][]MetricPoint, len(sqlResult)) for resID, metricMap := range sqlResult { converted[resID] = convertAndDownsample(metricMap, chartDownsampleTarget) } return converted } result := queryBatch(resourceIDs, start) if result == nil { result = make(map[string]map[string][]MetricPoint) } // Collect resource IDs that returned no data for gap-fill retry. var missing []string for _, id := range resourceIDs { if _, ok := result[id]; !ok { missing = append(missing, id) } } if len(missing) > 0 { gapStart := end.Add(-chartGapFillLookbackWindow(duration)) if gapResult := queryBatch(missing, gapStart); gapResult != nil { for id, metricMap := range gapResult { result[id] = metricMap } } } return result } // GuestChartRequest identifies a single guest for batch chart retrieval. type GuestChartRequest struct { InMemoryKey string // key in the in-memory ring buffer SQLResourceID string // resource_id in the SQLite store } func normalizeChartMetricTypes(metricTypes []string) []string { if len(metricTypes) == 0 { return nil } seen := make(map[string]struct{}, len(metricTypes)) normalized := make([]string, 0, len(metricTypes)) for _, metricType := range metricTypes { canonical := strings.ToLower(strings.TrimSpace(metricType)) if canonical == "" { continue } if _, ok := seen[canonical]; ok { continue } seen[canonical] = struct{}{} normalized = append(normalized, canonical) } if len(normalized) == 0 { return nil } return normalized } func filterChartMetricMap(metricMap map[string][]MetricPoint, metricTypes []string) map[string][]MetricPoint { if len(metricTypes) == 0 || len(metricMap) == 0 { return metricMap } filtered := make(map[string][]MetricPoint, len(metricTypes)) for _, metricType := range metricTypes { if points, ok := metricMap[metricType]; ok { filtered[metricType] = points } } return filtered } func (m *Monitor) shouldLoadGuestChartInMemory(inMemoryKey string, metricTypes []string, duration time.Duration) bool { if m == nil || m.metricsHistory == nil { return false } if m.metricsStore == nil || duration <= inMemoryChartThreshold { return true } return m.metricsHistory.GuestMetricCoverageSpan(inMemoryKey, metricTypes, duration) >= chartCoverageRequiredSpan(duration) } func (m *Monitor) shouldLoadNodeChartInMemory(nodeID string, metricTypes []string, duration time.Duration) bool { if m == nil || m.metricsHistory == nil { return false } if m.metricsStore == nil || duration <= inMemoryChartThreshold { return true } return m.metricsHistory.NodeMetricCoverageSpan(nodeID, metricTypes, duration) >= chartCoverageRequiredSpan(duration) } // GetGuestMetricsForChartBatch returns chart metrics for multiple guests of the // same SQL resource type, using batch SQL queries instead of N individual // queries. Results are keyed by SQLResourceID. func (m *Monitor) GetGuestMetricsForChartBatch( sqlResourceType string, requests []GuestChartRequest, duration time.Duration, metricTypes ...string, ) map[string]map[string][]MetricPoint { if m == nil || len(requests) == 0 { return nil } requestedMetricTypes := normalizeChartMetricTypes(metricTypes) if mock.IsMockEnabled() { result := make(map[string]map[string][]MetricPoint, len(requests)) for _, req := range requests { inMemory := m.GetGuestMetrics(req.InMemoryKey, duration) result[req.SQLResourceID] = filterChartMetricMap(m.mockGuestMetricsForChart( req.InMemoryKey, sqlResourceType, req.SQLResourceID, duration, inMemory, ), requestedMetricTypes) } return result } result := make(map[string]map[string][]MetricPoint, len(requests)) // Phase 1: Check in-memory for all guests and identify which need fallback. var needFallback []string for _, req := range requests { inMemory := map[string][]MetricPoint{} if m.shouldLoadGuestChartInMemory(req.InMemoryKey, requestedMetricTypes, duration) { inMemory = filterChartMetricMap(m.GetGuestMetrics(req.InMemoryKey, duration), requestedMetricTypes) } result[req.SQLResourceID] = inMemory if hasSufficientChartMapCoverage(inMemory, duration) { continue } needFallback = append(needFallback, req.SQLResourceID) } if len(needFallback) == 0 { return result } // Phase 2: Batch-query store, trying all resource type candidates and // keeping the best coverage per ID. Ties on span are broken by total // point count, matching the single-resource alias resolution logic. storeResults := make(map[string]map[string][]MetricPoint) if m.metricsStore != nil { for _, candidate := range monitorStoreResourceTypeCandidates(sqlResourceType) { batch := m.queryStoreBatchMetricMapWithGapFill(candidate, needFallback, duration, requestedMetricTypes) for id, metricMap := range batch { if existing, ok := storeResults[id]; ok { newSpan := chartMapCoverageSpan(metricMap) oldSpan := chartMapCoverageSpan(existing) if newSpan < oldSpan || (newSpan == oldSpan && monitorMetricMapPointCount(metricMap) <= monitorMetricMapPointCount(existing)) { continue } } storeResults[id] = metricMap } } } nativeResults := m.nativeGuestMetricHistory(sqlResourceType, duration) // Phase 3: Merge fallback data per metric type. for _, id := range needFallback { best := cloneMetricPointMap(result[id]) storeData, ok := storeResults[id] if ok { best = mergeGuestMetricHistory(best, filterChartMetricMap(storeData, requestedMetricTypes), duration) } nativeData, ok := nativeResults[id] if !ok { result[id] = best continue } result[id] = mergeGuestMetricHistory(best, filterChartMetricMap(nativeData, requestedMetricTypes), duration) } return result } // GetNodeMetricsForChartBatch returns chart metrics for multiple nodes, // using batch SQL queries instead of N×M individual queries (where M is the // number of metric types). Results are keyed by nodeID, then by metric type. func (m *Monitor) GetNodeMetricsForChartBatch( nodeIDs []string, metricTypes []string, duration time.Duration, ) map[string]map[string][]MetricPoint { if m == nil || len(nodeIDs) == 0 { return nil } if mock.IsMockEnabled() { result := make(map[string]map[string][]MetricPoint, len(nodeIDs)) for _, nodeID := range nodeIDs { nodeResult := make(map[string][]MetricPoint, len(metricTypes)) for _, metricType := range metricTypes { var inMemoryPoints []MetricPoint if m != nil && m.metricsHistory != nil { inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration) } nodeResult[metricType] = m.mockNodeMetricsForChart(nodeID, metricType, duration, inMemoryPoints) } result[nodeID] = nodeResult } return result } result := make(map[string]map[string][]MetricPoint, len(nodeIDs)) // Phase 1: Check in-memory for all nodes and identify which need store. var needStore []string for _, nid := range nodeIDs { nodeResult := make(map[string][]MetricPoint, len(metricTypes)) allSufficient := true loadInMemory := m.shouldLoadNodeChartInMemory(nid, metricTypes, duration) for _, mt := range metricTypes { if !loadInMemory { nodeResult[mt] = nil allSufficient = false continue } points := m.metricsHistory.GetNodeMetrics(nid, mt, duration) nodeResult[mt] = points if m.metricsStore != nil && !hasSufficientChartSeriesCoverage(points, duration) { allSufficient = false } } result[nid] = nodeResult if !allSufficient { needStore = append(needStore, nid) } } if len(needStore) == 0 { return result } // Phase 2: Batch-query store for all nodes that need it. batchResult := m.queryStoreBatchMetricMapWithGapFill("node", needStore, duration, metricTypes) // Phase 3: Merge — per metric type, use store data if better coverage. for _, nid := range needStore { storeData, ok := batchResult[nid] if !ok { continue } for _, mt := range metricTypes { storePoints, found := storeData[mt] if !found { continue } inMemory := result[nid][mt] if chartSeriesCoverageSpan(storePoints) <= chartSeriesCoverageSpan(inMemory) { continue } result[nid][mt] = storePoints } } return result } // GetStorageMetricsForChartBatch returns chart metrics for multiple storage // pools, using batch SQL queries instead of N individual queries. // Results are keyed by storageID. func (m *Monitor) GetStorageMetricsForChartBatch( storageIDs []string, duration time.Duration, ) map[string]map[string][]MetricPoint { if m == nil || len(storageIDs) == 0 { return nil } result := make(map[string]map[string][]MetricPoint, len(storageIDs)) if mock.IsMockEnabled() { for _, sid := range storageIDs { inMemory := map[string][]MetricPoint{} if m.metricsHistory != nil { inMemory = m.metricsHistory.GetAllStorageMetrics(sid, duration) } result[sid] = m.mockStorageMetricsForChartCached(sid, duration, inMemory) } return result } // Phase 1: Check in-memory for all storage pools. var needStore []string for _, sid := range storageIDs { inMemory := map[string][]MetricPoint{} if m.metricsHistory != nil { inMemory = m.metricsHistory.GetAllStorageMetrics(sid, duration) } if m.metricsStore == nil { result[sid] = inMemory continue } if hasSufficientChartMapCoverageForMetrics(inMemory, duration, storageChartMetricTypes) { result[sid] = inMemory continue } needStore = append(needStore, sid) result[sid] = inMemory // keep as fallback } if len(needStore) == 0 { return result } // Phase 2: Batch-query store. batchResult := m.queryStoreBatchMetricMapWithGapFill("storage", needStore, duration, storageChartMetricTypes) // Phase 3: Merge — use store data if better coverage. for _, id := range needStore { storeData, ok := batchResult[id] if !ok { continue } result[id] = mergeMetricHistory(result[id], storeData, duration) } return result } // GetStorageCapacityMetricsForSummaryBatch returns only the canonical capacity // metrics required for the compact dashboard storage summary card. func (m *Monitor) GetStorageCapacityMetricsForSummaryBatch( storageIDs []string, duration time.Duration, ) map[string]map[string][]MetricPoint { if m == nil || len(storageIDs) == 0 { return nil } result := make(map[string]map[string][]MetricPoint, len(storageIDs)) if mock.IsMockEnabled() { for _, sid := range storageIDs { inMemory := map[string][]MetricPoint{} if m.metricsHistory != nil { inMemory = filterMetricPointMap( m.metricsHistory.GetAllStorageMetrics(sid, duration), storageSummaryChartMetricTypes, ) } fullMetrics := m.mockStorageMetricsForChartCached(sid, duration, inMemory) result[sid] = filterMetricPointMap(fullMetrics, storageSummaryChartMetricTypes) } return result } var needStore []string for _, sid := range storageIDs { inMemory := map[string][]MetricPoint{} if m.metricsHistory != nil { inMemory = filterMetricPointMap( m.metricsHistory.GetAllStorageMetrics(sid, duration), storageSummaryChartMetricTypes, ) } if m.metricsStore == nil { result[sid] = inMemory continue } if hasSufficientChartMapCoverageForMetrics(inMemory, duration, storageSummaryChartMetricTypes) { result[sid] = inMemory continue } needStore = append(needStore, sid) result[sid] = inMemory } if len(needStore) == 0 { return result } batchResult := m.queryStoreBatchMetricMapWithGapFill( "storage", needStore, duration, storageSummaryChartMetricTypes, ) for _, sid := range needStore { storeData, ok := batchResult[sid] if !ok { continue } result[sid] = mergeMetricHistory(result[sid], storeData, duration) } return result } func (m *Monitor) GetStorageSummaryCapacityTrend(duration time.Duration) ([]MetricPoint, int64) { if m == nil { return nil, 0 } if mock.IsMockEnabled() { return m.mockStorageSummaryCapacityTrendCached(duration) } readState := m.GetUnifiedReadStateOrSnapshot() if readState == nil { return nil, 0 } storageIDs := make([]string, 0, len(readState.StoragePools())) for _, pool := range readState.StoragePools() { if pool == nil { continue } storageID := strings.TrimSpace(pool.SourceID()) if storageID == "" { continue } storageIDs = append(storageIDs, storageID) } return buildStorageSummaryCapacityTrend( m.GetStorageCapacityMetricsForSummaryBatch(storageIDs, duration), ) } func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration) map[string]map[string][]MetricPoint { providers := m.supplementalProviderSnapshot() if len(providers) == 0 { return nil } orgID := "default" if m != nil { if trimmed := strings.TrimSpace(m.GetOrgID()); trimmed != "" { orgID = trimmed } } history := make(map[string]map[string][]MetricPoint) for _, provider := range providers { historyProvider, ok := provider.(MonitorGuestMetricHistoryProvider) if !ok { continue } nativeHistory := historyProvider.GuestMetricHistory(m, orgID, resourceType, duration) for resourceID, metricMap := range nativeHistory { if strings.TrimSpace(resourceID) == "" || len(metricMap) == 0 { continue } existing := history[resourceID] history[resourceID] = mergeGuestMetricHistory(existing, metricMap, duration) } } if len(history) == 0 { return nil } return history } func chartGapFillLookbackWindow(duration time.Duration) time.Duration { lookback := duration * chartGapFillLookbackMultiplier if lookback < chartGapFillLookbackMin { lookback = chartGapFillLookbackMin } if lookback > chartGapFillLookbackMax { lookback = chartGapFillLookbackMax } return lookback } func convertSeriesAndDownsample(points []metrics.MetricPoint, target int) []MetricPoint { converted := make([]MetricPoint, len(points)) for i, p := range points { converted[i] = MetricPoint{Value: p.Value, Timestamp: p.Timestamp} } return lttb(converted, target) } func (m *Monitor) queryStoreMetricMapWithGapFill(resourceType, resourceID string, duration time.Duration) (map[string][]MetricPoint, bool) { if m == nil || m.metricsStore == nil { return nil, false } end := time.Now() start := end.Add(-duration) query := func(from time.Time) (map[string][]MetricPoint, bool) { sqlResult, err := m.metricsStore.QueryAll(resourceType, resourceID, from, end, 0) if err != nil || len(sqlResult) == 0 { return nil, false } return convertAndDownsample(sqlResult, chartDownsampleTarget), true } if result, ok := query(start); ok { return result, true } return query(end.Add(-chartGapFillLookbackWindow(duration))) } func (m *Monitor) queryStoreMetricMapWithGapFillAliases(resourceType, resourceID string, duration time.Duration) (map[string][]MetricPoint, bool) { var ( best map[string][]MetricPoint bestSpan time.Duration bestPoints int found bool ) for _, candidate := range monitorStoreResourceTypeCandidates(resourceType) { result, ok := m.queryStoreMetricMapWithGapFill(candidate, resourceID, duration) if !ok { continue } span := chartMapCoverageSpan(result) points := monitorMetricMapPointCount(result) if !found || span > bestSpan || (span == bestSpan && points > bestPoints) { best = result bestSpan = span bestPoints = points found = true } } return best, found } func (m *Monitor) queryStoreMetricSeriesWithGapFill(resourceType, resourceID, metricType string, duration time.Duration) ([]MetricPoint, bool) { if m == nil || m.metricsStore == nil { return nil, false } end := time.Now() start := end.Add(-duration) query := func(from time.Time) ([]MetricPoint, bool) { sqlPoints, err := m.metricsStore.Query(resourceType, resourceID, metricType, from, end, 0) if err != nil || len(sqlPoints) == 0 { return nil, false } return convertSeriesAndDownsample(sqlPoints, chartDownsampleTarget), true } if points, ok := query(start); ok { return points, true } return query(end.Add(-chartGapFillLookbackWindow(duration))) } // convertAndDownsample converts pkg/metrics.MetricPoint slices to // internal/monitoring.MetricPoint slices and applies LTTB downsampling. func convertAndDownsample(sqlResult map[string][]metrics.MetricPoint, target int) map[string][]MetricPoint { result := make(map[string][]MetricPoint, len(sqlResult)) for metric, points := range sqlResult { converted := make([]MetricPoint, len(points)) for i, p := range points { converted[i] = MetricPoint{Value: p.Value, Timestamp: p.Timestamp} } result[metric] = lttb(converted, target) } return result } func cloneMetricSeries(points []MetricPoint) []MetricPoint { if len(points) == 0 { return nil } cloned := make([]MetricPoint, len(points)) copy(cloned, points) return cloned } func cloneMetricPointMap(metrics map[string][]MetricPoint) map[string][]MetricPoint { if len(metrics) == 0 { return map[string][]MetricPoint{} } cloned := make(map[string][]MetricPoint, len(metrics)) for metricType, points := range metrics { cloned[metricType] = cloneMetricSeries(points) } return cloned } func mockChartPointTarget(duration time.Duration) int { switch { case duration <= time.Hour: return mockChartPointTargetShortRange case duration <= 6*time.Hour: return mockChartPointTargetMedium case duration <= 24*time.Hour: return mockChartPointTargetLong default: return mockChartPointTargetExtended } } func downsampleMetricSeriesForMockChart(points []MetricPoint, duration time.Duration) []MetricPoint { target := mockChartPointTarget(duration) return lttb(cloneMetricSeries(points), target) } func downsampleMetricMapForMockChart(metrics map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint { if len(metrics) == 0 { return map[string][]MetricPoint{} } downsampled := make(map[string][]MetricPoint, len(metrics)) for metricType, points := range metrics { downsampled[metricType] = downsampleMetricSeriesForMockChart(points, duration) } return downsampled } func normalizeMockChartMetricTypes(metricTypes []string) string { if len(metricTypes) == 0 { return "" } normalized := make([]string, 0, len(metricTypes)) for _, metricType := range metricTypes { trimmed := strings.TrimSpace(metricType) if trimmed == "" { continue } normalized = append(normalized, trimmed) } if len(normalized) == 0 { return "" } sort.Strings(normalized) return strings.Join(normalized, ",") } func (m *Monitor) readMockChartMetricMapCache( key mockChartMetricMapCacheKey, ) (map[string][]MetricPoint, bool) { if m == nil { return nil, false } m.mockChartCacheMu.RLock() defer m.mockChartCacheMu.RUnlock() if m.mockChartMapCache == nil { return nil, false } cached, ok := m.mockChartMapCache[key] if !ok { return nil, false } return cloneMetricPointMap(cached), true } func (m *Monitor) writeMockChartMetricMapCache( key mockChartMetricMapCacheKey, metrics map[string][]MetricPoint, ) map[string][]MetricPoint { cloned := cloneMetricPointMap(metrics) if m == nil { return cloned } m.mockChartCacheMu.Lock() defer m.mockChartCacheMu.Unlock() if m.mockChartMapCache == nil { m.mockChartMapCache = make(map[mockChartMetricMapCacheKey]map[string][]MetricPoint) } m.mockChartMapCache[key] = cloneMetricPointMap(cloned) return cloned } func (m *Monitor) invalidateMockChartCaches() { if m == nil { return } m.mockChartCacheMu.Lock() defer m.mockChartCacheMu.Unlock() if m.mockChartMapCache != nil { clear(m.mockChartMapCache) } } type mockWorkloadChartPrewarmRequests struct { vms []GuestChartRequest containers []GuestChartRequest pods []GuestChartRequest dockerContainers []GuestChartRequest } func appendMockGuestChartRequest( requests []GuestChartRequest, seen map[string]struct{}, inMemoryKey string, sqlResourceID string, ) []GuestChartRequest { inMemoryKey = strings.TrimSpace(inMemoryKey) sqlResourceID = strings.TrimSpace(sqlResourceID) if inMemoryKey == "" || sqlResourceID == "" { return requests } identity := inMemoryKey + "\x00" + sqlResourceID if _, ok := seen[identity]; ok { return requests } seen[identity] = struct{}{} return append(requests, GuestChartRequest{ InMemoryKey: inMemoryKey, SQLResourceID: sqlResourceID, }) } func mockMetricIDFromTargetOrSource(target *unifiedresources.MetricsTarget, sourceID string) string { if target != nil { if metricID := strings.TrimSpace(target.ResourceID); metricID != "" { return metricID } } return strings.TrimSpace(sourceID) } func mockKubernetesPodChartMetricID(pod *unifiedresources.PodView) string { if pod == nil { return "" } clusterKey := strings.TrimSpace(pod.ClusterID()) if clusterKey == "" { clusterKey = strings.TrimSpace(pod.ClusterName()) } podKey := strings.TrimSpace(pod.PodUID()) if podKey == "" { namespace := strings.TrimSpace(pod.Namespace()) name := strings.TrimSpace(pod.Name()) if namespace != "" || name != "" { podKey = namespace + "/" + name } } if clusterKey == "" || podKey == "" { return "" } return "k8s:" + clusterKey + ":pod:" + podKey } func mockWorkloadChartRequestsForReadState( readState unifiedresources.ReadState, ) mockWorkloadChartPrewarmRequests { if readState == nil { return mockWorkloadChartPrewarmRequests{} } requests := mockWorkloadChartPrewarmRequests{ vms: make([]GuestChartRequest, 0, len(readState.VMs())), containers: make([]GuestChartRequest, 0, len(readState.Containers())), pods: make([]GuestChartRequest, 0, len(readState.Pods())), dockerContainers: make([]GuestChartRequest, 0, len(readState.DockerContainers())), } seenVMs := make(map[string]struct{}, len(readState.VMs())) seenContainers := make(map[string]struct{}, len(readState.Containers())) seenPods := make(map[string]struct{}, len(readState.Pods())) seenDockerContainers := make(map[string]struct{}, len(readState.DockerContainers())) for _, vm := range readState.VMs() { if vm == nil { continue } metricID := mockMetricIDFromTargetOrSource(vm.MetricsTarget(), vm.SourceID()) requests.vms = appendMockGuestChartRequest(requests.vms, seenVMs, metricID, metricID) } for _, container := range readState.Containers() { if container == nil { continue } metricID := mockMetricIDFromTargetOrSource(container.MetricsTarget(), container.SourceID()) requests.containers = appendMockGuestChartRequest( requests.containers, seenContainers, metricID, metricID, ) } for _, pod := range readState.Pods() { metricID := mockKubernetesPodChartMetricID(pod) requests.pods = appendMockGuestChartRequest(requests.pods, seenPods, metricID, metricID) } for _, container := range readState.DockerContainers() { if container == nil { continue } metricID := mockMetricIDFromTargetOrSource(container.MetricsTarget(), container.ContainerID()) requests.dockerContainers = appendMockGuestChartRequest( requests.dockerContainers, seenDockerContainers, "docker:"+metricID, metricID, ) } return requests } func (m *Monitor) prewarmMockWorkloadChartCaches(duration time.Duration) { if m == nil || duration <= 0 { return } requests := mockWorkloadChartRequestsForReadState(m.GetUnifiedReadStateOrSnapshot()) if len(requests.vms) > 0 { _ = m.GetGuestMetricsForChartBatch("vm", requests.vms, duration) } if len(requests.containers) > 0 { _ = m.GetGuestMetricsForChartBatch("container", requests.containers, duration) } if len(requests.pods) > 0 { _ = m.GetGuestMetricsForChartBatch("k8s", requests.pods, duration) } if len(requests.dockerContainers) > 0 { _ = m.GetGuestMetricsForChartBatch("dockerContainer", requests.dockerContainers, duration) } } func (m *Monitor) prewarmMockDashboardChartCaches() { if m == nil || !mock.IsMockEnabled() { return } _, _ = m.mockStorageSummaryCapacityTrendCached(24 * time.Hour) m.prewarmMockWorkloadChartCaches(mockDashboardWorkloadPrewarmDuration) } func (m *Monitor) mockGuestMetricsForChart( inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration, inMemoryResult map[string][]MetricPoint, ) map[string][]MetricPoint { key := mockChartMetricMapCacheKey{ kind: "guest", resourceType: strings.TrimSpace(sqlResourceType), resourceID: strings.TrimSpace(sqlResourceID), aux: strings.TrimSpace(inMemoryKey), duration: duration, } if cached, ok := m.readMockChartMetricMapCache(key); ok { return cached } var computed map[string][]MetricPoint switch { case hasSufficientChartMapCoverage(inMemoryResult, duration): computed = downsampleMetricMapForMockChart(inMemoryResult, duration) case len(sqlResourceID) > 0: if synthetic := mockGuestMetricsForChart(sqlResourceType, sqlResourceID, duration); len(synthetic) > 0 { computed = downsampleMetricMapForMockChart(synthetic, duration) } } if computed == nil { computed = downsampleMetricMapForMockChart(inMemoryResult, duration) } return m.writeMockChartMetricMapCache(key, computed) } func (m *Monitor) mockNodeMetricsForChart( nodeID string, metricType string, duration time.Duration, inMemoryPoints []MetricPoint, ) []MetricPoint { cacheKey := mockChartMetricMapCacheKey{ kind: "node", resourceType: "node", resourceID: strings.TrimSpace(nodeID), aux: normalizeMockChartMetricTypes([]string{metricType}), duration: duration, } if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok { return cloneMetricSeries(cached[metricType]) } computed := map[string][]MetricPoint{} if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) { computed[metricType] = downsampleMetricSeriesForMockChart(inMemoryPoints, duration) } else { computed[metricType] = downsampleMetricSeriesForMockChart( mockNodeMetricsForChart(nodeID, []string{metricType}, duration)[metricType], duration, ) } return cloneMetricSeries(m.writeMockChartMetricMapCache(cacheKey, computed)[metricType]) } func (m *Monitor) mockDiskMetricsForChart( resourceID string, metricType string, duration time.Duration, inMemoryPoints []MetricPoint, ) []MetricPoint { cacheKey := mockChartMetricMapCacheKey{ kind: "disk", resourceType: "disk", resourceID: strings.TrimSpace(resourceID), aux: normalizeMockChartMetricTypes([]string{metricType}), duration: duration, } if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok { return cloneMetricSeries(cached[metricType]) } computed := map[string][]MetricPoint{} if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) { computed[metricType] = downsampleMetricSeriesForMockChart(inMemoryPoints, duration) } else { computed[metricType] = downsampleMetricSeriesForMockChart( mockDiskMetricsForChart(resourceID, []string{metricType}, duration)[metricType], duration, ) } return cloneMetricSeries(m.writeMockChartMetricMapCache(cacheKey, computed)[metricType]) } func (m *Monitor) mockStorageMetricsForChartCached( storageID string, duration time.Duration, inMemoryResult map[string][]MetricPoint, ) map[string][]MetricPoint { cacheKey := mockChartMetricMapCacheKey{ kind: "storage", resourceType: "storage", resourceID: strings.TrimSpace(storageID), duration: duration, } if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok { return cached } var computed map[string][]MetricPoint if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, storageChartMetricTypes) { computed = downsampleMetricMapForMockChart(inMemoryResult, duration) } else { computed = downsampleMetricMapForMockChart( m.mockStorageMetricsForChart(storageID, duration, inMemoryResult), duration, ) } return m.writeMockChartMetricMapCache(cacheKey, computed) } func (m *Monitor) mockStorageSummaryCapacityTrendCached(duration time.Duration) ([]MetricPoint, int64) { cacheKey := mockChartMetricMapCacheKey{ kind: "storage-summary", resourceType: "storage", resourceID: "__aggregate__", duration: duration, } if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok { return cached["capacity"], oldestMetricSeriesTimestamp(cached["capacity"]) } computed := m.mockStorageSummaryCapacityTrend(duration) cached := m.writeMockChartMetricMapCache(cacheKey, map[string][]MetricPoint{ "capacity": computed, }) return cached["capacity"], oldestMetricSeriesTimestamp(cached["capacity"]) } func mergeGuestMetricHistory(base, candidate map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint { return mergeMetricHistory(base, candidate, duration) } func mergeMetricHistory(base, candidate map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint { if len(candidate) == 0 { return cloneMetricPointMap(base) } merged := cloneMetricPointMap(base) for metricType, candidateSeries := range candidate { if !shouldPreferMetricSeries(merged[metricType], candidateSeries, duration) { continue } merged[metricType] = cloneMetricSeries(candidateSeries) } return merged } func filterMetricPointMap(metricMap map[string][]MetricPoint, metricTypes []string) map[string][]MetricPoint { if len(metricTypes) == 0 { return cloneMetricPointMap(metricMap) } filtered := make(map[string][]MetricPoint, len(metricTypes)) for _, metricType := range metricTypes { if len(metricMap[metricType]) == 0 { continue } filtered[metricType] = cloneMetricSeries(metricMap[metricType]) } return filtered } func buildStorageSummaryCapacityTrend( poolMetrics map[string]map[string][]MetricPoint, ) ([]MetricPoint, int64) { type aggregateBucket struct { used float64 avail float64 hasUsed bool hasAvail bool } buckets := make(map[int64]*aggregateBucket) var oldestTimestamp int64 for _, metrics := range poolMetrics { for _, point := range metrics["used"] { timestamp := point.Timestamp.UnixMilli() bucket := buckets[timestamp] if bucket == nil { bucket = &aggregateBucket{} buckets[timestamp] = bucket } bucket.used += point.Value bucket.hasUsed = true if oldestTimestamp == 0 || timestamp < oldestTimestamp { oldestTimestamp = timestamp } } for _, point := range metrics["avail"] { timestamp := point.Timestamp.UnixMilli() bucket := buckets[timestamp] if bucket == nil { bucket = &aggregateBucket{} buckets[timestamp] = bucket } bucket.avail += point.Value bucket.hasAvail = true if oldestTimestamp == 0 || timestamp < oldestTimestamp { oldestTimestamp = timestamp } } } if len(buckets) == 0 { return nil, oldestTimestamp } timestamps := make([]int64, 0, len(buckets)) for timestamp := range buckets { timestamps = append(timestamps, timestamp) } sort.Slice(timestamps, func(i, j int) bool { return timestamps[i] < timestamps[j] }) out := make([]MetricPoint, 0, len(timestamps)) for _, timestamp := range timestamps { bucket := buckets[timestamp] if bucket == nil || !bucket.hasUsed || !bucket.hasAvail { continue } total := bucket.used + bucket.avail if math.IsNaN(total) || math.IsInf(total, 0) || total <= 0 { continue } out = append(out, MetricPoint{ Timestamp: time.UnixMilli(timestamp), Value: (bucket.used / total) * 100, }) } return out, oldestTimestamp } func oldestMetricSeriesTimestamp(points []MetricPoint) int64 { var oldest int64 for _, point := range points { timestamp := point.Timestamp.UnixMilli() if oldest == 0 || timestamp < oldest { oldest = timestamp } } return oldest } func shouldPreferMetricSeries(current, candidate []MetricPoint, duration time.Duration) bool { if len(candidate) == 0 { return false } if len(current) == 0 { return true } candidateSpan := chartSeriesCoverageSpan(candidate) currentSpan := chartSeriesCoverageSpan(current) currentSufficient := hasSufficientChartSeriesCoverage(current, duration) candidateSufficient := hasSufficientChartSeriesCoverage(candidate, duration) switch { case currentSufficient && !candidateSufficient: return false case !currentSufficient && candidateSufficient: return true } if candidateSpan > currentSpan { return true } if candidateSpan < currentSpan { return false } return len(candidate) > len(current) } func chartCoverageRequiredSpan(duration time.Duration) time.Duration { if duration <= 0 { return 0 } required := (duration * shortRangeCoverageRatioNum) / shortRangeCoverageRatioDen slack := duration / 10 if slack > shortRangeCoverageMaxSlack { slack = shortRangeCoverageMaxSlack } if required <= slack { return 0 } return required - slack } func chartSeriesCoverageSpan(points []MetricPoint) time.Duration { if len(points) < 2 { return 0 } oldest := points[0].Timestamp newest := points[0].Timestamp for i := 1; i < len(points); i++ { ts := points[i].Timestamp if ts.Before(oldest) { oldest = ts } if ts.After(newest) { newest = ts } } if !newest.After(oldest) { return 0 } return newest.Sub(oldest) } func chartMapCoverageSpan(metrics map[string][]MetricPoint) time.Duration { var best time.Duration for _, points := range metrics { span := chartSeriesCoverageSpan(points) if span > best { best = span } } return best } func monitorMetricMapPointCount(metrics map[string][]MetricPoint) int { total := 0 for _, points := range metrics { total += len(points) } return total } func monitorGuestMetricKeyCandidates(key string) []string { trimmed := strings.TrimSpace(key) if trimmed == "" { return []string{""} } return []string{trimmed} } func monitorStoreResourceTypeCandidates(resourceType string) []string { normalized := strings.TrimSpace(resourceType) switch normalized { case "agent": return []string{"agent"} case "dockerContainer": return []string{"dockerContainer", "docker"} case "docker": return []string{"docker", "dockerContainer"} default: return []string{normalized} } } func hasSufficientChartSeriesCoverage(points []MetricPoint, duration time.Duration) bool { return chartSeriesCoverageSpan(points) >= chartCoverageRequiredSpan(duration) } func hasSufficientChartMapCoverage(metrics map[string][]MetricPoint, duration time.Duration) bool { return chartMapCoverageSpan(metrics) >= chartCoverageRequiredSpan(duration) } func hasSufficientChartMapCoverageForMetrics(metrics map[string][]MetricPoint, duration time.Duration, metricTypes []string) bool { if len(metricTypes) == 0 { return hasSufficientChartMapCoverage(metrics, duration) } for _, metricType := range metricTypes { if !hasSufficientChartSeriesCoverage(metrics[metricType], duration) { return false } } return true }