mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 14:07:31 +00:00
b4be3bd55b
Mock chart history thinned each metric with LTTB on its own. LTTB keeps the points whose values stand out, so cpu, memory and disk of one guest kept different timestamps: on the public demo a 24h container history had eight ~48-point series spread over 162 distinct timestamps. The drawer History panels break a line wherever another series in the panel has a reading this one lacks, so every mock chart drew as dots with a "Missing observations" note. Real history does not do this because the poller writes all of a resource's metrics at the same times. Thin mock series onto an epoch-aligned grid instead: keep the first reading in each bucket plus the newest reading, at their own timestamps. Which readings survive now depends only on when they were taken, so series sampled together keep identical timestamps, values stay real readings, and point targets are unchanged. The synthetic generator no longer pre-thins to 500 points, since every caller thins onto the grid.
1497 lines
46 KiB
Go
1497 lines
46 KiB
Go
package monitoring
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
|
||
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
||
)
|
||
|
||
// inMemoryChartThreshold is the max duration reliably covered by the in-memory
|
||
// metrics buffer. With 1000 data-points at a 10 s polling interval the buffer
|
||
// holds ~2.8 h of data; 2 h is a safe conservative cut-off.
|
||
const inMemoryChartThreshold = 2 * time.Hour
|
||
|
||
// chartDownsampleTarget is the number of points returned per metric when
|
||
// falling back to the persistent store. 500 is more than enough for any
|
||
// sparkline or thumbnail chart.
|
||
const chartDownsampleTarget = 500
|
||
|
||
const (
|
||
// When the requested chart window has no data (for example after running in
|
||
// mock mode for a while), query a broader historical window so sparklines
|
||
// can render immediately instead of waiting for enough fresh samples.
|
||
chartGapFillLookbackMultiplier = 12
|
||
chartGapFillLookbackMin = 6 * time.Hour
|
||
chartGapFillLookbackMax = 7 * 24 * time.Hour
|
||
)
|
||
|
||
// For short chart ranges (<= inMemoryChartThreshold), we prefer in-memory data
|
||
// for freshness unless coverage is too shallow (for example after a restart).
|
||
// In that case we fall back to SQLite history to avoid mostly-empty charts.
|
||
const (
|
||
shortRangeCoverageRatioNum = 3
|
||
shortRangeCoverageRatioDen = 4
|
||
shortRangeCoverageMaxSlack = 2 * time.Minute
|
||
)
|
||
|
||
const (
|
||
mockChartPointTargetShortRange = 20
|
||
mockChartPointTargetMedium = 32
|
||
mockChartPointTargetLong = 48
|
||
mockChartPointTargetExtended = 64
|
||
)
|
||
|
||
const (
|
||
mockDashboardWorkloadPrewarmDuration = time.Hour
|
||
// Prewarming every guest creates an estate-sized chart cache once per mock
|
||
// sampler tick. A first-viewport cohort removes cold-start latency while
|
||
// keeping the rest available through the same lazy deterministic path.
|
||
mockDashboardWorkloadPrewarmLimitPerType = 16
|
||
)
|
||
|
||
var storageChartMetricTypes = []string{"usage", "used", "avail", "total"}
|
||
|
||
type mockChartMetricMapCacheKey struct {
|
||
kind string
|
||
resourceType string
|
||
resourceID string
|
||
aux string
|
||
duration time.Duration
|
||
}
|
||
|
||
// MonitorGuestMetricHistoryProvider optionally exposes source-native guest
|
||
// metric history through the canonical chart boundary when local Pulse history
|
||
// is shallow.
|
||
type MonitorGuestMetricHistoryProvider interface {
|
||
GuestMetricHistory(m *Monitor, orgID string, resourceType string, duration time.Duration) map[string]map[string][]MetricPoint
|
||
}
|
||
|
||
// GetGuestMetrics returns historical metrics for a guest
|
||
func (m *Monitor) GetGuestMetrics(guestID string, duration time.Duration) map[string][]MetricPoint {
|
||
if m == nil || m.metricsHistory == nil {
|
||
return map[string][]MetricPoint{}
|
||
}
|
||
|
||
candidates := monitorGuestMetricKeyCandidates(guestID)
|
||
best := map[string][]MetricPoint{}
|
||
bestSpan := time.Duration(0)
|
||
bestPoints := 0
|
||
selected := false
|
||
|
||
for _, candidate := range candidates {
|
||
result := m.metricsHistory.GetAllGuestMetrics(candidate, duration)
|
||
span := chartMapCoverageSpan(result)
|
||
points := monitorMetricMapPointCount(result)
|
||
if !selected || span > bestSpan || (span == bestSpan && points > bestPoints) {
|
||
best = result
|
||
bestSpan = span
|
||
bestPoints = points
|
||
selected = true
|
||
}
|
||
}
|
||
|
||
return best
|
||
}
|
||
|
||
// GetGuestMetricsForChart returns guest metrics optimised for chart display.
|
||
// Short ranges are served from the in-memory ring buffer; longer ranges fall
|
||
// back to the persistent SQLite store with LTTB downsampling.
|
||
//
|
||
// inMemoryKey is the key used in the in-memory buffer (e.g. "docker:abc123").
|
||
// sqlResourceType/sqlResourceID are the type/id used in the SQLite store
|
||
// (e.g. "dockerContainer"/"abc123").
|
||
func (m *Monitor) GetGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID string, duration time.Duration) map[string][]MetricPoint {
|
||
if mock.IsMockEnabled() {
|
||
return m.mockGuestMetricsForChart(inMemoryKey, sqlResourceType, sqlResourceID, duration, m.GetGuestMetrics(inMemoryKey, duration))
|
||
}
|
||
|
||
inMemoryResult := map[string][]MetricPoint{}
|
||
if m.shouldLoadGuestChartInMemory(inMemoryKey, nil, duration) {
|
||
inMemoryResult = m.GetGuestMetrics(inMemoryKey, duration)
|
||
}
|
||
|
||
if hasSufficientChartMapCoverage(inMemoryResult, duration) {
|
||
return inMemoryResult
|
||
}
|
||
|
||
// inMemoryResult is a fresh map from GetGuestMetrics (slices already
|
||
// owned); merge mutates and returns it in place — no clone needed.
|
||
best := inMemoryResult
|
||
if m.metricsStore != nil {
|
||
if converted, ok := m.queryStoreMetricMapWithGapFillAliases(sqlResourceType, sqlResourceID, duration); ok {
|
||
best = mergeGuestMetricHistory(best, converted, duration)
|
||
}
|
||
}
|
||
if nativeHistory := m.nativeGuestMetricHistory(sqlResourceType, duration); len(nativeHistory) > 0 {
|
||
if nativeMetrics, ok := nativeHistory[sqlResourceID]; ok {
|
||
best = mergeGuestMetricHistory(best, nativeMetrics, duration)
|
||
}
|
||
}
|
||
return best
|
||
}
|
||
|
||
// GetNodeMetrics returns historical metrics for a node
|
||
func (m *Monitor) GetNodeMetrics(nodeID string, metricType string, duration time.Duration) []MetricPoint {
|
||
return m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration)
|
||
}
|
||
|
||
// GetNodeMetricsForChart returns node metrics for a single metric type,
|
||
// falling back to SQLite + LTTB for longer ranges.
|
||
func (m *Monitor) GetNodeMetricsForChart(nodeID, metricType string, duration time.Duration) []MetricPoint {
|
||
var inMemoryPoints []MetricPoint
|
||
if mock.IsMockEnabled() {
|
||
if m != nil && m.metricsHistory != nil {
|
||
inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration)
|
||
}
|
||
return m.mockNodeMetricsForChart(nodeID, metricType, duration, inMemoryPoints)
|
||
}
|
||
if m != nil && m.shouldLoadNodeChartInMemory(nodeID, []string{metricType}, duration) {
|
||
inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration)
|
||
}
|
||
|
||
if m.metricsStore == nil {
|
||
return inMemoryPoints
|
||
}
|
||
if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) {
|
||
return inMemoryPoints
|
||
}
|
||
|
||
downsampled, ok := m.queryStoreMetricSeriesWithGapFill("node", nodeID, metricType, duration)
|
||
if !ok {
|
||
return inMemoryPoints
|
||
}
|
||
if chartSeriesCoverageSpan(downsampled) <= chartSeriesCoverageSpan(inMemoryPoints) {
|
||
return inMemoryPoints
|
||
}
|
||
return downsampled
|
||
}
|
||
|
||
// GetStorageMetrics returns historical metrics for storage
|
||
func (m *Monitor) GetStorageMetrics(storageID string, duration time.Duration) map[string][]MetricPoint {
|
||
if m == nil || m.metricsHistory == nil {
|
||
return map[string][]MetricPoint{}
|
||
}
|
||
return m.metricsHistory.GetAllStorageMetrics(storageID, duration)
|
||
}
|
||
|
||
// GetDiskMetrics returns historical metrics for a physical disk.
|
||
func (m *Monitor) GetDiskMetrics(resourceID string, metricType string, duration time.Duration) []MetricPoint {
|
||
if m == nil || m.metricsHistory == nil {
|
||
return nil
|
||
}
|
||
return m.metricsHistory.GetDiskMetrics(resourceID, metricType, duration)
|
||
}
|
||
|
||
// GetDiskMetricsForChart returns physical-disk metrics optimized for chart
|
||
// display, preferring in-memory data for freshness and falling back to the
|
||
// persistent store and source-native temperature history when coverage is shallow.
|
||
func (m *Monitor) GetDiskMetricsForChart(resourceID string, metricType string, duration time.Duration) []MetricPoint {
|
||
inMemoryPoints := m.GetDiskMetrics(resourceID, metricType, duration)
|
||
if mock.IsMockEnabled() {
|
||
return m.mockDiskMetricsForChart(resourceID, metricType, duration, inMemoryPoints)
|
||
}
|
||
|
||
if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) {
|
||
return inMemoryPoints
|
||
}
|
||
|
||
best := cloneMetricSeries(inMemoryPoints)
|
||
if converted, ok := m.queryStoreMetricMapWithGapFill("disk", resourceID, duration); ok {
|
||
if candidate := converted[metricType]; shouldPreferMetricSeries(best, candidate, duration) {
|
||
best = cloneMetricSeries(candidate)
|
||
}
|
||
}
|
||
if metricType == "smart_temp" && !hasSufficientChartSeriesCoverage(best, duration) {
|
||
if nativePoints := m.nativePhysicalDiskTemperatureHistory(duration)[resourceID]; len(nativePoints) > 0 {
|
||
nativePoints = lttb(nativePoints, chartDownsampleTarget)
|
||
if shouldPreferMetricSeries(best, nativePoints, duration) {
|
||
best = cloneMetricSeries(nativePoints)
|
||
}
|
||
}
|
||
}
|
||
|
||
return best
|
||
}
|
||
|
||
// GetStorageMetricsForChart returns storage metrics, falling back to SQLite +
|
||
// LTTB for longer ranges.
|
||
func (m *Monitor) GetStorageMetricsForChart(storageID string, duration time.Duration) map[string][]MetricPoint {
|
||
inMemoryResult := map[string][]MetricPoint{}
|
||
if m != nil && m.metricsHistory != nil {
|
||
inMemoryResult = m.metricsHistory.GetAllStorageMetrics(storageID, duration)
|
||
}
|
||
if mock.IsMockEnabled() {
|
||
return m.mockStorageMetricsForChartCached(storageID, duration, inMemoryResult, m.GetUnifiedReadStateOrSnapshot)
|
||
}
|
||
if m.metricsStore == nil {
|
||
return inMemoryResult
|
||
}
|
||
if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, storageChartMetricTypes) {
|
||
return inMemoryResult
|
||
}
|
||
|
||
// inMemoryResult is a fresh map from GetAllStorageMetrics; merge mutates
|
||
// in place.
|
||
best := inMemoryResult
|
||
if converted, ok := m.queryStoreMetricMapWithGapFill("storage", storageID, duration); ok {
|
||
best = mergeMetricHistory(best, converted, duration)
|
||
}
|
||
return best
|
||
}
|
||
|
||
// DiskChartEntry holds temperature chart data and display metadata for a
|
||
// single physical disk.
|
||
type DiskChartEntry struct {
|
||
Name string `json:"name"`
|
||
Node string `json:"node"`
|
||
Instance string `json:"instance"`
|
||
Temperature []MetricPoint `json:"temperature"`
|
||
}
|
||
|
||
// GetPhysicalDiskTemperatureCharts returns temperature time-series for all
|
||
// physical disks that have temperature data. The Monitor owns state access
|
||
// (monitoring is exempt from the ReadState-only rule), so this keeps state
|
||
// reads out of API handler code.
|
||
//
|
||
// Uses batch queries to load all disk metrics in 1-2 SQL calls instead of N
|
||
// (one per disk), avoiding N+1 query patterns on systems with many disks.
|
||
func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[string]DiskChartEntry {
|
||
if m == nil {
|
||
return nil
|
||
}
|
||
mockEnabled := mock.IsMockEnabled()
|
||
|
||
readState := m.GetUnifiedReadStateOrSnapshot()
|
||
if readState == nil {
|
||
return nil
|
||
}
|
||
metricsTargetStore := m.currentMetricsTargetStore()
|
||
|
||
// Phase 1: Collect disk metadata and resource IDs.
|
||
type diskMeta struct {
|
||
resourceID string
|
||
name string
|
||
node string
|
||
instance string
|
||
temperatureCollected bool
|
||
}
|
||
var disks []diskMeta
|
||
for _, disk := range readState.PhysicalDisks() {
|
||
if disk == nil || disk.Temperature() <= 0 {
|
||
continue
|
||
}
|
||
resourceID := ""
|
||
if metricsTargetStore != nil {
|
||
if target := metricsTargetStore.MetricsTargetForResource(disk.ID()); target != nil {
|
||
resourceID = strings.TrimSpace(target.ResourceID)
|
||
}
|
||
}
|
||
if resourceID == "" {
|
||
resourceID = strings.TrimSpace(disk.MetricResourceID())
|
||
}
|
||
if resourceID == "" {
|
||
continue
|
||
}
|
||
name := strings.TrimSpace(disk.Model())
|
||
if name == "" {
|
||
name = strings.TrimSpace(disk.DevPath())
|
||
}
|
||
disks = append(disks, diskMeta{
|
||
resourceID: resourceID,
|
||
name: name,
|
||
node: strings.TrimSpace(disk.Node()),
|
||
instance: strings.TrimSpace(disk.Instance()),
|
||
temperatureCollected: diskinventory.TemperatureCollected(disk.Temperature(), disk.Collection()),
|
||
})
|
||
}
|
||
|
||
if len(disks) == 0 {
|
||
return make(map[string]DiskChartEntry)
|
||
}
|
||
|
||
// Phase 2: Batch-load metrics for all disks (1-2 queries instead of N).
|
||
resourceIDs := make([]string, len(disks))
|
||
for i, d := range disks {
|
||
resourceIDs[i] = d.resourceID
|
||
}
|
||
inMemoryHistory := make(map[string][]MetricPoint, len(disks))
|
||
if m.metricsHistory != nil {
|
||
for _, d := range disks {
|
||
inMemoryHistory[d.resourceID] = m.metricsHistory.GetDiskMetrics(d.resourceID, "smart_temp", duration)
|
||
}
|
||
}
|
||
var batchMetrics map[string]map[string][]MetricPoint
|
||
var nativeHistory map[string][]MetricPoint
|
||
if !mockEnabled {
|
||
batchMetrics = m.queryStoreBatchMetricMapWithGapFill("disk", resourceIDs, duration, []string{"smart_temp"})
|
||
nativeHistory = m.nativePhysicalDiskTemperatureHistory(duration)
|
||
}
|
||
|
||
// Phase 3: Build result entries.
|
||
result := make(map[string]DiskChartEntry, len(disks))
|
||
for _, d := range disks {
|
||
tempPoints := cloneMetricSeries(inMemoryHistory[d.resourceID])
|
||
if mockEnabled {
|
||
tempPoints = m.mockDiskMetricsForChart(d.resourceID, "smart_temp", duration, tempPoints)
|
||
} else {
|
||
if resMetrics, ok := batchMetrics[d.resourceID]; ok {
|
||
if pts, found := resMetrics["smart_temp"]; found {
|
||
if shouldPreferMetricSeries(tempPoints, pts, duration) {
|
||
tempPoints = cloneMetricSeries(pts)
|
||
}
|
||
}
|
||
}
|
||
if nativePoints, ok := nativeHistory[d.resourceID]; ok {
|
||
nativePoints = lttb(nativePoints, chartDownsampleTarget)
|
||
if shouldPreferMetricSeries(tempPoints, nativePoints, duration) {
|
||
tempPoints = cloneMetricSeries(nativePoints)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Neither a current nor a retained temperature is two historical
|
||
// observations. Keep empty and single-point histories intact instead
|
||
// of inventing a flat line. A collected reading still supplies current
|
||
// disk metadata; a retained-only reading with no history supplies none.
|
||
if len(tempPoints) == 0 && !d.temperatureCollected {
|
||
continue
|
||
}
|
||
|
||
result[d.resourceID] = DiskChartEntry{
|
||
Name: d.name,
|
||
Node: d.node,
|
||
Instance: d.instance,
|
||
Temperature: tempPoints,
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func (m *Monitor) nativePhysicalDiskTemperatureHistory(duration time.Duration) map[string][]MetricPoint {
|
||
if mock.IsMockEnabled() {
|
||
return nil
|
||
}
|
||
|
||
providers := m.supplementalProviderSnapshot()
|
||
if len(providers) == 0 {
|
||
return nil
|
||
}
|
||
|
||
orgID := "default"
|
||
if m != nil {
|
||
if trimmed := strings.TrimSpace(m.GetOrgID()); trimmed != "" {
|
||
orgID = trimmed
|
||
}
|
||
}
|
||
|
||
history := make(map[string][]MetricPoint)
|
||
for _, provider := range providers {
|
||
historyProvider, ok := provider.(MonitorPhysicalDiskTemperatureHistoryProvider)
|
||
if !ok {
|
||
continue
|
||
}
|
||
nativeHistory := historyProvider.PhysicalDiskTemperatureHistory(m, orgID, duration)
|
||
for resourceID, points := range nativeHistory {
|
||
if strings.TrimSpace(resourceID) == "" || len(points) == 0 {
|
||
continue
|
||
}
|
||
if existing, ok := history[resourceID]; !ok || chartSeriesCoverageSpan(points) > chartSeriesCoverageSpan(existing) {
|
||
history[resourceID] = points
|
||
}
|
||
}
|
||
}
|
||
if len(history) == 0 {
|
||
return nil
|
||
}
|
||
return history
|
||
}
|
||
|
||
func (m *Monitor) currentMetricsTargetStore() MetricsTargetResourceStore {
|
||
if m == nil {
|
||
return nil
|
||
}
|
||
|
||
m.mu.RLock()
|
||
defer m.mu.RUnlock()
|
||
|
||
store, ok := m.resourceStore.(MetricsTargetResourceStore)
|
||
if !ok {
|
||
return nil
|
||
}
|
||
return store
|
||
}
|
||
|
||
// queryStoreBatchMetricMapWithGapFill queries metrics for multiple resource IDs
|
||
// of the same type in a single batch. If some resources have no data in the
|
||
// primary window, re-queries those with a broader lookback window.
|
||
// Returns map[resourceID]map[metricType][]MetricPoint (already downsampled).
|
||
func (m *Monitor) queryStoreBatchMetricMapWithGapFill(resourceType string, resourceIDs []string, duration time.Duration, metricTypes []string) map[string]map[string][]MetricPoint {
|
||
if m == nil || m.metricsStore == nil || len(resourceIDs) == 0 {
|
||
return nil
|
||
}
|
||
|
||
end := time.Now()
|
||
start := end.Add(-duration)
|
||
|
||
// queryBatch fetches and downsamples metrics for the given IDs.
|
||
queryBatch := func(ids []string, from time.Time) map[string]map[string][]MetricPoint {
|
||
sqlResult, err := m.metricsStore.QueryMetricTypesBatch(resourceType, ids, metricTypes, from, end, 0)
|
||
if err != nil || len(sqlResult) == 0 {
|
||
return nil
|
||
}
|
||
// Convert metrics.MetricPoint → monitoring.MetricPoint and downsample.
|
||
converted := make(map[string]map[string][]MetricPoint, len(sqlResult))
|
||
for resID, metricMap := range sqlResult {
|
||
converted[resID] = convertAndDownsample(metricMap, chartDownsampleTarget)
|
||
}
|
||
return converted
|
||
}
|
||
|
||
result := queryBatch(resourceIDs, start)
|
||
if result == nil {
|
||
result = make(map[string]map[string][]MetricPoint)
|
||
}
|
||
|
||
// Collect resource IDs that returned no data for gap-fill retry.
|
||
var missing []string
|
||
for _, id := range resourceIDs {
|
||
if _, ok := result[id]; !ok {
|
||
missing = append(missing, id)
|
||
}
|
||
}
|
||
|
||
if len(missing) > 0 {
|
||
gapStart := end.Add(-chartGapFillLookbackWindow(duration))
|
||
if gapResult := queryBatch(missing, gapStart); gapResult != nil {
|
||
for id, metricMap := range gapResult {
|
||
result[id] = metricMap
|
||
}
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
// GuestChartRequest identifies a single guest for batch chart retrieval.
|
||
type GuestChartRequest struct {
|
||
InMemoryKey string // key in the in-memory ring buffer
|
||
SQLResourceID string // resource_id in the SQLite store
|
||
}
|
||
|
||
func normalizeChartMetricTypes(metricTypes []string) []string {
|
||
if len(metricTypes) == 0 {
|
||
return nil
|
||
}
|
||
|
||
seen := make(map[string]struct{}, len(metricTypes))
|
||
normalized := make([]string, 0, len(metricTypes))
|
||
for _, metricType := range metricTypes {
|
||
canonical := strings.ToLower(strings.TrimSpace(metricType))
|
||
if canonical == "" {
|
||
continue
|
||
}
|
||
if _, ok := seen[canonical]; ok {
|
||
continue
|
||
}
|
||
seen[canonical] = struct{}{}
|
||
normalized = append(normalized, canonical)
|
||
}
|
||
if len(normalized) == 0 {
|
||
return nil
|
||
}
|
||
return normalized
|
||
}
|
||
|
||
func filterChartMetricMap(metricMap map[string][]MetricPoint, metricTypes []string) map[string][]MetricPoint {
|
||
if len(metricTypes) == 0 || len(metricMap) == 0 {
|
||
return metricMap
|
||
}
|
||
|
||
filtered := make(map[string][]MetricPoint, len(metricTypes))
|
||
for _, metricType := range metricTypes {
|
||
if points, ok := metricMap[metricType]; ok {
|
||
filtered[metricType] = points
|
||
}
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
func (m *Monitor) shouldLoadGuestChartInMemory(inMemoryKey string, metricTypes []string, duration time.Duration) bool {
|
||
if m == nil || m.metricsHistory == nil {
|
||
return false
|
||
}
|
||
if m.metricsStore == nil || duration <= inMemoryChartThreshold {
|
||
return true
|
||
}
|
||
return m.metricsHistory.GuestMetricCoverageSpan(inMemoryKey, metricTypes, duration) >= chartCoverageRequiredSpan(duration)
|
||
}
|
||
|
||
func (m *Monitor) shouldLoadNodeChartInMemory(nodeID string, metricTypes []string, duration time.Duration) bool {
|
||
if m == nil || m.metricsHistory == nil {
|
||
return false
|
||
}
|
||
if m.metricsStore == nil || duration <= inMemoryChartThreshold {
|
||
return true
|
||
}
|
||
return m.metricsHistory.NodeMetricCoverageSpan(nodeID, metricTypes, duration) >= chartCoverageRequiredSpan(duration)
|
||
}
|
||
|
||
// GetGuestMetricsForChartBatch returns chart metrics for multiple guests of the
|
||
// same SQL resource type, using batch SQL queries instead of N individual
|
||
// queries. Results are keyed by SQLResourceID.
|
||
func (m *Monitor) GetGuestMetricsForChartBatch(
|
||
sqlResourceType string,
|
||
requests []GuestChartRequest,
|
||
duration time.Duration,
|
||
metricTypes ...string,
|
||
) map[string]map[string][]MetricPoint {
|
||
if m == nil || len(requests) == 0 {
|
||
return nil
|
||
}
|
||
|
||
requestedMetricTypes := normalizeChartMetricTypes(metricTypes)
|
||
|
||
if mock.IsMockEnabled() {
|
||
result := make(map[string]map[string][]MetricPoint, len(requests))
|
||
for _, req := range requests {
|
||
inMemory := m.GetGuestMetrics(req.InMemoryKey, duration)
|
||
result[req.SQLResourceID] = filterChartMetricMap(m.mockGuestMetricsForChart(
|
||
req.InMemoryKey,
|
||
sqlResourceType,
|
||
req.SQLResourceID,
|
||
duration,
|
||
inMemory,
|
||
), requestedMetricTypes)
|
||
}
|
||
return result
|
||
}
|
||
|
||
result := make(map[string]map[string][]MetricPoint, len(requests))
|
||
|
||
// Phase 1: Check in-memory for all guests and identify which need fallback.
|
||
var needFallback []string
|
||
for _, req := range requests {
|
||
inMemory := map[string][]MetricPoint{}
|
||
if m.shouldLoadGuestChartInMemory(req.InMemoryKey, requestedMetricTypes, duration) {
|
||
inMemory = filterChartMetricMap(m.GetGuestMetrics(req.InMemoryKey, duration), requestedMetricTypes)
|
||
}
|
||
result[req.SQLResourceID] = inMemory
|
||
if hasSufficientChartMapCoverage(inMemory, duration) {
|
||
continue
|
||
}
|
||
needFallback = append(needFallback, req.SQLResourceID)
|
||
}
|
||
|
||
if len(needFallback) == 0 {
|
||
return result
|
||
}
|
||
|
||
// Phase 2: Batch-query store, trying all resource type candidates and
|
||
// keeping the best coverage per ID. Ties on span are broken by total
|
||
// point count, matching the single-resource alias resolution logic.
|
||
storeResults := make(map[string]map[string][]MetricPoint)
|
||
if m.metricsStore != nil {
|
||
for _, candidate := range monitorStoreResourceTypeCandidates(sqlResourceType) {
|
||
batch := m.queryStoreBatchMetricMapWithGapFill(candidate, needFallback, duration, requestedMetricTypes)
|
||
for id, metricMap := range batch {
|
||
if existing, ok := storeResults[id]; ok {
|
||
newSpan := chartMapCoverageSpan(metricMap)
|
||
oldSpan := chartMapCoverageSpan(existing)
|
||
if newSpan < oldSpan || (newSpan == oldSpan && monitorMetricMapPointCount(metricMap) <= monitorMetricMapPointCount(existing)) {
|
||
continue
|
||
}
|
||
}
|
||
storeResults[id] = metricMap
|
||
}
|
||
}
|
||
}
|
||
nativeResults := m.nativeGuestMetricHistory(sqlResourceType, duration)
|
||
|
||
// Phase 3: Merge fallback data per metric type.
|
||
// result[id] is already owned (set from filterChartMetricMap output above);
|
||
// merge mutates in place.
|
||
for _, id := range needFallback {
|
||
best := result[id]
|
||
if storeData, ok := storeResults[id]; ok {
|
||
best = mergeGuestMetricHistory(best, filterChartMetricMap(storeData, requestedMetricTypes), duration)
|
||
}
|
||
if nativeData, ok := nativeResults[id]; ok {
|
||
best = mergeGuestMetricHistory(best, filterChartMetricMap(nativeData, requestedMetricTypes), duration)
|
||
}
|
||
result[id] = best
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
// GetNodeMetricsForChartBatch returns chart metrics for multiple nodes,
|
||
// using batch SQL queries instead of N×M individual queries (where M is the
|
||
// number of metric types). Results are keyed by nodeID, then by metric type.
|
||
func (m *Monitor) GetNodeMetricsForChartBatch(
|
||
nodeIDs []string,
|
||
metricTypes []string,
|
||
duration time.Duration,
|
||
) map[string]map[string][]MetricPoint {
|
||
if m == nil || len(nodeIDs) == 0 {
|
||
return nil
|
||
}
|
||
|
||
if mock.IsMockEnabled() {
|
||
result := make(map[string]map[string][]MetricPoint, len(nodeIDs))
|
||
for _, nodeID := range nodeIDs {
|
||
nodeResult := make(map[string][]MetricPoint, len(metricTypes))
|
||
for _, metricType := range metricTypes {
|
||
var inMemoryPoints []MetricPoint
|
||
if m != nil && m.metricsHistory != nil {
|
||
inMemoryPoints = m.metricsHistory.GetNodeMetrics(nodeID, metricType, duration)
|
||
}
|
||
nodeResult[metricType] = m.mockNodeMetricsForChart(nodeID, metricType, duration, inMemoryPoints)
|
||
}
|
||
result[nodeID] = nodeResult
|
||
}
|
||
return result
|
||
}
|
||
|
||
result := make(map[string]map[string][]MetricPoint, len(nodeIDs))
|
||
|
||
// Phase 1: Check in-memory for all nodes and identify which need store.
|
||
var needStore []string
|
||
for _, nid := range nodeIDs {
|
||
nodeResult := make(map[string][]MetricPoint, len(metricTypes))
|
||
|
||
allSufficient := true
|
||
loadInMemory := m.shouldLoadNodeChartInMemory(nid, metricTypes, duration)
|
||
for _, mt := range metricTypes {
|
||
if !loadInMemory {
|
||
nodeResult[mt] = nil
|
||
allSufficient = false
|
||
continue
|
||
}
|
||
|
||
points := m.metricsHistory.GetNodeMetrics(nid, mt, duration)
|
||
nodeResult[mt] = points
|
||
if m.metricsStore != nil && !hasSufficientChartSeriesCoverage(points, duration) {
|
||
allSufficient = false
|
||
}
|
||
}
|
||
result[nid] = nodeResult
|
||
if !allSufficient {
|
||
needStore = append(needStore, nid)
|
||
}
|
||
}
|
||
|
||
if len(needStore) == 0 {
|
||
return result
|
||
}
|
||
|
||
// Phase 2: Batch-query store for all nodes that need it.
|
||
batchResult := m.queryStoreBatchMetricMapWithGapFill("node", needStore, duration, metricTypes)
|
||
|
||
// Phase 3: Merge — per metric type, use store data if better coverage.
|
||
for _, nid := range needStore {
|
||
storeData, ok := batchResult[nid]
|
||
if !ok {
|
||
continue
|
||
}
|
||
for _, mt := range metricTypes {
|
||
storePoints, found := storeData[mt]
|
||
if !found {
|
||
continue
|
||
}
|
||
inMemory := result[nid][mt]
|
||
if chartSeriesCoverageSpan(storePoints) <= chartSeriesCoverageSpan(inMemory) {
|
||
continue
|
||
}
|
||
result[nid][mt] = storePoints
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
// GetStorageMetricsForChartBatch returns chart metrics for multiple storage
|
||
// pools, using batch SQL queries instead of N individual queries.
|
||
// Results are keyed by storageID.
|
||
func (m *Monitor) GetStorageMetricsForChartBatch(
|
||
storageIDs []string,
|
||
duration time.Duration,
|
||
) map[string]map[string][]MetricPoint {
|
||
if m == nil || len(storageIDs) == 0 {
|
||
return nil
|
||
}
|
||
|
||
result := make(map[string]map[string][]MetricPoint, len(storageIDs))
|
||
|
||
if mock.IsMockEnabled() {
|
||
// One read of the estate view for every pool that needs a total.
|
||
readState := sync.OnceValue(m.GetUnifiedReadStateOrSnapshot)
|
||
for _, sid := range storageIDs {
|
||
inMemory := map[string][]MetricPoint{}
|
||
if m.metricsHistory != nil {
|
||
inMemory = m.metricsHistory.GetAllStorageMetrics(sid, duration)
|
||
}
|
||
result[sid] = m.mockStorageMetricsForChartCached(sid, duration, inMemory, readState)
|
||
}
|
||
return result
|
||
}
|
||
|
||
// Phase 1: Check in-memory for all storage pools.
|
||
var needStore []string
|
||
for _, sid := range storageIDs {
|
||
inMemory := map[string][]MetricPoint{}
|
||
if m.metricsHistory != nil {
|
||
inMemory = m.metricsHistory.GetAllStorageMetrics(sid, duration)
|
||
}
|
||
if m.metricsStore == nil {
|
||
result[sid] = inMemory
|
||
continue
|
||
}
|
||
if hasSufficientChartMapCoverageForMetrics(inMemory, duration, storageChartMetricTypes) {
|
||
result[sid] = inMemory
|
||
continue
|
||
}
|
||
needStore = append(needStore, sid)
|
||
result[sid] = inMemory // keep as fallback
|
||
}
|
||
|
||
if len(needStore) == 0 {
|
||
return result
|
||
}
|
||
|
||
// Phase 2: Batch-query store.
|
||
batchResult := m.queryStoreBatchMetricMapWithGapFill("storage", needStore, duration, storageChartMetricTypes)
|
||
|
||
// Phase 3: Merge — use store data if better coverage.
|
||
for _, id := range needStore {
|
||
storeData, ok := batchResult[id]
|
||
if !ok {
|
||
continue
|
||
}
|
||
result[id] = mergeMetricHistory(result[id], storeData, duration)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func (m *Monitor) nativeGuestMetricHistory(resourceType string, duration time.Duration) map[string]map[string][]MetricPoint {
|
||
providers := m.supplementalProviderSnapshot()
|
||
if len(providers) == 0 {
|
||
return nil
|
||
}
|
||
|
||
orgID := "default"
|
||
if m != nil {
|
||
if trimmed := strings.TrimSpace(m.GetOrgID()); trimmed != "" {
|
||
orgID = trimmed
|
||
}
|
||
}
|
||
|
||
history := make(map[string]map[string][]MetricPoint)
|
||
for _, provider := range providers {
|
||
historyProvider, ok := provider.(MonitorGuestMetricHistoryProvider)
|
||
if !ok {
|
||
continue
|
||
}
|
||
nativeHistory := historyProvider.GuestMetricHistory(m, orgID, resourceType, duration)
|
||
for resourceID, metricMap := range nativeHistory {
|
||
if strings.TrimSpace(resourceID) == "" || len(metricMap) == 0 {
|
||
continue
|
||
}
|
||
existing := history[resourceID]
|
||
history[resourceID] = mergeGuestMetricHistory(existing, metricMap, duration)
|
||
}
|
||
}
|
||
if len(history) == 0 {
|
||
return nil
|
||
}
|
||
return history
|
||
}
|
||
|
||
func chartGapFillLookbackWindow(duration time.Duration) time.Duration {
|
||
lookback := duration * chartGapFillLookbackMultiplier
|
||
if lookback < chartGapFillLookbackMin {
|
||
lookback = chartGapFillLookbackMin
|
||
}
|
||
if lookback > chartGapFillLookbackMax {
|
||
lookback = chartGapFillLookbackMax
|
||
}
|
||
return lookback
|
||
}
|
||
|
||
func convertSeriesAndDownsample(points []metrics.MetricPoint, target int) []MetricPoint {
|
||
converted := make([]MetricPoint, len(points))
|
||
for i, p := range points {
|
||
converted[i] = MetricPoint{Value: p.Value, Timestamp: p.Timestamp}
|
||
}
|
||
return lttb(converted, target)
|
||
}
|
||
|
||
func (m *Monitor) queryStoreMetricMapWithGapFill(resourceType, resourceID string, duration time.Duration) (map[string][]MetricPoint, bool) {
|
||
if m == nil || m.metricsStore == nil {
|
||
return nil, false
|
||
}
|
||
|
||
end := time.Now()
|
||
start := end.Add(-duration)
|
||
|
||
query := func(from time.Time) (map[string][]MetricPoint, bool) {
|
||
sqlResult, err := m.metricsStore.QueryAll(resourceType, resourceID, from, end, 0)
|
||
if err != nil || len(sqlResult) == 0 {
|
||
return nil, false
|
||
}
|
||
return convertAndDownsample(sqlResult, chartDownsampleTarget), true
|
||
}
|
||
|
||
if result, ok := query(start); ok {
|
||
return result, true
|
||
}
|
||
|
||
return query(end.Add(-chartGapFillLookbackWindow(duration)))
|
||
}
|
||
|
||
func (m *Monitor) queryStoreMetricMapWithGapFillAliases(resourceType, resourceID string, duration time.Duration) (map[string][]MetricPoint, bool) {
|
||
var (
|
||
best map[string][]MetricPoint
|
||
bestSpan time.Duration
|
||
bestPoints int
|
||
found bool
|
||
)
|
||
|
||
for _, candidate := range monitorStoreResourceTypeCandidates(resourceType) {
|
||
result, ok := m.queryStoreMetricMapWithGapFill(candidate, resourceID, duration)
|
||
if !ok {
|
||
continue
|
||
}
|
||
|
||
span := chartMapCoverageSpan(result)
|
||
points := monitorMetricMapPointCount(result)
|
||
if !found || span > bestSpan || (span == bestSpan && points > bestPoints) {
|
||
best = result
|
||
bestSpan = span
|
||
bestPoints = points
|
||
found = true
|
||
}
|
||
}
|
||
|
||
return best, found
|
||
}
|
||
|
||
func (m *Monitor) queryStoreMetricSeriesWithGapFill(resourceType, resourceID, metricType string, duration time.Duration) ([]MetricPoint, bool) {
|
||
if m == nil || m.metricsStore == nil {
|
||
return nil, false
|
||
}
|
||
|
||
end := time.Now()
|
||
start := end.Add(-duration)
|
||
|
||
query := func(from time.Time) ([]MetricPoint, bool) {
|
||
sqlPoints, err := m.metricsStore.Query(resourceType, resourceID, metricType, from, end, 0)
|
||
if err != nil || len(sqlPoints) == 0 {
|
||
return nil, false
|
||
}
|
||
return convertSeriesAndDownsample(sqlPoints, chartDownsampleTarget), true
|
||
}
|
||
|
||
if points, ok := query(start); ok {
|
||
return points, true
|
||
}
|
||
|
||
return query(end.Add(-chartGapFillLookbackWindow(duration)))
|
||
}
|
||
|
||
// convertAndDownsample converts pkg/metrics.MetricPoint slices to
|
||
// internal/monitoring.MetricPoint slices and applies LTTB downsampling.
|
||
func convertAndDownsample(sqlResult map[string][]metrics.MetricPoint, target int) map[string][]MetricPoint {
|
||
result := make(map[string][]MetricPoint, len(sqlResult))
|
||
for metric, points := range sqlResult {
|
||
converted := make([]MetricPoint, len(points))
|
||
for i, p := range points {
|
||
converted[i] = MetricPoint{Value: p.Value, Timestamp: p.Timestamp}
|
||
}
|
||
result[metric] = lttb(converted, target)
|
||
}
|
||
return result
|
||
}
|
||
|
||
func cloneMetricSeries(points []MetricPoint) []MetricPoint {
|
||
if len(points) == 0 {
|
||
return nil
|
||
}
|
||
cloned := make([]MetricPoint, len(points))
|
||
copy(cloned, points)
|
||
return cloned
|
||
}
|
||
|
||
func cloneMetricPointMap(metrics map[string][]MetricPoint) map[string][]MetricPoint {
|
||
if len(metrics) == 0 {
|
||
return map[string][]MetricPoint{}
|
||
}
|
||
cloned := make(map[string][]MetricPoint, len(metrics))
|
||
for metricType, points := range metrics {
|
||
cloned[metricType] = cloneMetricSeries(points)
|
||
}
|
||
return cloned
|
||
}
|
||
|
||
func mockChartPointTarget(duration time.Duration) int {
|
||
switch {
|
||
case duration <= time.Hour:
|
||
return mockChartPointTargetShortRange
|
||
case duration <= 6*time.Hour:
|
||
return mockChartPointTargetMedium
|
||
case duration <= 24*time.Hour:
|
||
return mockChartPointTargetLong
|
||
default:
|
||
return mockChartPointTargetExtended
|
||
}
|
||
}
|
||
|
||
// downsampleMetricSeriesForMockChart thins a mock series onto the shared mock
|
||
// chart grid (see sampleMetricSeriesOnMockChartGrid). It does not use LTTB:
|
||
// LTTB keeps the points whose values stand out, so each metric of a resource
|
||
// kept its own timestamps, and drawer history panels, which break a line
|
||
// where another series in the panel has a reading this one lacks, drew mock
|
||
// charts as scattered dots.
|
||
func downsampleMetricSeriesForMockChart(points []MetricPoint, duration time.Duration) []MetricPoint {
|
||
return sampleMetricSeriesOnMockChartGrid(points, mockChartGridInterval(duration))
|
||
}
|
||
|
||
func downsampleMetricMapForMockChart(metrics map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint {
|
||
if len(metrics) == 0 {
|
||
return map[string][]MetricPoint{}
|
||
}
|
||
downsampled := make(map[string][]MetricPoint, len(metrics))
|
||
for metricType, points := range metrics {
|
||
downsampled[metricType] = downsampleMetricSeriesForMockChart(points, duration)
|
||
}
|
||
return downsampled
|
||
}
|
||
|
||
func normalizeMockChartMetricTypes(metricTypes []string) string {
|
||
if len(metricTypes) == 0 {
|
||
return ""
|
||
}
|
||
normalized := make([]string, 0, len(metricTypes))
|
||
for _, metricType := range metricTypes {
|
||
trimmed := strings.TrimSpace(metricType)
|
||
if trimmed == "" {
|
||
continue
|
||
}
|
||
normalized = append(normalized, trimmed)
|
||
}
|
||
if len(normalized) == 0 {
|
||
return ""
|
||
}
|
||
sort.Strings(normalized)
|
||
return strings.Join(normalized, ",")
|
||
}
|
||
|
||
func (m *Monitor) readMockChartMetricMapCache(
|
||
key mockChartMetricMapCacheKey,
|
||
) (map[string][]MetricPoint, bool) {
|
||
if m == nil {
|
||
return nil, false
|
||
}
|
||
|
||
m.mockChartCacheMu.RLock()
|
||
defer m.mockChartCacheMu.RUnlock()
|
||
|
||
if m.mockChartMapCache == nil {
|
||
return nil, false
|
||
}
|
||
cached, ok := m.mockChartMapCache[key]
|
||
if !ok {
|
||
return nil, false
|
||
}
|
||
return cloneMetricPointMap(cached), true
|
||
}
|
||
|
||
func (m *Monitor) writeMockChartMetricMapCache(
|
||
key mockChartMetricMapCacheKey,
|
||
metrics map[string][]MetricPoint,
|
||
) map[string][]MetricPoint {
|
||
cloned := cloneMetricPointMap(metrics)
|
||
if m == nil {
|
||
return cloned
|
||
}
|
||
|
||
m.mockChartCacheMu.Lock()
|
||
defer m.mockChartCacheMu.Unlock()
|
||
|
||
if m.mockChartMapCache == nil {
|
||
m.mockChartMapCache = make(map[mockChartMetricMapCacheKey]map[string][]MetricPoint)
|
||
}
|
||
m.mockChartMapCache[key] = cloneMetricPointMap(cloned)
|
||
return cloned
|
||
}
|
||
|
||
func (m *Monitor) invalidateMockChartCaches() {
|
||
if m == nil {
|
||
return
|
||
}
|
||
|
||
m.mockChartCacheMu.Lock()
|
||
defer m.mockChartCacheMu.Unlock()
|
||
|
||
if m.mockChartMapCache != nil {
|
||
clear(m.mockChartMapCache)
|
||
}
|
||
}
|
||
|
||
type mockWorkloadChartPrewarmRequests struct {
|
||
vms []GuestChartRequest
|
||
containers []GuestChartRequest
|
||
pods []GuestChartRequest
|
||
dockerContainers []GuestChartRequest
|
||
}
|
||
|
||
func appendMockGuestChartRequest(
|
||
requests []GuestChartRequest,
|
||
seen map[string]struct{},
|
||
inMemoryKey string,
|
||
sqlResourceID string,
|
||
) []GuestChartRequest {
|
||
inMemoryKey = strings.TrimSpace(inMemoryKey)
|
||
sqlResourceID = strings.TrimSpace(sqlResourceID)
|
||
if inMemoryKey == "" || sqlResourceID == "" {
|
||
return requests
|
||
}
|
||
|
||
identity := inMemoryKey + "\x00" + sqlResourceID
|
||
if _, ok := seen[identity]; ok {
|
||
return requests
|
||
}
|
||
seen[identity] = struct{}{}
|
||
|
||
return append(requests, GuestChartRequest{
|
||
InMemoryKey: inMemoryKey,
|
||
SQLResourceID: sqlResourceID,
|
||
})
|
||
}
|
||
|
||
func mockMetricIDFromTargetOrSource(target *unifiedresources.MetricsTarget, sourceID string) string {
|
||
if target != nil {
|
||
if metricID := strings.TrimSpace(target.ResourceID); metricID != "" {
|
||
return metricID
|
||
}
|
||
}
|
||
return strings.TrimSpace(sourceID)
|
||
}
|
||
|
||
func mockKubernetesPodChartMetricID(pod *unifiedresources.PodView) string {
|
||
if pod == nil {
|
||
return ""
|
||
}
|
||
|
||
clusterKey := strings.TrimSpace(pod.ClusterID())
|
||
if clusterKey == "" {
|
||
clusterKey = strings.TrimSpace(pod.ClusterName())
|
||
}
|
||
podKey := strings.TrimSpace(pod.PodUID())
|
||
if podKey == "" {
|
||
namespace := strings.TrimSpace(pod.Namespace())
|
||
name := strings.TrimSpace(pod.Name())
|
||
if namespace != "" || name != "" {
|
||
podKey = namespace + "/" + name
|
||
}
|
||
}
|
||
if clusterKey == "" || podKey == "" {
|
||
return ""
|
||
}
|
||
return "k8s:" + clusterKey + ":pod:" + podKey
|
||
}
|
||
|
||
func mockWorkloadChartRequestsForReadState(
|
||
readState unifiedresources.ReadState,
|
||
) mockWorkloadChartPrewarmRequests {
|
||
if readState == nil {
|
||
return mockWorkloadChartPrewarmRequests{}
|
||
}
|
||
|
||
requests := mockWorkloadChartPrewarmRequests{
|
||
vms: make([]GuestChartRequest, 0, len(readState.VMs())),
|
||
containers: make([]GuestChartRequest, 0, len(readState.Containers())),
|
||
pods: make([]GuestChartRequest, 0, len(readState.Pods())),
|
||
dockerContainers: make([]GuestChartRequest, 0, len(readState.DockerContainers())),
|
||
}
|
||
seenVMs := make(map[string]struct{}, len(readState.VMs()))
|
||
seenContainers := make(map[string]struct{}, len(readState.Containers()))
|
||
seenPods := make(map[string]struct{}, len(readState.Pods()))
|
||
seenDockerContainers := make(map[string]struct{}, len(readState.DockerContainers()))
|
||
|
||
for _, vm := range readState.VMs() {
|
||
if vm == nil {
|
||
continue
|
||
}
|
||
metricID := mockMetricIDFromTargetOrSource(vm.MetricsTarget(), vm.SourceID())
|
||
requests.vms = appendMockGuestChartRequest(requests.vms, seenVMs, metricID, metricID)
|
||
}
|
||
for _, container := range readState.Containers() {
|
||
if container == nil {
|
||
continue
|
||
}
|
||
metricID := mockMetricIDFromTargetOrSource(container.MetricsTarget(), container.SourceID())
|
||
requests.containers = appendMockGuestChartRequest(
|
||
requests.containers,
|
||
seenContainers,
|
||
metricID,
|
||
metricID,
|
||
)
|
||
}
|
||
for _, pod := range readState.Pods() {
|
||
metricID := mockKubernetesPodChartMetricID(pod)
|
||
requests.pods = appendMockGuestChartRequest(requests.pods, seenPods, metricID, metricID)
|
||
}
|
||
for _, container := range readState.DockerContainers() {
|
||
if container == nil {
|
||
continue
|
||
}
|
||
metricID := mockMetricIDFromTargetOrSource(container.MetricsTarget(), container.ContainerID())
|
||
requests.dockerContainers = appendMockGuestChartRequest(
|
||
requests.dockerContainers,
|
||
seenDockerContainers,
|
||
"docker:"+metricID,
|
||
metricID,
|
||
)
|
||
}
|
||
|
||
return requests
|
||
}
|
||
|
||
func boundMockWorkloadChartPrewarmRequests(
|
||
requests mockWorkloadChartPrewarmRequests,
|
||
) mockWorkloadChartPrewarmRequests {
|
||
bound := func(values []GuestChartRequest) []GuestChartRequest {
|
||
if len(values) <= mockDashboardWorkloadPrewarmLimitPerType {
|
||
return values
|
||
}
|
||
return values[:mockDashboardWorkloadPrewarmLimitPerType]
|
||
}
|
||
requests.vms = bound(requests.vms)
|
||
requests.containers = bound(requests.containers)
|
||
requests.pods = bound(requests.pods)
|
||
requests.dockerContainers = bound(requests.dockerContainers)
|
||
return requests
|
||
}
|
||
|
||
func (m *Monitor) prewarmMockWorkloadChartCaches(duration time.Duration) {
|
||
if m == nil || duration <= 0 {
|
||
return
|
||
}
|
||
|
||
requests := boundMockWorkloadChartPrewarmRequests(
|
||
mockWorkloadChartRequestsForReadState(m.GetUnifiedStructureReadState()),
|
||
)
|
||
if len(requests.vms) > 0 {
|
||
_ = m.GetGuestMetricsForChartBatch("vm", requests.vms, duration)
|
||
}
|
||
if len(requests.containers) > 0 {
|
||
_ = m.GetGuestMetricsForChartBatch("container", requests.containers, duration)
|
||
}
|
||
if len(requests.pods) > 0 {
|
||
_ = m.GetGuestMetricsForChartBatch("k8s", requests.pods, duration)
|
||
}
|
||
if len(requests.dockerContainers) > 0 {
|
||
_ = m.GetGuestMetricsForChartBatch("dockerContainer", requests.dockerContainers, duration)
|
||
}
|
||
}
|
||
|
||
func (m *Monitor) prewarmMockDashboardChartCaches() {
|
||
if m == nil || !mock.IsMockEnabled() {
|
||
return
|
||
}
|
||
|
||
m.prewarmMockWorkloadChartCaches(mockDashboardWorkloadPrewarmDuration)
|
||
}
|
||
|
||
func (m *Monitor) mockGuestMetricsForChart(
|
||
inMemoryKey,
|
||
sqlResourceType,
|
||
sqlResourceID string,
|
||
duration time.Duration,
|
||
inMemoryResult map[string][]MetricPoint,
|
||
) map[string][]MetricPoint {
|
||
key := mockChartMetricMapCacheKey{
|
||
kind: "guest",
|
||
resourceType: strings.TrimSpace(sqlResourceType),
|
||
resourceID: strings.TrimSpace(sqlResourceID),
|
||
aux: strings.TrimSpace(inMemoryKey),
|
||
duration: duration,
|
||
}
|
||
if cached, ok := m.readMockChartMetricMapCache(key); ok {
|
||
return cached
|
||
}
|
||
|
||
// Resolve mock coverage per series, not per resource. A resource can have a
|
||
// full window of CPU/memory/disk while I/O or thermal series are absent; a
|
||
// resource-level coverage check would then return the partial map unchanged
|
||
// and leave whole drawer groups stuck in "Collecting history".
|
||
synthetic := mockGuestMetricsForChart(sqlResourceType, sqlResourceID, duration)
|
||
computed := make(map[string][]MetricPoint, len(inMemoryResult)+len(synthetic))
|
||
for metricType, points := range inMemoryResult {
|
||
if hasSufficientChartSeriesCoverage(points, duration) {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(points, duration)
|
||
continue
|
||
}
|
||
if fallback := synthetic[metricType]; len(fallback) > 0 {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(fallback, duration)
|
||
continue
|
||
}
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(points, duration)
|
||
}
|
||
for metricType, points := range synthetic {
|
||
if len(computed[metricType]) > 0 {
|
||
continue
|
||
}
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(points, duration)
|
||
}
|
||
return m.writeMockChartMetricMapCache(key, computed)
|
||
}
|
||
|
||
func (m *Monitor) mockNodeMetricsForChart(
|
||
nodeID string,
|
||
metricType string,
|
||
duration time.Duration,
|
||
inMemoryPoints []MetricPoint,
|
||
) []MetricPoint {
|
||
cacheKey := mockChartMetricMapCacheKey{
|
||
kind: "node",
|
||
resourceType: "node",
|
||
resourceID: strings.TrimSpace(nodeID),
|
||
aux: normalizeMockChartMetricTypes([]string{metricType}),
|
||
duration: duration,
|
||
}
|
||
if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok {
|
||
return cloneMetricSeries(cached[metricType])
|
||
}
|
||
|
||
computed := map[string][]MetricPoint{}
|
||
if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(inMemoryPoints, duration)
|
||
} else {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(
|
||
mockNodeMetricsForChart(nodeID, []string{metricType}, duration)[metricType],
|
||
duration,
|
||
)
|
||
}
|
||
return cloneMetricSeries(m.writeMockChartMetricMapCache(cacheKey, computed)[metricType])
|
||
}
|
||
|
||
func (m *Monitor) mockDiskMetricsForChart(
|
||
resourceID string,
|
||
metricType string,
|
||
duration time.Duration,
|
||
inMemoryPoints []MetricPoint,
|
||
) []MetricPoint {
|
||
cacheKey := mockChartMetricMapCacheKey{
|
||
kind: "disk",
|
||
resourceType: "disk",
|
||
resourceID: strings.TrimSpace(resourceID),
|
||
aux: normalizeMockChartMetricTypes([]string{metricType}),
|
||
duration: duration,
|
||
}
|
||
if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok {
|
||
return cloneMetricSeries(cached[metricType])
|
||
}
|
||
|
||
computed := map[string][]MetricPoint{}
|
||
if hasSufficientChartSeriesCoverage(inMemoryPoints, duration) {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(inMemoryPoints, duration)
|
||
} else {
|
||
computed[metricType] = downsampleMetricSeriesForMockChart(
|
||
mockDiskMetricsForChart(resourceID, []string{metricType}, duration)[metricType],
|
||
duration,
|
||
)
|
||
}
|
||
return cloneMetricSeries(m.writeMockChartMetricMapCache(cacheKey, computed)[metricType])
|
||
}
|
||
|
||
func (m *Monitor) mockStorageMetricsForChartCached(
|
||
storageID string,
|
||
duration time.Duration,
|
||
inMemoryResult map[string][]MetricPoint,
|
||
readState func() unifiedresources.ReadState,
|
||
) map[string][]MetricPoint {
|
||
cacheKey := mockChartMetricMapCacheKey{
|
||
kind: "storage",
|
||
resourceType: "storage",
|
||
resourceID: strings.TrimSpace(storageID),
|
||
duration: duration,
|
||
}
|
||
if cached, ok := m.readMockChartMetricMapCache(cacheKey); ok {
|
||
return cached
|
||
}
|
||
|
||
var computed map[string][]MetricPoint
|
||
if hasSufficientChartMapCoverageForMetrics(inMemoryResult, duration, storageChartMetricTypes) {
|
||
computed = downsampleMetricMapForMockChart(inMemoryResult, duration)
|
||
} else {
|
||
computed = downsampleMetricMapForMockChart(
|
||
m.mockStorageMetricsForChart(storageID, duration, inMemoryResult, readState),
|
||
duration,
|
||
)
|
||
}
|
||
return m.writeMockChartMetricMapCache(cacheKey, computed)
|
||
}
|
||
|
||
func mergeGuestMetricHistory(base, candidate map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint {
|
||
return mergeMetricHistory(base, candidate, duration)
|
||
}
|
||
|
||
// mergeMetricHistory selects the better series per metric between base and
|
||
// candidate and returns the merged map.
|
||
//
|
||
// Ownership: caller transfers ownership of both maps (and their slice values)
|
||
// to this function. The returned map reuses base's storage when present and
|
||
// may share slices with candidate. Callers must not retain references to
|
||
// base or candidate after this call, and must not pass shared/borrowed maps.
|
||
func mergeMetricHistory(base, candidate map[string][]MetricPoint, duration time.Duration) map[string][]MetricPoint {
|
||
if len(candidate) == 0 {
|
||
return base
|
||
}
|
||
if base == nil {
|
||
base = make(map[string][]MetricPoint, len(candidate))
|
||
}
|
||
for metricType, candidateSeries := range candidate {
|
||
if !shouldPreferMetricSeries(base[metricType], candidateSeries, duration) {
|
||
continue
|
||
}
|
||
base[metricType] = candidateSeries
|
||
}
|
||
return base
|
||
}
|
||
|
||
func shouldPreferMetricSeries(current, candidate []MetricPoint, duration time.Duration) bool {
|
||
if len(candidate) == 0 {
|
||
return false
|
||
}
|
||
if len(current) == 0 {
|
||
return true
|
||
}
|
||
|
||
candidateSpan := chartSeriesCoverageSpan(candidate)
|
||
currentSpan := chartSeriesCoverageSpan(current)
|
||
currentSufficient := hasSufficientChartSeriesCoverage(current, duration)
|
||
candidateSufficient := hasSufficientChartSeriesCoverage(candidate, duration)
|
||
switch {
|
||
case currentSufficient && !candidateSufficient:
|
||
return false
|
||
case !currentSufficient && candidateSufficient:
|
||
return true
|
||
}
|
||
|
||
if candidateSpan > currentSpan {
|
||
return true
|
||
}
|
||
if candidateSpan < currentSpan {
|
||
return false
|
||
}
|
||
return len(candidate) > len(current)
|
||
}
|
||
|
||
func chartCoverageRequiredSpan(duration time.Duration) time.Duration {
|
||
if duration <= 0 {
|
||
return 0
|
||
}
|
||
|
||
required := (duration * shortRangeCoverageRatioNum) / shortRangeCoverageRatioDen
|
||
slack := duration / 10
|
||
if slack > shortRangeCoverageMaxSlack {
|
||
slack = shortRangeCoverageMaxSlack
|
||
}
|
||
if required <= slack {
|
||
return 0
|
||
}
|
||
return required - slack
|
||
}
|
||
|
||
func chartSeriesCoverageSpan(points []MetricPoint) time.Duration {
|
||
if len(points) < 2 {
|
||
return 0
|
||
}
|
||
|
||
oldest := points[0].Timestamp
|
||
newest := points[0].Timestamp
|
||
for i := 1; i < len(points); i++ {
|
||
ts := points[i].Timestamp
|
||
if ts.Before(oldest) {
|
||
oldest = ts
|
||
}
|
||
if ts.After(newest) {
|
||
newest = ts
|
||
}
|
||
}
|
||
if !newest.After(oldest) {
|
||
return 0
|
||
}
|
||
return newest.Sub(oldest)
|
||
}
|
||
|
||
func chartMapCoverageSpan(metrics map[string][]MetricPoint) time.Duration {
|
||
var best time.Duration
|
||
for _, points := range metrics {
|
||
span := chartSeriesCoverageSpan(points)
|
||
if span > best {
|
||
best = span
|
||
}
|
||
}
|
||
return best
|
||
}
|
||
|
||
func monitorMetricMapPointCount(metrics map[string][]MetricPoint) int {
|
||
total := 0
|
||
for _, points := range metrics {
|
||
total += len(points)
|
||
}
|
||
return total
|
||
}
|
||
|
||
func monitorGuestMetricKeyCandidates(key string) []string {
|
||
trimmed := strings.TrimSpace(key)
|
||
if trimmed == "" {
|
||
return []string{""}
|
||
}
|
||
return []string{trimmed}
|
||
}
|
||
|
||
func monitorStoreResourceTypeCandidates(resourceType string) []string {
|
||
normalized := strings.TrimSpace(resourceType)
|
||
switch normalized {
|
||
case "agent":
|
||
return []string{"agent"}
|
||
case "dockerContainer":
|
||
return []string{"dockerContainer", "docker"}
|
||
case "docker":
|
||
return []string{"docker", "dockerContainer"}
|
||
default:
|
||
return []string{normalized}
|
||
}
|
||
}
|
||
|
||
func hasSufficientChartSeriesCoverage(points []MetricPoint, duration time.Duration) bool {
|
||
return chartSeriesCoverageSpan(points) >= chartCoverageRequiredSpan(duration)
|
||
}
|
||
|
||
func hasSufficientChartMapCoverage(metrics map[string][]MetricPoint, duration time.Duration) bool {
|
||
return chartMapCoverageSpan(metrics) >= chartCoverageRequiredSpan(duration)
|
||
}
|
||
|
||
func hasSufficientChartMapCoverageForMetrics(metrics map[string][]MetricPoint, duration time.Duration, metricTypes []string) bool {
|
||
if len(metricTypes) == 0 {
|
||
return hasSufficientChartMapCoverage(metrics, duration)
|
||
}
|
||
for _, metricType := range metricTypes {
|
||
if !hasSufficientChartSeriesCoverage(metrics[metricType], duration) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|