mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
pulse-triage[bot]
4c57c21bcc
Use the existing tenant-scoped native path when local disk history is shallow. Fill missing successful reporting series once from Netdata without retrying permission or transport failures, preserve independent partial series and original Celsius sample times, reject ambiguous or revoked identities, and stop fabricating sparkline history from a live reading. Cover the real authenticated client/provider/poller/History route and its failure boundaries. Change-source: pulse-maintainer
1199 lines
41 KiB
Go
1199 lines
41 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"math"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
|
)
|
|
|
|
type stubDiskTemperatureHistoryProvider struct {
|
|
history map[string][]MetricPoint
|
|
}
|
|
|
|
func (s *stubDiskTemperatureHistoryProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {
|
|
return nil
|
|
}
|
|
|
|
func (s *stubDiskTemperatureHistoryProvider) PhysicalDiskTemperatureHistory(*Monitor, string, time.Duration) map[string][]MetricPoint {
|
|
if len(s.history) == 0 {
|
|
return nil
|
|
}
|
|
result := make(map[string][]MetricPoint, len(s.history))
|
|
for resourceID, points := range s.history {
|
|
copied := make([]MetricPoint, len(points))
|
|
copy(copied, points)
|
|
result[resourceID] = copied
|
|
}
|
|
return result
|
|
}
|
|
|
|
type stubGuestMetricHistoryProvider struct {
|
|
history map[string]map[string][]MetricPoint
|
|
}
|
|
|
|
func (s *stubGuestMetricHistoryProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {
|
|
return nil
|
|
}
|
|
|
|
func (s *stubGuestMetricHistoryProvider) GuestMetricHistory(*Monitor, string, string, time.Duration) map[string]map[string][]MetricPoint {
|
|
if len(s.history) == 0 {
|
|
return nil
|
|
}
|
|
result := make(map[string]map[string][]MetricPoint, len(s.history))
|
|
for resourceID, metricMap := range s.history {
|
|
cloned := make(map[string][]MetricPoint, len(metricMap))
|
|
for metricType, points := range metricMap {
|
|
copied := make([]MetricPoint, len(points))
|
|
copy(copied, points)
|
|
cloned[metricType] = copied
|
|
}
|
|
result[resourceID] = cloned
|
|
}
|
|
return result
|
|
}
|
|
|
|
func newChartFallbackTestMonitor(t *testing.T) *Monitor {
|
|
t.Helper()
|
|
|
|
cfg := metrics.DefaultConfig(t.TempDir())
|
|
cfg.RetentionRaw = 90 * 24 * time.Hour
|
|
cfg.RetentionMinute = 90 * 24 * time.Hour
|
|
cfg.RetentionHourly = 90 * 24 * time.Hour
|
|
cfg.RetentionDaily = 90 * 24 * time.Hour
|
|
cfg.WriteBufferSize = 64
|
|
|
|
store, err := metrics.NewStore(cfg)
|
|
if err != nil {
|
|
t.Fatalf("failed to create metrics store: %v", err)
|
|
}
|
|
t.Cleanup(func() {
|
|
_ = store.Close()
|
|
})
|
|
|
|
return &Monitor{
|
|
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
|
|
metricsStore: store,
|
|
}
|
|
}
|
|
|
|
func writeRawMetricBatch(t *testing.T, store *metrics.Store, resourceType, resourceID, metricType string, points []MetricPoint) {
|
|
t.Helper()
|
|
|
|
records := make([]metrics.WriteMetric, len(points))
|
|
for i, point := range points {
|
|
records[i] = metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: point.Value,
|
|
Timestamp: point.Timestamp,
|
|
Tier: metrics.TierRaw,
|
|
}
|
|
}
|
|
store.WriteBatchSync(records)
|
|
}
|
|
|
|
func assertFollowsCanonicalMockSeries(t *testing.T, points []MetricPoint, resourceType, resourceID, metricType string) {
|
|
t.Helper()
|
|
if len(points) < 2 {
|
|
t.Fatalf("expected canonical mock series with at least 2 points, got %+v", points)
|
|
}
|
|
|
|
checkIndices := []int{0, len(points) / 2, len(points) - 1}
|
|
seen := map[int]struct{}{}
|
|
for _, idx := range checkIndices {
|
|
if idx < 0 || idx >= len(points) {
|
|
continue
|
|
}
|
|
if _, ok := seen[idx]; ok {
|
|
continue
|
|
}
|
|
seen[idx] = struct{}{}
|
|
|
|
want := mock.SampleMetric(resourceType, resourceID, metricType, points[idx].Timestamp)
|
|
if diff := math.Abs(points[idx].Value - want); diff > 1e-9 {
|
|
t.Fatalf("points[%d] = %+v, want canonical %.12f", idx, points[idx], want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_ShortRangeFallsBackToStoreWhenInMemoryCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
inMemoryKey := "agent:host-1"
|
|
monitor.metricsHistory.AddGuestMetric(inMemoryKey, "cpu", 15, now.Add(-4*time.Minute))
|
|
monitor.metricsHistory.AddGuestMetric(inMemoryKey, "cpu", 18, now.Add(-1*time.Minute))
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-58 * time.Minute), Value: 41},
|
|
{Timestamp: now.Add(-30 * time.Minute), Value: 43},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 46},
|
|
})
|
|
|
|
result := monitor.GetGuestMetricsForChart(inMemoryKey, "agent", "host-1", duration)
|
|
if got, wantMin := chartMapCoverageSpan(result), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected store-backed coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_LongRangeUsesInMemoryWhenCoverageSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 12 * time.Hour
|
|
|
|
inMemoryKey := "agent:host-1"
|
|
for _, offset := range []time.Duration{-11 * time.Hour, -6 * time.Hour, -1 * time.Minute} {
|
|
monitor.metricsHistory.AddGuestMetric(inMemoryKey, "cpu", 17, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 91},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 94},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 99},
|
|
})
|
|
|
|
result := monitor.GetGuestMetricsForChart(inMemoryKey, "agent", "host-1", duration)
|
|
cpuPoints := result["cpu"]
|
|
if len(cpuPoints) == 0 {
|
|
t.Fatal("expected in-memory guest chart points")
|
|
}
|
|
if cpuPoints[0].Value != 17 {
|
|
t.Fatalf("expected long-range in-memory guest history to win, got first value %.2f", cpuPoints[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChart_ShortRangeUsesInMemoryWhenCoverageSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
for _, offset := range []time.Duration{-59 * time.Minute, -30 * time.Minute, -1 * time.Minute} {
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "cpu", 11, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-59 * time.Minute), Value: 99},
|
|
{Timestamp: now.Add(-30 * time.Minute), Value: 99},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 99},
|
|
})
|
|
|
|
result := monitor.GetNodeMetricsForChart("node-1", "cpu", duration)
|
|
if len(result) == 0 {
|
|
t.Fatal("expected non-empty node chart series")
|
|
}
|
|
if result[0].Value != 11 {
|
|
t.Fatalf("expected in-memory series to be preferred, got first value %.2f", result[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChart_LongRangeUsesInMemoryWhenCoverageSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 12 * time.Hour
|
|
|
|
for _, offset := range []time.Duration{-11 * time.Hour, -6 * time.Hour, -1 * time.Minute} {
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "cpu", 13, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 88},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 92},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 96},
|
|
})
|
|
|
|
result := monitor.GetNodeMetricsForChart("node-1", "cpu", duration)
|
|
if len(result) == 0 {
|
|
t.Fatal("expected in-memory node chart series")
|
|
}
|
|
if result[0].Value != 13 {
|
|
t.Fatalf("expected long-range in-memory node history to win, got first value %.2f", result[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChart_ShortRangeFallsBackToStoreWhenInMemoryCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "cpu", 10, now.Add(-5*time.Minute))
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "cpu", 12, now.Add(-1*time.Minute))
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-57 * time.Minute), Value: 60},
|
|
{Timestamp: now.Add(-25 * time.Minute), Value: 62},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 65},
|
|
})
|
|
|
|
result := monitor.GetNodeMetricsForChart("node-1", "cpu", duration)
|
|
if got, wantMin := chartSeriesCoverageSpan(result), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected store-backed node coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_ShortRangeFallsBackToStoreWhenInMemoryCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "usage", 20, now.Add(-3*time.Minute))
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "usage", 21, now.Add(-1*time.Minute))
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "storage", "storage-1", "usage", []MetricPoint{
|
|
{Timestamp: now.Add(-56 * time.Minute), Value: 71},
|
|
{Timestamp: now.Add(-28 * time.Minute), Value: 73},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 75},
|
|
})
|
|
|
|
result := monitor.GetStorageMetricsForChart("storage-1", duration)
|
|
if got, wantMin := chartMapCoverageSpan(result), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected store-backed storage coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_LongRangeUsesInMemoryWhenCoverageSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 12 * time.Hour
|
|
|
|
for _, offset := range []time.Duration{-11 * time.Hour, -6 * time.Hour, -1 * time.Minute} {
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "usage", 26, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "storage", "storage-1", "usage", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 71},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 74},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 79},
|
|
})
|
|
|
|
result := monitor.GetStorageMetricsForChart("storage-1", duration)
|
|
usagePoints := result["usage"]
|
|
if len(usagePoints) == 0 {
|
|
t.Fatal("expected in-memory storage chart points")
|
|
}
|
|
if usagePoints[0].Value != 26 {
|
|
t.Fatalf("expected long-range in-memory storage history to win, got first value %.2f", usagePoints[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_MergesStorageMetricsPerMetricType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
for _, offset := range []time.Duration{-58 * time.Minute, -30 * time.Minute, -1 * time.Minute} {
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "usage", 42, now.Add(offset))
|
|
}
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "used", 420, now.Add(-2*time.Minute))
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "avail", 580, now.Add(-2*time.Minute))
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "storage", "storage-1", "used", []MetricPoint{
|
|
{Timestamp: now.Add(-58 * time.Minute), Value: 390},
|
|
{Timestamp: now.Add(-30 * time.Minute), Value: 405},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 420},
|
|
})
|
|
writeRawMetricBatch(t, monitor.metricsStore, "storage", "storage-1", "avail", []MetricPoint{
|
|
{Timestamp: now.Add(-58 * time.Minute), Value: 610},
|
|
{Timestamp: now.Add(-30 * time.Minute), Value: 595},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 580},
|
|
})
|
|
|
|
result := monitor.GetStorageMetricsForChart("storage-1", duration)
|
|
if len(result["usage"]) != 3 {
|
|
t.Fatalf("expected in-memory usage series to remain intact, got %d points", len(result["usage"]))
|
|
}
|
|
if got, wantMin := chartSeriesCoverageSpan(result["used"]), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected store-backed used coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
if got, wantMin := chartSeriesCoverageSpan(result["avail"]), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected store-backed avail coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
}
|
|
|
|
func TestGetPhysicalDiskTemperatureCharts_UsesUnifiedReadStateDiskViews(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(models.StateSnapshot{
|
|
PhysicalDisks: []models.PhysicalDisk{
|
|
{
|
|
ID: "disk-1",
|
|
Node: "node-a",
|
|
Instance: "lab-a",
|
|
DevPath: "/dev/nvme0n1",
|
|
Model: "Samsung PM9A3",
|
|
Serial: "SERIAL-DISK-1",
|
|
Temperature: 42,
|
|
LastChecked: time.Now().UTC(),
|
|
},
|
|
},
|
|
})
|
|
|
|
monitor := &Monitor{
|
|
state: models.NewState(),
|
|
resourceStore: unifiedresources.NewMonitorAdapter(registry),
|
|
}
|
|
|
|
charts := monitor.GetPhysicalDiskTemperatureCharts(30 * time.Minute)
|
|
entry, ok := charts["SERIAL-DISK-1"]
|
|
if !ok {
|
|
t.Fatalf("expected chart entry for canonical disk metric ID, got %#v", charts)
|
|
}
|
|
if entry.Name != "Samsung PM9A3" {
|
|
t.Fatalf("chart name = %q, want Samsung PM9A3", entry.Name)
|
|
}
|
|
if entry.Node != "node-a" {
|
|
t.Fatalf("chart node = %q, want node-a", entry.Node)
|
|
}
|
|
if entry.Instance != "lab-a" {
|
|
t.Fatalf("chart instance = %q, want lab-a", entry.Instance)
|
|
}
|
|
if len(entry.Temperature) != 0 {
|
|
t.Fatalf("a current temperature must not fabricate historical samples: %+v", entry.Temperature)
|
|
}
|
|
}
|
|
|
|
func TestGetPhysicalDiskTemperatureCharts_UsesInMemoryHistoryWithoutMetricsStore(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(models.StateSnapshot{
|
|
PhysicalDisks: []models.PhysicalDisk{
|
|
{
|
|
ID: "disk-1",
|
|
Node: "node-a",
|
|
Instance: "lab-a",
|
|
DevPath: "/dev/nvme0n1",
|
|
Model: "Samsung PM9A3",
|
|
Serial: "SERIAL-DISK-1",
|
|
Temperature: 42,
|
|
LastChecked: time.Now().UTC(),
|
|
},
|
|
},
|
|
})
|
|
|
|
monitor := &Monitor{
|
|
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
|
|
state: models.NewState(),
|
|
resourceStore: unifiedresources.NewMonitorAdapter(registry),
|
|
}
|
|
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
for _, sample := range []struct {
|
|
offset time.Duration
|
|
value float64
|
|
}{
|
|
{-58 * time.Minute, 39},
|
|
{-30 * time.Minute, 41},
|
|
{-1 * time.Minute, 42},
|
|
} {
|
|
monitor.metricsHistory.AddDiskMetric("SERIAL-DISK-1", "smart_temp", sample.value, now.Add(sample.offset))
|
|
}
|
|
|
|
charts := monitor.GetPhysicalDiskTemperatureCharts(time.Hour)
|
|
entry, ok := charts["SERIAL-DISK-1"]
|
|
if !ok {
|
|
t.Fatalf("expected chart entry for canonical disk metric ID, got %#v", charts)
|
|
}
|
|
if got, wantMin := chartSeriesCoverageSpan(entry.Temperature), 45*time.Minute; got < wantMin {
|
|
t.Fatalf("expected in-memory disk temperature coverage >= %s, got %s", wantMin, got)
|
|
}
|
|
}
|
|
|
|
func TestGetPhysicalDiskTemperatureCharts_UsesNativeHistoryWhenStoreCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(models.StateSnapshot{
|
|
PhysicalDisks: []models.PhysicalDisk{
|
|
{
|
|
ID: "disk-1",
|
|
Node: "truenas-main",
|
|
Instance: "",
|
|
DevPath: "/dev/sda",
|
|
Model: "Seagate Exos X18",
|
|
Serial: "SERIAL-DISK-1",
|
|
Temperature: 34,
|
|
LastChecked: time.Now().UTC(),
|
|
},
|
|
},
|
|
})
|
|
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
monitor := &Monitor{
|
|
state: models.NewState(),
|
|
resourceStore: unifiedresources.NewMonitorAdapter(registry),
|
|
supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
|
|
unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
|
|
history: map[string][]MetricPoint{
|
|
"SERIAL-DISK-1": {
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 29},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 31},
|
|
{Timestamp: now, Value: 34},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
charts := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
|
|
entry, ok := charts["SERIAL-DISK-1"]
|
|
if !ok {
|
|
t.Fatalf("expected chart entry for canonical disk metric id, got %#v", charts)
|
|
}
|
|
if len(entry.Temperature) != 3 {
|
|
t.Fatalf("expected native history points instead of padded fallback, got %+v", entry.Temperature)
|
|
}
|
|
if entry.Temperature[0].Value != 29 || entry.Temperature[len(entry.Temperature)-1].Value != 34 {
|
|
t.Fatalf("expected native history values to win, got %+v", entry.Temperature)
|
|
}
|
|
}
|
|
|
|
func TestGetPhysicalDiskTemperatureCharts_PrefersSeededMockHistoryInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
graph := mock.CurrentFixtureGraph()
|
|
if len(graph.PlatformFixtures.TrueNAS.Disks) == 0 {
|
|
t.Fatal("expected mock mode TrueNAS disks")
|
|
}
|
|
resourceID := trueNASDiskMetricsResourceID(graph.PlatformFixtures.TrueNAS.Disks[0])
|
|
if resourceID == "" {
|
|
t.Fatal("expected canonical TrueNAS disk metric id")
|
|
}
|
|
|
|
now := time.Now().UTC().Truncate(time.Minute)
|
|
monitor := &Monitor{
|
|
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
|
|
state: models.NewState(),
|
|
supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
|
|
unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
|
|
history: map[string][]MetricPoint{
|
|
resourceID: {
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 29},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 31},
|
|
{Timestamp: now, Value: 34},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
for _, sample := range []struct {
|
|
offset time.Duration
|
|
value float64
|
|
}{
|
|
{-58 * time.Minute, 32},
|
|
{-30 * time.Minute, 33},
|
|
{-1 * time.Minute, 34},
|
|
} {
|
|
monitor.metricsHistory.AddDiskMetric(resourceID, "smart_temp", sample.value, now.Add(sample.offset))
|
|
}
|
|
|
|
charts := monitor.GetPhysicalDiskTemperatureCharts(time.Hour)
|
|
entry, ok := charts[resourceID]
|
|
if !ok {
|
|
t.Fatalf("expected chart entry for canonical disk metric id, got %#v", charts)
|
|
}
|
|
if len(entry.Temperature) != 3 {
|
|
t.Fatalf("expected seeded in-memory history to win, got %+v", entry.Temperature)
|
|
}
|
|
if entry.Temperature[0].Value != 32 || entry.Temperature[len(entry.Temperature)-1].Value != 34 {
|
|
t.Fatalf("expected seeded mock history values to win, got %+v", entry.Temperature)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_UsesCanonicalMockSamplerInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "memory", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Minute), Value: 12},
|
|
{Timestamp: now.Add(-1 * time.Minute), Value: 91},
|
|
})
|
|
|
|
result := monitor.GetGuestMetricsForChart("vm-1", "vm", "vm-1", duration)
|
|
assertFollowsCanonicalMockSeries(t, result["memory"], "vm", "vm-1", "memory")
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_SynthesizesMockHostTemperature(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
for _, tc := range []struct {
|
|
name string
|
|
inMemoryKey string
|
|
sqlResourceType string
|
|
resourceID string
|
|
}{
|
|
{name: "agent", inMemoryKey: "agent:host-1", sqlResourceType: "agent", resourceID: "host-1"},
|
|
{name: "docker host", inMemoryKey: "dockerHost:runtime-1", sqlResourceType: "dockerHost", resourceID: "runtime-1"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
result := monitor.GetGuestMetricsForChart(tc.inMemoryKey, tc.sqlResourceType, tc.resourceID, time.Hour)
|
|
if len(result["temperature"]) < 2 {
|
|
t.Fatalf("expected drawable mock temperature history, got %+v", result["temperature"])
|
|
}
|
|
assertFollowsCanonicalMockSeries(t, result["temperature"], tc.sqlResourceType, tc.resourceID, "temperature")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_SupplementsPartialMockHistoryPerSeries(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
for _, sample := range []struct {
|
|
offset time.Duration
|
|
value float64
|
|
}{
|
|
{-58 * time.Minute, 12.5},
|
|
{-30 * time.Minute, 37.25},
|
|
{-1 * time.Minute, 81.75},
|
|
} {
|
|
monitor.metricsHistory.AddGuestMetric("docker:container-1", "cpu", sample.value, now.Add(sample.offset))
|
|
}
|
|
|
|
result := monitor.GetGuestMetricsForChart(
|
|
"docker:container-1",
|
|
"dockerContainer",
|
|
"container-1",
|
|
duration,
|
|
)
|
|
if len(result["cpu"]) != 3 || result["cpu"][0].Value != 12.5 || result["cpu"][2].Value != 81.75 {
|
|
t.Fatalf("expected seeded CPU history to win, got %+v", result["cpu"])
|
|
}
|
|
for _, metricType := range []string{"netin", "netout", "diskread", "diskwrite"} {
|
|
if len(result[metricType]) < 2 {
|
|
t.Fatalf("expected missing %s history to be synthesized, got %+v", metricType, result[metricType])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_PrefersSeededMockHistoryInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
for _, sample := range []struct {
|
|
offset time.Duration
|
|
value float64
|
|
}{
|
|
{-58 * time.Minute, 12.5},
|
|
{-30 * time.Minute, 37.25},
|
|
{-1 * time.Minute, 81.75},
|
|
} {
|
|
monitor.metricsHistory.AddGuestMetric("vm-1", "memory", sample.value, now.Add(sample.offset))
|
|
}
|
|
|
|
result := monitor.GetGuestMetricsForChart("vm-1", "vm", "vm-1", duration)
|
|
if len(result["memory"]) != 3 {
|
|
t.Fatalf("expected seeded in-memory guest history to win, got %+v", result["memory"])
|
|
}
|
|
if result["memory"][0].Value != 12.5 || result["memory"][len(result["memory"])-1].Value != 81.75 {
|
|
t.Fatalf("expected seeded guest values to win, got %+v", result["memory"])
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_PrefersSeededMockHistoryInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
for _, sample := range []struct {
|
|
offset time.Duration
|
|
value float64
|
|
}{
|
|
{-58 * time.Minute, 9.5},
|
|
{-30 * time.Minute, 26.0},
|
|
{-1 * time.Minute, 64.25},
|
|
} {
|
|
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", sample.value, now.Add(sample.offset))
|
|
}
|
|
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
|
|
InMemoryKey: "vm-1",
|
|
SQLResourceID: "vm-1",
|
|
}}, duration)
|
|
if len(result["vm-1"]["cpu"]) != 3 {
|
|
t.Fatalf("expected seeded in-memory guest batch history to win, got %+v", result["vm-1"]["cpu"])
|
|
}
|
|
if result["vm-1"]["cpu"][0].Value != 9.5 || result["vm-1"]["cpu"][len(result["vm-1"]["cpu"])-1].Value != 64.25 {
|
|
t.Fatalf("expected seeded guest batch values to win, got %+v", result["vm-1"]["cpu"])
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_DownsamplesDenseSeededMockHistory(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Minute)
|
|
duration := time.Hour
|
|
|
|
for i := 0; i < 61; i++ {
|
|
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", float64(i), now.Add(-time.Duration(60-i)*time.Minute))
|
|
}
|
|
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", []GuestChartRequest{{
|
|
InMemoryKey: "vm-1",
|
|
SQLResourceID: "vm-1",
|
|
}}, duration)
|
|
cpu := result["vm-1"]["cpu"]
|
|
if len(cpu) == 0 {
|
|
t.Fatal("expected downsampled seeded mock history")
|
|
}
|
|
if len(cpu) >= 61 {
|
|
t.Fatalf("expected dense seeded mock history to be downsampled, got %d points", len(cpu))
|
|
}
|
|
if len(cpu) > 20 {
|
|
t.Fatalf("expected 1h mock guest history to cap at 20 points, got %d", len(cpu))
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_UsesCanonicalMockSamplerInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
graph := mock.CurrentFixtureGraph()
|
|
if len(graph.State.Storage) == 0 {
|
|
t.Fatal("expected canonical mock storage fixtures")
|
|
}
|
|
storageID := graph.State.Storage[0].ID
|
|
if storageID == "" {
|
|
t.Fatal("expected canonical storage id")
|
|
}
|
|
|
|
monitor := &Monitor{
|
|
metricsHistory: NewMetricsHistory(1024, 24*time.Hour),
|
|
state: models.NewState(),
|
|
}
|
|
|
|
result := monitor.GetStorageMetricsForChart(storageID, time.Hour)
|
|
assertFollowsCanonicalMockSeries(t, result["usage"], "storage", storageID, "usage")
|
|
if len(result["used"]) == 0 || len(result["avail"]) == 0 || len(result["total"]) == 0 {
|
|
t.Fatalf("expected full storage capacity series, got %+v", result)
|
|
}
|
|
total := result["total"][len(result["total"])-1].Value
|
|
if total <= 0 {
|
|
t.Fatalf("expected positive storage total, got %+v", result["total"])
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_PrefersSeededMockHistoryInMockMode(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
type sample struct {
|
|
offset time.Duration
|
|
value float64
|
|
}
|
|
metricsByType := map[string][]sample{
|
|
"usage": {{-58 * time.Minute, 40}, {-30 * time.Minute, 41}, {-1 * time.Minute, 42}},
|
|
"used": {{-58 * time.Minute, 400}, {-30 * time.Minute, 410}, {-1 * time.Minute, 420}},
|
|
"avail": {{-58 * time.Minute, 600}, {-30 * time.Minute, 590}, {-1 * time.Minute, 580}},
|
|
"total": {{-58 * time.Minute, 1000}, {-30 * time.Minute, 1000}, {-1 * time.Minute, 1000}},
|
|
}
|
|
for metricType, samples := range metricsByType {
|
|
for _, sample := range samples {
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", metricType, sample.value, now.Add(sample.offset))
|
|
}
|
|
}
|
|
|
|
result := monitor.GetStorageMetricsForChart("storage-1", duration)
|
|
if len(result["usage"]) != 3 {
|
|
t.Fatalf("expected seeded in-memory storage history to win, got %+v", result["usage"])
|
|
}
|
|
if result["usage"][0].Value != 40 || result["usage"][len(result["usage"])-1].Value != 42 {
|
|
t.Fatalf("expected seeded storage values to win, got %+v", result["usage"])
|
|
}
|
|
if len(result["used"]) != 3 || len(result["avail"]) != 3 || len(result["total"]) != 3 {
|
|
t.Fatalf("expected complete seeded storage metrics, got %+v", result)
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChart_DownsamplesDenseSeededMockHistory(t *testing.T) {
|
|
previous := mock.IsMockEnabled()
|
|
mustSetMockEnabled(t, true)
|
|
defer mustSetMockEnabled(t, previous)
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Minute)
|
|
duration := time.Hour
|
|
|
|
for i := 0; i < 61; i++ {
|
|
ts := now.Add(-time.Duration(60-i) * time.Minute)
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "usage", float64(40+i%5), ts)
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "used", float64(400+i), ts)
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "avail", float64(600-i), ts)
|
|
monitor.metricsHistory.AddStorageMetric("storage-1", "total", 1000, ts)
|
|
}
|
|
|
|
result := monitor.GetStorageMetricsForChart("storage-1", duration)
|
|
if len(result["usage"]) == 0 {
|
|
t.Fatal("expected downsampled seeded storage history")
|
|
}
|
|
if len(result["usage"]) >= 61 {
|
|
t.Fatalf("expected dense seeded storage history to be downsampled, got %d points", len(result["usage"]))
|
|
}
|
|
if len(result["usage"]) > 20 {
|
|
t.Fatalf("expected 1h mock storage history to cap at 20 points, got %d", len(result["usage"]))
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_UsesGapFillLookbackWhenRequestedRangeIsEmpty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "agent", "host-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-4 * time.Hour), Value: 37},
|
|
{Timestamp: now.Add(-3*time.Hour - 30*time.Minute), Value: 39},
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 42},
|
|
})
|
|
|
|
result := monitor.GetGuestMetricsForChart("agent:host-1", "agent", "host-1", duration)
|
|
cpu := result["cpu"]
|
|
if len(cpu) == 0 {
|
|
t.Fatal("expected gap-fill fallback to return historical cpu points")
|
|
}
|
|
if cpu[len(cpu)-1].Timestamp.Before(now.Add(-4*time.Hour)) || cpu[len(cpu)-1].Timestamp.After(now.Add(-3*time.Hour+time.Minute)) {
|
|
t.Fatalf("expected latest fallback point to come from older store data window, got %s", cpu[len(cpu)-1].Timestamp)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChart_UsesNativeHistoryWhenStoreCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "agent", "truenas-main", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 41},
|
|
})
|
|
monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
|
|
unifiedresources.SourceTrueNAS: &stubGuestMetricHistoryProvider{
|
|
history: map[string]map[string][]MetricPoint{
|
|
"truenas-main": {
|
|
"cpu": {
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 20},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 28},
|
|
{Timestamp: now, Value: 34},
|
|
},
|
|
"memory": {
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 45},
|
|
{Timestamp: now, Value: 62},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
result := monitor.GetGuestMetricsForChart("agent:truenas-main", "agent", "truenas-main", duration)
|
|
if got := len(result["cpu"]); got != 3 {
|
|
t.Fatalf("expected native cpu history to win, got %+v", result["cpu"])
|
|
}
|
|
if got := result["cpu"][2].Value; got != 34 {
|
|
t.Fatalf("expected latest native cpu value 34, got %+v", result["cpu"])
|
|
}
|
|
if got := len(result["memory"]); got != 2 {
|
|
t.Fatalf("expected native memory history, got %+v", result["memory"])
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChart_UsesGapFillLookbackWhenRequestedRangeIsEmpty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-5 * time.Hour), Value: 21},
|
|
{Timestamp: now.Add(-4*time.Hour - 30*time.Minute), Value: 24},
|
|
{Timestamp: now.Add(-4 * time.Hour), Value: 26},
|
|
})
|
|
|
|
result := monitor.GetNodeMetricsForChart("node-1", "cpu", duration)
|
|
if len(result) == 0 {
|
|
t.Fatal("expected gap-fill fallback to return historical node cpu points")
|
|
}
|
|
if result[len(result)-1].Timestamp.Before(now.Add(-5*time.Hour)) || result[len(result)-1].Timestamp.After(now.Add(-4*time.Hour+time.Minute)) {
|
|
t.Fatalf("expected latest fallback node point to come from older store data window, got %s", result[len(result)-1].Timestamp)
|
|
}
|
|
}
|
|
|
|
// ---------- Batch method tests ----------
|
|
|
|
func TestGetGuestMetricsForChartBatch_BatchesStoreQueries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour // beyond inMemoryChartThreshold → forces store path
|
|
|
|
// Write store data for 3 guests.
|
|
for _, id := range []string{"vm-1", "vm-2", "vm-3"} {
|
|
writeRawMetricBatch(t, monitor.metricsStore, "vm", id, "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 10},
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 20},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 30},
|
|
})
|
|
writeRawMetricBatch(t, monitor.metricsStore, "vm", id, "memory", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 50},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 60},
|
|
})
|
|
}
|
|
|
|
requests := []GuestChartRequest{
|
|
{InMemoryKey: "vm-1", SQLResourceID: "vm-1"},
|
|
{InMemoryKey: "vm-2", SQLResourceID: "vm-2"},
|
|
{InMemoryKey: "vm-3", SQLResourceID: "vm-3"},
|
|
}
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", requests, duration)
|
|
|
|
// Verify all 3 guests have data.
|
|
for _, id := range []string{"vm-1", "vm-2", "vm-3"} {
|
|
metrics, ok := result[id]
|
|
if !ok {
|
|
t.Fatalf("missing result for %s", id)
|
|
}
|
|
if len(metrics["cpu"]) == 0 {
|
|
t.Errorf("%s: expected cpu points from store, got none", id)
|
|
}
|
|
if len(metrics["memory"]) == 0 {
|
|
t.Errorf("%s: expected memory points from store, got none", id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_PrefersInMemoryForShortRanges(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := time.Hour // short range
|
|
|
|
// Seed sufficient in-memory coverage.
|
|
for _, offset := range []time.Duration{-55 * time.Minute, -30 * time.Minute, -5 * time.Minute} {
|
|
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", 11, now.Add(offset))
|
|
}
|
|
|
|
// Write different values to store to distinguish sources.
|
|
writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-55 * time.Minute), Value: 99},
|
|
{Timestamp: now.Add(-30 * time.Minute), Value: 99},
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 99},
|
|
})
|
|
|
|
requests := []GuestChartRequest{{InMemoryKey: "vm-1", SQLResourceID: "vm-1"}}
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", requests, duration)
|
|
|
|
cpuPoints := result["vm-1"]["cpu"]
|
|
if len(cpuPoints) == 0 {
|
|
t.Fatal("expected non-empty cpu series")
|
|
}
|
|
// In-memory values are 11; store values are 99. Should prefer in-memory.
|
|
if cpuPoints[0].Value != 11 {
|
|
t.Fatalf("expected in-memory value 11, got %.2f (store leak)", cpuPoints[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_PrefersInMemoryWhenLongRangeCoverageIsSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 12 * time.Hour
|
|
|
|
for _, offset := range []time.Duration{-11 * time.Hour, -6 * time.Hour, -5 * time.Minute} {
|
|
monitor.metricsHistory.AddGuestMetric("vm-1", "cpu", 19, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 89},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 93},
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 98},
|
|
})
|
|
|
|
requests := []GuestChartRequest{{InMemoryKey: "vm-1", SQLResourceID: "vm-1"}}
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", requests, duration)
|
|
|
|
cpuPoints := result["vm-1"]["cpu"]
|
|
if len(cpuPoints) == 0 {
|
|
t.Fatal("expected non-empty cpu series")
|
|
}
|
|
if cpuPoints[0].Value != 19 {
|
|
t.Fatalf("expected long-range in-memory batch series to win, got %.2f", cpuPoints[0].Value)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_UsesNativeAgentHistoryWhenStoreCoverageShallow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "agent", "truenas-main", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 41},
|
|
})
|
|
monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
|
|
unifiedresources.SourceTrueNAS: &stubGuestMetricHistoryProvider{
|
|
history: map[string]map[string][]MetricPoint{
|
|
"truenas-main": {
|
|
"cpu": {
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 20},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 28},
|
|
{Timestamp: now, Value: 34},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
requests := []GuestChartRequest{{InMemoryKey: "agent:truenas-main", SQLResourceID: "truenas-main"}}
|
|
result := monitor.GetGuestMetricsForChartBatch("agent", requests, duration)
|
|
|
|
cpuPoints := result["truenas-main"]["cpu"]
|
|
if len(cpuPoints) != 3 {
|
|
t.Fatalf("expected native batch cpu history to win, got %+v", cpuPoints)
|
|
}
|
|
if cpuPoints[2].Value != 34 {
|
|
t.Fatalf("expected latest batch cpu value 34, got %+v", cpuPoints)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_EmptyRequests(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
result := monitor.GetGuestMetricsForChartBatch("vm", nil, time.Hour)
|
|
if result != nil {
|
|
t.Fatalf("expected nil for empty requests, got %v", result)
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_DifferentInMemoryAndSQLKeys(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
// Docker containers use "docker:<id>" as in-memory key but just "<id>" as SQL ID.
|
|
monitor.metricsHistory.AddGuestMetric("docker:dc1", "cpu", 5, now.Add(-10*time.Minute))
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "dockerContainer", "dc1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 40},
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 45},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 50},
|
|
})
|
|
|
|
requests := []GuestChartRequest{{InMemoryKey: "docker:dc1", SQLResourceID: "dc1"}}
|
|
result := monitor.GetGuestMetricsForChartBatch("dockerContainer", requests, duration)
|
|
|
|
cpuPoints := result["dc1"]["cpu"]
|
|
if len(cpuPoints) == 0 {
|
|
t.Fatal("expected store-backed cpu points for docker container")
|
|
}
|
|
}
|
|
|
|
func TestGetGuestMetricsForChartBatch_PicksBestAliasCandidate(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
// Write sparse data under "dockerContainer" (canonical) and better data
|
|
// under the legacy "docker" alias. The batch path should pick the alias
|
|
// with better coverage, matching single-resource alias resolution.
|
|
writeRawMetricBatch(t, monitor.metricsStore, "dockerContainer", "dc1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-10 * time.Minute), Value: 1},
|
|
})
|
|
writeRawMetricBatch(t, monitor.metricsStore, "docker", "dc1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 40},
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 45},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 50},
|
|
})
|
|
|
|
requests := []GuestChartRequest{{InMemoryKey: "docker:dc1", SQLResourceID: "dc1"}}
|
|
result := monitor.GetGuestMetricsForChartBatch("dockerContainer", requests, duration)
|
|
|
|
cpuPoints := result["dc1"]["cpu"]
|
|
if len(cpuPoints) < 2 {
|
|
t.Fatalf("expected multi-point series from best alias, got %d points", len(cpuPoints))
|
|
}
|
|
span := chartSeriesCoverageSpan(cpuPoints)
|
|
if span < time.Hour {
|
|
t.Fatalf("expected coverage span >= 1h from legacy alias, got %s", span)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChartBatch_BatchesMultipleNodesAndMetricTypes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
// Write store data for 2 nodes with multiple metric types.
|
|
for _, nid := range []string{"node-1", "node-2"} {
|
|
for _, mt := range []string{"cpu", "memory", "disk"} {
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", nid, mt, []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 10},
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 20},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 30},
|
|
})
|
|
}
|
|
}
|
|
|
|
metricTypes := []string{"cpu", "memory", "disk", "netin", "netout"}
|
|
result := monitor.GetNodeMetricsForChartBatch([]string{"node-1", "node-2"}, metricTypes, duration)
|
|
|
|
for _, nid := range []string{"node-1", "node-2"} {
|
|
nodeMetrics, ok := result[nid]
|
|
if !ok {
|
|
t.Fatalf("missing result for %s", nid)
|
|
}
|
|
for _, mt := range []string{"cpu", "memory", "disk"} {
|
|
if len(nodeMetrics[mt]) == 0 {
|
|
t.Errorf("%s/%s: expected points from store, got none", nid, mt)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChartBatch_PrefersInMemoryWhenLongRangeCoverageIsSufficient(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 12 * time.Hour
|
|
|
|
for _, offset := range []time.Duration{-11 * time.Hour, -6 * time.Hour, -5 * time.Minute} {
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "cpu", 14, now.Add(offset))
|
|
monitor.metricsHistory.AddNodeMetric("node-1", "memory", 37, now.Add(offset))
|
|
}
|
|
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "cpu", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 84},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 88},
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 94},
|
|
})
|
|
writeRawMetricBatch(t, monitor.metricsStore, "node", "node-1", "memory", []MetricPoint{
|
|
{Timestamp: now.Add(-11 * time.Hour), Value: 74},
|
|
{Timestamp: now.Add(-6 * time.Hour), Value: 78},
|
|
{Timestamp: now.Add(-5 * time.Minute), Value: 82},
|
|
})
|
|
|
|
result := monitor.GetNodeMetricsForChartBatch([]string{"node-1"}, []string{"cpu", "memory"}, duration)
|
|
|
|
if cpuPoints := result["node-1"]["cpu"]; len(cpuPoints) == 0 || cpuPoints[0].Value != 14 {
|
|
t.Fatalf("expected long-range in-memory node batch cpu history to win, got %+v", cpuPoints)
|
|
}
|
|
if memoryPoints := result["node-1"]["memory"]; len(memoryPoints) == 0 || memoryPoints[0].Value != 37 {
|
|
t.Fatalf("expected long-range in-memory node batch memory history to win, got %+v", memoryPoints)
|
|
}
|
|
}
|
|
|
|
func TestGetNodeMetricsForChartBatch_EmptyNodeIDs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
result := monitor.GetNodeMetricsForChartBatch(nil, []string{"cpu"}, time.Hour)
|
|
if result != nil {
|
|
t.Fatalf("expected nil for empty nodeIDs, got %v", result)
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChartBatch_BatchesMultiplePools(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
now := time.Now().UTC().Truncate(time.Second)
|
|
duration := 4 * time.Hour
|
|
|
|
for _, sid := range []string{"storage-1", "storage-2"} {
|
|
writeRawMetricBatch(t, monitor.metricsStore, "storage", sid, "usage", []MetricPoint{
|
|
{Timestamp: now.Add(-3 * time.Hour), Value: 60},
|
|
{Timestamp: now.Add(-2 * time.Hour), Value: 65},
|
|
{Timestamp: now.Add(-1 * time.Hour), Value: 70},
|
|
})
|
|
}
|
|
|
|
result := monitor.GetStorageMetricsForChartBatch([]string{"storage-1", "storage-2"}, duration)
|
|
|
|
for _, sid := range []string{"storage-1", "storage-2"} {
|
|
metrics, ok := result[sid]
|
|
if !ok {
|
|
t.Fatalf("missing result for %s", sid)
|
|
}
|
|
if len(metrics["usage"]) == 0 {
|
|
t.Errorf("%s: expected usage points from store, got none", sid)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestGetStorageMetricsForChartBatch_EmptyStorageIDs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
monitor := newChartFallbackTestMonitor(t)
|
|
result := monitor.GetStorageMetricsForChartBatch(nil, time.Hour)
|
|
if result != nil {
|
|
t.Fatalf("expected nil for empty storageIDs, got %v", result)
|
|
}
|
|
}
|