Align mock chart seeding with live timeline

This commit is contained in:
rcourtman
2026-04-01 14:05:26 +01:00
parent 1339e17eaa
commit ca78a10978
3 changed files with 175 additions and 78 deletions
@@ -133,6 +133,9 @@ window instead of stitching a second live tail onto the end of seeded
sparklines. Monitoring must not let provider-owned mock resources receive a
duplicate generic unified-resource writer that appends a divergent recent tail
after the canonical mock sampler has already seeded and extended that series.
The seed path must therefore include the canonical terminal `now` sample on
its tiered timeline instead of generating history only up to before `now` and
then appending a separately anchored current-value tail afterward.
That same chart boundary also owns role-shaped realism. Seeded history,
synthetic summary fallbacks, and runtime mock writes must derive their bounds
and curve shape from the same canonical resource-role registry, so database,
+27 -75
View File
@@ -501,14 +501,15 @@ func generateFlatSeries(current float64, points int, min, max, span float64, rng
}
// buildTieredTimestamps generates a sorted list of timestamps with denser
// intervals for recent data:
// intervals for recent data, including the canonical terminal sample at now:
//
// Last 2h: 1min intervals (~120 points)
// 2h24h: 2min intervals (~660 points)
// 24hend: ~65min intervals (variable)
//
// This ensures short time ranges (1h, 4h) have enough data points without
// needing an API-level fallback layer.
// needing an API-level fallback layer, and it keeps seeded history on the same
// timeline the live mock sampler would have produced.
func buildTieredTimestamps(now time.Time, totalDuration time.Duration) []time.Time {
// Each segment covers [now - startOffset, now - endOffset) and is walked
// chronologically from oldest to newest. Segments are defined oldest-first
@@ -547,6 +548,10 @@ func buildTieredTimestamps(now time.Time, totalDuration time.Duration) []time.Ti
}
}
if len(timestamps) == 0 || !timestamps[len(timestamps)-1].Equal(now) {
timestamps = append(timestamps, now)
}
return timestamps
}
@@ -559,13 +564,11 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
return
}
// Build a tiered timestamp list so short time ranges (1h, 4h) have dense
// data without needing an API-level fallback layer.
// Build one canonical timestamp list so seeded history and subsequent live
// mock ticks sample the same runtime timeline model.
// Last 2h: 1min intervals (~120 points)
// 2h24h: 2min intervals (~660 points)
// 24h90d: ~65min intervals (~1920 points)
// The current "now" point is appended explicitly by each recorder so the
// seed and live sampler share one canonical terminal timestamp.
seedTimestamps := buildTieredTimestamps(now, seedDuration)
const seedBatchSize = 5000
numPoints := len(seedTimestamps)
@@ -603,34 +606,30 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
return
}
series := mockmodel.StorageCapacitySeriesForTimestampsWithRole(
currentUsed,
currentTotal,
currentUsage := clampFloat((currentUsed/currentTotal)*100, 0, 100)
usageSeries := GenerateSeededResourceMetricSeriesForTimestamps(
currentUsage,
seedTimestamps,
mock.MetricSeed("storage", storageID, "usage"),
mock.MetricRole("storage", storageID),
"storage",
storageID,
"usage",
styleFlat,
)
for i := 0; i < numPoints; i++ {
ts := seedTimestamps[i]
mh.AddStorageMetric(storageID, "usage", series.Usage[i], ts)
mh.AddStorageMetric(storageID, "used", series.Used[i], ts)
mh.AddStorageMetric(storageID, "avail", series.Avail[i], ts)
mh.AddStorageMetric(storageID, "total", series.Total[i], ts)
queueMetric("storage", storageID, "usage", series.Usage[i], ts)
queueMetric("storage", storageID, "used", series.Used[i], ts)
queueMetric("storage", storageID, "avail", series.Avail[i], ts)
queueMetric("storage", storageID, "total", series.Total[i], ts)
usage := clampFloat(usageSeries[i], 0, 100)
used := currentTotal * (usage / 100.0)
avail := math.Max(0, currentTotal-used)
mh.AddStorageMetric(storageID, "usage", usage, ts)
mh.AddStorageMetric(storageID, "used", used, ts)
mh.AddStorageMetric(storageID, "avail", avail, ts)
mh.AddStorageMetric(storageID, "total", currentTotal, ts)
queueMetric("storage", storageID, "usage", usage, ts)
queueMetric("storage", storageID, "used", used, ts)
queueMetric("storage", storageID, "avail", avail, ts)
queueMetric("storage", storageID, "total", currentTotal, ts)
}
last := numPoints - 1
mh.AddStorageMetric(storageID, "usage", series.Usage[last], now)
mh.AddStorageMetric(storageID, "used", series.Used[last], now)
mh.AddStorageMetric(storageID, "avail", series.Avail[last], now)
mh.AddStorageMetric(storageID, "total", series.Total[last], now)
queueMetric("storage", storageID, "usage", series.Usage[last], now)
queueMetric("storage", storageID, "used", series.Used[last], now)
queueMetric("storage", storageID, "avail", series.Avail[last], now)
queueMetric("storage", storageID, "total", series.Total[last], now)
}
recordNode := func(node models.Node) {
@@ -656,13 +655,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
queueMetric("node", node.ID, "disk", diskSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
mh.AddNodeMetric(node.ID, "cpu", node.CPU*100, now)
mh.AddNodeMetric(node.ID, "memory", node.Memory.Usage, now)
mh.AddNodeMetric(node.ID, "disk", node.Disk.Usage, now)
queueMetric("node", node.ID, "cpu", node.CPU*100, now)
queueMetric("node", node.ID, "memory", node.Memory.Usage, now)
queueMetric("node", node.ID, "disk", node.Disk.Usage, now)
}
recordGuest := func(
@@ -748,35 +740,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
}
}
// Ensure the latest point lands at "now" for full-range charts.
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "cpu", cpuPercent, now)
mh.AddGuestMetric(metricID, "memory", memPercent, now)
}
queueMetric(storeType, storeID, "cpu", cpuPercent, now)
queueMetric(storeType, storeID, "memory", memPercent, now)
if includeDisk {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "disk", diskPercent, now)
}
queueMetric(storeType, storeID, "disk", diskPercent, now)
}
if includeDiskIO {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "diskread", diskRead, now)
mh.AddGuestMetric(metricID, "diskwrite", diskWrite, now)
}
queueMetric(storeType, storeID, "diskread", diskRead, now)
queueMetric(storeType, storeID, "diskwrite", diskWrite, now)
}
if includeNetwork {
for _, metricID := range uniqueMetricIDs {
mh.AddGuestMetric(metricID, "netin", netIn, now)
mh.AddGuestMetric(metricID, "netout", netOut, now)
}
queueMetric(storeType, storeID, "netin", netIn, now)
queueMetric(storeType, storeID, "netout", netOut, now)
}
}
log.Debug().Int("count", len(state.Nodes)).Msg("mock seeding: processing nodes")
@@ -921,9 +884,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now)
}
log.Debug().Int("count", len(state.CephClusters)).Msg("mock seeding: processing ceph clusters")
@@ -954,8 +914,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
queueMetric("ceph", cephID, "usage", usageSeries[i], ts)
}
// Ensure the latest point lands at "now" for full-range charts.
queueMetric("ceph", cephID, "usage", cluster.UsagePercent, now)
}
log.Debug().Int("count", len(state.DockerHosts)).Msg("mock seeding: processing docker hosts")
@@ -1079,8 +1037,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
mh.AddGuestMetric(poolKey, "disk", diskSeries[i], ts)
queueMetric("storage", poolKey, "usage", diskSeries[i], ts)
}
mh.AddGuestMetric(poolKey, "disk", diskPercent, now)
queueMetric("storage", poolKey, "usage", diskPercent, now)
recordStorageTimeline(poolKey, float64(pool.UsedBytes), float64(pool.TotalBytes))
}
@@ -1106,8 +1062,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
mh.AddGuestMetric(dsKey, "disk", diskSeries[i], ts)
queueMetric("storage", dsKey, "usage", diskSeries[i], ts)
}
mh.AddGuestMetric(dsKey, "disk", diskPercent, now)
queueMetric("storage", dsKey, "usage", diskPercent, now)
recordStorageTimeline(dsKey, float64(dataset.UsedBytes), float64(totalBytes))
}
@@ -1136,8 +1090,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
mh.AddDiskMetric(resourceID, "smart_temp", tempSeries[i], ts)
queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts)
}
mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now)
}
for _, app := range trueNASFixtures.Apps {
@@ -19,7 +19,7 @@ func fixtureGraphWithState(state models.StateSnapshot) mock.FixtureGraph {
return mock.FixtureGraph{State: state}
}
func TestBuildTieredTimestamps_LeavesTerminalNowToRecorders(t *testing.T) {
func TestBuildTieredTimestamps_IncludesCanonicalTerminalNow(t *testing.T) {
now := time.Date(2026, time.March, 31, 12, 0, 0, 0, time.UTC)
timestamps := buildTieredTimestamps(now, time.Hour)
@@ -28,8 +28,8 @@ func TestBuildTieredTimestamps_LeavesTerminalNowToRecorders(t *testing.T) {
}
last := timestamps[len(timestamps)-1]
if !last.Before(now) {
t.Fatalf("expected seed timestamps to stop before now, got %v with now=%v", last, now)
if !last.Equal(now) {
t.Fatalf("expected seed timestamps to include terminal now, got %v with now=%v", last, now)
}
}
@@ -925,3 +925,145 @@ func TestGenerateSeededMetricSeriesForTimestamps_UsesSameTimelineAsMockRuntime(t
}
}
}
func TestSeedMockMetricsHistory_StaysContinuousWithSubsequentLiveMockTicks(t *testing.T) {
now := time.Date(2026, time.March, 31, 12, 0, 0, 0, time.UTC)
next := now.Add(time.Minute)
const storageTotal = int64(2 * 1024 * 1024 * 1024 * 1024)
storageUsageAt := func(at time.Time) (float64, int64, int64) {
usage := mock.SampleMetric("storage", "storage-tail", "usage", at)
used := int64(math.Round((float64(storageTotal) * usage) / 100.0))
if used < 0 {
used = 0
}
if used > storageTotal {
used = storageTotal
}
return usage, used, storageTotal - used
}
vmMemoryNow := mock.SampleMetric("vm", "vm-tail", "memory", now)
vmDiskNow := mock.SampleMetric("vm", "vm-tail", "disk", now)
storageUsageNow, storageUsedNow, storageFreeNow := storageUsageAt(now)
seedState := models.StateSnapshot{
VMs: []models.VM{
{
ID: "vm-tail",
Status: "running",
CPU: mock.SampleMetric("vm", "vm-tail", "cpu", now) / 100.0,
Memory: models.Memory{
Usage: vmMemoryNow,
Total: 16 * 1024 * 1024 * 1024,
Used: int64(math.Round((16 * 1024 * 1024 * 1024) * (vmMemoryNow / 100.0))),
},
Disk: models.Disk{
Usage: vmDiskNow,
Total: 512 * 1024 * 1024 * 1024,
Used: int64(math.Round((512 * 1024 * 1024 * 1024) * (vmDiskNow / 100.0))),
},
NetworkIn: mock.SampleMetricInt("vm", "vm-tail", "netin", now),
NetworkOut: mock.SampleMetricInt("vm", "vm-tail", "netout", now),
DiskRead: mock.SampleMetricInt("vm", "vm-tail", "diskread", now),
DiskWrite: mock.SampleMetricInt("vm", "vm-tail", "diskwrite", now),
},
},
Storage: []models.Storage{
{
ID: "storage-tail",
Status: "available",
Total: storageTotal,
Used: storageUsedNow,
Free: storageFreeNow,
Usage: storageUsageNow,
},
},
}
mh := NewMetricsHistory(5000, 7*24*time.Hour)
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(seedState), now, 7*24*time.Hour, time.Minute)
vmCPUSeeded := mh.GetGuestMetrics("vm-tail", "cpu", 7*24*time.Hour)
if len(vmCPUSeeded) == 0 {
t.Fatal("expected seeded vm cpu history")
}
for i, point := range vmCPUSeeded {
want := mock.SampleMetric("vm", "vm-tail", "cpu", point.Timestamp)
if diff := math.Abs(point.Value - want); diff > 1e-9 {
t.Fatalf("expected seeded vm cpu point %d to follow canonical runtime timeline: got=%f want=%f ts=%v", i, point.Value, want, point.Timestamp)
}
}
storageSeeded := mh.GetAllStorageMetrics("storage-tail", 7*24*time.Hour)["usage"]
if len(storageSeeded) == 0 {
t.Fatal("expected seeded storage usage history")
}
for i, point := range storageSeeded {
want := mock.SampleMetric("storage", "storage-tail", "usage", point.Timestamp)
if diff := math.Abs(point.Value - want); diff > 1e-9 {
t.Fatalf("expected seeded storage usage point %d to follow canonical runtime timeline: got=%f want=%f ts=%v", i, point.Value, want, point.Timestamp)
}
}
vmMemoryNext := mock.SampleMetric("vm", "vm-tail", "memory", next)
vmDiskNext := mock.SampleMetric("vm", "vm-tail", "disk", next)
storageUsageNext, storageUsedNext, storageFreeNext := storageUsageAt(next)
liveState := models.StateSnapshot{
VMs: []models.VM{
{
ID: "vm-tail",
Status: "running",
CPU: mock.SampleMetric("vm", "vm-tail", "cpu", next) / 100.0,
Memory: models.Memory{
Usage: vmMemoryNext,
Total: 16 * 1024 * 1024 * 1024,
Used: int64(math.Round((16 * 1024 * 1024 * 1024) * (vmMemoryNext / 100.0))),
},
Disk: models.Disk{
Usage: vmDiskNext,
Total: 512 * 1024 * 1024 * 1024,
Used: int64(math.Round((512 * 1024 * 1024 * 1024) * (vmDiskNext / 100.0))),
},
NetworkIn: mock.SampleMetricInt("vm", "vm-tail", "netin", next),
NetworkOut: mock.SampleMetricInt("vm", "vm-tail", "netout", next),
DiskRead: mock.SampleMetricInt("vm", "vm-tail", "diskread", next),
DiskWrite: mock.SampleMetricInt("vm", "vm-tail", "diskwrite", next),
},
},
Storage: []models.Storage{
{
ID: "storage-tail",
Status: "available",
Total: storageTotal,
Used: storageUsedNext,
Free: storageFreeNext,
Usage: storageUsageNext,
},
},
}
recordMockStateToMetricsHistory(mh, nil, fixtureGraphWithState(liveState), next)
vmCPUAfterTick := mh.GetGuestMetrics("vm-tail", "cpu", 7*24*time.Hour)
if got := vmCPUAfterTick[len(vmCPUAfterTick)-1].Timestamp; !got.Equal(next) {
t.Fatalf("expected latest vm cpu point at %v, got %v", next, got)
}
if got, want := vmCPUAfterTick[len(vmCPUAfterTick)-1].Value, mock.SampleMetric("vm", "vm-tail", "cpu", next); math.Abs(got-want) > 1e-9 {
t.Fatalf("expected live vm cpu tick to continue canonical runtime timeline: got=%f want=%f", got, want)
}
if got := vmCPUAfterTick[len(vmCPUAfterTick)-2].Timestamp; !got.Equal(now) {
t.Fatalf("expected penultimate vm cpu point to remain anchored at seed now %v, got %v", now, got)
}
storageAfterTick := mh.GetAllStorageMetrics("storage-tail", 7*24*time.Hour)["usage"]
if got := storageAfterTick[len(storageAfterTick)-1].Timestamp; !got.Equal(next) {
t.Fatalf("expected latest storage usage point at %v, got %v", next, got)
}
if got, want := storageAfterTick[len(storageAfterTick)-1].Value, mock.SampleMetric("storage", "storage-tail", "usage", next); math.Abs(got-want) > 1e-9 {
t.Fatalf("expected live storage usage tick to continue canonical runtime timeline: got=%f want=%f", got, want)
}
if got := storageAfterTick[len(storageAfterTick)-2].Timestamp; !got.Equal(now) {
t.Fatalf("expected penultimate storage point to remain anchored at seed now %v, got %v", now, got)
}
}