feat(truenas): add native disk chart fallback

This commit is contained in:
rcourtman
2026-03-29 21:17:51 +01:00
parent eebf5bd678
commit a71f370731
13 changed files with 794 additions and 10 deletions
@@ -245,6 +245,13 @@ physical-disk resources such as TrueNAS disks into the shared `disk`
metrics-store contract via the existing SMART-temperature writer, so physical
disk charts and disk-health consumers read one history path instead of a
TrueNAS-only temperature cache.
That same boundary now also owns native disk-history fallback when Pulse's own
history is shallow. `internal/truenas/client.go`,
`internal/truenas/provider.go`, `internal/monitoring/truenas_poller.go`, and
`internal/monitoring/monitor_metrics.go` must route TrueNAS `disktemp`
reporting history through the shared physical-disk chart path, so canonical
disk charts can render real provider-backed history instead of flat padding
after restarts or immediately after onboarding.
That same monitoring boundary now also owns modern TrueNAS app workload
telemetry. `internal/truenas/client.go`, `internal/truenas/provider.go`, and
`internal/monitoring/monitor.go` must ingest `app.stats` through the official
@@ -207,6 +207,38 @@ func TestUnifiedPhysicalDiskMetricsUseCanonicalDiskHistoryPath(t *testing.T) {
}
}
func TestUnifiedPhysicalDiskMetricsAllowNativeHistoryProviders(t *testing.T) {
monitorData, err := os.ReadFile("monitor.go")
if err != nil {
t.Fatalf("failed to read monitor.go: %v", err)
}
monitorSource := string(monitorData)
monitorSnippets := []string{
"type MonitorPhysicalDiskTemperatureHistoryProvider interface {",
`PhysicalDiskTemperatureHistory(m *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint`,
}
for _, snippet := range monitorSnippets {
if !strings.Contains(monitorSource, snippet) {
t.Fatalf("monitor.go must contain %q", snippet)
}
}
pollerData, err := os.ReadFile("truenas_poller.go")
if err != nil {
t.Fatalf("failed to read truenas_poller.go: %v", err)
}
pollerSource := string(pollerData)
pollerSnippets := []string{
"func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint {",
"entry.provider.PhysicalDiskTemperatureHistory(ctx, duration)",
}
for _, snippet := range pollerSnippets {
if !strings.Contains(pollerSource, snippet) {
t.Fatalf("truenas_poller.go must contain %q", snippet)
}
}
}
func TestTrueNASSystemTelemetryUsesCanonicalHostTemperatureModel(t *testing.T) {
clientData, err := os.ReadFile(filepath.Join("..", "truenas", "client.go"))
if err != nil {
+7
View File
@@ -142,6 +142,13 @@ type MonitorSupplementalRecordsProvider interface {
SupplementalRecords(m *Monitor, orgID string) []unifiedresources.IngestRecord
}
// MonitorPhysicalDiskTemperatureHistoryProvider optionally exposes source-native
// physical-disk temperature history through the canonical monitoring chart
// boundary when Pulse's own stored history is shallow.
type MonitorPhysicalDiskTemperatureHistoryProvider interface {
PhysicalDiskTemperatureHistory(m *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint
}
func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
baseName := strings.TrimSpace(nodeName)
if baseName == "" {
+43
View File
@@ -215,6 +215,7 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s
resourceIDs[i] = d.resourceID
}
batchMetrics := m.queryStoreBatchMetricMapWithGapFill("disk", resourceIDs, duration)
nativeHistory := m.nativePhysicalDiskTemperatureHistory(duration)
// Phase 3: Build result entries.
result := make(map[string]DiskChartEntry, len(disks))
@@ -225,6 +226,13 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s
tempPoints = pts
}
}
if nativePoints, ok := nativeHistory[d.resourceID]; ok {
nativePoints = lttb(nativePoints, chartDownsampleTarget)
if chartSeriesCoverageSpan(nativePoints) > chartSeriesCoverageSpan(tempPoints) &&
(!hasSufficientChartSeriesCoverage(tempPoints, duration) || len(tempPoints) < 2) {
tempPoints = nativePoints
}
}
// Sparklines require >= 2 points. If the store returned 0 or 1 points
// but the disk has a live temperature reading, pad to 2 points so the
@@ -248,6 +256,41 @@ func (m *Monitor) GetPhysicalDiskTemperatureCharts(duration time.Duration) map[s
return result
}
func (m *Monitor) nativePhysicalDiskTemperatureHistory(duration time.Duration) map[string][]MetricPoint {
providers := m.supplementalProviderSnapshot()
if len(providers) == 0 {
return nil
}
orgID := "default"
if m != nil {
if trimmed := strings.TrimSpace(m.GetOrgID()); trimmed != "" {
orgID = trimmed
}
}
history := make(map[string][]MetricPoint)
for _, provider := range providers {
historyProvider, ok := provider.(MonitorPhysicalDiskTemperatureHistoryProvider)
if !ok {
continue
}
nativeHistory := historyProvider.PhysicalDiskTemperatureHistory(m, orgID, duration)
for resourceID, points := range nativeHistory {
if strings.TrimSpace(resourceID) == "" || len(points) == 0 {
continue
}
if existing, ok := history[resourceID]; !ok || chartSeriesCoverageSpan(points) > chartSeriesCoverageSpan(existing) {
history[resourceID] = points
}
}
}
if len(history) == 0 {
return nil
}
return history
}
func (m *Monitor) currentMetricsTargetStore() MetricsTargetResourceStore {
if m == nil {
return nil
@@ -9,6 +9,27 @@ import (
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
)
type stubDiskTemperatureHistoryProvider struct {
history map[string][]MetricPoint
}
func (s *stubDiskTemperatureHistoryProvider) SupplementalRecords(*Monitor, string) []unifiedresources.IngestRecord {
return nil
}
func (s *stubDiskTemperatureHistoryProvider) PhysicalDiskTemperatureHistory(*Monitor, string, time.Duration) map[string][]MetricPoint {
if len(s.history) == 0 {
return nil
}
result := make(map[string][]MetricPoint, len(s.history))
for resourceID, points := range s.history {
copied := make([]MetricPoint, len(points))
copy(copied, points)
result[resourceID] = copied
}
return result
}
func newChartFallbackTestMonitor(t *testing.T) *Monitor {
t.Helper()
@@ -191,6 +212,55 @@ func TestGetPhysicalDiskTemperatureCharts_UsesUnifiedReadStateDiskViews(t *testi
}
}
func TestGetPhysicalDiskTemperatureCharts_UsesNativeHistoryWhenStoreCoverageShallow(t *testing.T) {
t.Parallel()
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{
ID: "disk-1",
Node: "truenas-main",
Instance: "",
DevPath: "/dev/sda",
Model: "Seagate Exos X18",
Serial: "SERIAL-DISK-1",
Temperature: 34,
LastChecked: time.Now().UTC(),
},
},
})
now := time.Now().UTC().Truncate(time.Second)
monitor := &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(registry),
supplementalProviders: map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
history: map[string][]MetricPoint{
"SERIAL-DISK-1": {
{Timestamp: now.Add(-2 * time.Hour), Value: 29},
{Timestamp: now.Add(-1 * time.Hour), Value: 31},
{Timestamp: now, Value: 34},
},
},
},
},
}
charts := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
entry, ok := charts["SERIAL-DISK-1"]
if !ok {
t.Fatalf("expected chart entry for canonical disk metric id, got %#v", charts)
}
if len(entry.Temperature) != 3 {
t.Fatalf("expected native history points instead of padded fallback, got %+v", entry.Temperature)
}
if entry.Temperature[0].Value != 29 || entry.Temperature[len(entry.Temperature)-1].Value != 34 {
t.Fatalf("expected native history values to win, got %+v", entry.Temperature)
}
}
func TestGetGuestMetricsForChart_UsesGapFillLookbackWhenRequestedRangeIsEmpty(t *testing.T) {
t.Parallel()
@@ -7,6 +7,8 @@ import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@@ -27,6 +29,8 @@ const (
SLONodeChartBatchP95 = 35 * time.Millisecond
SLOGuestChartBatchGitHubActionsP95 = 220 * time.Millisecond
SLONodeChartBatchGitHubActionsP95 = 140 * time.Millisecond
SLOPhysicalDiskChartFallbackP95 = 30 * time.Millisecond
SLOPhysicalDiskChartFallbackGHA = 120 * time.Millisecond
monitoringSLOIterations = 120
)
@@ -231,3 +235,66 @@ func TestSLO_GetNodeMetricsForChartBatch(t *testing.T) {
t.Errorf("SLO VIOLATION: p95=%v exceeds target %v", p95, target)
}
}
func TestSLO_GetPhysicalDiskTemperatureCharts_WithNativeHistoryFallback(t *testing.T) {
skipMonitoringSLOUnderRace(t)
suppressMonitoringTestLogs(t)
monitor := newChartFallbackTestMonitor(t)
now := time.Now().UTC().Truncate(time.Second)
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(models.StateSnapshot{
PhysicalDisks: []models.PhysicalDisk{
{
ID: "disk-1",
Node: "truenas-main",
DevPath: "/dev/sda",
Model: "Seagate Exos X18",
Serial: "SERIAL-DISK-1",
Temperature: 34,
LastChecked: now,
},
},
})
monitor.resourceStore = unifiedresources.NewMonitorAdapter(registry)
monitor.supplementalProviders = map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider{
unifiedresources.SourceTrueNAS: &stubDiskTemperatureHistoryProvider{
history: map[string][]MetricPoint{
"SERIAL-DISK-1": {
{Timestamp: now.Add(-2 * time.Hour), Value: 29},
{Timestamp: now.Add(-1 * time.Hour), Value: 31},
{Timestamp: now, Value: 34},
},
},
},
}
sanity := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
entry, ok := sanity["SERIAL-DISK-1"]
if !ok {
t.Fatalf("sanity: expected chart entry for canonical disk metric id, got %#v", sanity)
}
if len(entry.Temperature) != 3 {
t.Fatalf("sanity: expected native history points instead of padded fallback, got %+v", entry.Temperature)
}
latencies := measureMonitoringLatencies(t, func() {
result := monitor.GetPhysicalDiskTemperatureCharts(4 * time.Hour)
entry, ok := result["SERIAL-DISK-1"]
if !ok {
t.Fatalf("expected chart entry for canonical disk metric id, got %#v", result)
}
if len(entry.Temperature) != 3 {
t.Fatalf("expected native history points instead of padded fallback, got %+v", entry.Temperature)
}
})
target := effectiveMonitoringSLOTarget(SLOPhysicalDiskChartFallbackP95, SLOPhysicalDiskChartFallbackGHA)
p95 := monitoringPercentile(latencies, 0.95)
t.Logf("GetPhysicalDiskTemperatureCharts(native-history fallback) p50=%v p95=%v p99=%v SLO=%v",
monitoringPercentile(latencies, 0.50), p95, monitoringPercentile(latencies, 0.99), target)
if p95 > target {
t.Errorf("SLO VIOLATION: p95=%v exceeds target %v", p95, target)
}
}
+55
View File
@@ -23,6 +23,8 @@ import (
const defaultTrueNASPollInterval = 60 * time.Second
const defaultTrueNASHistoryReadTimeout = 10 * time.Second
// TrueNASPoller manages periodic polling of configured TrueNAS connections.
type TrueNASPoller struct {
multiTenant *config.MultiTenantPersistence
@@ -447,6 +449,59 @@ func (p *TrueNASPoller) SupplementalRecords(_ *Monitor, orgID string) []unifiedr
return p.GetCurrentRecordsForOrg(orgID)
}
// PhysicalDiskTemperatureHistory exposes native TrueNAS disk temperature
// history through the canonical monitoring chart boundary.
func (p *TrueNASPoller) PhysicalDiskTemperatureHistory(_ *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint {
if p == nil || !truenas.IsFeatureEnabled() {
return nil
}
orgID = strings.TrimSpace(orgID)
if orgID == "" {
orgID = "default"
}
entries := p.providerEntriesForOrg(orgID)
if len(entries) == 0 {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), defaultTrueNASHistoryReadTimeout)
defer cancel()
history := make(map[string][]MetricPoint)
for _, entry := range entries {
nativeHistory, err := entry.provider.PhysicalDiskTemperatureHistory(ctx, duration)
if err != nil {
log.Warn().
Str("component", "truenas_poller").
Str("action", "disk_temperature_history").
Str("org_id", orgID).
Str("connection_id", strings.TrimSpace(entry.connectionID)).
Err(err).
Msg("TrueNAS poller failed to read native disk temperature history")
continue
}
for resourceID, points := range nativeHistory {
if strings.TrimSpace(resourceID) == "" || len(points) == 0 {
continue
}
converted := make([]MetricPoint, len(points))
for i, point := range points {
converted[i] = MetricPoint{
Timestamp: point.Timestamp,
Value: point.Value,
}
}
history[resourceID] = converted
}
}
if len(history) == 0 {
return nil
}
return history
}
// SnapshotOwnedSources declares source-native ingest ownership for legacy
// snapshot suppression (default/global call path).
func (p *TrueNASPoller) SnapshotOwnedSources() []unifiedresources.DataSource {
@@ -249,6 +249,80 @@ func TestTrueNASPollerRecordsMetrics(t *testing.T) {
}
}
func TestTrueNASPollerPhysicalDiskTemperatureHistoryUsesTenantScopedProvider(t *testing.T) {
previous := truenas.IsFeatureEnabled()
truenas.SetFeatureEnabled(true)
t.Cleanup(func() { truenas.SetFeatureEnabled(previous) })
fixtures := truenas.DefaultFixtures()
now := time.Date(2026, 3, 29, 20, 0, 0, 0, time.UTC)
fetcher := &controllableTrueNASHistoryFetcher{
snapshot: &fixtures,
history: map[string][]truenas.TimeSeriesPoint{
"sda": {
{Timestamp: now.Add(-2 * time.Hour), Value: 30},
{Timestamp: now.Add(-1 * time.Hour), Value: 32},
{Timestamp: now, Value: 34},
},
},
}
provider := truenas.NewLiveProvider(fetcher)
if err := provider.Refresh(context.Background()); err != nil {
t.Fatalf("Refresh() error = %v", err)
}
poller := NewTrueNASPoller(nil, time.Minute, nil)
poller.providersByOrg["default"] = map[string]*truenas.Provider{
"conn-1": provider,
}
history := poller.PhysicalDiskTemperatureHistory(nil, "default", 4*time.Hour)
points, ok := history["ZL0A1234"]
if !ok {
t.Fatalf("expected canonical metric resource id ZL0A1234, got %#v", history)
}
if len(points) != 3 || points[len(points)-1].Value != 34 {
t.Fatalf("unexpected tenant-scoped disk history: %+v", points)
}
}
type controllableTrueNASHistoryFetcher struct {
snapshot *truenas.FixtureSnapshot
history map[string][]truenas.TimeSeriesPoint
}
func (s *controllableTrueNASHistoryFetcher) Fetch(context.Context) (*truenas.FixtureSnapshot, error) {
if s == nil || s.snapshot == nil {
return nil, nil
}
copied := *s.snapshot
copied.Disks = append([]truenas.Disk(nil), s.snapshot.Disks...)
copied.Pools = append([]truenas.Pool(nil), s.snapshot.Pools...)
copied.Datasets = append([]truenas.Dataset(nil), s.snapshot.Datasets...)
copied.Alerts = append([]truenas.Alert(nil), s.snapshot.Alerts...)
copied.Apps = append([]truenas.App(nil), s.snapshot.Apps...)
copied.ZFSSnapshots = append([]truenas.ZFSSnapshot(nil), s.snapshot.ZFSSnapshots...)
copied.ReplicationTasks = append([]truenas.ReplicationTask(nil), s.snapshot.ReplicationTasks...)
return &copied, nil
}
func (s *controllableTrueNASHistoryFetcher) DiskTemperatureHistory(_ context.Context, identifiers []string, _ time.Duration) (map[string][]truenas.TimeSeriesPoint, error) {
result := make(map[string][]truenas.TimeSeriesPoint)
for _, identifier := range identifiers {
points, ok := s.history[identifier]
if !ok || len(points) == 0 {
continue
}
copied := make([]truenas.TimeSeriesPoint, len(points))
copy(copied, points)
result[identifier] = copied
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
type pollerControlFetcher struct {
snapshot *truenas.FixtureSnapshot
startCalls []string
+203 -6
View File
@@ -348,6 +348,27 @@ func (c *Client) GetDiskTemperatures(ctx context.Context) (map[string]int, error
return c.getDiskTemperaturesWithFallback(ctx, nil)
}
// GetDiskTemperatureHistory returns recent disk temperature series by TrueNAS
// disk identifier using the native reporting API.
func (c *Client) GetDiskTemperatureHistory(ctx context.Context, identifiers []string, duration time.Duration) (map[string][]TimeSeriesPoint, error) {
identifiers = dedupeStrings(identifiers)
if len(identifiers) == 0 {
return nil, fmt.Errorf("truenas disk temperature history requires disk identifiers")
}
conn, err := c.dialRPC(ctx)
if err != nil {
return nil, err
}
defer func() { _ = conn.Close() }()
rpc := &trueNASRPCClient{conn: conn, nextID: 1}
if err := rpc.authenticate(ctx, c.config); err != nil {
return nil, err
}
return rpc.getDiskTemperatureHistory(ctx, identifiers, duration)
}
func (c *Client) getDiskTemperaturesWithFallback(ctx context.Context, identifiers []string) (map[string]int, error) {
var response any
restErr := c.getJSON(ctx, http.MethodGet, "/disk/temperatures", &response)
@@ -1378,6 +1399,43 @@ func (c *trueNASRPCClient) getDiskTemperatureAggregates(ctx context.Context, ide
return parseDiskTemperatureAggregates(response, windowDays), nil
}
func (c *trueNASRPCClient) getDiskTemperatureHistory(ctx context.Context, identifiers []string, duration time.Duration) (map[string][]TimeSeriesPoint, error) {
if c == nil || c.conn == nil {
return nil, fmt.Errorf("truenas rpc connection is nil")
}
identifiers = dedupeStrings(identifiers)
if len(identifiers) == 0 {
return nil, fmt.Errorf("truenas rpc disk temperature history query requires at least one identifier")
}
if duration <= 0 {
duration = 24 * time.Hour
}
end := time.Now().Unix()
start := end - int64(duration.Seconds())
if start <= 0 {
start = end
}
graphs := make([]map[string]any, 0, len(identifiers))
for _, identifier := range identifiers {
graphs = append(graphs, map[string]any{
"name": "disktemp",
"identifier": identifier,
})
}
response, err := c.getReportingDataWithQuery(ctx, graphs, map[string]any{
"aggregate": false,
"start": start,
"end": end,
})
if err != nil {
return nil, err
}
return parseReportingDiskTemperatureHistory(response), nil
}
func (c *trueNASRPCClient) getReportingData(ctx context.Context, graphs []map[string]any) ([]trueNASReportingGetDataResponse, error) {
if c == nil || c.conn == nil {
return nil, fmt.Errorf("truenas rpc connection is nil")
@@ -1392,15 +1450,29 @@ func (c *trueNASRPCClient) getReportingData(ctx context.Context, graphs []map[st
start = end
}
var response []trueNASReportingGetDataResponse
return c.getReportingDataWithQuery(ctx, graphs, map[string]any{
"aggregate": true,
"start": start,
"end": end,
})
}
func (c *trueNASRPCClient) getReportingDataWithQuery(ctx context.Context, graphs []map[string]any, query map[string]any) ([]trueNASReportingGetDataResponse, error) {
if c == nil || c.conn == nil {
return nil, fmt.Errorf("truenas rpc connection is nil")
}
if len(graphs) == 0 {
return nil, fmt.Errorf("truenas reporting query requires at least one graph")
}
if len(query) == 0 {
return nil, fmt.Errorf("truenas reporting query requires options")
}
params := []any{
graphs,
map[string]any{
"aggregate": true,
"start": start,
"end": end,
},
query,
}
var response []trueNASReportingGetDataResponse
if err := c.call(ctx, "reporting.get_data", params, &response); err != nil {
return nil, err
}
@@ -1615,6 +1687,41 @@ func parseReportingDiskTemperatures(responses []trueNASReportingGetDataResponse)
return temperatures
}
func parseReportingDiskTemperatureHistory(responses []trueNASReportingGetDataResponse) map[string][]TimeSeriesPoint {
if len(responses) == 0 {
return nil
}
history := make(map[string][]TimeSeriesPoint)
for _, response := range responses {
if strings.TrimSpace(strings.ToLower(response.Name)) != "disktemp" {
continue
}
identifier := readStringAny(map[string]any{"identifier": response.Identifier}, "identifier")
if identifier == "" {
continue
}
points := make([]TimeSeriesPoint, 0, len(response.Data))
for _, raw := range response.Data {
timestamp, value, ok := parseReportingSeriesPoint(raw, response.Legend)
if !ok || timestamp.IsZero() {
continue
}
points = append(points, TimeSeriesPoint{Timestamp: timestamp, Value: value})
}
if len(points) == 0 {
continue
}
history[identifier] = points
}
if len(history) == 0 {
return nil
}
return history
}
func parseDiskTemperatureAggregates(raw any, defaultWindowDays int) map[string]DiskTemperatureAggregate {
if defaultWindowDays <= 0 {
defaultWindowDays = defaultDiskTemperatureAggregateWindowDays
@@ -1703,6 +1810,96 @@ func readFloatValueAny(record map[string]any, keys ...string) (float64, bool) {
return 0, false
}
func parseReportingSeriesPoint(raw any, legends []string) (time.Time, float64, bool) {
switch typed := raw.(type) {
case []any:
if len(typed) < 2 {
return time.Time{}, 0, false
}
timestamp, ok := parseReportingTimestampAny(typed[0])
if !ok {
return time.Time{}, 0, false
}
if parsed, ok := parseFloat64Any(typed[1]); ok {
return timestamp, parsed, true
}
values := extractReportingLegendFloatValues(typed[1:], legends)
for _, legend := range legends {
if value, ok := values[legend]; ok {
return timestamp, value, true
}
}
case map[string]any:
timestamp, ok := parseReportingTimestampAny(
firstNonNilMapValue(typed, "timestamp", "time", "ts", "x"),
)
if !ok {
return time.Time{}, 0, false
}
values := extractReportingLegendFloatValues(typed, legends)
for _, legend := range legends {
if value, ok := values[legend]; ok {
return timestamp, value, true
}
}
if value, ok := readFloatValueAny(typed, "value", "y", "temperature"); ok {
return timestamp, value, true
}
}
return time.Time{}, 0, false
}
func firstNonNilMapValue(record map[string]any, keys ...string) any {
for _, key := range keys {
if value, ok := record[key]; ok && value != nil {
return value
}
}
return nil
}
func parseReportingTimestampAny(raw any) (time.Time, bool) {
switch typed := raw.(type) {
case time.Time:
if typed.IsZero() {
return time.Time{}, false
}
return typed.UTC(), true
case string:
trimmed := strings.TrimSpace(typed)
if trimmed == "" {
return time.Time{}, false
}
if parsed, err := time.Parse(time.RFC3339, trimmed); err == nil {
return parsed.UTC(), true
}
if integer, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
return unixReportingTimestamp(integer), true
}
case json.Number:
if integer, err := typed.Int64(); err == nil {
return unixReportingTimestamp(integer), true
}
if value, err := typed.Float64(); err == nil {
return unixReportingTimestamp(int64(math.Round(value))), true
}
default:
if value, ok := parseFloat64Any(raw); ok {
return unixReportingTimestamp(int64(math.Round(value))), true
}
}
return time.Time{}, false
}
func unixReportingTimestamp(value int64) time.Time {
switch {
case value >= 1_000_000_000_000:
return time.UnixMilli(value).UTC()
default:
return time.Unix(value, 0).UTC()
}
}
func readIntValueAny(record map[string]any, keys ...string) int {
for _, key := range keys {
value, ok := record[key]
+72
View File
@@ -763,6 +763,78 @@ func TestGetDisksIncludesDiskTemperatureAggregatesFromRPC(t *testing.T) {
}
}
func TestGetDiskTemperatureHistoryUsesReportingRPC(t *testing.T) {
server := newMockServerWithRPC(t, defaultAPIResponses(), nil, func(t *testing.T, conn *websocket.Conn) {
authReq := readRPCRequest(t, conn)
if authReq.Method != "auth.login_with_api_key" {
t.Fatalf("expected api-key auth method, got %q", authReq.Method)
}
writeRPCResult(t, conn, authReq.ID, true)
historyReq := readRPCRequest(t, conn)
if historyReq.Method != "reporting.get_data" {
t.Fatalf("expected reporting.get_data, got %q", historyReq.Method)
}
params, ok := historyReq.Params.([]any)
if !ok || len(params) != 2 {
t.Fatalf("unexpected history params: %#v", historyReq.Params)
}
graphs, ok := params[0].([]any)
if !ok || len(graphs) != 1 {
t.Fatalf("unexpected history graphs: %#v", params[0])
}
graph, ok := graphs[0].(map[string]any)
if !ok {
t.Fatalf("unexpected history graph entry: %#v", graphs[0])
}
if got := readStringAny(graph, "name"); got != "disktemp" {
t.Fatalf("expected disktemp graph, got %q", got)
}
if got := readStringAny(graph, "identifier"); got != "sda" {
t.Fatalf("expected sda identifier, got %q", got)
}
query, ok := params[1].(map[string]any)
if !ok {
t.Fatalf("unexpected history query: %#v", params[1])
}
if aggregate := query["aggregate"]; aggregate != false {
t.Fatalf("expected aggregate=false for history query, got %#v", aggregate)
}
now := time.Now().UTC().Truncate(time.Second)
writeRPCResult(t, conn, historyReq.ID, []map[string]any{{
"name": "disktemp",
"identifier": "sda",
"legend": []string{"temperature"},
"data": []any{
[]any{now.Add(-2 * time.Hour).Unix(), 30.0},
[]any{now.Add(-1 * time.Hour).Unix(), 31.5},
[]any{now.Unix(), 33.0},
},
"aggregations": map[string]any{},
"start": now.Add(-2 * time.Hour).Unix(),
"end": now.Unix(),
}})
})
t.Cleanup(server.Close)
client := mustClientForServer(t, server.URL, ClientConfig{APIKey: "api-key"})
history, err := client.GetDiskTemperatureHistory(context.Background(), []string{"sda"}, 4*time.Hour)
if err != nil {
t.Fatalf("GetDiskTemperatureHistory() error = %v", err)
}
points, ok := history["sda"]
if !ok {
t.Fatalf("expected history for sda, got %#v", history)
}
if len(points) != 3 {
t.Fatalf("expected 3 history points, got %+v", points)
}
if points[0].Value != 30.0 || points[2].Value != 33.0 {
t.Fatalf("unexpected history values: %+v", points)
}
}
func TestClientHandlesHTTPAndDecodeErrors(t *testing.T) {
t.Run("non-2xx response", func(t *testing.T) {
server := newMockServer(t, map[string]apiResponse{
+98
View File
@@ -56,6 +56,10 @@ type appReadFetcher interface {
ReadAppLogs(ctx context.Context, appName, containerID string, tailLines int) ([]AppLogLine, error)
}
type physicalDiskHistoryFetcher interface {
DiskTemperatureHistory(ctx context.Context, identifiers []string, duration time.Duration) (map[string][]TimeSeriesPoint, error)
}
// APIFetcher loads snapshots from the live TrueNAS API client.
type APIFetcher struct {
Client *Client
@@ -98,6 +102,13 @@ func (f *APIFetcher) ReadAppLogs(ctx context.Context, appName, containerID strin
return f.Client.GetAppLogs(ctx, appName, containerID, tailLines)
}
func (f *APIFetcher) DiskTemperatureHistory(ctx context.Context, identifiers []string, duration time.Duration) (map[string][]TimeSeriesPoint, error) {
if f == nil || f.Client == nil {
return nil, fmt.Errorf("truenas api fetcher client is nil")
}
return f.Client.GetDiskTemperatureHistory(ctx, identifiers, duration)
}
// FixtureFetcher loads snapshots from static fixture data.
type FixtureFetcher struct {
Snapshot FixtureSnapshot
@@ -255,6 +266,67 @@ func (p *Provider) GetAppConfig(_ context.Context, appID string) (*AppConfigResu
return result, nil
}
// PhysicalDiskTemperatureHistory returns canonical physical-disk temperature
// series keyed by the shared physical-disk metrics resource IDs.
func (p *Provider) PhysicalDiskTemperatureHistory(ctx context.Context, duration time.Duration) (map[string][]TimeSeriesPoint, error) {
if p == nil {
return nil, fmt.Errorf("truenas provider is nil")
}
historyFetcher, ok := p.fetcher.(physicalDiskHistoryFetcher)
if !ok {
return nil, fmt.Errorf("truenas provider fetcher does not support physical disk history")
}
snapshot := p.Snapshot()
if snapshot == nil {
return nil, fmt.Errorf("truenas provider has no cached snapshot")
}
identifiers := make([]string, 0, len(snapshot.Disks))
metricIDsByIdentifier := make(map[string]string, len(snapshot.Disks)*3)
for _, disk := range snapshot.Disks {
metricID := trueNASDiskMetricResourceID(disk)
if metricID == "" {
continue
}
if name := strings.TrimSpace(disk.Name); name != "" {
identifiers = append(identifiers, name)
}
for _, key := range trueNASDiskHistoryLookupKeys(disk) {
if _, exists := metricIDsByIdentifier[key]; !exists {
metricIDsByIdentifier[key] = metricID
}
}
}
identifiers = dedupeStrings(identifiers)
if len(identifiers) == 0 {
return nil, nil
}
nativeHistory, err := historyFetcher.DiskTemperatureHistory(ctx, identifiers, duration)
if err != nil {
return nil, err
}
if len(nativeHistory) == 0 {
return nil, nil
}
historyByMetricID := make(map[string][]TimeSeriesPoint, len(nativeHistory))
for identifier, points := range nativeHistory {
metricID := metricIDsByIdentifier[strings.TrimSpace(identifier)]
if metricID == "" || len(points) == 0 {
continue
}
copied := make([]TimeSeriesPoint, len(points))
copy(copied, points)
historyByMetricID[metricID] = copied
}
if len(historyByMetricID) == 0 {
return nil, nil
}
return historyByMetricID, nil
}
// Close releases resources held by the active fetcher, if supported.
func (p *Provider) Close() {
if p == nil || p.fetcher == nil {
@@ -1336,6 +1408,32 @@ func temperatureAggregateMetaFromTrueNASDisk(disk Disk) *unifiedresources.Temper
}
}
func trueNASDiskHistoryLookupKeys(disk Disk) []string {
return dedupeStrings([]string{
strings.TrimSpace(disk.Name),
strings.TrimSpace(disk.ID),
strings.TrimSpace(disk.Serial),
})
}
func trueNASDiskMetricResourceID(disk Disk) string {
devPath := ""
if name := strings.TrimSpace(disk.Name); name != "" {
devPath = "/dev/" + name
}
meta := &unifiedresources.PhysicalDiskMeta{
DevPath: devPath,
Serial: strings.TrimSpace(disk.Serial),
DiskType: strings.TrimSpace(disk.Transport),
SizeBytes: disk.SizeBytes,
}
fallback := strings.TrimSpace(disk.ID)
if fallback == "" {
fallback = strings.TrimSpace(disk.Name)
}
return unifiedresources.PhysicalDiskMetaMetricID(meta, fallback)
}
func parentPoolFromDataset(datasetName string) string {
parts := strings.SplitN(strings.TrimSpace(datasetName), "/", 2)
if len(parts) == 0 {
+59 -4
View File
@@ -38,10 +38,11 @@ func (s *closableStubFetcher) Close() {
}
type controllableStubFetcher struct {
snapshot *FixtureSnapshot
startCalls []string
stopCalls []string
logReads []appLogReadCall
snapshot *FixtureSnapshot
startCalls []string
stopCalls []string
logReads []appLogReadCall
diskHistory map[string][]TimeSeriesPoint
}
type appLogReadCall struct {
@@ -92,6 +93,26 @@ func (s *controllableStubFetcher) ReadAppLogs(_ context.Context, appName, contai
}, nil
}
func (s *controllableStubFetcher) DiskTemperatureHistory(_ context.Context, identifiers []string, _ time.Duration) (map[string][]TimeSeriesPoint, error) {
if len(s.diskHistory) == 0 {
return nil, nil
}
result := make(map[string][]TimeSeriesPoint)
for _, identifier := range identifiers {
points, ok := s.diskHistory[identifier]
if !ok || len(points) == 0 {
continue
}
copied := make([]TimeSeriesPoint, len(points))
copy(copied, points)
result[identifier] = copied
}
if len(result) == 0 {
return nil, nil
}
return result, nil
}
func TestFixtureFetcherReturnsSnapshotCopy(t *testing.T) {
fixtures := DefaultFixtures()
fetcher := &FixtureFetcher{Snapshot: fixtures}
@@ -758,3 +779,37 @@ func TestRecordsProjectDiskTemperatureAggregatesIntoCanonicalMetadata(t *testing
t.Fatal("expected sda physical disk record")
}
func TestProviderPhysicalDiskTemperatureHistoryUsesCanonicalMetricIDs(t *testing.T) {
fixtures := DefaultFixtures()
now := time.Date(2026, 3, 29, 20, 0, 0, 0, time.UTC)
fetcher := &controllableStubFetcher{
snapshot: &fixtures,
diskHistory: map[string][]TimeSeriesPoint{
"sda": {
{Timestamp: now.Add(-2 * time.Hour), Value: 30},
{Timestamp: now.Add(-1 * time.Hour), Value: 32},
{Timestamp: now, Value: 34},
},
},
}
provider := NewLiveProvider(fetcher)
if err := provider.Refresh(context.Background()); err != nil {
t.Fatalf("Refresh() error = %v", err)
}
history, err := provider.PhysicalDiskTemperatureHistory(context.Background(), 4*time.Hour)
if err != nil {
t.Fatalf("PhysicalDiskTemperatureHistory() error = %v", err)
}
points, ok := history["ZL0A1234"]
if !ok {
t.Fatalf("expected canonical disk metric id ZL0A1234, got keys %#v", history)
}
if len(points) != 3 {
t.Fatalf("expected 3 temperature history points, got %+v", points)
}
if points[len(points)-1].Value != 34 {
t.Fatalf("expected latest point value 34, got %+v", points)
}
}
+7
View File
@@ -81,6 +81,13 @@ type DiskTemperatureAggregate struct {
MaxCelsius float64
}
// TimeSeriesPoint stores one provider-native metric point before it is mapped
// onto the canonical monitoring/chart surface.
type TimeSeriesPoint struct {
Timestamp time.Time
Value float64
}
// Alert mirrors a TrueNAS alert listing entry.
type Alert struct {
ID string