diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 419c34448e..a16a714d8b 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -764,7 +764,8 @@ Downstream readers keep that distinction too. The disk temperature charts do not pad a series to now with the retained value. The AI chat context, Patrol and the AI disk tools present it only as a last-known value with its reason. The performance report and the reporting runtime snapshot leave it out of their -disk tables. +disk tables. The performance report colours a collected reading by the +tenant's alert disk temperature thresholds for the disk type, not a fixed line. An enabled availability target assigned to a host agent creates an agent-lifecycle lease for that exact target/agent pairing. First assignment diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 692424c56d..24d5382258 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1702,10 +1702,10 @@ normalization restores a non-positive entry to its default. A disk is hot from its trigger. Alerts and Patrol findings stay open until the reading falls to the clear value, or under the trigger when there is no band below it. The Physical Disks verdict and the TrueNAS Health cell judge only the -current reading against the trigger. Judges that still differ: TrueNAS disk -temperature alerts use `TrueNASDiskDefaults.Temperature` (a flat 55/50), the -TrueNAS disk drawer tones the reading from a fixed 55C, and PDF reports colour -disk temperatures at a fixed 50/60C. +current reading against the trigger. PDF reports colour disk temperatures by +the same thresholds. Judges that still differ: TrueNAS disk temperature alerts +use `TrueNASDiskDefaults.Temperature` (a flat 55/50), and the TrueNAS disk +drawer tones the reading from a fixed 55C. `TestDiskTemperatureThresholdMatchesCheckHostPolicy` in `internal/alerts/threshold_resolution_shared_test.go` pins per-type resolution, a raised NVMe trigger that `CheckHost` also honours, the copy, the disabled default and diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index e04dff160b..21b9ae0ad6 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -6349,6 +6349,15 @@ when `diskinventory.TemperatureCollected` holds, so a retained value is reported as 0 (no reading) rather than tabulated as measured. Proof: `TestContract_ReportsOmitRetainedDiskTemperatures` in `internal/api/contract_test.go`. +Each reported disk also carries its alert disk temperature thresholds +(`reporting.DiskInfo.TemperatureWarning` / `TemperatureCritical`, the clear +value and the trigger). `reportDiskTemperatureThresholds` resolves them from +the tenant's alert manager (`alerts.Manager.DiskTemperatureThreshold`, the +factory policy when there is none). The PDF disk table colours a reading amber +from the clear value and red from the trigger, as the Physical Disks Temp +column does, and leaves it plain when disk temperature alerting is off. It no +longer uses a fixed 50/60C. Proof: +`TestContract_ReportsCarryDiskTemperatureAlertThresholds`. That same metrics-history contract also owns Kubernetes pod identity normalization. `/api/metrics-store/history` must accept legacy bare pod IDs such as `cluster-1:pod:pod-1`, canonicalize them onto the unified pod metrics diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 079fabf486..6607d9eed8 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -6779,7 +6779,9 @@ judges the current reading, with no memory of an alert. A disk temperature alert held between its clear value and trigger shows an amber Temp cell, not `Running Hot`. A switched-off agent Disk Temp default resolves to null, which leaves the reading uncoloured and never hot, as alerting is off for it. -Per-host Disk Temp overrides do not yet reach Physical Disks. +Per-host Disk Temp overrides do not yet reach Physical Disks. PDF performance +reports colour a disk reading by the same per-type thresholds, which +`reporting.DiskInfo` carries. `frontend-modern/src/features/storageBackups/__tests__/diskPresentation.test.ts` pins cell-and-verdict agreement for every type from 40C to 80C under factory and raised triggers, plus retained readings, failure evidence, sorting and diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index 17faf9067b..1e0a0292d1 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -5286,6 +5286,70 @@ func TestContract_ReportsOmitRetainedDiskTemperatures(t *testing.T) { } } +// Report disk tables colour a reading by the disk's alert disk temperature +// thresholds (the tenant's per-type policy), not a fixed 50/60C: amber from the +// clear value, red from the trigger, as the Physical Disks Temp column does. +func TestContract_ReportsCarryDiskTemperatureAlertThresholds(t *testing.T) { + engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} + original := reporting.GetEngine() + reporting.SetEngine(engine) + t.Cleanup(func() { reporting.SetEngine(original) }) + + state := models.NewState() + state.Nodes = []models.Node{{ID: "node-1", Name: "node-a", Status: "online"}} + disk := func(id, devPath, diskType string, temperature int) unifiedresources.Resource { + return unifiedresources.Resource{ + ID: id, Type: unifiedresources.ResourceTypePhysicalDisk, Name: id, ParentName: "node-a", + Identity: unifiedresources.ResourceIdentity{Hostnames: []string{"node-a"}}, + PhysicalDisk: &unifiedresources.PhysicalDiskMeta{ + DevPath: devPath, DiskType: diskType, Health: "PASSED", Wearout: -1, Temperature: temperature, + Collection: &diskinventory.CollectionStatus{Temperature: diskinventory.Available("smartctl")}, + }, + } + } + monitor := newReportingMonitorForTest(t, state, []unifiedresources.Resource{ + disk("disk-nvme", "/dev/nvme0n1", "nvme", 63), + disk("disk-sata", "/dev/sda", "sata", 56), + }) + handler := NewReportingHandlers(newReportingMTMForTest(t, monitor), nil) + thresholdsByDevice := func() map[string][2]float64 { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/api/reporting?format=pdf&resourceType=node&resourceId=node-1", nil) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + handler.HandleGenerateReport(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("expected 200, got %d body=%s", rec.Code, rec.Body.String()) + } + got := map[string][2]float64{} + for _, d := range engine.lastReq.Disks { + got[d.Device] = [2]float64{d.TemperatureWarning, d.TemperatureCritical} + } + return got + } + + // No alert manager: the factory per-type policy. + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {65, 70}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("factory report disk thresholds = %v, want %v", got, want) + } + + manager := alerts.NewManager() + cfg := manager.GetConfig() + cfg.DiskTempByType["nvme"] = alerts.HysteresisThreshold{Trigger: 75, Clear: 70} + manager.UpdateConfig(cfg) + setUnexportedField(t, monitor, "alertManager", manager) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {70, 75}, "/dev/sda": {50, 55}}; !reflect.DeepEqual(got, want) { + t.Fatalf("configured report disk thresholds = %v, want %v", got, want) + } + + cfg = manager.GetConfig() + cfg.AgentDefaults.DiskTemperature = &alerts.HysteresisThreshold{Trigger: 0, Clear: 0} + manager.UpdateConfig(cfg) + if got, want := thresholdsByDevice(), map[string][2]float64{"/dev/nvme0n1": {0, 0}, "/dev/sda": {0, 0}}; !reflect.DeepEqual(got, want) { + t.Fatalf("report disk thresholds with disk temperature alerting off = %v, want %v", got, want) + } +} + func TestContract_ReportingRequestCarriesEntitledReportBranding(t *testing.T) { engine := &stubReportingEngine{data: []byte("report"), contentType: "application/pdf"} original := reporting.GetEngine() diff --git a/internal/api/metrics_reporting_handlers.go b/internal/api/metrics_reporting_handlers.go index f363c6c0b2..3cf94d3f68 100644 --- a/internal/api/metrics_reporting_handlers.go +++ b/internal/api/metrics_reporting_handlers.go @@ -14,6 +14,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/ai" + "github.com/rcourtman/pulse-go-rewrite/internal/alerts" "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/models" "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" @@ -334,6 +335,9 @@ type reportingEnrichmentSnapshot struct { RecentlyResolved []models.ResolvedAlert LegacyBackups models.PVEBackups Resources []unifiedresources.Resource + // alertManager supplies the tenant's disk temperature thresholds. Nil + // means the factory alert configuration. + alertManager *alerts.Manager } func emptyReportingEnrichmentSnapshot() reportingEnrichmentSnapshot { @@ -405,6 +409,7 @@ func (h *ReportingHandlers) getReportingEnrichmentSnapshot(ctx context.Context, RecentlyResolved: monitor.RecentlyResolvedSnapshot(), LegacyBackups: monitor.PVEBackupsSnapshot(), Resources: unifiedResources, + alertManager: monitor.GetAlertManager(), } snapshot.normalizeCollections() return snapshot, true @@ -876,20 +881,39 @@ func (h *ReportingHandlers) enrichNodeReport(req *reporting.MetricReportRequest, continue } pd := r.PhysicalDisk + temperatureWarning, temperatureCritical := reportDiskTemperatureThresholds(snapshot.alertManager, pd.DiskType) req.Disks = append(req.Disks, reporting.DiskInfo{ - Device: pd.DevPath, - Model: pd.Model, - Serial: pd.Serial, - Type: pd.DiskType, - Size: pd.SizeBytes, - Health: pd.Health, - Temperature: reportDiskTemperature(*pd), - WearLevel: pd.Wearout, + Device: pd.DevPath, + Model: pd.Model, + Serial: pd.Serial, + Type: pd.DiskType, + Size: pd.SizeBytes, + Health: pd.Health, + Temperature: reportDiskTemperature(*pd), + WearLevel: pd.Wearout, + TemperatureWarning: temperatureWarning, + TemperatureCritical: temperatureCritical, }) } } } +// reportDiskTemperatureThresholds returns the alert disk temperature thresholds +// a report colours a disk reading by: the clear value and the trigger, from +// the same policy as disk temperature alerts and the Physical Disks Temp +// column. Zero means disk temperature alerting is off for the disk. +func reportDiskTemperatureThresholds(manager *alerts.Manager, diskType string) (float64, float64) { + threshold := manager.DiskTemperatureThreshold(diskType) + if threshold == nil || threshold.Trigger <= 0 { + return 0, 0 + } + warning := threshold.Clear + if warning <= 0 || warning > threshold.Trigger { + warning = threshold.Trigger + } + return warning, threshold.Trigger +} + // reportDiskTemperature returns the disk temperature a report may tabulate: // one the current observation collected. Normalization may keep a last-known // temperature it did not collect (a disk in standby, a host agent past its diff --git a/pkg/reporting/pdf.go b/pkg/reporting/pdf.go index b1db00fdc1..1cf9f3d575 100644 --- a/pkg/reporting/pdf.go +++ b/pkg/reporting/pdf.go @@ -1670,9 +1670,10 @@ func (g *PDFGenerator) writeDisksSection(pdf *fpdf.Fpdf, data *ReportData) { tempStr := "-" if disk.Temperature > 0 { tempStr = fmt.Sprintf("%dC", disk.Temperature) - if disk.Temperature >= 60 { + switch diskTemperatureSeverity(disk) { + case "critical": pdf.SetTextColor(colorDanger[0], colorDanger[1], colorDanger[2]) - } else if disk.Temperature >= 50 { + case "warning": pdf.SetTextColor(colorWarning[0], colorWarning[1], colorWarning[2]) } } @@ -2808,3 +2809,20 @@ func formatDuration(d time.Duration) string { } return fmt.Sprintf("%d %s", minutes, minWord) } + +// diskTemperatureSeverity colours a disk reading by its alert disk temperature +// thresholds: "critical" from the trigger, "warning" from the clear value, and +// "" below it or when the disk carries no thresholds. +func diskTemperatureSeverity(disk DiskInfo) string { + if disk.Temperature <= 0 || disk.TemperatureCritical <= 0 { + return "" + } + reading := float64(disk.Temperature) + if reading >= disk.TemperatureCritical { + return "critical" + } + if disk.TemperatureWarning > 0 && reading >= disk.TemperatureWarning { + return "warning" + } + return "" +} diff --git a/pkg/reporting/pdf_additional_test.go b/pkg/reporting/pdf_additional_test.go index 49ca2c3fc9..d190c3a4b2 100644 --- a/pkg/reporting/pdf_additional_test.go +++ b/pkg/reporting/pdf_additional_test.go @@ -57,3 +57,26 @@ func TestFormatDuration(t *testing.T) { t.Fatalf("expected 2 days, 1 hour, got %q", got) } } + +// Disk temperatures in a report take the disk's alert thresholds, not a fixed +// 50/60C: an NVMe at 63C under a 70C trigger stays uncoloured, a SATA disk at +// 56C past its 55C trigger is red, and a disk without thresholds is plain. +func TestDiskTemperatureSeverityFollowsAlertThresholds(t *testing.T) { + cases := []struct { + disk DiskInfo + want string + }{ + {DiskInfo{Temperature: 63, TemperatureWarning: 65, TemperatureCritical: 70}, ""}, + {DiskInfo{Temperature: 66, TemperatureWarning: 65, TemperatureCritical: 70}, "warning"}, + {DiskInfo{Temperature: 70, TemperatureWarning: 65, TemperatureCritical: 70}, "critical"}, + {DiskInfo{Temperature: 56, TemperatureWarning: 50, TemperatureCritical: 55}, "critical"}, + {DiskInfo{Temperature: 72, TemperatureWarning: 70, TemperatureCritical: 75}, "warning"}, + {DiskInfo{Temperature: 80}, ""}, + {DiskInfo{Temperature: 0, TemperatureWarning: 50, TemperatureCritical: 55}, ""}, + } + for _, tc := range cases { + if got := diskTemperatureSeverity(tc.disk); got != tc.want { + t.Errorf("diskTemperatureSeverity(%+v) = %q, want %q", tc.disk, got, tc.want) + } + } +} diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index cc424e6bc8..66d8e6babf 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -153,6 +153,12 @@ type DiskInfo struct { Health string // PASSED, FAILED, UNKNOWN Temperature int // Celsius WearLevel int // 0-100, percentage of life REMAINING (100 = healthy, 0 = end of life, -1 = unknown) + // TemperatureWarning and TemperatureCritical are the disk's alert disk + // temperature thresholds in Celsius (the clear value and the trigger). The + // reading is amber from Warning and red from Critical, as in the Physical + // Disks Temp column. A zero Critical leaves the reading uncoloured. + TemperatureWarning float64 + TemperatureCritical float64 } // MultiReportRequest defines the parameters for generating a multi-resource report.