From 871ec31bf69c7cd2b121e45ef4539fea00a9209a Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Sun, 10 May 2026 22:55:42 +0200 Subject: [PATCH] feat(backend/health): surface SignalR free-tier quota in /health/detailed HealthCheckService now runs a fourth check (SignalR Quota) alongside Storage, Processing, and Agent Binaries. Live read of ConnectionCount (max/1h) and MessageCount (total since 00:00 UTC) via the same IAzureMonitorMetricsReader that backs the maintenance-time Ops watcher - no cache; the data is already 3-5min stale from Azure Monitor's side, and a /health/detailed re-check is expected to give fresh values during incident response. Reuses MaintenanceService's internal classifier (CalculatePercent, ClassifySignalRQuotaTier) so the watcher and the dashboard agree on the 80%/95% boundaries down to the floor-vs-ceil rounding. Tier mapping: None -> healthy, Warning -> warning, Critical -> unhealthy. Worst-of-both-metrics wins. PerformAllChecksAsync now uses a tri-state combiner (any unhealthy / any warning / else healthy) so a Warning tier correctly tints the OverallStatus bar yellow instead of red. Fail-soft: - SignalRResourceId unset -> Status="unknown" with "watcher not configured" (excluded from OverallStatus combiner so dev/local stays healthy) - Both metric reads null -> Status="unhealthy" pointing at the most likely cause (Function App MI missing 'Monitoring Reader' on the SignalR resource) - One read null -> "no data" for that row, normal classification for the other Frontend unchanged: the existing /health-check Backend Services card loop renders the new check with Connections, Messages, and Resource as Details. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../Services/HealthCheckService.cs | 137 +++++++++++++++++- 1 file changed, 134 insertions(+), 3 deletions(-) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs b/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs index e527f5c0..76dcb0f0 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/HealthCheckService.cs @@ -1,4 +1,6 @@ +using AutopilotMonitor.Functions.Services.Monitoring; using AutopilotMonitor.Shared; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Services; @@ -11,15 +13,21 @@ public class HealthCheckService private readonly ILogger _logger; private readonly AdminConfigurationService _adminConfigService; private readonly IHttpClientFactory _httpClientFactory; + private readonly IAzureMonitorMetricsReader _metricsReader; + private readonly IConfiguration _configuration; public HealthCheckService( ILogger logger, AdminConfigurationService adminConfigService, - IHttpClientFactory httpClientFactory) + IHttpClientFactory httpClientFactory, + IAzureMonitorMetricsReader metricsReader, + IConfiguration configuration) { _logger = logger; _adminConfigService = adminConfigService; _httpClientFactory = httpClientFactory; + _metricsReader = metricsReader; + _configuration = configuration; } /// @@ -36,11 +44,19 @@ public class HealthCheckService var checks = await Task.WhenAll( CheckStorageBackendAsync(), CheckProcessingBackendAsync(), - CheckAgentBinariesAsync() + CheckAgentBinariesAsync(), + CheckSignalRQuotaAsync() ); result.Checks.AddRange(checks); - result.OverallStatus = result.Checks.All(c => c.Status == "healthy") ? "healthy" : "unhealthy"; + + // Tri-state combiner: any unhealthy → unhealthy; any warning (no unhealthy) → warning; + // else healthy. "unknown" checks (e.g. SignalR watcher not configured) don't affect overall. + var rated = result.Checks.Where(c => c.Status != "unknown").ToList(); + result.OverallStatus = + rated.Any(c => c.Status == "unhealthy") ? "unhealthy" : + rated.Any(c => c.Status == "warning") ? "warning" : + "healthy"; return result; } @@ -152,6 +168,121 @@ public class HealthCheckService return check; } + + /// + /// Reads the same SignalR ConnectionCount + MessageCount metrics that the Ops watcher + /// (see SignalR-quota partial) classifies, and surfaces + /// the live state as a health-check entry. Live read on each call — no cache. The + /// underlying is fail-soft (returns null on + /// auth/throttle/missing-data), so this method classifies that case as "unhealthy" with + /// a clear message rather than crashing the dashboard. + /// + private async Task CheckSignalRQuotaAsync() + { + var check = new HealthCheck + { + Name = "SignalR Quota", + Description = "Free-tier connection + daily message usage" + }; + + var resourceId = _configuration["SignalRResourceId"]; + if (string.IsNullOrWhiteSpace(resourceId)) + { + check.Status = "unknown"; + check.Message = "SignalR quota watcher not configured (SignalRResourceId app setting unset)"; + return check; + } + + var connectionLimit = ParsePositiveInt( + "SignalRFreeTierConnectionLimit", MaintenanceService.DefaultSignalRConnectionLimit); + var messageLimit = ParsePositiveLong( + "SignalRFreeTierDailyMessageLimit", MaintenanceService.DefaultSignalRDailyMessageLimit); + + var nowUtc = DateTimeOffset.UtcNow; + var startOfDayUtc = new DateTimeOffset(nowUtc.UtcDateTime.Date, TimeSpan.Zero); + + // Two parallel Azure Monitor reads — same shape as the watcher uses. + var connectionsTask = _metricsReader.GetMaximumAsync( + resourceId, "ConnectionCount", TimeSpan.FromHours(1), CancellationToken.None); + var messagesTask = _metricsReader.GetTotalAsync( + resourceId, "MessageCount", startOfDayUtc, nowUtc, CancellationToken.None); + + await Task.WhenAll(connectionsTask, messagesTask); + var observedConnections = await connectionsTask; + var observedMessages = await messagesTask; + + // Both reads failed → watcher is blind. Most likely cause: managed identity + // missing the "Monitoring Reader" role on the SignalR resource. + if (observedConnections is null && observedMessages is null) + { + check.Status = "unhealthy"; + check.Message = "Azure Monitor returned no data for ConnectionCount or MessageCount — check the Function App's managed identity has 'Monitoring Reader' on the SignalR resource"; + check.Details = new Dictionary + { + ["Resource"] = resourceId + }; + return check; + } + + var details = new Dictionary(); + var worstTier = MaintenanceService.SignalRQuotaTier.None; + + if (observedConnections is not null) + { + var observedInt = (int)Math.Ceiling(observedConnections.Value); + var percent = MaintenanceService.CalculatePercent(observedInt, connectionLimit); + var tier = MaintenanceService.ClassifySignalRQuotaTier(percent); + if (tier > worstTier) worstTier = tier; + details["Connections (max/1h)"] = $"{observedInt}/{connectionLimit} ({percent}%)"; + } + else + { + details["Connections (max/1h)"] = "no data"; + } + + if (observedMessages is not null) + { + var observedLong = (long)Math.Ceiling(observedMessages.Value); + var percent = MaintenanceService.CalculatePercent(observedLong, messageLimit); + var tier = MaintenanceService.ClassifySignalRQuotaTier(percent); + if (tier > worstTier) worstTier = tier; + details["Messages (today, UTC)"] = $"{observedLong:N0}/{messageLimit:N0} ({percent}%)"; + } + else + { + details["Messages (today, UTC)"] = "no data"; + } + + details["Resource"] = resourceId; + + check.Status = worstTier switch + { + MaintenanceService.SignalRQuotaTier.Critical => "unhealthy", + MaintenanceService.SignalRQuotaTier.Warning => "warning", + _ => "healthy" + }; + + check.Message = worstTier switch + { + MaintenanceService.SignalRQuotaTier.Critical => "Free-tier limit nearly saturated — scale to Standard before clients get 429'd", + MaintenanceService.SignalRQuotaTier.Warning => "Approaching free-tier limit — consider scaling to Standard", + _ => "Within free-tier limits" + }; + check.Details = details; + return check; + } + + private int ParsePositiveInt(string key, int fallback) + { + var raw = _configuration[key]; + return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback; + } + + private long ParsePositiveLong(string key, long fallback) + { + var raw = _configuration[key]; + return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback; + } } ///