feat(backend/health): surface SignalR free-tier quota in /health/detailed

HealthCheckService now runs a fourth check (SignalR Quota) alongside Storage,
Processing, and Agent Binaries. Live read of ConnectionCount (max/1h) and
MessageCount (total since 00:00 UTC) via the same IAzureMonitorMetricsReader
that backs the maintenance-time Ops watcher - no cache; the data is already
3-5min stale from Azure Monitor's side, and a /health/detailed re-check is
expected to give fresh values during incident response.

Reuses MaintenanceService's internal classifier (CalculatePercent,
ClassifySignalRQuotaTier) so the watcher and the dashboard agree on the
80%/95% boundaries down to the floor-vs-ceil rounding.

Tier mapping: None -> healthy, Warning -> warning, Critical -> unhealthy.
Worst-of-both-metrics wins. PerformAllChecksAsync now uses a tri-state
combiner (any unhealthy / any warning / else healthy) so a Warning tier
correctly tints the OverallStatus bar yellow instead of red.

Fail-soft:
- SignalRResourceId unset -> Status="unknown" with "watcher not configured"
  (excluded from OverallStatus combiner so dev/local stays healthy)
- Both metric reads null -> Status="unhealthy" pointing at the most likely
  cause (Function App MI missing 'Monitoring Reader' on the SignalR resource)
- One read null -> "no data" for that row, normal classification for the other

Frontend unchanged: the existing /health-check Backend Services card loop
renders the new check with Connections, Messages, and Resource as Details.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-10 22:55:42 +02:00
parent 5cb7480bb4
commit 871ec31bf6
@@ -1,4 +1,6 @@
using AutopilotMonitor.Functions.Services.Monitoring;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services;
@@ -11,15 +13,21 @@ public class HealthCheckService
private readonly ILogger<HealthCheckService> _logger;
private readonly AdminConfigurationService _adminConfigService;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IAzureMonitorMetricsReader _metricsReader;
private readonly IConfiguration _configuration;
public HealthCheckService(
ILogger<HealthCheckService> logger,
AdminConfigurationService adminConfigService,
IHttpClientFactory httpClientFactory)
IHttpClientFactory httpClientFactory,
IAzureMonitorMetricsReader metricsReader,
IConfiguration configuration)
{
_logger = logger;
_adminConfigService = adminConfigService;
_httpClientFactory = httpClientFactory;
_metricsReader = metricsReader;
_configuration = configuration;
}
/// <summary>
@@ -36,11 +44,19 @@ public class HealthCheckService
var checks = await Task.WhenAll(
CheckStorageBackendAsync(),
CheckProcessingBackendAsync(),
CheckAgentBinariesAsync()
CheckAgentBinariesAsync(),
CheckSignalRQuotaAsync()
);
result.Checks.AddRange(checks);
result.OverallStatus = result.Checks.All(c => c.Status == "healthy") ? "healthy" : "unhealthy";
// Tri-state combiner: any unhealthy → unhealthy; any warning (no unhealthy) → warning;
// else healthy. "unknown" checks (e.g. SignalR watcher not configured) don't affect overall.
var rated = result.Checks.Where(c => c.Status != "unknown").ToList();
result.OverallStatus =
rated.Any(c => c.Status == "unhealthy") ? "unhealthy" :
rated.Any(c => c.Status == "warning") ? "warning" :
"healthy";
return result;
}
@@ -152,6 +168,121 @@ public class HealthCheckService
return check;
}
/// <summary>
/// Reads the same SignalR ConnectionCount + MessageCount metrics that the Ops watcher
/// (see <see cref="MaintenanceService"/> SignalR-quota partial) classifies, and surfaces
/// the live state as a health-check entry. Live read on each call — no cache. The
/// underlying <see cref="IAzureMonitorMetricsReader"/> is fail-soft (returns null on
/// auth/throttle/missing-data), so this method classifies that case as "unhealthy" with
/// a clear message rather than crashing the dashboard.
/// </summary>
private async Task<HealthCheck> CheckSignalRQuotaAsync()
{
var check = new HealthCheck
{
Name = "SignalR Quota",
Description = "Free-tier connection + daily message usage"
};
var resourceId = _configuration["SignalRResourceId"];
if (string.IsNullOrWhiteSpace(resourceId))
{
check.Status = "unknown";
check.Message = "SignalR quota watcher not configured (SignalRResourceId app setting unset)";
return check;
}
var connectionLimit = ParsePositiveInt(
"SignalRFreeTierConnectionLimit", MaintenanceService.DefaultSignalRConnectionLimit);
var messageLimit = ParsePositiveLong(
"SignalRFreeTierDailyMessageLimit", MaintenanceService.DefaultSignalRDailyMessageLimit);
var nowUtc = DateTimeOffset.UtcNow;
var startOfDayUtc = new DateTimeOffset(nowUtc.UtcDateTime.Date, TimeSpan.Zero);
// Two parallel Azure Monitor reads — same shape as the watcher uses.
var connectionsTask = _metricsReader.GetMaximumAsync(
resourceId, "ConnectionCount", TimeSpan.FromHours(1), CancellationToken.None);
var messagesTask = _metricsReader.GetTotalAsync(
resourceId, "MessageCount", startOfDayUtc, nowUtc, CancellationToken.None);
await Task.WhenAll(connectionsTask, messagesTask);
var observedConnections = await connectionsTask;
var observedMessages = await messagesTask;
// Both reads failed → watcher is blind. Most likely cause: managed identity
// missing the "Monitoring Reader" role on the SignalR resource.
if (observedConnections is null && observedMessages is null)
{
check.Status = "unhealthy";
check.Message = "Azure Monitor returned no data for ConnectionCount or MessageCount — check the Function App's managed identity has 'Monitoring Reader' on the SignalR resource";
check.Details = new Dictionary<string, object>
{
["Resource"] = resourceId
};
return check;
}
var details = new Dictionary<string, object>();
var worstTier = MaintenanceService.SignalRQuotaTier.None;
if (observedConnections is not null)
{
var observedInt = (int)Math.Ceiling(observedConnections.Value);
var percent = MaintenanceService.CalculatePercent(observedInt, connectionLimit);
var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
if (tier > worstTier) worstTier = tier;
details["Connections (max/1h)"] = $"{observedInt}/{connectionLimit} ({percent}%)";
}
else
{
details["Connections (max/1h)"] = "no data";
}
if (observedMessages is not null)
{
var observedLong = (long)Math.Ceiling(observedMessages.Value);
var percent = MaintenanceService.CalculatePercent(observedLong, messageLimit);
var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
if (tier > worstTier) worstTier = tier;
details["Messages (today, UTC)"] = $"{observedLong:N0}/{messageLimit:N0} ({percent}%)";
}
else
{
details["Messages (today, UTC)"] = "no data";
}
details["Resource"] = resourceId;
check.Status = worstTier switch
{
MaintenanceService.SignalRQuotaTier.Critical => "unhealthy",
MaintenanceService.SignalRQuotaTier.Warning => "warning",
_ => "healthy"
};
check.Message = worstTier switch
{
MaintenanceService.SignalRQuotaTier.Critical => "Free-tier limit nearly saturated — scale to Standard before clients get 429'd",
MaintenanceService.SignalRQuotaTier.Warning => "Approaching free-tier limit — consider scaling to Standard",
_ => "Within free-tier limits"
};
check.Details = details;
return check;
}
private int ParsePositiveInt(string key, int fallback)
{
var raw = _configuration[key];
return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
}
private long ParsePositiveLong(string key, long fallback)
{
var raw = _configuration[key];
return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
}
}
/// <summary>