mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
feat(backend/health): surface SignalR free-tier quota in /health/detailed
HealthCheckService now runs a fourth check (SignalR Quota) alongside Storage, Processing, and Agent Binaries. Live read of ConnectionCount (max/1h) and MessageCount (total since 00:00 UTC) via the same IAzureMonitorMetricsReader that backs the maintenance-time Ops watcher - no cache; the data is already 3-5min stale from Azure Monitor's side, and a /health/detailed re-check is expected to give fresh values during incident response. Reuses MaintenanceService's internal classifier (CalculatePercent, ClassifySignalRQuotaTier) so the watcher and the dashboard agree on the 80%/95% boundaries down to the floor-vs-ceil rounding. Tier mapping: None -> healthy, Warning -> warning, Critical -> unhealthy. Worst-of-both-metrics wins. PerformAllChecksAsync now uses a tri-state combiner (any unhealthy / any warning / else healthy) so a Warning tier correctly tints the OverallStatus bar yellow instead of red. Fail-soft: - SignalRResourceId unset -> Status="unknown" with "watcher not configured" (excluded from OverallStatus combiner so dev/local stays healthy) - Both metric reads null -> Status="unhealthy" pointing at the most likely cause (Function App MI missing 'Monitoring Reader' on the SignalR resource) - One read null -> "no data" for that row, normal classification for the other Frontend unchanged: the existing /health-check Backend Services card loop renders the new check with Connections, Messages, and Resource as Details. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
using AutopilotMonitor.Functions.Services.Monitoring;
|
||||
using AutopilotMonitor.Shared;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services;
|
||||
@@ -11,15 +13,21 @@ public class HealthCheckService
|
||||
private readonly ILogger<HealthCheckService> _logger;
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly IAzureMonitorMetricsReader _metricsReader;
|
||||
private readonly IConfiguration _configuration;
|
||||
|
||||
public HealthCheckService(
|
||||
ILogger<HealthCheckService> logger,
|
||||
AdminConfigurationService adminConfigService,
|
||||
IHttpClientFactory httpClientFactory)
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IAzureMonitorMetricsReader metricsReader,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
_logger = logger;
|
||||
_adminConfigService = adminConfigService;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_metricsReader = metricsReader;
|
||||
_configuration = configuration;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -36,11 +44,19 @@ public class HealthCheckService
|
||||
var checks = await Task.WhenAll(
|
||||
CheckStorageBackendAsync(),
|
||||
CheckProcessingBackendAsync(),
|
||||
CheckAgentBinariesAsync()
|
||||
CheckAgentBinariesAsync(),
|
||||
CheckSignalRQuotaAsync()
|
||||
);
|
||||
|
||||
result.Checks.AddRange(checks);
|
||||
result.OverallStatus = result.Checks.All(c => c.Status == "healthy") ? "healthy" : "unhealthy";
|
||||
|
||||
// Tri-state combiner: any unhealthy → unhealthy; any warning (no unhealthy) → warning;
|
||||
// else healthy. "unknown" checks (e.g. SignalR watcher not configured) don't affect overall.
|
||||
var rated = result.Checks.Where(c => c.Status != "unknown").ToList();
|
||||
result.OverallStatus =
|
||||
rated.Any(c => c.Status == "unhealthy") ? "unhealthy" :
|
||||
rated.Any(c => c.Status == "warning") ? "warning" :
|
||||
"healthy";
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -152,6 +168,121 @@ public class HealthCheckService
|
||||
|
||||
return check;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads the same SignalR ConnectionCount + MessageCount metrics that the Ops watcher
|
||||
/// (see <see cref="MaintenanceService"/> SignalR-quota partial) classifies, and surfaces
|
||||
/// the live state as a health-check entry. Live read on each call — no cache. The
|
||||
/// underlying <see cref="IAzureMonitorMetricsReader"/> is fail-soft (returns null on
|
||||
/// auth/throttle/missing-data), so this method classifies that case as "unhealthy" with
|
||||
/// a clear message rather than crashing the dashboard.
|
||||
/// </summary>
|
||||
private async Task<HealthCheck> CheckSignalRQuotaAsync()
|
||||
{
|
||||
var check = new HealthCheck
|
||||
{
|
||||
Name = "SignalR Quota",
|
||||
Description = "Free-tier connection + daily message usage"
|
||||
};
|
||||
|
||||
var resourceId = _configuration["SignalRResourceId"];
|
||||
if (string.IsNullOrWhiteSpace(resourceId))
|
||||
{
|
||||
check.Status = "unknown";
|
||||
check.Message = "SignalR quota watcher not configured (SignalRResourceId app setting unset)";
|
||||
return check;
|
||||
}
|
||||
|
||||
var connectionLimit = ParsePositiveInt(
|
||||
"SignalRFreeTierConnectionLimit", MaintenanceService.DefaultSignalRConnectionLimit);
|
||||
var messageLimit = ParsePositiveLong(
|
||||
"SignalRFreeTierDailyMessageLimit", MaintenanceService.DefaultSignalRDailyMessageLimit);
|
||||
|
||||
var nowUtc = DateTimeOffset.UtcNow;
|
||||
var startOfDayUtc = new DateTimeOffset(nowUtc.UtcDateTime.Date, TimeSpan.Zero);
|
||||
|
||||
// Two parallel Azure Monitor reads — same shape as the watcher uses.
|
||||
var connectionsTask = _metricsReader.GetMaximumAsync(
|
||||
resourceId, "ConnectionCount", TimeSpan.FromHours(1), CancellationToken.None);
|
||||
var messagesTask = _metricsReader.GetTotalAsync(
|
||||
resourceId, "MessageCount", startOfDayUtc, nowUtc, CancellationToken.None);
|
||||
|
||||
await Task.WhenAll(connectionsTask, messagesTask);
|
||||
var observedConnections = await connectionsTask;
|
||||
var observedMessages = await messagesTask;
|
||||
|
||||
// Both reads failed → watcher is blind. Most likely cause: managed identity
|
||||
// missing the "Monitoring Reader" role on the SignalR resource.
|
||||
if (observedConnections is null && observedMessages is null)
|
||||
{
|
||||
check.Status = "unhealthy";
|
||||
check.Message = "Azure Monitor returned no data for ConnectionCount or MessageCount — check the Function App's managed identity has 'Monitoring Reader' on the SignalR resource";
|
||||
check.Details = new Dictionary<string, object>
|
||||
{
|
||||
["Resource"] = resourceId
|
||||
};
|
||||
return check;
|
||||
}
|
||||
|
||||
var details = new Dictionary<string, object>();
|
||||
var worstTier = MaintenanceService.SignalRQuotaTier.None;
|
||||
|
||||
if (observedConnections is not null)
|
||||
{
|
||||
var observedInt = (int)Math.Ceiling(observedConnections.Value);
|
||||
var percent = MaintenanceService.CalculatePercent(observedInt, connectionLimit);
|
||||
var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
|
||||
if (tier > worstTier) worstTier = tier;
|
||||
details["Connections (max/1h)"] = $"{observedInt}/{connectionLimit} ({percent}%)";
|
||||
}
|
||||
else
|
||||
{
|
||||
details["Connections (max/1h)"] = "no data";
|
||||
}
|
||||
|
||||
if (observedMessages is not null)
|
||||
{
|
||||
var observedLong = (long)Math.Ceiling(observedMessages.Value);
|
||||
var percent = MaintenanceService.CalculatePercent(observedLong, messageLimit);
|
||||
var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
|
||||
if (tier > worstTier) worstTier = tier;
|
||||
details["Messages (today, UTC)"] = $"{observedLong:N0}/{messageLimit:N0} ({percent}%)";
|
||||
}
|
||||
else
|
||||
{
|
||||
details["Messages (today, UTC)"] = "no data";
|
||||
}
|
||||
|
||||
details["Resource"] = resourceId;
|
||||
|
||||
check.Status = worstTier switch
|
||||
{
|
||||
MaintenanceService.SignalRQuotaTier.Critical => "unhealthy",
|
||||
MaintenanceService.SignalRQuotaTier.Warning => "warning",
|
||||
_ => "healthy"
|
||||
};
|
||||
|
||||
check.Message = worstTier switch
|
||||
{
|
||||
MaintenanceService.SignalRQuotaTier.Critical => "Free-tier limit nearly saturated — scale to Standard before clients get 429'd",
|
||||
MaintenanceService.SignalRQuotaTier.Warning => "Approaching free-tier limit — consider scaling to Standard",
|
||||
_ => "Within free-tier limits"
|
||||
};
|
||||
check.Details = details;
|
||||
return check;
|
||||
}
|
||||
|
||||
private int ParsePositiveInt(string key, int fallback)
|
||||
{
|
||||
var raw = _configuration[key];
|
||||
return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
private long ParsePositiveLong(string key, long fallback)
|
||||
{
|
||||
var raw = _configuration[key];
|
||||
return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
Reference in New Issue
Block a user