From bc3cefa790ac35113d72de2a9cb839ae7f8a7cd7 Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Fri, 8 May 2026 18:06:56 +0200 Subject: [PATCH] feat(backend/observability): SignalR free-quota watcher with tiered OpsEvents Hourly TimerFunction queries Azure Monitor for SignalR ConnectionCount (max over last hour) and MessageCount (sum since UTC midnight) and emits Warning (>=80%) / Error (>=95%) OpsEvents when the free-tier limits (20 connections, 20k messages/day) are approached. Mirror of the cert-expiry pattern - global TenantId=null events, dedup'd to one emission per EventType per UTC day. Fail-soft: skips silently when SignalRResourceId app setting is unset, logs warning on Azure Monitor auth/query errors. Function App's managed identity needs 'Monitoring Reader' on the SignalR resource. Also called from the manual maintenance trigger so 'Run Maintenance Now' exercises every health check. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../MaintenanceServiceSignalRQuotaTests.cs | 180 +++++++++++++++ .../AutopilotMonitor.Functions.csproj | 1 + .../Admin/SignalRQuotaTimerFunction.cs | 45 ++++ .../AutopilotMonitor.Functions/Program.cs | 3 + .../MaintenanceService.SignalRQuota.cs | 208 ++++++++++++++++++ .../Services/MaintenanceService.cs | 13 ++ .../Monitoring/AzureMonitorMetricsReader.cs | 101 +++++++++ .../Monitoring/IAzureMonitorMetricsReader.cs | 31 +++ .../Services/OpsEventService.cs | 24 ++ .../local.settings.json.template | 7 +- 10 files changed, 612 insertions(+), 1 deletion(-) create mode 100644 src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceSignalRQuotaTests.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Functions/Admin/SignalRQuotaTimerFunction.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.SignalRQuota.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Monitoring/AzureMonitorMetricsReader.cs create mode 100644 src/Backend/AutopilotMonitor.Functions/Services/Monitoring/IAzureMonitorMetricsReader.cs diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceSignalRQuotaTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceSignalRQuotaTests.cs new file mode 100644 index 00000000..d9bb97ec --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceSignalRQuotaTests.cs @@ -0,0 +1,180 @@ +using AutopilotMonitor.Functions.Services; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Tier-boundary + percent-math tests for the SignalR free-tier quota watcher. +/// Mirrors the cert-expiry tests' approach: pure-function coverage of the +/// boundary risks (off-by-one, floor vs ceil, zero-division). Full-service +/// smoke tests would require constructing 16 dependencies; the percent + +/// classifier are the parts most likely to break the watcher silently. +/// +public class MaintenanceServiceSignalRQuotaTests +{ + // --- ClassifySignalRQuotaTier --- + // Thresholds (per MaintenanceService.SignalRQuota.cs): + // >= 95% -> Critical + // >= 80% -> Warning + // else -> None + + [Theory] + [InlineData(0)] + [InlineData(50)] + [InlineData(79)] + public void ClassifySignalRQuotaTier_BelowWarning_ReturnsNone(int percent) + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.None, + MaintenanceService.ClassifySignalRQuotaTier(percent)); + } + + [Theory] + [InlineData(80)] + [InlineData(85)] + [InlineData(94)] + public void ClassifySignalRQuotaTier_InWarningBand_ReturnsWarning(int percent) + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.Warning, + MaintenanceService.ClassifySignalRQuotaTier(percent)); + } + + [Theory] + [InlineData(95)] + [InlineData(99)] + [InlineData(100)] + [InlineData(120)] + public void ClassifySignalRQuotaTier_AtOrAboveCritical_ReturnsCritical(int percent) + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.Critical, + MaintenanceService.ClassifySignalRQuotaTier(percent)); + } + + [Fact] + public void ClassifySignalRQuotaTier_BoundaryAt79_IsBelowWarning() + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.None, + MaintenanceService.ClassifySignalRQuotaTier(79)); + } + + [Fact] + public void ClassifySignalRQuotaTier_BoundaryAt80_IsWarningNotNone() + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.Warning, + MaintenanceService.ClassifySignalRQuotaTier(80)); + } + + [Fact] + public void ClassifySignalRQuotaTier_BoundaryAt94_IsWarningNotCritical() + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.Warning, + MaintenanceService.ClassifySignalRQuotaTier(94)); + } + + [Fact] + public void ClassifySignalRQuotaTier_BoundaryAt95_IsCriticalNotWarning() + { + Assert.Equal( + MaintenanceService.SignalRQuotaTier.Critical, + MaintenanceService.ClassifySignalRQuotaTier(95)); + } + + // --- CalculatePercent --- + // Floors to integer to match what an operator sees on the dashboard. + // Critical contract: 79.99% must NOT round up to 80% (warning trigger). + + [Theory] + [InlineData(16, 20, 80)] // exact 80% + [InlineData(19, 20, 95)] // exact 95% + [InlineData(20, 20, 100)] + [InlineData(0, 20, 0)] + [InlineData(10, 20, 50)] + public void CalculatePercent_ExactValues_RoundsCorrectly(double observed, double limit, int expected) + { + Assert.Equal(expected, MaintenanceService.CalculatePercent(observed, limit)); + } + + [Fact] + public void CalculatePercent_JustUnderWarning_FloorsTo79() + { + // 15.99 / 20 = 79.95% -> floor -> 79 -> stays None tier + Assert.Equal(79, MaintenanceService.CalculatePercent(15.99, 20)); + } + + [Fact] + public void CalculatePercent_JustUnderCritical_FloorsTo94() + { + // 18.99 / 20 = 94.95% -> floor -> 94 -> stays Warning tier + Assert.Equal(94, MaintenanceService.CalculatePercent(18.99, 20)); + } + + [Fact] + public void CalculatePercent_ZeroLimit_ReturnsZero() + { + // Defensive: misconfigured limit must not divide by zero or NaN-poison + // the tier classification. + Assert.Equal(0, MaintenanceService.CalculatePercent(50, 0)); + } + + [Fact] + public void CalculatePercent_NegativeLimit_ReturnsZero() + { + Assert.Equal(0, MaintenanceService.CalculatePercent(50, -1)); + } + + [Fact] + public void CalculatePercent_OverLimit_ReturnsAbove100() + { + // Over-quota must surface as >100% so Critical fires, not silently cap. + Assert.Equal(150, MaintenanceService.CalculatePercent(30, 20)); + } + + [Theory] + [InlineData(16000, 20000, 80)] + [InlineData(19000, 20000, 95)] + [InlineData(15999, 20000, 79)] + [InlineData(20000, 20000, 100)] + public void CalculatePercent_MessageScale_RoundsCorrectly(long observed, long limit, int expected) + { + // Same math but at the message-quota scale (20k cap, not 20). + // Verifies the double-precision path doesn't drift on larger numbers. + Assert.Equal(expected, MaintenanceService.CalculatePercent(observed, limit)); + } + + // --- End-to-end tier mapping (the user-visible behavior) --- + // Combines CalculatePercent + ClassifySignalRQuotaTier the way the watcher + // actually uses them. Catches drift if either function changes + // independently of the other. + + [Theory] + [InlineData(15, 20, "None")] + [InlineData(16, 20, "Warning")] + [InlineData(18, 20, "Warning")] + [InlineData(19, 20, "Critical")] + [InlineData(20, 20, "Critical")] + [InlineData(25, 20, "Critical")] + public void ConnectionScenarios_MapToCorrectTier(int observed, int limit, string expectedTier) + { + var percent = MaintenanceService.CalculatePercent(observed, limit); + var tier = MaintenanceService.ClassifySignalRQuotaTier(percent); + Assert.Equal(expectedTier, tier.ToString()); + } + + [Theory] + [InlineData(15999, 20000, "None")] + [InlineData(16000, 20000, "Warning")] + [InlineData(18999, 20000, "Warning")] + [InlineData(19000, 20000, "Critical")] + [InlineData(20000, 20000, "Critical")] + [InlineData(25000, 20000, "Critical")] + public void MessageScenarios_MapToCorrectTier(long observed, long limit, string expectedTier) + { + var percent = MaintenanceService.CalculatePercent(observed, limit); + var tier = MaintenanceService.ClassifySignalRQuotaTier(percent); + Assert.Equal(expectedTier, tier.ToString()); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj b/src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj index 14e926d5..7ca94ab7 100644 --- a/src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj +++ b/src/Backend/AutopilotMonitor.Functions/AutopilotMonitor.Functions.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/SignalRQuotaTimerFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/SignalRQuotaTimerFunction.cs new file mode 100644 index 00000000..ee880337 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/SignalRQuotaTimerFunction.cs @@ -0,0 +1,45 @@ +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Admin +{ + /// + /// Timer-triggered watcher for the Azure SignalR Free-tier limits (20 + /// concurrent connections / 20,000 messages per UTC day). Runs every full + /// hour and emits OpsEvents when the configured thresholds are crossed so + /// operators get a Telegram ping before clients start getting 429'd. + /// + /// Separate from the 2h MaintenanceFunction: the SignalR limits can be + /// breached by a brief connection spike, so the cadence is tighter. Cost + /// impact is negligible (~720 extra executions/month, ~$0.04). + /// + public class SignalRQuotaTimerFunction + { + private readonly MaintenanceService _maintenanceService; + private readonly ILogger _logger; + + public SignalRQuotaTimerFunction( + MaintenanceService maintenanceService, + ILogger logger) + { + _maintenanceService = maintenanceService; + _logger = logger; + } + + /// + /// NCRONTAB "0 0 * * * *" - every hour on the minute. The watcher itself + /// is idempotent (one OpsEvent per EventType per UTC day) and skips + /// silently when SignalRResourceId is unset, so this trigger is safe to + /// run in environments without managed-identity wiring. + /// + [Function("SignalRQuotaTimer")] + public async Task Run([TimerTrigger("0 0 * * * *")] object timer, CancellationToken ct) + { + _logger.LogInformation("SignalR quota watcher fired"); + await _maintenanceService.CheckSignalRQuotaAsync(ct); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index bcde7718..8ebe4e77 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -154,6 +154,9 @@ builder.Services.AddSingleton(); builder.Services.AddHttpClient() .AddPolicyHandler((sp, _) => sp.GetRequiredService().ExternalDataApi); builder.Services.AddSingleton(); +builder.Services.AddSingleton< + AutopilotMonitor.Functions.Services.Monitoring.IAzureMonitorMetricsReader, + AutopilotMonitor.Functions.Services.Monitoring.AzureMonitorMetricsReader>(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.SignalRQuota.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.SignalRQuota.cs new file mode 100644 index 00000000..0e48f4e0 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.SignalRQuota.cs @@ -0,0 +1,208 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Shared.DataAccess; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// SignalR Free-tier quota watcher. Reads ConnectionCount (max over the last + /// hour) and MessageCount (total since UTC midnight) from Azure Monitor and + /// emits tiered OpsEvents when either approaches the documented Free-tier + /// caps - 20 concurrent connections and 20,000 messages per day. The + /// operator can then scale to Standard before clients get 429'd. + /// + /// Globally scoped (TenantId = null) - SignalR is one shared resource for + /// the whole Function App, not a per-tenant concern. + /// + public partial class MaintenanceService + { + // Tier thresholds, expressed as percent of the configured cap. + // Order matters: most-severe first. + internal const int SignalRQuotaCriticalPercent = 95; + internal const int SignalRQuotaWarningPercent = 80; + + // Default Free-tier caps as documented by Microsoft. Overridable via + // app settings so an operator who has scaled to a paid tier can re-target + // the watcher without a code change (or disable it by setting the cap + // very high). + internal const int DefaultSignalRConnectionLimit = 20; + internal const long DefaultSignalRDailyMessageLimit = 20_000; + + // Dedup prefix - all SignalR EventTypes start with this, so the seen-index + // can be built with a single Security-category read. + internal const string SignalRQuotaEventTypePrefix = "SignalR"; + + internal enum SignalRQuotaTier { None, Warning, Critical } + + /// + /// Pure tier classifier - extracted so the boundary math is unit-testable + /// without constructing a full MaintenanceService. + /// + internal static SignalRQuotaTier ClassifySignalRQuotaTier(int percent) + { + if (percent >= SignalRQuotaCriticalPercent) return SignalRQuotaTier.Critical; + if (percent >= SignalRQuotaWarningPercent) return SignalRQuotaTier.Warning; + return SignalRQuotaTier.None; + } + + internal static int CalculatePercent(double observed, double limit) + { + if (limit <= 0) return 0; + // Floor to avoid 79.6% triggering an 80% warning. Matches what the + // operator sees on the Azure dashboard, which also rounds down. + return (int)Math.Floor(observed / limit * 100.0); + } + + /// + /// Reads SignalR ConnectionCount + MessageCount metrics from Azure Monitor + /// and emits Warning/Error OpsEvents when the Free-tier limits are + /// approached. Dedup: one event per EventType per UTC day. Fail-soft - + /// missing config or Azure Monitor errors yield a single log line, never + /// a thrown exception. + /// + public async Task CheckSignalRQuotaAsync(CancellationToken ct = default) + { + var resourceId = _configuration["SignalRResourceId"]; + if (string.IsNullOrWhiteSpace(resourceId)) + { + _logger.LogInformation( + "SignalR quota watcher skipped: SignalRResourceId app setting not configured"); + return; + } + + var connectionLimit = ParseIntSetting( + "SignalRFreeTierConnectionLimit", DefaultSignalRConnectionLimit); + var messageLimit = ParseLongSetting( + "SignalRFreeTierDailyMessageLimit", DefaultSignalRDailyMessageLimit); + + var seenToday = await BuildSignalRQuotaSeenIndexAsync(); + + await CheckConnectionsAsync(resourceId, connectionLimit, seenToday, ct); + await CheckMessagesAsync(resourceId, messageLimit, seenToday, ct); + } + + private async Task CheckConnectionsAsync( + string resourceId, int limit, HashSet seenToday, CancellationToken ct) + { + var observed = await _metricsReader.GetMaximumAsync( + resourceId, "ConnectionCount", TimeSpan.FromHours(1), ct); + if (observed is null) + { + _logger.LogDebug("SignalR ConnectionCount metric returned no data points"); + return; + } + + var observedInt = (int)Math.Ceiling(observed.Value); + var percent = CalculatePercent(observedInt, limit); + var tier = ClassifySignalRQuotaTier(percent); + + switch (tier) + { + case SignalRQuotaTier.Critical: + if (seenToday.Add("SignalRConnectionsCritical")) + { + await _opsEventService.RecordSignalRConnectionsCriticalAsync( + observedInt, limit, percent, resourceId); + _logger.LogError( + "SignalR Critical: {Observed}/{Limit} connections ({Percent}%)", + observedInt, limit, percent); + } + break; + case SignalRQuotaTier.Warning: + if (seenToday.Add("SignalRConnectionsHigh")) + { + await _opsEventService.RecordSignalRConnectionsHighAsync( + observedInt, limit, percent, resourceId); + _logger.LogWarning( + "SignalR Warning: {Observed}/{Limit} connections ({Percent}%)", + observedInt, limit, percent); + } + break; + default: + _logger.LogInformation( + "SignalR connections healthy: {Observed}/{Limit} ({Percent}%)", + observedInt, limit, percent); + break; + } + } + + private async Task CheckMessagesAsync( + string resourceId, long limit, HashSet seenToday, CancellationToken ct) + { + // Quota is daily and resets at 00:00 UTC, so query that exact window. + var nowUtc = DateTimeOffset.UtcNow; + var startOfDayUtc = new DateTimeOffset(nowUtc.UtcDateTime.Date, TimeSpan.Zero); + + var observed = await _metricsReader.GetTotalAsync( + resourceId, "MessageCount", startOfDayUtc, nowUtc, ct); + if (observed is null) + { + _logger.LogDebug("SignalR MessageCount metric returned no data points"); + return; + } + + var observedLong = (long)Math.Ceiling(observed.Value); + var percent = CalculatePercent(observedLong, limit); + var tier = ClassifySignalRQuotaTier(percent); + + switch (tier) + { + case SignalRQuotaTier.Critical: + if (seenToday.Add("SignalRMessagesCritical")) + { + await _opsEventService.RecordSignalRMessagesCriticalAsync( + observedLong, limit, percent, resourceId); + _logger.LogError( + "SignalR Critical: {Observed}/{Limit} messages today ({Percent}%)", + observedLong, limit, percent); + } + break; + case SignalRQuotaTier.Warning: + if (seenToday.Add("SignalRMessagesHigh")) + { + await _opsEventService.RecordSignalRMessagesHighAsync( + observedLong, limit, percent, resourceId); + _logger.LogWarning( + "SignalR Warning: {Observed}/{Limit} messages today ({Percent}%)", + observedLong, limit, percent); + } + break; + default: + _logger.LogInformation( + "SignalR messages healthy: {Observed}/{Limit} today ({Percent}%)", + observedLong, limit, percent); + break; + } + } + + private async Task> BuildSignalRQuotaSeenIndexAsync() + { + var todayUtc = DateTime.UtcNow.Date; + var todaysSecurityEvents = await _opsEventRepo.GetOpsEventsAsync( + category: OpsEventCategory.Security, + dateFrom: todayUtc, + dateTo: todayUtc.AddDays(1)); + + return todaysSecurityEvents + .Where(e => e.EventType.StartsWith(SignalRQuotaEventTypePrefix, StringComparison.Ordinal)) + .Select(e => e.EventType) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + } + + private int ParseIntSetting(string key, int fallback) + { + var raw = _configuration[key]; + return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback; + } + + private long ParseLongSetting(string key, long fallback) + { + var raw = _configuration[key]; + return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback; + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index c380b3ff..d01ac73c 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -5,10 +5,12 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using AutopilotMonitor.Functions.Services.Monitoring; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -53,6 +55,8 @@ namespace AutopilotMonitor.Functions.Services private readonly IDistressReportRepository _distressReportRepo; private readonly IOpsEventRepository _opsEventRepo; private readonly OpsEventService _opsEventService; + private readonly IAzureMonitorMetricsReader _metricsReader; + private readonly IConfiguration _configuration; private readonly IHttpClientFactory _httpClientFactory; private readonly ILogger _logger; @@ -73,6 +77,8 @@ namespace AutopilotMonitor.Functions.Services IDistressReportRepository distressReportRepo, IOpsEventRepository opsEventRepo, OpsEventService opsEventService, + IAzureMonitorMetricsReader metricsReader, + IConfiguration configuration, IHttpClientFactory httpClientFactory, ILogger logger) { @@ -88,6 +94,8 @@ namespace AutopilotMonitor.Functions.Services _distressReportRepo = distressReportRepo; _opsEventRepo = opsEventRepo; _opsEventService = opsEventService; + _metricsReader = metricsReader; + _configuration = configuration; _httpClientFactory = httpClientFactory; _logger = logger; } @@ -170,6 +178,11 @@ namespace AutopilotMonitor.Functions.Services // Mirror the timer path: check embedded Intune cert bundle for // expiring members so manual triggers also exercise the watcher. await CheckEmbeddedCertExpiryAsync(); + + // Same parity for SignalR quota - the dedicated 1h timer is the + // primary cadence, but operators triggering maintenance manually + // expect every health/quota check to run. + await CheckSignalRQuotaAsync(); } await RecomputePlatformStatsAsync(); diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/AzureMonitorMetricsReader.cs b/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/AzureMonitorMetricsReader.cs new file mode 100644 index 00000000..08eea4c9 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/AzureMonitorMetricsReader.cs @@ -0,0 +1,101 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Identity; +using Azure.Monitor.Query; +using Azure.Monitor.Query.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services.Monitoring +{ + /// + /// Default backed by Azure.Monitor.Query. + /// Authenticates via — in production this + /// resolves to the Function App's managed identity, which must be granted + /// "Monitoring Reader" on the queried resource. + /// + public sealed class AzureMonitorMetricsReader : IAzureMonitorMetricsReader + { + private readonly MetricsQueryClient _client; + private readonly ILogger _logger; + + public AzureMonitorMetricsReader(ILogger logger) + { + _client = new MetricsQueryClient(new DefaultAzureCredential()); + _logger = logger; + } + + public async Task GetMaximumAsync( + string resourceId, string metricName, TimeSpan window, CancellationToken ct) + { + try + { + var response = await _client.QueryResourceAsync( + resourceId, + new[] { metricName }, + new MetricsQueryOptions + { + TimeRange = new QueryTimeRange(window), + Aggregations = { MetricAggregationType.Maximum }, + }, + ct); + + var metric = response.Value.Metrics.FirstOrDefault(); + if (metric is null) return null; + + var values = metric.TimeSeries + .SelectMany(t => t.Values) + .Select(v => v.Maximum) + .Where(v => v.HasValue) + .Select(v => v!.Value) + .ToList(); + + return values.Count == 0 ? (double?)null : values.Max(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Azure Monitor query failed for {Metric} on {ResourceId} (window {Window})", + metricName, resourceId, window); + return null; + } + } + + public async Task GetTotalAsync( + string resourceId, string metricName, DateTimeOffset from, DateTimeOffset to, CancellationToken ct) + { + try + { + var response = await _client.QueryResourceAsync( + resourceId, + new[] { metricName }, + new MetricsQueryOptions + { + TimeRange = new QueryTimeRange(from, to), + Aggregations = { MetricAggregationType.Total }, + }, + ct); + + var metric = response.Value.Metrics.FirstOrDefault(); + if (metric is null) return null; + + var values = metric.TimeSeries + .SelectMany(t => t.Values) + .Select(v => v.Total) + .Where(v => v.HasValue) + .Select(v => v!.Value) + .ToList(); + + return values.Count == 0 ? (double?)null : values.Sum(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Azure Monitor query failed for {Metric} on {ResourceId} (range {From}..{To})", + metricName, resourceId, from, to); + return null; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/IAzureMonitorMetricsReader.cs b/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/IAzureMonitorMetricsReader.cs new file mode 100644 index 00000000..35c57f1c --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/Monitoring/IAzureMonitorMetricsReader.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace AutopilotMonitor.Functions.Services.Monitoring +{ + /// + /// Thin abstraction over Azure Monitor metrics so watchers can be unit-tested + /// without spinning up a real MetricsQueryClient. Reads are scoped to a + /// single Azure resource (e.g. an Azure SignalR Service instance). + /// + public interface IAzureMonitorMetricsReader + { + /// + /// Returns the maximum value observed for on + /// within the trailing + /// (now − window … now). Null when no data points were returned. + /// + Task GetMaximumAsync( + string resourceId, string metricName, TimeSpan window, CancellationToken ct); + + /// + /// Returns the total sum of on + /// across the inclusive UTC range + /// [, ]. Null when no data + /// points were returned. + /// + Task GetTotalAsync( + string resourceId, string metricName, DateTimeOffset from, DateTimeOffset to, CancellationToken ct); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 0d2db78f..6ba18239 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -112,6 +112,30 @@ namespace AutopilotMonitor.Functions.Services "No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients", null, "System.Maintenance", new { }); + public Task RecordSignalRConnectionsHighAsync(int observed, int limit, int percent, string resourceId) + => WriteAsync(OpsEventCategory.Security, "SignalRConnectionsHigh", OpsEventSeverity.Warning, + $"SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - watch for 429s; consider scaling to Standard before saturation", + null, "System.Monitoring", + new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId }); + + public Task RecordSignalRConnectionsCriticalAsync(int observed, int limit, int percent, string resourceId) + => WriteAsync(OpsEventCategory.Security, "SignalRConnectionsCritical", OpsEventSeverity.Error, + $"CRITICAL: SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - new client connections will be 429'd at 100%; scale to Standard now", + null, "System.Monitoring", + new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId }); + + public Task RecordSignalRMessagesHighAsync(long observed, long limit, int percent, string resourceId) + => WriteAsync(OpsEventCategory.Security, "SignalRMessagesHigh", OpsEventSeverity.Warning, + $"SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - quota resets at 00:00 UTC; consider scaling if pattern persists", + null, "System.Monitoring", + new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId }); + + public Task RecordSignalRMessagesCriticalAsync(long observed, long limit, int percent, string resourceId) + => WriteAsync(OpsEventCategory.Security, "SignalRMessagesCritical", OpsEventSeverity.Error, + $"CRITICAL: SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - hub will throttle at 100% until 00:00 UTC reset; scale to Standard now", + null, "System.Monitoring", + new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId }); + // ── Tenant ───────────────────────────────────────────────────────────── public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary deletedCounts) diff --git a/src/Backend/AutopilotMonitor.Functions/local.settings.json.template b/src/Backend/AutopilotMonitor.Functions/local.settings.json.template index b90bcfc5..e9d2c895 100644 --- a/src/Backend/AutopilotMonitor.Functions/local.settings.json.template +++ b/src/Backend/AutopilotMonitor.Functions/local.settings.json.template @@ -4,6 +4,11 @@ "AzureWebJobsStorage": "UseDevelopmentStorage=true", "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated", "AzureTableStorageConnectionString": "UseDevelopmentStorage=true", - "AzureBlobStorageConnectionString": "UseDevelopmentStorage=true" + "AzureBlobStorageConnectionString": "UseDevelopmentStorage=true", + + "// SignalR free-tier quota watcher": "Optional. Set to the full Azure resource id of the SignalR Service to enable hourly OpsEvents when the 20-connections / 20k-messages-per-day free-tier limits are approached. Function App's managed identity needs 'Monitoring Reader' on this resource. Leave unset to skip the watcher.", + "SignalRResourceId": "", + "SignalRFreeTierConnectionLimit": "20", + "SignalRFreeTierDailyMessageLimit": "20000" } }