feat(backend/observability): SignalR free-quota watcher with tiered OpsEvents

Hourly TimerFunction queries Azure Monitor for SignalR ConnectionCount (max over last hour) and MessageCount (sum since UTC midnight) and emits Warning (>=80%) / Error (>=95%) OpsEvents when the free-tier limits (20 connections, 20k messages/day) are approached. Mirror of the cert-expiry pattern - global TenantId=null events, dedup'd to one emission per EventType per UTC day. Fail-soft: skips silently when SignalRResourceId app setting is unset, logs warning on Azure Monitor auth/query errors. Function App's managed identity needs 'Monitoring Reader' on the SignalR resource. Also called from the manual maintenance trigger so 'Run Maintenance Now' exercises every health check. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-10-24 04:32:32 +00:00 · 2026-05-08 18:06:56 +02:00
parent 5a4309dc9a
commit bc3cefa790
10 changed files with 612 additions and 1 deletions
@@ -0,0 +1,180 @@
+using AutopilotMonitor.Functions.Services;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+/// <summary>
+/// Tier-boundary + percent-math tests for the SignalR free-tier quota watcher.
+/// Mirrors the cert-expiry tests' approach: pure-function coverage of the
+/// boundary risks (off-by-one, floor vs ceil, zero-division). Full-service
+/// smoke tests would require constructing 16 dependencies; the percent +
+/// classifier are the parts most likely to break the watcher silently.
+/// </summary>
+public class MaintenanceServiceSignalRQuotaTests
+{
+    // --- ClassifySignalRQuotaTier ---
+    // Thresholds (per MaintenanceService.SignalRQuota.cs):
+    //   >= 95% -> Critical
+    //   >= 80% -> Warning
+    //   else   -> None
+
+    [Theory]
+    [InlineData(0)]
+    [InlineData(50)]
+    [InlineData(79)]
+    public void ClassifySignalRQuotaTier_BelowWarning_ReturnsNone(int percent)
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.None,
+            MaintenanceService.ClassifySignalRQuotaTier(percent));
+    }
+
+    [Theory]
+    [InlineData(80)]
+    [InlineData(85)]
+    [InlineData(94)]
+    public void ClassifySignalRQuotaTier_InWarningBand_ReturnsWarning(int percent)
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.Warning,
+            MaintenanceService.ClassifySignalRQuotaTier(percent));
+    }
+
+    [Theory]
+    [InlineData(95)]
+    [InlineData(99)]
+    [InlineData(100)]
+    [InlineData(120)]
+    public void ClassifySignalRQuotaTier_AtOrAboveCritical_ReturnsCritical(int percent)
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.Critical,
+            MaintenanceService.ClassifySignalRQuotaTier(percent));
+    }
+
+    [Fact]
+    public void ClassifySignalRQuotaTier_BoundaryAt79_IsBelowWarning()
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.None,
+            MaintenanceService.ClassifySignalRQuotaTier(79));
+    }
+
+    [Fact]
+    public void ClassifySignalRQuotaTier_BoundaryAt80_IsWarningNotNone()
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.Warning,
+            MaintenanceService.ClassifySignalRQuotaTier(80));
+    }
+
+    [Fact]
+    public void ClassifySignalRQuotaTier_BoundaryAt94_IsWarningNotCritical()
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.Warning,
+            MaintenanceService.ClassifySignalRQuotaTier(94));
+    }
+
+    [Fact]
+    public void ClassifySignalRQuotaTier_BoundaryAt95_IsCriticalNotWarning()
+    {
+        Assert.Equal(
+            MaintenanceService.SignalRQuotaTier.Critical,
+            MaintenanceService.ClassifySignalRQuotaTier(95));
+    }
+
+    // --- CalculatePercent ---
+    // Floors to integer to match what an operator sees on the dashboard.
+    // Critical contract: 79.99% must NOT round up to 80% (warning trigger).
+
+    [Theory]
+    [InlineData(16, 20, 80)]   // exact 80%
+    [InlineData(19, 20, 95)]   // exact 95%
+    [InlineData(20, 20, 100)]
+    [InlineData(0, 20, 0)]
+    [InlineData(10, 20, 50)]
+    public void CalculatePercent_ExactValues_RoundsCorrectly(double observed, double limit, int expected)
+    {
+        Assert.Equal(expected, MaintenanceService.CalculatePercent(observed, limit));
+    }
+
+    [Fact]
+    public void CalculatePercent_JustUnderWarning_FloorsTo79()
+    {
+        // 15.99 / 20 = 79.95% -> floor -> 79 -> stays None tier
+        Assert.Equal(79, MaintenanceService.CalculatePercent(15.99, 20));
+    }
+
+    [Fact]
+    public void CalculatePercent_JustUnderCritical_FloorsTo94()
+    {
+        // 18.99 / 20 = 94.95% -> floor -> 94 -> stays Warning tier
+        Assert.Equal(94, MaintenanceService.CalculatePercent(18.99, 20));
+    }
+
+    [Fact]
+    public void CalculatePercent_ZeroLimit_ReturnsZero()
+    {
+        // Defensive: misconfigured limit must not divide by zero or NaN-poison
+        // the tier classification.
+        Assert.Equal(0, MaintenanceService.CalculatePercent(50, 0));
+    }
+
+    [Fact]
+    public void CalculatePercent_NegativeLimit_ReturnsZero()
+    {
+        Assert.Equal(0, MaintenanceService.CalculatePercent(50, -1));
+    }
+
+    [Fact]
+    public void CalculatePercent_OverLimit_ReturnsAbove100()
+    {
+        // Over-quota must surface as >100% so Critical fires, not silently cap.
+        Assert.Equal(150, MaintenanceService.CalculatePercent(30, 20));
+    }
+
+    [Theory]
+    [InlineData(16000, 20000, 80)]
+    [InlineData(19000, 20000, 95)]
+    [InlineData(15999, 20000, 79)]
+    [InlineData(20000, 20000, 100)]
+    public void CalculatePercent_MessageScale_RoundsCorrectly(long observed, long limit, int expected)
+    {
+        // Same math but at the message-quota scale (20k cap, not 20).
+        // Verifies the double-precision path doesn't drift on larger numbers.
+        Assert.Equal(expected, MaintenanceService.CalculatePercent(observed, limit));
+    }
+
+    // --- End-to-end tier mapping (the user-visible behavior) ---
+    // Combines CalculatePercent + ClassifySignalRQuotaTier the way the watcher
+    // actually uses them. Catches drift if either function changes
+    // independently of the other.
+
+    [Theory]
+    [InlineData(15, 20, "None")]
+    [InlineData(16, 20, "Warning")]
+    [InlineData(18, 20, "Warning")]
+    [InlineData(19, 20, "Critical")]
+    [InlineData(20, 20, "Critical")]
+    [InlineData(25, 20, "Critical")]
+    public void ConnectionScenarios_MapToCorrectTier(int observed, int limit, string expectedTier)
+    {
+        var percent = MaintenanceService.CalculatePercent(observed, limit);
+        var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
+        Assert.Equal(expectedTier, tier.ToString());
+    }
+
+    [Theory]
+    [InlineData(15999, 20000, "None")]
+    [InlineData(16000, 20000, "Warning")]
+    [InlineData(18999, 20000, "Warning")]
+    [InlineData(19000, 20000, "Critical")]
+    [InlineData(20000, 20000, "Critical")]
+    [InlineData(25000, 20000, "Critical")]
+    public void MessageScenarios_MapToCorrectTier(long observed, long limit, string expectedTier)
+    {
+        var percent = MaintenanceService.CalculatePercent(observed, limit);
+        var tier = MaintenanceService.ClassifySignalRQuotaTier(percent);
+        Assert.Equal(expectedTier, tier.ToString());
+    }
+}
@@ -29,6 +29,7 @@
  <ItemGroup>
    <FrameworkReference Include="Microsoft.AspNetCore.App" />
    <PackageReference Include="Azure.Identity" Version="1.21.0" />
+    <PackageReference Include="Azure.Monitor.Query" Version="1.5.0" />
    <PackageReference Include="Microsoft.ApplicationInsights.WorkerService" Version="2.23.0" />
    <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="2.51.0" />
    <PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.50.0" />
@@ -0,0 +1,45 @@
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Functions.Admin
+{
+    /// <summary>
+    /// Timer-triggered watcher for the Azure SignalR Free-tier limits (20
+    /// concurrent connections / 20,000 messages per UTC day). Runs every full
+    /// hour and emits OpsEvents when the configured thresholds are crossed so
+    /// operators get a Telegram ping before clients start getting 429'd.
+    ///
+    /// Separate from the 2h MaintenanceFunction: the SignalR limits can be
+    /// breached by a brief connection spike, so the cadence is tighter. Cost
+    /// impact is negligible (~720 extra executions/month, ~$0.04).
+    /// </summary>
+    public class SignalRQuotaTimerFunction
+    {
+        private readonly MaintenanceService _maintenanceService;
+        private readonly ILogger<SignalRQuotaTimerFunction> _logger;
+
+        public SignalRQuotaTimerFunction(
+            MaintenanceService maintenanceService,
+            ILogger<SignalRQuotaTimerFunction> logger)
+        {
+            _maintenanceService = maintenanceService;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// NCRONTAB "0 0 * * * *" - every hour on the minute. The watcher itself
+        /// is idempotent (one OpsEvent per EventType per UTC day) and skips
+        /// silently when SignalRResourceId is unset, so this trigger is safe to
+        /// run in environments without managed-identity wiring.
+        /// </summary>
+        [Function("SignalRQuotaTimer")]
+        public async Task Run([TimerTrigger("0 0 * * * *")] object timer, CancellationToken ct)
+        {
+            _logger.LogInformation("SignalR quota watcher fired");
+            await _maintenanceService.CheckSignalRQuotaAsync(ct);
+        }
+    }
+}
@@ -154,6 +154,9 @@ builder.Services.AddSingleton<ImeLogPatternService>();
 builder.Services.AddHttpClient<GitHubRuleRepository>()
    .AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
 builder.Services.AddSingleton<MaintenanceService>();
+builder.Services.AddSingleton<
+    AutopilotMonitor.Functions.Services.Monitoring.IAzureMonitorMetricsReader,
+    AutopilotMonitor.Functions.Services.Monitoring.AzureMonitorMetricsReader>();
 builder.Services.AddSingleton<OpsAlertDispatchService>();
 builder.Services.AddSingleton<OpsEventService>();
 builder.Services.AddSingleton<BlockedDeviceService>();
@@ -0,0 +1,208 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using AutopilotMonitor.Shared.DataAccess;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Services
+{
+    /// <summary>
+    /// SignalR Free-tier quota watcher. Reads ConnectionCount (max over the last
+    /// hour) and MessageCount (total since UTC midnight) from Azure Monitor and
+    /// emits tiered OpsEvents when either approaches the documented Free-tier
+    /// caps - 20 concurrent connections and 20,000 messages per day. The
+    /// operator can then scale to Standard before clients get 429'd.
+    ///
+    /// Globally scoped (TenantId = null) - SignalR is one shared resource for
+    /// the whole Function App, not a per-tenant concern.
+    /// </summary>
+    public partial class MaintenanceService
+    {
+        // Tier thresholds, expressed as percent of the configured cap.
+        // Order matters: most-severe first.
+        internal const int SignalRQuotaCriticalPercent = 95;
+        internal const int SignalRQuotaWarningPercent = 80;
+
+        // Default Free-tier caps as documented by Microsoft. Overridable via
+        // app settings so an operator who has scaled to a paid tier can re-target
+        // the watcher without a code change (or disable it by setting the cap
+        // very high).
+        internal const int DefaultSignalRConnectionLimit = 20;
+        internal const long DefaultSignalRDailyMessageLimit = 20_000;
+
+        // Dedup prefix - all SignalR EventTypes start with this, so the seen-index
+        // can be built with a single Security-category read.
+        internal const string SignalRQuotaEventTypePrefix = "SignalR";
+
+        internal enum SignalRQuotaTier { None, Warning, Critical }
+
+        /// <summary>
+        /// Pure tier classifier - extracted so the boundary math is unit-testable
+        /// without constructing a full MaintenanceService.
+        /// </summary>
+        internal static SignalRQuotaTier ClassifySignalRQuotaTier(int percent)
+        {
+            if (percent >= SignalRQuotaCriticalPercent) return SignalRQuotaTier.Critical;
+            if (percent >= SignalRQuotaWarningPercent)  return SignalRQuotaTier.Warning;
+            return SignalRQuotaTier.None;
+        }
+
+        internal static int CalculatePercent(double observed, double limit)
+        {
+            if (limit <= 0) return 0;
+            // Floor to avoid 79.6% triggering an 80% warning. Matches what the
+            // operator sees on the Azure dashboard, which also rounds down.
+            return (int)Math.Floor(observed / limit * 100.0);
+        }
+
+        /// <summary>
+        /// Reads SignalR ConnectionCount + MessageCount metrics from Azure Monitor
+        /// and emits Warning/Error OpsEvents when the Free-tier limits are
+        /// approached. Dedup: one event per EventType per UTC day. Fail-soft -
+        /// missing config or Azure Monitor errors yield a single log line, never
+        /// a thrown exception.
+        /// </summary>
+        public async Task CheckSignalRQuotaAsync(CancellationToken ct = default)
+        {
+            var resourceId = _configuration["SignalRResourceId"];
+            if (string.IsNullOrWhiteSpace(resourceId))
+            {
+                _logger.LogInformation(
+                    "SignalR quota watcher skipped: SignalRResourceId app setting not configured");
+                return;
+            }
+
+            var connectionLimit = ParseIntSetting(
+                "SignalRFreeTierConnectionLimit", DefaultSignalRConnectionLimit);
+            var messageLimit = ParseLongSetting(
+                "SignalRFreeTierDailyMessageLimit", DefaultSignalRDailyMessageLimit);
+
+            var seenToday = await BuildSignalRQuotaSeenIndexAsync();
+
+            await CheckConnectionsAsync(resourceId, connectionLimit, seenToday, ct);
+            await CheckMessagesAsync(resourceId, messageLimit, seenToday, ct);
+        }
+
+        private async Task CheckConnectionsAsync(
+            string resourceId, int limit, HashSet<string> seenToday, CancellationToken ct)
+        {
+            var observed = await _metricsReader.GetMaximumAsync(
+                resourceId, "ConnectionCount", TimeSpan.FromHours(1), ct);
+            if (observed is null)
+            {
+                _logger.LogDebug("SignalR ConnectionCount metric returned no data points");
+                return;
+            }
+
+            var observedInt = (int)Math.Ceiling(observed.Value);
+            var percent = CalculatePercent(observedInt, limit);
+            var tier = ClassifySignalRQuotaTier(percent);
+
+            switch (tier)
+            {
+                case SignalRQuotaTier.Critical:
+                    if (seenToday.Add("SignalRConnectionsCritical"))
+                    {
+                        await _opsEventService.RecordSignalRConnectionsCriticalAsync(
+                            observedInt, limit, percent, resourceId);
+                        _logger.LogError(
+                            "SignalR Critical: {Observed}/{Limit} connections ({Percent}%)",
+                            observedInt, limit, percent);
+                    }
+                    break;
+                case SignalRQuotaTier.Warning:
+                    if (seenToday.Add("SignalRConnectionsHigh"))
+                    {
+                        await _opsEventService.RecordSignalRConnectionsHighAsync(
+                            observedInt, limit, percent, resourceId);
+                        _logger.LogWarning(
+                            "SignalR Warning: {Observed}/{Limit} connections ({Percent}%)",
+                            observedInt, limit, percent);
+                    }
+                    break;
+                default:
+                    _logger.LogInformation(
+                        "SignalR connections healthy: {Observed}/{Limit} ({Percent}%)",
+                        observedInt, limit, percent);
+                    break;
+            }
+        }
+
+        private async Task CheckMessagesAsync(
+            string resourceId, long limit, HashSet<string> seenToday, CancellationToken ct)
+        {
+            // Quota is daily and resets at 00:00 UTC, so query that exact window.
+            var nowUtc = DateTimeOffset.UtcNow;
+            var startOfDayUtc = new DateTimeOffset(nowUtc.UtcDateTime.Date, TimeSpan.Zero);
+
+            var observed = await _metricsReader.GetTotalAsync(
+                resourceId, "MessageCount", startOfDayUtc, nowUtc, ct);
+            if (observed is null)
+            {
+                _logger.LogDebug("SignalR MessageCount metric returned no data points");
+                return;
+            }
+
+            var observedLong = (long)Math.Ceiling(observed.Value);
+            var percent = CalculatePercent(observedLong, limit);
+            var tier = ClassifySignalRQuotaTier(percent);
+
+            switch (tier)
+            {
+                case SignalRQuotaTier.Critical:
+                    if (seenToday.Add("SignalRMessagesCritical"))
+                    {
+                        await _opsEventService.RecordSignalRMessagesCriticalAsync(
+                            observedLong, limit, percent, resourceId);
+                        _logger.LogError(
+                            "SignalR Critical: {Observed}/{Limit} messages today ({Percent}%)",
+                            observedLong, limit, percent);
+                    }
+                    break;
+                case SignalRQuotaTier.Warning:
+                    if (seenToday.Add("SignalRMessagesHigh"))
+                    {
+                        await _opsEventService.RecordSignalRMessagesHighAsync(
+                            observedLong, limit, percent, resourceId);
+                        _logger.LogWarning(
+                            "SignalR Warning: {Observed}/{Limit} messages today ({Percent}%)",
+                            observedLong, limit, percent);
+                    }
+                    break;
+                default:
+                    _logger.LogInformation(
+                        "SignalR messages healthy: {Observed}/{Limit} today ({Percent}%)",
+                        observedLong, limit, percent);
+                    break;
+            }
+        }
+
+        private async Task<HashSet<string>> BuildSignalRQuotaSeenIndexAsync()
+        {
+            var todayUtc = DateTime.UtcNow.Date;
+            var todaysSecurityEvents = await _opsEventRepo.GetOpsEventsAsync(
+                category: OpsEventCategory.Security,
+                dateFrom: todayUtc,
+                dateTo: todayUtc.AddDays(1));
+
+            return todaysSecurityEvents
+                .Where(e => e.EventType.StartsWith(SignalRQuotaEventTypePrefix, StringComparison.Ordinal))
+                .Select(e => e.EventType)
+                .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        }
+
+        private int ParseIntSetting(string key, int fallback)
+        {
+            var raw = _configuration[key];
+            return int.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
+        }
+
+        private long ParseLongSetting(string key, long fallback)
+        {
+            var raw = _configuration[key];
+            return long.TryParse(raw, out var parsed) && parsed > 0 ? parsed : fallback;
+        }
+    }
+}
@@ -5,10 +5,12 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Services.Monitoring;
 using AutopilotMonitor.Shared.DataAccess;
 using AutopilotMonitor.Shared.Models;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
+using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;

@@ -53,6 +55,8 @@ namespace AutopilotMonitor.Functions.Services
        private readonly IDistressReportRepository _distressReportRepo;
        private readonly IOpsEventRepository _opsEventRepo;
        private readonly OpsEventService _opsEventService;
+        private readonly IAzureMonitorMetricsReader _metricsReader;
+        private readonly IConfiguration _configuration;
        private readonly IHttpClientFactory _httpClientFactory;
        private readonly ILogger<MaintenanceService> _logger;

@@ -73,6 +77,8 @@ namespace AutopilotMonitor.Functions.Services
            IDistressReportRepository distressReportRepo,
            IOpsEventRepository opsEventRepo,
            OpsEventService opsEventService,
+            IAzureMonitorMetricsReader metricsReader,
+            IConfiguration configuration,
            IHttpClientFactory httpClientFactory,
            ILogger<MaintenanceService> logger)
        {
@@ -88,6 +94,8 @@ namespace AutopilotMonitor.Functions.Services
            _distressReportRepo = distressReportRepo;
            _opsEventRepo = opsEventRepo;
            _opsEventService = opsEventService;
+            _metricsReader = metricsReader;
+            _configuration = configuration;
            _httpClientFactory = httpClientFactory;
            _logger = logger;
        }
@@ -170,6 +178,11 @@ namespace AutopilotMonitor.Functions.Services
                    // Mirror the timer path: check embedded Intune cert bundle for
                    // expiring members so manual triggers also exercise the watcher.
                    await CheckEmbeddedCertExpiryAsync();
+
+                    // Same parity for SignalR quota - the dedicated 1h timer is the
+                    // primary cadence, but operators triggering maintenance manually
+                    // expect every health/quota check to run.
+                    await CheckSignalRQuotaAsync();
                }

                await RecomputePlatformStatsAsync();
@@ -0,0 +1,101 @@
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure.Identity;
+using Azure.Monitor.Query;
+using Azure.Monitor.Query.Models;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Services.Monitoring
+{
+    /// <summary>
+    /// Default <see cref="IAzureMonitorMetricsReader"/> backed by Azure.Monitor.Query.
+    /// Authenticates via <see cref="DefaultAzureCredential"/> — in production this
+    /// resolves to the Function App's managed identity, which must be granted
+    /// "Monitoring Reader" on the queried resource.
+    /// </summary>
+    public sealed class AzureMonitorMetricsReader : IAzureMonitorMetricsReader
+    {
+        private readonly MetricsQueryClient _client;
+        private readonly ILogger<AzureMonitorMetricsReader> _logger;
+
+        public AzureMonitorMetricsReader(ILogger<AzureMonitorMetricsReader> logger)
+        {
+            _client = new MetricsQueryClient(new DefaultAzureCredential());
+            _logger = logger;
+        }
+
+        public async Task<double?> GetMaximumAsync(
+            string resourceId, string metricName, TimeSpan window, CancellationToken ct)
+        {
+            try
+            {
+                var response = await _client.QueryResourceAsync(
+                    resourceId,
+                    new[] { metricName },
+                    new MetricsQueryOptions
+                    {
+                        TimeRange = new QueryTimeRange(window),
+                        Aggregations = { MetricAggregationType.Maximum },
+                    },
+                    ct);
+
+                var metric = response.Value.Metrics.FirstOrDefault();
+                if (metric is null) return null;
+
+                var values = metric.TimeSeries
+                    .SelectMany(t => t.Values)
+                    .Select(v => v.Maximum)
+                    .Where(v => v.HasValue)
+                    .Select(v => v!.Value)
+                    .ToList();
+
+                return values.Count == 0 ? (double?)null : values.Max();
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex,
+                    "Azure Monitor query failed for {Metric} on {ResourceId} (window {Window})",
+                    metricName, resourceId, window);
+                return null;
+            }
+        }
+
+        public async Task<double?> GetTotalAsync(
+            string resourceId, string metricName, DateTimeOffset from, DateTimeOffset to, CancellationToken ct)
+        {
+            try
+            {
+                var response = await _client.QueryResourceAsync(
+                    resourceId,
+                    new[] { metricName },
+                    new MetricsQueryOptions
+                    {
+                        TimeRange = new QueryTimeRange(from, to),
+                        Aggregations = { MetricAggregationType.Total },
+                    },
+                    ct);
+
+                var metric = response.Value.Metrics.FirstOrDefault();
+                if (metric is null) return null;
+
+                var values = metric.TimeSeries
+                    .SelectMany(t => t.Values)
+                    .Select(v => v.Total)
+                    .Where(v => v.HasValue)
+                    .Select(v => v!.Value)
+                    .ToList();
+
+                return values.Count == 0 ? (double?)null : values.Sum();
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning(ex,
+                    "Azure Monitor query failed for {Metric} on {ResourceId} (range {From}..{To})",
+                    metricName, resourceId, from, to);
+                return null;
+            }
+        }
+    }
+}
@@ -0,0 +1,31 @@
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace AutopilotMonitor.Functions.Services.Monitoring
+{
+    /// <summary>
+    /// Thin abstraction over Azure Monitor metrics so watchers can be unit-tested
+    /// without spinning up a real <c>MetricsQueryClient</c>. Reads are scoped to a
+    /// single Azure resource (e.g. an Azure SignalR Service instance).
+    /// </summary>
+    public interface IAzureMonitorMetricsReader
+    {
+        /// <summary>
+        /// Returns the maximum value observed for <paramref name="metricName"/> on
+        /// <paramref name="resourceId"/> within the trailing <paramref name="window"/>
+        /// (now − window … now). Null when no data points were returned.
+        /// </summary>
+        Task<double?> GetMaximumAsync(
+            string resourceId, string metricName, TimeSpan window, CancellationToken ct);
+
+        /// <summary>
+        /// Returns the total sum of <paramref name="metricName"/> on
+        /// <paramref name="resourceId"/> across the inclusive UTC range
+        /// [<paramref name="from"/>, <paramref name="to"/>]. Null when no data
+        /// points were returned.
+        /// </summary>
+        Task<double?> GetTotalAsync(
+            string resourceId, string metricName, DateTimeOffset from, DateTimeOffset to, CancellationToken ct);
+    }
+}
@@ -112,6 +112,30 @@ namespace AutopilotMonitor.Functions.Services
                "No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients",
                null, "System.Maintenance", new { });

+        public Task RecordSignalRConnectionsHighAsync(int observed, int limit, int percent, string resourceId)
+            => WriteAsync(OpsEventCategory.Security, "SignalRConnectionsHigh", OpsEventSeverity.Warning,
+                $"SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - watch for 429s; consider scaling to Standard before saturation",
+                null, "System.Monitoring",
+                new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
+
+        public Task RecordSignalRConnectionsCriticalAsync(int observed, int limit, int percent, string resourceId)
+            => WriteAsync(OpsEventCategory.Security, "SignalRConnectionsCritical", OpsEventSeverity.Error,
+                $"CRITICAL: SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - new client connections will be 429'd at 100%; scale to Standard now",
+                null, "System.Monitoring",
+                new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
+
+        public Task RecordSignalRMessagesHighAsync(long observed, long limit, int percent, string resourceId)
+            => WriteAsync(OpsEventCategory.Security, "SignalRMessagesHigh", OpsEventSeverity.Warning,
+                $"SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - quota resets at 00:00 UTC; consider scaling if pattern persists",
+                null, "System.Monitoring",
+                new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
+
+        public Task RecordSignalRMessagesCriticalAsync(long observed, long limit, int percent, string resourceId)
+            => WriteAsync(OpsEventCategory.Security, "SignalRMessagesCritical", OpsEventSeverity.Error,
+                $"CRITICAL: SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - hub will throttle at 100% until 00:00 UTC reset; scale to Standard now",
+                null, "System.Monitoring",
+                new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
+
        // ── Tenant ─────────────────────────────────────────────────────────────

        public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary<string, int> deletedCounts)
@@ -4,6 +4,11 @@
    "AzureWebJobsStorage": "UseDevelopmentStorage=true",
    "FUNCTIONS_WORKER_RUNTIME": "dotnet-isolated",
    "AzureTableStorageConnectionString": "UseDevelopmentStorage=true",
-    "AzureBlobStorageConnectionString": "UseDevelopmentStorage=true"
+    "AzureBlobStorageConnectionString": "UseDevelopmentStorage=true",
+
+    "// SignalR free-tier quota watcher": "Optional. Set to the full Azure resource id of the SignalR Service to enable hourly OpsEvents when the 20-connections / 20k-messages-per-day free-tier limits are approached. Function App's managed identity needs 'Monitoring Reader' on this resource. Leave unset to skip the watcher.",
+    "SignalRResourceId": "",
+    "SignalRFreeTierConnectionLimit": "20",
+    "SignalRFreeTierDailyMessageLimit": "20000"
  }
 }