feat(backend/security): cert-expiry watcher in maintenance with tiered OpsEvents

Adds proactive monitoring of the embedded Intune CA bundle so the operator
gets a Telegram ping months before agent mTLS would break. Bundle rotation
(re-exporting fresh PEMs from a corporate-enrolled device) is the only fix
and needs lead time - today's outage from 9af3de15 was rooted in exactly
this blind spot.

Hooks into the existing 2h MaintenanceService.RunAllAsync timer, iterates
the embedded roots + intermediates, and emits tiered OpsEvents on the
Security category:

  daysUntilExpiry  | Severity | EventType
  -----------------+----------+----------------------------
   <= 7 (or past)  | Critical | EmbeddedCertExpired
   <= 30           | Error    | EmbeddedCertExpiringUrgent
   <= 90           | Warning  | EmbeddedCertExpiringSoon
   else            | -        | (silent)

Plus a Critical EmbeddedCertBundleEmpty event if the loader returns no
roots at all (server misconfiguration / failed deploy).

Dedup is one event per cert per tier per UTC day, by querying today's
Security OpsEvents and matching on EventType + thumbprint extracted from
the Details JSON. Cold starts (Flex Consumption scale-to-zero) rebuild
the dedup index from the table, so repeated process restarts do not
multiply Telegram pings.

Operator action required to receive Telegram alerts: configure four
OpsAlertRules in admin UI - one per new EventType - with appropriate
MinSeverity. The OpsEvents themselves land in the table regardless,
visible in the Ops dashboard.

Tests: 23 new (tier-classifier boundary + dedup-key extraction + JSON
edge cases). Full suite 1098/1098 green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/Backend/AutopilotMonitor.Functions.Tests/MaintenanceServiceCertExpiryTests.cs

@@ -0,0 +1,118 @@
+using AutopilotMonitor.Functions.Services;
+
+namespace AutopilotMonitor.Functions.Tests;
+
+/// <summary>
+/// Tier-boundary + dedup-key tests for the embedded-cert expiry watcher.
+/// Full-service smoke tests would require constructing 14 dependencies; these
+/// pure-function tests cover the boundary risks (off-by-one, wrong tier) that
+/// are most likely to break the watcher silently.
+/// </summary>
+public class MaintenanceServiceCertExpiryTests
+{
+    // --- ClassifyCertExpiryTier ---
+    // Thresholds (per MaintenanceService.CertExpiry.cs):
+    //   <= 7d  -> EmbeddedCertExpired (Critical)
+    //   <= 30d -> EmbeddedCertExpiringUrgent (Error)
+    //   <= 90d -> EmbeddedCertExpiringSoon (Warning)
+    //   else   -> null (silent)
+
+    [Theory]
+    [InlineData(91, null)]
+    [InlineData(180, null)]
+    [InlineData(3650, null)]
+    public void ClassifyCertExpiryTier_FarFromExpiry_ReturnsNull(int days, string? expected)
+    {
+        Assert.Equal(expected, MaintenanceService.ClassifyCertExpiryTier(days));
+    }
+
+    [Theory]
+    [InlineData(90)]
+    [InlineData(60)]
+    [InlineData(31)]
+    public void ClassifyCertExpiryTier_WithinSoonWindow_ReturnsExpiringSoon(int days)
+    {
+        Assert.Equal("EmbeddedCertExpiringSoon", MaintenanceService.ClassifyCertExpiryTier(days));
+    }
+
+    [Theory]
+    [InlineData(30)]
+    [InlineData(15)]
+    [InlineData(8)]
+    public void ClassifyCertExpiryTier_WithinUrgentWindow_ReturnsExpiringUrgent(int days)
+    {
+        Assert.Equal("EmbeddedCertExpiringUrgent", MaintenanceService.ClassifyCertExpiryTier(days));
+    }
+
+    [Theory]
+    [InlineData(7)]
+    [InlineData(1)]
+    [InlineData(0)]
+    [InlineData(-1)]
+    [InlineData(-365)]
+    public void ClassifyCertExpiryTier_WithinCriticalWindowOrExpired_ReturnsExpired(int days)
+    {
+        Assert.Equal("EmbeddedCertExpired", MaintenanceService.ClassifyCertExpiryTier(days));
+    }
+
+    [Fact]
+    public void ClassifyCertExpiryTier_BoundaryAt91_IsAboveSoonTier()
+    {
+        Assert.Null(MaintenanceService.ClassifyCertExpiryTier(91));
+    }
+
+    [Fact]
+    public void ClassifyCertExpiryTier_BoundaryAt31_IsSoonNotUrgent()
+    {
+        Assert.Equal("EmbeddedCertExpiringSoon", MaintenanceService.ClassifyCertExpiryTier(31));
+    }
+
+    [Fact]
+    public void ClassifyCertExpiryTier_BoundaryAt8_IsUrgentNotExpired()
+    {
+        Assert.Equal("EmbeddedCertExpiringUrgent", MaintenanceService.ClassifyCertExpiryTier(8));
+    }
+
+    // --- ExtractThumbprint (used to build the dedup key from existing OpsEvents) ---
+
+    [Fact]
+    public void ExtractThumbprint_WithValidJson_ReturnsThumbprint()
+    {
+        var json = """{"role":"Root","subject":"CN=Foo","thumbprint":"ABC123","daysUntilExpiry":45}""";
+        Assert.Equal("ABC123", MaintenanceService.ExtractThumbprint(json));
+    }
+
+    [Fact]
+    public void ExtractThumbprint_WithMissingProperty_ReturnsEmpty()
+    {
+        var json = """{"role":"Root","subject":"CN=Foo"}""";
+        Assert.Equal("", MaintenanceService.ExtractThumbprint(json));
+    }
+
+    [Fact]
+    public void ExtractThumbprint_WithNull_ReturnsEmpty()
+    {
+        Assert.Equal("", MaintenanceService.ExtractThumbprint(null));
+    }
+
+    [Fact]
+    public void ExtractThumbprint_WithEmpty_ReturnsEmpty()
+    {
+        Assert.Equal("", MaintenanceService.ExtractThumbprint(""));
+    }
+
+    [Fact]
+    public void ExtractThumbprint_WithMalformedJson_ReturnsEmpty()
+    {
+        Assert.Equal("", MaintenanceService.ExtractThumbprint("not json {"));
+    }
+
+    [Fact]
+    public void ExtractThumbprint_WithNumericThumbprint_ReturnsEmpty()
+    {
+        // Defensive: real OpsEvent details always serialize thumbprint as string,
+        // but if a non-string value sneaks in we must not crash or mis-key.
+        var json = """{"thumbprint":12345}""";
+        Assert.Equal("", MaintenanceService.ExtractThumbprint(json));
+    }
+}

src/Backend/AutopilotMonitor.Functions/Security/CertificateValidator.cs

@@ -40,6 +40,20 @@ namespace AutopilotMonitor.Functions.Security
         private static readonly Lazy<X509Certificate2[]> _intuneIntermediateCerts =
             new(() => LoadIntuneCerts("intune-intermediate-"));
 
+        /// <summary>
+        /// Returns the embedded Intune root certificates loaded into the validator.
+        /// Used by maintenance jobs (e.g. cert-expiry monitor) that need to inspect
+        /// the bundle without re-loading PEM resources. Treat the returned array as
+        /// read-only; the chain validator caches it for process lifetime.
+        /// </summary>
+        internal static IReadOnlyList<X509Certificate2> GetEmbeddedRoots() => _intuneRootCerts.Value;
+
+        /// <summary>
+        /// Returns the embedded Intune intermediate certificates loaded into the validator.
+        /// See <see cref="GetEmbeddedRoots"/> for usage notes.
+        /// </summary>
+        internal static IReadOnlyList<X509Certificate2> GetEmbeddedIntermediates() => _intuneIntermediateCerts.Value;
+
         private class CachedValidationResult
         {
             public CertificateValidationResult Result { get; set; } = null!;

src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.CertExpiry.cs

@@ -0,0 +1,141 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text.Json;
+using System.Threading.Tasks;
+using AutopilotMonitor.Functions.Security;
+using AutopilotMonitor.Shared.DataAccess;
+using Microsoft.Extensions.Logging;
+
+namespace AutopilotMonitor.Functions.Services
+{
+    /// <summary>
+    /// Cert-expiry monitoring leg of MaintenanceService. Runs on the existing 2h
+    /// timer and emits tiered OpsEvents for embedded Intune CA certs nearing
+    /// NotAfter, so the operator gets a Telegram ping months before agent mTLS
+    /// would break - bundle rotation is the only fix and needs lead time to
+    /// source new PEMs from a corporate-enrolled device.
+    /// </summary>
+    public partial class MaintenanceService
+    {
+        // Tier thresholds, expressed as days remaining until NotAfter.
+        // Order matters: most-severe first.
+        internal const int CertExpiryCriticalThresholdDays = 7;
+        internal const int CertExpiryUrgentThresholdDays = 30;
+        internal const int CertExpirySoonThresholdDays = 90;
+
+        // Emit-once-per-UTC-day dedup. Runtime caches by (eventType, thumbprint)
+        // tuple from today's Security ops events. A cold start re-builds from the
+        // table, so Flex Consumption scale-to-zero does not multiply pings.
+        internal const string CertExpiryEventTypePrefix = "EmbeddedCert";
+
+        /// <summary>
+        /// Pure tier classifier - returns the OpsEvent EventType for a given
+        /// daysUntilExpiry, or null if the cert is comfortably far from expiry.
+        /// Extracted so the tier boundaries are unit-testable without
+        /// constructing a full MaintenanceService.
+        /// </summary>
+        internal static string? ClassifyCertExpiryTier(int daysUntilExpiry)
+        {
+            if (daysUntilExpiry <= CertExpiryCriticalThresholdDays) return "EmbeddedCertExpired";
+            if (daysUntilExpiry <= CertExpiryUrgentThresholdDays) return "EmbeddedCertExpiringUrgent";
+            if (daysUntilExpiry <= CertExpirySoonThresholdDays) return "EmbeddedCertExpiringSoon";
+            return null;
+        }
+
+        /// <summary>
+        /// Inspects the embedded Intune CA bundle (roots + intermediates) and
+        /// emits Warning/Error/Critical OpsEvents for any cert near expiry.
+        /// Dedup: one event per cert per tier per UTC day, by querying today's
+        /// Security ops events.
+        /// </summary>
+        public async Task CheckEmbeddedCertExpiryAsync()
+        {
+            var roots = CertificateValidator.GetEmbeddedRoots();
+            var intermediates = CertificateValidator.GetEmbeddedIntermediates();
+            var seenToday = await BuildCertExpirySeenIndexAsync();
+
+            if (roots.Count == 0)
+            {
+                if (seenToday.Add($"EmbeddedCertBundleEmpty|"))
+                    await _opsEventService.RecordEmbeddedCertBundleEmptyAsync();
+                _logger.LogError("Embedded Intune root bundle is empty - agent mTLS will fail closed");
+                return;
+            }
+
+            await EvaluateCertBucketAsync(roots, "Root", seenToday);
+            await EvaluateCertBucketAsync(intermediates, "Intermediate", seenToday);
+        }
+
+        private async Task EvaluateCertBucketAsync(
+            IReadOnlyList<X509Certificate2> certs, string role, HashSet<string> seenToday)
+        {
+            var now = DateTime.UtcNow;
+            foreach (var cert in certs)
+            {
+                var notAfterUtc = cert.NotAfter.ToUniversalTime();
+                var daysLeft = (int)Math.Floor((notAfterUtc - now).TotalDays);
+                var eventType = ClassifyCertExpiryTier(daysLeft);
+                if (eventType is null) continue;
+
+                var dedupKey = $"{eventType}|{cert.Thumbprint}";
+                if (!seenToday.Add(dedupKey))
+                {
+                    _logger.LogDebug("Cert expiry already emitted today: {Key}", dedupKey);
+                    continue;
+                }
+
+                switch (eventType)
+                {
+                    case "EmbeddedCertExpired":
+                        await _opsEventService.RecordEmbeddedCertExpiredAsync(
+                            role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft);
+                        break;
+                    case "EmbeddedCertExpiringUrgent":
+                        await _opsEventService.RecordEmbeddedCertExpiringUrgentAsync(
+                            role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft);
+                        break;
+                    case "EmbeddedCertExpiringSoon":
+                        await _opsEventService.RecordEmbeddedCertExpiringSoonAsync(
+                            role, cert.Subject, cert.Thumbprint, notAfterUtc, daysLeft);
+                        break;
+                }
+            }
+        }
+
+        private async Task<HashSet<string>> BuildCertExpirySeenIndexAsync()
+        {
+            var todayUtc = DateTime.UtcNow.Date;
+            var todaysSecurityEvents = await _opsEventRepo.GetOpsEventsAsync(
+                category: OpsEventCategory.Security,
+                dateFrom: todayUtc,
+                dateTo: todayUtc.AddDays(1));
+
+            return todaysSecurityEvents
+                .Where(e => e.EventType.StartsWith(CertExpiryEventTypePrefix, StringComparison.Ordinal))
+                .Select(e => $"{e.EventType}|{ExtractThumbprint(e.Details)}")
+                .ToHashSet(StringComparer.OrdinalIgnoreCase);
+        }
+
+        internal static string ExtractThumbprint(string? detailsJson)
+        {
+            if (string.IsNullOrEmpty(detailsJson)) return string.Empty;
+            try
+            {
+                using var doc = JsonDocument.Parse(detailsJson);
+                if (doc.RootElement.ValueKind == JsonValueKind.Object &&
+                    doc.RootElement.TryGetProperty("thumbprint", out var t) &&
+                    t.ValueKind == JsonValueKind.String)
+                {
+                    return t.GetString() ?? string.Empty;
+                }
+            }
+            catch (JsonException)
+            {
+                // Malformed details JSON - dedup key falls back to empty thumbprint
+            }
+            return string.Empty;
+        }
+    }
+}

src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs

@@ -111,6 +111,7 @@ namespace AutopilotMonitor.Functions.Services
                 await CleanupOldOpsEventsAsync();
                 await CleanupOrphanedEventsAsync();
                 await CheckAgentBlobStorageAsync();
+                await CheckEmbeddedCertExpiryAsync();
                 await RecomputePlatformStatsAsync();
 
                 // Backfill and repair tasks run only via manual trigger (RunManualAsync)

src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs

@@ -87,6 +87,31 @@ namespace AutopilotMonitor.Functions.Services
                 $"Agent version pattern '{pattern}' blocked",
                 null, blockedBy, new { pattern });
 
+        public Task RecordEmbeddedCertExpiringSoonAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
+            => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringSoon", OpsEventSeverity.Warning,
+                $"Embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - rotate the embedded PEM bundle before {notAfterUtc:yyyy-MM-dd}",
+                null, "System.Maintenance",
+                new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
+
+        public Task RecordEmbeddedCertExpiringUrgentAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
+            => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringUrgent", OpsEventSeverity.Error,
+                $"URGENT: embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - agent mTLS will break without bundle rotation",
+                null, "System.Maintenance",
+                new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
+
+        public Task RecordEmbeddedCertExpiredAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
+            => WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpired", OpsEventSeverity.Critical,
+                daysUntilExpiry < 0
+                    ? $"CRITICAL: embedded Intune {role.ToLowerInvariant()} '{subject}' EXPIRED {-daysUntilExpiry}d ago ({notAfterUtc:u}) - agent mTLS validation broken"
+                    : $"CRITICAL: embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u})",
+                null, "System.Maintenance",
+                new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
+
+        public Task RecordEmbeddedCertBundleEmptyAsync()
+            => WriteAsync(OpsEventCategory.Security, "EmbeddedCertBundleEmpty", OpsEventSeverity.Critical,
+                "No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients",
+                null, "System.Maintenance", new { });
+
         // ── Tenant ─────────────────────────────────────────────────────────────
 
         public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary<string, int> deletedCounts)