Backend Resilience & Cache Enhancements

src/Agent/AutopilotMonitor.Agent/buildcounter.txt

@@ -1 +1 @@
-441
+442

src/Backend/AutopilotMonitor.Functions/Security/AutopilotDeviceValidator.cs

@@ -100,8 +100,8 @@ namespace AutopilotMonitor.Functions.Security
         {
             try
             {
-                var accessToken = await _graphTokenService.GetAccessTokenAsync(tenantId);
-                if (string.IsNullOrEmpty(accessToken))
+                var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId);
+                if (string.IsNullOrEmpty(tokenResult.AccessToken))
                 {
                     return new AutopilotDeviceValidationResult
                     {
@@ -113,7 +113,7 @@ namespace AutopilotMonitor.Functions.Security
                 }
 
                 var graphClient = _httpClientFactory.CreateClient();
-                graphClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
+                graphClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", tokenResult.AccessToken);
 
                 // For windowsAutopilotDeviceIdentities, eq on serialNumber is unreliable and often returns 400.
                 // Use contains for server-side narrowing, then perform exact match client-side.

src/Backend/AutopilotMonitor.Functions/Security/CorporateIdentifierValidator.cs

@@ -99,8 +99,8 @@ namespace AutopilotMonitor.Functions.Security
         {
             try
             {
-                var accessToken = await _graphTokenService.GetAccessTokenAsync(tenantId);
-                if (string.IsNullOrEmpty(accessToken))
+                var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId);
+                if (string.IsNullOrEmpty(tokenResult.AccessToken))
                 {
                     return new CorporateIdentifierValidationResult
                     {
@@ -111,7 +111,7 @@ namespace AutopilotMonitor.Functions.Security
                 }
 
                 var graphClient = _httpClientFactory.CreateClient();
-                graphClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", accessToken);
+                graphClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", tokenResult.AccessToken);
 
                 // POST https://graph.microsoft.com/beta/deviceManagement/importedDeviceIdentities/searchExistingIdentities
                 var identifier = $"{normalizedManufacturer},{normalizedModel},{normalizedSerial}";

src/Backend/AutopilotMonitor.Functions/Security/GraphTokenService.cs

@@ -43,24 +43,37 @@ namespace AutopilotMonitor.Functions.Security
                 return cached;
             }
 
-            var accessToken = await GetAccessTokenAsync(tenantId);
+            var tokenResult = await GetAccessTokenAsync(tenantId);
             var result = new GraphConsentStatusResult
             {
-                IsConsented = !string.IsNullOrWhiteSpace(accessToken),
-                Message = string.IsNullOrWhiteSpace(accessToken)
-                    ? "Admin consent for Graph application permissions is missing or app credentials are invalid."
-                    : "Admin consent is available for this tenant."
+                IsConsented = !string.IsNullOrWhiteSpace(tokenResult.AccessToken),
+                IsTransient = tokenResult.IsTransient,
+                Message = !string.IsNullOrWhiteSpace(tokenResult.AccessToken)
+                    ? "Admin consent is available for this tenant."
+                    : tokenResult.IsTransient
+                        ? "Could not verify consent status due to a transient error. Will retry on next request."
+                        : "Admin consent for Graph application permissions is missing or app credentials are invalid."
             };
 
-            _cache.Set(cacheKey, result, new MemoryCacheEntryOptions
+            // Only cache definitive results — never cache transient failures
+            if (!tokenResult.IsTransient)
             {
-                AbsoluteExpirationRelativeToNow = ConsentStatusTtl
-            });
+                _cache.Set(cacheKey, result, new MemoryCacheEntryOptions
+                {
+                    AbsoluteExpirationRelativeToNow = ConsentStatusTtl
+                });
+            }
+            else
+            {
+                _logger.LogWarning(
+                    "Consent status check for tenant {TenantId} returned transient error — result NOT cached",
+                    tenantId);
+            }
 
             return result;
         }
 
-        public async Task<string?> GetAccessTokenAsync(string tenantId)
+        public async Task<GraphTokenResult> GetAccessTokenAsync(string tenantId)
         {
             var clientId = _configuration["EntraId:ClientId"];
             var clientSecret = _configuration["EntraId:ClientSecret"];
@@ -69,7 +82,7 @@ namespace AutopilotMonitor.Functions.Security
             {
                 _logger.LogError(
                     "Device validation is enabled but Entra ID app credentials are not configured. Set EntraId:ClientId and EntraId:ClientSecret.");
-                return null;
+                return GraphTokenResult.PermanentFailure();
             }
 
             var tokenUrl = $"https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token";
@@ -80,64 +93,131 @@ namespace AutopilotMonitor.Functions.Security
             var retryDelays = new[] { TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30) };
             string? responseBody = null;
             int attempt = 0;
+            bool lastAttemptWasTransient = false;
 
             while (true)
             {
-                var tokenClient = _httpClientFactory.CreateClient();
-                var body = new FormUrlEncodedContent(new Dictionary<string, string>
+                try
                 {
-                    ["client_id"] = clientId,
-                    ["client_secret"] = clientSecret,
-                    ["scope"] = "https://graph.microsoft.com/.default",
-                    ["grant_type"] = "client_credentials"
-                });
+                    var tokenClient = _httpClientFactory.CreateClient();
+                    var body = new FormUrlEncodedContent(new Dictionary<string, string>
+                    {
+                        ["client_id"] = clientId,
+                        ["client_secret"] = clientSecret,
+                        ["scope"] = "https://graph.microsoft.com/.default",
+                        ["grant_type"] = "client_credentials"
+                    });
 
-                var response = await tokenClient.PostAsync(tokenUrl, body);
-                responseBody = await response.Content.ReadAsStringAsync();
+                    var response = await tokenClient.PostAsync(tokenUrl, body);
+                    responseBody = await response.Content.ReadAsStringAsync();
 
-                if (response.IsSuccessStatusCode)
-                {
-                    var tokenJson = JsonConvert.DeserializeObject<JObject>(responseBody);
-                    return tokenJson?["access_token"]?.ToString();
-                }
+                    if (response.IsSuccessStatusCode)
+                    {
+                        var tokenJson = JsonConvert.DeserializeObject<JObject>(responseBody);
+                        var accessToken = tokenJson?["access_token"]?.ToString();
+                        return GraphTokenResult.Success(accessToken);
+                    }
 
-                // Only retry on transient consent-propagation errors (unauthorized_client, invalid_client).
-                // Do not retry on permanent errors like invalid credentials or tenant not found.
-                var isRetryable = response.StatusCode == System.Net.HttpStatusCode.Unauthorized
-                    || response.StatusCode == System.Net.HttpStatusCode.BadRequest
-                    && (responseBody.Contains("unauthorized_client", StringComparison.OrdinalIgnoreCase)
-                        || responseBody.Contains("AADSTS700016", StringComparison.OrdinalIgnoreCase)
-                        || responseBody.Contains("AADSTS7000215", StringComparison.OrdinalIgnoreCase));
+                    // Classify the error: consent-propagation errors are retryable but ultimately permanent (no consent).
+                    // Server errors (500, 502, 503, 504) and timeouts (408) are truly transient.
+                    var statusCode = (int)response.StatusCode;
+                    var isConsentError = response.StatusCode == System.Net.HttpStatusCode.BadRequest
+                        && (responseBody.Contains("unauthorized_client", StringComparison.OrdinalIgnoreCase)
+                            || responseBody.Contains("AADSTS700016", StringComparison.OrdinalIgnoreCase)
+                            || responseBody.Contains("AADSTS7000215", StringComparison.OrdinalIgnoreCase));
+                    var isServerError = statusCode == 408 || statusCode == 429
+                        || statusCode >= 500 && statusCode <= 599;
+                    var isRetryable = response.StatusCode == System.Net.HttpStatusCode.Unauthorized
+                        || isConsentError || isServerError;
 
-                if (!isRetryable || attempt >= retryDelays.Length)
-                {
-                    _logger.LogWarning(
-                        "Failed to acquire Graph token for tenant {TenantId} after {Attempts} attempt(s). Status: {StatusCode}. Body: {ResponseBody}",
+                    lastAttemptWasTransient = isServerError;
+
+                    if (!isRetryable || attempt >= retryDelays.Length)
+                    {
+                        _logger.LogWarning(
+                            "Failed to acquire Graph token for tenant {TenantId} after {Attempts} attempt(s). Status: {StatusCode}. Body: {ResponseBody}",
+                            tenantId,
+                            attempt + 1,
+                            statusCode,
+                            responseBody);
+
+                        // Server errors are transient (infrastructure issue), consent errors are permanent (no consent granted)
+                        return lastAttemptWasTransient
+                            ? GraphTokenResult.TransientFailure()
+                            : GraphTokenResult.PermanentFailure();
+                    }
+
+                    var delay = retryDelays[attempt];
+
+                    // Respect Retry-After header from Azure AD / Graph throttling
+                    if (response.Headers.RetryAfter?.Delta is TimeSpan retryAfterDelta && retryAfterDelta > delay)
+                    {
+                        delay = retryAfterDelta;
+                    }
+
+                    _logger.LogInformation(
+                        "Graph token acquisition for tenant {TenantId} returned {StatusCode} (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}s — likely Azure AD consent propagation delay.",
                         tenantId,
+                        statusCode,
                         attempt + 1,
-                        (int)response.StatusCode,
-                        responseBody);
-                    return null;
+                        retryDelays.Length + 1,
+                        delay.TotalSeconds);
+
+                    await Task.Delay(delay);
+                    attempt++;
                 }
+                catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
+                {
+                    // Network errors and timeouts are transient
+                    if (attempt >= retryDelays.Length)
+                    {
+                        _logger.LogWarning(ex,
+                            "Graph token acquisition for tenant {TenantId} failed with network error after {Attempts} attempt(s)",
+                            tenantId, attempt + 1);
+                        return GraphTokenResult.TransientFailure();
+                    }
 
-                var delay = retryDelays[attempt];
-                _logger.LogInformation(
-                    "Graph token acquisition for tenant {TenantId} returned {StatusCode} (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}s — likely Azure AD consent propagation delay.",
-                    tenantId,
-                    (int)response.StatusCode,
-                    attempt + 1,
-                    retryDelays.Length + 1,
-                    delay.TotalSeconds);
+                    var delay = retryDelays[attempt];
+                    _logger.LogWarning(ex,
+                        "Graph token acquisition for tenant {TenantId} network error (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}s",
+                        tenantId, attempt + 1, retryDelays.Length + 1, delay.TotalSeconds);
 
-                await Task.Delay(delay);
-                attempt++;
+                    await Task.Delay(delay);
+                    attempt++;
+                }
             }
         }
     }
 
+    /// <summary>
+    /// Result of a Graph token acquisition attempt.
+    /// Distinguishes between success, permanent failure (no consent), and transient failure (network/server error).
+    /// </summary>
+    public class GraphTokenResult
+    {
+        public string? AccessToken { get; private set; }
+
+        /// <summary>
+        /// True when the failure is transient (network error, server error, timeout).
+        /// Transient failures should NOT be cached as "no consent".
+        /// </summary>
+        public bool IsTransient { get; private set; }
+
+        public static GraphTokenResult Success(string? token) => new() { AccessToken = token };
+        public static GraphTokenResult PermanentFailure() => new() { AccessToken = null, IsTransient = false };
+        public static GraphTokenResult TransientFailure() => new() { AccessToken = null, IsTransient = true };
+    }
+
     public class GraphConsentStatusResult
     {
         public bool IsConsented { get; set; }
+
+        /// <summary>
+        /// True when the result is due to a transient error (network/server issue).
+        /// Callers should treat this as "unknown" rather than "not consented".
+        /// </summary>
+        public bool IsTransient { get; set; }
+
         public string? Message { get; set; }
     }
 }

src/Backend/AutopilotMonitor.Functions/Services/BlockedVersionService.cs

@@ -24,9 +24,12 @@ namespace AutopilotMonitor.Functions.Services
         private readonly TableStorageService _storageService;
         private readonly ILogger<BlockedVersionService> _logger;
 
+        private static readonly TimeSpan CacheRefreshInterval = TimeSpan.FromMinutes(5);
+
         // In-memory cache of all version block rules
         private readonly ConcurrentDictionary<string, VersionBlockCacheEntry> _cache = new(StringComparer.OrdinalIgnoreCase);
         private volatile bool _loaded;
+        private DateTime _lastLoadedUtc = DateTime.MinValue;
 
         public BlockedVersionService(TableStorageService storageService, ILogger<BlockedVersionService> logger)
         {
@@ -43,7 +46,7 @@ namespace AutopilotMonitor.Functions.Services
             if (string.IsNullOrEmpty(agentVersion))
                 return (false, "Block", null);
 
-            if (!_loaded)
+            if (!_loaded || DateTime.UtcNow - _lastLoadedUtc > CacheRefreshInterval)
                 await LoadBlockListAsync();
 
             string? matchedAction = null;
@@ -272,22 +275,36 @@ namespace AutopilotMonitor.Functions.Services
             try
             {
                 var tableClient = _storageService.GetTableServiceClient().GetTableClient(Constants.TableNames.BlockedVersions);
+                var freshCache = new ConcurrentDictionary<string, VersionBlockCacheEntry>(StringComparer.OrdinalIgnoreCase);
 
                 await foreach (var entity in tableClient.QueryAsync<TableEntity>(e => e.PartitionKey == "global"))
                 {
                     var pattern = entity.GetString("VersionPattern") ?? DecodeRowKey(entity.RowKey);
-                    _cache[pattern] = new VersionBlockCacheEntry
+                    freshCache[pattern] = new VersionBlockCacheEntry
                     {
                         Action = entity.GetString("Action") ?? "Block"
                     };
                 }
 
-                _logger.LogDebug("Loaded {Count} version block rules", _cache.Count);
+                // Atomically replace cache contents
+                _cache.Clear();
+                foreach (var kvp in freshCache)
+                {
+                    _cache[kvp.Key] = kvp.Value;
+                }
+
+                _lastLoadedUtc = DateTime.UtcNow;
+                _logger.LogDebug("Loaded {Count} version block rules (refresh interval: {Interval}min)", _cache.Count, CacheRefreshInterval.TotalMinutes);
             }
             catch (Exception ex)
             {
                 _logger.LogError(ex, "Failed to load version block rules");
-                _loaded = false;
+                // On error: if we had data before, keep it (stale > nothing). Only reset _loaded if we never loaded.
+                if (_lastLoadedUtc == DateTime.MinValue)
+                {
+                    _loaded = false;
+                }
+                // Otherwise keep _loaded=true and _lastLoadedUtc as-is so we retry on next interval
             }
         }
 

src/Backend/AutopilotMonitor.Functions/Services/GitHubRuleRepository.cs

@@ -1,3 +1,7 @@
+using System;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
 using AutopilotMonitor.Shared.Models;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
@@ -8,6 +12,7 @@ namespace AutopilotMonitor.Functions.Services
     /// <summary>
     /// Fetches rule definitions from GitHub (raw.githubusercontent.com).
     /// Uses the CI-generated combined files from rules/dist/.
+    /// Includes retry logic with exponential backoff for transient failures.
     /// </summary>
     public class GitHubRuleRepository
     {
@@ -15,6 +20,13 @@ namespace AutopilotMonitor.Functions.Services
         private readonly ILogger<GitHubRuleRepository> _logger;
         private readonly string _baseUrl;
 
+        private static readonly TimeSpan[] RetryDelays = new[]
+        {
+            TimeSpan.FromSeconds(2),
+            TimeSpan.FromSeconds(5),
+            TimeSpan.FromSeconds(10)
+        };
+
         public GitHubRuleRepository(HttpClient httpClient, IConfiguration config, ILogger<GitHubRuleRepository> logger)
         {
             _httpClient = httpClient;
@@ -26,34 +38,94 @@ namespace AutopilotMonitor.Functions.Services
         public async Task<List<GatherRule>> FetchGatherRulesAsync()
         {
             var url = $"{_baseUrl}/dist/gather-rules.json";
-            _logger.LogInformation($"Fetching gather rules from {url}");
-            var json = await _httpClient.GetStringAsync(url);
+            _logger.LogInformation("Fetching gather rules from {Url}", url);
+            var json = await FetchWithRetryAsync(url);
             var wrapper = JsonConvert.DeserializeObject<RulesWrapper<GatherRule>>(json);
             var rules = wrapper?.Rules ?? new List<GatherRule>();
-            _logger.LogInformation($"Fetched {rules.Count} gather rules from GitHub");
+            _logger.LogInformation("Fetched {Count} gather rules from GitHub", rules.Count);
             return rules;
         }
 
         public async Task<List<AnalyzeRule>> FetchAnalyzeRulesAsync()
         {
             var url = $"{_baseUrl}/dist/analyze-rules.json";
-            _logger.LogInformation($"Fetching analyze rules from {url}");
-            var json = await _httpClient.GetStringAsync(url);
+            _logger.LogInformation("Fetching analyze rules from {Url}", url);
+            var json = await FetchWithRetryAsync(url);
             var wrapper = JsonConvert.DeserializeObject<RulesWrapper<AnalyzeRule>>(json);
             var rules = wrapper?.Rules ?? new List<AnalyzeRule>();
-            _logger.LogInformation($"Fetched {rules.Count} analyze rules from GitHub");
+            _logger.LogInformation("Fetched {Count} analyze rules from GitHub", rules.Count);
             return rules;
         }
 
         public async Task<List<ImeLogPattern>> FetchImeLogPatternsAsync()
         {
             var url = $"{_baseUrl}/dist/ime-log-patterns.json";
-            _logger.LogInformation($"Fetching IME log patterns from {url}");
-            var json = await _httpClient.GetStringAsync(url);
+            _logger.LogInformation("Fetching IME log patterns from {Url}", url);
+            var json = await FetchWithRetryAsync(url);
             var wrapper = JsonConvert.DeserializeObject<RulesWrapper<ImeLogPattern>>(json);
             var patterns = wrapper?.Rules ?? new List<ImeLogPattern>();
-            _logger.LogInformation($"Fetched {patterns.Count} IME log patterns from GitHub");
+            _logger.LogInformation("Fetched {Count} IME log patterns from GitHub", patterns.Count);
             return patterns;
         }
+
+        /// <summary>
+        /// Fetches a URL with retry logic and exponential backoff.
+        /// Retries on transient HTTP errors (408, 429, 5xx) and network exceptions.
+        /// </summary>
+        private async Task<string> FetchWithRetryAsync(string url)
+        {
+            for (int attempt = 0; attempt <= RetryDelays.Length; attempt++)
+            {
+                try
+                {
+                    var response = await _httpClient.GetAsync(url);
+
+                    if (response.IsSuccessStatusCode)
+                    {
+                        return await response.Content.ReadAsStringAsync();
+                    }
+
+                    var statusCode = (int)response.StatusCode;
+                    var isTransient = statusCode == 408 || statusCode == 429
+                        || statusCode >= 500 && statusCode <= 599;
+
+                    if (!isTransient || attempt >= RetryDelays.Length)
+                    {
+                        // Non-transient error or retries exhausted — throw to caller
+                        var body = await response.Content.ReadAsStringAsync();
+                        _logger.LogWarning(
+                            "GitHub fetch failed for {Url} after {Attempts} attempt(s). Status: {StatusCode}. Body: {Body}",
+                            url, attempt + 1, statusCode, body);
+                        response.EnsureSuccessStatusCode(); // Throws HttpRequestException
+                    }
+
+                    var delay = RetryDelays[attempt];
+
+                    // Respect Retry-After header (GitHub rate limiting)
+                    if (response.Headers.RetryAfter?.Delta is TimeSpan retryAfterDelta && retryAfterDelta > delay)
+                    {
+                        delay = retryAfterDelta;
+                    }
+
+                    _logger.LogWarning(
+                        "GitHub fetch for {Url} returned {StatusCode} (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}s",
+                        url, statusCode, attempt + 1, RetryDelays.Length + 1, delay.TotalSeconds);
+
+                    await Task.Delay(delay);
+                }
+                catch (Exception ex) when (attempt < RetryDelays.Length && (ex is HttpRequestException or TaskCanceledException))
+                {
+                    var delay = RetryDelays[attempt];
+                    _logger.LogWarning(ex,
+                        "GitHub fetch for {Url} failed with network error (attempt {Attempt}/{MaxAttempts}). Retrying in {Delay}s",
+                        url, attempt + 1, RetryDelays.Length + 1, delay.TotalSeconds);
+
+                    await Task.Delay(delay);
+                }
+            }
+
+            // Should not reach here, but just in case
+            throw new HttpRequestException($"Failed to fetch {url} after {RetryDelays.Length + 1} attempts");
+        }
     }
 }

src/Backend/AutopilotMonitor.Functions/Services/RateLimitService.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Extensions.Caching.Memory;
@@ -16,6 +17,11 @@ namespace AutopilotMonitor.Functions.Services
         private readonly ILogger<RateLimitService> _logger;
         private readonly TimeSpan _windowDuration;
 
+        // Dedicated lock objects per device to avoid race conditions when cache evicts the request history list.
+        // Using the list itself as a lock target is unsafe because cache eviction can create a new list instance,
+        // causing different threads to lock on different objects.
+        private readonly ConcurrentDictionary<string, object> _locks = new();
+
         public RateLimitService(IMemoryCache cache, ILogger<RateLimitService> logger)
         {
             _cache = cache;
@@ -44,14 +50,16 @@ namespace AutopilotMonitor.Functions.Services
             var now = DateTime.UtcNow;
 
             // Get or create request history for this device
+            var lockObj = _locks.GetOrAdd(cacheKey, _ => new object());
             var requestHistory = _cache.GetOrCreate(cacheKey, entry =>
             {
                 entry.SlidingExpiration = _windowDuration;
                 entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromHours(1);
+                entry.RegisterPostEvictionCallback((key, _, _, _) => _locks.TryRemove((string)key, out _));
                 return new List<DateTime>();
-            });
+            })!;
 
-            lock (requestHistory!)
+            lock (lockObj)
             {
                 // Remove requests outside the sliding window
                 var windowStart = now.Subtract(_windowDuration);

src/Backend/AutopilotMonitor.Functions/Services/TeamsNotificationService.cs

@@ -115,21 +115,14 @@ namespace AutopilotMonitor.Functions.Services
                            .Replace("\"context\":", "\"@context\":");
 
                 var content = new StringContent(json, Encoding.UTF8, "application/json");
-                var response = await _http.PostAsync(webhookUrl, content);
+                var sent = await PostWithRetryAsync(webhookUrl, content, $"enrollment:{deviceName}");
 
-                if (!response.IsSuccessStatusCode)
-                {
-                    var body = await response.Content.ReadAsStringAsync();
-                    _logger.LogWarning($"Teams webhook returned {(int)response.StatusCode}: {body}");
-                }
-                else
-                {
-                    _logger.LogInformation($"Teams enrollment notification sent for device '{deviceName}' (success={success})");
-                }
+                if (sent)
+                    _logger.LogInformation("Teams enrollment notification sent for device '{DeviceName}' (success={Success})", deviceName, success);
             }
             catch (Exception ex)
             {
-                _logger.LogWarning(ex, $"Failed to send Teams enrollment notification for device '{deviceName}'");
+                _logger.LogWarning(ex, "Failed to send Teams enrollment notification for device '{DeviceName}'", deviceName);
             }
         }
 
@@ -195,24 +188,58 @@ namespace AutopilotMonitor.Functions.Services
                            .Replace("\"context\":", "\"@context\":");
 
                 var content = new StringContent(json, Encoding.UTF8, "application/json");
-                var response = await _http.PostAsync(webhookUrl, content);
+                var sent = await PostWithRetryAsync(webhookUrl, content, $"whiteglove:{deviceName}");
 
-                if (!response.IsSuccessStatusCode)
-                {
-                    var body = await response.Content.ReadAsStringAsync();
-                    _logger.LogWarning($"Teams webhook returned {(int)response.StatusCode}: {body}");
-                }
-                else
-                {
-                    _logger.LogInformation($"Teams WhiteGlove notification sent for device '{deviceName}' (success={success})");
-                }
+                if (sent)
+                    _logger.LogInformation("Teams WhiteGlove notification sent for device '{DeviceName}' (success={Success})", deviceName, success);
             }
             catch (Exception ex)
             {
-                _logger.LogWarning(ex, $"Failed to send Teams WhiteGlove notification for device '{deviceName}'");
+                _logger.LogWarning(ex, "Failed to send Teams WhiteGlove notification for device '{DeviceName}'", deviceName);
             }
         }
 
+        /// <summary>
+        /// Posts content to a webhook URL with a single retry on transient failures (429, 5xx, network errors).
+        /// </summary>
+        private async Task<bool> PostWithRetryAsync(string webhookUrl, StringContent content, string context)
+        {
+            const int maxAttempts = 2;
+            for (int attempt = 1; attempt <= maxAttempts; attempt++)
+            {
+                try
+                {
+                    var response = await _http.PostAsync(webhookUrl, content);
+
+                    if (response.IsSuccessStatusCode)
+                        return true;
+
+                    var statusCode = (int)response.StatusCode;
+                    var isTransient = statusCode == 429 || statusCode >= 500 && statusCode <= 599;
+
+                    if (!isTransient || attempt >= maxAttempts)
+                    {
+                        var body = await response.Content.ReadAsStringAsync();
+                        _logger.LogWarning("Teams webhook returned {StatusCode} for {Context}: {Body}", statusCode, context, body);
+                        return false;
+                    }
+
+                    _logger.LogWarning(
+                        "Teams webhook returned {StatusCode} for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
+                        statusCode, context, attempt, maxAttempts);
+                    await Task.Delay(TimeSpan.FromSeconds(2));
+                }
+                catch (Exception ex) when (attempt < maxAttempts && (ex is HttpRequestException or TaskCanceledException))
+                {
+                    _logger.LogWarning(ex,
+                        "Teams webhook network error for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
+                        context, attempt, maxAttempts);
+                    await Task.Delay(TimeSpan.FromSeconds(2));
+                }
+            }
+            return false;
+        }
+
         private static string BuildHardwareText(string? manufacturer, string? model)
         {
             var parts = new[]

src/Backend/AutopilotMonitor.Functions/Services/TelegramNotificationService.cs

@@ -59,16 +59,9 @@ namespace AutopilotMonitor.Functions.Services
 
                 var json = JsonConvert.SerializeObject(payload);
                 var content = new StringContent(json, Encoding.UTF8, "application/json");
-                var response = await _http.PostAsync(webhookUrl, content);
+                var sent = await PostWithRetryAsync(webhookUrl, content, $"signup:{tenantId}");
 
-                if (!response.IsSuccessStatusCode)
-                {
-                    var body = await response.Content.ReadAsStringAsync();
-                    _logger.LogWarning(
-                        "Telegram webhook returned {StatusCode} for tenant {TenantId}: {Body}",
-                        (int)response.StatusCode, tenantId, body);
-                }
-                else
+                if (sent)
                 {
                     _logger.LogInformation(
                         "Telegram Private Preview signup notification sent for tenant {TenantId}, UPN {Upn}",
@@ -107,16 +100,9 @@ namespace AutopilotMonitor.Functions.Services
 
                 var json = JsonConvert.SerializeObject(payload);
                 var content = new StringContent(json, Encoding.UTF8, "application/json");
-                var response = await _http.PostAsync(webhookUrl, content);
+                var sent = await PostWithRetryAsync(webhookUrl, content, $"report:{reportId}");
 
-                if (!response.IsSuccessStatusCode)
-                {
-                    var body = await response.Content.ReadAsStringAsync();
-                    _logger.LogWarning(
-                        "Telegram webhook returned {StatusCode} for session report {ReportId}: {Body}",
-                        (int)response.StatusCode, reportId, body);
-                }
-                else
+                if (sent)
                 {
                     _logger.LogInformation(
                         "Telegram session report notification sent for report {ReportId}", reportId);
@@ -128,6 +114,47 @@ namespace AutopilotMonitor.Functions.Services
             }
         }
 
+        /// <summary>
+        /// Posts content to a webhook URL with a single retry on transient failures (429, 5xx, network errors).
+        /// </summary>
+        private async Task<bool> PostWithRetryAsync(string webhookUrl, StringContent content, string context)
+        {
+            const int maxAttempts = 2;
+            for (int attempt = 1; attempt <= maxAttempts; attempt++)
+            {
+                try
+                {
+                    var response = await _http.PostAsync(webhookUrl, content);
+
+                    if (response.IsSuccessStatusCode)
+                        return true;
+
+                    var statusCode = (int)response.StatusCode;
+                    var isTransient = statusCode == 429 || statusCode >= 500 && statusCode <= 599;
+
+                    if (!isTransient || attempt >= maxAttempts)
+                    {
+                        var body = await response.Content.ReadAsStringAsync();
+                        _logger.LogWarning("Telegram webhook returned {StatusCode} for {Context}: {Body}", statusCode, context, body);
+                        return false;
+                    }
+
+                    _logger.LogWarning(
+                        "Telegram webhook returned {StatusCode} for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
+                        statusCode, context, attempt, maxAttempts);
+                    await Task.Delay(TimeSpan.FromSeconds(2));
+                }
+                catch (Exception ex) when (attempt < maxAttempts && (ex is HttpRequestException or TaskCanceledException))
+                {
+                    _logger.LogWarning(ex,
+                        "Telegram webhook network error for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
+                        context, attempt, maxAttempts);
+                    await Task.Delay(TimeSpan.FromSeconds(2));
+                }
+            }
+            return false;
+        }
+
         private async Task<string?> GetWebhookUrlAsync()
         {
             try