From 13dd6723da486c941fda27d25f906772d076aa4b Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 10 Aug 2026 13:32:10 +0200 Subject: [PATCH] agent: classify platform-403 as endpoint-unavailable + self-update retry after await-enrollment Field case sits-d.cloud 2026-08-09: a pinned pre-cutover agent (2.0.1282) hit the stopped legacy Function App. Azure answers with a platform-level 403 HTML page, which the agent reported as "device is not authorized" - support chases a non-existent authorization problem, and the stranded build never rescues itself because the startup self-update check runs pre-enrollment where OOBE has no network. Two fixes: - BackendApiClient reads the 401/403 body: an HTML response cannot come from our backend, so it is classified as endpoint stopped/retired/ proxied (BackendAuthException.EndpointUnavailable, page title in the message) instead of a device-authorization verdict. - AgentBootstrap retries the self-update check once after the MDM certificate appears (proof of connectivity) when the startup check never reached the manifest, with relaxed timeouts (10s check / 60s download). An applied update restarts into a current binary BEFORE session registration hits a retired endpoint. Marker files are also written on this trigger so agent_version_check reflects the latest verdict. Co-Authored-By: Claude Fable 5 --- ...BackendApiClientAuthClassificationTests.cs | 110 ++++++++++++++++++ .../Monitoring/Runtime/SelfUpdater.cs | 59 +++++++--- .../Monitoring/Transport/BackendApiClient.cs | 110 ++++++++++++++++-- .../Runtime/AgentBootstrap.cs | 42 +++++++ 4 files changed, 294 insertions(+), 27 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs new file mode 100644 index 00000000..426569fe --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs @@ -0,0 +1,110 @@ +#nullable enable +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport +{ + /// + /// 401/403 classification in : a backend JSON 403 is a real + /// device-authorization verdict, while an HTML body means the response came from the Azure + /// platform / an edge proxy (stopped or retired app) and must NOT be reported as + /// "device is not authorized". Field case sits-d.cloud 2026-08-09: a pre-cutover agent hit + /// the stopped legacy Function App and support chased a non-existent authorization problem. + /// + public sealed class BackendApiClientAuthClassificationTests + { + private sealed class StubHandler : HttpMessageHandler + { + private readonly Func _responder; + public StubHandler(Func responder) => _responder = responder; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => Task.FromResult(_responder(request)); + } + + private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType) + { + var handler = new StubHandler(_ => new HttpResponseMessage(status) + { + Content = new StringContent(body, Encoding.UTF8, mediaType) + }); + return new BackendApiClient( + new HttpClient(handler), + baseUrl: "https://backend.example", + manufacturer: "Contoso", + model: "TestBox", + serialNumber: "SN123", + useBootstrapTokenAuth: false, + bootstrapToken: null, + agentVersion: "2.0.0", + logger: null); + } + + [Fact] + public async Task Backend_json_403_is_reported_as_device_not_authorized() + { + using var client = BuildClient( + HttpStatusCode.Forbidden, + "{\"success\":false,\"message\":\"Device validation failed\"}", + "application/json"); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.Equal(403, ex.StatusCode); + Assert.False(ex.EndpointUnavailable); + Assert.Contains("The device is not authorized", ex.Message); + } + + [Fact] + public async Task Platform_html_403_is_reported_as_endpoint_unavailable_with_page_title() + { + using var client = BuildClient( + HttpStatusCode.Forbidden, + "Web App - Unavailablestopped", + "text/html"); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.Equal(403, ex.StatusCode); + Assert.True(ex.EndpointUnavailable); + Assert.Contains("Web App - Unavailable", ex.Message); + Assert.Contains("NOT a device-authorization failure", ex.Message); + Assert.DoesNotContain("Check client certificate", ex.Message); + } + + [Fact] + public async Task Html_body_with_generic_content_type_is_still_sniffed_as_platform_page() + { + using var client = BuildClient( + HttpStatusCode.Forbidden, + " Error 403 - This web app is stopped.", + "text/plain"); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.True(ex.EndpointUnavailable); + Assert.Contains("HTML platform error page", ex.Message); + } + + [Fact] + public async Task Backend_401_keeps_authorization_message_and_status() + { + using var client = BuildClient( + HttpStatusCode.Unauthorized, + "{\"success\":false,\"message\":\"certificate rejected\"}", + "application/json"); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.Equal(401, ex.StatusCode); + Assert.False(ex.EndpointUnavailable); + Assert.Contains("The device is not authorized", ex.Message); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/SelfUpdater.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/SelfUpdater.cs index a5d35494..0dc5c220 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/SelfUpdater.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Runtime/SelfUpdater.cs @@ -88,6 +88,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime /// public static string BackendExpectedSha256 { get; set; } + /// + /// True once a run actually obtained a version + /// from the manifest (regardless of whether an update was then needed/applied). + /// + /// Consumed by the --await-enrollment path: the startup check runs pre-enrollment + /// where OOBE frequently has no network yet, so a failed check there says nothing about + /// the latest version. Once the MDM certificate appears the device provably has + /// connectivity — the bootstrap retries the check exactly when this is still false. + /// + /// + public static bool LastVersionCheckSucceeded { get; private set; } + /// /// Optional: graceful-shutdown request hook, set by the runtime host once its /// shutdown machinery is armed (before the orchestrator starts) and cleared in its @@ -130,14 +142,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime /// to cover hotfixes that reuse the same version number. /// /// - /// "startup" (default) or "runtime_hash_mismatch". Recorded in the marker file so the - /// agent_version_check event (outcome=updated) can distinguish the two paths. + /// "startup" (default), "await_enrollment" (post-cert retry in the bootstrap), or + /// "runtime_hash_mismatch". Recorded in the marker file so the agent_version_check + /// event (outcome=updated) can distinguish the paths. /// /// /// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve /// Autopilot boot speed. Runtime trigger passes 60000 because monitoring is already running /// and we know an update is required. /// + /// + /// Overrides the version-v2.json fetch timeout (default 2500ms, tuned for boot speed). + /// The await-enrollment retry passes a relaxed value: enrollment just completed, the + /// device provably has network, and a stranded old build's last self-rescue chance is + /// worth a few extra seconds before session registration. + /// /// /// When false (default), a latest version strictly lower than /// is rejected even when is true — prevents the production @@ -151,7 +170,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime bool forceUpdate = false, string triggerReason = "startup", int? downloadTimeoutMsOverride = null, - bool allowDowngrade = false) + bool allowDowngrade = false, + int? versionCheckTimeoutMsOverride = null) { Action log = msg => { @@ -160,7 +180,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime }; var isStartupTrigger = string.Equals(triggerReason, "startup", StringComparison.Ordinal); + // The await-enrollment retry replaces the startup verdict — its markers must win so + // the agent_version_check event reflects the LATEST check, not the pre-network one. + var writeMarkers = isStartupTrigger || string.Equals(triggerReason, "await_enrollment", StringComparison.Ordinal); var downloadTimeoutMs = downloadTimeoutMsOverride ?? DownloadTimeoutMs; + var versionCheckTimeoutMs = versionCheckTimeoutMsOverride ?? VersionCheckTimeoutMs; var totalSw = Stopwatch.StartNew(); long versionCheckMs = 0, downloadMs = 0, verifyMs = 0, extractMs = 0, swapMs = 0; long zipSizeBytes = 0; @@ -174,16 +198,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime try { - // Step 1: Fetch version-v2.json (2.5s timeout) + // Step 1: Fetch version-v2.json (default 2.5s timeout, relaxed on retry triggers) var versionSw = Stopwatch.StartNew(); string manifestSha256; - (latestVersion, manifestSha256) = await GetLatestVersionAsync(log); + (latestVersion, manifestSha256) = await GetLatestVersionAsync(log, versionCheckTimeoutMs); versionSw.Stop(); versionCheckMs = versionSw.ElapsedMilliseconds; + LastVersionCheckSucceeded = latestVersion != null; if (latestVersion == null) { - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("version_check_failed", currentVersion, null, "version-v2.json fetch failed or timed out", log); return; // Could not determine latest version — continue with current } @@ -213,7 +238,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime if (!forceUpdate) { log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})"); - if (isStartupTrigger) + if (writeMarkers) WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log); return; } @@ -230,7 +255,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime if (!forceUpdate) { log($"Self-update: cannot compare versions (current={currentVersion}, latest={latestVersion}) — skipping"); - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("version_compare_failed", currentVersion, latestVersion, "version parse failed", log); return; } @@ -249,7 +274,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime downloadMs = downloadTimerSw.ElapsedMilliseconds; if (!downloadOk) { - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("download_failed", currentVersion, latestVersion, $"ZIP download failed within {downloadTimeoutMs}ms", log); return; } @@ -282,7 +307,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime if (!integrityOk) { CleanupStaging(null, zipPath); - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("integrity_mismatch", currentVersion, latestVersion, "SHA-256 mismatch on downloaded ZIP", log); return; } @@ -297,7 +322,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime extractMs = extractSw.ElapsedMilliseconds; if (!extractOk) { - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("extract_failed", currentVersion, latestVersion, "ZIP extraction failed", log); return; } @@ -310,7 +335,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime { log("Self-update: staging validation failed — AutopilotMonitor.Agent.exe not found in ZIP"); CleanupStaging(stagingDir, zipPath); - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("extract_failed", currentVersion, latestVersion, "AutopilotMonitor.Agent.exe not found in ZIP", log); return; } @@ -323,7 +348,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime if (!swapOk) { CleanupStaging(stagingDir, zipPath); - if (isStartupTrigger) + if (writeMarkers) WriteSkipMarker("swap_failed", currentVersion, latestVersion, "file swap failed", log); return; } @@ -355,7 +380,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime catch (Exception ex) { log($"Self-update: unexpected error — {ex.Message}. Continuing with current version."); - if (isStartupTrigger) + if (writeMarkers) { try { WriteSkipMarker("unexpected_error", currentVersion, latestVersion, ex.Message, log); } catch { } } @@ -366,7 +391,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime /// Fetches version-v2.json from blob storage and returns the version string and optional SHA-256 hash. /// Returns (null, null) if the check fails or times out. /// - private static async Task<(string version, string sha256)> GetLatestVersionAsync(Action log) + private static async Task<(string version, string sha256)> GetLatestVersionAsync(Action log, int timeoutMs) { try { @@ -375,7 +400,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime var versionUrl = $"{Constants.AgentDownloadBaseUrl}/{Constants.AgentVersionFileNameForLine(2)}"; using (var handler = new HttpClientHandler()) - using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(VersionCheckTimeoutMs) }) + using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(timeoutMs) }) { var json = await client.GetStringAsync(versionUrl); var obj = JObject.Parse(json); @@ -396,7 +421,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime } catch (TaskCanceledException) { - log($"Self-update: version check timed out ({VersionCheckTimeoutMs}ms) — skipping update"); + log($"Self-update: version check timed out ({timeoutMs}ms) — skipping update"); return (null, null); } catch (HttpRequestException ex) when (ex.Message.Contains("404")) diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs index 2494e1ed..260c8aba 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs @@ -137,7 +137,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport continue; } - ThrowOnAuthFailure(response); + await ThrowOnAuthFailureAsync(response).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -236,7 +236,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport using (response) { - ThrowOnAuthFailure(response); + await ThrowOnAuthFailureAsync(response).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -282,16 +282,97 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// /// Throws for 401/403 responses so callers can /// distinguish authentication failures from transient server errors. + /// + /// Field case (sits-d.cloud, 2026-08-09): a pre-cutover agent build pointed at a + /// retired Function App; Azure answers for a stopped app with a platform-level 403 + /// carrying an HTML error page ("Web App - Unavailable"). Our backend never returns + /// HTML on 401/403 — an HTML body therefore means the response did not come from the + /// backend's auth layer at all (stopped/retired app, edge proxy, WAF). Reporting that + /// as "device is not authorized" sends troubleshooting down the wrong path, so the + /// two cases get distinct messages and the exception carries a flag. + /// /// - private static void ThrowOnAuthFailure(HttpResponseMessage response) + private static async Task ThrowOnAuthFailureAsync(HttpResponseMessage response) { - if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized || - response.StatusCode == System.Net.HttpStatusCode.Forbidden) + if (response.StatusCode != System.Net.HttpStatusCode.Unauthorized && + response.StatusCode != System.Net.HttpStatusCode.Forbidden) { + return; + } + + var statusText = $"{(int)response.StatusCode} {response.StatusCode}"; + + string body = null; + try + { + body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + } + catch + { + // Body is diagnostic-only; classification below treats null as non-HTML. + } + + if (LooksLikeHtmlErrorPage(response, body)) + { + var title = TryExtractHtmlTitle(body); + var titleText = title != null ? $" (\"{title}\")" : string.Empty; + var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)"; throw new BackendAuthException( - $"Backend returned {(int)response.StatusCode} {response.StatusCode}. " + - "The device is not authorized. Check client certificate and Autopilot device validation.", - (int)response.StatusCode); + $"Backend returned {statusText} with an HTML platform error page{titleText} from '{host}'. " + + "This response did not come from the Autopilot Monitor backend — the endpoint appears stopped, " + + "retired, or intercepted by a proxy. This is NOT a device-authorization failure; an outdated " + + "agent build pointing at a stale ApiBaseUrl is the most likely cause.", + (int)response.StatusCode, + endpointUnavailable: true); + } + + throw new BackendAuthException( + $"Backend returned {statusText}. " + + "The device is not authorized. Check client certificate and Autopilot device validation.", + (int)response.StatusCode); + } + + /// + /// True when a 401/403 response is an HTML page rather than a backend JSON/plain-text + /// reply. Checks the declared Content-Type first, then falls back to sniffing the body + /// prefix so a platform page served with a generic Content-Type is still recognized. + /// + private static bool LooksLikeHtmlErrorPage(HttpResponseMessage response, string body) + { + var mediaType = response.Content?.Headers?.ContentType?.MediaType; + if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase)) + return true; + + if (string.IsNullOrWhiteSpace(body)) + return false; + + var head = body.TrimStart(); + return head.StartsWith(" + /// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so the + /// log line names the platform page without dumping markup. Null when absent/oversized. + /// + private static string TryExtractHtmlTitle(string body) + { + if (string.IsNullOrEmpty(body)) return null; + try + { + var match = System.Text.RegularExpressions.Regex.Match( + body, @"]*>\s*(.*?)\s*", + System.Text.RegularExpressions.RegexOptions.IgnoreCase | + System.Text.RegularExpressions.RegexOptions.Singleline, + TimeSpan.FromMilliseconds(250)); + if (!match.Success) return null; + + var title = match.Groups[1].Value; + return title.Length == 0 ? null : (title.Length > 80 ? title.Substring(0, 80) : title); + } + catch + { + return null; } } @@ -330,7 +411,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport } /// - /// Thrown when the backend returns 401 or 403, indicating the device is not authorized. + /// Thrown when the backend returns 401 or 403, indicating the device is not authorized — + /// or, when is set, that the 401/403 was a platform-level + /// HTML response (stopped/retired app, edge proxy) rather than a backend auth verdict. /// public class BackendAuthException : Exception { @@ -340,9 +423,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// public int StatusCode { get; } - public BackendAuthException(string message, int statusCode = 0) : base(message) + /// + /// True when the 401/403 carried an HTML platform error page — the endpoint itself is + /// stopped/retired/proxied and the status says nothing about this device's authorization. + /// + public bool EndpointUnavailable { get; } + + public BackendAuthException(string message, int statusCode = 0, bool endpointUnavailable = false) : base(message) { StatusCode = statusCode; + EndpointUnavailable = endpointUnavailable; } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs index 546060a5..899b49cc 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentBootstrap.cs @@ -121,6 +121,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime // Await-enrollment is one-shot — remove the persisted config so subsequent restarts proceed normally. Program.DeleteAwaitEnrollmentConfig(dataDirectory, logger); + + RetrySelfUpdateAfterEnrollment(logger, consoleMode); } // Event-driven TenantId wait — kicks in only when the user opted-in via @@ -152,6 +154,46 @@ namespace AutopilotMonitor.Agent.V2.Runtime isWhiteGloveResume: isWhiteGloveResume, cleanupServiceFactory: cleanupServiceFactory); } + + /// + /// One-shot self-update retry after the MDM certificate appeared. The startup check in + /// Main runs pre-enrollment, where an --await-enrollment install typically has no + /// network yet — its failure says nothing about the latest version. The certificate is + /// proof of connectivity, and this is the last chance to swap a stranded old build + /// (stale baked-in ApiBaseUrl) for a current one BEFORE session registration hits a + /// retired endpoint (field case sits-d.cloud, 2026-08-09). Timeouts are relaxed vs. the + /// boot-speed-tuned startup check; enrollment is already past OOBE's critical path here. + /// On update success the process restarts and this method never returns; the + /// await-enrollment config was already removed, so the new binary proceeds normally. + /// + private static void RetrySelfUpdateAfterEnrollment(AgentLogger logger, bool consoleMode) + { + if (SelfUpdater.LastVersionCheckSucceeded) + return; // Startup check reached the manifest — its verdict stands. + + try + { + logger.Info("Await-enrollment: startup self-update check had failed (no network in early OOBE) — retrying now that enrollment proves connectivity."); + var agentDir = System.IO.Path.GetDirectoryName( + System.Reflection.Assembly.GetExecutingAssembly().Location) ?? string.Empty; + SelfUpdater.CheckAndApplyUpdateAsync( + currentVersion: Program.GetAgentVersion(), + agentDir: agentDir, + consoleMode: consoleMode, + triggerReason: "await_enrollment", + downloadTimeoutMsOverride: 60000, + // Fresh --await-enrollment installs have no cached remote config yet, so + // there is no admin downgrade override to honour on this path. + allowDowngrade: false, + versionCheckTimeoutMsOverride: 10000).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + // Same contract as the startup call site: the updater swallows its own errors; + // anything escaping must not break bootstrap — the current binary continues. + logger.Warning($"Await-enrollment: self-update retry failed unexpectedly: {ex.Message}. Continuing with current version."); + } + } } ///