mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
agent: classify platform-403 as endpoint-unavailable + self-update retry after await-enrollment
Field case sits-d.cloud 2026-08-09: a pinned pre-cutover agent (2.0.1282) hit the stopped legacy Function App. Azure answers with a platform-level 403 HTML page, which the agent reported as "device is not authorized" - support chases a non-existent authorization problem, and the stranded build never rescues itself because the startup self-update check runs pre-enrollment where OOBE has no network. Two fixes: - BackendApiClient reads the 401/403 body: an HTML response cannot come from our backend, so it is classified as endpoint stopped/retired/ proxied (BackendAuthException.EndpointUnavailable, page title in the message) instead of a device-authorization verdict. - AgentBootstrap retries the self-update check once after the MDM certificate appears (proof of connectivity) when the startup check never reached the manifest, with relaxed timeouts (10s check / 60s download). An applied update restarts into a current binary BEFORE session registration hits a retired endpoint. Marker files are also written on this trigger so agent_version_check reflects the latest verdict. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+110
@@ -0,0 +1,110 @@
|
|||||||
|
#nullable enable
|
||||||
|
using System;
|
||||||
|
using System.Net;
|
||||||
|
using System.Net.Http;
|
||||||
|
using System.Text;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||||
|
using Xunit;
|
||||||
|
|
||||||
|
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// 401/403 classification in <see cref="BackendApiClient"/>: a backend JSON 403 is a real
|
||||||
|
/// device-authorization verdict, while an HTML body means the response came from the Azure
|
||||||
|
/// platform / an edge proxy (stopped or retired app) and must NOT be reported as
|
||||||
|
/// "device is not authorized". Field case sits-d.cloud 2026-08-09: a pre-cutover agent hit
|
||||||
|
/// the stopped legacy Function App and support chased a non-existent authorization problem.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class BackendApiClientAuthClassificationTests
|
||||||
|
{
|
||||||
|
private sealed class StubHandler : HttpMessageHandler
|
||||||
|
{
|
||||||
|
private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
|
||||||
|
public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder) => _responder = responder;
|
||||||
|
|
||||||
|
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||||
|
=> Task.FromResult(_responder(request));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType)
|
||||||
|
{
|
||||||
|
var handler = new StubHandler(_ => new HttpResponseMessage(status)
|
||||||
|
{
|
||||||
|
Content = new StringContent(body, Encoding.UTF8, mediaType)
|
||||||
|
});
|
||||||
|
return new BackendApiClient(
|
||||||
|
new HttpClient(handler),
|
||||||
|
baseUrl: "https://backend.example",
|
||||||
|
manufacturer: "Contoso",
|
||||||
|
model: "TestBox",
|
||||||
|
serialNumber: "SN123",
|
||||||
|
useBootstrapTokenAuth: false,
|
||||||
|
bootstrapToken: null,
|
||||||
|
agentVersion: "2.0.0",
|
||||||
|
logger: null);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Backend_json_403_is_reported_as_device_not_authorized()
|
||||||
|
{
|
||||||
|
using var client = BuildClient(
|
||||||
|
HttpStatusCode.Forbidden,
|
||||||
|
"{\"success\":false,\"message\":\"Device validation failed\"}",
|
||||||
|
"application/json");
|
||||||
|
|
||||||
|
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||||
|
|
||||||
|
Assert.Equal(403, ex.StatusCode);
|
||||||
|
Assert.False(ex.EndpointUnavailable);
|
||||||
|
Assert.Contains("The device is not authorized", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Platform_html_403_is_reported_as_endpoint_unavailable_with_page_title()
|
||||||
|
{
|
||||||
|
using var client = BuildClient(
|
||||||
|
HttpStatusCode.Forbidden,
|
||||||
|
"<!DOCTYPE html><html><head><title>Web App - Unavailable</title></head><body>stopped</body></html>",
|
||||||
|
"text/html");
|
||||||
|
|
||||||
|
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||||
|
|
||||||
|
Assert.Equal(403, ex.StatusCode);
|
||||||
|
Assert.True(ex.EndpointUnavailable);
|
||||||
|
Assert.Contains("Web App - Unavailable", ex.Message);
|
||||||
|
Assert.Contains("NOT a device-authorization failure", ex.Message);
|
||||||
|
Assert.DoesNotContain("Check client certificate", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Html_body_with_generic_content_type_is_still_sniffed_as_platform_page()
|
||||||
|
{
|
||||||
|
using var client = BuildClient(
|
||||||
|
HttpStatusCode.Forbidden,
|
||||||
|
" <html><body>Error 403 - This web app is stopped.</body></html>",
|
||||||
|
"text/plain");
|
||||||
|
|
||||||
|
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||||
|
|
||||||
|
Assert.True(ex.EndpointUnavailable);
|
||||||
|
Assert.Contains("HTML platform error page", ex.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task Backend_401_keeps_authorization_message_and_status()
|
||||||
|
{
|
||||||
|
using var client = BuildClient(
|
||||||
|
HttpStatusCode.Unauthorized,
|
||||||
|
"{\"success\":false,\"message\":\"certificate rejected\"}",
|
||||||
|
"application/json");
|
||||||
|
|
||||||
|
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||||
|
|
||||||
|
Assert.Equal(401, ex.StatusCode);
|
||||||
|
Assert.False(ex.EndpointUnavailable);
|
||||||
|
Assert.Contains("The device is not authorized", ex.Message);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -88,6 +88,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public static string BackendExpectedSha256 { get; set; }
|
public static string BackendExpectedSha256 { get; set; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// True once a <see cref="CheckAndApplyUpdateAsync"/> run actually obtained a version
|
||||||
|
/// from the manifest (regardless of whether an update was then needed/applied).
|
||||||
|
/// <para>
|
||||||
|
/// Consumed by the <c>--await-enrollment</c> path: the startup check runs pre-enrollment
|
||||||
|
/// where OOBE frequently has no network yet, so a failed check there says nothing about
|
||||||
|
/// the latest version. Once the MDM certificate appears the device provably has
|
||||||
|
/// connectivity — the bootstrap retries the check exactly when this is still false.
|
||||||
|
/// </para>
|
||||||
|
/// </summary>
|
||||||
|
public static bool LastVersionCheckSucceeded { get; private set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Optional: graceful-shutdown request hook, set by the runtime host once its
|
/// Optional: graceful-shutdown request hook, set by the runtime host once its
|
||||||
/// shutdown machinery is armed (before the orchestrator starts) and cleared in its
|
/// shutdown machinery is armed (before the orchestrator starts) and cleared in its
|
||||||
@@ -130,14 +142,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
/// to cover hotfixes that reuse the same version number.
|
/// to cover hotfixes that reuse the same version number.
|
||||||
/// </param>
|
/// </param>
|
||||||
/// <param name="triggerReason">
|
/// <param name="triggerReason">
|
||||||
/// "startup" (default) or "runtime_hash_mismatch". Recorded in the marker file so the
|
/// "startup" (default), "await_enrollment" (post-cert retry in the bootstrap), or
|
||||||
/// agent_version_check event (outcome=updated) can distinguish the two paths.
|
/// "runtime_hash_mismatch". Recorded in the marker file so the agent_version_check
|
||||||
|
/// event (outcome=updated) can distinguish the paths.
|
||||||
/// </param>
|
/// </param>
|
||||||
/// <param name="downloadTimeoutMsOverride">
|
/// <param name="downloadTimeoutMsOverride">
|
||||||
/// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve
|
/// Overrides the ZIP download timeout. Startup path leaves null (fast-fail, 10s) to preserve
|
||||||
/// Autopilot boot speed. Runtime trigger passes 60000 because monitoring is already running
|
/// Autopilot boot speed. Runtime trigger passes 60000 because monitoring is already running
|
||||||
/// and we know an update is required.
|
/// and we know an update is required.
|
||||||
/// </param>
|
/// </param>
|
||||||
|
/// <param name="versionCheckTimeoutMsOverride">
|
||||||
|
/// Overrides the version-v2.json fetch timeout (default 2500ms, tuned for boot speed).
|
||||||
|
/// The await-enrollment retry passes a relaxed value: enrollment just completed, the
|
||||||
|
/// device provably has network, and a stranded old build's last self-rescue chance is
|
||||||
|
/// worth a few extra seconds before session registration.
|
||||||
|
/// </param>
|
||||||
/// <param name="allowDowngrade">
|
/// <param name="allowDowngrade">
|
||||||
/// When false (default), a latest version strictly lower than <paramref name="currentVersion"/>
|
/// When false (default), a latest version strictly lower than <paramref name="currentVersion"/>
|
||||||
/// is rejected even when <paramref name="forceUpdate"/> is true — prevents the production
|
/// is rejected even when <paramref name="forceUpdate"/> is true — prevents the production
|
||||||
@@ -151,7 +170,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
bool forceUpdate = false,
|
bool forceUpdate = false,
|
||||||
string triggerReason = "startup",
|
string triggerReason = "startup",
|
||||||
int? downloadTimeoutMsOverride = null,
|
int? downloadTimeoutMsOverride = null,
|
||||||
bool allowDowngrade = false)
|
bool allowDowngrade = false,
|
||||||
|
int? versionCheckTimeoutMsOverride = null)
|
||||||
{
|
{
|
||||||
Action<string> log = msg =>
|
Action<string> log = msg =>
|
||||||
{
|
{
|
||||||
@@ -160,7 +180,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
};
|
};
|
||||||
|
|
||||||
var isStartupTrigger = string.Equals(triggerReason, "startup", StringComparison.Ordinal);
|
var isStartupTrigger = string.Equals(triggerReason, "startup", StringComparison.Ordinal);
|
||||||
|
// The await-enrollment retry replaces the startup verdict — its markers must win so
|
||||||
|
// the agent_version_check event reflects the LATEST check, not the pre-network one.
|
||||||
|
var writeMarkers = isStartupTrigger || string.Equals(triggerReason, "await_enrollment", StringComparison.Ordinal);
|
||||||
var downloadTimeoutMs = downloadTimeoutMsOverride ?? DownloadTimeoutMs;
|
var downloadTimeoutMs = downloadTimeoutMsOverride ?? DownloadTimeoutMs;
|
||||||
|
var versionCheckTimeoutMs = versionCheckTimeoutMsOverride ?? VersionCheckTimeoutMs;
|
||||||
var totalSw = Stopwatch.StartNew();
|
var totalSw = Stopwatch.StartNew();
|
||||||
long versionCheckMs = 0, downloadMs = 0, verifyMs = 0, extractMs = 0, swapMs = 0;
|
long versionCheckMs = 0, downloadMs = 0, verifyMs = 0, extractMs = 0, swapMs = 0;
|
||||||
long zipSizeBytes = 0;
|
long zipSizeBytes = 0;
|
||||||
@@ -174,16 +198,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Step 1: Fetch version-v2.json (2.5s timeout)
|
// Step 1: Fetch version-v2.json (default 2.5s timeout, relaxed on retry triggers)
|
||||||
var versionSw = Stopwatch.StartNew();
|
var versionSw = Stopwatch.StartNew();
|
||||||
string manifestSha256;
|
string manifestSha256;
|
||||||
(latestVersion, manifestSha256) = await GetLatestVersionAsync(log);
|
(latestVersion, manifestSha256) = await GetLatestVersionAsync(log, versionCheckTimeoutMs);
|
||||||
versionSw.Stop();
|
versionSw.Stop();
|
||||||
versionCheckMs = versionSw.ElapsedMilliseconds;
|
versionCheckMs = versionSw.ElapsedMilliseconds;
|
||||||
|
LastVersionCheckSucceeded = latestVersion != null;
|
||||||
|
|
||||||
if (latestVersion == null)
|
if (latestVersion == null)
|
||||||
{
|
{
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("version_check_failed", currentVersion, null, "version-v2.json fetch failed or timed out", log);
|
WriteSkipMarker("version_check_failed", currentVersion, null, "version-v2.json fetch failed or timed out", log);
|
||||||
return; // Could not determine latest version — continue with current
|
return; // Could not determine latest version — continue with current
|
||||||
}
|
}
|
||||||
@@ -213,7 +238,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
if (!forceUpdate)
|
if (!forceUpdate)
|
||||||
{
|
{
|
||||||
log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})");
|
log($"Self-update: current version {currentVersion} is up to date (latest: {latestVersion})");
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log);
|
WriteCheckedMarker(currentVersion, latestVersion, versionCheckMs, log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -230,7 +255,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
if (!forceUpdate)
|
if (!forceUpdate)
|
||||||
{
|
{
|
||||||
log($"Self-update: cannot compare versions (current={currentVersion}, latest={latestVersion}) — skipping");
|
log($"Self-update: cannot compare versions (current={currentVersion}, latest={latestVersion}) — skipping");
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("version_compare_failed", currentVersion, latestVersion, "version parse failed", log);
|
WriteSkipMarker("version_compare_failed", currentVersion, latestVersion, "version parse failed", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -249,7 +274,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
downloadMs = downloadTimerSw.ElapsedMilliseconds;
|
downloadMs = downloadTimerSw.ElapsedMilliseconds;
|
||||||
if (!downloadOk)
|
if (!downloadOk)
|
||||||
{
|
{
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("download_failed", currentVersion, latestVersion, $"ZIP download failed within {downloadTimeoutMs}ms", log);
|
WriteSkipMarker("download_failed", currentVersion, latestVersion, $"ZIP download failed within {downloadTimeoutMs}ms", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -282,7 +307,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
if (!integrityOk)
|
if (!integrityOk)
|
||||||
{
|
{
|
||||||
CleanupStaging(null, zipPath);
|
CleanupStaging(null, zipPath);
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("integrity_mismatch", currentVersion, latestVersion, "SHA-256 mismatch on downloaded ZIP", log);
|
WriteSkipMarker("integrity_mismatch", currentVersion, latestVersion, "SHA-256 mismatch on downloaded ZIP", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -297,7 +322,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
extractMs = extractSw.ElapsedMilliseconds;
|
extractMs = extractSw.ElapsedMilliseconds;
|
||||||
if (!extractOk)
|
if (!extractOk)
|
||||||
{
|
{
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("extract_failed", currentVersion, latestVersion, "ZIP extraction failed", log);
|
WriteSkipMarker("extract_failed", currentVersion, latestVersion, "ZIP extraction failed", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -310,7 +335,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
{
|
{
|
||||||
log("Self-update: staging validation failed — AutopilotMonitor.Agent.exe not found in ZIP");
|
log("Self-update: staging validation failed — AutopilotMonitor.Agent.exe not found in ZIP");
|
||||||
CleanupStaging(stagingDir, zipPath);
|
CleanupStaging(stagingDir, zipPath);
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("extract_failed", currentVersion, latestVersion, "AutopilotMonitor.Agent.exe not found in ZIP", log);
|
WriteSkipMarker("extract_failed", currentVersion, latestVersion, "AutopilotMonitor.Agent.exe not found in ZIP", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -323,7 +348,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
if (!swapOk)
|
if (!swapOk)
|
||||||
{
|
{
|
||||||
CleanupStaging(stagingDir, zipPath);
|
CleanupStaging(stagingDir, zipPath);
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
WriteSkipMarker("swap_failed", currentVersion, latestVersion, "file swap failed", log);
|
WriteSkipMarker("swap_failed", currentVersion, latestVersion, "file swap failed", log);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -355,7 +380,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
log($"Self-update: unexpected error — {ex.Message}. Continuing with current version.");
|
log($"Self-update: unexpected error — {ex.Message}. Continuing with current version.");
|
||||||
if (isStartupTrigger)
|
if (writeMarkers)
|
||||||
{
|
{
|
||||||
try { WriteSkipMarker("unexpected_error", currentVersion, latestVersion, ex.Message, log); } catch { }
|
try { WriteSkipMarker("unexpected_error", currentVersion, latestVersion, ex.Message, log); } catch { }
|
||||||
}
|
}
|
||||||
@@ -366,7 +391,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
/// Fetches version-v2.json from blob storage and returns the version string and optional SHA-256 hash.
|
/// Fetches version-v2.json from blob storage and returns the version string and optional SHA-256 hash.
|
||||||
/// Returns (null, null) if the check fails or times out.
|
/// Returns (null, null) if the check fails or times out.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static async Task<(string version, string sha256)> GetLatestVersionAsync(Action<string> log)
|
private static async Task<(string version, string sha256)> GetLatestVersionAsync(Action<string> log, int timeoutMs)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@@ -375,7 +400,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
var versionUrl = $"{Constants.AgentDownloadBaseUrl}/{Constants.AgentVersionFileNameForLine(2)}";
|
var versionUrl = $"{Constants.AgentDownloadBaseUrl}/{Constants.AgentVersionFileNameForLine(2)}";
|
||||||
|
|
||||||
using (var handler = new HttpClientHandler())
|
using (var handler = new HttpClientHandler())
|
||||||
using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(VersionCheckTimeoutMs) })
|
using (var client = new HttpClient(handler) { Timeout = TimeSpan.FromMilliseconds(timeoutMs) })
|
||||||
{
|
{
|
||||||
var json = await client.GetStringAsync(versionUrl);
|
var json = await client.GetStringAsync(versionUrl);
|
||||||
var obj = JObject.Parse(json);
|
var obj = JObject.Parse(json);
|
||||||
@@ -396,7 +421,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
|||||||
}
|
}
|
||||||
catch (TaskCanceledException)
|
catch (TaskCanceledException)
|
||||||
{
|
{
|
||||||
log($"Self-update: version check timed out ({VersionCheckTimeoutMs}ms) — skipping update");
|
log($"Self-update: version check timed out ({timeoutMs}ms) — skipping update");
|
||||||
return (null, null);
|
return (null, null);
|
||||||
}
|
}
|
||||||
catch (HttpRequestException ex) when (ex.Message.Contains("404"))
|
catch (HttpRequestException ex) when (ex.Message.Contains("404"))
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
ThrowOnAuthFailure(response);
|
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||||
@@ -236,7 +236,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
|||||||
|
|
||||||
using (response)
|
using (response)
|
||||||
{
|
{
|
||||||
ThrowOnAuthFailure(response);
|
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||||
@@ -282,17 +282,98 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Throws <see cref="BackendAuthException"/> for 401/403 responses so callers can
|
/// Throws <see cref="BackendAuthException"/> for 401/403 responses so callers can
|
||||||
/// distinguish authentication failures from transient server errors.
|
/// distinguish authentication failures from transient server errors.
|
||||||
|
/// <para>
|
||||||
|
/// Field case (sits-d.cloud, 2026-08-09): a pre-cutover agent build pointed at a
|
||||||
|
/// retired Function App; Azure answers for a stopped app with a platform-level 403
|
||||||
|
/// carrying an HTML error page ("Web App - Unavailable"). Our backend never returns
|
||||||
|
/// HTML on 401/403 — an HTML body therefore means the response did not come from the
|
||||||
|
/// backend's auth layer at all (stopped/retired app, edge proxy, WAF). Reporting that
|
||||||
|
/// as "device is not authorized" sends troubleshooting down the wrong path, so the
|
||||||
|
/// two cases get distinct messages and the exception carries a flag.
|
||||||
|
/// </para>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static void ThrowOnAuthFailure(HttpResponseMessage response)
|
private static async Task ThrowOnAuthFailureAsync(HttpResponseMessage response)
|
||||||
{
|
{
|
||||||
if (response.StatusCode == System.Net.HttpStatusCode.Unauthorized ||
|
if (response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
|
||||||
response.StatusCode == System.Net.HttpStatusCode.Forbidden)
|
response.StatusCode != System.Net.HttpStatusCode.Forbidden)
|
||||||
{
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var statusText = $"{(int)response.StatusCode} {response.StatusCode}";
|
||||||
|
|
||||||
|
string body = null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
body = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Body is diagnostic-only; classification below treats null as non-HTML.
|
||||||
|
}
|
||||||
|
|
||||||
|
if (LooksLikeHtmlErrorPage(response, body))
|
||||||
|
{
|
||||||
|
var title = TryExtractHtmlTitle(body);
|
||||||
|
var titleText = title != null ? $" (\"{title}\")" : string.Empty;
|
||||||
|
var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)";
|
||||||
throw new BackendAuthException(
|
throw new BackendAuthException(
|
||||||
$"Backend returned {(int)response.StatusCode} {response.StatusCode}. " +
|
$"Backend returned {statusText} with an HTML platform error page{titleText} from '{host}'. " +
|
||||||
|
"This response did not come from the Autopilot Monitor backend — the endpoint appears stopped, " +
|
||||||
|
"retired, or intercepted by a proxy. This is NOT a device-authorization failure; an outdated " +
|
||||||
|
"agent build pointing at a stale ApiBaseUrl is the most likely cause.",
|
||||||
|
(int)response.StatusCode,
|
||||||
|
endpointUnavailable: true);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new BackendAuthException(
|
||||||
|
$"Backend returned {statusText}. " +
|
||||||
"The device is not authorized. Check client certificate and Autopilot device validation.",
|
"The device is not authorized. Check client certificate and Autopilot device validation.",
|
||||||
(int)response.StatusCode);
|
(int)response.StatusCode);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// True when a 401/403 response is an HTML page rather than a backend JSON/plain-text
|
||||||
|
/// reply. Checks the declared Content-Type first, then falls back to sniffing the body
|
||||||
|
/// prefix so a platform page served with a generic Content-Type is still recognized.
|
||||||
|
/// </summary>
|
||||||
|
private static bool LooksLikeHtmlErrorPage(HttpResponseMessage response, string body)
|
||||||
|
{
|
||||||
|
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
|
||||||
|
if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase))
|
||||||
|
return true;
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
var head = body.TrimStart();
|
||||||
|
return head.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase) ||
|
||||||
|
head.StartsWith("<html", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so the
|
||||||
|
/// log line names the platform page without dumping markup. Null when absent/oversized.
|
||||||
|
/// </summary>
|
||||||
|
private static string TryExtractHtmlTitle(string body)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(body)) return null;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var match = System.Text.RegularExpressions.Regex.Match(
|
||||||
|
body, @"<title[^>]*>\s*(.*?)\s*</title>",
|
||||||
|
System.Text.RegularExpressions.RegexOptions.IgnoreCase |
|
||||||
|
System.Text.RegularExpressions.RegexOptions.Singleline,
|
||||||
|
TimeSpan.FromMilliseconds(250));
|
||||||
|
if (!match.Success) return null;
|
||||||
|
|
||||||
|
var title = match.Groups[1].Value;
|
||||||
|
return title.Length == 0 ? null : (title.Length > 80 ? title.Substring(0, 80) : title);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -330,7 +411,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Thrown when the backend returns 401 or 403, indicating the device is not authorized.
|
/// Thrown when the backend returns 401 or 403, indicating the device is not authorized —
|
||||||
|
/// or, when <see cref="EndpointUnavailable"/> is set, that the 401/403 was a platform-level
|
||||||
|
/// HTML response (stopped/retired app, edge proxy) rather than a backend auth verdict.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class BackendAuthException : Exception
|
public class BackendAuthException : Exception
|
||||||
{
|
{
|
||||||
@@ -340,9 +423,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public int StatusCode { get; }
|
public int StatusCode { get; }
|
||||||
|
|
||||||
public BackendAuthException(string message, int statusCode = 0) : base(message)
|
/// <summary>
|
||||||
|
/// True when the 401/403 carried an HTML platform error page — the endpoint itself is
|
||||||
|
/// stopped/retired/proxied and the status says nothing about this device's authorization.
|
||||||
|
/// </summary>
|
||||||
|
public bool EndpointUnavailable { get; }
|
||||||
|
|
||||||
|
public BackendAuthException(string message, int statusCode = 0, bool endpointUnavailable = false) : base(message)
|
||||||
{
|
{
|
||||||
StatusCode = statusCode;
|
StatusCode = statusCode;
|
||||||
|
EndpointUnavailable = endpointUnavailable;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -121,6 +121,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
|||||||
|
|
||||||
// Await-enrollment is one-shot — remove the persisted config so subsequent restarts proceed normally.
|
// Await-enrollment is one-shot — remove the persisted config so subsequent restarts proceed normally.
|
||||||
Program.DeleteAwaitEnrollmentConfig(dataDirectory, logger);
|
Program.DeleteAwaitEnrollmentConfig(dataDirectory, logger);
|
||||||
|
|
||||||
|
RetrySelfUpdateAfterEnrollment(logger, consoleMode);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Event-driven TenantId wait — kicks in only when the user opted-in via
|
// Event-driven TenantId wait — kicks in only when the user opted-in via
|
||||||
@@ -152,6 +154,46 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
|||||||
isWhiteGloveResume: isWhiteGloveResume,
|
isWhiteGloveResume: isWhiteGloveResume,
|
||||||
cleanupServiceFactory: cleanupServiceFactory);
|
cleanupServiceFactory: cleanupServiceFactory);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// One-shot self-update retry after the MDM certificate appeared. The startup check in
|
||||||
|
/// <c>Main</c> runs pre-enrollment, where an --await-enrollment install typically has no
|
||||||
|
/// network yet — its failure says nothing about the latest version. The certificate is
|
||||||
|
/// proof of connectivity, and this is the last chance to swap a stranded old build
|
||||||
|
/// (stale baked-in ApiBaseUrl) for a current one BEFORE session registration hits a
|
||||||
|
/// retired endpoint (field case sits-d.cloud, 2026-08-09). Timeouts are relaxed vs. the
|
||||||
|
/// boot-speed-tuned startup check; enrollment is already past OOBE's critical path here.
|
||||||
|
/// On update success the process restarts and this method never returns; the
|
||||||
|
/// await-enrollment config was already removed, so the new binary proceeds normally.
|
||||||
|
/// </summary>
|
||||||
|
private static void RetrySelfUpdateAfterEnrollment(AgentLogger logger, bool consoleMode)
|
||||||
|
{
|
||||||
|
if (SelfUpdater.LastVersionCheckSucceeded)
|
||||||
|
return; // Startup check reached the manifest — its verdict stands.
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
logger.Info("Await-enrollment: startup self-update check had failed (no network in early OOBE) — retrying now that enrollment proves connectivity.");
|
||||||
|
var agentDir = System.IO.Path.GetDirectoryName(
|
||||||
|
System.Reflection.Assembly.GetExecutingAssembly().Location) ?? string.Empty;
|
||||||
|
SelfUpdater.CheckAndApplyUpdateAsync(
|
||||||
|
currentVersion: Program.GetAgentVersion(),
|
||||||
|
agentDir: agentDir,
|
||||||
|
consoleMode: consoleMode,
|
||||||
|
triggerReason: "await_enrollment",
|
||||||
|
downloadTimeoutMsOverride: 60000,
|
||||||
|
// Fresh --await-enrollment installs have no cached remote config yet, so
|
||||||
|
// there is no admin downgrade override to honour on this path.
|
||||||
|
allowDowngrade: false,
|
||||||
|
versionCheckTimeoutMsOverride: 10000).GetAwaiter().GetResult();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// Same contract as the startup call site: the updater swallows its own errors;
|
||||||
|
// anything escaping must not break bootstrap — the current binary continues.
|
||||||
|
logger.Warning($"Await-enrollment: self-update retry failed unexpectedly: {ex.Message}. Continuing with current version.");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
Reference in New Issue
Block a user