mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-23 20:26:10 +00:00
Agent: an intercepted connection is no longer an auth failure
A security client that inspects TLS strips the device certificate on the way to the backend; the platform then answers 403 "Client Certificate Required" itself, or a proxy answers with its own page. The agent counted these like backend rejections: five in a row shut it down, the scheduled task restarted it at every boot, and only the 48-hour emergency break removed it. - Origin of every 401/403: only a response that echoes the request's X-Correlation-ID is the backend's verdict and counts toward the auth shutdown; the stopped-app page still counts; anything else is foreign (client_certificate_required or foreign_response) and never counts. - Telemetry: a foreign answer keeps the batch and pauses uploads with a backoff from 30 s up to 15 min, without a timer; collectors and the decision engine keep running, the terminal phase gets one upload attempt, and the agent still leaves the device when the enrollment ends. - Distress type ClientCertificateNotReceived (9), once per process, with operation, status, page title and the issuer of the server certificate; both handlers observe it through a validation callback that returns exactly sslPolicyErrors == None. - Registration: an intercepted start retries on at most InterceptedRegistrationRetryBoots further boots (config channel, default 1, agent clamp 0..5, ConfigVersion 41), then ends monitoring like the emergency break. The break, this give-up and the config kill signal now share Program.EndMonitoringAtStart. - Evidence: telemetry_upload_blocked carries reason, operation, statusCode, pageTitle, serverCertIssuer and firstBlockedAtUtc; agent_started carries priorRegistration* after failed starts (a local network fingerprint; only whether it changed leaves the device); agent_metrics_snapshot counts 2xx responses without the echo. - Backend: distress type 9, the buffer default, InterceptedRuns on the registration-failure summary; tests pin the correlation echo on 401/403 and the middleware's place in the pipeline. - Web: distress labels for TPM Incompatible and Cert Not Received; config report row. - MCP: session coverage names intercepted uploads and the failed registrations before the start. D-349 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+1
@@ -50,6 +50,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
nameof(AgentConfigResponse.MaxBatchSize),
|
||||
nameof(AgentConfigResponse.MaxAuthFailures),
|
||||
nameof(AgentConfigResponse.AuthFailureTimeoutMinutes),
|
||||
nameof(AgentConfigResponse.InterceptedRegistrationRetryBoots),
|
||||
// Merge reads two Collectors knobs (AgentMaxLifetimeMinutes, HelloWaitTimeoutSeconds);
|
||||
// the remaining CollectorConfiguration flags flow untouched to DefaultComponentFactory.
|
||||
nameof(AgentConfigResponse.Collectors),
|
||||
|
||||
+46
@@ -55,6 +55,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
UnrestrictedMode = true,
|
||||
MaxAuthFailures = 7,
|
||||
AuthFailureTimeoutMinutes = 15,
|
||||
InterceptedRegistrationRetryBoots = 2,
|
||||
EnableImeMatchLog = true,
|
||||
Collectors = new CollectorConfiguration
|
||||
{
|
||||
@@ -98,6 +99,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(45, agent.HelloWaitTimeoutSeconds);
|
||||
Assert.Equal(7, agent.MaxAuthFailures);
|
||||
Assert.Equal(15, agent.AuthFailureTimeoutMinutes);
|
||||
Assert.Equal(2, agent.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.True(result.UnrestrictedModeChanged);
|
||||
@@ -468,6 +470,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(0, agent.AgentMaxLifetimeMinutes);
|
||||
}
|
||||
|
||||
// ============================================================= Intercepted-registration buffer
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)] // clean up at the first intercepted registration
|
||||
[InlineData(3)]
|
||||
public void Merge_copies_intercepted_registration_retry_boots(int remoteValue)
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = remoteValue;
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(remoteValue, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Merge_keeps_the_agent_default_when_the_backend_sends_no_intercepted_registration_retry_boots()
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = null; // backend that predates the field
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(1, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(-1, 0)]
|
||||
[InlineData(int.MinValue, 0)]
|
||||
[InlineData(6, 5)]
|
||||
[InlineData(int.MaxValue, 5)]
|
||||
public void Merge_clamps_intercepted_registration_retry_boots_to_zero_through_five(int remoteValue, int expected)
|
||||
{
|
||||
var agent = NewAgentConfig();
|
||||
var remote = FullRemote();
|
||||
remote.InterceptedRegistrationRetryBoots = remoteValue;
|
||||
|
||||
RemoteConfigMerger.Merge(agent, remote);
|
||||
|
||||
Assert.Equal(expected, agent.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
// ============================================================= Guard
|
||||
|
||||
[Fact]
|
||||
|
||||
+17
@@ -214,6 +214,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Null(loaded.LatestAgentSha256);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(3)]
|
||||
[InlineData(null)] // backend that predates the field — the agent keeps its default
|
||||
public void RoundTrip_preserves_intercepted_registration_retry_boots(int? value)
|
||||
{
|
||||
// An intercepted start gets no live config, so the registration buffer it decides on
|
||||
// is the cached value: it must survive the write + read strip untouched.
|
||||
var probe = NewProbe();
|
||||
|
||||
probe.DoCache(new AgentConfigResponse { InterceptedRegistrationRetryBoots = value });
|
||||
var loaded = probe.DoLoad();
|
||||
|
||||
Assert.NotNull(loaded);
|
||||
Assert.Equal(value, loaded.InterceptedRegistrationRetryBoots);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
private CacheProbe NewProbe()
|
||||
|
||||
+31
-4
@@ -7,6 +7,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -101,6 +102,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(401, svc.LastFetchAuthStatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_InterceptedAnswer_FallsBack_WithoutCounting_AndReportsOnce()
|
||||
{
|
||||
// A 403 something on the network path answered (TLS inspection): same fallback as an
|
||||
// auth failure, but the tracker does not count it — it sends ClientCertificateNotReceived.
|
||||
var distress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0,
|
||||
AutopilotMonitor.DecisionCore.Engine.SystemClock.Instance, _logger, distress);
|
||||
var svc = NewService(BackendBehaviour.AlwaysIntercepted403, tracker);
|
||||
|
||||
var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
|
||||
Assert.Equal(1, svc.LastFetchAttempts);
|
||||
Assert.Equal(403, svc.LastFetchAuthStatusCode);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
var call = Assert.Single(distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.StartsWith("agent/config http 403 client_certificate_required", call.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_RetryDisabled_OneAttemptOnly()
|
||||
{
|
||||
@@ -252,10 +275,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
/// <summary>All 4 initial attempts fail; the NEXT call (the recovery fetch) succeeds.</summary>
|
||||
FailAllThenSucceed,
|
||||
AlwaysAuthFailure401,
|
||||
AlwaysIntercepted403,
|
||||
AlwaysReturnNull,
|
||||
}
|
||||
|
||||
private FakeRemoteConfigService NewService(BackendBehaviour behaviour)
|
||||
private FakeRemoteConfigService NewService(BackendBehaviour behaviour, AuthFailureTracker? tracker = null)
|
||||
{
|
||||
// Pass a throwaway BackendApiClient — the fake overrides CallBackendAsync
|
||||
// before it ever hits the wire.
|
||||
@@ -269,7 +293,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
bootstrapToken: null,
|
||||
agentVersion: "0.0.0",
|
||||
logger: _logger);
|
||||
return new FakeRemoteConfigService(apiClient, _logger, behaviour);
|
||||
return new FakeRemoteConfigService(apiClient, _logger, behaviour, tracker);
|
||||
}
|
||||
|
||||
private sealed class FakeRemoteConfigService : RemoteConfigService
|
||||
@@ -277,8 +301,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
private readonly BackendBehaviour _behaviour;
|
||||
private int _calls;
|
||||
|
||||
public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour)
|
||||
: base(apiClient, "00000000-0000-0000-0000-000000000001", logger)
|
||||
public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour, AuthFailureTracker? tracker)
|
||||
: base(apiClient, "00000000-0000-0000-0000-000000000001", logger, authFailureTracker: tracker)
|
||||
{
|
||||
_behaviour = behaviour;
|
||||
}
|
||||
@@ -305,6 +329,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
throw new HttpRequestException("simulated network failure");
|
||||
case BackendBehaviour.AlwaysAuthFailure401:
|
||||
throw new BackendAuthException("Unauthorized", 401);
|
||||
case BackendBehaviour.AlwaysIntercepted403:
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", 403,
|
||||
classification: InterceptionFixtures.CertificateRequired());
|
||||
case BackendBehaviour.AlwaysReturnNull:
|
||||
return Task.FromResult<AgentConfigResponse>(null!);
|
||||
default:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Harness
|
||||
{
|
||||
/// <summary>
|
||||
/// Records every distress dispatch instead of sending it. The real anti-flood rules of
|
||||
/// <see cref="DistressReporter"/> are bypassed on purpose: tests assert what the caller dispatched.
|
||||
/// </summary>
|
||||
internal sealed class FakeDistressReporter : DistressReporter
|
||||
{
|
||||
public sealed class Call
|
||||
{
|
||||
public DistressErrorType ErrorType { get; set; }
|
||||
public string Message { get; set; } = default!;
|
||||
public int? HttpStatusCode { get; set; }
|
||||
}
|
||||
|
||||
public List<Call> Calls { get; } = new List<Call>();
|
||||
|
||||
/// <summary>
|
||||
/// The send each dispatch returns; null = already completed. Lets a test keep a send in
|
||||
/// flight to prove a caller waits for it (or stops waiting at its bound).
|
||||
/// </summary>
|
||||
public Func<Task>? Send { get; set; }
|
||||
|
||||
public FakeDistressReporter()
|
||||
: base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null)
|
||||
{
|
||||
Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode });
|
||||
return Send?.Invoke() ?? Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// The local network fingerprint a failed registration is stored with: SHA-256 over the
|
||||
/// connection type and gateway <see cref="NetworkChangeDetector"/> captures, cut to 16 hex
|
||||
/// characters; unknown (null) whenever either input is missing. Gateways are documentation
|
||||
/// addresses (RFC 5737).
|
||||
/// </summary>
|
||||
public sealed class NetworkFingerprintTests
|
||||
{
|
||||
private static NetworkChangeDetector.NetworkStateSnapshot State(string? connectionType, string? gateway, bool hasNetwork = true) =>
|
||||
new NetworkChangeDetector.NetworkStateSnapshot
|
||||
{
|
||||
HasNetwork = hasNetwork,
|
||||
ConnectionType = connectionType,
|
||||
Gateway = gateway,
|
||||
};
|
||||
|
||||
[Fact]
|
||||
public void Same_connection_type_and_gateway_give_the_same_short_hex_fingerprint()
|
||||
{
|
||||
var first = NetworkFingerprint.Of(State("WiFi", "192.0.2.1"));
|
||||
var second = NetworkFingerprint.Of(State("WiFi", "192.0.2.1"));
|
||||
|
||||
Assert.Equal(first, second);
|
||||
Assert.Equal(NetworkFingerprint.Length, first!.Length);
|
||||
Assert.True(first.All(c => (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')), first);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("WiFi", "198.51.100.1")] // another gateway
|
||||
[InlineData("Ethernet", "192.0.2.1")] // another connection type
|
||||
public void Another_gateway_or_connection_type_is_another_network(string connectionType, string gateway)
|
||||
{
|
||||
Assert.NotEqual(
|
||||
NetworkFingerprint.Of(State("WiFi", "192.0.2.1")),
|
||||
NetworkFingerprint.Of(State(connectionType, gateway)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_network_has_no_fingerprint()
|
||||
{
|
||||
Assert.Null(NetworkFingerprint.Of(null));
|
||||
Assert.Null(NetworkFingerprint.Of(State("WiFi", "192.0.2.1", hasNetwork: false)));
|
||||
Assert.Null(NetworkFingerprint.Of(State("WiFi", null)));
|
||||
Assert.Null(NetworkFingerprint.Of(State(null, "192.0.2.1")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("aaaa", "aaaa", false)]
|
||||
[InlineData("aaaa", "bbbb", true)]
|
||||
public void Differs_compares_known_fingerprints(string before, string after, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, NetworkFingerprint.Differs(before, after));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, "aaaa")]
|
||||
[InlineData("aaaa", null)]
|
||||
[InlineData(null, null)]
|
||||
public void Differs_is_unknown_when_either_side_is(string? before, string? after)
|
||||
{
|
||||
Assert.Null(NetworkFingerprint.Differs(before, after));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TryCapture_reads_this_machine_without_throwing()
|
||||
{
|
||||
// Machine-dependent: either no active interface (null) or a well-formed fingerprint
|
||||
// that matches the detector's own capture.
|
||||
var captured = NetworkFingerprint.TryCapture();
|
||||
|
||||
if (captured != null)
|
||||
{
|
||||
Assert.Equal(NetworkFingerprint.Length, captured.Length);
|
||||
Assert.Equal(NetworkFingerprint.Of(NetworkChangeDetector.CaptureInterfaceState()), captured);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// The orchestrator side of an intercepted upload: one <c>telemetry_upload_blocked</c> with the
|
||||
/// evidence (queued behind the intercepted batch, never uploaded while paused), and the one
|
||||
/// terminal attempt — requested by <c>DrainSpool</c> or made by <see cref="EnrollmentOrchestrator.Stop"/>.
|
||||
/// </summary>
|
||||
[Collection("SerialThreading")]
|
||||
public sealed class EnrollmentOrchestratorInterceptionTests
|
||||
{
|
||||
private const int WaitMs = 10_000;
|
||||
|
||||
private static TelemetryItemDraft Draft() =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "T1_S1",
|
||||
rowKey: "0000000001",
|
||||
payloadJson: "{}",
|
||||
isSessionScoped: true,
|
||||
requiresImmediateFlush: false);
|
||||
|
||||
// The signal and the decision transition of the same post mention the event type too.
|
||||
private static bool IsBlockedEvent(TelemetryItem item) =>
|
||||
item.Kind == TelemetryItemKind.Event
|
||||
&& item.PayloadJson.Contains(AutopilotMonitor.Shared.Constants.EventTypes.TelemetryUploadBlocked);
|
||||
|
||||
private static TelemetryItem? FindBlockedEvent(EnrollmentOrchestrator sut) =>
|
||||
sut.SpoolForTests!.Peek(1000).FirstOrDefault(IsBlockedEvent);
|
||||
|
||||
// Dates stay strings: the wire format of firstBlockedAtUtc is what is under test.
|
||||
private static JObject Parse(string json)
|
||||
{
|
||||
using var reader = new JsonTextReader(new StringReader(json)) { DateParseHandling = DateParseHandling.None };
|
||||
return JObject.Load(reader);
|
||||
}
|
||||
|
||||
/// <summary>Starts the orchestrator and drives the first upload into an interception.</summary>
|
||||
private static async Task<EnrollmentOrchestrator> StartInterceptedAsync(EnrollmentOrchestratorRig rig)
|
||||
{
|
||||
rig.Uploader.QueueIntercepted(5);
|
||||
var sut = rig.Build();
|
||||
sut.Start();
|
||||
sut.SpoolForTests!.Enqueue(Draft());
|
||||
await sut.DrainAsync();
|
||||
Assert.True(sut.Transport.IsInterceptionPaused);
|
||||
return sut;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Interception_queues_one_telemetry_upload_blocked_with_the_evidence_and_wakeups_stay_offline()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
try
|
||||
{
|
||||
TelemetryItem? blocked = null;
|
||||
Assert.True(SpinWait.SpinUntil(() => (blocked = FindBlockedEvent(sut)) != null, WaitMs),
|
||||
"telemetry_upload_blocked never reached the spool.");
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
var data = (JObject)Parse(blocked!.PayloadJson)["Data"]!;
|
||||
Assert.Equal(
|
||||
new[] { "firstBlockedAtUtc", "operation", "pageTitle", "reason", "serverCertIssuer", "statusCode" },
|
||||
data.Properties().Select(p => p.Name).OrderBy(n => n, StringComparer.Ordinal));
|
||||
Assert.Equal("client_certificate_required", (string?)data["reason"]);
|
||||
Assert.Equal("agent/telemetry", (string?)data["operation"]);
|
||||
Assert.Equal(JTokenType.Integer, data["statusCode"]!.Type);
|
||||
Assert.Equal(403, (int)data["statusCode"]!);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, (string?)data["pageTitle"]);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, (string?)data["serverCertIssuer"]);
|
||||
Assert.Equal(
|
||||
EnrollmentOrchestratorRig.DefaultClockStart.ToString("o", CultureInfo.InvariantCulture),
|
||||
(string?)data["firstBlockedAtUtc"]);
|
||||
|
||||
// The marker was an immediate-upload item; neither its wakeup nor a further drain
|
||||
// touches the network while the pause holds.
|
||||
var drain = await sut.DrainAsync();
|
||||
Assert.Contains("upload intercepted", drain.LastErrorReason);
|
||||
Assert.Equal(calls, rig.Uploader.CallCount);
|
||||
Assert.Single(sut.SpoolForTests!.Peek(1000), IsBlockedEvent);
|
||||
}
|
||||
finally
|
||||
{
|
||||
sut.Stop();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_attempt_requested_by_the_termination_runs_once_and_Stop_adds_none()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
Assert.True(SpinWait.SpinUntil(() => FindBlockedEvent(sut) != null, WaitMs));
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
sut.RequestTerminalUploadAttempt();
|
||||
Assert.True(SpinWait.SpinUntil(() => rig.Uploader.CallCount == calls + 1, WaitMs),
|
||||
"the woken drain loop did not make the terminal attempt.");
|
||||
|
||||
sut.Stop();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stop_makes_the_one_terminal_attempt_when_the_termination_did_not()
|
||||
{
|
||||
using var rig = new EnrollmentOrchestratorRig();
|
||||
var sut = await StartInterceptedAsync(rig);
|
||||
var calls = rig.Uploader.CallCount;
|
||||
|
||||
sut.Stop();
|
||||
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
}
|
||||
}
|
||||
}
|
||||
+32
-1
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
@@ -10,6 +11,7 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
@@ -96,6 +98,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
Assert.Equal(TimeSpan.FromSeconds(5), client.Timeout);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_without_observer_keeps_the_default_validation()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var cert = CreateSelfSignedCert();
|
||||
|
||||
using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger);
|
||||
|
||||
Assert.Null(handler.ServerCertificateCustomValidationCallback);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_with_observer_installs_an_observing_callback_that_never_weakens_validation()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
using var cert = CreateSelfSignedCert();
|
||||
var observer = new ServerCertificateObserver();
|
||||
|
||||
using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger, observer);
|
||||
var callback = handler.ServerCertificateCustomValidationCallback;
|
||||
|
||||
Assert.NotNull(callback);
|
||||
Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateChainErrors));
|
||||
Assert.True(callback(null!, cert, null!, SslPolicyErrors.None));
|
||||
Assert.Equal(cert.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CreateHandler_throws_when_resolver_returns_null()
|
||||
{
|
||||
@@ -161,7 +192,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
/// we scan the type hierarchy for the first <see cref="HttpMessageHandler"/>-typed
|
||||
/// instance field to stay version-tolerant.
|
||||
/// </summary>
|
||||
private static HttpMessageHandler GetPipelineHead(HttpClient client)
|
||||
internal static HttpMessageHandler GetPipelineHead(HttpClient client)
|
||||
{
|
||||
const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public;
|
||||
for (var t = (Type?)client.GetType(); t != null; t = t.BaseType)
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
using System.IO;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Runtime;
|
||||
using Xunit;
|
||||
|
||||
@@ -68,6 +74,43 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.False(bundle.HasClientCertificate);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildAuthClients_carries_the_server_certificate_observer_for_both_stacks()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var bundle = BackendClientFactory.BuildAuthClients(MinimalBootstrapTokenConfig(), agentVersion: "1.2.3.4", logger: NewLogger(tmp.Path));
|
||||
|
||||
Assert.NotNull(bundle.ServerCertificates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildBackendApiHttpClient_installs_the_observing_callback_on_the_inner_handler()
|
||||
{
|
||||
var observer = new ServerCertificateObserver();
|
||||
using var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA");
|
||||
|
||||
using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4", observer);
|
||||
var inner = InnerHandler(client);
|
||||
var callback = inner.ServerCertificateCustomValidationCallback;
|
||||
|
||||
Assert.NotNull(callback);
|
||||
Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateNameMismatch));
|
||||
Assert.True(callback(null!, cert, null!, SslPolicyErrors.None));
|
||||
Assert.Equal(cert.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildBackendApiHttpClient_without_observer_keeps_the_default_validation()
|
||||
{
|
||||
using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4");
|
||||
|
||||
Assert.Null(InnerHandler(client).ServerCertificateCustomValidationCallback);
|
||||
}
|
||||
|
||||
// Pipeline: NetworkMetricsRecordingHandler → HttpClientHandler.
|
||||
private static HttpClientHandler InnerHandler(HttpClient client) =>
|
||||
(HttpClientHandler)((DelegatingHandler)MtlsHttpClientFactoryTests.GetPipelineHead(client)).InnerHandler;
|
||||
|
||||
// =========================================================== TelemetryClientResult
|
||||
|
||||
[Fact]
|
||||
|
||||
+14
-12
@@ -122,20 +122,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
// ============================================================ failed-registration record
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
public void RecordFailedRun_records_a_failed_run_with_its_window_and_bounded_error()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
var returned = BackendSessionRegistration.RecordFailedRun(
|
||||
persistence, SessionRegistrationResult.Failed(new string('e', 1000)),
|
||||
started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true,
|
||||
configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path));
|
||||
configFetchOutcome: "UsedDefaults", networkFingerprint: null, logger: NewLogger(tmp.Path));
|
||||
|
||||
var record = persistence.LoadRegistrationFailure();
|
||||
Assert.NotNull(record);
|
||||
Assert.Equal(1, record!.FailedRuns);
|
||||
Assert.Equal(0, record.InterceptedRuns);
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length);
|
||||
Assert.False(record.NetworkLinkUpAtStart);
|
||||
@@ -143,33 +144,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.Equal(1204.5, record.AttemptWindowSeconds);
|
||||
Assert.Equal("UsedDefaults", record.ConfigFetchOutcome);
|
||||
Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc);
|
||||
// The caller decides on the record as written, including this run.
|
||||
Assert.Equal(1, returned!.FailedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_records_auth_failures_and_clears_on_success()
|
||||
public void RecordFailedRun_records_auth_failures_and_a_success_clears_them()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var persistence = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
BackendSessionRegistration.RecordFailedRun(
|
||||
persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"),
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
at, at.AddSeconds(3), true, true, "Succeeded", null, NewLogger(tmp.Path));
|
||||
Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome);
|
||||
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }),
|
||||
at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path));
|
||||
BackendSessionRegistration.ClearAfterSuccess(persistence, () => null, NewLogger(tmp.Path));
|
||||
Assert.Null(persistence.LoadRegistrationFailure());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationRun_without_persistence_is_a_no_op()
|
||||
public void Recording_without_persistence_is_a_no_op()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
BackendSessionRegistration.RecordRegistrationRun(
|
||||
Assert.Null(BackendSessionRegistration.RecordFailedRun(
|
||||
null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow,
|
||||
null, null, null, NewLogger(tmp.Path));
|
||||
null, null, null, null, NewLogger(tmp.Path)));
|
||||
Assert.Null(BackendSessionRegistration.ClearAfterSuccess(null, () => null, NewLogger(tmp.Path)));
|
||||
}
|
||||
|
||||
// ============================================================ emergency-break message
|
||||
|
||||
+477
@@ -0,0 +1,477 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Runtime;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
using ProgramAlias = AutopilotMonitor.Agent.V2.Program;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
{
|
||||
/// <summary>
|
||||
/// A registration that something on the network path answered at process start (client
|
||||
/// certificate interception plan, K4/K5): a buffer of further boots, then the emergency break's
|
||||
/// cleanup; the pending ClientCertificateNotReceived distress leaves the machine first; a later
|
||||
/// successful registration reports the failed starts on <c>agent_started</c>. Each start is
|
||||
/// ended the way <see cref="BackendSessionRegistration.Register"/> ends it: record the failed
|
||||
/// run, then conclude it.
|
||||
/// </summary>
|
||||
public sealed class InterceptedRegistrationTests : IDisposable
|
||||
{
|
||||
private static readonly DateTime At = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly SessionIdPersistence _persistence;
|
||||
private readonly string _stateDir;
|
||||
private int _cleanupCalls;
|
||||
|
||||
public InterceptedRegistrationTests()
|
||||
{
|
||||
_logger = new AgentLogger(Path.Combine(_tmp.Path, "logs"), AgentLogLevel.Info);
|
||||
_persistence = new SessionIdPersistence(_tmp.Path);
|
||||
_persistence.GetOrCreate(_logger);
|
||||
_stateDir = Path.Combine(_tmp.Path, "State");
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
// ============================================================ K4: buffer, then clean up
|
||||
|
||||
[Fact]
|
||||
public void Agent_default_buffer_allows_one_further_boot_then_cleans_up()
|
||||
{
|
||||
var config = new AgentConfiguration();
|
||||
Assert.Equal(1, config.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(1, record.FailedRuns);
|
||||
Assert.Equal(1, record.InterceptedRuns);
|
||||
Assert.Equal("AuthFailed", record.Outcome);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config)); // the give-up keeps exit 6
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)] // clean up at the first intercepted start
|
||||
[InlineData(2)]
|
||||
[InlineData(5)]
|
||||
public void Gives_up_on_the_first_intercepted_start_beyond_the_buffer(int buffer)
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = buffer };
|
||||
|
||||
for (var start = 1; start <= buffer; start++)
|
||||
{
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
Assert.False(GaveUp, $"gave up on start {start} with a buffer of {buffer}");
|
||||
}
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, 1)]
|
||||
[InlineData(false, 0)]
|
||||
public void Give_up_runs_the_cleanup_only_with_SelfDestructOnComplete_like_the_emergency_break(
|
||||
bool selfDestructOnComplete, int expectedCleanupCalls)
|
||||
{
|
||||
var config = new AgentConfiguration
|
||||
{
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
SelfDestructOnComplete = selfDestructOnComplete,
|
||||
};
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_intercepted_start_takes_the_buffer_from_the_cached_config()
|
||||
{
|
||||
// The config fetch of an intercepted start is intercepted too: it falls back to the
|
||||
// cache, and the registration decides on the cached buffer, not the agent default.
|
||||
var config = new AgentConfiguration();
|
||||
var service = new InterceptedConfigService(_logger, cached: new AgentConfigResponse
|
||||
{
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
SelfDestructOnComplete = true,
|
||||
});
|
||||
|
||||
var fetched = await service.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
Assert.Equal(RemoteConfigFetchOutcome.FromCache, service.LastFetchOutcome);
|
||||
RemoteConfigMerger.Merge(config, fetched, _logger);
|
||||
Assert.Equal(0, config.InterceptedRegistrationRetryBoots);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Backend_rejection_and_network_failure_keep_their_exit_codes_and_never_give_up()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 };
|
||||
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(2, record.FailedRuns);
|
||||
Assert.Equal(0, record.InterceptedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Only_an_intercepted_start_gives_up_even_when_the_count_already_exceeds_the_buffer()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 };
|
||||
Boot(Intercepted(), config);
|
||||
Boot(Intercepted(), config);
|
||||
config.InterceptedRegistrationRetryBoots = 0; // e.g. a later cached config lowered it
|
||||
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.Equal(0, _cleanupCalls);
|
||||
var record = _persistence.LoadRegistrationFailure()!;
|
||||
Assert.Equal(4, record.FailedRuns);
|
||||
Assert.Equal(2, record.InterceptedRuns); // unchanged by the other failures
|
||||
Assert.Equal("Failed", record.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Failures_of_another_kind_between_interceptions_do_not_reset_the_count()
|
||||
{
|
||||
var config = new AgentConfiguration(); // buffer 1
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
Assert.Equal(7, Boot(NetworkFailure(), config));
|
||||
Assert.Equal(6, Boot(BackendRejected(), config));
|
||||
Assert.False(GaveUp);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config)); // second interception since the last success
|
||||
|
||||
AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Only_a_successful_registration_resets_the_count()
|
||||
{
|
||||
var config = new AgentConfiguration(); // buffer 1
|
||||
Boot(Intercepted(), config);
|
||||
BackendSessionRegistration.ClearAfterSuccess(_persistence, () => "fp-office", _logger);
|
||||
|
||||
Assert.Equal(6, Boot(Intercepted(), config));
|
||||
|
||||
Assert.False(GaveUp);
|
||||
Assert.Equal(1, _persistence.LoadRegistrationFailure()!.InterceptedRuns);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Without_a_written_record_the_buffer_cannot_be_enforced_and_the_next_boot_retries()
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 };
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record: null, config, authFailureTracker: null,
|
||||
_persistence, _stateDir, CleanupFactory, _logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.False(GaveUp);
|
||||
Assert.True(_persistence.SessionExists());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true)]
|
||||
[InlineData(false)]
|
||||
public async Task The_registration_answer_decides_whether_the_start_counts_as_intercepted(bool foreign)
|
||||
{
|
||||
// Through the real client: no correlation echo + the platform's "Client Certificate
|
||||
// Required" page is an interception; a JSON 403 that echoes the id is the backend's verdict.
|
||||
using var handler = new RecordingHttpMessageHandler();
|
||||
if (foreign)
|
||||
handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
else
|
||||
handler.QueueStatus(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}");
|
||||
using var http = new HttpClient(handler);
|
||||
var apiClient = new BackendApiClient(
|
||||
http, "https://backend.test", manufacturer: "m", model: "d", serialNumber: "s",
|
||||
useBootstrapTokenAuth: false, bootstrapToken: null, agentVersion: "0.0.0", logger: _logger);
|
||||
var distress = new FakeDistressReporter();
|
||||
var config = new AgentConfiguration
|
||||
{
|
||||
ApiBaseUrl = "https://backend.test",
|
||||
TenantId = "00000000-0000-0000-0000-000000000001",
|
||||
SessionId = _persistence.GetOrCreate(_logger),
|
||||
InterceptedRegistrationRetryBoots = 0,
|
||||
};
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: _logger,
|
||||
authFailureTracker: NewTracker(distress),
|
||||
deviceHardware: ("m", "d", "s"),
|
||||
networkLinkWait: _ => Task.CompletedTask);
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome);
|
||||
Assert.Equal(foreign, result.IsIntercepted);
|
||||
Assert.Equal(
|
||||
foreign ? DistressErrorType.ClientCertificateNotReceived : DistressErrorType.DeviceNotRegistered,
|
||||
Assert.Single(distress.Calls).ErrorType);
|
||||
|
||||
Assert.Equal(6, Boot(result, config));
|
||||
Assert.Equal(foreign, GaveUp);
|
||||
}
|
||||
|
||||
// ============================================================ the distress leaves first
|
||||
|
||||
[Fact]
|
||||
public void The_exit_waits_for_the_pending_ClientCertificateNotReceived_send()
|
||||
{
|
||||
var record = RecordFailed(Intercepted(), "fp-office");
|
||||
var (tracker, inFlight) = DispatchSlowCertificateDistress();
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record, new AgentConfiguration(), tracker,
|
||||
_persistence, _stateDir, CleanupFactory, _logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.True(inFlight.IsCompleted);
|
||||
Assert.False(GaveUp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_give_up_waits_for_the_pending_send_before_it_cleans_up()
|
||||
{
|
||||
var record = RecordFailed(Intercepted(), "fp-office");
|
||||
var (tracker, inFlight) = DispatchSlowCertificateDistress();
|
||||
bool? sentBeforeCleanup = null;
|
||||
|
||||
var exit = BackendSessionRegistration.ConcludeFailedRun(
|
||||
Intercepted(), record, new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 }, tracker,
|
||||
_persistence, _stateDir,
|
||||
() =>
|
||||
{
|
||||
sentBeforeCleanup = inFlight.IsCompleted;
|
||||
throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests");
|
||||
},
|
||||
_logger);
|
||||
|
||||
Assert.Equal(6, exit);
|
||||
Assert.True(sentBeforeCleanup);
|
||||
Assert.True(GaveUp);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_wait_for_the_distress_is_bounded_by_its_own_http_timeout()
|
||||
{
|
||||
Assert.True(BackendSessionRegistration.PendingDistressBound <= TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
// ============================================================ K5: measurement point
|
||||
|
||||
[Theory]
|
||||
[InlineData("fp-office", "fp-office", false)]
|
||||
[InlineData("fp-office", "fp-home", true)]
|
||||
public void A_success_after_failed_starts_reports_the_record_it_clears(
|
||||
string failedOn, string registeredOn, bool networkChanged)
|
||||
{
|
||||
var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 };
|
||||
Boot(NetworkFailure(), config, fingerprint: "fp-elsewhere");
|
||||
Boot(Intercepted(), config, fingerprint: failedOn); // the latest failed start's network counts
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn, _logger)!;
|
||||
|
||||
Assert.Equal(2, prior.FailedRuns);
|
||||
Assert.Equal(1, prior.InterceptedRuns);
|
||||
Assert.Equal("AuthFailed", prior.LastOutcome);
|
||||
Assert.Equal(networkChanged, prior.NetworkChanged);
|
||||
Assert.Null(_persistence.LoadRegistrationFailureRecord());
|
||||
|
||||
var data = EmitAgentStarted(prior);
|
||||
Assert.Equal(2, data["priorRegistrationFailedRuns"]);
|
||||
Assert.Equal(1, data["priorRegistrationInterceptedRuns"]);
|
||||
Assert.Equal("AuthFailed", data["priorRegistrationLastOutcome"]);
|
||||
Assert.Equal(networkChanged, data["priorRegistrationNetworkChanged"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null, "fp-home")] // the failed start's network was unknown
|
||||
[InlineData("fp-office", null)] // this start's network is unknown
|
||||
public void Network_change_is_omitted_when_either_fingerprint_is_unknown(string? failedOn, string? registeredOn)
|
||||
{
|
||||
Boot(Intercepted(), new AgentConfiguration(), fingerprint: failedOn);
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn!, _logger)!;
|
||||
|
||||
Assert.Null(prior.NetworkChanged);
|
||||
var data = EmitAgentStarted(prior);
|
||||
Assert.Equal(1, data["priorRegistrationFailedRuns"]);
|
||||
Assert.False(data.ContainsKey("priorRegistrationNetworkChanged"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_success_without_failed_starts_reports_nothing_and_never_reads_the_network()
|
||||
{
|
||||
var captures = 0;
|
||||
|
||||
var prior = BackendSessionRegistration.ClearAfterSuccess(
|
||||
_persistence, () => { captures++; return "fp-office"; }, _logger);
|
||||
|
||||
Assert.Null(prior);
|
||||
Assert.Equal(0, captures);
|
||||
Assert.DoesNotContain(EmitAgentStarted(null).Keys, k => k.StartsWith("priorRegistration", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_network_fingerprint_stays_on_the_device()
|
||||
{
|
||||
Boot(Intercepted(), new AgentConfiguration(), fingerprint: "0123456789abcdef");
|
||||
|
||||
// Kept locally beside the record ...
|
||||
Assert.Contains("0123456789abcdef", File.ReadAllText(Path.Combine(_tmp.Path, "registration-failure.json")));
|
||||
Assert.Equal("0123456789abcdef", _persistence.LoadRegistrationFailureRecord()!.NetworkFingerprint);
|
||||
|
||||
// ... while the summary the emergency break sends cannot carry it.
|
||||
var wire = JsonConvert.SerializeObject(new AgentErrorReport
|
||||
{
|
||||
PriorRegistrationFailure = _persistence.LoadRegistrationFailure(),
|
||||
});
|
||||
Assert.DoesNotContain("0123456789abcdef", wire);
|
||||
Assert.DoesNotContain("Fingerprint", wire);
|
||||
Assert.Contains("\"InterceptedRuns\":1", wire);
|
||||
}
|
||||
|
||||
// ============================================================ helpers
|
||||
|
||||
private static SessionRegistrationResult Intercepted() =>
|
||||
SessionRegistrationResult.AuthFailed(403, "The device certificate did not reach the backend.", intercepted: true);
|
||||
|
||||
private static SessionRegistrationResult BackendRejected() =>
|
||||
SessionRegistrationResult.AuthFailed(403, "Backend returned 403 Forbidden. The device is not authorized.");
|
||||
|
||||
private static SessionRegistrationResult NetworkFailure() =>
|
||||
SessionRegistrationResult.Failed("No such host is known.");
|
||||
|
||||
private bool GaveUp => File.Exists(Path.Combine(_stateDir, "enrollment-complete.marker"));
|
||||
|
||||
private AuthFailureTracker NewTracker(FakeDistressReporter distress) =>
|
||||
new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance, _logger, distress);
|
||||
|
||||
/// <summary>
|
||||
/// What the registration helper does when the answer did not come from the backend: the
|
||||
/// tracker dispatches ClientCertificateNotReceived — here a send that stays in flight for a
|
||||
/// second, started right before the conclusion so nothing else can use the time up.
|
||||
/// </summary>
|
||||
private (AuthFailureTracker Tracker, Task InFlight) DispatchSlowCertificateDistress()
|
||||
{
|
||||
Task? inFlight = null;
|
||||
var distress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(TimeSpan.FromSeconds(1)) };
|
||||
var tracker = NewTracker(distress);
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, Assert.Single(distress.Calls).ErrorType);
|
||||
Assert.False(inFlight!.IsCompleted);
|
||||
return (tracker, inFlight);
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary? RecordFailed(SessionRegistrationResult result, string? fingerprint) =>
|
||||
BackendSessionRegistration.RecordFailedRun(
|
||||
_persistence, result, At, At.AddSeconds(2), linkUpAtStart: true, linkUpAtEnd: true,
|
||||
configFetchOutcome: "FromCache", networkFingerprint: fingerprint, logger: _logger);
|
||||
|
||||
/// <summary>One start whose registration failed with <paramref name="result"/>, ended the way Register ends it.</summary>
|
||||
private int Boot(SessionRegistrationResult result, AgentConfiguration config, string? fingerprint = "fp-office")
|
||||
{
|
||||
var record = RecordFailed(result, fingerprint);
|
||||
return BackendSessionRegistration.ConcludeFailedRun(
|
||||
result, record, config, authFailureTracker: null, _persistence, _stateDir, CleanupFactory, _logger);
|
||||
}
|
||||
|
||||
private CleanupService CleanupFactory()
|
||||
{
|
||||
_cleanupCalls++;
|
||||
// The real factory spawns a PowerShell cleanup script; throwing makes the cleanup retry
|
||||
// swallow and log it, so the call is counted without a side effect (as in ProgramGuardsTests).
|
||||
throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests");
|
||||
}
|
||||
|
||||
/// <summary>The emergency break's end state (ProgramGuardsTests): marker, session files gone, cleanup per SelfDestructOnComplete.</summary>
|
||||
private void AssertCleanedUpLikeTheEmergencyBreak(int expectedCleanupCalls)
|
||||
{
|
||||
Assert.True(GaveUp);
|
||||
Assert.False(_persistence.SessionExists());
|
||||
Assert.Null(_persistence.LoadRegistrationFailure());
|
||||
Assert.Equal(expectedCleanupCalls, _cleanupCalls);
|
||||
// The next start ends at the bootstrap guard, exactly as after the break.
|
||||
Assert.True(ProgramAlias.CheckEnrollmentCompleteMarker(
|
||||
_stateDir, selfDestructOnComplete: false, cleanupServiceFactory: null, _logger, consoleMode: false));
|
||||
}
|
||||
|
||||
private Dictionary<string, object> EmitAgentStarted(PriorRegistrationFailures? prior)
|
||||
{
|
||||
var sink = new FakeSignalIngressSink();
|
||||
LifecycleEmitters.EmitAgentStarted(
|
||||
new InformationalEventPost(sink, SystemClock.Instance, _logger),
|
||||
new AgentConfiguration { SessionId = "S1", TenantId = "00000000-0000-0000-0000-000000000001", CommandLineArgs = string.Empty },
|
||||
new ProgramAlias.PreviousExitSummary { ExitType = "reboot_kill" },
|
||||
"2.0.test",
|
||||
remoteConfigService: null,
|
||||
_logger,
|
||||
priorRegistrationFailures: prior);
|
||||
return (Dictionary<string, object>)Assert.Single(sink.Posted).TypedPayload!;
|
||||
}
|
||||
|
||||
/// <summary>A start whose config fetch is answered by the interception: it falls back to the cache.</summary>
|
||||
private sealed class InterceptedConfigService : RemoteConfigService
|
||||
{
|
||||
private readonly AgentConfigResponse _cached;
|
||||
|
||||
public InterceptedConfigService(AgentLogger logger, AgentConfigResponse cached)
|
||||
: base(
|
||||
new BackendApiClient(
|
||||
httpClient: new HttpClient(), baseUrl: "http://localhost", manufacturer: string.Empty,
|
||||
model: string.Empty, serialNumber: string.Empty, useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null, agentVersion: "0.0.0", logger: logger),
|
||||
"00000000-0000-0000-0000-000000000001",
|
||||
logger)
|
||||
{
|
||||
_cached = cached;
|
||||
}
|
||||
|
||||
protected override Task<AgentConfigResponse> CallBackendAsync() =>
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", 403,
|
||||
classification: InterceptionFixtures.CertificateRequired());
|
||||
|
||||
protected override AgentConfigResponse LoadCachedConfig() => _cached;
|
||||
|
||||
protected override Task WaitForNetworkLinkAsync() => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
+145
-34
@@ -1,11 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -261,8 +262,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_endpoint_unavailable_suppresses_distress_but_still_counts()
|
||||
public void RecordFailure_only_the_stopped_app_page_counts_without_distress()
|
||||
{
|
||||
// Rewritten deliberately for the client-certificate interception plan: the former
|
||||
// "endpoint unavailable" flag covered every HTML platform page, so the platform's
|
||||
// "Client Certificate Required" page (TLS inspection on the path) shut healthy agents
|
||||
// down. Now only the stopped-app page keeps that semantics — it counts toward the
|
||||
// shutdown (a dead endpoint must not retry forever) without a distress report that
|
||||
// could not reach it — while the certificate page never counts (tests below).
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
@@ -270,19 +277,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
// Platform-403s from a stopped endpoint: a distress report cannot reach it and the
|
||||
// DeviceNotRegistered classification would be wrong — nothing must be dispatched.
|
||||
tracker.RecordFailure(403, "agent/config", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/register-session", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.CertificateRequired());
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp());
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.StoppedApp());
|
||||
|
||||
Assert.Empty(fakeDistress.Calls);
|
||||
// The shutdown thresholds still apply — a dead endpoint must not retry forever.
|
||||
Assert.Equal(2, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(1, events);
|
||||
Assert.DoesNotContain(fakeDistress.Calls, c => c.ErrorType == DistressErrorType.DeviceNotRegistered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_endpoint_unavailable_does_not_consume_streak_report_slot()
|
||||
public void RecordFailure_stopped_app_page_does_not_consume_streak_report_slot()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
@@ -290,7 +297,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
|
||||
// The streak opens with a platform-403 (no dispatch), then a live backend answers
|
||||
// with a genuine 403 — that one must still produce the streak's single report.
|
||||
tracker.RecordFailure(403, "agent/config", endpointUnavailable: true);
|
||||
tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp());
|
||||
tracker.RecordFailure(403, "agent/config");
|
||||
tracker.RecordFailure(403, "agent/config");
|
||||
|
||||
@@ -298,6 +305,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, fakeDistress.Calls[0].ErrorType);
|
||||
}
|
||||
|
||||
// ============================================================= Foreign answers (interception)
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_never_reaches_the_threshold()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 10, clock, NewLogger(), new FakeDistressReporter());
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
for (int i = 0; i < 10; i++)
|
||||
{
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
clock.Now = clock.Now.AddMinutes(3); // also past the time window
|
||||
}
|
||||
|
||||
Assert.Equal(0, events);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_dispatches_one_ClientCertificateNotReceived_per_process()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordSuccess(); // a success re-arms the streak report, never the per-process one
|
||||
tracker.RecordFailure(401, "agent/telemetry", InterceptionFixtures.CertificateRequired(401));
|
||||
|
||||
var call = Assert.Single(fakeDistress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.Equal(403, call.HttpStatusCode);
|
||||
// Operation, status, sub-reason, page title, issuer — in that order, within the 256 budget.
|
||||
Assert.StartsWith("agent/register-session http 403 client_certificate_required;", call.Message);
|
||||
Assert.True(
|
||||
call.Message.IndexOf(InterceptionFixtures.PlatformPageTitle, StringComparison.Ordinal) <
|
||||
call.Message.IndexOf(InterceptionFixtures.InspectionIssuer, StringComparison.Ordinal));
|
||||
Assert.True(call.Message.Length <= 256, $"distress message is {call.Message.Length} characters");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordFailure_foreign_within_a_backend_streak_neither_resets_nor_advances_it()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var fakeDistress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
int events = 0;
|
||||
tracker.ThresholdExceeded += (_, _) => events++;
|
||||
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.Equal(2, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(0, events);
|
||||
|
||||
tracker.RecordFailure(403, "agent/telemetry"); // the third backend rejection trips it
|
||||
|
||||
Assert.Equal(3, tracker.ConsecutiveFailures);
|
||||
Assert.Equal(1, events);
|
||||
Assert.Equal(1, fakeDistress.Calls.Count(c => c.ErrorType == DistressErrorType.DeviceNotRegistered));
|
||||
}
|
||||
|
||||
// ============================================================= Pending foreign distress
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_returns_at_once_when_nothing_was_dispatched()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), new FakeDistressReporter());
|
||||
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_waits_for_the_send_to_finish()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
Task? inFlight = null;
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(200) };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.FromSeconds(10)));
|
||||
Assert.True(inFlight!.IsCompleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_stops_at_the_bound()
|
||||
{
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var neverSent = new TaskCompletionSource<bool>();
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired());
|
||||
|
||||
var watch = Stopwatch.StartNew();
|
||||
var finished = tracker.WaitForPendingForeignDistress(TimeSpan.FromMilliseconds(100));
|
||||
watch.Stop();
|
||||
|
||||
Assert.False(finished);
|
||||
Assert.True(watch.Elapsed < TimeSpan.FromSeconds(5), $"waited {watch.Elapsed}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WaitForPendingForeignDistress_never_waits_for_a_backend_streak_report()
|
||||
{
|
||||
// Only the ClientCertificateNotReceived send is kept: a failed start must not block
|
||||
// on anything else.
|
||||
var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc));
|
||||
var neverSent = new TaskCompletionSource<bool>();
|
||||
var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task };
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress);
|
||||
|
||||
tracker.RecordFailure(403, "agent/register-session");
|
||||
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(fakeDistress.Calls).ErrorType);
|
||||
Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordSuccess_rearms_distress_dispatch_for_the_next_streak()
|
||||
{
|
||||
@@ -329,29 +463,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(1, events);
|
||||
}
|
||||
|
||||
private sealed class FakeDistressReporter : DistressReporter
|
||||
{
|
||||
public sealed class Call
|
||||
{
|
||||
public DistressErrorType ErrorType { get; set; }
|
||||
public string Message { get; set; } = default!;
|
||||
public int? HttpStatusCode { get; set; }
|
||||
}
|
||||
|
||||
public List<Call> Calls { get; } = new List<Call>();
|
||||
|
||||
public FakeDistressReporter()
|
||||
: base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null)
|
||||
{
|
||||
Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode });
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock
|
||||
{
|
||||
public DateTime Now { get; set; }
|
||||
|
||||
@@ -298,9 +298,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RecordRegistrationFailure_counts_intercepted_runs_and_keeps_the_latest_network_fingerprint()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var sut = new SessionIdPersistence(tmp.Path);
|
||||
var at = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc);
|
||||
var intercepted = FailedRun(at, outcome: "AuthFailed");
|
||||
intercepted.InterceptedRuns = 1;
|
||||
|
||||
sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-office");
|
||||
sut.RecordRegistrationFailure(FailedRun(at.AddHours(1)), networkFingerprint: null);
|
||||
var merged = sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-home");
|
||||
|
||||
Assert.Equal(3, merged!.FailedRuns);
|
||||
Assert.Equal(2, merged.InterceptedRuns);
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord();
|
||||
Assert.Equal(3, record!.Summary.FailedRuns);
|
||||
Assert.Equal(2, record.Summary.InterceptedRuns);
|
||||
Assert.Equal("fp-home", record.NetworkFingerprint);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_record_from_an_agent_before_the_interception_buffer_still_loads()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"),
|
||||
"{\"FailedRuns\":2,\"FirstFailedAtUtc\":\"2026-09-01T08:00:00Z\",\"LastFailedAtUtc\":\"2026-09-02T08:00:00Z\"," +
|
||||
"\"Outcome\":\"AuthFailed\",\"LastError\":\"x\",\"NetworkLinkUpAtStart\":true,\"NetworkLinkUpAtEnd\":true," +
|
||||
"\"AttemptWindowSeconds\":3.0,\"ConfigFetchOutcome\":\"FromCache\"}");
|
||||
|
||||
var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord();
|
||||
|
||||
Assert.Equal(2, record!.Summary.FailedRuns);
|
||||
Assert.Equal(0, record.Summary.InterceptedRuns);
|
||||
Assert.Null(record.NetworkFingerprint);
|
||||
Assert.Equal(new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), record.Summary.FirstFailedAtUtc);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not json")]
|
||||
[InlineData("{\"FailedRuns\":0}")]
|
||||
[InlineData("null")]
|
||||
[InlineData("[]")]
|
||||
[InlineData("")]
|
||||
public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content)
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
|
||||
+31
@@ -6,6 +6,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
@@ -96,6 +97,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security
|
||||
Assert.Equal(1, tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_registration_ends_auth_failed_without_counting_and_reports_once()
|
||||
{
|
||||
// Something on the network path answered (TLS inspection): the run still ends
|
||||
// AuthFailed (exit 6, unchanged until the interception buffer exists), but the tracker
|
||||
// does not count it and sends ClientCertificateNotReceived instead of DeviceNotRegistered.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = NewLogger(tmp);
|
||||
var config = NewConfig();
|
||||
|
||||
var apiClient = new FakeApiClient(_ =>
|
||||
throw new BackendAuthException("The device certificate did not reach the backend.", statusCode: 403,
|
||||
classification: InterceptionFixtures.CertificateRequired()));
|
||||
var distress = new FakeDistressReporter();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0,
|
||||
clock: SystemClock.Instance, logger: logger, distressReporter: distress);
|
||||
|
||||
var result = await SessionRegistrationHelper.RegisterWithRetryAsync(
|
||||
apiClient, config, agentVersion: "0.0.0", logger: logger, authFailureTracker: tracker,
|
||||
networkLinkWait: _ => Task.CompletedTask);
|
||||
|
||||
Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome);
|
||||
Assert.Equal(403, result.HttpStatusCode);
|
||||
Assert.Equal(1, apiClient.CallCount);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures);
|
||||
var call = Assert.Single(distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType);
|
||||
Assert.StartsWith("agent/register-session http 403 client_certificate_required", call.Message);
|
||||
}
|
||||
|
||||
// ── SESSION-OWNER-BINDING ────────────────────────────────────────────
|
||||
|
||||
private static BackendAuthException OwnerMismatch()
|
||||
|
||||
+54
@@ -165,6 +165,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
public IReadOnlyCollection<string> StarvedAlreadyReportedOverride { get; set; } = Array.Empty<string>();
|
||||
public List<string> TerminationActionLog { get; } = new List<string>();
|
||||
public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero;
|
||||
public int TerminalUploadAttemptRequests;
|
||||
// Session bc803955 — DevicePrep app-wave settle wait. Zero (default) disables the
|
||||
// wait so the existing timing-sensitive tests stay untouched; the dedicated settle
|
||||
// tests opt in with a real budget.
|
||||
@@ -267,6 +268,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
public long IngressPendingSignalCount => _rig.IngressPendingSignalCountAccessor!();
|
||||
public bool CanObserveSpool => _rig.PendingItemCountAccessor != null;
|
||||
public int SpoolPendingItemCount => _rig.PendingItemCountAccessor!();
|
||||
|
||||
public void RequestTerminalUploadAttempt()
|
||||
{
|
||||
Interlocked.Increment(ref _rig.TerminalUploadAttemptRequests);
|
||||
_rig.TerminationActionLog.Add("terminalUploadAttempt");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>ARCH-F2 — recording shutdown gate (no real shutdown.exe in tests).</summary>
|
||||
@@ -489,6 +496,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone));
|
||||
|
||||
Assert.Equal(0, spoolPolls);
|
||||
Assert.Equal(0, rig.TerminalUploadAttemptRequests);
|
||||
Assert.Equal(0, rig.DiagnosticsUploads);
|
||||
Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
@@ -497,6 +505,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
public enum TerminalExit { SelfDestruct, StandaloneReboot, WhiteGlovePart1 }
|
||||
|
||||
[Theory]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.StoppedWaiting, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.StandaloneReboot)]
|
||||
[InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed, TerminalExit.WhiteGlovePart1)]
|
||||
public void Handle_while_uploads_are_intercepted_makes_one_terminal_attempt_then_exits_as_usual(
|
||||
EnrollmentTerminationReason reason, EnrollmentTerminationOutcome outcome, SessionStage stage, TerminalExit exit)
|
||||
{
|
||||
// Enrollment end during an interception pause: no deferral. DrainSpool asks for the one
|
||||
// terminal upload attempt, waits its bounded budget (the spool never empties — every
|
||||
// upload is answered on the network path), then the exit runs exactly as for any
|
||||
// session. Unsent telemetry is lost, as today.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = stage }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromMilliseconds(200);
|
||||
rig.PendingItemCountAccessor = () => 7;
|
||||
rig.DiagnosticsResult = new DiagnosticsUploadResult
|
||||
{
|
||||
ErrorCode = "The device certificate did not reach the backend (TLS inspection or a proxy on the network path): 403 Forbidden ...",
|
||||
};
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: exit != TerminalExit.StandaloneReboot);
|
||||
cfg.RebootOnComplete = exit == TerminalExit.StandaloneReboot;
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, Args(reason, outcome, stage));
|
||||
|
||||
Assert.Equal(1, rig.TerminalUploadAttemptRequests);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
Assert.True(rig.TerminationActionLog.IndexOf("terminalUploadAttempt") < rig.TerminationActionLog.IndexOf("signalShutdown"));
|
||||
if (exit == TerminalExit.WhiteGlovePart1)
|
||||
{
|
||||
Assert.True(rig.SessionPersistence.IsWhiteGloveResume());
|
||||
Assert.Equal(0, rig.CleanupService.Invocations);
|
||||
return;
|
||||
}
|
||||
|
||||
Assert.Equal(1, rig.DiagnosticsUploads);
|
||||
Assert.Contains(Constants.EventTypes.DiagnosticsUploadFailed, rig.EmittedEventTypes);
|
||||
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
Assert.Equal(exit == TerminalExit.SelfDestruct ? 1 : 0, rig.CleanupService.Invocations);
|
||||
Assert.Equal(exit == TerminalExit.StandaloneReboot ? 1 : 0, rig.RebootInvocations);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics()
|
||||
{
|
||||
|
||||
+148
-18
@@ -6,16 +6,19 @@ using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// 401/403 classification in <see cref="BackendApiClient"/>: a backend JSON 403 is a real
|
||||
/// device-authorization verdict, while an HTML body means the response came from the Azure
|
||||
/// platform / an edge proxy (stopped or retired app) and must NOT be reported as
|
||||
/// "device is not authorized". Field case 2026-08-09: a pre-cutover agent hit
|
||||
/// the stopped legacy Function App and support chased a non-existent authorization problem.
|
||||
/// 401/403 classification in <see cref="BackendApiClient"/>: only a response that echoes the
|
||||
/// request's correlation id is the backend's device-authorization verdict. The platform page of
|
||||
/// a stopped app (field case 2026-08-09: a pre-cutover agent hit the stopped legacy Function App
|
||||
/// and support chased a non-existent authorization problem) and anything answered on the network
|
||||
/// path (TLS inspection: "Client Certificate Required", proxy pages) must NOT be reported as
|
||||
/// "device is not authorized".
|
||||
/// </summary>
|
||||
public sealed class BackendApiClientAuthClassificationTests
|
||||
{
|
||||
@@ -24,17 +27,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
private readonly Func<HttpRequestMessage, HttpResponseMessage> _responder;
|
||||
public StubHandler(Func<HttpRequestMessage, HttpResponseMessage> responder) => _responder = responder;
|
||||
|
||||
public int Calls { get; private set; }
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
=> Task.FromResult(_responder(request));
|
||||
{
|
||||
Calls++;
|
||||
var response = _responder(request);
|
||||
response.RequestMessage = request; // as HttpClientHandler does
|
||||
return Task.FromResult(response);
|
||||
}
|
||||
}
|
||||
|
||||
private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType)
|
||||
{
|
||||
var handler = new StubHandler(_ => new HttpResponseMessage(status)
|
||||
/// <param name="fromBackend">
|
||||
/// True: the response echoes the request's X-Correlation-ID like the backend's middleware.
|
||||
/// False: a platform or proxy answer without the echo.
|
||||
/// </param>
|
||||
private static StubHandler Answer(HttpStatusCode status, string body, string mediaType, bool fromBackend) =>
|
||||
new StubHandler(request =>
|
||||
{
|
||||
Content = new StringContent(body, Encoding.UTF8, mediaType)
|
||||
var response = new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) };
|
||||
if (fromBackend && request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids))
|
||||
response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids);
|
||||
return response;
|
||||
});
|
||||
return new BackendApiClient(
|
||||
|
||||
private static BackendApiClient BuildClient(HttpMessageHandler handler, ServerCertificateObserver? serverCertificates = null) =>
|
||||
new BackendApiClient(
|
||||
new HttpClient(handler),
|
||||
baseUrl: "https://backend.example",
|
||||
manufacturer: "Contoso",
|
||||
@@ -43,8 +61,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
useBootstrapTokenAuth: false,
|
||||
bootstrapToken: null,
|
||||
agentVersion: "2.0.0",
|
||||
logger: null);
|
||||
}
|
||||
logger: null,
|
||||
serverCertificates: serverCertificates);
|
||||
|
||||
private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType, bool fromBackend = true) =>
|
||||
BuildClient(Answer(status, body, mediaType, fromBackend));
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_json_403_is_reported_as_device_not_authorized()
|
||||
@@ -134,16 +155,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
public async Task Platform_html_403_is_reported_as_endpoint_unavailable_with_page_title()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
"<!DOCTYPE html><html><head><title>Web App - Unavailable</title></head><body>stopped</body></html>",
|
||||
"text/html");
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal(403, ex.StatusCode);
|
||||
Assert.True(ex.EndpointUnavailable);
|
||||
Assert.Contains("Web App - Unavailable", ex.Message);
|
||||
Assert.False(ex.IsForeignResponse);
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, ex.Origin);
|
||||
Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message);
|
||||
Assert.Contains("NOT a device-authorization failure", ex.Message);
|
||||
Assert.Contains("stale ApiBaseUrl", ex.Message);
|
||||
Assert.DoesNotContain("Check client certificate", ex.Message);
|
||||
}
|
||||
|
||||
@@ -153,7 +175,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
" <html><body>Error 403 - This web app is stopped.</body></html>",
|
||||
"text/plain");
|
||||
"text/plain",
|
||||
fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
@@ -161,6 +184,113 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains("HTML platform error page", ex.Message);
|
||||
}
|
||||
|
||||
// ── answered on the network path (TLS inspection, proxy) ─────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Certificate_required_page_says_the_certificate_did_not_reach_the_backend()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal(403, ex.StatusCode);
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.False(ex.EndpointUnavailable);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, ex.Classification.ForeignReason);
|
||||
Assert.Null(ex.ErrorCode);
|
||||
// The cause leads: RegistrationFailureSummary.LastError keeps 256 characters.
|
||||
Assert.StartsWith("The device certificate did not reach the backend (TLS inspection or a proxy on the network path)", ex.Message);
|
||||
Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message);
|
||||
Assert.True(ex.Message.IndexOf("Web App - Unavailable", StringComparison.Ordinal) < 256);
|
||||
Assert.Contains("NOT a device-authorization failure", ex.Message);
|
||||
Assert.DoesNotContain("stale ApiBaseUrl", ex.Message);
|
||||
Assert.DoesNotContain("Check client certificate", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Proxy_page_keeps_the_phrase_and_title_ANALYZE_NET_001_matches()
|
||||
{
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html", fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(
|
||||
() => client.GetDiagnosticsUploadUrlAsync("tenant-1", "session-1", "package.zip"));
|
||||
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, ex.Classification.ForeignReason);
|
||||
Assert.Contains("HTML platform error page (\"Example Gateway - Notification\")", ex.Message);
|
||||
Assert.Contains("403", ex.Message);
|
||||
Assert.DoesNotContain("stale ApiBaseUrl", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Json_403_without_the_echo_is_foreign_and_its_errorCode_is_ignored()
|
||||
{
|
||||
// An errorCode in a body the backend did not author must not drive a session rotation.
|
||||
using var client = BuildClient(
|
||||
HttpStatusCode.Forbidden,
|
||||
"{\"success\":false,\"errorCode\":\"session_owner_mismatch\"}",
|
||||
"application/json",
|
||||
fromBackend: false);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(
|
||||
() => client.RegisterSessionAsync(new AutopilotMonitor.Shared.Models.SessionRegistration()));
|
||||
|
||||
Assert.True(ex.IsForeignResponse);
|
||||
Assert.Null(ex.ErrorCode);
|
||||
Assert.Contains("a response without the backend's correlation echo", ex.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Foreign_answer_carries_the_observed_server_certificate_issuer()
|
||||
{
|
||||
var observer = new ServerCertificateObserver();
|
||||
using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA"))
|
||||
{
|
||||
observer.Validate(cert, System.Net.Security.SslPolicyErrors.None);
|
||||
}
|
||||
using var client = BuildClient(
|
||||
Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false),
|
||||
observer);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackendAuthException>(() => client.GetAgentConfigAsync("tenant-1"));
|
||||
|
||||
Assert.Equal("CN=Example Inspection CA", ex.Classification.ServerCertIssuer);
|
||||
}
|
||||
|
||||
// ── agent/error: a report the backend never received is not delivered ──
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.OK, true, true)]
|
||||
[InlineData(HttpStatusCode.InternalServerError, true, true)] // unchanged: only the round-trip counts
|
||||
[InlineData(HttpStatusCode.Forbidden, true, true)] // the backend answered
|
||||
[InlineData(HttpStatusCode.Forbidden, false, false)] // answered on the network path
|
||||
[InlineData(HttpStatusCode.Unauthorized, false, false)]
|
||||
public async Task ReportAgentError_counts_as_delivered_unless_the_backend_never_answered(
|
||||
HttpStatusCode status, bool fromBackend, bool expectedDelivered)
|
||||
{
|
||||
using var client = BuildClient(status, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend);
|
||||
|
||||
var delivered = await client.ReportAgentErrorAsync(new AgentErrorReport { TenantId = "tenant-1", SessionId = "session-1" });
|
||||
|
||||
Assert.Equal(expectedDelivered, delivered);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_emergency_report_rolls_its_reservation_back()
|
||||
{
|
||||
var handler = Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false);
|
||||
using var client = BuildClient(handler);
|
||||
var reporter = new EmergencyReporter(client, "session-1", "tenant-1", "2.0.0", logger: null);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "first");
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "second");
|
||||
|
||||
// Not delivered → no dedup, no cooldown: the same key is attempted again.
|
||||
Assert.Equal(2, handler.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_401_keeps_authorization_message_and_status()
|
||||
{
|
||||
|
||||
+32
@@ -156,6 +156,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains("BackendTelemetryUploader: ingest PERMANENT", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UploadBatch_logs_the_whole_interception_finding_at_warning()
|
||||
{
|
||||
// The old PERMANENT line carried the first 200 body characters, which ended before the
|
||||
// platform's "Client Certificate Required" text. The INTERCEPTED line names the finding.
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http,
|
||||
baseUrl: "https://backend.test",
|
||||
tenantId: "T1",
|
||||
logger: logger);
|
||||
|
||||
var result = await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
Assert.True(result.IsIntercepted);
|
||||
|
||||
Assert.True(handler.Captured[0].TryGetHeader("X-Correlation-ID", out var corr));
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains("BackendTelemetryUploader: ingest INTERCEPTED", logContent);
|
||||
Assert.Contains($"corr={corr}", logContent);
|
||||
Assert.Contains("status=403", logContent);
|
||||
Assert.Contains("reason=client_certificate_required", logContent);
|
||||
Assert.Contains($"pageTitle=\"{InterceptionFixtures.PlatformPageTitle}\"", logContent);
|
||||
Assert.Contains("serverCertIssuer=\"(unknown)\"", logContent);
|
||||
Assert.DoesNotContain("ingest PERMANENT", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task UploadBatch_empty_batch_no_log_no_network()
|
||||
{
|
||||
|
||||
+194
-2
@@ -4,7 +4,16 @@ using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using Xunit;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -51,7 +60,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
string? model = "Latitude",
|
||||
string? serial = "ABC123",
|
||||
string? bootstrap = null,
|
||||
string? version = "11.0.0")
|
||||
string? version = "11.0.0",
|
||||
AuthFailureTracker? tracker = null,
|
||||
ServerCertificateObserver? serverCertificates = null,
|
||||
NetworkMetrics? networkMetrics = null)
|
||||
{
|
||||
Http = new HttpClient(Handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
Sut = new BackendTelemetryUploader(
|
||||
@@ -62,7 +74,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
model: model,
|
||||
serialNumber: serial,
|
||||
bootstrapToken: bootstrap,
|
||||
agentVersion: version);
|
||||
agentVersion: version,
|
||||
authFailureTracker: tracker,
|
||||
serverCertificates: serverCertificates,
|
||||
networkMetrics: networkMetrics);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
@@ -245,6 +260,85 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Null(result.ErrorReason);
|
||||
}
|
||||
|
||||
// ============================================================ 2xx without the correlation echo
|
||||
|
||||
[Fact]
|
||||
public async Task Ok_with_the_echo_is_a_success_and_not_counted()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueStatus(HttpStatusCode.OK);
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal(0, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ok_without_the_echo_is_still_a_success_and_only_counted()
|
||||
{
|
||||
// A 200 that did not come from the backend (e.g. a block page served with 200): the
|
||||
// upload path behaves exactly as before — control signals are still read — and the
|
||||
// only difference is the counter.
|
||||
var metrics = new NetworkMetrics();
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance,
|
||||
new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info));
|
||||
tracker.RecordFailure(403, "agent/telemetry");
|
||||
using var rig = new Rig(tracker: tracker, networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "<html><title>Example</title></html>");
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "{\"deviceKillSignal\":true}", mediaType: "application/json");
|
||||
|
||||
var page = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
var signals = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(2) }, CancellationToken.None);
|
||||
|
||||
Assert.True(page.Success);
|
||||
Assert.Null(page.ErrorReason);
|
||||
Assert.True(signals.Success);
|
||||
Assert.True(signals.DeviceKillSignal);
|
||||
Assert.Equal(0, tracker.ConsecutiveFailures); // a 2xx still resets the auth streak
|
||||
Assert.Equal(2, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Answers_other_than_2xx_never_count_as_ok_without_echo()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.ServiceUnavailable, "<html>busy</html>");
|
||||
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, metrics.OkWithoutEchoCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_count_reaches_the_agent_metrics_snapshot()
|
||||
{
|
||||
var metrics = new NetworkMetrics();
|
||||
using var rig = new Rig(networkMetrics: metrics);
|
||||
rig.Handler.QueueForeign(HttpStatusCode.OK, "<html><title>Example</title></html>");
|
||||
await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
var sink = new FakeSignalIngressSink();
|
||||
using var tmp = new TempDirectory();
|
||||
var collector = new AgentSelfMetricsCollector(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(sink, SystemClock.Instance),
|
||||
networkMetrics: metrics,
|
||||
logger: new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info),
|
||||
agentVersion: "2.0.test");
|
||||
collector.CollectSafe();
|
||||
|
||||
var snapshot = (IDictionary<string, object>)Assert.Single(sink.Posted, p =>
|
||||
p.TypedPayload is IDictionary<string, object> d && d.ContainsKey("agent_version")).TypedPayload!;
|
||||
Assert.Equal(1L, snapshot["telemetry_upload_ok_without_echo"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.Unauthorized)]
|
||||
[InlineData(HttpStatusCode.Forbidden)]
|
||||
@@ -261,6 +355,104 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Contains(((int)status).ToString(), result.ErrorReason);
|
||||
}
|
||||
|
||||
// ============================================================ 401/403 origin
|
||||
|
||||
private sealed class OriginRig : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
|
||||
public FakeDistressReporter Distress { get; } = new FakeDistressReporter();
|
||||
public AuthFailureTracker Tracker { get; }
|
||||
public Rig Upload { get; }
|
||||
|
||||
public OriginRig()
|
||||
{
|
||||
Tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance,
|
||||
new AgentLogger(_tmp.Path, AgentLogLevel.Info), Distress);
|
||||
var serverCertificates = new ServerCertificateObserver();
|
||||
using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA"))
|
||||
{
|
||||
serverCertificates.Validate(cert, System.Net.Security.SslPolicyErrors.None);
|
||||
}
|
||||
Upload = new Rig(tracker: Tracker, serverCertificates: serverCertificates);
|
||||
}
|
||||
|
||||
public Task<UploadResult> UploadAsync() => Upload.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Upload.Dispose();
|
||||
_tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backend_403_is_an_auth_failure_that_counts_and_reports_DeviceNotRegistered()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueStatus(HttpStatusCode.Forbidden, body: "{\"error\":\"Device validation failed\"}");
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsAuthFailure);
|
||||
Assert.False(result.IsIntercepted);
|
||||
Assert.Equal(1, rig.Tracker.ConsecutiveFailures);
|
||||
Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(rig.Distress.Calls).ErrorType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Stopped_app_403_counts_like_an_auth_failure_without_distress()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsAuthFailure);
|
||||
Assert.False(result.IsIntercepted);
|
||||
Assert.Equal(1, rig.Tracker.ConsecutiveFailures);
|
||||
Assert.Empty(rig.Distress.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Certificate_required_403_is_intercepted_never_counted_and_reported_once()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
await rig.UploadAsync();
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.True(result.IsIntercepted);
|
||||
Assert.False(result.IsAuthFailure);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.Equal(403, result.Interception!.StatusCode);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, result.Interception.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, result.Interception.PageTitle);
|
||||
Assert.Equal("CN=Example Inspection CA", result.Interception.ServerCertIssuer);
|
||||
Assert.Equal(0, rig.Tracker.ConsecutiveFailures);
|
||||
var distress = Assert.Single(rig.Distress.Calls);
|
||||
Assert.Equal(DistressErrorType.ClientCertificateNotReceived, distress.ErrorType);
|
||||
Assert.Contains("agent/telemetry http 403 client_certificate_required", distress.Message);
|
||||
Assert.Contains("CN=Example Inspection CA", distress.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Proxy_page_403_is_intercepted_as_foreign_response()
|
||||
{
|
||||
using var rig = new OriginRig();
|
||||
rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage);
|
||||
|
||||
var result = await rig.UploadAsync();
|
||||
|
||||
Assert.True(result.IsIntercepted);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, result.Interception!.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.ProxyPageTitle, result.Interception.PageTitle);
|
||||
Assert.Equal(0, rig.Tracker.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.BadRequest)] // 400
|
||||
[InlineData(HttpStatusCode.NotFound)] // 404
|
||||
|
||||
+11
@@ -74,6 +74,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>Queues a 403 the platform's "Client Certificate Required" page answered (an interception).</summary>
|
||||
public FakeBackendTelemetryUploader QueueIntercepted(int times = 1)
|
||||
{
|
||||
for (int i = 0; i < times; i++)
|
||||
{
|
||||
_script.Enqueue(_ => UploadResult.Intercepted(
|
||||
"intercepted: http 403 (client_certificate_required)", InterceptionFixtures.CertificateRequired()));
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Synthetic copies of the pages an agent gets instead of a backend answer: the Azure platform
|
||||
/// page when no client certificate arrived, the platform page of a stopped app, and a proxy's
|
||||
/// block page. Rebuilt from the observed structure and wording; none of them is a captured
|
||||
/// artifact. The style block keeps the platform marker text past the first 200 characters, where
|
||||
/// the old upload log excerpt ended.
|
||||
/// </summary>
|
||||
internal static class InterceptionFixtures
|
||||
{
|
||||
public const string PlatformPageTitle = "Web App - Unavailable";
|
||||
public const string ProxyPageTitle = "Example Gateway - Notification";
|
||||
public const string InspectionIssuer = "CN=Example Inspection CA, O=Example Security Ltd, C=US";
|
||||
|
||||
private const string PlatformPageHead =
|
||||
"<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\r\n" +
|
||||
"<html xmlns=\"http://www.w3.org/1999/xhtml\">\r\n<head>\r\n<title>Web App - Unavailable</title>\r\n" +
|
||||
"<style type=\"text/css\">\r\n html { height: 100%; width: 100%; }\r\n" +
|
||||
" #content { margin: 0 auto; padding: 40px 0 0; font-family: \"Segoe UI\", Arial, sans-serif; }\r\n" +
|
||||
" h3 { color: #3a3a3a; font-size: 26px; font-weight: 300; }\r\n</style>\r\n</head>\r\n<body>\r\n<div id=\"content\">\r\n";
|
||||
|
||||
public const string CertificateRequiredPage =
|
||||
PlatformPageHead +
|
||||
"<h3>Error 403 - Forbidden: Client Certificate Required.</h3>\r\n" +
|
||||
"<p>The web app you have attempted to reach requires a client certificate and does not accept any requests without one.</p>\r\n" +
|
||||
"</div>\r\n</body>\r\n</html>\r\n";
|
||||
|
||||
public const string StoppedAppPage =
|
||||
PlatformPageHead +
|
||||
"<h3>Error 403 - This web app is stopped.</h3>\r\n" +
|
||||
"<p>The web app you have attempted to reach is currently stopped and does not accept any requests.</p>\r\n" +
|
||||
"</div>\r\n</body>\r\n</html>\r\n";
|
||||
|
||||
public const string ProxyPage =
|
||||
"<!DOCTYPE html>\r\n<html><head><meta charset=\"utf-8\"><title>Example Gateway - Notification</title></head>\r\n" +
|
||||
"<body><h1>Access blocked</h1><p>The request was blocked by the security policy of your organization.</p></body></html>\r\n";
|
||||
|
||||
/// <summary>The platform's "Client Certificate Required" page seen through an inspecting proxy.</summary>
|
||||
public static ResponseClassification CertificateRequired(int statusCode = 403) =>
|
||||
ResponseClassification.ForForeign(statusCode, ResponseClassification.ClientCertificateRequired, PlatformPageTitle, isHtml: true, InspectionIssuer);
|
||||
|
||||
public static ResponseClassification StoppedApp(int statusCode = 403) =>
|
||||
ResponseClassification.ForPlatformStopped(statusCode, PlatformPageTitle, isHtml: true, serverCertIssuer: null);
|
||||
}
|
||||
}
|
||||
+18
-1
@@ -7,6 +7,7 @@ using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
@@ -23,17 +24,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
|
||||
public IReadOnlyList<CapturedRequest> Captured => _captured;
|
||||
|
||||
/// <summary>
|
||||
/// Queues a response from the backend: like the backend's correlation middleware, it echoes
|
||||
/// the request's <c>X-Correlation-ID</c> — the proof of origin a 401/403 is classified by.
|
||||
/// </summary>
|
||||
public RecordingHttpMessageHandler QueueStatus(HttpStatusCode status, string? body = null)
|
||||
{
|
||||
_script.Enqueue(_ =>
|
||||
_script.Enqueue(request =>
|
||||
{
|
||||
var resp = new HttpResponseMessage(status);
|
||||
if (body != null) resp.Content = new StringContent(body, Encoding.UTF8, "application/json");
|
||||
if (request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids))
|
||||
resp.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids);
|
||||
return resp;
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Queues a response that something on the network path produced (platform page, proxy
|
||||
/// page): no correlation echo.
|
||||
/// </summary>
|
||||
public RecordingHttpMessageHandler QueueForeign(HttpStatusCode status, string body, string mediaType = "text/html")
|
||||
{
|
||||
_script.Enqueue(_ => new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) });
|
||||
return this;
|
||||
}
|
||||
|
||||
public RecordingHttpMessageHandler QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
#nullable enable
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="ResponseOriginClassifier"/>: only the echo of the request's correlation id proves
|
||||
/// that a 401/403 came from the backend. Fixtures are synthetic copies of the platform and proxy
|
||||
/// pages (<see cref="InterceptionFixtures"/>).
|
||||
/// </summary>
|
||||
public sealed class ResponseOriginClassifierTests
|
||||
{
|
||||
private const string SentId = "0123456789abcdef0123456789abcdef";
|
||||
|
||||
private static HttpResponseMessage Response(HttpStatusCode status, string? body, string mediaType, string? echo = null)
|
||||
{
|
||||
var response = new HttpResponseMessage(status);
|
||||
if (body != null) response.Content = new StringContent(body, Encoding.UTF8, mediaType);
|
||||
if (echo != null) response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, echo);
|
||||
return response;
|
||||
}
|
||||
|
||||
private static ResponseClassification Classify(HttpResponseMessage response, string? issuer = null)
|
||||
{
|
||||
var body = ResponseOriginClassifier.ReadBodyPrefixAsync(response).GetAwaiter().GetResult();
|
||||
return ResponseOriginClassifier.Classify(response, SentId, body, issuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Platform_certificate_required_page_is_foreign_client_certificate_required()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html");
|
||||
|
||||
var c = Classify(response, InterceptionFixtures.InspectionIssuer);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.True(c.IsForeign);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle);
|
||||
Assert.True(c.IsHtml);
|
||||
Assert.Equal(403, c.StatusCode);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, c.ServerCertIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Platform_stopped_app_page_is_platform_stopped()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, c.Origin);
|
||||
Assert.False(c.IsForeign);
|
||||
Assert.Null(c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Proxy_page_is_foreign_response_with_its_title()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason);
|
||||
Assert.Equal(InterceptionFixtures.ProxyPageTitle, c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Json_403_with_echoed_correlation_id_is_backend()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Json_403_without_echo_is_foreign_response()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, c.Origin);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason);
|
||||
Assert.False(c.IsHtml);
|
||||
Assert.Null(c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Html_page_with_echo_is_backend_because_the_header_wins()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Echo_of_another_request_id_is_not_proof()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Unauthorized, "{}", "application/json", echo: "ffffffffffffffffffffffffffffffff");
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, Classify(response).Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_sent_id_can_never_be_echoed()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, "{}", "application/json", echo: SentId);
|
||||
|
||||
Assert.Equal(ResponseOrigin.Foreign, ResponseOriginClassifier.Classify(response, sentCorrelationId: null, bodyPrefix: "{}").Origin);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Markers_match_case_insensitively_and_in_plain_text()
|
||||
{
|
||||
using var stopped = Response(HttpStatusCode.Forbidden, "error 403 - this web app is stopped.", "text/plain");
|
||||
using var certificate = Response(HttpStatusCode.Forbidden, "Forbidden: CLIENT CERTIFICATE REQUIRED", "text/plain");
|
||||
|
||||
Assert.Equal(ResponseOrigin.PlatformStopped, Classify(stopped).Origin);
|
||||
var c = Classify(certificate);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason);
|
||||
Assert.False(c.IsHtml);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Html_served_with_generic_content_type_is_sniffed()
|
||||
{
|
||||
using var response = Response(HttpStatusCode.Forbidden, " <html><head><title>Blocked</title></head></html>", "text/plain");
|
||||
|
||||
var c = Classify(response);
|
||||
|
||||
Assert.True(c.IsHtml);
|
||||
Assert.Equal("Blocked", c.PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Page_title_is_capped_at_80_characters()
|
||||
{
|
||||
var longTitle = new string('x', 120);
|
||||
using var response = Response(HttpStatusCode.Forbidden, $"<html><head><title>{longTitle}</title></head></html>", "text/html");
|
||||
|
||||
Assert.Equal(new string('x', 80), Classify(response).PageTitle);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Body_prefix_is_capped_and_a_marker_beyond_it_is_not_seen()
|
||||
{
|
||||
var body = new string(' ', ResponseOriginClassifier.MaxBodyChars) + "Client Certificate Required";
|
||||
using var response = Response(HttpStatusCode.Forbidden, body, "text/plain");
|
||||
|
||||
var prefix = await ResponseOriginClassifier.ReadBodyPrefixAsync(response);
|
||||
|
||||
Assert.Equal(ResponseOriginClassifier.MaxBodyChars, prefix!.Length);
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, ResponseOriginClassifier.Classify(response, SentId, prefix).ForeignReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Missing_body_reads_as_null_and_classifies_as_foreign_response()
|
||||
{
|
||||
using var response = new HttpResponseMessage(HttpStatusCode.Forbidden);
|
||||
|
||||
Assert.Null(await ResponseOriginClassifier.ReadBodyPrefixAsync(response));
|
||||
Assert.Equal(ResponseClassification.ForeignResponse, Classify(response).ForeignReason);
|
||||
}
|
||||
}
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// <see cref="ServerCertificateObserver"/> must never weaken TLS validation: the installed
|
||||
/// callback returns exactly the verdict .NET Framework applies without one
|
||||
/// (<c>sslPolicyErrors == None</c>) and only records the issuer as evidence.
|
||||
/// </summary>
|
||||
public sealed class ServerCertificateObserverTests
|
||||
{
|
||||
internal static X509Certificate2 SelfSigned(string subject)
|
||||
{
|
||||
using (var rsa = RSA.Create(2048))
|
||||
{
|
||||
var request = new CertificateRequest(subject, rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1);
|
||||
return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1));
|
||||
}
|
||||
}
|
||||
|
||||
private static Func<HttpRequestMessage, X509Certificate2, X509Chain, SslPolicyErrors, bool> AttachedCallback(ServerCertificateObserver observer)
|
||||
{
|
||||
var handler = new HttpClientHandler();
|
||||
observer.Attach(handler);
|
||||
return handler.ServerCertificateCustomValidationCallback!;
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SslPolicyErrors.None, true)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNameMismatch, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
[InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)]
|
||||
public void Callback_accepts_only_a_certificate_without_policy_errors(SslPolicyErrors errors, bool expected)
|
||||
{
|
||||
using var certificate = SelfSigned("CN=Example Inspection CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
|
||||
Assert.Equal(expected, callback(null!, certificate, null!, errors));
|
||||
// Recorded whatever the verdict: a rejected inspection certificate is evidence too.
|
||||
Assert.Equal(certificate.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Last_validated_connection_wins()
|
||||
{
|
||||
using var first = SelfSigned("CN=Example Backend CA");
|
||||
using var second = SelfSigned("CN=Example Inspection CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
|
||||
Assert.Null(observer.LastIssuer);
|
||||
callback(null!, first, null!, SslPolicyErrors.None);
|
||||
callback(null!, second, null!, SslPolicyErrors.None);
|
||||
|
||||
Assert.Equal(second.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Missing_certificate_is_rejected_and_keeps_the_last_issuer()
|
||||
{
|
||||
using var certificate = SelfSigned("CN=Example Backend CA");
|
||||
var observer = new ServerCertificateObserver();
|
||||
var callback = AttachedCallback(observer);
|
||||
callback(null!, certificate, null!, SslPolicyErrors.None);
|
||||
|
||||
Assert.False(callback(null!, null!, null!, SslPolicyErrors.RemoteCertificateNotAvailable));
|
||||
Assert.Equal(certificate.Issuer, observer.LastIssuer);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Long_issuer_is_capped()
|
||||
{
|
||||
using var certificate = SelfSigned($"CN=Example Inspection CA, O={new string('o', 200)}");
|
||||
var observer = new ServerCertificateObserver();
|
||||
|
||||
observer.Validate(certificate, SslPolicyErrors.None);
|
||||
|
||||
Assert.Equal(ServerCertificateObserver.MaxIssuerLength, observer.LastIssuer!.Length);
|
||||
Assert.StartsWith(observer.LastIssuer, certificate.Issuer, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
}
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// The transport-local pause after an intercepted upload (a 401/403 something on the network
|
||||
/// path answered): no HTTP while it holds, backoff 30 s → 15 min, a success ends it, the
|
||||
/// terminal phase gets exactly one attempt despite it, and it is never an auth failure.
|
||||
/// Driven by the virtual clock — the pause is a "not before", never a timer.
|
||||
/// </summary>
|
||||
public sealed class TelemetryUploadOrchestratorInterceptionTests
|
||||
{
|
||||
private static readonly DateTime Start = new DateTime(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Rig : IDisposable
|
||||
{
|
||||
private int _rowKey;
|
||||
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public VirtualClock Clock { get; } = new VirtualClock(Start);
|
||||
public FakeBackendTelemetryUploader Uploader { get; } = new FakeBackendTelemetryUploader();
|
||||
public TelemetryUploadOrchestrator Sut { get; }
|
||||
|
||||
public Rig()
|
||||
{
|
||||
var spool = new TelemetrySpool(Tmp.Path, Clock);
|
||||
Sut = new TelemetryUploadOrchestrator(spool, Uploader, Clock, batchSize: 10, retryBackoffs: new[] { TimeSpan.Zero });
|
||||
}
|
||||
|
||||
public void Enqueue(int count = 1)
|
||||
{
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
var rowKey = (++_rowKey).ToString("D10");
|
||||
Sut.Enqueue(new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "T1_S1",
|
||||
rowKey: rowKey,
|
||||
payloadJson: "{}",
|
||||
isSessionScoped: true,
|
||||
requiresImmediateFlush: false));
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose() { Sut.Dispose(); Tmp.Dispose(); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Intercepted_upload_retains_the_batch_and_later_drains_make_no_http_call()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(2);
|
||||
rig.Uploader.QueueIntercepted();
|
||||
|
||||
var first = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, first.FailedBatches);
|
||||
Assert.Equal(0, first.UploadedItems);
|
||||
Assert.Contains("intercepted", first.LastErrorReason);
|
||||
Assert.Equal(-1, rig.Sut.LastUploadedItemId);
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
Assert.False(rig.Sut.IsPaused);
|
||||
|
||||
// Periodic drains and immediate-flush wakeups (a new item, another drain) during the
|
||||
// pause: no HTTP, and an expected state rather than another failed batch.
|
||||
rig.Enqueue();
|
||||
var paused = await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(29));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Equal(0, paused.FailedBatches);
|
||||
Assert.Contains("upload intercepted, next attempt not before", paused.LastErrorReason);
|
||||
Assert.Equal(-1, rig.Sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Backoff_runs_30s_1m_2m_5m_10m_then_15m_capped()
|
||||
{
|
||||
var expected = new[] { 30, 60, 120, 300, 600, 900, 900 }.Select(s => TimeSpan.FromSeconds(s)).ToArray();
|
||||
Assert.Equal(expected.Take(6), TelemetryUploadOrchestrator.InterceptionBackoffs);
|
||||
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(expected.Length);
|
||||
|
||||
foreach (var delay in expected)
|
||||
{
|
||||
var calls = rig.Uploader.CallCount;
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
|
||||
rig.Clock.Advance(delay - TimeSpan.FromSeconds(1));
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(calls + 1, rig.Uploader.CallCount);
|
||||
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A clock that can be set back, as NTP does to a device that booted with a future RTC time.</summary>
|
||||
private sealed class SettableClock : AutopilotMonitor.DecisionCore.Engine.IClock
|
||||
{
|
||||
public SettableClock(DateTime utcNow) { UtcNow = utcNow; }
|
||||
public DateTime UtcNow { get; set; }
|
||||
public Task Delay(TimeSpan delay, System.Threading.CancellationToken cancellationToken = default) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Clock_set_back_past_the_pause_start_ends_the_pause_instead_of_stretching_it()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var clock = new SettableClock(Start);
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueIntercepted(2);
|
||||
using var sut = new TelemetryUploadOrchestrator(new TelemetrySpool(tmp.Path, clock), uploader, clock, retryBackoffs: new[] { TimeSpan.Zero });
|
||||
sut.Enqueue(new TelemetryItemDraft(TelemetryItemKind.Event, "T1_S1", "0000000001", "{}", isSessionScoped: true));
|
||||
|
||||
await sut.DrainAllAsync();
|
||||
clock.UtcNow = Start.AddHours(-2);
|
||||
|
||||
Assert.False(sut.IsInterceptionPaused);
|
||||
await sut.DrainAllAsync();
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.True(sut.IsInterceptionPaused); // re-armed from the corrected clock
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Successful_upload_ends_the_pause_and_resets_the_backoff()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(3);
|
||||
rig.Uploader.QueueIntercepted(2).QueueOk().QueueIntercepted();
|
||||
|
||||
await rig.Sut.DrainAllAsync(); // intercepted, 30 s
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync(); // intercepted, 1 min
|
||||
rig.Clock.Advance(TimeSpan.FromMinutes(1));
|
||||
var cleared = await rig.Sut.DrainAllAsync(); // the path is clear
|
||||
|
||||
Assert.Equal(3, cleared.UploadedItems);
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
|
||||
rig.Enqueue();
|
||||
await rig.Sut.DrainAllAsync(); // intercepted again: back to 30 s
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(29));
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(1));
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_drain_makes_exactly_one_attempt_despite_the_pause()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(3);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
|
||||
Assert.True(rig.Sut.GrantTerminalAttempt());
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, terminal.FailedBatches);
|
||||
Assert.True(rig.Sut.IsInterceptionPaused);
|
||||
|
||||
// One attempt per process: a later terminal drain (orchestrator.Stop) honours the pause.
|
||||
rig.Sut.GrantTerminalAttempt();
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Terminal_attempt_that_gets_through_drains_everything()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue(25);
|
||||
rig.Uploader.QueueIntercepted().QueueOk(3);
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
rig.Sut.GrantTerminalAttempt();
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(25, terminal.UploadedItems);
|
||||
Assert.Equal(4, rig.Uploader.CallCount);
|
||||
Assert.Equal(24, rig.Sut.LastUploadedItemId);
|
||||
Assert.False(rig.Sut.IsInterceptionPaused);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Grant_without_a_pause_changes_nothing_until_an_interception_meets_it()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
Assert.False(rig.Sut.GrantTerminalAttempt());
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueOk().QueueIntercepted(3);
|
||||
|
||||
var normal = await rig.Sut.DrainAllAsync();
|
||||
Assert.Equal(1, normal.UploadedItems);
|
||||
|
||||
rig.Enqueue();
|
||||
await rig.Sut.DrainAllAsync(); // intercepted during the terminal phase
|
||||
await rig.Sut.DrainAllAsync(); // the granted attempt
|
||||
await rig.Sut.DrainAllAsync(); // pause honoured again
|
||||
|
||||
Assert.Equal(3, rig.Uploader.CallCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DeviceBlocked_pause_still_skips_terminal_drains()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueOkWithSignals(deviceBlocked: true, unblockAt: Start.AddMinutes(10));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
Assert.True(rig.Sut.IsPaused);
|
||||
|
||||
rig.Enqueue();
|
||||
Assert.False(rig.Sut.GrantTerminalAttempt());
|
||||
var terminal = await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, rig.Uploader.CallCount);
|
||||
Assert.Contains("device blocked", terminal.LastErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task First_interception_is_reported_once_and_never_as_a_blocked_or_auth_failure()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted(2);
|
||||
var reports = new List<InterceptionReport>();
|
||||
var blocked = 0;
|
||||
rig.Sut.UploadIntercepted += (_, r) => reports.Add(r);
|
||||
rig.Sut.UploadBlocked += (_, __) => blocked++;
|
||||
|
||||
await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(2, rig.Uploader.CallCount);
|
||||
var report = Assert.Single(reports);
|
||||
Assert.Equal(Start, report.FirstBlockedAtUtc);
|
||||
Assert.Equal(ResponseClassification.ClientCertificateRequired, report.Interception.ForeignReason);
|
||||
Assert.Equal(403, report.Interception.StatusCode);
|
||||
Assert.Equal(InterceptionFixtures.InspectionIssuer, report.Interception.ServerCertIssuer);
|
||||
Assert.Equal(0, blocked);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Permanent_block_report_keeps_its_own_once_per_process_gate()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.Enqueue();
|
||||
rig.Uploader.QueueIntercepted().QueuePermanent("http 400: contract").QueuePermanent("http 400: contract");
|
||||
int intercepted = 0, blocked = 0;
|
||||
rig.Sut.UploadIntercepted += (_, __) => intercepted++;
|
||||
rig.Sut.UploadBlocked += (_, __) => blocked++;
|
||||
|
||||
await rig.Sut.DrainAllAsync();
|
||||
rig.Clock.Advance(TimeSpan.FromSeconds(30));
|
||||
await rig.Sut.DrainAllAsync();
|
||||
await rig.Sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(3, rig.Uploader.CallCount);
|
||||
Assert.Equal(1, intercepted);
|
||||
Assert.Equal(1, blocked);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,6 +195,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public int AbsoluteMaxSessionHours { get; set; } = 48;
|
||||
|
||||
/// <summary>
|
||||
/// Further boots the agent may try after a session registration was answered by a
|
||||
/// 401/403 the backend did not author, before it cleans up like the emergency break.
|
||||
/// Merged from remote config (also from the cache, since an intercepted start gets no
|
||||
/// live config), clamped to 0..5. Default: 1.
|
||||
/// </summary>
|
||||
public int InterceptedRegistrationRetryBoots { get; set; } = 1;
|
||||
|
||||
/// <summary>
|
||||
/// Bootstrap token for pre-MDM auth during OOBE.
|
||||
/// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg.
|
||||
|
||||
@@ -28,6 +28,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public static class RemoteConfigMerger
|
||||
{
|
||||
/// <summary>
|
||||
/// Upper bound for <see cref="AgentConfiguration.InterceptedRegistrationRetryBoots"/>: each
|
||||
/// extra boot keeps an intercepted device's agent installed until its next boot.
|
||||
/// </summary>
|
||||
private const int MaxInterceptedRegistrationRetryBoots = 5;
|
||||
|
||||
/// <summary>
|
||||
/// Overwrites the relevant <paramref name="agentConfig"/> fields with values from
|
||||
/// <paramref name="remoteConfig"/>. Remote wins unconditionally for every field that has
|
||||
@@ -174,6 +180,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
agentConfig.MaxAuthFailures = remoteConfig.MaxAuthFailures;
|
||||
agentConfig.AuthFailureTimeoutMinutes = remoteConfig.AuthFailureTimeoutMinutes;
|
||||
|
||||
// ---------------- Intercepted-registration buffer — null (a backend that predates the
|
||||
// field) keeps the current value; anything else is clamped to
|
||||
// 0..MaxInterceptedRegistrationRetryBoots.
|
||||
if (remoteConfig.InterceptedRegistrationRetryBoots.HasValue)
|
||||
{
|
||||
var requested = remoteConfig.InterceptedRegistrationRetryBoots.Value;
|
||||
var bounded = Math.Max(0, Math.Min(MaxInterceptedRegistrationRetryBoots, requested));
|
||||
if (bounded != requested)
|
||||
logger?.Warning($"Remote config: InterceptedRegistrationRetryBoots {requested} is outside 0..{MaxInterceptedRegistrationRetryBoots} — using {bounded}.");
|
||||
agentConfig.InterceptedRegistrationRetryBoots = bounded;
|
||||
}
|
||||
|
||||
// ---------------- Nested CollectorConfiguration — only the two knobs that have a
|
||||
// V2 consumer on AgentConfiguration itself. The rest flow through
|
||||
// the untouched remoteConfig.Collectors to DefaultComponentFactory.
|
||||
|
||||
@@ -178,10 +178,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
{
|
||||
// Auth failures are NOT retryable — 401/403 won't go away with another
|
||||
// attempt; record + bail straight to cache/defaults so an operator can
|
||||
// see the AuthCertificateRejected/DeviceNotRegistered distress signal
|
||||
// without 100 s of needless retry delay.
|
||||
_logger.Warning($"Config fetch authentication failed: {ex.Message}");
|
||||
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.EndpointUnavailable);
|
||||
// see the distress signal without 100 s of needless retry delay. A 401/403
|
||||
// from something on the network path is not counted by the tracker; it
|
||||
// reports ClientCertificateNotReceived instead.
|
||||
_logger.Warning(ex.IsForeignResponse
|
||||
? $"Config fetch intercepted on the network path: {ex.Message}"
|
||||
: $"Config fetch authentication failed: {ex.Message}");
|
||||
_authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.Classification);
|
||||
lastException = ex;
|
||||
LastFetchAuthStatusCode = ex.StatusCode;
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Short local fingerprint of the network a start ran on: the first <see cref="Length"/> hex
|
||||
/// characters of SHA-256 over the active interface's connection type and default gateway, as
|
||||
/// <see cref="NetworkChangeDetector"/> captures them. Stored beside the failed-registration
|
||||
/// record and compared by the next successful registration; only that comparison (changed or
|
||||
/// not) leaves the device, never a fingerprint.
|
||||
/// </summary>
|
||||
internal static class NetworkFingerprint
|
||||
{
|
||||
internal const int Length = 16;
|
||||
|
||||
/// <summary>Fingerprint of the current network; null when it is unknown. Never throws.</summary>
|
||||
public static string? TryCapture()
|
||||
{
|
||||
try
|
||||
{
|
||||
return Of(NetworkChangeDetector.CaptureInterfaceState());
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Fingerprint of a captured state; null without an active interface, type or gateway.</summary>
|
||||
internal static string? Of(NetworkChangeDetector.NetworkStateSnapshot? state)
|
||||
{
|
||||
if (state == null || !state.HasNetwork
|
||||
|| string.IsNullOrEmpty(state.ConnectionType) || string.IsNullOrEmpty(state.Gateway))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return StartupEventGate.HashString(state.ConnectionType + "|" + state.Gateway).Substring(0, Length);
|
||||
}
|
||||
|
||||
/// <summary>True/false when both fingerprints are known, null when either is unknown.</summary>
|
||||
public static bool? Differs(string? before, string? after) =>
|
||||
before == null || after == null ? (bool?)null : !string.Equals(before, after, StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
+9
-6
@@ -85,12 +85,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
|
||||
protected override void Collect()
|
||||
{
|
||||
// Full path writes 38 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (5) + network delta (9: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency) + IME tracker health
|
||||
// (9) + signal pipeline (9: 5 counters, stage, applied ordinal, wg level+score).
|
||||
// cap=38 → HashHelpers.GetPrime(38)=41 buckets → no resize on the last key.
|
||||
var data = new Dictionary<string, object>(capacity: 38, StringComparer.Ordinal)
|
||||
// Full path writes 48 keys: agent_version + process metrics (5: cpu, ws, private,
|
||||
// threads, handles) + spool stats (5) + network (10: requests, failures,
|
||||
// bytes_up/down, avg_latency, total_up/down/requests/latency, upload 2xx without
|
||||
// echo) + IME tracker health (18) + signal pipeline (9: 5 counters, stage, applied
|
||||
// ordinal, wg level+score). cap=48 → no resize on the last key.
|
||||
var data = new Dictionary<string, object>(capacity: 48, StringComparer.Ordinal)
|
||||
{
|
||||
{ "agent_version", _agentVersion }
|
||||
};
|
||||
@@ -160,6 +160,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
}
|
||||
|
||||
// --- Network delta ---
|
||||
// Cumulative and independent of the delta baseline: telemetry uploads answered 2xx
|
||||
// without the backend's correlation echo (counted only — see BackendTelemetryUploader).
|
||||
data["telemetry_upload_ok_without_echo"] = _networkMetrics.OkWithoutEchoCount;
|
||||
try
|
||||
{
|
||||
var currentNet = _networkMetrics.GetSnapshot();
|
||||
|
||||
+30
-18
@@ -37,6 +37,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
private const int ConnectivityCheckDelayMs = 5000;
|
||||
private const int ConnectivityCheckTimeoutMs = 8000;
|
||||
private const string Source = "NetworkChangeDetector";
|
||||
private const string WifiConnectionType = "WiFi";
|
||||
|
||||
/// <summary>
|
||||
/// Well-known MDM/Entra endpoints that must be reachable for enrollment to succeed.
|
||||
@@ -312,6 +313,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private NetworkStateSnapshot CaptureCurrentState()
|
||||
{
|
||||
var snapshot = CaptureInterfaceState(_logger);
|
||||
|
||||
// Collect WiFi SSID synchronously (needed for before/after comparison)
|
||||
if (snapshot.ConnectionType == WifiConnectionType)
|
||||
{
|
||||
Guid? nicGuid = Guid.TryParse(snapshot.InterfaceId, out var parsedGuid) ? parsedGuid : (Guid?)null;
|
||||
var wifiInfo = CollectWiFiInfo(nicGuid);
|
||||
snapshot.WifiSsid = wifiInfo.ssid;
|
||||
snapshot.WifiSignalPercent = wifiInfo.signal;
|
||||
snapshot.WifiRadioType = wifiInfo.radioType;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Interface-level state of the active network (<see cref="NetworkInterfaceLocator"/>):
|
||||
/// connection type, adapter, first IPv4 address, first gateway — no WiFi details. Shared
|
||||
/// with <see cref="NetworkFingerprint"/> so the registration record fingerprints the network
|
||||
/// exactly as this detector sees it. Never throws: a failed read is "no network".
|
||||
/// </summary>
|
||||
internal static NetworkStateSnapshot CaptureInterfaceState(AgentLogger logger = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -346,13 +370,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
}
|
||||
}
|
||||
|
||||
var snapshot = new NetworkStateSnapshot
|
||||
return new NetworkStateSnapshot
|
||||
{
|
||||
HasNetwork = true,
|
||||
AdapterName = activeNic.Name,
|
||||
AdapterDescription = activeNic.Description,
|
||||
InterfaceType = activeNic.NetworkInterfaceType.ToString(),
|
||||
ConnectionType = isWifi ? "WiFi" : "Ethernet",
|
||||
ConnectionType = isWifi ? WifiConnectionType : "Ethernet",
|
||||
MacAddress = activeNic.GetPhysicalAddress().ToString(),
|
||||
InterfaceId = activeNic.Id,
|
||||
LinkSpeedMbps = activeNic.Speed / 1_000_000,
|
||||
@@ -360,22 +384,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
Gateway = gateway,
|
||||
CapturedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
// Collect WiFi SSID synchronously (needed for before/after comparison)
|
||||
if (isWifi)
|
||||
{
|
||||
Guid? nicGuid = Guid.TryParse(activeNic.Id, out var parsedGuid) ? parsedGuid : (Guid?)null;
|
||||
var wifiInfo = CollectWiFiInfo(nicGuid);
|
||||
snapshot.WifiSsid = wifiInfo.ssid;
|
||||
snapshot.WifiSignalPercent = wifiInfo.signal;
|
||||
snapshot.WifiRadioType = wifiInfo.radioType;
|
||||
}
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}");
|
||||
logger?.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}");
|
||||
return new NetworkStateSnapshot { HasNetwork = false, CapturedAt = DateTime.UtcNow };
|
||||
}
|
||||
}
|
||||
@@ -434,7 +446,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
if (snapshot == null || !snapshot.HasNetwork)
|
||||
return "None";
|
||||
|
||||
if (snapshot.ConnectionType == "WiFi" && !string.IsNullOrEmpty(snapshot.WifiSsid))
|
||||
if (snapshot.ConnectionType == WifiConnectionType && !string.IsNullOrEmpty(snapshot.WifiSsid))
|
||||
return $"WiFi '{snapshot.WifiSsid}'";
|
||||
|
||||
return $"{snapshot.ConnectionType} ({snapshot.IpAddress ?? "no IP"})";
|
||||
@@ -465,7 +477,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
if (before.ConnectionType != after.ConnectionType)
|
||||
return "type_change";
|
||||
|
||||
if (before.ConnectionType == "WiFi" && after.ConnectionType == "WiFi" &&
|
||||
if (before.ConnectionType == WifiConnectionType && after.ConnectionType == WifiConnectionType &&
|
||||
before.WifiSsid != after.WifiSsid)
|
||||
return "ssid_change";
|
||||
|
||||
@@ -479,7 +491,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
// Inner type: network state snapshot
|
||||
// -------------------------------------------------------------------
|
||||
|
||||
private class NetworkStateSnapshot
|
||||
internal sealed class NetworkStateSnapshot
|
||||
{
|
||||
public bool HasNetwork { get; set; }
|
||||
public string AdapterName { get; set; }
|
||||
|
||||
+114
-105
@@ -1,9 +1,11 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
@@ -40,6 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private readonly string _bootstrapToken;
|
||||
private readonly string _agentVersion;
|
||||
private readonly Logging.AgentLogger _logger;
|
||||
private readonly ServerCertificateObserver _serverCertificates;
|
||||
|
||||
/// <summary>
|
||||
/// Protected ctor for testability — <c>SessionRegistrationHelperTests.FakeApiClient</c>
|
||||
@@ -48,6 +51,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// </summary>
|
||||
protected BackendApiClient() { }
|
||||
|
||||
/// <param name="serverCertificates">
|
||||
/// Observer installed on the <paramref name="httpClient"/>'s handler; its issuer becomes
|
||||
/// evidence on a 401/403 that did not come from the backend. Null = no evidence.
|
||||
/// </param>
|
||||
public BackendApiClient(
|
||||
HttpClient httpClient,
|
||||
string baseUrl,
|
||||
@@ -57,7 +64,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
bool useBootstrapTokenAuth,
|
||||
string bootstrapToken,
|
||||
string agentVersion,
|
||||
Logging.AgentLogger logger)
|
||||
Logging.AgentLogger logger,
|
||||
ServerCertificateObserver serverCertificates = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("baseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -69,6 +77,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
_bootstrapToken = bootstrapToken;
|
||||
_agentVersion = agentVersion;
|
||||
_logger = logger;
|
||||
_serverCertificates = serverCertificates;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -107,7 +116,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
using (var httpRequest = new HttpRequestMessage(HttpMethod.Get, url))
|
||||
{
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
_logger?.Debug($"GetAgentConfigAsync: GET {url} (attempt {attempt}/{maxAttempts})");
|
||||
|
||||
HttpResponseMessage response;
|
||||
@@ -137,7 +146,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
continue;
|
||||
}
|
||||
|
||||
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||
await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
@@ -173,9 +182,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// emergency-break path passes a longer <paramref name="timeout"/> because its first
|
||||
/// request of a freshly booted process pays cold TLS + DNS + a possible backend
|
||||
/// cold start (observed 503s take well over 5 s).
|
||||
/// Returns <c>true</c> when the HTTP round-trip completed (the endpoint always
|
||||
/// answers 200), <c>false</c> on timeout/transport failure — the caller-side
|
||||
/// anti-flood layer uses this to decide whether the report counts as delivered.
|
||||
/// Returns <c>true</c> when the HTTP round-trip completed (the endpoint answers 200 for
|
||||
/// every report it receives), <c>false</c> on timeout/transport failure or when a
|
||||
/// 401/403 came from something other than the backend (the report never arrived) — the
|
||||
/// caller-side anti-flood layer uses this to decide whether the report counts as delivered.
|
||||
/// Virtual for tests.
|
||||
/// </summary>
|
||||
public virtual async Task<bool> ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null)
|
||||
@@ -195,13 +205,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
// Backend security validation requires tenant id before parsing the body.
|
||||
httpRequest.Headers.Add("X-Tenant-Id", report.TenantId);
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
|
||||
using (var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(5)))
|
||||
using (var response = await _httpClient.SendAsync(httpRequest, cts.Token).ConfigureAwait(false))
|
||||
{
|
||||
// Response status is deliberately ignored — endpoint always returns 200.
|
||||
return true;
|
||||
// Every status counts as delivered (unchanged) — except a 401/403 without the
|
||||
// backend's correlation echo: then the report never reached the backend.
|
||||
if (!IsAuthFailureStatus(response.StatusCode))
|
||||
return true;
|
||||
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var classification = ResponseOriginClassifier.Classify(response, correlationId, body);
|
||||
if (!classification.IsForeign)
|
||||
return true;
|
||||
|
||||
_logger?.Debug(
|
||||
$"ReportAgentErrorAsync: http {(int)response.StatusCode} did not come from the backend " +
|
||||
$"({classification.ForeignReason}) — report not delivered.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -221,7 +243,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
||||
})
|
||||
{
|
||||
AddSecurityHeaders(httpRequest);
|
||||
var correlationId = AddSecurityHeaders(httpRequest);
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
@@ -236,7 +258,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
|
||||
using (response)
|
||||
{
|
||||
await ThrowOnAuthFailureAsync(response).ConfigureAwait(false);
|
||||
await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false);
|
||||
await ThrowOnCodedConflictAsync(response).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
@@ -284,61 +306,76 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// Throws <see cref="BackendAuthException"/> for 401/403 responses so callers can
|
||||
/// distinguish authentication failures from transient server errors.
|
||||
/// <para>
|
||||
/// Field case (2026-08-09): a pre-cutover agent build pointed at a
|
||||
/// retired Function App; Azure answers for a stopped app with a platform-level 403
|
||||
/// carrying an HTML error page ("Web App - Unavailable"). Our backend never returns
|
||||
/// HTML on 401/403 — an HTML body therefore means the response did not come from the
|
||||
/// backend's auth layer at all (stopped/retired app, edge proxy, WAF). Reporting that
|
||||
/// as "device is not authorized" sends troubleshooting down the wrong path, so the
|
||||
/// two cases get distinct messages and the exception carries a flag.
|
||||
/// Only a response that echoes this request's <c>X-Correlation-ID</c> is the backend's
|
||||
/// verdict (<see cref="ResponseOriginClassifier"/>). Anything else never came from the
|
||||
/// backend's auth layer: the platform page of a stopped app (field case 2026-08-09: a
|
||||
/// pre-cutover build pointed at a retired Function App), or the platform's "Client
|
||||
/// Certificate Required" page / a proxy page when TLS inspection on the network path
|
||||
/// strips the device certificate. Reporting those as "device is not authorized" sends
|
||||
/// troubleshooting down the wrong path, so each origin gets its own message and the
|
||||
/// exception carries the classification.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static async Task ThrowOnAuthFailureAsync(HttpResponseMessage response)
|
||||
private async Task ThrowOnAuthFailureAsync(HttpResponseMessage response, string correlationId)
|
||||
{
|
||||
if (response.StatusCode != System.Net.HttpStatusCode.Unauthorized &&
|
||||
response.StatusCode != System.Net.HttpStatusCode.Forbidden)
|
||||
{
|
||||
if (!IsAuthFailureStatus(response.StatusCode))
|
||||
return;
|
||||
}
|
||||
|
||||
var statusText = $"{(int)response.StatusCode} {response.StatusCode}";
|
||||
var statusCode = (int)response.StatusCode;
|
||||
var statusText = $"{statusCode} {response.StatusCode}";
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var classification = ResponseOriginClassifier.Classify(
|
||||
response, correlationId, body, _serverCertificates?.LastIssuer);
|
||||
|
||||
string body = null;
|
||||
try
|
||||
if (classification.Origin != ResponseOrigin.Backend)
|
||||
{
|
||||
body = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Body is diagnostic-only; classification below treats null as non-HTML.
|
||||
}
|
||||
|
||||
if (LooksLikeHtmlErrorPage(response, body))
|
||||
{
|
||||
var title = TryExtractHtmlTitle(body);
|
||||
var titleText = title != null ? $" (\"{title}\")" : string.Empty;
|
||||
var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)";
|
||||
throw new BackendAuthException(
|
||||
$"Backend returned {statusText} with an HTML platform error page{titleText} from '{host}'. " +
|
||||
"This response did not come from the Autopilot Monitor backend — the endpoint appears stopped, " +
|
||||
"retired, or intercepted by a proxy. This is NOT a device-authorization failure; an outdated " +
|
||||
"agent build pointing at a stale ApiBaseUrl is the most likely cause.",
|
||||
(int)response.StatusCode,
|
||||
endpointUnavailable: true);
|
||||
DescribeNonBackendAnswer(response, statusText, classification),
|
||||
statusCode,
|
||||
classification: classification);
|
||||
}
|
||||
|
||||
// Backend JSON bodies may carry a machine-readable errorCode (Shared
|
||||
// Constants.AgentErrorCodes) that tells the caller how to react — e.g.
|
||||
// session_owner_mismatch → rotate the session id instead of treating this as a
|
||||
// certificate rejection. Absent/unparseable → null, classic auth failure.
|
||||
var errorCode = TryExtractErrorCode(body);
|
||||
throw new BackendAuthException(
|
||||
$"Backend returned {statusText}. " +
|
||||
"The device is not authorized. Check client certificate and Autopilot device validation.",
|
||||
(int)response.StatusCode,
|
||||
errorCode: errorCode);
|
||||
statusCode,
|
||||
errorCode: TryExtractErrorCode(body),
|
||||
classification: classification);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Message for a 401/403 the backend did not author. The cause leads because
|
||||
/// <c>RegistrationFailureSummary.LastError</c> keeps only 256 characters; HTML answers
|
||||
/// keep the phrase "HTML platform error page" plus the page title (ANALYZE-NET-001 matches both).
|
||||
/// </summary>
|
||||
private static string DescribeNonBackendAnswer(HttpResponseMessage response, string statusText, ResponseClassification classification)
|
||||
{
|
||||
var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)";
|
||||
var titleText = classification.PageTitle != null ? $" (\"{classification.PageTitle}\")" : string.Empty;
|
||||
var htmlPage = $"an HTML platform error page{titleText}";
|
||||
|
||||
if (classification.Origin == ResponseOrigin.PlatformStopped)
|
||||
{
|
||||
return $"Backend returned {statusText} with {(classification.IsHtml ? htmlPage : "a platform error page")} from '{host}'. " +
|
||||
"This response did not come from the Autopilot Monitor backend — the endpoint appears stopped or retired. " +
|
||||
"This is NOT a device-authorization failure; an outdated agent build pointing at a stale ApiBaseUrl " +
|
||||
"is the most likely cause.";
|
||||
}
|
||||
|
||||
var cause = classification.ForeignReason == ResponseClassification.ClientCertificateRequired
|
||||
? "The device certificate did not reach the backend (TLS inspection or a proxy on the network path)"
|
||||
: "Something on the network path answered instead of the backend (TLS inspection, a proxy or a security gateway)";
|
||||
var answer = classification.IsHtml ? htmlPage : "a response without the backend's correlation echo";
|
||||
return $"{cause}: {statusText} from '{host}' was {answer}. This is NOT a device-authorization failure.";
|
||||
}
|
||||
|
||||
private static bool IsAuthFailureStatus(HttpStatusCode statusCode) =>
|
||||
statusCode == HttpStatusCode.Unauthorized || statusCode == HttpStatusCode.Forbidden;
|
||||
|
||||
/// <summary>
|
||||
/// Throws <see cref="BackendRefusalException"/> for a 409 whose JSON body carries a
|
||||
/// machine-readable <c>errorCode</c> — register-session answers
|
||||
@@ -395,62 +432,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when a 401/403 response is an HTML page rather than a backend JSON/plain-text
|
||||
/// reply. Checks the declared Content-Type first, then falls back to sniffing the body
|
||||
/// prefix so a platform page served with a generic Content-Type is still recognized.
|
||||
/// Adds device / version / bootstrap-token security headers and returns the request's
|
||||
/// correlation id. Client certificate is sent at the TLS layer via the externally-supplied
|
||||
/// HttpClient's HttpClientHandler.
|
||||
/// </summary>
|
||||
private static bool LooksLikeHtmlErrorPage(HttpResponseMessage response, string body)
|
||||
{
|
||||
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
|
||||
if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return false;
|
||||
|
||||
var head = body.TrimStart();
|
||||
return head.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase) ||
|
||||
head.StartsWith("<html", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so the
|
||||
/// log line names the platform page without dumping markup. Null when absent/oversized.
|
||||
/// </summary>
|
||||
private static string TryExtractHtmlTitle(string body)
|
||||
{
|
||||
if (string.IsNullOrEmpty(body)) return null;
|
||||
try
|
||||
{
|
||||
var match = System.Text.RegularExpressions.Regex.Match(
|
||||
body, @"<title[^>]*>\s*(.*?)\s*</title>",
|
||||
System.Text.RegularExpressions.RegexOptions.IgnoreCase |
|
||||
System.Text.RegularExpressions.RegexOptions.Singleline,
|
||||
TimeSpan.FromMilliseconds(250));
|
||||
if (!match.Success) return null;
|
||||
|
||||
var title = match.Groups[1].Value;
|
||||
return title.Length == 0 ? null : (title.Length > 80 ? title.Substring(0, 80) : title);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds device / version / bootstrap-token security headers. Client certificate is sent
|
||||
/// at the TLS layer via the externally-supplied HttpClient's HttpClientHandler.
|
||||
/// </summary>
|
||||
private void AddSecurityHeaders(HttpRequestMessage request)
|
||||
private string AddSecurityHeaders(HttpRequestMessage request)
|
||||
{
|
||||
// Per-request correlation id for end-to-end tracing (mirrors BackendTelemetryUploader).
|
||||
// The backend's CorrelationIdMiddleware reads X-Correlation-ID; without it the backend
|
||||
// mints a fresh id per inbound request and agent↔backend calls cannot be tied together.
|
||||
// Generated per request so each attempt (e.g. config-fetch 503 retries) is independently
|
||||
// traceable.
|
||||
// traceable — and so the echo on a 401/403 proves which request the backend answered.
|
||||
var correlationId = Guid.NewGuid().ToString("N");
|
||||
request.Headers.Add("X-Correlation-ID", correlationId);
|
||||
request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId);
|
||||
_logger?.Debug($"BackendApiClient: {request.Method} {request.RequestUri?.AbsolutePath} (corr={correlationId})");
|
||||
|
||||
if (_useBootstrapTokenAuth && !string.IsNullOrEmpty(_bootstrapToken))
|
||||
@@ -464,6 +458,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
request.Headers.Add("X-Device-SerialNumber", _serialNumber);
|
||||
if (!string.IsNullOrEmpty(_agentVersion))
|
||||
request.Headers.Add("X-Agent-Version", _agentVersion);
|
||||
|
||||
return correlationId;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
@@ -473,23 +469,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Thrown when the backend returns 401 or 403, indicating the device is not authorized —
|
||||
/// or, when <see cref="EndpointUnavailable"/> is set, that the 401/403 was a platform-level
|
||||
/// HTML response (stopped/retired app, edge proxy) rather than a backend auth verdict.
|
||||
/// Thrown for a 401 or 403. <see cref="Origin"/> says who answered: the backend (the device is
|
||||
/// not authorized), the platform page of a stopped app (<see cref="EndpointUnavailable"/>), or
|
||||
/// something on the network path (<see cref="IsForeignResponse"/>) — the last two say nothing
|
||||
/// about this device's authorization.
|
||||
/// </summary>
|
||||
public class BackendAuthException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// HTTP status code returned by the backend (401 or 403).
|
||||
/// HTTP status code of the response (401 or 403).
|
||||
/// Used by the DistressReporter to classify the failure type.
|
||||
/// </summary>
|
||||
public int StatusCode { get; }
|
||||
|
||||
/// <summary>Classification and evidence of the response; <see cref="ResponseOrigin.Backend"/> when constructed without one.</summary>
|
||||
public ResponseClassification Classification { get; }
|
||||
|
||||
public ResponseOrigin Origin => Classification.Origin;
|
||||
|
||||
/// <summary>
|
||||
/// True when the 401/403 carried an HTML platform error page — the endpoint itself is
|
||||
/// stopped/retired/proxied and the status says nothing about this device's authorization.
|
||||
/// True only for the platform page of a stopped app: the endpoint itself is down. Counts
|
||||
/// toward the auth shutdown like a backend rejection, without a distress report.
|
||||
/// </summary>
|
||||
public bool EndpointUnavailable { get; }
|
||||
public bool EndpointUnavailable => Origin == ResponseOrigin.PlatformStopped;
|
||||
|
||||
/// <summary>
|
||||
/// True when something other than the backend answered (the platform's "Client Certificate
|
||||
/// Required" page, a proxy page): the device certificate did not reach the backend. Never
|
||||
/// counts as an authentication failure.
|
||||
/// </summary>
|
||||
public bool IsForeignResponse => Origin == ResponseOrigin.Foreign;
|
||||
|
||||
/// <summary>
|
||||
/// Machine-readable code from the backend's JSON error body (see Shared
|
||||
@@ -497,11 +506,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// </summary>
|
||||
public string ErrorCode { get; }
|
||||
|
||||
public BackendAuthException(string message, int statusCode = 0, bool endpointUnavailable = false, string errorCode = null) : base(message)
|
||||
public BackendAuthException(string message, int statusCode = 0, string errorCode = null, ResponseClassification classification = null) : base(message)
|
||||
{
|
||||
StatusCode = statusCode;
|
||||
EndpointUnavailable = endpointUnavailable;
|
||||
ErrorCode = errorCode;
|
||||
Classification = classification ?? ResponseClassification.ForBackend(statusCode);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private long _totalBytesUp;
|
||||
private long _totalBytesDown;
|
||||
private long _totalLatencyMs;
|
||||
private long _okWithoutEchoCount;
|
||||
|
||||
public void RecordRequest(long bytesUp, long bytesDown, long latencyMs, bool failed)
|
||||
{
|
||||
@@ -25,6 +26,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
Interlocked.Increment(ref _failureCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts a 2xx telemetry upload whose response did not echo the request's
|
||||
/// <c>X-Correlation-ID</c>: something other than the backend may have answered (a block page
|
||||
/// served with 200). Observed only — the upload still counts as a success.
|
||||
/// </summary>
|
||||
public void RecordOkWithoutEcho() => Interlocked.Increment(ref _okWithoutEchoCount);
|
||||
|
||||
/// <summary>Cumulative <see cref="RecordOkWithoutEcho"/> count since the process started.</summary>
|
||||
public long OkWithoutEchoCount => Interlocked.Read(ref _okWithoutEchoCount);
|
||||
|
||||
/// <summary>
|
||||
/// Returns an immutable snapshot of all counters for delta calculation.
|
||||
/// </summary>
|
||||
|
||||
@@ -10,6 +10,7 @@ using AutopilotMonitor.Agent.V2.Core.Persistence;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Events;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Signals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.DecisionCore.Classifiers;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
@@ -315,6 +316,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public long IngressPendingSignalCount => _ingress?.PendingSignalCount ?? 0L;
|
||||
|
||||
/// <summary>
|
||||
/// Terminal phase: grants the transport its one upload attempt despite an active
|
||||
/// interception pause (<see cref="TelemetryUploadOrchestrator.GrantTerminalAttempt"/>) and,
|
||||
/// only while that pause holds, wakes the drain loop so the attempt runs inside the caller's
|
||||
/// bounded drain window. Without an interception pause nothing changes. No-op before
|
||||
/// <see cref="Start"/>.
|
||||
/// </summary>
|
||||
public void RequestTerminalUploadAttempt()
|
||||
{
|
||||
var transport = _transport;
|
||||
if (transport == null) return;
|
||||
if (transport.GrantTerminalAttempt())
|
||||
OnImmediateFlushRequested(this, EventArgs.Empty);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Typed read-model surfaces returned by <see cref="IComponentFactory.CreateCollectorHosts"/>
|
||||
/// (ARCH-F4). Set during <see cref="Start"/> step 14; <c>null</c> before Start and when
|
||||
@@ -384,6 +400,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// here cannot deadlock the drain.
|
||||
_transport.BatchPoisoned += OnBatchPoisoned;
|
||||
_transport.UploadBlocked += OnUploadBlocked;
|
||||
_transport.UploadIntercepted += OnUploadIntercepted;
|
||||
|
||||
// The spool reference reaches peripheral collectors (AgentSelfMetricsCollector via
|
||||
// PeriodicCollectorLifecycleHost) as a CreateCollectorHosts parameter at step 14,
|
||||
@@ -1004,11 +1021,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
catch (AggregateException) { /* wrapped OperationCanceledException */ }
|
||||
catch (Exception ex) { _logger.Warning($"EnrollmentOrchestrator: drain loop join failed: {ex.Message}"); }
|
||||
|
||||
// 5) Terminaler Drain — finale Batches.
|
||||
// 5) Terminaler Drain — finale Batches. An active interception pause still lets the
|
||||
// process make its one terminal attempt here, unless DrainSpool already used it.
|
||||
try
|
||||
{
|
||||
using (var timeoutCts = new CancellationTokenSource(_terminalDrainTimeout))
|
||||
{
|
||||
_transport?.GrantTerminalAttempt();
|
||||
_transport?.DrainAllAsync(timeoutCts.Token).GetAwaiter().GetResult();
|
||||
}
|
||||
}
|
||||
@@ -1155,6 +1174,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the first intercepted upload of the process as <c>telemetry_upload_blocked</c>
|
||||
/// with the interception evidence. Gated once per process by the transport, independently
|
||||
/// of <see cref="OnUploadBlocked"/>. Queued behind the intercepted batch like every other
|
||||
/// item: it reaches the backend only if the path clears in this run, and its arrival time
|
||||
/// marks the end of the interception.
|
||||
/// </summary>
|
||||
private void OnUploadIntercepted(object? sender, InterceptionReport report)
|
||||
{
|
||||
var interception = report.Interception;
|
||||
_logger.Warning(
|
||||
$"EnrollmentOrchestrator: telemetry upload intercepted on the network path ({interception.ForeignReason}, " +
|
||||
$"http {interception.StatusCode}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " +
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\") — uploads pause with backoff; " +
|
||||
"the agent keeps observing.");
|
||||
|
||||
var ingress = _ingress;
|
||||
if (ingress == null) return;
|
||||
try
|
||||
{
|
||||
new InformationalEventPost(ingress, _clock).Emit(new Shared.Models.EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.TelemetryUploadBlocked,
|
||||
Severity = Shared.Models.EventSeverity.Warning,
|
||||
Source = "TelemetryUploadOrchestrator",
|
||||
Phase = Shared.Models.EnrollmentPhase.Unknown,
|
||||
Message = "Telemetry upload intercepted on the network path: the answer did not come from the "
|
||||
+ "backend, so the device certificate did not reach it. Uploads pause with backoff and "
|
||||
+ "resume once the path clears; this marker is queued behind the intercepted batch.",
|
||||
Timestamp = _clock.UtcNow,
|
||||
ImmediateUpload = true,
|
||||
Data = new Dictionary<string, object>(StringComparer.Ordinal)
|
||||
{
|
||||
["reason"] = interception.ForeignReason ?? ResponseClassification.ForeignResponse,
|
||||
["operation"] = "agent/telemetry",
|
||||
["statusCode"] = interception.StatusCode,
|
||||
["pageTitle"] = interception.PageTitle ?? string.Empty,
|
||||
["serverCertIssuer"] = interception.ServerCertIssuer ?? string.Empty,
|
||||
["firstBlockedAtUtc"] = report.FirstBlockedAtUtc.ToString("o", CultureInfo.InvariantCulture),
|
||||
},
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error("EnrollmentOrchestrator: failed to post telemetry_upload_blocked (interception).", ex);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Test helpers
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -5,6 +5,7 @@ using System.Net.Http;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
@@ -40,10 +41,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// (<see cref="HttpClient"/>'s <c>_handler</c>-Feld ist internal und in net48
|
||||
/// reflection-unzuverlässig).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <paramref name="serverCertificates"/>, when supplied, observes the server certificate of
|
||||
/// every new connection without changing the validation verdict.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static HttpClientHandler CreateHandler(
|
||||
ICertificateResolver resolver,
|
||||
AgentLogger logger)
|
||||
AgentLogger logger,
|
||||
ServerCertificateObserver? serverCertificates = null)
|
||||
{
|
||||
if (resolver == null) throw new ArgumentNullException(nameof(resolver));
|
||||
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
||||
@@ -62,6 +68,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate,
|
||||
};
|
||||
handler.ClientCertificates.Add(cert);
|
||||
serverCertificates?.Attach(handler);
|
||||
|
||||
logger.Info(
|
||||
$"MtlsHttpClientFactory: handler ready (cert thumbprint={cert.Thumbprint}).");
|
||||
@@ -85,9 +92,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
ICertificateResolver resolver,
|
||||
AgentLogger logger,
|
||||
TimeSpan? timeout = null,
|
||||
NetworkMetrics? metrics = null)
|
||||
NetworkMetrics? metrics = null,
|
||||
ServerCertificateObserver? serverCertificates = null)
|
||||
{
|
||||
var innerHandler = CreateHandler(resolver, logger);
|
||||
var innerHandler = CreateHandler(resolver, logger, serverCertificates);
|
||||
HttpMessageHandler pipelineHead = innerHandler;
|
||||
if (metrics != null)
|
||||
{
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
@@ -29,6 +31,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// twice about a certificate it already rejected.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only a 401/403 the backend authored counts (<see cref="ResponseOriginClassifier"/>). One
|
||||
/// from something on the network path (TLS inspection, a proxy page) never counts and never
|
||||
/// touches the streak — it says nothing about this device's authorization — and is reported
|
||||
/// once per process as <see cref="DistressErrorType.ClientCertificateNotReceived"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Thread-safety: consecutive-count uses <see cref="Interlocked"/>; the first-failure timestamp
|
||||
/// is protected by a single monitor lock. <see cref="ThresholdExceeded"/> is raised outside the
|
||||
/// lock to prevent handler re-entrancy deadlocks.
|
||||
@@ -47,6 +55,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private DateTime? _firstFailureUtc;
|
||||
private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot
|
||||
private int _distressDispatched; // 0/1 per failure streak — reset by RecordSuccess
|
||||
private int _foreignDistressDispatched; // 0/1 per process — never reset
|
||||
private Task _foreignDistressSend; // the per-process ClientCertificateNotReceived send; null until dispatched
|
||||
|
||||
public AuthFailureTracker(
|
||||
int maxFailures,
|
||||
@@ -90,16 +100,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// dispatch further distress signals. Emits <see cref="ThresholdExceeded"/> the first
|
||||
/// time either ceiling is crossed; subsequent calls after termination is armed are no-ops.
|
||||
/// </summary>
|
||||
/// <param name="endpointUnavailable">
|
||||
/// True when the 401/403 was a platform-level HTML response (stopped/retired app, edge
|
||||
/// proxy — see <see cref="BackendAuthException.EndpointUnavailable"/>). Suppresses the
|
||||
/// distress dispatch: the report would go to the very endpoint that just answered with a
|
||||
/// platform error page, and its DeviceNotRegistered classification would be wrong anyway.
|
||||
/// Counting and the shutdown thresholds are unaffected. Should a later failure in the
|
||||
/// same streak come from a live backend, that one still dispatches the streak's report.
|
||||
/// <param name="classification">
|
||||
/// Who answered (<see cref="ResponseOriginClassifier"/>); null = the backend.
|
||||
/// <see cref="ResponseOrigin.PlatformStopped"/> counts but suppresses the distress dispatch:
|
||||
/// the report would go to the very endpoint that just answered with its stopped page, and
|
||||
/// DeviceNotRegistered would be wrong anyway; a later failure in the same streak from a live
|
||||
/// backend still dispatches the streak's report. <see cref="ResponseOrigin.Foreign"/> never
|
||||
/// counts — see <see cref="RecordForeignResponse"/>.
|
||||
/// </param>
|
||||
public void RecordFailure(int statusCode, string operation, bool endpointUnavailable = false)
|
||||
public void RecordFailure(int statusCode, string operation, ResponseClassification classification = null)
|
||||
{
|
||||
if (classification?.IsForeign == true)
|
||||
{
|
||||
RecordForeignResponse(statusCode, operation, classification);
|
||||
return;
|
||||
}
|
||||
|
||||
if (Volatile.Read(ref _thresholdFired) == 1) return;
|
||||
|
||||
var count = Interlocked.Increment(ref _consecutiveFailures);
|
||||
@@ -114,9 +130,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
|
||||
// V1 parity — only one distress report per failure streak. Repeat hits use the local
|
||||
// log only; the backend already knows from the first report that the device is
|
||||
// unauthorized. Endpoint-unavailable failures never dispatch (nothing to send to) but
|
||||
// unauthorized. Stopped-endpoint failures never dispatch (nothing to send to) but
|
||||
// also never consume the streak's report slot.
|
||||
if (endpointUnavailable)
|
||||
if (classification?.Origin == ResponseOrigin.PlatformStopped)
|
||||
{
|
||||
if (_distressReporter != null && Volatile.Read(ref _distressDispatched) == 0)
|
||||
{
|
||||
@@ -180,6 +196,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
_logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 that something on the network path authored. Never counts and never touches
|
||||
/// the streak — an interception says nothing about this device's authorization, and five
|
||||
/// of them must not shut a healthy agent down. The first one per process dispatches
|
||||
/// <see cref="DistressErrorType.ClientCertificateNotReceived"/>: the distress channel needs
|
||||
/// no client certificate, so it gets through where the request did not. Message order is
|
||||
/// operation, status, sub-reason, page title, server-certificate issuer (the reporter cuts
|
||||
/// at 256 characters).
|
||||
/// </summary>
|
||||
private void RecordForeignResponse(int statusCode, string operation, ResponseClassification classification)
|
||||
{
|
||||
var finding =
|
||||
$"{operation} http {statusCode} {classification.ForeignReason}; " +
|
||||
$"page \"{classification.PageTitle ?? "(none)"}\"; " +
|
||||
$"server cert issuer \"{classification.ServerCertIssuer ?? "(unknown)"}\"";
|
||||
_logger.Info($"AuthFailureTracker: answer did not come from the backend — not an authentication failure, not counted ({finding}).");
|
||||
|
||||
if (_distressReporter == null || Interlocked.CompareExchange(ref _foreignDistressDispatched, 1, 0) != 0)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
// Fire-and-forget like every other dispatch here, but kept: a caller about to end
|
||||
// the process waits for it (WaitForPendingForeignDistress).
|
||||
Volatile.Write(ref _foreignDistressSend, _distressReporter.TrySendAsync(
|
||||
DistressErrorType.ClientCertificateNotReceived, finding, httpStatusCode: statusCode));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"AuthFailureTracker: distress dispatch threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives this process's <see cref="DistressErrorType.ClientCertificateNotReceived"/> send up
|
||||
/// to <paramref name="bound"/> to leave the machine. The dispatch is fire-and-forget; a caller
|
||||
/// about to end the process (a start whose registration failed exits or cleans up right away)
|
||||
/// waits here so the report is not abandoned mid-flight. Returns true when nothing is pending
|
||||
/// or the send finished within the bound. Never throws.
|
||||
/// </summary>
|
||||
public bool WaitForPendingForeignDistress(TimeSpan bound)
|
||||
{
|
||||
var send = Volatile.Read(ref _foreignDistressSend);
|
||||
if (send == null || send.IsCompleted) return true;
|
||||
try
|
||||
{
|
||||
return send.Wait(bound);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return true; // faulted or cancelled: finished, nothing left to wait for
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class AuthFailureThresholdEventArgs : EventArgs
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
{
|
||||
@@ -27,6 +29,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
private readonly string _registrationFailurePath;
|
||||
private readonly object _lockObject = new object();
|
||||
|
||||
// Agent-local key in registration-failure.json; RegistrationFailureSummary has no such member.
|
||||
private const string LocalNetworkFingerprintKey = "NetworkFingerprint";
|
||||
|
||||
public SessionIdPersistence(string dataDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dataDirectory))
|
||||
@@ -170,22 +175,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
|
||||
/// <summary>
|
||||
/// Folds one failed registration run into <c>registration-failure.json</c>: the run
|
||||
/// counter grows, the first failure time is kept, everything else describes the latest
|
||||
/// run. A run that never registers exits and leaves nothing else behind — the emergency
|
||||
/// break reads this file so the backend learns why the session never appeared.
|
||||
/// Best-effort; never throws.
|
||||
/// counters grow (an intercepted run — <c>InterceptedRuns</c> > 0 on
|
||||
/// <paramref name="latestRun"/> — also counts toward
|
||||
/// <see cref="RegistrationFailureSummary.InterceptedRuns"/>), the first failure time is kept,
|
||||
/// everything else describes the latest run. A run that never registers exits and leaves
|
||||
/// nothing else behind — the emergency break reads this file so the backend learns why the
|
||||
/// session never appeared. <paramref name="networkFingerprint"/> (the latest run's network)
|
||||
/// is stored beside the summary, never in it, so it cannot reach the wire. Written through to
|
||||
/// disk (L.12): the process exits right after and the next boot decides on the record.
|
||||
/// Returns the merged summary, null when it could not be written. Never throws.
|
||||
/// </summary>
|
||||
public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null)
|
||||
public RegistrationFailureSummary RecordRegistrationFailure(
|
||||
RegistrationFailureSummary latestRun, AgentLogger logger = null, string networkFingerprint = null)
|
||||
{
|
||||
if (latestRun == null) return;
|
||||
if (latestRun == null) return null;
|
||||
lock (_lockObject)
|
||||
{
|
||||
try
|
||||
{
|
||||
var previous = LoadRegistrationFailureInternal();
|
||||
var previous = LoadRegistrationFailureInternal()?.Summary;
|
||||
var merged = new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = (previous?.FailedRuns ?? 0) + 1,
|
||||
InterceptedRuns = (previous?.InterceptedRuns ?? 0) + (latestRun.InterceptedRuns > 0 ? 1 : 0),
|
||||
FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc,
|
||||
LastFailedAtUtc = latestRun.LastFailedAtUtc,
|
||||
Outcome = latestRun.Outcome,
|
||||
@@ -195,18 +207,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
AttemptWindowSeconds = latestRun.AttemptWindowSeconds,
|
||||
ConfigFetchOutcome = latestRun.ConfigFetchOutcome,
|
||||
};
|
||||
File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged));
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}.");
|
||||
WriteRegistrationFailure(merged, networkFingerprint);
|
||||
logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns} ({merged.InterceptedRuns} intercepted).");
|
||||
return merged;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The failed-registration record since the last success, or <c>null</c> (none, unreadable).</summary>
|
||||
/// <summary>
|
||||
/// The failed-registration record since the last success, or <c>null</c> (none, unreadable).
|
||||
/// The summary only — what the emergency break may send.
|
||||
/// </summary>
|
||||
public RegistrationFailureSummary LoadRegistrationFailure()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal()?.Summary;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The failed-registration record plus the agent-local network fingerprint of its latest
|
||||
/// run, or <c>null</c> (none, unreadable). For the next successful start's comparison only.
|
||||
/// </summary>
|
||||
public RegistrationFailureRecord LoadRegistrationFailureRecord()
|
||||
{
|
||||
lock (_lockObject) return LoadRegistrationFailureInternal();
|
||||
}
|
||||
@@ -252,13 +278,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
|
||||
private RegistrationFailureSummary LoadRegistrationFailureInternal()
|
||||
private RegistrationFailureRecord LoadRegistrationFailureInternal()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!File.Exists(_registrationFailurePath)) return null;
|
||||
var record = JsonConvert.DeserializeObject<RegistrationFailureSummary>(File.ReadAllText(_registrationFailurePath));
|
||||
return record != null && record.FailedRuns > 0 ? record : null;
|
||||
var json = JObject.Parse(File.ReadAllText(_registrationFailurePath));
|
||||
var summary = json.ToObject<RegistrationFailureSummary>();
|
||||
return summary != null && summary.FailedRuns > 0
|
||||
? new RegistrationFailureRecord(summary, (string)json[LocalNetworkFingerprintKey])
|
||||
: null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -267,6 +296,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The summary's own properties plus one agent-local key: the fingerprint sits next to the
|
||||
/// wire type, never inside it (the summary deserializes without it).
|
||||
/// </summary>
|
||||
private void WriteRegistrationFailure(RegistrationFailureSummary summary, string networkFingerprint)
|
||||
{
|
||||
var json = JObject.FromObject(summary);
|
||||
if (!string.IsNullOrEmpty(networkFingerprint))
|
||||
json[LocalNetworkFingerprintKey] = networkFingerprint;
|
||||
|
||||
var bytes = Encoding.UTF8.GetBytes(json.ToString(Formatting.None));
|
||||
using (var stream = new FileStream(
|
||||
_registrationFailurePath, FileMode.Create, FileAccess.Write, FileShare.None,
|
||||
bufferSize: 4096, options: FileOptions.WriteThrough))
|
||||
{
|
||||
stream.Write(bytes, 0, bytes.Length);
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDelete(string path, AgentLogger logger)
|
||||
{
|
||||
try { if (File.Exists(path)) File.Delete(path); }
|
||||
@@ -305,4 +354,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <c>registration-failure.json</c> as the agent keeps it: the <see cref="RegistrationFailureSummary"/>
|
||||
/// the emergency break may send, plus the network fingerprint of the latest failed run, which
|
||||
/// stays on the device — only the next successful start's comparison (changed or not) is reported.
|
||||
/// </summary>
|
||||
public sealed class RegistrationFailureRecord
|
||||
{
|
||||
public RegistrationFailureRecord(RegistrationFailureSummary summary, string networkFingerprint)
|
||||
{
|
||||
Summary = summary ?? throw new ArgumentNullException(nameof(summary));
|
||||
NetworkFingerprint = networkFingerprint;
|
||||
}
|
||||
|
||||
public RegistrationFailureSummary Summary { get; }
|
||||
|
||||
/// <summary>Fingerprint of the latest failed run's network; null when it was unknown.</summary>
|
||||
public string NetworkFingerprint { get; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,7 +33,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <item>On <see cref="BackendAuthException"/> (401/403) → report to the
|
||||
/// <see cref="AuthFailureTracker"/> (which fires the first-failure distress + may trip
|
||||
/// the shutdown threshold) and return <see cref="SessionRegistrationOutcome.AuthFailed"/>
|
||||
/// without retrying — the backend has definitively rejected the device cert.</item>
|
||||
/// without retrying — the backend has definitively rejected the device cert. A 401/403
|
||||
/// something on the network path answered ends the same way, marked
|
||||
/// <see cref="SessionRegistrationResult.IsIntercepted"/>.</item>
|
||||
/// <item>Exception to the above — a 403 or 409 carrying error code
|
||||
/// <c>session_owner_mismatch</c> (SESSION-OWNER-BINDING): the persisted SessionId names a
|
||||
/// session that is not this device's — bound to another device identity (403, typically an
|
||||
@@ -135,8 +137,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
logger.Error($"Session owner mismatch reported by backend but session rotation failed: {rotateEx.Message}", rotateEx);
|
||||
if (ex is BackendAuthException authEx)
|
||||
{
|
||||
authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.EndpointUnavailable);
|
||||
return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message);
|
||||
authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.Classification);
|
||||
return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message, authEx.IsForeignResponse);
|
||||
}
|
||||
return SessionRegistrationResult.Failed(ex.Message);
|
||||
}
|
||||
@@ -154,12 +156,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
}
|
||||
catch (BackendAuthException ex)
|
||||
{
|
||||
logger.Error($"Session registration authentication failed ({ex.StatusCode}): {ex.Message}");
|
||||
logger.Error(ex.IsForeignResponse
|
||||
? $"Session registration intercepted on the network path ({ex.StatusCode}): {ex.Message}"
|
||||
: $"Session registration authentication failed ({ex.StatusCode}): {ex.Message}");
|
||||
// Feed the central auth-failure tracker so the first-failure distress dispatches
|
||||
// and the shutdown threshold advances. V1 parity: no retry on auth-failure —
|
||||
// backend has definitively rejected the cert/token.
|
||||
authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.EndpointUnavailable);
|
||||
return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message);
|
||||
// backend has definitively rejected the cert/token. A 401/403 from something on
|
||||
// the network path is not counted; the tracker reports it once per process as
|
||||
// ClientCertificateNotReceived, and the run ends AuthFailed marked intercepted
|
||||
// (the caller's buffer decides between a further boot and cleaning up).
|
||||
authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.Classification);
|
||||
return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message, ex.IsForeignResponse);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -274,7 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// <summary>Backend responded with <c>Success=true</c>. The session row is live.</summary>
|
||||
Succeeded = 0,
|
||||
|
||||
/// <summary>401/403 from the backend — cert/token rejected. No retry was attempted.</summary>
|
||||
/// <summary>
|
||||
/// 401/403 — cert/token rejected by the backend, or answered by something on the network
|
||||
/// path (<see cref="SessionRegistrationResult.IsIntercepted"/>). No retry was attempted.
|
||||
/// </summary>
|
||||
AuthFailed = 1,
|
||||
|
||||
/// <summary>All attempts failed with non-auth errors (network, 5xx, malformed response).</summary>
|
||||
@@ -299,25 +309,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
/// </summary>
|
||||
public string RotatedFromSessionId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// True when the run ended on a 401/403 that something on the network path answered (TLS
|
||||
/// inspection, a proxy page — <see cref="BackendAuthException.IsForeignResponse"/>), not
|
||||
/// the backend. Such starts count toward the intercepted-registration buffer.
|
||||
/// </summary>
|
||||
public bool IsIntercepted { get; }
|
||||
|
||||
private SessionRegistrationResult(
|
||||
SessionRegistrationOutcome outcome,
|
||||
RegisterSessionResponse response,
|
||||
int httpStatusCode,
|
||||
string errorMessage,
|
||||
string rotatedFromSessionId = null)
|
||||
string rotatedFromSessionId = null,
|
||||
bool isIntercepted = false)
|
||||
{
|
||||
Outcome = outcome;
|
||||
Response = response;
|
||||
HttpStatusCode = httpStatusCode;
|
||||
ErrorMessage = errorMessage;
|
||||
RotatedFromSessionId = rotatedFromSessionId;
|
||||
IsIntercepted = isIntercepted;
|
||||
}
|
||||
|
||||
public static SessionRegistrationResult Succeeded(RegisterSessionResponse response, string rotatedFromSessionId = null)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.Succeeded, response, 200, null, rotatedFromSessionId);
|
||||
|
||||
public static SessionRegistrationResult AuthFailed(int statusCode, string message)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message);
|
||||
public static SessionRegistrationResult AuthFailed(int statusCode, string message, bool intercepted = false)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message, isIntercepted: intercepted);
|
||||
|
||||
public static SessionRegistrationResult Failed(string message)
|
||||
=> new SessionRegistrationResult(SessionRegistrationOutcome.Failed, null, 0, message);
|
||||
|
||||
@@ -1343,6 +1343,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
return;
|
||||
}
|
||||
|
||||
// An interception pause (uploads answered by something on the network path) would
|
||||
// otherwise skip this drain entirely: the terminal phase gets exactly one attempt, then
|
||||
// termination goes on as usual — whatever did not go up is lost, as with any session.
|
||||
try { _drain.RequestTerminalUploadAttempt(); }
|
||||
catch (Exception ex) { _logger.Warning($"EnrollmentTerminationHandler: terminal upload attempt request threw: {ex.Message}"); }
|
||||
|
||||
// Block briefly so pending events can land before the next destructive step
|
||||
// (shutdown.exe, self-destruct). Two phases share the same bounded budget:
|
||||
//
|
||||
|
||||
@@ -54,5 +54,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// <see cref="CanObserveSpool"/> is true.
|
||||
/// </summary>
|
||||
int SpoolPendingItemCount { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Called once at the start of every <c>DrainSpool</c>: lets the transport make its one
|
||||
/// terminal upload attempt even while an interception pause holds uploads back (production:
|
||||
/// <see cref="Orchestration.EnrollmentOrchestrator.RequestTerminalUploadAttempt"/>).
|
||||
/// Without an interception pause it changes nothing.
|
||||
/// </summary>
|
||||
void RequestTerminalUploadAttempt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,5 +31,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
public bool CanObserveSpool => true;
|
||||
|
||||
public int SpoolPendingItemCount => _orchestrator.PendingItemCount;
|
||||
|
||||
public void RequestTerminalUploadAttempt() => _orchestrator.RequestTerminalUploadAttempt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>Who answered a 401/403 — see <see cref="ResponseOriginClassifier"/>.</summary>
|
||||
public enum ResponseOrigin
|
||||
{
|
||||
/// <summary>The backend: the response echoes the request's <c>X-Correlation-ID</c>. A real authorization verdict.</summary>
|
||||
Backend = 0,
|
||||
|
||||
/// <summary>
|
||||
/// The platform page of a stopped web app (no echo, "This web app is stopped"). The endpoint
|
||||
/// itself is down; counts toward the auth shutdown so a dead endpoint does not keep the agent alive.
|
||||
/// </summary>
|
||||
PlatformStopped = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Anything else without the echo: the platform's "Client Certificate Required" page (the device
|
||||
/// certificate did not reach the backend) or a page of a proxy on the network path. Never an
|
||||
/// authorization verdict.
|
||||
/// </summary>
|
||||
Foreign = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A classified 401/403 plus the evidence the agent reports about it (log, distress,
|
||||
/// <c>telemetry_upload_blocked</c>). Built by <see cref="ResponseOriginClassifier.Classify"/>.
|
||||
/// </summary>
|
||||
public sealed class ResponseClassification
|
||||
{
|
||||
/// <summary>Foreign sub-reason: the platform answered "Client Certificate Required".</summary>
|
||||
public const string ClientCertificateRequired = "client_certificate_required";
|
||||
|
||||
/// <summary>Foreign sub-reason: any other answer that did not come from the backend (e.g. a proxy page).</summary>
|
||||
public const string ForeignResponse = "foreign_response";
|
||||
|
||||
private ResponseClassification(
|
||||
ResponseOrigin origin,
|
||||
int statusCode,
|
||||
string? foreignReason,
|
||||
string? pageTitle,
|
||||
bool isHtml,
|
||||
string? serverCertIssuer)
|
||||
{
|
||||
Origin = origin;
|
||||
StatusCode = statusCode;
|
||||
ForeignReason = foreignReason;
|
||||
PageTitle = pageTitle;
|
||||
IsHtml = isHtml;
|
||||
ServerCertIssuer = serverCertIssuer;
|
||||
}
|
||||
|
||||
public ResponseOrigin Origin { get; }
|
||||
|
||||
/// <summary>HTTP status of the classified response (401 or 403).</summary>
|
||||
public int StatusCode { get; }
|
||||
|
||||
/// <summary><see cref="ClientCertificateRequired"/> or <see cref="ForeignResponse"/> for <see cref="ResponseOrigin.Foreign"/>; null otherwise.</summary>
|
||||
public string? ForeignReason { get; }
|
||||
|
||||
/// <summary>HTML <c><title></c> of the page (at most 80 characters), null when absent.</summary>
|
||||
public string? PageTitle { get; }
|
||||
|
||||
/// <summary>True when the body is an HTML page (declared <c>text/html</c> or sniffed).</summary>
|
||||
public bool IsHtml { get; }
|
||||
|
||||
/// <summary>Issuer of the server certificate of the last validated backend connection (<see cref="ServerCertificateObserver"/>); null when unknown.</summary>
|
||||
public string? ServerCertIssuer { get; }
|
||||
|
||||
public bool IsForeign => Origin == ResponseOrigin.Foreign;
|
||||
|
||||
internal static ResponseClassification ForBackend(int statusCode) =>
|
||||
new ResponseClassification(ResponseOrigin.Backend, statusCode, null, null, isHtml: false, serverCertIssuer: null);
|
||||
|
||||
internal static ResponseClassification ForPlatformStopped(int statusCode, string? pageTitle, bool isHtml, string? serverCertIssuer) =>
|
||||
new ResponseClassification(ResponseOrigin.PlatformStopped, statusCode, null, pageTitle, isHtml, serverCertIssuer);
|
||||
|
||||
internal static ResponseClassification ForForeign(int statusCode, string foreignReason, string? pageTitle, bool isHtml, string? serverCertIssuer) =>
|
||||
new ResponseClassification(ResponseOrigin.Foreign, statusCode, foreignReason, pageTitle, isHtml, serverCertIssuer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Decides whether a 401/403 came from the backend at all. The backend's correlation
|
||||
/// middleware echoes the request's <c>X-Correlation-ID</c> on every response it writes (the
|
||||
/// agent sends a fresh id per request); the Azure platform's own pages and a proxy's pages never
|
||||
/// carry it. The echo is therefore the only proof of origin — body shape, content type and
|
||||
/// title are evidence, not proof. Shared by every agent call site that can receive a 401/403.
|
||||
/// </summary>
|
||||
public static class ResponseOriginClassifier
|
||||
{
|
||||
/// <summary>Request header the backend echoes on every response it writes.</summary>
|
||||
public const string CorrelationIdHeader = "X-Correlation-ID";
|
||||
|
||||
/// <summary>Only the head of a body is inspected; platform and proxy pages are a few KB.</summary>
|
||||
public const int MaxBodyChars = 16 * 1024;
|
||||
|
||||
private const int MaxTitleLength = 80;
|
||||
|
||||
// Text of the platform page of a stopped web app ("Error 403 - This web app is stopped.").
|
||||
private const string StoppedAppMarker = "This web app is stopped";
|
||||
|
||||
// Text of the platform page served when no client certificate arrived
|
||||
// ("Error 403 - Forbidden: Client Certificate Required.").
|
||||
private const string ClientCertificateRequiredMarker = "Client Certificate Required";
|
||||
|
||||
/// <summary>
|
||||
/// Classifies a 401/403. <paramref name="sentCorrelationId"/> is the id this request carried;
|
||||
/// <paramref name="bodyPrefix"/> is the head of the body (<see cref="ReadBodyPrefixAsync"/>);
|
||||
/// <paramref name="serverCertIssuer"/> is passed through as evidence. An echo always wins:
|
||||
/// a backend page is the backend's verdict whatever it looks like.
|
||||
/// </summary>
|
||||
public static ResponseClassification Classify(
|
||||
HttpResponseMessage response,
|
||||
string? sentCorrelationId,
|
||||
string? bodyPrefix,
|
||||
string? serverCertIssuer = null)
|
||||
{
|
||||
if (response == null) throw new ArgumentNullException(nameof(response));
|
||||
|
||||
var statusCode = (int)response.StatusCode;
|
||||
if (EchoesCorrelationId(response, sentCorrelationId))
|
||||
return ResponseClassification.ForBackend(statusCode);
|
||||
|
||||
var body = Cap(bodyPrefix);
|
||||
var isHtml = LooksLikeHtml(response, body);
|
||||
var title = TryExtractHtmlTitle(body);
|
||||
|
||||
if (Contains(body, StoppedAppMarker))
|
||||
return ResponseClassification.ForPlatformStopped(statusCode, title, isHtml, serverCertIssuer);
|
||||
|
||||
var reason = Contains(body, ClientCertificateRequiredMarker)
|
||||
? ResponseClassification.ClientCertificateRequired
|
||||
: ResponseClassification.ForeignResponse;
|
||||
return ResponseClassification.ForForeign(statusCode, reason, title, isHtml, serverCertIssuer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The first <see cref="MaxBodyChars"/> characters of the body, null when there is none or it
|
||||
/// cannot be read. Never throws — the body is evidence only.
|
||||
/// </summary>
|
||||
public static async Task<string?> ReadBodyPrefixAsync(HttpResponseMessage response)
|
||||
{
|
||||
var content = response?.Content;
|
||||
if (content == null) return null;
|
||||
try
|
||||
{
|
||||
return Cap(await content.ReadAsStringAsync().ConfigureAwait(false));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when <paramref name="response"/> carries <paramref name="sentCorrelationId"/> in
|
||||
/// <see cref="CorrelationIdHeader"/> — the backend wrote it. Also used on 2xx answers, where
|
||||
/// a missing echo is only counted (<c>BackendTelemetryUploader</c>), never acted on.
|
||||
/// </summary>
|
||||
public static bool EchoesCorrelationId(HttpResponseMessage response, string? sentCorrelationId)
|
||||
{
|
||||
if (response == null) throw new ArgumentNullException(nameof(response));
|
||||
if (string.IsNullOrEmpty(sentCorrelationId)) return false;
|
||||
if (!response.Headers.TryGetValues(CorrelationIdHeader, out var values)) return false;
|
||||
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (string.Equals(value?.Trim(), sentCorrelationId, StringComparison.Ordinal)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True when the body is an HTML page rather than a JSON/plain-text reply. Checks the declared
|
||||
/// Content-Type first, then sniffs the body prefix so a page served with a generic
|
||||
/// Content-Type is still recognized.
|
||||
/// </summary>
|
||||
private static bool LooksLikeHtml(HttpResponseMessage response, string? body)
|
||||
{
|
||||
var mediaType = response.Content?.Headers?.ContentType?.MediaType;
|
||||
if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(body))
|
||||
return false;
|
||||
|
||||
var head = body!.TrimStart();
|
||||
return head.StartsWith("<!DOCTYPE", StringComparison.OrdinalIgnoreCase) ||
|
||||
head.StartsWith("<html", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so logs and
|
||||
/// reports name the page without dumping markup. Null when absent, empty or the regex times out.
|
||||
/// </summary>
|
||||
private static string? TryExtractHtmlTitle(string? body)
|
||||
{
|
||||
if (string.IsNullOrEmpty(body)) return null;
|
||||
try
|
||||
{
|
||||
var match = Regex.Match(
|
||||
body, @"<title[^>]*>\s*(.*?)\s*</title>",
|
||||
RegexOptions.IgnoreCase | RegexOptions.Singleline,
|
||||
TimeSpan.FromMilliseconds(250));
|
||||
if (!match.Success) return null;
|
||||
|
||||
var title = match.Groups[1].Value;
|
||||
return title.Length == 0 ? null : (title.Length > MaxTitleLength ? title.Substring(0, MaxTitleLength) : title);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Contains(string? body, string marker) =>
|
||||
body != null && body.IndexOf(marker, StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
private static string? Cap(string? body) =>
|
||||
body != null && body.Length > MaxBodyChars ? body.Substring(0, MaxBodyChars) : body;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using System.Threading;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Remembers the issuer of the server certificate of the last TLS connection a backend
|
||||
/// <see cref="HttpClientHandler"/> validated. Evidence for a 401/403 that did not come from the
|
||||
/// backend: a TLS-inspecting proxy presents a certificate from its own CA, which names the
|
||||
/// product; a Microsoft issuer means the device certificate got lost on the device itself.
|
||||
/// <para>
|
||||
/// Observation only: the callback returns exactly <c>sslPolicyErrors == None</c>, the verdict
|
||||
/// .NET Framework applies to a client connection without a callback, so validation is never
|
||||
/// weakened. The callback runs during the handshake of a new connection; a reused keep-alive
|
||||
/// connection is not validated again.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ServerCertificateObserver
|
||||
{
|
||||
/// <summary>Longest issuer kept. The CN leads the distinguished name and survives the cut.</summary>
|
||||
internal const int MaxIssuerLength = 128;
|
||||
|
||||
private string? _lastIssuer;
|
||||
|
||||
/// <summary>Issuer of the last validated server certificate, null before the first handshake.</summary>
|
||||
public string? LastIssuer => Volatile.Read(ref _lastIssuer);
|
||||
|
||||
/// <summary>Installs the observing validation callback on <paramref name="handler"/>.</summary>
|
||||
public void Attach(HttpClientHandler handler)
|
||||
{
|
||||
if (handler == null) throw new ArgumentNullException(nameof(handler));
|
||||
handler.ServerCertificateCustomValidationCallback =
|
||||
(_, certificate, _, sslPolicyErrors) => Validate(certificate, sslPolicyErrors);
|
||||
}
|
||||
|
||||
/// <summary>Records the issuer and returns the unchanged default verdict.</summary>
|
||||
internal bool Validate(X509Certificate2? certificate, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
Record(certificate);
|
||||
return sslPolicyErrors == SslPolicyErrors.None;
|
||||
}
|
||||
|
||||
private void Record(X509Certificate2? certificate)
|
||||
{
|
||||
if (certificate == null) return;
|
||||
try
|
||||
{
|
||||
var issuer = certificate.Issuer;
|
||||
if (string.IsNullOrEmpty(issuer)) return;
|
||||
Volatile.Write(ref _lastIssuer, issuer.Length > MaxIssuerLength ? issuer.Substring(0, MaxIssuerLength) : issuer);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Observation must never fail a handshake.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+48
-22
@@ -9,6 +9,7 @@ using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -49,7 +50,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
private readonly string? _agentVersion;
|
||||
private readonly AuthFailureTracker? _authFailureTracker;
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
private readonly ServerCertificateObserver? _serverCertificates;
|
||||
private readonly NetworkMetrics? _networkMetrics;
|
||||
|
||||
/// <param name="serverCertificates">
|
||||
/// Observer installed on the <paramref name="httpClient"/>'s handler; its issuer becomes
|
||||
/// evidence on an intercepted upload. Null = no evidence.
|
||||
/// </param>
|
||||
/// <param name="networkMetrics">
|
||||
/// Counts 2xx answers without the correlation echo (<c>agent_metrics_snapshot</c>). Null = not counted.
|
||||
/// </param>
|
||||
public BackendTelemetryUploader(
|
||||
HttpClient httpClient,
|
||||
string baseUrl,
|
||||
@@ -60,7 +70,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
string? bootstrapToken = null,
|
||||
string? agentVersion = null,
|
||||
AuthFailureTracker? authFailureTracker = null,
|
||||
Logging.AgentLogger? logger = null)
|
||||
Logging.AgentLogger? logger = null,
|
||||
ServerCertificateObserver? serverCertificates = null,
|
||||
NetworkMetrics? networkMetrics = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -75,6 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
_agentVersion = agentVersion;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
_logger = logger;
|
||||
_serverCertificates = serverCertificates;
|
||||
_networkMetrics = networkMetrics;
|
||||
}
|
||||
|
||||
public async Task<UploadResult> UploadBatchAsync(
|
||||
@@ -117,7 +131,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
||||
request.Content.Headers.ContentEncoding.Add("gzip");
|
||||
AddSecurityHeaders(request);
|
||||
request.Headers.Add("X-Correlation-ID", correlationId);
|
||||
request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId);
|
||||
|
||||
// P14: device-clock send time, one value per request. Lets the backend separate
|
||||
// spool delay (SentAt − OccurredUtc, both device frame) from device-vs-server
|
||||
@@ -154,7 +168,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
using (response)
|
||||
{
|
||||
var result = await MapResponseAsync(response).ConfigureAwait(false);
|
||||
var result = await MapResponseAsync(response, correlationId).ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
if (result.Success)
|
||||
@@ -165,6 +179,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else if (result.Interception != null)
|
||||
{
|
||||
// The whole page finding instead of a body excerpt: the 200-character
|
||||
// excerpt used to end before the "Client Certificate Required" text.
|
||||
var interception = result.Interception;
|
||||
_logger?.Warning(
|
||||
$"BackendTelemetryUploader: ingest INTERCEPTED — the answer did not come from the backend " +
|
||||
$"(corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={interception.StatusCode}, " +
|
||||
$"reason={interception.ForeignReason}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " +
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\").");
|
||||
}
|
||||
else if (result.IsSessionGone)
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
@@ -179,7 +204,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response)
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response, string correlationId)
|
||||
{
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
@@ -189,22 +214,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// backend hiccup does not eventually shut the agent down.
|
||||
_authFailureTracker?.RecordSuccess();
|
||||
|
||||
// A 2xx without the backend's correlation echo may be a block page served with 200.
|
||||
// Counted only (agent_metrics_snapshot) and still a success: whether every real
|
||||
// backend 2xx carries the echo is measured before anything is ever enforced on it.
|
||||
if (!ResponseOriginClassifier.EchoesCorrelationId(response, correlationId))
|
||||
_networkMetrics?.RecordOkWithoutEcho();
|
||||
|
||||
// M4.6.ε — parse backend-to-agent control signals from the 2xx response body.
|
||||
// Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok().
|
||||
return await TryReadControlSignalsAsync(response).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var reason = await TryReadReasonAsync(response).ConfigureAwait(false);
|
||||
var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false);
|
||||
var reason = string.IsNullOrEmpty(body) ? string.Empty : (body!.Length <= 200 ? body : body.Substring(0, 200));
|
||||
var shortReason = $"http {statusCode}{(string.IsNullOrEmpty(reason) ? string.Empty : ": " + reason)}";
|
||||
|
||||
switch (response.StatusCode)
|
||||
{
|
||||
case HttpStatusCode.Unauthorized: // 401
|
||||
case HttpStatusCode.Forbidden: // 403
|
||||
{
|
||||
// Feed the central tracker so the agent can shut down after MaxAuthFailures
|
||||
// instead of retrying a permanently-revoked certificate forever.
|
||||
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry");
|
||||
return UploadResult.Unauthorized($"unauthorized: {shortReason}");
|
||||
// instead of retrying a permanently-revoked certificate forever. It counts only
|
||||
// what the backend (or the platform page of a stopped endpoint) answered; an
|
||||
// answer from the network path is reported once and never counted.
|
||||
var classification = ResponseOriginClassifier.Classify(
|
||||
response, correlationId, body, _serverCertificates?.LastIssuer);
|
||||
_authFailureTracker?.RecordFailure(statusCode, "agent/telemetry", classification);
|
||||
return classification.IsForeign
|
||||
? UploadResult.Intercepted($"intercepted: http {statusCode} ({classification.ForeignReason})", classification)
|
||||
: UploadResult.Unauthorized($"unauthorized: {shortReason}");
|
||||
}
|
||||
|
||||
case HttpStatusCode.RequestTimeout: // 408
|
||||
case (HttpStatusCode)429: // TooManyRequests — enum available net48 SP1+, cast for safety
|
||||
@@ -238,20 +278,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
return UploadResult.Transient(shortReason);
|
||||
}
|
||||
|
||||
private static async Task<string> TryReadReasonAsync(HttpResponseMessage response)
|
||||
{
|
||||
try
|
||||
{
|
||||
var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
if (string.IsNullOrEmpty(body)) return string.Empty;
|
||||
return body.Length <= 200 ? body : body.Substring(0, 200);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// M4.6.ε — parse backend-to-agent control signals from a 2xx batch-response body. Fields
|
||||
/// are <b>optional</b>: an empty body or a body without any of these fields is treated as
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// The first intercepted upload of the process, carried by
|
||||
/// <see cref="TelemetryUploadOrchestrator.UploadIntercepted"/> so the orchestrator can surface a
|
||||
/// <c>telemetry_upload_blocked</c> event with the evidence.
|
||||
/// </summary>
|
||||
public sealed class InterceptionReport
|
||||
{
|
||||
public InterceptionReport(ResponseClassification interception, DateTime firstBlockedAtUtc)
|
||||
{
|
||||
Interception = interception ?? throw new ArgumentNullException(nameof(interception));
|
||||
FirstBlockedAtUtc = firstBlockedAtUtc;
|
||||
}
|
||||
|
||||
/// <summary>Status, sub-reason, page title and server-certificate issuer of the intercepted upload.</summary>
|
||||
public ResponseClassification Interception { get; }
|
||||
|
||||
/// <summary>When the first upload was intercepted (agent clock, UTC).</summary>
|
||||
public DateTime FirstBlockedAtUtc { get; }
|
||||
}
|
||||
}
|
||||
+153
-8
@@ -25,6 +25,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
TimeSpan.FromMilliseconds(1600),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Pause after each consecutive intercepted upload (the last entry repeats). Every attempt
|
||||
/// is a POST of the head batch; the platform or proxy rejects it before the backend sees it.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<TimeSpan> InterceptionBackoffs = new[]
|
||||
{
|
||||
TimeSpan.FromSeconds(30),
|
||||
TimeSpan.FromMinutes(1),
|
||||
TimeSpan.FromMinutes(2),
|
||||
TimeSpan.FromMinutes(5),
|
||||
TimeSpan.FromMinutes(10),
|
||||
TimeSpan.FromMinutes(15),
|
||||
};
|
||||
|
||||
private const int TerminalAttemptNotGranted = 0;
|
||||
private const int TerminalAttemptGranted = 1;
|
||||
private const int TerminalAttemptUsed = 2;
|
||||
|
||||
private readonly ITelemetrySpool _spool;
|
||||
private readonly IBackendTelemetryUploader _uploader;
|
||||
private readonly IClock _clock;
|
||||
@@ -35,6 +53,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// M4.6.ε — backend-to-agent control signals.
|
||||
private DateTime? _pausedUntilUtc;
|
||||
|
||||
// Interception pause — a "not before" for the next upload after an intercepted one, modelled
|
||||
// on the DeviceBlocked pause: drains compare it with the clock and return without HTTP, no
|
||||
// timer (L.7). One immutable window so the termination thread reads it atomically; null =
|
||||
// never intercepted or cleared by a success. The backoff step and the one-shot report are
|
||||
// only touched under the drain guard.
|
||||
private InterceptionWindow? _interceptionWindow;
|
||||
private int _interceptionBackoffStep;
|
||||
private bool _interceptionReported;
|
||||
|
||||
// The one upload attempt the terminal phase may make despite the interception pause.
|
||||
private int _terminalAttemptState;
|
||||
|
||||
// P1 — current effective batch size. Shrinks (never grows) when the backend returns 413,
|
||||
// so an oversized batch is split down instead of discarded. Only mutated under the drain
|
||||
// guard. Floor 1.
|
||||
@@ -113,6 +143,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public event EventHandler<string>? UploadBlocked;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) for the first intercepted upload
|
||||
/// (<see cref="UploadResult.IsIntercepted"/>), independent of <see cref="UploadBlocked"/>.
|
||||
/// Like that event, anything the handler posts queues behind the intercepted batch and
|
||||
/// reaches the backend only once the path clears in the same run.
|
||||
/// </summary>
|
||||
public event EventHandler<InterceptionReport>? UploadIntercepted;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) when the backend answered
|
||||
/// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never
|
||||
@@ -128,6 +166,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public bool IsPaused => _pausedUntilUtc.HasValue && _pausedUntilUtc.Value > _clock.UtcNow;
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> while the pause after an intercepted upload has not expired. Drains return
|
||||
/// without HTTP in this state — except the one attempt <see cref="GrantTerminalAttempt"/>
|
||||
/// allows. A successful upload ends the pause and resets the backoff.
|
||||
/// </summary>
|
||||
public bool IsInterceptionPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
var window = Volatile.Read(ref _interceptionWindow);
|
||||
if (window == null) return false;
|
||||
|
||||
// Device clocks jump (RTC, NTP). A clock set back past the pause start ends the
|
||||
// pause instead of stretching it by the jump; the next attempt re-arms it.
|
||||
var now = _clock.UtcNow;
|
||||
return now >= window.FromUtc && now < window.UntilUtc;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Grants the single upload attempt per process that the terminal phase may make despite an
|
||||
/// active interception pause; the first drain that meets the pause afterwards uses it.
|
||||
/// Idempotent. Without an interception pause the grant changes nothing. The DeviceBlocked
|
||||
/// pause is not affected and still skips terminal drains. Returns
|
||||
/// <see cref="IsInterceptionPaused"/>, so the caller knows whether waking the drain pays off.
|
||||
/// </summary>
|
||||
public bool GrantTerminalAttempt()
|
||||
{
|
||||
Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptGranted, TerminalAttemptNotGranted);
|
||||
return IsInterceptionPaused;
|
||||
}
|
||||
|
||||
public long LastUploadedItemId => _spool.LastUploadedItemId;
|
||||
|
||||
public TelemetryItem Enqueue(TelemetryItemDraft draft)
|
||||
@@ -150,6 +220,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
List<PoisonReport>? pendingPoison = null;
|
||||
string? pendingBlockedReason = null;
|
||||
string? pendingSessionGoneReason = null;
|
||||
InterceptionReport? pendingInterception = null;
|
||||
DrainResult result;
|
||||
|
||||
try
|
||||
@@ -158,12 +229,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
int failedBatches = 0;
|
||||
string? lastError = null;
|
||||
|
||||
// P1 — shared success-side control-signal handling. Applied to BOTH the normal
|
||||
// success path and the poison-survivor re-upload, so a DeviceBlocked / DeviceKill /
|
||||
// AdminAction / Actions response on the survivor upload is honoured identically and
|
||||
// never silently dropped. Returns true if the backend just quarantined the device.
|
||||
bool CollectControlSignals(UploadResult o)
|
||||
// P1 — shared success-side handling. Applied to BOTH the normal success path and
|
||||
// the poison-survivor re-upload, so a DeviceBlocked / DeviceKill / AdminAction /
|
||||
// Actions response on the survivor upload is honoured identically and never silently
|
||||
// dropped, and either success ends an interception pause. Returns true if the
|
||||
// backend just quarantined the device.
|
||||
bool HandleSuccessfulUpload(UploadResult o)
|
||||
{
|
||||
EndInterceptionPause();
|
||||
ApplyControlSignals(o); // mutates _pausedUntilUtc synchronously (next IsPaused sees it)
|
||||
if (CarriesSignal(o)) (pendingSignals ??= new List<UploadResult>()).Add(o);
|
||||
return o.DeviceBlocked;
|
||||
@@ -182,6 +255,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
var batch = _spool.Peek(_effectiveBatchSize);
|
||||
if (batch.Count == 0) break;
|
||||
|
||||
// Interception pause: no HTTP before the not-before passes — periodic drains and
|
||||
// immediate-flush wakeups alike — except the one attempt the terminal phase is
|
||||
// granted. Checked after the Peek so only a real attempt uses the grant.
|
||||
if (IsInterceptionPaused && !TryUseTerminalAttempt())
|
||||
{
|
||||
lastError = $"upload intercepted, next attempt not before {_interceptionWindow?.UntilUtc:O}";
|
||||
break;
|
||||
}
|
||||
|
||||
var outcome = await TryUploadWithRetryAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (outcome.Success)
|
||||
@@ -192,7 +274,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// Stop draining immediately if the backend just quarantined the device —
|
||||
// the next loop iteration would see IsPaused and break anyway, but there
|
||||
// is no point in even Peeking another batch.
|
||||
if (CollectControlSignals(outcome)) break;
|
||||
if (HandleSuccessfulUpload(outcome)) break;
|
||||
}
|
||||
else if (outcome.Interception != null)
|
||||
{
|
||||
// Something on the network path answered instead of the backend (TLS
|
||||
// inspection, proxy page). RETAIN (cursor stays), pause with backoff, and
|
||||
// surface the first one once per process — never as an auth failure.
|
||||
failedBatches++;
|
||||
lastError = outcome.ErrorReason;
|
||||
StartInterceptionPause(outcome.Interception);
|
||||
if (!_interceptionReported)
|
||||
{
|
||||
_interceptionReported = true;
|
||||
pendingInterception = new InterceptionReport(outcome.Interception, _clock.UtcNow);
|
||||
}
|
||||
break;
|
||||
}
|
||||
else if (outcome.IsPoison)
|
||||
{
|
||||
@@ -236,7 +333,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// P1 fix: honour any control signals the survivor re-upload carried
|
||||
// (DeviceBlocked / DeviceKill / AdminAction / Actions) — same as the
|
||||
// normal success path; previously these were silently dropped.
|
||||
deviceBlockedBySurvivors = CollectControlSignals(survivorOutcome);
|
||||
deviceBlockedBySurvivors = HandleSuccessfulUpload(survivorOutcome);
|
||||
}
|
||||
|
||||
// Survivors uploaded (or none) — advance past the whole batch; the named items
|
||||
@@ -316,7 +413,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
// One-shot loud marker so a stuck non-transient, non-auth block is visible on
|
||||
// the backend instead of only in the agent log (transient = normal retry;
|
||||
// auth = the session is already shutting down via the auth path).
|
||||
// a backend auth rejection = the auth path shuts the agent down; an
|
||||
// interception has its own one-shot report above).
|
||||
if (!outcome.IsTransient && !outcome.IsAuthFailure && !_uploadBlockedReported)
|
||||
{
|
||||
_uploadBlockedReported = true;
|
||||
@@ -362,9 +460,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
if (pendingInterception != null)
|
||||
{
|
||||
try { UploadIntercepted?.Invoke(this, pendingInterception); }
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Arms the interception pause with the next backoff step (the last step repeats). Called
|
||||
/// under the drain guard.
|
||||
/// </summary>
|
||||
private void StartInterceptionPause(ResponseClassification interception)
|
||||
{
|
||||
var delay = InterceptionBackoffs[Math.Min(_interceptionBackoffStep, InterceptionBackoffs.Count - 1)];
|
||||
_interceptionBackoffStep = Math.Min(_interceptionBackoffStep + 1, InterceptionBackoffs.Count - 1);
|
||||
var now = _clock.UtcNow;
|
||||
Volatile.Write(ref _interceptionWindow, new InterceptionWindow(now, now + delay));
|
||||
|
||||
// The uploader already logged the page finding at Warning; this line adds the schedule.
|
||||
_logger?.Info(
|
||||
$"TelemetryUploadOrchestrator: upload intercepted ({interception.ForeignReason}, http {interception.StatusCode}) — " +
|
||||
$"batch retained, next attempt not before {now + delay:O} ({delay.TotalSeconds:0}s).");
|
||||
}
|
||||
|
||||
/// <summary>A successful upload ends the interception pause and resets the backoff. Called under the drain guard.</summary>
|
||||
private void EndInterceptionPause()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _interceptionWindow, null) == null) return;
|
||||
|
||||
_interceptionBackoffStep = 0;
|
||||
_logger?.Info("TelemetryUploadOrchestrator: upload path clear again after an interception — uploads resume.");
|
||||
}
|
||||
|
||||
private bool TryUseTerminalAttempt() =>
|
||||
Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptUsed, TerminalAttemptGranted) == TerminalAttemptGranted;
|
||||
|
||||
private sealed class InterceptionWindow
|
||||
{
|
||||
public InterceptionWindow(DateTime fromUtc, DateTime untilUtc)
|
||||
{
|
||||
FromUtc = fromUtc;
|
||||
UntilUtc = untilUtc;
|
||||
}
|
||||
|
||||
public DateTime FromUtc { get; }
|
||||
public DateTime UntilUtc { get; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of items at the head of <paramref name="batch"/> whose partition key is not the
|
||||
/// current session's. 0 when no current key is configured (every batch counts as own).
|
||||
|
||||
@@ -31,7 +31,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
bool requiresSplit = false,
|
||||
bool isPoison = false,
|
||||
IReadOnlyList<string>? poisonRowKeys = null,
|
||||
bool isSessionGone = false)
|
||||
bool isSessionGone = false,
|
||||
ResponseClassification? interception = null)
|
||||
{
|
||||
Success = success;
|
||||
ErrorReason = errorReason;
|
||||
@@ -46,6 +47,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
IsPoison = isPoison;
|
||||
PoisonRowKeys = poisonRowKeys;
|
||||
IsSessionGone = isSessionGone;
|
||||
Interception = interception;
|
||||
}
|
||||
|
||||
public bool Success { get; }
|
||||
@@ -93,6 +95,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public bool IsSessionGone { get; }
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 that did not come from the backend (no <c>X-Correlation-ID</c> echo): the
|
||||
/// platform's "Client Certificate Required" page or a proxy page — TLS inspection on the
|
||||
/// network path. Not an auth failure: never counted by the <c>AuthFailureTracker</c>. The
|
||||
/// orchestrator retains the batch and pauses uploads with backoff.
|
||||
/// </summary>
|
||||
public bool IsIntercepted => Interception != null;
|
||||
|
||||
/// <summary>Status, sub-reason, page title and server-certificate issuer of the interception; null unless <see cref="IsIntercepted"/>.</summary>
|
||||
public ResponseClassification? Interception { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Non-terminal quarantine signalled by the backend. The agent should stop draining
|
||||
/// uploads until <see cref="UnblockAt"/> (if set) elapses — the session stays alive so
|
||||
@@ -186,5 +199,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isSessionGone: true);
|
||||
|
||||
/// <summary>
|
||||
/// A 401/403 something on the network path answered (<see cref="ResponseOrigin.Foreign"/>).
|
||||
/// Flagged <see cref="IsIntercepted"/>, never <see cref="IsAuthFailure"/>.
|
||||
/// </summary>
|
||||
public static UploadResult Intercepted(string reason, ResponseClassification interception)
|
||||
{
|
||||
if (interception == null) throw new ArgumentNullException(nameof(interception));
|
||||
if (!interception.IsForeign) throw new ArgumentException("Only a foreign response is an interception.", nameof(interception));
|
||||
return new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
interception: interception);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,10 +256,10 @@ namespace AutopilotMonitor.Agent.V2
|
||||
/// <summary>
|
||||
/// <c>true</c> when the file-based <c>enrollment-complete.marker</c> signals that a
|
||||
/// previous session already completed enrollment — written by
|
||||
/// <see cref="CheckSessionAgeEmergencyBreak"/> right before it fires the cleanup retry,
|
||||
/// so that a next-boot observing the marker exits cleanly even if the cleanup retry
|
||||
/// failed to remove the Scheduled Task. Triggers another self-destruct attempt when
|
||||
/// <paramref name="selfDestructOnComplete"/> is set.
|
||||
/// <see cref="EndMonitoringAtStart"/> (emergency break, intercepted-registration give-up,
|
||||
/// config kill signal) right before it fires the cleanup retry, so that a next-boot observing the marker exits
|
||||
/// cleanly even if the cleanup retry failed to remove the Scheduled Task. Triggers another
|
||||
/// self-destruct attempt when <paramref name="selfDestructOnComplete"/> is set.
|
||||
/// <para>
|
||||
/// The <c>HKLM\SOFTWARE\AutopilotMonitor\Deployed</c> registry value is NOT consulted
|
||||
/// here: it is a bootstrap-script reentry lock (read by Install-AutopilotMonitor-v2.ps1
|
||||
@@ -350,23 +350,10 @@ namespace AutopilotMonitor.Agent.V2
|
||||
try { onBreakFired?.Invoke(sessionAgeHours); }
|
||||
catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); }
|
||||
|
||||
// Write enrollment-complete.marker so the next start exits cleanly even if
|
||||
// the cleanup retry below fails (Scheduled Task still alive, files locked, …).
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName);
|
||||
File.WriteAllText(markerPath,
|
||||
$"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)");
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
|
||||
if (selfDestructOnComplete)
|
||||
{
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, "emergency_break");
|
||||
}
|
||||
|
||||
persistence.Delete(logger);
|
||||
EndMonitoringAtStart(
|
||||
stateDirectory, persistence, selfDestructOnComplete, cleanupServiceFactory, logger,
|
||||
markerText: $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)",
|
||||
cleanupReason: "emergency_break");
|
||||
logger.Info("Emergency break: cleanup completed. Agent will exit.");
|
||||
return true;
|
||||
}
|
||||
@@ -377,6 +364,40 @@ namespace AutopilotMonitor.Agent.V2
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends monitoring at process start: writes <c>enrollment-complete.marker</c> first so the
|
||||
/// next start exits cleanly even if the cleanup retry fails (Scheduled Task still alive,
|
||||
/// files locked, …), runs the cleanup retry when <paramref name="runSelfDestruct"/> is set
|
||||
/// (the emergency break and the give-up pass <c>SelfDestructOnComplete</c>, the kill signal
|
||||
/// forces it), and deletes the persisted session files. Shared by the session-age emergency
|
||||
/// break, the intercepted-registration give-up
|
||||
/// (<c>BackendSessionRegistration.ConcludeFailedRun</c>) and the config-channel kill signal
|
||||
/// so all three leave the device in the same state. Never throws.
|
||||
/// </summary>
|
||||
internal static void EndMonitoringAtStart(
|
||||
string stateDirectory,
|
||||
SessionIdPersistence persistence,
|
||||
bool runSelfDestruct,
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger,
|
||||
string markerText,
|
||||
string cleanupReason)
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
File.WriteAllText(Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName), markerText);
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
|
||||
if (runSelfDestruct)
|
||||
{
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, cleanupReason);
|
||||
}
|
||||
|
||||
persistence?.Delete(logger);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Kill-switch on the control channel: honours a <c>DeviceKillSignal</c> served on the
|
||||
/// <c>agent/config</c> response. Config is fetched at every process start (boot via
|
||||
@@ -409,22 +430,15 @@ namespace AutopilotMonitor.Agent.V2
|
||||
if (consoleMode)
|
||||
Console.Out.WriteLine("KILL: administrator issued a remote kill signal. Cleaning up and exiting.");
|
||||
|
||||
// Write enrollment-complete.marker first so the next start exits cleanly even if
|
||||
// the cleanup below fails (Scheduled Task still alive, files locked, …) — same
|
||||
// guarantee-of-termination pattern as the session-age emergency break.
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(stateDirectory);
|
||||
var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName);
|
||||
File.WriteAllText(markerPath, $"Kill signal at {DateTime.UtcNow:O} (config channel)");
|
||||
}
|
||||
catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); }
|
||||
SessionIdPersistence persistence = null;
|
||||
try { persistence = new SessionIdPersistence(dataDirectory); }
|
||||
catch (Exception ex) { logger.Warning($"Failed to open the persisted session after kill: {ex.Message}"); }
|
||||
|
||||
// forceSelfDestruct — no SelfDestructOnComplete gate, matching the telemetry-kill.
|
||||
TryRetryCleanup(cleanupServiceFactory, logger, "kill_signal");
|
||||
|
||||
try { new SessionIdPersistence(dataDirectory).Delete(logger); }
|
||||
catch (Exception ex) { logger.Warning($"Failed to delete persisted session after kill: {ex.Message}"); }
|
||||
EndMonitoringAtStart(
|
||||
stateDirectory, persistence, runSelfDestruct: true, cleanupServiceFactory, logger,
|
||||
markerText: $"Kill signal at {DateTime.UtcNow:O} (config channel)",
|
||||
cleanupReason: "kill_signal");
|
||||
|
||||
logger.Info("Kill signal handled — agent exiting.");
|
||||
return true;
|
||||
|
||||
@@ -269,13 +269,17 @@ namespace AutopilotMonitor.Agent.V2
|
||||
|
||||
// Phase 6 (POST /api/agent/register-session with retry + outcome-based exit-code
|
||||
// mapping + on-failure client disposal) is encapsulated in BackendSessionRegistration.
|
||||
// Exit codes: 6 = AuthFailed, 7 = anything else. See Runtime/BackendSessionRegistration.cs.
|
||||
// Exit codes: 6 = AuthFailed, 7 = anything else. A registration intercepted on more
|
||||
// starts than the buffer allows cleans up like the emergency break (state folder +
|
||||
// cleanup factory) and still exits 6. See Runtime/BackendSessionRegistration.cs.
|
||||
var registration = Runtime.BackendSessionRegistration.Register(
|
||||
bootstrap.AgentConfig, auth, telemetry.MtlsHttpClient, GetAgentVersion(), consoleMode, logger,
|
||||
rotateSession: () => Runtime.BackendSessionRegistration.RotateSession(
|
||||
bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger),
|
||||
sessionPersistence: bootstrap.SessionPersistence,
|
||||
configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString());
|
||||
configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString(),
|
||||
stateDirectory: stateSubdir,
|
||||
cleanupServiceFactory: bootstrap.CleanupServiceFactory);
|
||||
if (registration.ShouldExit)
|
||||
{
|
||||
return registration.ExitCode;
|
||||
|
||||
@@ -345,7 +345,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
lifecyclePost = new InformationalEventPost(ingress, SystemClock.Instance, logger);
|
||||
LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, remoteConfigService, logger,
|
||||
rotatedFromSessionId: registrationResult?.RotatedFromSessionId);
|
||||
rotatedFromSessionId: registrationResult?.RotatedFromSessionId,
|
||||
priorRegistrationFailures: registration.PriorFailures);
|
||||
// Wire-visible signal that the agent is running on defaults/cache
|
||||
// rather than the live tenant config — closes the cold-start
|
||||
// blind spot observed in session 8f2bef72 (2026-05-19).
|
||||
|
||||
@@ -7,6 +7,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -76,7 +77,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
logger.Debug("Client certificate authentication disabled.");
|
||||
}
|
||||
|
||||
var apiHttpClient = BuildBackendApiHttpClient(networkMetrics, clientCertificate, agentVersion);
|
||||
// One observer for both backend handler stacks (this one and the mTLS telemetry stack
|
||||
// in BuildTelemetryClients): the issuer of the last validated server certificate is
|
||||
// the evidence for a 401/403 that something on the network path answered.
|
||||
var serverCertificates = new ServerCertificateObserver();
|
||||
|
||||
var apiHttpClient = BuildBackendApiHttpClient(networkMetrics, clientCertificate, agentVersion, serverCertificates);
|
||||
|
||||
var backendApiClient = new BackendApiClient(
|
||||
httpClient: apiHttpClient,
|
||||
@@ -87,7 +93,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
useBootstrapTokenAuth: agentConfig.UseBootstrapTokenAuth,
|
||||
bootstrapToken: agentConfig.BootstrapToken,
|
||||
agentVersion: agentVersion,
|
||||
logger: logger);
|
||||
logger: logger,
|
||||
serverCertificates: serverCertificates);
|
||||
|
||||
// M4.6.γ — Emergency + Distress reporters. Plumbed into RemoteConfigService so
|
||||
// Config-fetch failures (auth vs network) flow to the correct channel. Also fires
|
||||
@@ -141,7 +148,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
serialNumber: hardware.SerialNumber,
|
||||
distressReporter: distressReporter,
|
||||
emergencyReporter: emergencyReporter,
|
||||
authFailureTracker: authFailureTracker);
|
||||
authFailureTracker: authFailureTracker,
|
||||
serverCertificates: serverCertificates);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -168,7 +176,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
mtlsHttpClient = MtlsHttpClientFactory.Create(
|
||||
resolver: new DefaultCertificateResolver(),
|
||||
logger: logger,
|
||||
metrics: auth.NetworkMetrics);
|
||||
metrics: auth.NetworkMetrics,
|
||||
serverCertificates: auth.ServerCertificates);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
@@ -189,7 +198,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
bootstrapToken: agentConfig.UseBootstrapTokenAuth ? agentConfig.BootstrapToken : null,
|
||||
agentVersion: agentVersion,
|
||||
authFailureTracker: auth.AuthFailureTracker,
|
||||
logger: logger); // Plan §5 Fix 5 — upload-cadence logging
|
||||
logger: logger, // Plan §5 Fix 5 — upload-cadence logging
|
||||
serverCertificates: auth.ServerCertificates,
|
||||
networkMetrics: auth.NetworkMetrics);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -211,14 +222,16 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// <item><see cref="NetworkMetricsRecordingHandler"/> — records every SendAsync into
|
||||
/// the shared <see cref="NetworkMetrics"/> counter.</item>
|
||||
/// <item><see cref="HttpClientHandler"/> — TLS, optional client cert,
|
||||
/// gzip/deflate response decompression.</item>
|
||||
/// gzip/deflate response decompression, optional observing server-certificate
|
||||
/// callback (<paramref name="serverCertificates"/>; never changes the verdict).</item>
|
||||
/// </list>
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static HttpClient BuildBackendApiHttpClient(
|
||||
NetworkMetrics networkMetrics,
|
||||
X509Certificate2 clientCertificate,
|
||||
string agentVersion)
|
||||
string agentVersion,
|
||||
ServerCertificateObserver serverCertificates = null)
|
||||
{
|
||||
var inner = new HttpClientHandler();
|
||||
if (inner.SupportsAutomaticDecompression)
|
||||
@@ -229,6 +242,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
inner.ClientCertificates.Add(clientCertificate);
|
||||
}
|
||||
serverCertificates?.Attach(inner);
|
||||
|
||||
var pipeline = new NetworkMetricsRecordingHandler(networkMetrics, inner);
|
||||
var client = new HttpClient(pipeline)
|
||||
@@ -270,6 +284,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
public EmergencyReporter EmergencyReporter { get; }
|
||||
public AuthFailureTracker AuthFailureTracker { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Server-certificate observer shared by the BackendApiClient handler and the mTLS
|
||||
/// telemetry handler built in Phase 5.
|
||||
/// </summary>
|
||||
public ServerCertificateObserver ServerCertificates { get; }
|
||||
|
||||
public BackendAuthBundle(
|
||||
BackendApiClient backendApiClient,
|
||||
NetworkMetrics networkMetrics,
|
||||
@@ -279,7 +299,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
string serialNumber,
|
||||
DistressReporter distressReporter,
|
||||
EmergencyReporter emergencyReporter,
|
||||
AuthFailureTracker authFailureTracker)
|
||||
AuthFailureTracker authFailureTracker,
|
||||
ServerCertificateObserver serverCertificates)
|
||||
{
|
||||
BackendApiClient = backendApiClient;
|
||||
NetworkMetrics = networkMetrics;
|
||||
@@ -290,6 +311,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
DistressReporter = distressReporter;
|
||||
EmergencyReporter = emergencyReporter;
|
||||
AuthFailureTracker = authFailureTracker;
|
||||
ServerCertificates = serverCertificates;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
@@ -19,10 +21,22 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// no-op and session status / phase / admin-overrides / validator reconcile all break.
|
||||
/// On registration failure we follow V1's rule: collectors MUST NOT start and the
|
||||
/// agent exits cleanly; the Scheduled Task has only a boot trigger, so the next attempt is
|
||||
/// the next boot. The failed run is recorded for the emergency break report.
|
||||
/// the next boot. The failed run is recorded for the emergency break report. A registration
|
||||
/// that something on the network path answered gets <c>InterceptedRegistrationRetryBoots</c>
|
||||
/// further boots, then the agent cleans up like the emergency break (<see cref="ConcludeFailedRun"/>).
|
||||
/// A successful registration after failed starts reports them on <c>agent_started</c>
|
||||
/// (<see cref="ClearAfterSuccess"/>).
|
||||
/// </summary>
|
||||
internal static class BackendSessionRegistration
|
||||
{
|
||||
/// <summary>
|
||||
/// How long a failed start waits for the pending ClientCertificateNotReceived distress before
|
||||
/// it exits or cleans up — the distress reporter's own HTTP timeout.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan PendingDistressBound = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <param name="stateDirectory">State folder for <c>enrollment-complete.marker</c> on the intercepted-registration give-up.</param>
|
||||
/// <param name="cleanupServiceFactory">Self-destruct for the give-up (run when <c>SelfDestructOnComplete</c>).</param>
|
||||
public static SessionRegistrationOutcomeResult Register(
|
||||
AgentConfiguration agentConfig,
|
||||
BackendAuthBundle auth,
|
||||
@@ -32,7 +46,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
AgentLogger logger,
|
||||
Func<string> rotateSession = null,
|
||||
SessionIdPersistence sessionPersistence = null,
|
||||
string configFetchOutcome = null)
|
||||
string configFetchOutcome = null,
|
||||
string stateDirectory = null,
|
||||
Func<CleanupService> cleanupServiceFactory = null)
|
||||
{
|
||||
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
|
||||
if (auth == null) throw new ArgumentNullException(nameof(auth));
|
||||
@@ -57,36 +73,41 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
rotateSession: rotateSession)
|
||||
.GetAwaiter().GetResult();
|
||||
|
||||
RecordRegistrationRun(
|
||||
sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow,
|
||||
linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, logger);
|
||||
|
||||
if (registrationResult.Outcome != SessionRegistrationOutcome.Succeeded)
|
||||
if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
logger.Error(
|
||||
$"=== SESSION REGISTRATION FAILED ({registrationResult.Outcome}: {registrationResult.ErrorMessage}) — " +
|
||||
"collectors will NOT start to prevent orphaned events. ===");
|
||||
if (consoleMode)
|
||||
Console.Error.WriteLine($"FATAL: session registration failed ({registrationResult.Outcome}). Agent exiting.");
|
||||
try { mtlsHttpClient?.Dispose(); } catch { }
|
||||
try { auth.BackendApiClient?.Dispose(); } catch { }
|
||||
// Exit code differs so the diag skill can distinguish Auth vs Network in Scheduled-Task history.
|
||||
return SessionRegistrationOutcomeResult.Exit(
|
||||
registrationResult.Outcome == SessionRegistrationOutcome.AuthFailed ? 6 : 7);
|
||||
return SessionRegistrationOutcomeResult.Continue(
|
||||
registrationResult, ClearAfterSuccess(sessionPersistence, NetworkFingerprint.TryCapture, logger));
|
||||
}
|
||||
|
||||
return SessionRegistrationOutcomeResult.Continue(registrationResult);
|
||||
var record = RecordFailedRun(
|
||||
sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow,
|
||||
linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, NetworkFingerprint.TryCapture(), logger);
|
||||
|
||||
logger.Error(
|
||||
$"=== SESSION REGISTRATION FAILED ({registrationResult.Outcome}: {registrationResult.ErrorMessage}) — " +
|
||||
"collectors will NOT start to prevent orphaned events. ===");
|
||||
if (consoleMode)
|
||||
Console.Error.WriteLine($"FATAL: session registration failed ({registrationResult.Outcome}). Agent exiting.");
|
||||
try { mtlsHttpClient?.Dispose(); } catch { }
|
||||
try { auth.BackendApiClient?.Dispose(); } catch { }
|
||||
|
||||
return SessionRegistrationOutcomeResult.Exit(ConcludeFailedRun(
|
||||
registrationResult, record, agentConfig, auth.AuthFailureTracker,
|
||||
sessionPersistence, stateDirectory, cleanupServiceFactory, logger));
|
||||
}
|
||||
|
||||
/// <summary>Longest <see cref="RegistrationFailureSummary.LastError"/> kept on disk and on the wire.</summary>
|
||||
internal const int MaxRecordedErrorLength = 256;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps <c>registration-failure.json</c> in step with this run: a failed run is folded
|
||||
/// into the record (the emergency break later reports it — for a session that never
|
||||
/// registered, the only account of why), a successful one clears it. Never throws.
|
||||
/// Folds a failed run into <c>registration-failure.json</c> (the emergency break later reports
|
||||
/// it — for a session that never registered, the only account of why) and returns the record
|
||||
/// including this run; null without persistence or when it could not be written. An
|
||||
/// intercepted run also counts toward <see cref="RegistrationFailureSummary.InterceptedRuns"/>,
|
||||
/// which only a successful registration resets. <paramref name="networkFingerprint"/> is this
|
||||
/// run's network; it is kept beside the record and never leaves the device. Never throws.
|
||||
/// </summary>
|
||||
internal static void RecordRegistrationRun(
|
||||
internal static RegistrationFailureSummary RecordFailedRun(
|
||||
SessionIdPersistence sessionPersistence,
|
||||
SessionRegistrationResult registrationResult,
|
||||
DateTime startedUtc,
|
||||
@@ -94,24 +115,20 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
bool? linkUpAtStart,
|
||||
bool? linkUpAtEnd,
|
||||
string configFetchOutcome,
|
||||
string networkFingerprint,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (sessionPersistence == null || registrationResult == null) return;
|
||||
if (sessionPersistence == null || registrationResult == null) return null;
|
||||
try
|
||||
{
|
||||
if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded)
|
||||
{
|
||||
sessionPersistence.ClearRegistrationFailure(logger);
|
||||
return;
|
||||
}
|
||||
|
||||
var error = registrationResult.ErrorMessage;
|
||||
if (error != null && error.Length > MaxRecordedErrorLength)
|
||||
error = error.Substring(0, MaxRecordedErrorLength);
|
||||
|
||||
sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary
|
||||
return sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = 1,
|
||||
InterceptedRuns = registrationResult.IsIntercepted ? 1 : 0,
|
||||
FirstFailedAtUtc = endedUtc,
|
||||
LastFailedAtUtc = endedUtc,
|
||||
Outcome = registrationResult.Outcome.ToString(),
|
||||
@@ -120,14 +137,99 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
NetworkLinkUpAtEnd = linkUpAtEnd,
|
||||
AttemptWindowSeconds = Math.Round(Math.Max(0, (endedUtc - startedUtc).TotalSeconds), 1),
|
||||
ConfigFetchOutcome = configFetchOutcome,
|
||||
}, logger);
|
||||
}, logger, networkFingerprint);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"Registration run record failed (best-effort): {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A successful registration clears <c>registration-failure.json</c>. When earlier starts of
|
||||
/// this session had failed, the cleared record comes back for this run's <c>agent_started</c>
|
||||
/// — until now the success erased it silently and late successes were uncountable. The
|
||||
/// network fingerprints are compared here (<paramref name="captureNetworkFingerprint"/> runs
|
||||
/// only when there is a record) and never leave the device. Null when there was no record.
|
||||
/// Never throws.
|
||||
/// </summary>
|
||||
internal static PriorRegistrationFailures ClearAfterSuccess(
|
||||
SessionIdPersistence sessionPersistence,
|
||||
Func<string> captureNetworkFingerprint,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (sessionPersistence == null) return null;
|
||||
try
|
||||
{
|
||||
var cleared = sessionPersistence.LoadRegistrationFailureRecord();
|
||||
sessionPersistence.ClearRegistrationFailure(logger);
|
||||
if (cleared == null) return null;
|
||||
|
||||
var prior = new PriorRegistrationFailures(
|
||||
cleared.Summary.FailedRuns,
|
||||
cleared.Summary.InterceptedRuns,
|
||||
cleared.Summary.Outcome,
|
||||
NetworkFingerprint.Differs(cleared.NetworkFingerprint, captureNetworkFingerprint?.Invoke()));
|
||||
logger?.Info(
|
||||
$"Session registered after {prior.FailedRuns} failed start(s) ({prior.InterceptedRuns} intercepted, " +
|
||||
$"last: {prior.LastOutcome}, network changed: {prior.NetworkChanged?.ToString() ?? "unknown"}).");
|
||||
return prior;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger?.Warning($"Registration run record failed (best-effort): {ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The end of a start whose registration failed. First the ClientCertificateNotReceived
|
||||
/// distress, which the tracker dispatched fire-and-forget, gets up to
|
||||
/// <see cref="PendingDistressBound"/> to leave the machine: this process is about to exit or
|
||||
/// clean up, and nothing else waits for it. Then the intercepted-registration buffer: when
|
||||
/// this start was intercepted and the intercepted starts since the last success (this one
|
||||
/// included) exceed <c>InterceptedRegistrationRetryBoots</c>, monitoring ends exactly like
|
||||
/// the 48-hour emergency break (<see cref="Program.EndMonitoringAtStart"/>). Every other
|
||||
/// failure — a backend 401/403, a network failure, an interception within the buffer, a
|
||||
/// record that could not be written — leaves everything in place for the next boot. Returns
|
||||
/// the exit code: 6 for an auth failure (the give-up included: it is still one), 7 otherwise.
|
||||
/// </summary>
|
||||
internal static int ConcludeFailedRun(
|
||||
SessionRegistrationResult registrationResult,
|
||||
RegistrationFailureSummary record,
|
||||
AgentConfiguration agentConfig,
|
||||
AuthFailureTracker authFailureTracker,
|
||||
SessionIdPersistence sessionPersistence,
|
||||
string stateDirectory,
|
||||
Func<CleanupService> cleanupServiceFactory,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (authFailureTracker != null && !authFailureTracker.WaitForPendingForeignDistress(PendingDistressBound))
|
||||
logger.Warning($"ClientCertificateNotReceived distress still in flight after {PendingDistressBound.TotalSeconds:F0}s — continuing without it.");
|
||||
|
||||
var buffer = agentConfig.InterceptedRegistrationRetryBoots;
|
||||
if (registrationResult.IsIntercepted && record != null && record.InterceptedRuns > buffer)
|
||||
{
|
||||
logger.Warning(
|
||||
$"=== REGISTRATION INTERCEPTED ON {record.InterceptedRuns} START(S) SINCE THE LAST SUCCESS " +
|
||||
$"(buffer: {buffer} further boot(s)) — cleaning up like the emergency break. ===");
|
||||
Program.EndMonitoringAtStart(
|
||||
stateDirectory, sessionPersistence, agentConfig.SelfDestructOnComplete, cleanupServiceFactory, logger,
|
||||
markerText: $"Intercepted registration at {DateTime.UtcNow:O} ({record.InterceptedRuns} intercepted start(s), buffer {buffer})",
|
||||
cleanupReason: "intercepted_registration");
|
||||
}
|
||||
else if (registrationResult.IsIntercepted)
|
||||
{
|
||||
logger.Info(
|
||||
$"Registration intercepted on {record?.InterceptedRuns.ToString() ?? "an unrecorded number of"} start(s) " +
|
||||
$"since the last success (buffer: {buffer} further boot(s)) — the next boot retries.");
|
||||
}
|
||||
|
||||
// Exit code differs so the diag skill can distinguish Auth vs Network in Scheduled-Task history.
|
||||
return registrationResult.Outcome == SessionRegistrationOutcome.AuthFailed ? 6 : 7;
|
||||
}
|
||||
|
||||
private static bool? TryIsNetworkLinkUp()
|
||||
{
|
||||
try { return NetworkInterface.GetIsNetworkAvailable(); }
|
||||
@@ -266,7 +368,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// Phase 6 outcome — either an early exit (V1 parity: 6 = AuthFailed, 7 = anything else)
|
||||
/// or a Continue payload carrying the successful <see cref="SessionRegistrationResult"/>
|
||||
/// for downstream consumers (initial-signal posting, admin-preemption handling,
|
||||
/// session-started anchor, AdminAction surface in the wired telemetry response).
|
||||
/// session-started anchor, AdminAction surface in the wired telemetry response) and the
|
||||
/// failed starts this registration followed (<see cref="PriorFailures"/>, for <c>agent_started</c>).
|
||||
/// </summary>
|
||||
internal sealed class SessionRegistrationOutcomeResult
|
||||
{
|
||||
@@ -274,17 +377,50 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
public int ExitCode { get; }
|
||||
public SessionRegistrationResult Registration { get; }
|
||||
|
||||
private SessionRegistrationOutcomeResult(bool shouldExit, int exitCode, SessionRegistrationResult registration)
|
||||
/// <summary>The failed-registration record this success cleared; null when there was none.</summary>
|
||||
public PriorRegistrationFailures PriorFailures { get; }
|
||||
|
||||
private SessionRegistrationOutcomeResult(
|
||||
bool shouldExit, int exitCode, SessionRegistrationResult registration, PriorRegistrationFailures priorFailures)
|
||||
{
|
||||
ShouldExit = shouldExit;
|
||||
ExitCode = exitCode;
|
||||
Registration = registration;
|
||||
PriorFailures = priorFailures;
|
||||
}
|
||||
|
||||
public static SessionRegistrationOutcomeResult Exit(int code)
|
||||
=> new SessionRegistrationOutcomeResult(true, code, null);
|
||||
=> new SessionRegistrationOutcomeResult(true, code, null, null);
|
||||
|
||||
public static SessionRegistrationOutcomeResult Continue(SessionRegistrationResult registration)
|
||||
=> new SessionRegistrationOutcomeResult(false, 0, registration);
|
||||
public static SessionRegistrationOutcomeResult Continue(
|
||||
SessionRegistrationResult registration, PriorRegistrationFailures priorFailures = null)
|
||||
=> new SessionRegistrationOutcomeResult(false, 0, registration, priorFailures);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// What the failed starts before a successful registration left in the record it cleared,
|
||||
/// reported once on that run's <c>agent_started</c> (<c>priorRegistration*</c> keys): whether a
|
||||
/// further boot rescues intercepted or failed starts becomes countable. Only the fingerprint
|
||||
/// comparison is carried, never a fingerprint.
|
||||
/// </summary>
|
||||
internal sealed class PriorRegistrationFailures
|
||||
{
|
||||
public PriorRegistrationFailures(int failedRuns, int interceptedRuns, string lastOutcome, bool? networkChanged)
|
||||
{
|
||||
FailedRuns = failedRuns;
|
||||
InterceptedRuns = interceptedRuns;
|
||||
LastOutcome = lastOutcome;
|
||||
NetworkChanged = networkChanged;
|
||||
}
|
||||
|
||||
public int FailedRuns { get; }
|
||||
|
||||
public int InterceptedRuns { get; }
|
||||
|
||||
/// <summary>Registration outcome of the latest failed start (e.g. <c>AuthFailed</c>, <c>Failed</c>).</summary>
|
||||
public string LastOutcome { get; }
|
||||
|
||||
/// <summary>The latest failed start ran on a different network than this one; null when either side is unknown.</summary>
|
||||
public bool? NetworkChanged { get; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// and the tenant-influenced runtime knobs so dashboards can classify crash-loops,
|
||||
/// backend-rejected sessions and forced self-destruct runs. Phase stays
|
||||
/// <see cref="EnrollmentPhase.Unknown"/> — the event is NOT a phase declaration.
|
||||
/// <paramref name="priorRegistrationFailures"/> (failed starts this run's registration
|
||||
/// followed) adds the <c>priorRegistration*</c> keys; null leaves them out.
|
||||
/// </summary>
|
||||
public static void EmitAgentStarted(
|
||||
InformationalEventPost post,
|
||||
@@ -43,7 +45,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
string agentVersion,
|
||||
RemoteConfigService remoteConfigService,
|
||||
AgentLogger logger,
|
||||
string rotatedFromSessionId = null)
|
||||
string rotatedFromSessionId = null,
|
||||
PriorRegistrationFailures priorRegistrationFailures = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -95,6 +98,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
if (previousExit?.LastBootUtc.HasValue == true)
|
||||
data["previousBootUtc"] = previousExit.LastBootUtc.Value.ToString("o");
|
||||
|
||||
// Earlier starts of this session failed to register and this one succeeded: the
|
||||
// record the success cleared, so late registrations (and whether the network
|
||||
// changed in between) are countable. Each key only when known.
|
||||
if (priorRegistrationFailures != null)
|
||||
{
|
||||
data["priorRegistrationFailedRuns"] = priorRegistrationFailures.FailedRuns;
|
||||
data["priorRegistrationInterceptedRuns"] = priorRegistrationFailures.InterceptedRuns;
|
||||
if (!string.IsNullOrEmpty(priorRegistrationFailures.LastOutcome))
|
||||
data["priorRegistrationLastOutcome"] = priorRegistrationFailures.LastOutcome;
|
||||
if (priorRegistrationFailures.NetworkChanged.HasValue)
|
||||
data["priorRegistrationNetworkChanged"] = priorRegistrationFailures.NetworkChanged.Value;
|
||||
}
|
||||
|
||||
post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = agentConfig.SessionId,
|
||||
|
||||
@@ -47,6 +47,7 @@ public class AgentConfigResolverTests
|
||||
Assert.True(r.EnableGeoLocation);
|
||||
Assert.Equal(5, r.MaxAuthFailures);
|
||||
Assert.Equal(0, r.AuthFailureTimeoutMinutes);
|
||||
Assert.Equal(1, r.InterceptedRegistrationRetryBoots); // backend constant, no tenant or operator setting
|
||||
Assert.Equal("Info", r.LogLevel);
|
||||
Assert.Equal(10, r.RebootDelaySeconds);
|
||||
Assert.Equal(60, r.EnrollmentSummaryTimeoutSeconds);
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Middleware;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The agent treats a 401/403 as this backend's verdict only when the response echoes the
|
||||
/// <c>X-Correlation-ID</c> it sent; the platform's own pages and a proxy's pages carry none. The
|
||||
/// middleware stamps the header onto the shared <see cref="HttpContext"/> response before the rest
|
||||
/// of the pipeline runs, so the stamp does not depend on the status a later step writes: a
|
||||
/// middleware envelope (<see cref="ApiErrorWriter"/>) and a function response alike — with the
|
||||
/// ASP.NET Core integration a worker <c>HttpResponseData</c> is a view over that same response,
|
||||
/// and its header set starts from the headers already on it.
|
||||
/// </summary>
|
||||
public class CorrelationIdMiddlewareTests
|
||||
{
|
||||
private const string HeaderName = "X-Correlation-ID";
|
||||
|
||||
/// <summary>The agent's format: a fresh <c>Guid.ToString("N")</c> per request.</summary>
|
||||
private const string AgentCorrelationId = "7c9e6679a1b24f3e8d5c0b1a2f3e4d5c";
|
||||
|
||||
private sealed class Run
|
||||
{
|
||||
public required HttpContext Http { get; init; }
|
||||
public required IDictionary<object, object> Items { get; init; }
|
||||
|
||||
public string EchoedId => Http.Response.Headers[HeaderName].ToString();
|
||||
|
||||
public string Body()
|
||||
{
|
||||
Http.Response.Body.Position = 0;
|
||||
return new StreamReader(Http.Response.Body, Encoding.UTF8).ReadToEnd();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Runs the middleware; <paramref name="answer"/> is whatever answers the request after it.</summary>
|
||||
private static async Task<Run> InvokeAsync(string? inboundId, Func<FunctionContext, HttpContext, Task> answer)
|
||||
{
|
||||
var http = new DefaultHttpContext();
|
||||
http.Response.Body = new MemoryStream();
|
||||
if (inboundId != null)
|
||||
http.Request.Headers[HeaderName] = inboundId;
|
||||
var items = new Dictionary<object, object> { ["HttpRequestContext"] = http };
|
||||
var context = new Mock<FunctionContext>();
|
||||
context.SetupGet(c => c.Items).Returns(items);
|
||||
|
||||
var sut = new CorrelationIdMiddleware(NullLogger<CorrelationIdMiddleware>.Instance);
|
||||
await sut.Invoke(context.Object, ctx => answer(ctx, http));
|
||||
return new Run { Http = http, Items = items };
|
||||
}
|
||||
|
||||
/// <summary>A function's answer: status and JSON body on the shared response.</summary>
|
||||
private static Func<FunctionContext, HttpContext, Task> FunctionAnswers(HttpStatusCode status) => (_, http) =>
|
||||
{
|
||||
http.Response.StatusCode = (int)status;
|
||||
http.Response.ContentType = "application/json";
|
||||
return http.Response.WriteAsync("{\"success\":false}");
|
||||
};
|
||||
|
||||
// ── A valid inbound id is echoed on every status ─────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.OK)] // ingest 2xx
|
||||
[InlineData(HttpStatusCode.Unauthorized)] // ValidateSecurityAsync → ResponseHelper.ErrorAsync
|
||||
[InlineData(HttpStatusCode.Forbidden)] // the same, plus RegisterSession's and GetDiagnosticsUploadUrl's own 403 bodies
|
||||
public async Task A_valid_inbound_id_is_echoed_whatever_status_the_function_answers_with(HttpStatusCode status)
|
||||
{
|
||||
var run = await InvokeAsync(AgentCorrelationId, FunctionAnswers(status));
|
||||
|
||||
Assert.Equal((int)status, run.Http.Response.StatusCode);
|
||||
Assert.Equal(AgentCorrelationId, run.EchoedId);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.Unauthorized)]
|
||||
[InlineData(HttpStatusCode.Forbidden)]
|
||||
public async Task A_valid_inbound_id_is_echoed_on_a_middleware_error_envelope_and_in_its_body(HttpStatusCode status)
|
||||
{
|
||||
var run = await InvokeAsync(AgentCorrelationId, (ctx, http) => ApiErrorWriter.WriteAsync(
|
||||
http, ctx.GetCorrelationId(), status, ApiErrorWriter.DefaultCode(status), "Request rejected."));
|
||||
|
||||
Assert.Equal((int)status, run.Http.Response.StatusCode);
|
||||
Assert.Equal(AgentCorrelationId, run.EchoedId);
|
||||
using var body = JsonDocument.Parse(run.Body());
|
||||
Assert.Equal(AgentCorrelationId, body.RootElement.GetProperty("correlationId").GetString());
|
||||
}
|
||||
|
||||
public static TheoryData<string> InsideTheAllowList => new()
|
||||
{
|
||||
AgentCorrelationId,
|
||||
"Az_09-mixed",
|
||||
new string('a', 128),
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(InsideTheAllowList))]
|
||||
public async Task Every_id_inside_the_allow_list_is_echoed_verbatim(string inboundId)
|
||||
{
|
||||
var run = await InvokeAsync(inboundId, FunctionAnswers(HttpStatusCode.Forbidden));
|
||||
|
||||
Assert.Equal(inboundId, run.EchoedId);
|
||||
Assert.Equal(inboundId, run.Items["CorrelationId"]);
|
||||
}
|
||||
|
||||
// ── Anything else is replaced, never echoed ──────────────────────────────────
|
||||
|
||||
public static TheoryData<string> OutsideTheAllowList => new()
|
||||
{
|
||||
"",
|
||||
"7c9e6679 a1b2",
|
||||
"7c9e6679\r\nX-Forged: 1",
|
||||
"7c9e6679.a1b2",
|
||||
"7c9e6679-ä",
|
||||
new string('a', 129),
|
||||
};
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(OutsideTheAllowList))]
|
||||
public async Task An_inbound_id_outside_the_allow_list_is_replaced_not_echoed(string inboundId)
|
||||
{
|
||||
var run = await InvokeAsync(inboundId, FunctionAnswers(HttpStatusCode.Forbidden));
|
||||
|
||||
Assert.NotEqual(inboundId, run.EchoedId);
|
||||
AssertFreshId(run);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_request_without_an_id_gets_a_fresh_one()
|
||||
{
|
||||
var run = await InvokeAsync(null, FunctionAnswers(HttpStatusCode.OK));
|
||||
|
||||
AssertFreshId(run);
|
||||
}
|
||||
|
||||
/// <summary>A minted id is a compact GUID, and the request's logs and error bodies carry the same value.</summary>
|
||||
private static void AssertFreshId(Run run)
|
||||
{
|
||||
Assert.Matches("^[0-9a-f]{32}$", run.EchoedId);
|
||||
Assert.Equal(run.EchoedId, run.Items["CorrelationId"]);
|
||||
}
|
||||
}
|
||||
@@ -454,6 +454,7 @@ public class DistressValidationTests
|
||||
[InlineData(DistressErrorType.ConfigFetchDenied)]
|
||||
[InlineData(DistressErrorType.SessionRegistrationDenied)]
|
||||
[InlineData(DistressErrorType.TpmPssUnsupported)]
|
||||
[InlineData(DistressErrorType.ClientCertificateNotReceived)]
|
||||
public void EnumIsDefined_AllValidErrorTypes_ReturnsTrue(DistressErrorType errorType)
|
||||
{
|
||||
Assert.True(Enum.IsDefined(typeof(DistressErrorType), errorType));
|
||||
@@ -462,7 +463,7 @@ public class DistressValidationTests
|
||||
[Theory]
|
||||
[InlineData(99)]
|
||||
[InlineData(-1)]
|
||||
[InlineData(9)]
|
||||
[InlineData(10)]
|
||||
[InlineData(999)]
|
||||
public void EnumIsDefined_InvalidValues_ReturnsFalse(int rawValue)
|
||||
{
|
||||
@@ -470,11 +471,11 @@ public class DistressValidationTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DistressErrorType_HasExactly9Values()
|
||||
public void DistressErrorType_HasExactly10Values()
|
||||
{
|
||||
// If someone adds a new enum value, this test forces them to also add test coverage
|
||||
var values = Enum.GetValues(typeof(DistressErrorType));
|
||||
Assert.Equal(9, values.Length);
|
||||
Assert.Equal(10, values.Length);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
|
||||
@@ -64,6 +64,29 @@ public class MiddlewarePipelineOrderTests
|
||||
Assert.Equal(1, Count(src, $"UseMiddleware<{nameof(PolicyEnforcementMiddleware)}>()"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Correlation_id_is_stamped_before_every_middleware_that_can_answer_early()
|
||||
{
|
||||
// The agent treats a 401/403 as this backend's verdict only when it echoes the X-Correlation-ID
|
||||
// the agent sent (CorrelationIdMiddlewareTests). A step that answers before the stamp would
|
||||
// send an error the agent cannot attribute to this backend.
|
||||
var src = ProgramSource();
|
||||
|
||||
var correlation = src.IndexOf($"UseMiddleware<{nameof(CorrelationIdMiddleware)}>()", System.StringComparison.Ordinal);
|
||||
Assert.True(correlation >= 0, "CorrelationIdMiddleware is not registered in Program.cs");
|
||||
Assert.Equal(1, Count(src, $"UseMiddleware<{nameof(CorrelationIdMiddleware)}>()"));
|
||||
|
||||
foreach (var answering in new[]
|
||||
{
|
||||
nameof(GlobalExceptionMiddleware), nameof(AuthenticationMiddleware), nameof(ClientAppBindingMiddleware),
|
||||
nameof(PolicyEnforcementMiddleware), nameof(UserRateLimitMiddleware), nameof(McpQuotaEnforcementMiddleware),
|
||||
})
|
||||
{
|
||||
var at = src.IndexOf($"UseMiddleware<{answering}>()", System.StringComparison.Ordinal);
|
||||
Assert.True(at < 0 || correlation < at, $"CorrelationIdMiddleware must run before {answering}");
|
||||
}
|
||||
}
|
||||
|
||||
private static int Count(string haystack, string needle)
|
||||
{
|
||||
var count = 0;
|
||||
|
||||
@@ -342,7 +342,7 @@ public class ReportAgentErrorFunctionTests
|
||||
await h.RunAsync(report, Read(null));
|
||||
|
||||
var details = Assert.Single(h.OpsEvents).Details;
|
||||
Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3", details);
|
||||
Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3,\"interceptedRuns\":2", details);
|
||||
Assert.Contains("\"networkLinkUpAtEnd\":false", details);
|
||||
Assert.Contains("\"attemptWindowSeconds\":1204.5", details);
|
||||
Assert.Contains("\"configFetchOutcome\":\"UsedDefaults\"", details);
|
||||
@@ -360,7 +360,8 @@ public class ReportAgentErrorFunctionTests
|
||||
|
||||
h.SessionRepo.Verify(r => r.StoreEventsBatchAsync(
|
||||
It.Is<List<EnrollmentEvent>>(e =>
|
||||
((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3))), Times.Once);
|
||||
((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3)
|
||||
&& ((Dictionary<string, object?>)e[0].Data["priorRegistrationFailure"])["interceptedRuns"]!.Equals(2))), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -394,6 +395,20 @@ public class ReportAgentErrorFunctionTests
|
||||
Assert.NotSame(raw, clean);
|
||||
}
|
||||
|
||||
// Intercepted runs are a subset of the failed runs; agents before the field send none (0).
|
||||
[Theory]
|
||||
[InlineData(-1, 0)]
|
||||
[InlineData(0, 0)]
|
||||
[InlineData(2, 2)]
|
||||
[InlineData(7, 3)]
|
||||
public void SanitizeRegistrationFailure_clamps_intercepted_runs_to_the_failed_runs(int interceptedRuns, int expected)
|
||||
{
|
||||
var raw = Failure();
|
||||
raw.InterceptedRuns = interceptedRuns;
|
||||
|
||||
Assert.Equal(expected, ReportAgentErrorFunction.SanitizeRegistrationFailure(raw)!.InterceptedRuns);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0)]
|
||||
[InlineData(-1)]
|
||||
@@ -409,6 +424,7 @@ public class ReportAgentErrorFunctionTests
|
||||
private static RegistrationFailureSummary Failure() => new()
|
||||
{
|
||||
FailedRuns = 3,
|
||||
InterceptedRuns = 2,
|
||||
FirstFailedAtUtc = Now.AddDays(-20),
|
||||
LastFailedAtUtc = Now.AddDays(-19),
|
||||
Outcome = "Failed",
|
||||
|
||||
@@ -196,6 +196,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
["AgentTimestamp"] = report.Timestamp.ToString("O"),
|
||||
["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationFailedRuns"] = priorRegistration?.FailedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationInterceptedRuns"] = priorRegistration?.InterceptedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty,
|
||||
["PriorRegistrationOutcome"] = priorRegistration?.Outcome ?? string.Empty,
|
||||
["PriorRegistrationLastError"] = priorRegistration?.LastError ?? string.Empty,
|
||||
["PriorRegistrationLinkUpAtStart"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtStart),
|
||||
@@ -486,7 +487,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
/// <summary>
|
||||
/// Bounds the device-supplied failed-registration record before it reaches App Insights, the
|
||||
/// ops feed or a timeline: strings are truncated, a record without a positive run count is
|
||||
/// dropped, negative windows are clamped. Returns a copy; null in, null out.
|
||||
/// dropped, negative windows are clamped, the intercepted runs are clamped to 0..FailedRuns
|
||||
/// (older agents omit them: 0). Returns a copy; null in, null out.
|
||||
/// </summary>
|
||||
internal static RegistrationFailureSummary? SanitizeRegistrationFailure(RegistrationFailureSummary? source)
|
||||
{
|
||||
@@ -494,6 +496,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
return new RegistrationFailureSummary
|
||||
{
|
||||
FailedRuns = source.FailedRuns,
|
||||
InterceptedRuns = Math.Clamp(source.InterceptedRuns, 0, source.FailedRuns),
|
||||
FirstFailedAtUtc = source.FirstFailedAtUtc,
|
||||
LastFailedAtUtc = source.LastFailedAtUtc,
|
||||
Outcome = Truncate(source.Outcome, MaxRegistrationTokenLength) ?? string.Empty,
|
||||
@@ -548,6 +551,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest
|
||||
data["priorRegistrationFailure"] = new Dictionary<string, object?>
|
||||
{
|
||||
["failedRuns"] = prior.FailedRuns,
|
||||
["interceptedRuns"] = prior.InterceptedRuns,
|
||||
["firstFailedAtUtc"] = prior.FirstFailedAtUtc.ToString("o"),
|
||||
["lastFailedAtUtc"] = prior.LastFailedAtUtc.ToString("o"),
|
||||
["outcome"] = prior.Outcome,
|
||||
|
||||
@@ -36,7 +36,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
public class AgentConfigResolver
|
||||
{
|
||||
/// <summary>Bumped when the wire shape gains a field the agent must understand.</summary>
|
||||
public const int ConfigVersion = 40; // EnableDoGroupIdAutoSet (Delivery Optimization group ID from network fingerprint)
|
||||
public const int ConfigVersion = 41; // InterceptedRegistrationRetryBoots (boots after an intercepted registration)
|
||||
|
||||
/// <summary>
|
||||
/// Served as <see cref="AgentConfigResponse.InterceptedRegistrationRetryBoots"/>. A code
|
||||
/// constant, no tenant or operator setting: changing it is a backend deploy.
|
||||
/// </summary>
|
||||
public const int DefaultInterceptedRegistrationRetryBoots = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The agent line whose hash oracle is served when no X-Agent-Version is available
|
||||
@@ -127,6 +133,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
EnableEspContinueAnywayObservation = tenantConfig.EnableEspContinueAnywayObservation ?? false,
|
||||
MaxAuthFailures = tenantConfig.MaxAuthFailures ?? 5,
|
||||
AuthFailureTimeoutMinutes = tenantConfig.AuthFailureTimeoutMinutes ?? 0,
|
||||
InterceptedRegistrationRetryBoots = DefaultInterceptedRegistrationRetryBoots,
|
||||
LogLevel = tenantConfig.LogLevel ?? "Info",
|
||||
RebootOnComplete = tenantConfig.RebootOnComplete ?? false,
|
||||
RebootDelaySeconds = tenantConfig.RebootDelaySeconds ?? 10,
|
||||
|
||||
@@ -1120,6 +1120,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
: new
|
||||
{
|
||||
failedRuns = failure.FailedRuns,
|
||||
interceptedRuns = failure.InterceptedRuns,
|
||||
firstFailedAtUtc = failure.FirstFailedAtUtc,
|
||||
lastFailedAtUtc = failure.LastFailedAtUtc,
|
||||
outcome = failure.Outcome,
|
||||
|
||||
@@ -54,6 +54,40 @@ describe('buildSessionCoverage — clean session', () => {
|
||||
});
|
||||
|
||||
describe('buildSessionCoverage — observation window', () => {
|
||||
it('reports starts that failed to register before the agent finally registered', () => {
|
||||
const c = buildSessionCoverage(SESSION, [
|
||||
ev('agent_started', 5, {
|
||||
priorRegistrationFailedRuns: '2',
|
||||
priorRegistrationInterceptedRuns: 2,
|
||||
priorRegistrationLastOutcome: 'AuthFailed',
|
||||
priorRegistrationNetworkChanged: 'True',
|
||||
}),
|
||||
]);
|
||||
expect(c.window.registrationRetries).toEqual([
|
||||
{ at: at(5), failedRuns: 2, interceptedRuns: 2, lastOutcome: 'AuthFailed', networkChanged: true },
|
||||
]);
|
||||
expect(c.gaps).toEqual([
|
||||
expect.stringContaining(`Before the agent start at ${at(5)}, 2 start(s) of this session failed to register, 2 of them answered by something on the network path instead of the backend; the network differed from the failed start`),
|
||||
]);
|
||||
});
|
||||
|
||||
it('keeps an unknown network flag unknown and ignores starts without failed registrations', () => {
|
||||
const c = buildSessionCoverage(SESSION, [
|
||||
ev('agent_started', 5, { priorRegistrationFailedRuns: 0 }),
|
||||
ev('agent_started', 50, { previousExitType: 'reboot_kill', priorRegistrationFailedRuns: 1, priorRegistrationLastOutcome: 'Failed' }),
|
||||
]);
|
||||
expect(c.window.registrationRetries).toEqual([
|
||||
{ at: at(50), failedRuns: 1, interceptedRuns: null, lastOutcome: 'Failed', networkChanged: null },
|
||||
]);
|
||||
const retryGap = c.gaps.find((g) => g.includes('failed to register')) ?? '';
|
||||
expect(retryGap).not.toContain('network');
|
||||
});
|
||||
|
||||
it('omits registrationRetries when no start reports failed registrations', () => {
|
||||
const c = buildSessionCoverage(SESSION, [ev('agent_started', 5)]);
|
||||
expect(c.window).not.toHaveProperty('registrationRetries');
|
||||
});
|
||||
|
||||
it('flags a missing agent_started', () => {
|
||||
const c = buildSessionCoverage(SESSION, [ev('esp_phase_changed', 10)]);
|
||||
expect(c.window.observedFrom).toBeNull();
|
||||
@@ -237,6 +271,46 @@ describe('buildSessionCoverage — collectors, upload, disk', () => {
|
||||
const c = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('telemetry_upload_poisoned', 1, {})]);
|
||||
expect(c.upload.poisonedItems).toBe(1);
|
||||
});
|
||||
|
||||
it('reports an interception on the network path instead of a backend rejection', () => {
|
||||
const c = buildSessionCoverage(SESSION, [
|
||||
ev('agent_started', 0),
|
||||
ev('telemetry_upload_blocked', 300, {
|
||||
reason: 'client_certificate_required',
|
||||
operation: 'agent/telemetry',
|
||||
statusCode: 403,
|
||||
pageTitle: 'Web App - Unavailable',
|
||||
serverCertIssuer: 'CN=Example Inspection Intermediate CA',
|
||||
firstBlockedAtUtc: at(290),
|
||||
}),
|
||||
]);
|
||||
expect(c.upload.blocked).toBe(true);
|
||||
expect(c.upload.intercepted).toEqual({
|
||||
reason: 'client_certificate_required',
|
||||
firstBlockedAtUtc: at(290),
|
||||
pageTitle: 'Web App - Unavailable',
|
||||
serverCertIssuer: 'CN=Example Inspection Intermediate CA',
|
||||
});
|
||||
expect(c.gaps).toEqual([
|
||||
expect.stringContaining(`Telemetry uploads were intercepted from ${at(290)}: the platform answered "Client Certificate Required"`),
|
||||
]);
|
||||
expect(c.gaps[0]).toContain('issued by "CN=Example Inspection Intermediate CA"');
|
||||
expect(c.gaps.join(' ')).not.toContain('permanent backend rejection');
|
||||
});
|
||||
|
||||
it('names the foreign page and keeps a backend rejection as its own gap', () => {
|
||||
const c = buildSessionCoverage(SESSION, [
|
||||
ev('agent_started', 0),
|
||||
ev('telemetry_upload_blocked', 100, { reason: 'permanent 4xx' }),
|
||||
ev('telemetry_upload_blocked', 200, { reason: 'foreign_response', pageTitle: 'Gateway - Notification' }),
|
||||
]);
|
||||
// No firstBlockedAtUtc on the event: fall back to the event's own timestamp.
|
||||
expect(c.upload.intercepted).toEqual({ reason: 'foreign_response', firstBlockedAtUtc: at(200), pageTitle: 'Gateway - Notification', serverCertIssuer: null });
|
||||
expect(c.gaps).toEqual([
|
||||
expect.stringContaining('blocked by a permanent backend rejection'),
|
||||
expect.stringContaining('something on the network path answered instead of the backend (page "Gateway - Notification")'),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildSessionCoverage — diagnostics package', () => {
|
||||
|
||||
@@ -247,6 +247,8 @@ export interface AgentConfigResponse {
|
||||
maxAuthFailures: number;
|
||||
/** Maximum time in minutes the agent keeps retrying after the first auth failure. 0 = disabled (no time limit, only MaxAuthFailures applies). Default: 0. */
|
||||
authFailureTimeoutMinutes: number;
|
||||
/** How many further boots the agent may try after a session registration was answered by a 401/403 this backend did not author (no echoed X-Correlation-ID: a proxy or the platform answered) before it cleans up like the 48-hour emergency break. 0 = clean up at the first intercepted registration. null = an older backend that does not send the field; the agent keeps its default. The agent clamps the value to 0..5 and keeps it in its config cache, because an intercepted start gets no live config. */
|
||||
interceptedRegistrationRetryBoots?: number;
|
||||
/** Log verbosity level for the agent. "Info" = normal messages, "Debug" = component state/decisions, "Verbose" = per-event tracing, "Trace" = full diagnostic output. Default: "Info" */
|
||||
logLevel: string;
|
||||
/** Whether to reboot the device after enrollment completes (and cleanup/self-destruct). Default: false */
|
||||
|
||||
@@ -49,6 +49,8 @@ export const COVERAGE_EVENT_FIELDS =
|
||||
'data.overwriteRewinds,data.overwriteBytesReprocessed,data.passMaxMs,data.passGapMaxMs,' +
|
||||
'data.collector,data.reason,data.errorType,' +
|
||||
'data.pendingItemCount,data.pendingBytes,data.kind,data.itemCount,data.disk_free_gb,' +
|
||||
'data.pageTitle,data.serverCertIssuer,data.firstBlockedAtUtc,' +
|
||||
'data.priorRegistrationFailedRuns,data.priorRegistrationInterceptedRuns,data.priorRegistrationLastOutcome,data.priorRegistrationNetworkChanged,' +
|
||||
'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind,' +
|
||||
'data.installingNames,data.downloadingNames,' +
|
||||
// Scripts block; scriptType/script_type already ride in SUMMARY_EVENT_FIELDS' own list.
|
||||
@@ -65,6 +67,11 @@ export interface CoverageWindow {
|
||||
agentStarts: number;
|
||||
/** Distinct `previousExitType` values reported by restarts (the first start has none). */
|
||||
previousExitTypes: string[];
|
||||
/**
|
||||
* Starts that registered only after earlier starts of the same session had failed to register
|
||||
* (`agent_started.priorRegistration*`). Those earlier boots observed nothing.
|
||||
*/
|
||||
registrationRetries?: RegistrationRetry[];
|
||||
lateStart: {
|
||||
bootToAgentStartSeconds: number | null;
|
||||
agentUptimeSeconds: number | null;
|
||||
@@ -116,10 +123,40 @@ export interface CollectorDegradation {
|
||||
at: string | null;
|
||||
}
|
||||
|
||||
export interface RegistrationRetry {
|
||||
/** The `agent_started` that finally registered. */
|
||||
at: string | null;
|
||||
failedRuns: number;
|
||||
/** Failed starts whose answer did not come from our backend (platform or proxy page). */
|
||||
interceptedRuns: number | null;
|
||||
lastOutcome: string | null;
|
||||
/** Network fingerprint (connection type + gateway) differed from the last failed start; null = unknown. */
|
||||
networkChanged: boolean | null;
|
||||
}
|
||||
|
||||
/** Answer reasons of `telemetry_upload_blocked` that mean "something on the network path answered, not the backend". */
|
||||
const INTERCEPTION_REASONS: ReadonlySet<string> = new Set(['client_certificate_required', 'foreign_response']);
|
||||
|
||||
export interface UploadInterception {
|
||||
/**
|
||||
* `client_certificate_required`: the platform answered "Client Certificate Required" — the
|
||||
* device certificate did not reach the backend. `foreign_response`: a page from something else
|
||||
* on the path, e.g. a proxy's block page.
|
||||
*/
|
||||
reason: string;
|
||||
/** First such answer, agent clock. */
|
||||
firstBlockedAtUtc: string | null;
|
||||
pageTitle: string | null;
|
||||
/** Issuer of the server certificate the agent saw; a proxy's own CA proves TLS inspection. */
|
||||
serverCertIssuer: string | null;
|
||||
}
|
||||
|
||||
export interface UploadCoverage {
|
||||
spoolPressure: boolean;
|
||||
poisonedItems: number;
|
||||
blocked: boolean;
|
||||
/** Present when the block was an interception on the network path (the event arrives only once the path cleared). */
|
||||
intercepted?: UploadInterception;
|
||||
backpressureEpisodes: number;
|
||||
diskSpaceLow: boolean;
|
||||
}
|
||||
@@ -180,6 +217,13 @@ function bool(v: unknown): boolean {
|
||||
return String(v ?? '').toLowerCase() === 'true';
|
||||
}
|
||||
|
||||
/** true / false / null (absent or unparseable) — for flags where "unknown" must not read as false. */
|
||||
function triBool(v: unknown): boolean | null {
|
||||
if (typeof v === 'boolean') return v;
|
||||
const s = String(v ?? '').trim().toLowerCase();
|
||||
return s === 'true' ? true : s === 'false' ? false : null;
|
||||
}
|
||||
|
||||
function countMap(v: unknown): Record<string, number> | undefined {
|
||||
if (!v || typeof v !== 'object' || Array.isArray(v)) return undefined;
|
||||
const out: Record<string, number> = {};
|
||||
@@ -285,6 +329,27 @@ export function buildSessionCoverage(
|
||||
const exits = previousExitTypes.length > 0 ? ` (previous exit: ${previousExitTypes.join(', ')})` : '';
|
||||
gaps.push(`The agent started ${starts.length} times${exits}: signals during the restart gaps were not observed.`);
|
||||
}
|
||||
// A start that failed to register exits before any collector runs and leaves no event of its
|
||||
// own; the start that finally registers reports the failed ones (agents after 2026-10).
|
||||
const registrationRetries: RegistrationRetry[] = [];
|
||||
for (const s of starts) {
|
||||
const failedRuns = num(s.data?.priorRegistrationFailedRuns);
|
||||
if (failedRuns === null || failedRuns <= 0) continue;
|
||||
registrationRetries.push({
|
||||
at: str(s.timestamp),
|
||||
failedRuns,
|
||||
interceptedRuns: num(s.data?.priorRegistrationInterceptedRuns),
|
||||
lastOutcome: str(s.data?.priorRegistrationLastOutcome),
|
||||
networkChanged: triBool(s.data?.priorRegistrationNetworkChanged),
|
||||
});
|
||||
}
|
||||
for (const r of registrationRetries) {
|
||||
const intercepted = (r.interceptedRuns ?? 0) > 0 ? `, ${r.interceptedRuns} of them answered by something on the network path instead of the backend` : '';
|
||||
const network = r.networkChanged === true ? '; the network differed from the failed start' : r.networkChanged === false ? '; same network as the failed start' : '';
|
||||
gaps.push(
|
||||
`Before the agent start at ${r.at ?? 'an unknown time'}, ${r.failedRuns} start(s) of this session failed to register${intercepted}${network}: nothing was observed during those boots.`,
|
||||
);
|
||||
}
|
||||
|
||||
// ── IME log tracker ─────────────────────────────────────────────────────────
|
||||
const degradedEv = first('ime_tracker_degraded');
|
||||
@@ -377,13 +442,26 @@ export function buildSessionCoverage(
|
||||
const spool = last('spool_pressure_detected');
|
||||
const poisoned = of('telemetry_upload_poisoned');
|
||||
const poisonedItems = poisoned.reduce((n, e) => n + (num(e.data?.itemCount) ?? 1), 0);
|
||||
const blocked = of('telemetry_upload_blocked').length > 0;
|
||||
const blockedEvents = of('telemetry_upload_blocked');
|
||||
const blocked = blockedEvents.length > 0;
|
||||
const isInterception = (e: Ev): boolean => INTERCEPTION_REASONS.has(str(e.data?.reason) ?? '');
|
||||
const interceptedEv = blockedEvents.find(isInterception);
|
||||
const rejectedByBackend = blockedEvents.some((e) => !isInterception(e));
|
||||
const intercepted: UploadInterception | undefined = interceptedEv
|
||||
? {
|
||||
reason: str(interceptedEv.data?.reason) ?? 'foreign_response',
|
||||
firstBlockedAtUtc: str(interceptedEv.data?.firstBlockedAtUtc) ?? str(interceptedEv.timestamp),
|
||||
pageTitle: str(interceptedEv.data?.pageTitle),
|
||||
serverCertIssuer: str(interceptedEv.data?.serverCertIssuer),
|
||||
}
|
||||
: undefined;
|
||||
const backpressureEpisodes = of('ingress_backpressure').length;
|
||||
const disk = last('disk_space_low');
|
||||
const upload: UploadCoverage = {
|
||||
spoolPressure: !!spool,
|
||||
poisonedItems,
|
||||
blocked,
|
||||
...(intercepted ? { intercepted } : {}),
|
||||
backpressureEpisodes,
|
||||
diskSpaceLow: !!disk,
|
||||
};
|
||||
@@ -395,9 +473,19 @@ export function buildSessionCoverage(
|
||||
const kinds = [...new Set(poisoned.map((e) => str(e.data?.kind)).filter((x): x is string => !!x))];
|
||||
gaps.push(`${poisonedItems} telemetry item(s) were quarantined as poison and never uploaded${kinds.length > 0 ? ` (${kinds.join(', ')})` : ''}; they remain in the spool inside the diagnostics ZIP.`);
|
||||
}
|
||||
if (blocked) {
|
||||
if (rejectedByBackend) {
|
||||
gaps.push('Telemetry upload was blocked by a permanent backend rejection at some point; later events reached the backend only once the block cleared.');
|
||||
}
|
||||
if (intercepted) {
|
||||
const issuer = intercepted.serverCertIssuer ? `; the server certificate the agent saw was issued by "${intercepted.serverCertIssuer}"` : '';
|
||||
const what =
|
||||
intercepted.reason === 'client_certificate_required'
|
||||
? 'the platform answered "Client Certificate Required", so the device certificate did not reach the backend (TLS inspection or a proxy on the network path)'
|
||||
: `something on the network path answered instead of the backend${intercepted.pageTitle ? ` (page "${intercepted.pageTitle}")` : ''}, typically a proxy or security gateway`;
|
||||
gaps.push(
|
||||
`Telemetry uploads were intercepted from ${intercepted.firstBlockedAtUtc ?? 'an unknown time'}: ${what}${issuer}. Events from that period reached the backend only after the path cleared.`,
|
||||
);
|
||||
}
|
||||
if (disk) {
|
||||
const free = num(disk.data?.disk_free_gb);
|
||||
gaps.push(`System drive ran low on space${free !== null ? ` (${free} GB free)` : ''}: spool and diagnostics writes may have failed.`);
|
||||
@@ -508,6 +596,7 @@ export function buildSessionCoverage(
|
||||
startedAtBackdatedSeconds,
|
||||
agentStarts: starts.length,
|
||||
previousExitTypes,
|
||||
...(registrationRetries.length > 0 ? { registrationRetries } : {}),
|
||||
lateStart,
|
||||
historicReplaySuppressed,
|
||||
},
|
||||
|
||||
@@ -694,7 +694,7 @@ namespace AutopilotMonitor.Shared
|
||||
public const string CollectorDegraded = "collector_degraded"; // One-shot Warning when a collector/watcher fails to arm (e.g. EventLogWatcher start error). Review MON-D1: without it a dead kernel watcher is indistinguishable from a genuine no-signal session.
|
||||
public const string StateQuarantineRecovered = "state_quarantine_recovered"; // Warning emitted on the start that honours a prior run's quarantine marker (3 consecutive journal-append failures). Review TRACE-H2: without it a corruption-recovery looks like a normal restart on the backend.
|
||||
public const string TelemetryUploadPoisoned = "telemetry_upload_poisoned"; // Warning when the drain quarantined a single oversized item (413 even on its own — the only locally-provable poison) to keep the pipeline flowing. Review TRACE-H1/P1.
|
||||
public const string TelemetryUploadBlocked = "telemetry_upload_blocked"; // Warning (one-shot) when the drain is blocked by a retained unknown permanent 4xx (400 contract bug / 404 route mismatch / tenant-validation). Data is NOT discarded. LOCAL diagnostic (agent log + diag ZIP): this marker is queued behind the blocking batch and only reaches the backend after the block clears — no out-of-band bypass by design.
|
||||
public const string TelemetryUploadBlocked = "telemetry_upload_blocked"; // Warning (one-shot each) when the drain is blocked by a retained unknown permanent 4xx (400 contract bug / 404 route mismatch / tenant-validation), or by an interception on the network path (data.reason client_certificate_required | foreign_response, plus operation, statusCode, pageTitle, serverCertIssuer, firstBlockedAtUtc). Data is NOT discarded. LOCAL diagnostic (agent log + diag ZIP): this marker is queued behind the blocking batch and only reaches the backend after the block clears — no out-of-band bypass by design.
|
||||
public const string SessionParkedWithoutDeadline = "session_parked_without_deadline"; // One-shot Warning tripwire (liveness plan PR1): a non-terminal session entered the post-AccountSetup dead-end zone (ESP final exit or advisory failure) with NO resolution-capable deadline armed. After the 2026-06-12 dead-end fixes this must never fire in production — every occurrence is a bug report for an unknown parking variant.
|
||||
public const string DiskSpaceLow = "disk_space_low"; // One-shot Warning (state-change-only, 1 GB hysteresis — NOT periodic) when the system drive drops below 2 GB free. Review MON-B10: disk-full was previously only visible inside Debug performance_snapshot payloads (filtered out of timelines), so a disk-starved enrollment carried no actionable threshold signal.
|
||||
public const string EspPolicyProviderStalled = "esp_policy_provider_stalled"; // One-shot-per-provider Warning: a registered EnrollmentStatusTracking CSP policy provider stayed incomplete >= 15 min (Setup/Apps TrackingPoliciesCreated != 1, or DevicePreparation InstallationState not in {2,3}). The ESP waits on Setup/Apps providers WITHOUT any OS timeout — e.g. pre-provisioning + co-management leaves only "ConfigMgr" registered (no "Sidecar" = IME) and the user ESP sits at "Apps (Identifying)" for days. Observational only; payload carries the full provider list. See EspPolicyProviderProbe.
|
||||
|
||||
@@ -94,6 +94,16 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public int AuthFailureTimeoutMinutes { get; set; } = 0;
|
||||
|
||||
/// <summary>
|
||||
/// How many further boots the agent may try after a session registration was answered by
|
||||
/// a 401/403 this backend did not author (no echoed <c>X-Correlation-ID</c>: a proxy or
|
||||
/// the platform answered) before it cleans up like the 48-hour emergency break.
|
||||
/// 0 = clean up at the first intercepted registration. null = an older backend that does
|
||||
/// not send the field; the agent keeps its default. The agent clamps the value to 0..5 and
|
||||
/// keeps it in its config cache, because an intercepted start gets no live config.
|
||||
/// </summary>
|
||||
public int? InterceptedRegistrationRetryBoots { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Log verbosity level for the agent.
|
||||
/// "Info" = normal messages, "Debug" = component state/decisions, "Verbose" = per-event tracing, "Trace" = full diagnostic output.
|
||||
|
||||
@@ -42,6 +42,16 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// firmware update or device replacement, not an agent change.
|
||||
/// </summary>
|
||||
TpmPssUnsupported = 8,
|
||||
|
||||
/// <summary>
|
||||
/// The agent got a 401/403 this backend did not author: the response does not echo the
|
||||
/// request's <c>X-Correlation-ID</c>. It is the platform's "Client Certificate Required"
|
||||
/// page or a proxy's own page, so the device certificate never reached the backend —
|
||||
/// typically TLS inspection on the network path. <see cref="DistressReport.Message"/>
|
||||
/// carries the operation, the status, the page title and the issuer of the server
|
||||
/// certificate the agent saw.
|
||||
/// </summary>
|
||||
ClientCertificateNotReceived = 9,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -14,6 +14,13 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// <summary>Process starts whose registration failed since the last success.</summary>
|
||||
public int FailedRuns { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Of <see cref="FailedRuns"/>, the starts whose registration was answered by a 401/403 that
|
||||
/// something on the network path authored (TLS inspection, a proxy page) rather than the
|
||||
/// backend. The agent gives up and cleans up once it exceeds the configured buffer.
|
||||
/// </summary>
|
||||
public int InterceptedRuns { get; set; }
|
||||
|
||||
/// <summary>UTC time the first of these runs gave up.</summary>
|
||||
public DateTime FirstFailedAtUtc { get; set; }
|
||||
|
||||
|
||||
@@ -27,6 +27,8 @@ const ERROR_TYPE_LABELS: Record<string, { label: string; color: string }> = {
|
||||
TenantRejected: { label: "Tenant Rejected", color: "bg-purple-100 text-purple-800 dark:bg-purple-900/40 dark:text-purple-300" },
|
||||
ConfigFetchDenied: { label: "Config Denied", color: "bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300" },
|
||||
SessionRegistrationDenied: { label: "Session Denied", color: "bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300" },
|
||||
TpmPssUnsupported: { label: "TPM Incompatible", color: "bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300" },
|
||||
ClientCertificateNotReceived: { label: "Cert Not Received", color: "bg-orange-100 text-orange-800 dark:bg-orange-900/40 dark:text-orange-300" },
|
||||
};
|
||||
|
||||
function ErrorTypeBadge({ errorType }: { errorType: string }) {
|
||||
|
||||
@@ -271,6 +271,8 @@ export const RUNTIME_FIELDS: Record<keyof AgentConfigResponse, RuntimeRow | Excl
|
||||
|
||||
maxAuthFailures: { section: "Auth Circuit Breaker", label: "Max Auth Failures", source: "tenant" },
|
||||
authFailureTimeoutMinutes: { section: "Auth Circuit Breaker", label: "Auth Failure Timeout (min)", source: "tenant" },
|
||||
// Backend constant; its C# initializer is null (an older backend sends nothing), so never "custom".
|
||||
interceptedRegistrationRetryBoots: { section: "Auth Circuit Breaker", label: "Intercepted Registration Retry Boots", source: "fixed", informational: true },
|
||||
|
||||
showEnrollmentSummary: { section: "Enrollment Summary", label: "Show Summary", source: "tenant" },
|
||||
enrollmentSummaryTimeoutSeconds: { section: "Enrollment Summary", label: "Timeout (sec)", source: "tenant" },
|
||||
|
||||
@@ -351,6 +351,7 @@ export const SHARED_MANIFEST = {
|
||||
"whiteGloveSealingPatternIds": [],
|
||||
"maxAuthFailures": 5,
|
||||
"authFailureTimeoutMinutes": 0,
|
||||
"interceptedRegistrationRetryBoots": null,
|
||||
"logLevel": "Info",
|
||||
"rebootOnComplete": false,
|
||||
"rebootDelaySeconds": 10,
|
||||
|
||||
@@ -347,6 +347,7 @@
|
||||
"whiteGloveSealingPatternIds": [],
|
||||
"maxAuthFailures": 5,
|
||||
"authFailureTimeoutMinutes": 0,
|
||||
"interceptedRegistrationRetryBoots": null,
|
||||
"logLevel": "Info",
|
||||
"rebootOnComplete": false,
|
||||
"rebootDelaySeconds": 10,
|
||||
@@ -1951,6 +1952,15 @@
|
||||
},
|
||||
"doc": "Maximum time in minutes the agent keeps retrying after the first auth failure. 0 = disabled (no time limit, only MaxAuthFailures applies). Default: 0."
|
||||
},
|
||||
{
|
||||
"name": "interceptedRegistrationRetryBoots",
|
||||
"type": {
|
||||
"kind": "primitive",
|
||||
"name": "number"
|
||||
},
|
||||
"optional": true,
|
||||
"doc": "How many further boots the agent may try after a session registration was answered by a 401/403 this backend did not author (no echoed X-Correlation-ID: a proxy or the platform answered) before it cleans up like the 48-hour emergency break. 0 = clean up at the first intercepted registration. null = an older backend that does not send the field; the agent keeps its default. The agent clamps the value to 0..5 and keeps it in its config cache, because an intercepted start gets no live config."
|
||||
},
|
||||
{
|
||||
"name": "logLevel",
|
||||
"type": {
|
||||
|
||||
@@ -247,6 +247,8 @@ export interface AgentConfigResponse {
|
||||
maxAuthFailures: number;
|
||||
/** Maximum time in minutes the agent keeps retrying after the first auth failure. 0 = disabled (no time limit, only MaxAuthFailures applies). Default: 0. */
|
||||
authFailureTimeoutMinutes: number;
|
||||
/** How many further boots the agent may try after a session registration was answered by a 401/403 this backend did not author (no echoed X-Correlation-ID: a proxy or the platform answered) before it cleans up like the 48-hour emergency break. 0 = clean up at the first intercepted registration. null = an older backend that does not send the field; the agent keeps its default. The agent clamps the value to 0..5 and keeps it in its config cache, because an intercepted start gets no live config. */
|
||||
interceptedRegistrationRetryBoots?: number;
|
||||
/** Log verbosity level for the agent. "Info" = normal messages, "Debug" = component state/decisions, "Verbose" = per-event tracing, "Trace" = full diagnostic output. Default: "Info" */
|
||||
logLevel: string;
|
||||
/** Whether to reboot the device after enrollment completes (and cleanup/self-destruct). Default: false */
|
||||
|
||||
Reference in New Issue
Block a user