diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerCompletenessTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerCompletenessTests.cs index 1185ecc3..b1d0ed6c 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerCompletenessTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerCompletenessTests.cs @@ -50,6 +50,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration nameof(AgentConfigResponse.MaxBatchSize), nameof(AgentConfigResponse.MaxAuthFailures), nameof(AgentConfigResponse.AuthFailureTimeoutMinutes), + nameof(AgentConfigResponse.InterceptedRegistrationRetryBoots), // Merge reads two Collectors knobs (AgentMaxLifetimeMinutes, HelloWaitTimeoutSeconds); // the remaining CollectorConfiguration flags flow untouched to DefaultComponentFactory. nameof(AgentConfigResponse.Collectors), diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerTests.cs index 8fee05d6..5aae2acf 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigMergerTests.cs @@ -55,6 +55,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration UnrestrictedMode = true, MaxAuthFailures = 7, AuthFailureTimeoutMinutes = 15, + InterceptedRegistrationRetryBoots = 2, EnableImeMatchLog = true, Collectors = new CollectorConfiguration { @@ -98,6 +99,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration Assert.Equal(45, agent.HelloWaitTimeoutSeconds); Assert.Equal(7, agent.MaxAuthFailures); Assert.Equal(15, agent.AuthFailureTimeoutMinutes); + Assert.Equal(2, agent.InterceptedRegistrationRetryBoots); Assert.NotNull(result); Assert.True(result.UnrestrictedModeChanged); @@ -468,6 +470,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration Assert.Equal(0, agent.AgentMaxLifetimeMinutes); } + // ============================================================= Intercepted-registration buffer + + [Theory] + [InlineData(0)] // clean up at the first intercepted registration + [InlineData(3)] + public void Merge_copies_intercepted_registration_retry_boots(int remoteValue) + { + var agent = NewAgentConfig(); + var remote = FullRemote(); + remote.InterceptedRegistrationRetryBoots = remoteValue; + + RemoteConfigMerger.Merge(agent, remote); + + Assert.Equal(remoteValue, agent.InterceptedRegistrationRetryBoots); + } + + [Fact] + public void Merge_keeps_the_agent_default_when_the_backend_sends_no_intercepted_registration_retry_boots() + { + var agent = NewAgentConfig(); + var remote = FullRemote(); + remote.InterceptedRegistrationRetryBoots = null; // backend that predates the field + + RemoteConfigMerger.Merge(agent, remote); + + Assert.Equal(1, agent.InterceptedRegistrationRetryBoots); + } + + [Theory] + [InlineData(-1, 0)] + [InlineData(int.MinValue, 0)] + [InlineData(6, 5)] + [InlineData(int.MaxValue, 5)] + public void Merge_clamps_intercepted_registration_retry_boots_to_zero_through_five(int remoteValue, int expected) + { + var agent = NewAgentConfig(); + var remote = FullRemote(); + remote.InterceptedRegistrationRetryBoots = remoteValue; + + RemoteConfigMerger.Merge(agent, remote); + + Assert.Equal(expected, agent.InterceptedRegistrationRetryBoots); + } + // ============================================================= Guard [Fact] diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceCacheHardeningTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceCacheHardeningTests.cs index d2f08227..9b043874 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceCacheHardeningTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceCacheHardeningTests.cs @@ -214,6 +214,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration Assert.Null(loaded.LatestAgentSha256); } + [Theory] + [InlineData(0)] + [InlineData(3)] + [InlineData(null)] // backend that predates the field — the agent keeps its default + public void RoundTrip_preserves_intercepted_registration_retry_boots(int? value) + { + // An intercepted start gets no live config, so the registration buffer it decides on + // is the cached value: it must survive the write + read strip untouched. + var probe = NewProbe(); + + probe.DoCache(new AgentConfigResponse { InterceptedRegistrationRetryBoots = value }); + var loaded = probe.DoLoad(); + + Assert.NotNull(loaded); + Assert.Equal(value, loaded.InterceptedRegistrationRetryBoots); + } + // ── Helpers ───────────────────────────────────────────────────────── private CacheProbe NewProbe() diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs index 7ffc6894..74fd4c11 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs @@ -7,6 +7,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; using AutopilotMonitor.Shared.Models; using Xunit; @@ -101,6 +102,28 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration Assert.Equal(401, svc.LastFetchAuthStatusCode); } + [Fact] + public async Task FetchConfig_InterceptedAnswer_FallsBack_WithoutCounting_AndReportsOnce() + { + // A 403 something on the network path answered (TLS inspection): same fallback as an + // auth failure, but the tracker does not count it — it sends ClientCertificateNotReceived. + var distress = new FakeDistressReporter(); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, + AutopilotMonitor.DecisionCore.Engine.SystemClock.Instance, _logger, distress); + var svc = NewService(BackendBehaviour.AlwaysIntercepted403, tracker); + + var result = await svc.FetchConfigAsync(retryOnTransientErrors: true); + + Assert.NotNull(result); + Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome); + Assert.Equal(1, svc.LastFetchAttempts); + Assert.Equal(403, svc.LastFetchAuthStatusCode); + Assert.Equal(0, tracker.ConsecutiveFailures); + var call = Assert.Single(distress.Calls); + Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType); + Assert.StartsWith("agent/config http 403 client_certificate_required", call.Message); + } + [Fact] public async Task FetchConfig_RetryDisabled_OneAttemptOnly() { @@ -252,10 +275,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration /// All 4 initial attempts fail; the NEXT call (the recovery fetch) succeeds. FailAllThenSucceed, AlwaysAuthFailure401, + AlwaysIntercepted403, AlwaysReturnNull, } - private FakeRemoteConfigService NewService(BackendBehaviour behaviour) + private FakeRemoteConfigService NewService(BackendBehaviour behaviour, AuthFailureTracker? tracker = null) { // Pass a throwaway BackendApiClient — the fake overrides CallBackendAsync // before it ever hits the wire. @@ -269,7 +293,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration bootstrapToken: null, agentVersion: "0.0.0", logger: _logger); - return new FakeRemoteConfigService(apiClient, _logger, behaviour); + return new FakeRemoteConfigService(apiClient, _logger, behaviour, tracker); } private sealed class FakeRemoteConfigService : RemoteConfigService @@ -277,8 +301,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration private readonly BackendBehaviour _behaviour; private int _calls; - public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour) - : base(apiClient, "00000000-0000-0000-0000-000000000001", logger) + public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour, AuthFailureTracker? tracker) + : base(apiClient, "00000000-0000-0000-0000-000000000001", logger, authFailureTracker: tracker) { _behaviour = behaviour; } @@ -305,6 +329,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration throw new HttpRequestException("simulated network failure"); case BackendBehaviour.AlwaysAuthFailure401: throw new BackendAuthException("Unauthorized", 401); + case BackendBehaviour.AlwaysIntercepted403: + throw new BackendAuthException("The device certificate did not reach the backend.", 403, + classification: InterceptionFixtures.CertificateRequired()); case BackendBehaviour.AlwaysReturnNull: return Task.FromResult(null!); default: diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Harness/FakeDistressReporter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Harness/FakeDistressReporter.cs new file mode 100644 index 00000000..2e7e9642 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Harness/FakeDistressReporter.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Shared.Models; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Harness +{ + /// + /// Records every distress dispatch instead of sending it. The real anti-flood rules of + /// are bypassed on purpose: tests assert what the caller dispatched. + /// + internal sealed class FakeDistressReporter : DistressReporter + { + public sealed class Call + { + public DistressErrorType ErrorType { get; set; } + public string Message { get; set; } = default!; + public int? HttpStatusCode { get; set; } + } + + public List Calls { get; } = new List(); + + /// + /// The send each dispatch returns; null = already completed. Lets a test keep a send in + /// flight to prove a caller waits for it (or stops waiting at its bound). + /// + public Func? Send { get; set; } + + public FakeDistressReporter() + : base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!) + { + } + + public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null) + { + Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode }); + return Send?.Invoke() ?? Task.CompletedTask; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/NetworkFingerprintTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/NetworkFingerprintTests.cs new file mode 100644 index 00000000..fb8a83e2 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/NetworkFingerprintTests.cs @@ -0,0 +1,85 @@ +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Monitoring; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring +{ + /// + /// The local network fingerprint a failed registration is stored with: SHA-256 over the + /// connection type and gateway captures, cut to 16 hex + /// characters; unknown (null) whenever either input is missing. Gateways are documentation + /// addresses (RFC 5737). + /// + public sealed class NetworkFingerprintTests + { + private static NetworkChangeDetector.NetworkStateSnapshot State(string? connectionType, string? gateway, bool hasNetwork = true) => + new NetworkChangeDetector.NetworkStateSnapshot + { + HasNetwork = hasNetwork, + ConnectionType = connectionType, + Gateway = gateway, + }; + + [Fact] + public void Same_connection_type_and_gateway_give_the_same_short_hex_fingerprint() + { + var first = NetworkFingerprint.Of(State("WiFi", "192.0.2.1")); + var second = NetworkFingerprint.Of(State("WiFi", "192.0.2.1")); + + Assert.Equal(first, second); + Assert.Equal(NetworkFingerprint.Length, first!.Length); + Assert.True(first.All(c => (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')), first); + } + + [Theory] + [InlineData("WiFi", "198.51.100.1")] // another gateway + [InlineData("Ethernet", "192.0.2.1")] // another connection type + public void Another_gateway_or_connection_type_is_another_network(string connectionType, string gateway) + { + Assert.NotEqual( + NetworkFingerprint.Of(State("WiFi", "192.0.2.1")), + NetworkFingerprint.Of(State(connectionType, gateway))); + } + + [Fact] + public void Unknown_network_has_no_fingerprint() + { + Assert.Null(NetworkFingerprint.Of(null)); + Assert.Null(NetworkFingerprint.Of(State("WiFi", "192.0.2.1", hasNetwork: false))); + Assert.Null(NetworkFingerprint.Of(State("WiFi", null))); + Assert.Null(NetworkFingerprint.Of(State(null, "192.0.2.1"))); + } + + [Theory] + [InlineData("aaaa", "aaaa", false)] + [InlineData("aaaa", "bbbb", true)] + public void Differs_compares_known_fingerprints(string before, string after, bool expected) + { + Assert.Equal(expected, NetworkFingerprint.Differs(before, after)); + } + + [Theory] + [InlineData(null, "aaaa")] + [InlineData("aaaa", null)] + [InlineData(null, null)] + public void Differs_is_unknown_when_either_side_is(string? before, string? after) + { + Assert.Null(NetworkFingerprint.Differs(before, after)); + } + + [Fact] + public void TryCapture_reads_this_machine_without_throwing() + { + // Machine-dependent: either no active interface (null) or a well-formed fingerprint + // that matches the detector's own capture. + var captured = NetworkFingerprint.TryCapture(); + + if (captured != null) + { + Assert.Equal(NetworkFingerprint.Length, captured.Length); + Assert.Equal(NetworkFingerprint.Of(NetworkChangeDetector.CaptureInterfaceState()), captured); + } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/EnrollmentOrchestratorInterceptionTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/EnrollmentOrchestratorInterceptionTests.cs new file mode 100644 index 00000000..39c3e827 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/EnrollmentOrchestratorInterceptionTests.cs @@ -0,0 +1,132 @@ +#nullable enable +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; +using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry; +using AutopilotMonitor.Shared.Models; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration +{ + /// + /// The orchestrator side of an intercepted upload: one telemetry_upload_blocked with the + /// evidence (queued behind the intercepted batch, never uploaded while paused), and the one + /// terminal attempt — requested by DrainSpool or made by . + /// + [Collection("SerialThreading")] + public sealed class EnrollmentOrchestratorInterceptionTests + { + private const int WaitMs = 10_000; + + private static TelemetryItemDraft Draft() => + new TelemetryItemDraft( + kind: TelemetryItemKind.Event, + partitionKey: "T1_S1", + rowKey: "0000000001", + payloadJson: "{}", + isSessionScoped: true, + requiresImmediateFlush: false); + + // The signal and the decision transition of the same post mention the event type too. + private static bool IsBlockedEvent(TelemetryItem item) => + item.Kind == TelemetryItemKind.Event + && item.PayloadJson.Contains(AutopilotMonitor.Shared.Constants.EventTypes.TelemetryUploadBlocked); + + private static TelemetryItem? FindBlockedEvent(EnrollmentOrchestrator sut) => + sut.SpoolForTests!.Peek(1000).FirstOrDefault(IsBlockedEvent); + + // Dates stay strings: the wire format of firstBlockedAtUtc is what is under test. + private static JObject Parse(string json) + { + using var reader = new JsonTextReader(new StringReader(json)) { DateParseHandling = DateParseHandling.None }; + return JObject.Load(reader); + } + + /// Starts the orchestrator and drives the first upload into an interception. + private static async Task StartInterceptedAsync(EnrollmentOrchestratorRig rig) + { + rig.Uploader.QueueIntercepted(5); + var sut = rig.Build(); + sut.Start(); + sut.SpoolForTests!.Enqueue(Draft()); + await sut.DrainAsync(); + Assert.True(sut.Transport.IsInterceptionPaused); + return sut; + } + + [Fact] + public async Task Interception_queues_one_telemetry_upload_blocked_with_the_evidence_and_wakeups_stay_offline() + { + using var rig = new EnrollmentOrchestratorRig(); + var sut = await StartInterceptedAsync(rig); + try + { + TelemetryItem? blocked = null; + Assert.True(SpinWait.SpinUntil(() => (blocked = FindBlockedEvent(sut)) != null, WaitMs), + "telemetry_upload_blocked never reached the spool."); + var calls = rig.Uploader.CallCount; + + var data = (JObject)Parse(blocked!.PayloadJson)["Data"]!; + Assert.Equal( + new[] { "firstBlockedAtUtc", "operation", "pageTitle", "reason", "serverCertIssuer", "statusCode" }, + data.Properties().Select(p => p.Name).OrderBy(n => n, StringComparer.Ordinal)); + Assert.Equal("client_certificate_required", (string?)data["reason"]); + Assert.Equal("agent/telemetry", (string?)data["operation"]); + Assert.Equal(JTokenType.Integer, data["statusCode"]!.Type); + Assert.Equal(403, (int)data["statusCode"]!); + Assert.Equal(InterceptionFixtures.PlatformPageTitle, (string?)data["pageTitle"]); + Assert.Equal(InterceptionFixtures.InspectionIssuer, (string?)data["serverCertIssuer"]); + Assert.Equal( + EnrollmentOrchestratorRig.DefaultClockStart.ToString("o", CultureInfo.InvariantCulture), + (string?)data["firstBlockedAtUtc"]); + + // The marker was an immediate-upload item; neither its wakeup nor a further drain + // touches the network while the pause holds. + var drain = await sut.DrainAsync(); + Assert.Contains("upload intercepted", drain.LastErrorReason); + Assert.Equal(calls, rig.Uploader.CallCount); + Assert.Single(sut.SpoolForTests!.Peek(1000), IsBlockedEvent); + } + finally + { + sut.Stop(); + } + } + + [Fact] + public async Task Terminal_attempt_requested_by_the_termination_runs_once_and_Stop_adds_none() + { + using var rig = new EnrollmentOrchestratorRig(); + var sut = await StartInterceptedAsync(rig); + Assert.True(SpinWait.SpinUntil(() => FindBlockedEvent(sut) != null, WaitMs)); + var calls = rig.Uploader.CallCount; + + sut.RequestTerminalUploadAttempt(); + Assert.True(SpinWait.SpinUntil(() => rig.Uploader.CallCount == calls + 1, WaitMs), + "the woken drain loop did not make the terminal attempt."); + + sut.Stop(); + Assert.Equal(calls + 1, rig.Uploader.CallCount); + } + + [Fact] + public async Task Stop_makes_the_one_terminal_attempt_when_the_termination_did_not() + { + using var rig = new EnrollmentOrchestratorRig(); + var sut = await StartInterceptedAsync(rig); + var calls = rig.Uploader.CallCount; + + sut.Stop(); + + Assert.Equal(calls + 1, rig.Uploader.CallCount); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/MtlsHttpClientFactoryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/MtlsHttpClientFactoryTests.cs index c58746e9..6930a5be 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/MtlsHttpClientFactoryTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Orchestration/MtlsHttpClientFactoryTests.cs @@ -2,6 +2,7 @@ using System; using System.Linq; using System.Net; using System.Net.Http; +using System.Net.Security; using System.Reflection; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; @@ -10,6 +11,7 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Transport; using Xunit; namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration @@ -96,6 +98,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration Assert.Equal(TimeSpan.FromSeconds(5), client.Timeout); } + [Fact] + public void CreateHandler_without_observer_keeps_the_default_validation() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + using var cert = CreateSelfSignedCert(); + + using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger); + + Assert.Null(handler.ServerCertificateCustomValidationCallback); + } + + [Fact] + public void CreateHandler_with_observer_installs_an_observing_callback_that_never_weakens_validation() + { + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + using var cert = CreateSelfSignedCert(); + var observer = new ServerCertificateObserver(); + + using var handler = MtlsHttpClientFactory.CreateHandler(new FakeCertificateResolver(cert), logger, observer); + var callback = handler.ServerCertificateCustomValidationCallback; + + Assert.NotNull(callback); + Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateChainErrors)); + Assert.True(callback(null!, cert, null!, SslPolicyErrors.None)); + Assert.Equal(cert.Issuer, observer.LastIssuer); + } + [Fact] public void CreateHandler_throws_when_resolver_returns_null() { @@ -161,7 +192,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration /// we scan the type hierarchy for the first -typed /// instance field to stay version-tolerant. /// - private static HttpMessageHandler GetPipelineHead(HttpClient client) + internal static HttpMessageHandler GetPipelineHead(HttpClient client) { const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public; for (var t = (Type?)client.GetType(); t != null; t = t.BaseType) diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendClientFactoryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendClientFactoryTests.cs index fe3fcb6d..8e4d8e27 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendClientFactoryTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendClientFactoryTests.cs @@ -1,7 +1,13 @@ using System.IO; +using System.Net.Http; +using System.Net.Security; using AutopilotMonitor.Agent.V2.Core.Configuration; using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.Agent.V2.Runtime; using Xunit; @@ -68,6 +74,43 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program Assert.False(bundle.HasClientCertificate); } + [Fact] + public void BuildAuthClients_carries_the_server_certificate_observer_for_both_stacks() + { + using var tmp = new TempDirectory(); + var bundle = BackendClientFactory.BuildAuthClients(MinimalBootstrapTokenConfig(), agentVersion: "1.2.3.4", logger: NewLogger(tmp.Path)); + + Assert.NotNull(bundle.ServerCertificates); + } + + [Fact] + public void BuildBackendApiHttpClient_installs_the_observing_callback_on_the_inner_handler() + { + var observer = new ServerCertificateObserver(); + using var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA"); + + using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4", observer); + var inner = InnerHandler(client); + var callback = inner.ServerCertificateCustomValidationCallback; + + Assert.NotNull(callback); + Assert.False(callback!(null!, cert, null!, SslPolicyErrors.RemoteCertificateNameMismatch)); + Assert.True(callback(null!, cert, null!, SslPolicyErrors.None)); + Assert.Equal(cert.Issuer, observer.LastIssuer); + } + + [Fact] + public void BuildBackendApiHttpClient_without_observer_keeps_the_default_validation() + { + using var client = BackendClientFactory.BuildBackendApiHttpClient(new NetworkMetrics(), clientCertificate: null, agentVersion: "1.2.3.4"); + + Assert.Null(InnerHandler(client).ServerCertificateCustomValidationCallback); + } + + // Pipeline: NetworkMetricsRecordingHandler → HttpClientHandler. + private static HttpClientHandler InnerHandler(HttpClient client) => + (HttpClientHandler)((DelegatingHandler)MtlsHttpClientFactoryTests.GetPipelineHead(client)).InnerHandler; + // =========================================================== TelemetryClientResult [Fact] diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendSessionRegistrationTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendSessionRegistrationTests.cs index b9fc22d9..937009a6 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendSessionRegistrationTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/BackendSessionRegistrationTests.cs @@ -122,20 +122,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program // ============================================================ failed-registration record [Fact] - public void RecordRegistrationRun_records_a_failed_run_with_its_window_and_bounded_error() + public void RecordFailedRun_records_a_failed_run_with_its_window_and_bounded_error() { using var tmp = new TempDirectory(); var persistence = new SessionIdPersistence(tmp.Path); var started = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc); - BackendSessionRegistration.RecordRegistrationRun( + var returned = BackendSessionRegistration.RecordFailedRun( persistence, SessionRegistrationResult.Failed(new string('e', 1000)), started, started.AddSeconds(1204.46), linkUpAtStart: false, linkUpAtEnd: true, - configFetchOutcome: "UsedDefaults", logger: NewLogger(tmp.Path)); + configFetchOutcome: "UsedDefaults", networkFingerprint: null, logger: NewLogger(tmp.Path)); var record = persistence.LoadRegistrationFailure(); Assert.NotNull(record); Assert.Equal(1, record!.FailedRuns); + Assert.Equal(0, record.InterceptedRuns); Assert.Equal("Failed", record.Outcome); Assert.Equal(BackendSessionRegistration.MaxRecordedErrorLength, record.LastError!.Length); Assert.False(record.NetworkLinkUpAtStart); @@ -143,33 +144,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program Assert.Equal(1204.5, record.AttemptWindowSeconds); Assert.Equal("UsedDefaults", record.ConfigFetchOutcome); Assert.Equal(started.AddSeconds(1204.46), record.LastFailedAtUtc); + // The caller decides on the record as written, including this run. + Assert.Equal(1, returned!.FailedRuns); } [Fact] - public void RecordRegistrationRun_records_auth_failures_and_clears_on_success() + public void RecordFailedRun_records_auth_failures_and_a_success_clears_them() { using var tmp = new TempDirectory(); var persistence = new SessionIdPersistence(tmp.Path); var at = new DateTime(2026, 8, 20, 9, 0, 0, DateTimeKind.Utc); - BackendSessionRegistration.RecordRegistrationRun( + BackendSessionRegistration.RecordFailedRun( persistence, SessionRegistrationResult.AuthFailed(401, "unauthorized"), - at, at.AddSeconds(3), true, true, "Succeeded", NewLogger(tmp.Path)); + at, at.AddSeconds(3), true, true, "Succeeded", null, NewLogger(tmp.Path)); Assert.Equal("AuthFailed", persistence.LoadRegistrationFailure()!.Outcome); - BackendSessionRegistration.RecordRegistrationRun( - persistence, SessionRegistrationResult.Succeeded(new RegisterSessionResponse { Success = true }), - at, at.AddSeconds(1), true, true, "Succeeded", NewLogger(tmp.Path)); + BackendSessionRegistration.ClearAfterSuccess(persistence, () => null, NewLogger(tmp.Path)); Assert.Null(persistence.LoadRegistrationFailure()); } [Fact] - public void RecordRegistrationRun_without_persistence_is_a_no_op() + public void Recording_without_persistence_is_a_no_op() { using var tmp = new TempDirectory(); - BackendSessionRegistration.RecordRegistrationRun( + Assert.Null(BackendSessionRegistration.RecordFailedRun( null, SessionRegistrationResult.Failed("x"), DateTime.UtcNow, DateTime.UtcNow, - null, null, null, NewLogger(tmp.Path)); + null, null, null, null, NewLogger(tmp.Path))); + Assert.Null(BackendSessionRegistration.ClearAfterSuccess(null, () => null, NewLogger(tmp.Path))); } // ============================================================ emergency-break message diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/InterceptedRegistrationTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/InterceptedRegistrationTests.cs new file mode 100644 index 00000000..c9666a55 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/InterceptedRegistrationTests.cs @@ -0,0 +1,477 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Configuration; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; +using AutopilotMonitor.Agent.V2.Runtime; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.Shared.Models; +using Newtonsoft.Json; +using Xunit; +using ProgramAlias = AutopilotMonitor.Agent.V2.Program; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Program +{ + /// + /// A registration that something on the network path answered at process start (client + /// certificate interception plan, K4/K5): a buffer of further boots, then the emergency break's + /// cleanup; the pending ClientCertificateNotReceived distress leaves the machine first; a later + /// successful registration reports the failed starts on agent_started. Each start is + /// ended the way ends it: record the failed + /// run, then conclude it. + /// + public sealed class InterceptedRegistrationTests : IDisposable + { + private static readonly DateTime At = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc); + + private readonly TempDirectory _tmp = new TempDirectory(); + private readonly AgentLogger _logger; + private readonly SessionIdPersistence _persistence; + private readonly string _stateDir; + private int _cleanupCalls; + + public InterceptedRegistrationTests() + { + _logger = new AgentLogger(Path.Combine(_tmp.Path, "logs"), AgentLogLevel.Info); + _persistence = new SessionIdPersistence(_tmp.Path); + _persistence.GetOrCreate(_logger); + _stateDir = Path.Combine(_tmp.Path, "State"); + } + + public void Dispose() => _tmp.Dispose(); + + // ============================================================ K4: buffer, then clean up + + [Fact] + public void Agent_default_buffer_allows_one_further_boot_then_cleans_up() + { + var config = new AgentConfiguration(); + Assert.Equal(1, config.InterceptedRegistrationRetryBoots); + + Assert.Equal(6, Boot(Intercepted(), config)); + + Assert.False(GaveUp); + Assert.True(_persistence.SessionExists()); + Assert.Equal(0, _cleanupCalls); + var record = _persistence.LoadRegistrationFailure()!; + Assert.Equal(1, record.FailedRuns); + Assert.Equal(1, record.InterceptedRuns); + Assert.Equal("AuthFailed", record.Outcome); + + Assert.Equal(6, Boot(Intercepted(), config)); // the give-up keeps exit 6 + + AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1); + } + + [Theory] + [InlineData(0)] // clean up at the first intercepted start + [InlineData(2)] + [InlineData(5)] + public void Gives_up_on_the_first_intercepted_start_beyond_the_buffer(int buffer) + { + var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = buffer }; + + for (var start = 1; start <= buffer; start++) + { + Assert.Equal(6, Boot(Intercepted(), config)); + Assert.False(GaveUp, $"gave up on start {start} with a buffer of {buffer}"); + } + + Assert.Equal(6, Boot(Intercepted(), config)); + + AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1); + } + + [Theory] + [InlineData(true, 1)] + [InlineData(false, 0)] + public void Give_up_runs_the_cleanup_only_with_SelfDestructOnComplete_like_the_emergency_break( + bool selfDestructOnComplete, int expectedCleanupCalls) + { + var config = new AgentConfiguration + { + InterceptedRegistrationRetryBoots = 0, + SelfDestructOnComplete = selfDestructOnComplete, + }; + + Assert.Equal(6, Boot(Intercepted(), config)); + + AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls); + } + + [Fact] + public async Task An_intercepted_start_takes_the_buffer_from_the_cached_config() + { + // The config fetch of an intercepted start is intercepted too: it falls back to the + // cache, and the registration decides on the cached buffer, not the agent default. + var config = new AgentConfiguration(); + var service = new InterceptedConfigService(_logger, cached: new AgentConfigResponse + { + InterceptedRegistrationRetryBoots = 0, + SelfDestructOnComplete = true, + }); + + var fetched = await service.FetchConfigAsync(retryOnTransientErrors: true); + Assert.Equal(RemoteConfigFetchOutcome.FromCache, service.LastFetchOutcome); + RemoteConfigMerger.Merge(config, fetched, _logger); + Assert.Equal(0, config.InterceptedRegistrationRetryBoots); + + Assert.Equal(6, Boot(Intercepted(), config)); + + AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1); + } + + [Fact] + public void Backend_rejection_and_network_failure_keep_their_exit_codes_and_never_give_up() + { + var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 }; + + Assert.Equal(6, Boot(BackendRejected(), config)); + Assert.Equal(7, Boot(NetworkFailure(), config)); + + Assert.False(GaveUp); + Assert.True(_persistence.SessionExists()); + Assert.Equal(0, _cleanupCalls); + var record = _persistence.LoadRegistrationFailure()!; + Assert.Equal(2, record.FailedRuns); + Assert.Equal(0, record.InterceptedRuns); + } + + [Fact] + public void Only_an_intercepted_start_gives_up_even_when_the_count_already_exceeds_the_buffer() + { + var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 }; + Boot(Intercepted(), config); + Boot(Intercepted(), config); + config.InterceptedRegistrationRetryBoots = 0; // e.g. a later cached config lowered it + + Assert.Equal(6, Boot(BackendRejected(), config)); + Assert.Equal(7, Boot(NetworkFailure(), config)); + + Assert.False(GaveUp); + Assert.Equal(0, _cleanupCalls); + var record = _persistence.LoadRegistrationFailure()!; + Assert.Equal(4, record.FailedRuns); + Assert.Equal(2, record.InterceptedRuns); // unchanged by the other failures + Assert.Equal("Failed", record.Outcome); + } + + [Fact] + public void Failures_of_another_kind_between_interceptions_do_not_reset_the_count() + { + var config = new AgentConfiguration(); // buffer 1 + + Assert.Equal(6, Boot(Intercepted(), config)); + Assert.Equal(7, Boot(NetworkFailure(), config)); + Assert.Equal(6, Boot(BackendRejected(), config)); + Assert.False(GaveUp); + + Assert.Equal(6, Boot(Intercepted(), config)); // second interception since the last success + + AssertCleanedUpLikeTheEmergencyBreak(expectedCleanupCalls: 1); + } + + [Fact] + public void Only_a_successful_registration_resets_the_count() + { + var config = new AgentConfiguration(); // buffer 1 + Boot(Intercepted(), config); + BackendSessionRegistration.ClearAfterSuccess(_persistence, () => "fp-office", _logger); + + Assert.Equal(6, Boot(Intercepted(), config)); + + Assert.False(GaveUp); + Assert.Equal(1, _persistence.LoadRegistrationFailure()!.InterceptedRuns); + } + + [Fact] + public void Without_a_written_record_the_buffer_cannot_be_enforced_and_the_next_boot_retries() + { + var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 }; + + var exit = BackendSessionRegistration.ConcludeFailedRun( + Intercepted(), record: null, config, authFailureTracker: null, + _persistence, _stateDir, CleanupFactory, _logger); + + Assert.Equal(6, exit); + Assert.False(GaveUp); + Assert.True(_persistence.SessionExists()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task The_registration_answer_decides_whether_the_start_counts_as_intercepted(bool foreign) + { + // Through the real client: no correlation echo + the platform's "Client Certificate + // Required" page is an interception; a JSON 403 that echoes the id is the backend's verdict. + using var handler = new RecordingHttpMessageHandler(); + if (foreign) + handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage); + else + handler.QueueStatus(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}"); + using var http = new HttpClient(handler); + var apiClient = new BackendApiClient( + http, "https://backend.test", manufacturer: "m", model: "d", serialNumber: "s", + useBootstrapTokenAuth: false, bootstrapToken: null, agentVersion: "0.0.0", logger: _logger); + var distress = new FakeDistressReporter(); + var config = new AgentConfiguration + { + ApiBaseUrl = "https://backend.test", + TenantId = "00000000-0000-0000-0000-000000000001", + SessionId = _persistence.GetOrCreate(_logger), + InterceptedRegistrationRetryBoots = 0, + }; + + var result = await SessionRegistrationHelper.RegisterWithRetryAsync( + apiClient, config, agentVersion: "0.0.0", logger: _logger, + authFailureTracker: NewTracker(distress), + deviceHardware: ("m", "d", "s"), + networkLinkWait: _ => Task.CompletedTask); + + Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome); + Assert.Equal(foreign, result.IsIntercepted); + Assert.Equal( + foreign ? DistressErrorType.ClientCertificateNotReceived : DistressErrorType.DeviceNotRegistered, + Assert.Single(distress.Calls).ErrorType); + + Assert.Equal(6, Boot(result, config)); + Assert.Equal(foreign, GaveUp); + } + + // ============================================================ the distress leaves first + + [Fact] + public void The_exit_waits_for_the_pending_ClientCertificateNotReceived_send() + { + var record = RecordFailed(Intercepted(), "fp-office"); + var (tracker, inFlight) = DispatchSlowCertificateDistress(); + + var exit = BackendSessionRegistration.ConcludeFailedRun( + Intercepted(), record, new AgentConfiguration(), tracker, + _persistence, _stateDir, CleanupFactory, _logger); + + Assert.Equal(6, exit); + Assert.True(inFlight.IsCompleted); + Assert.False(GaveUp); + } + + [Fact] + public void The_give_up_waits_for_the_pending_send_before_it_cleans_up() + { + var record = RecordFailed(Intercepted(), "fp-office"); + var (tracker, inFlight) = DispatchSlowCertificateDistress(); + bool? sentBeforeCleanup = null; + + var exit = BackendSessionRegistration.ConcludeFailedRun( + Intercepted(), record, new AgentConfiguration { InterceptedRegistrationRetryBoots = 0 }, tracker, + _persistence, _stateDir, + () => + { + sentBeforeCleanup = inFlight.IsCompleted; + throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests"); + }, + _logger); + + Assert.Equal(6, exit); + Assert.True(sentBeforeCleanup); + Assert.True(GaveUp); + } + + [Fact] + public void The_wait_for_the_distress_is_bounded_by_its_own_http_timeout() + { + Assert.True(BackendSessionRegistration.PendingDistressBound <= TimeSpan.FromSeconds(5)); + } + + // ============================================================ K5: measurement point + + [Theory] + [InlineData("fp-office", "fp-office", false)] + [InlineData("fp-office", "fp-home", true)] + public void A_success_after_failed_starts_reports_the_record_it_clears( + string failedOn, string registeredOn, bool networkChanged) + { + var config = new AgentConfiguration { InterceptedRegistrationRetryBoots = 5 }; + Boot(NetworkFailure(), config, fingerprint: "fp-elsewhere"); + Boot(Intercepted(), config, fingerprint: failedOn); // the latest failed start's network counts + + var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn, _logger)!; + + Assert.Equal(2, prior.FailedRuns); + Assert.Equal(1, prior.InterceptedRuns); + Assert.Equal("AuthFailed", prior.LastOutcome); + Assert.Equal(networkChanged, prior.NetworkChanged); + Assert.Null(_persistence.LoadRegistrationFailureRecord()); + + var data = EmitAgentStarted(prior); + Assert.Equal(2, data["priorRegistrationFailedRuns"]); + Assert.Equal(1, data["priorRegistrationInterceptedRuns"]); + Assert.Equal("AuthFailed", data["priorRegistrationLastOutcome"]); + Assert.Equal(networkChanged, data["priorRegistrationNetworkChanged"]); + } + + [Theory] + [InlineData(null, "fp-home")] // the failed start's network was unknown + [InlineData("fp-office", null)] // this start's network is unknown + public void Network_change_is_omitted_when_either_fingerprint_is_unknown(string? failedOn, string? registeredOn) + { + Boot(Intercepted(), new AgentConfiguration(), fingerprint: failedOn); + + var prior = BackendSessionRegistration.ClearAfterSuccess(_persistence, () => registeredOn!, _logger)!; + + Assert.Null(prior.NetworkChanged); + var data = EmitAgentStarted(prior); + Assert.Equal(1, data["priorRegistrationFailedRuns"]); + Assert.False(data.ContainsKey("priorRegistrationNetworkChanged")); + } + + [Fact] + public void A_success_without_failed_starts_reports_nothing_and_never_reads_the_network() + { + var captures = 0; + + var prior = BackendSessionRegistration.ClearAfterSuccess( + _persistence, () => { captures++; return "fp-office"; }, _logger); + + Assert.Null(prior); + Assert.Equal(0, captures); + Assert.DoesNotContain(EmitAgentStarted(null).Keys, k => k.StartsWith("priorRegistration", StringComparison.Ordinal)); + } + + [Fact] + public void The_network_fingerprint_stays_on_the_device() + { + Boot(Intercepted(), new AgentConfiguration(), fingerprint: "0123456789abcdef"); + + // Kept locally beside the record ... + Assert.Contains("0123456789abcdef", File.ReadAllText(Path.Combine(_tmp.Path, "registration-failure.json"))); + Assert.Equal("0123456789abcdef", _persistence.LoadRegistrationFailureRecord()!.NetworkFingerprint); + + // ... while the summary the emergency break sends cannot carry it. + var wire = JsonConvert.SerializeObject(new AgentErrorReport + { + PriorRegistrationFailure = _persistence.LoadRegistrationFailure(), + }); + Assert.DoesNotContain("0123456789abcdef", wire); + Assert.DoesNotContain("Fingerprint", wire); + Assert.Contains("\"InterceptedRuns\":1", wire); + } + + // ============================================================ helpers + + private static SessionRegistrationResult Intercepted() => + SessionRegistrationResult.AuthFailed(403, "The device certificate did not reach the backend.", intercepted: true); + + private static SessionRegistrationResult BackendRejected() => + SessionRegistrationResult.AuthFailed(403, "Backend returned 403 Forbidden. The device is not authorized."); + + private static SessionRegistrationResult NetworkFailure() => + SessionRegistrationResult.Failed("No such host is known."); + + private bool GaveUp => File.Exists(Path.Combine(_stateDir, "enrollment-complete.marker")); + + private AuthFailureTracker NewTracker(FakeDistressReporter distress) => + new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance, _logger, distress); + + /// + /// What the registration helper does when the answer did not come from the backend: the + /// tracker dispatches ClientCertificateNotReceived — here a send that stays in flight for a + /// second, started right before the conclusion so nothing else can use the time up. + /// + private (AuthFailureTracker Tracker, Task InFlight) DispatchSlowCertificateDistress() + { + Task? inFlight = null; + var distress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(TimeSpan.FromSeconds(1)) }; + var tracker = NewTracker(distress); + tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired()); + + Assert.Equal(DistressErrorType.ClientCertificateNotReceived, Assert.Single(distress.Calls).ErrorType); + Assert.False(inFlight!.IsCompleted); + return (tracker, inFlight); + } + + private RegistrationFailureSummary? RecordFailed(SessionRegistrationResult result, string? fingerprint) => + BackendSessionRegistration.RecordFailedRun( + _persistence, result, At, At.AddSeconds(2), linkUpAtStart: true, linkUpAtEnd: true, + configFetchOutcome: "FromCache", networkFingerprint: fingerprint, logger: _logger); + + /// One start whose registration failed with , ended the way Register ends it. + private int Boot(SessionRegistrationResult result, AgentConfiguration config, string? fingerprint = "fp-office") + { + var record = RecordFailed(result, fingerprint); + return BackendSessionRegistration.ConcludeFailedRun( + result, record, config, authFailureTracker: null, _persistence, _stateDir, CleanupFactory, _logger); + } + + private CleanupService CleanupFactory() + { + _cleanupCalls++; + // The real factory spawns a PowerShell cleanup script; throwing makes the cleanup retry + // swallow and log it, so the call is counted without a side effect (as in ProgramGuardsTests). + throw new InvalidOperationException("test-harness: cleanup factory must not spawn real PowerShell in unit tests"); + } + + /// The emergency break's end state (ProgramGuardsTests): marker, session files gone, cleanup per SelfDestructOnComplete. + private void AssertCleanedUpLikeTheEmergencyBreak(int expectedCleanupCalls) + { + Assert.True(GaveUp); + Assert.False(_persistence.SessionExists()); + Assert.Null(_persistence.LoadRegistrationFailure()); + Assert.Equal(expectedCleanupCalls, _cleanupCalls); + // The next start ends at the bootstrap guard, exactly as after the break. + Assert.True(ProgramAlias.CheckEnrollmentCompleteMarker( + _stateDir, selfDestructOnComplete: false, cleanupServiceFactory: null, _logger, consoleMode: false)); + } + + private Dictionary EmitAgentStarted(PriorRegistrationFailures? prior) + { + var sink = new FakeSignalIngressSink(); + LifecycleEmitters.EmitAgentStarted( + new InformationalEventPost(sink, SystemClock.Instance, _logger), + new AgentConfiguration { SessionId = "S1", TenantId = "00000000-0000-0000-0000-000000000001", CommandLineArgs = string.Empty }, + new ProgramAlias.PreviousExitSummary { ExitType = "reboot_kill" }, + "2.0.test", + remoteConfigService: null, + _logger, + priorRegistrationFailures: prior); + return (Dictionary)Assert.Single(sink.Posted).TypedPayload!; + } + + /// A start whose config fetch is answered by the interception: it falls back to the cache. + private sealed class InterceptedConfigService : RemoteConfigService + { + private readonly AgentConfigResponse _cached; + + public InterceptedConfigService(AgentLogger logger, AgentConfigResponse cached) + : base( + new BackendApiClient( + httpClient: new HttpClient(), baseUrl: "http://localhost", manufacturer: string.Empty, + model: string.Empty, serialNumber: string.Empty, useBootstrapTokenAuth: false, + bootstrapToken: null, agentVersion: "0.0.0", logger: logger), + "00000000-0000-0000-0000-000000000001", + logger) + { + _cached = cached; + } + + protected override Task CallBackendAsync() => + throw new BackendAuthException("The device certificate did not reach the backend.", 403, + classification: InterceptionFixtures.CertificateRequired()); + + protected override AgentConfigResponse LoadCachedConfig() => _cached; + + protected override Task WaitForNetworkLinkAsync() => Task.CompletedTask; + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs index f8896aaa..5a88cdc2 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/AuthFailureTrackerTests.cs @@ -1,11 +1,12 @@ using System; -using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Logging; -using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; using AutopilotMonitor.Shared.Models; using Xunit; @@ -261,8 +262,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security } [Fact] - public void RecordFailure_endpoint_unavailable_suppresses_distress_but_still_counts() + public void RecordFailure_only_the_stopped_app_page_counts_without_distress() { + // Rewritten deliberately for the client-certificate interception plan: the former + // "endpoint unavailable" flag covered every HTML platform page, so the platform's + // "Client Certificate Required" page (TLS inspection on the path) shut healthy agents + // down. Now only the stopped-app page keeps that semantics — it counts toward the + // shutdown (a dead endpoint must not retry forever) without a distress report that + // could not reach it — while the certificate page never counts (tests below). var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); var fakeDistress = new FakeDistressReporter(); var tracker = new AuthFailureTracker(maxFailures: 2, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); @@ -270,19 +277,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security int events = 0; tracker.ThresholdExceeded += (_, _) => events++; - // Platform-403s from a stopped endpoint: a distress report cannot reach it and the - // DeviceNotRegistered classification would be wrong — nothing must be dispatched. - tracker.RecordFailure(403, "agent/config", endpointUnavailable: true); - tracker.RecordFailure(403, "agent/register-session", endpointUnavailable: true); + tracker.RecordFailure(403, "agent/config", InterceptionFixtures.CertificateRequired()); + Assert.Equal(0, tracker.ConsecutiveFailures); + + tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp()); + tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.StoppedApp()); - Assert.Empty(fakeDistress.Calls); - // The shutdown thresholds still apply — a dead endpoint must not retry forever. Assert.Equal(2, tracker.ConsecutiveFailures); Assert.Equal(1, events); + Assert.DoesNotContain(fakeDistress.Calls, c => c.ErrorType == DistressErrorType.DeviceNotRegistered); } [Fact] - public void RecordFailure_endpoint_unavailable_does_not_consume_streak_report_slot() + public void RecordFailure_stopped_app_page_does_not_consume_streak_report_slot() { var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); var fakeDistress = new FakeDistressReporter(); @@ -290,7 +297,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security // The streak opens with a platform-403 (no dispatch), then a live backend answers // with a genuine 403 — that one must still produce the streak's single report. - tracker.RecordFailure(403, "agent/config", endpointUnavailable: true); + tracker.RecordFailure(403, "agent/config", InterceptionFixtures.StoppedApp()); tracker.RecordFailure(403, "agent/config"); tracker.RecordFailure(403, "agent/config"); @@ -298,6 +305,133 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security Assert.Equal(DistressErrorType.DeviceNotRegistered, fakeDistress.Calls[0].ErrorType); } + // ============================================================= Foreign answers (interception) + + [Fact] + public void RecordFailure_foreign_never_reaches_the_threshold() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 10, clock, NewLogger(), new FakeDistressReporter()); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + for (int i = 0; i < 10; i++) + { + tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired()); + clock.Now = clock.Now.AddMinutes(3); // also past the time window + } + + Assert.Equal(0, events); + Assert.Equal(0, tracker.ConsecutiveFailures); + } + + [Fact] + public void RecordFailure_foreign_dispatches_one_ClientCertificateNotReceived_per_process() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var fakeDistress = new FakeDistressReporter(); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); + + tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired()); + tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired()); + tracker.RecordSuccess(); // a success re-arms the streak report, never the per-process one + tracker.RecordFailure(401, "agent/telemetry", InterceptionFixtures.CertificateRequired(401)); + + var call = Assert.Single(fakeDistress.Calls); + Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType); + Assert.Equal(403, call.HttpStatusCode); + // Operation, status, sub-reason, page title, issuer — in that order, within the 256 budget. + Assert.StartsWith("agent/register-session http 403 client_certificate_required;", call.Message); + Assert.True( + call.Message.IndexOf(InterceptionFixtures.PlatformPageTitle, StringComparison.Ordinal) < + call.Message.IndexOf(InterceptionFixtures.InspectionIssuer, StringComparison.Ordinal)); + Assert.True(call.Message.Length <= 256, $"distress message is {call.Message.Length} characters"); + } + + [Fact] + public void RecordFailure_foreign_within_a_backend_streak_neither_resets_nor_advances_it() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var fakeDistress = new FakeDistressReporter(); + var tracker = new AuthFailureTracker(maxFailures: 3, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); + + int events = 0; + tracker.ThresholdExceeded += (_, _) => events++; + + tracker.RecordFailure(403, "agent/telemetry"); + tracker.RecordFailure(403, "agent/telemetry"); + tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired()); + tracker.RecordFailure(403, "agent/telemetry", InterceptionFixtures.CertificateRequired()); + + Assert.Equal(2, tracker.ConsecutiveFailures); + Assert.Equal(0, events); + + tracker.RecordFailure(403, "agent/telemetry"); // the third backend rejection trips it + + Assert.Equal(3, tracker.ConsecutiveFailures); + Assert.Equal(1, events); + Assert.Equal(1, fakeDistress.Calls.Count(c => c.ErrorType == DistressErrorType.DeviceNotRegistered)); + } + + // ============================================================= Pending foreign distress + + [Fact] + public void WaitForPendingForeignDistress_returns_at_once_when_nothing_was_dispatched() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), new FakeDistressReporter()); + + Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero)); + } + + [Fact] + public void WaitForPendingForeignDistress_waits_for_the_send_to_finish() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + Task? inFlight = null; + var fakeDistress = new FakeDistressReporter { Send = () => inFlight = Task.Delay(200) }; + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); + + tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired()); + + Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.FromSeconds(10))); + Assert.True(inFlight!.IsCompleted); + } + + [Fact] + public void WaitForPendingForeignDistress_stops_at_the_bound() + { + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var neverSent = new TaskCompletionSource(); + var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task }; + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); + tracker.RecordFailure(403, "agent/register-session", InterceptionFixtures.CertificateRequired()); + + var watch = Stopwatch.StartNew(); + var finished = tracker.WaitForPendingForeignDistress(TimeSpan.FromMilliseconds(100)); + watch.Stop(); + + Assert.False(finished); + Assert.True(watch.Elapsed < TimeSpan.FromSeconds(5), $"waited {watch.Elapsed}"); + } + + [Fact] + public void WaitForPendingForeignDistress_never_waits_for_a_backend_streak_report() + { + // Only the ClientCertificateNotReceived send is kept: a failed start must not block + // on anything else. + var clock = new FakeClock(new DateTime(2026, 4, 21, 10, 0, 0, DateTimeKind.Utc)); + var neverSent = new TaskCompletionSource(); + var fakeDistress = new FakeDistressReporter { Send = () => neverSent.Task }; + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, clock, NewLogger(), fakeDistress); + + tracker.RecordFailure(403, "agent/register-session"); + + Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(fakeDistress.Calls).ErrorType); + Assert.True(tracker.WaitForPendingForeignDistress(TimeSpan.Zero)); + } + [Fact] public void RecordSuccess_rearms_distress_dispatch_for_the_next_streak() { @@ -329,29 +463,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security Assert.Equal(1, events); } - private sealed class FakeDistressReporter : DistressReporter - { - public sealed class Call - { - public DistressErrorType ErrorType { get; set; } - public string Message { get; set; } = default!; - public int? HttpStatusCode { get; set; } - } - - public List Calls { get; } = new List(); - - public FakeDistressReporter() - : base(baseUrl: "https://example.test", tenantId: "t", manufacturer: "m", model: "d", serialNumber: "s", agentVersion: "0.0.0", logger: null!) - { - } - - public override Task TrySendAsync(DistressErrorType errorType, string message, int? httpStatusCode = null) - { - Calls.Add(new Call { ErrorType = errorType, Message = message, HttpStatusCode = httpStatusCode }); - return Task.CompletedTask; - } - } - private sealed class FakeClock : AutopilotMonitor.DecisionCore.Engine.IClock { public DateTime Now { get; set; } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionIdPersistenceTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionIdPersistenceTests.cs index 4f6cc85f..b6441945 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionIdPersistenceTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionIdPersistenceTests.cs @@ -298,9 +298,50 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security Assert.False(File.Exists(Path.Combine(tmp.Path, "registration-failure.json"))); } + [Fact] + public void RecordRegistrationFailure_counts_intercepted_runs_and_keeps_the_latest_network_fingerprint() + { + using var tmp = new TempDirectory(); + var sut = new SessionIdPersistence(tmp.Path); + var at = new DateTime(2026, 10, 12, 9, 0, 0, DateTimeKind.Utc); + var intercepted = FailedRun(at, outcome: "AuthFailed"); + intercepted.InterceptedRuns = 1; + + sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-office"); + sut.RecordRegistrationFailure(FailedRun(at.AddHours(1)), networkFingerprint: null); + var merged = sut.RecordRegistrationFailure(intercepted, networkFingerprint: "fp-home"); + + Assert.Equal(3, merged!.FailedRuns); + Assert.Equal(2, merged.InterceptedRuns); + var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord(); + Assert.Equal(3, record!.Summary.FailedRuns); + Assert.Equal(2, record.Summary.InterceptedRuns); + Assert.Equal("fp-home", record.NetworkFingerprint); + } + + [Fact] + public void A_record_from_an_agent_before_the_interception_buffer_still_loads() + { + using var tmp = new TempDirectory(); + File.WriteAllText(Path.Combine(tmp.Path, "registration-failure.json"), + "{\"FailedRuns\":2,\"FirstFailedAtUtc\":\"2026-09-01T08:00:00Z\",\"LastFailedAtUtc\":\"2026-09-02T08:00:00Z\"," + + "\"Outcome\":\"AuthFailed\",\"LastError\":\"x\",\"NetworkLinkUpAtStart\":true,\"NetworkLinkUpAtEnd\":true," + + "\"AttemptWindowSeconds\":3.0,\"ConfigFetchOutcome\":\"FromCache\"}"); + + var record = new SessionIdPersistence(tmp.Path).LoadRegistrationFailureRecord(); + + Assert.Equal(2, record!.Summary.FailedRuns); + Assert.Equal(0, record.Summary.InterceptedRuns); + Assert.Null(record.NetworkFingerprint); + Assert.Equal(new DateTime(2026, 9, 1, 8, 0, 0, DateTimeKind.Utc), record.Summary.FirstFailedAtUtc); + } + [Theory] [InlineData("not json")] [InlineData("{\"FailedRuns\":0}")] + [InlineData("null")] + [InlineData("[]")] + [InlineData("")] public void LoadRegistrationFailure_reports_nothing_for_a_corrupt_or_empty_record(string content) { using var tmp = new TempDirectory(); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionRegistrationHelperTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionRegistrationHelperTests.cs index 6fbaa36d..f9fa9166 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionRegistrationHelperTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Security/SessionRegistrationHelperTests.cs @@ -6,6 +6,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Transport; using AutopilotMonitor.DecisionCore.Engine; using AutopilotMonitor.Shared.Models; using Xunit; @@ -96,6 +97,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Security Assert.Equal(1, tracker.ConsecutiveFailures); } + [Fact] + public async Task Intercepted_registration_ends_auth_failed_without_counting_and_reports_once() + { + // Something on the network path answered (TLS inspection): the run still ends + // AuthFailed (exit 6, unchanged until the interception buffer exists), but the tracker + // does not count it and sends ClientCertificateNotReceived instead of DeviceNotRegistered. + using var tmp = new TempDirectory(); + var logger = NewLogger(tmp); + var config = NewConfig(); + + var apiClient = new FakeApiClient(_ => + throw new BackendAuthException("The device certificate did not reach the backend.", statusCode: 403, + classification: InterceptionFixtures.CertificateRequired())); + var distress = new FakeDistressReporter(); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, + clock: SystemClock.Instance, logger: logger, distressReporter: distress); + + var result = await SessionRegistrationHelper.RegisterWithRetryAsync( + apiClient, config, agentVersion: "0.0.0", logger: logger, authFailureTracker: tracker, + networkLinkWait: _ => Task.CompletedTask); + + Assert.Equal(SessionRegistrationOutcome.AuthFailed, result.Outcome); + Assert.Equal(403, result.HttpStatusCode); + Assert.Equal(1, apiClient.CallCount); + Assert.Equal(0, tracker.ConsecutiveFailures); + var call = Assert.Single(distress.Calls); + Assert.Equal(DistressErrorType.ClientCertificateNotReceived, call.ErrorType); + Assert.StartsWith("agent/register-session http 403 client_certificate_required", call.Message); + } + // ── SESSION-OWNER-BINDING ──────────────────────────────────────────── private static BackendAuthException OwnerMismatch() diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs index aa8b3b86..451cfab6 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs @@ -165,6 +165,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination public IReadOnlyCollection StarvedAlreadyReportedOverride { get; set; } = Array.Empty(); public List TerminationActionLog { get; } = new List(); public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero; + public int TerminalUploadAttemptRequests; // Session bc803955 — DevicePrep app-wave settle wait. Zero (default) disables the // wait so the existing timing-sensitive tests stay untouched; the dedicated settle // tests opt in with a real budget. @@ -267,6 +268,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination public long IngressPendingSignalCount => _rig.IngressPendingSignalCountAccessor!(); public bool CanObserveSpool => _rig.PendingItemCountAccessor != null; public int SpoolPendingItemCount => _rig.PendingItemCountAccessor!(); + + public void RequestTerminalUploadAttempt() + { + Interlocked.Increment(ref _rig.TerminalUploadAttemptRequests); + _rig.TerminationActionLog.Add("terminalUploadAttempt"); + } } /// ARCH-F2 — recording shutdown gate (no real shutdown.exe in tests). @@ -489,6 +496,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone)); Assert.Equal(0, spoolPolls); + Assert.Equal(0, rig.TerminalUploadAttemptRequests); Assert.Equal(0, rig.DiagnosticsUploads); Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes); Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes); @@ -497,6 +505,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination Assert.Equal(1, rig.ShutdownSignalled); } + public enum TerminalExit { SelfDestruct, StandaloneReboot, WhiteGlovePart1 } + + [Theory] + [InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.SelfDestruct)] + [InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed, TerminalExit.SelfDestruct)] + [InlineData(EnrollmentTerminationReason.StoppedWaiting, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)] + [InlineData(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.AwaitingHello, TerminalExit.SelfDestruct)] + [InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed, TerminalExit.StandaloneReboot)] + [InlineData(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed, TerminalExit.WhiteGlovePart1)] + public void Handle_while_uploads_are_intercepted_makes_one_terminal_attempt_then_exits_as_usual( + EnrollmentTerminationReason reason, EnrollmentTerminationOutcome outcome, SessionStage stage, TerminalExit exit) + { + // Enrollment end during an interception pause: no deferral. DrainSpool asks for the one + // terminal upload attempt, waits its bounded budget (the spool never empties — every + // upload is answered on the network path), then the exit runs exactly as for any + // session. Unsent telemetry is lost, as today. + using var rig = new Rig(); + rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = stage }.Build(); + rig.SpoolDrainPeriodOverride = TimeSpan.FromMilliseconds(200); + rig.PendingItemCountAccessor = () => 7; + rig.DiagnosticsResult = new DiagnosticsUploadResult + { + ErrorCode = "The device certificate did not reach the backend (TLS inspection or a proxy on the network path): 403 Forbidden ...", + }; + var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: exit != TerminalExit.StandaloneReboot); + cfg.RebootOnComplete = exit == TerminalExit.StandaloneReboot; + + rig.Build(cfg).Handle(sender: null!, Args(reason, outcome, stage)); + + Assert.Equal(1, rig.TerminalUploadAttemptRequests); + Assert.Equal(1, rig.ShutdownSignalled); + Assert.True(rig.TerminationActionLog.IndexOf("terminalUploadAttempt") < rig.TerminationActionLog.IndexOf("signalShutdown")); + if (exit == TerminalExit.WhiteGlovePart1) + { + Assert.True(rig.SessionPersistence.IsWhiteGloveResume()); + Assert.Equal(0, rig.CleanupService.Invocations); + return; + } + + Assert.Equal(1, rig.DiagnosticsUploads); + Assert.Contains(Constants.EventTypes.DiagnosticsUploadFailed, rig.EmittedEventTypes); + Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker"))); + Assert.Equal(exit == TerminalExit.SelfDestruct ? 1 : 0, rig.CleanupService.Invocations); + Assert.Equal(exit == TerminalExit.StandaloneReboot ? 1 : 0, rig.RebootInvocations); + } + [Fact] public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs index 5056925c..d08c8692 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendApiClientAuthClassificationTests.cs @@ -6,16 +6,19 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Transport; +using AutopilotMonitor.Shared.Models; using Xunit; namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport { /// - /// 401/403 classification in : a backend JSON 403 is a real - /// device-authorization verdict, while an HTML body means the response came from the Azure - /// platform / an edge proxy (stopped or retired app) and must NOT be reported as - /// "device is not authorized". Field case 2026-08-09: a pre-cutover agent hit - /// the stopped legacy Function App and support chased a non-existent authorization problem. + /// 401/403 classification in : only a response that echoes the + /// request's correlation id is the backend's device-authorization verdict. The platform page of + /// a stopped app (field case 2026-08-09: a pre-cutover agent hit the stopped legacy Function App + /// and support chased a non-existent authorization problem) and anything answered on the network + /// path (TLS inspection: "Client Certificate Required", proxy pages) must NOT be reported as + /// "device is not authorized". /// public sealed class BackendApiClientAuthClassificationTests { @@ -24,17 +27,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport private readonly Func _responder; public StubHandler(Func responder) => _responder = responder; + public int Calls { get; private set; } + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - => Task.FromResult(_responder(request)); + { + Calls++; + var response = _responder(request); + response.RequestMessage = request; // as HttpClientHandler does + return Task.FromResult(response); + } } - private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType) - { - var handler = new StubHandler(_ => new HttpResponseMessage(status) + /// + /// True: the response echoes the request's X-Correlation-ID like the backend's middleware. + /// False: a platform or proxy answer without the echo. + /// + private static StubHandler Answer(HttpStatusCode status, string body, string mediaType, bool fromBackend) => + new StubHandler(request => { - Content = new StringContent(body, Encoding.UTF8, mediaType) + var response = new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) }; + if (fromBackend && request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids)) + response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids); + return response; }); - return new BackendApiClient( + + private static BackendApiClient BuildClient(HttpMessageHandler handler, ServerCertificateObserver? serverCertificates = null) => + new BackendApiClient( new HttpClient(handler), baseUrl: "https://backend.example", manufacturer: "Contoso", @@ -43,8 +61,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport useBootstrapTokenAuth: false, bootstrapToken: null, agentVersion: "2.0.0", - logger: null); - } + logger: null, + serverCertificates: serverCertificates); + + private static BackendApiClient BuildClient(HttpStatusCode status, string body, string mediaType, bool fromBackend = true) => + BuildClient(Answer(status, body, mediaType, fromBackend)); [Fact] public async Task Backend_json_403_is_reported_as_device_not_authorized() @@ -134,16 +155,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport public async Task Platform_html_403_is_reported_as_endpoint_unavailable_with_page_title() { using var client = BuildClient( - HttpStatusCode.Forbidden, - "Web App - Unavailablestopped", - "text/html"); + HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html", fromBackend: false); var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); Assert.Equal(403, ex.StatusCode); Assert.True(ex.EndpointUnavailable); - Assert.Contains("Web App - Unavailable", ex.Message); + Assert.False(ex.IsForeignResponse); + Assert.Equal(ResponseOrigin.PlatformStopped, ex.Origin); + Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message); Assert.Contains("NOT a device-authorization failure", ex.Message); + Assert.Contains("stale ApiBaseUrl", ex.Message); Assert.DoesNotContain("Check client certificate", ex.Message); } @@ -153,7 +175,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport using var client = BuildClient( HttpStatusCode.Forbidden, " Error 403 - This web app is stopped.", - "text/plain"); + "text/plain", + fromBackend: false); var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); @@ -161,6 +184,113 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Contains("HTML platform error page", ex.Message); } + // ── answered on the network path (TLS inspection, proxy) ───────────── + + [Fact] + public async Task Certificate_required_page_says_the_certificate_did_not_reach_the_backend() + { + using var client = BuildClient( + HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.Equal(403, ex.StatusCode); + Assert.True(ex.IsForeignResponse); + Assert.False(ex.EndpointUnavailable); + Assert.Equal(ResponseClassification.ClientCertificateRequired, ex.Classification.ForeignReason); + Assert.Null(ex.ErrorCode); + // The cause leads: RegistrationFailureSummary.LastError keeps 256 characters. + Assert.StartsWith("The device certificate did not reach the backend (TLS inspection or a proxy on the network path)", ex.Message); + Assert.Contains("HTML platform error page (\"Web App - Unavailable\")", ex.Message); + Assert.True(ex.Message.IndexOf("Web App - Unavailable", StringComparison.Ordinal) < 256); + Assert.Contains("NOT a device-authorization failure", ex.Message); + Assert.DoesNotContain("stale ApiBaseUrl", ex.Message); + Assert.DoesNotContain("Check client certificate", ex.Message); + } + + [Fact] + public async Task Proxy_page_keeps_the_phrase_and_title_ANALYZE_NET_001_matches() + { + using var client = BuildClient( + HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html", fromBackend: false); + + var ex = await Assert.ThrowsAsync( + () => client.GetDiagnosticsUploadUrlAsync("tenant-1", "session-1", "package.zip")); + + Assert.True(ex.IsForeignResponse); + Assert.Equal(ResponseClassification.ForeignResponse, ex.Classification.ForeignReason); + Assert.Contains("HTML platform error page (\"Example Gateway - Notification\")", ex.Message); + Assert.Contains("403", ex.Message); + Assert.DoesNotContain("stale ApiBaseUrl", ex.Message); + } + + [Fact] + public async Task Json_403_without_the_echo_is_foreign_and_its_errorCode_is_ignored() + { + // An errorCode in a body the backend did not author must not drive a session rotation. + using var client = BuildClient( + HttpStatusCode.Forbidden, + "{\"success\":false,\"errorCode\":\"session_owner_mismatch\"}", + "application/json", + fromBackend: false); + + var ex = await Assert.ThrowsAsync( + () => client.RegisterSessionAsync(new AutopilotMonitor.Shared.Models.SessionRegistration())); + + Assert.True(ex.IsForeignResponse); + Assert.Null(ex.ErrorCode); + Assert.Contains("a response without the backend's correlation echo", ex.Message); + } + + [Fact] + public async Task Foreign_answer_carries_the_observed_server_certificate_issuer() + { + var observer = new ServerCertificateObserver(); + using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA")) + { + observer.Validate(cert, System.Net.Security.SslPolicyErrors.None); + } + using var client = BuildClient( + Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false), + observer); + + var ex = await Assert.ThrowsAsync(() => client.GetAgentConfigAsync("tenant-1")); + + Assert.Equal("CN=Example Inspection CA", ex.Classification.ServerCertIssuer); + } + + // ── agent/error: a report the backend never received is not delivered ── + + [Theory] + [InlineData(HttpStatusCode.OK, true, true)] + [InlineData(HttpStatusCode.InternalServerError, true, true)] // unchanged: only the round-trip counts + [InlineData(HttpStatusCode.Forbidden, true, true)] // the backend answered + [InlineData(HttpStatusCode.Forbidden, false, false)] // answered on the network path + [InlineData(HttpStatusCode.Unauthorized, false, false)] + public async Task ReportAgentError_counts_as_delivered_unless_the_backend_never_answered( + HttpStatusCode status, bool fromBackend, bool expectedDelivered) + { + using var client = BuildClient(status, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend); + + var delivered = await client.ReportAgentErrorAsync(new AgentErrorReport { TenantId = "tenant-1", SessionId = "session-1" }); + + Assert.Equal(expectedDelivered, delivered); + } + + [Fact] + public async Task Intercepted_emergency_report_rolls_its_reservation_back() + { + var handler = Answer(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", fromBackend: false); + using var client = BuildClient(handler); + var reporter = new EmergencyReporter(client, "session-1", "tenant-1", "2.0.0", logger: null); + + await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "first"); + await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "second"); + + // Not delivered → no dedup, no cooldown: the same key is attempted again. + Assert.Equal(2, handler.Calls); + } + [Fact] public async Task Backend_401_keeps_authorization_message_and_status() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderLoggingTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderLoggingTests.cs index d9ec99fa..0db3eb4a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderLoggingTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderLoggingTests.cs @@ -156,6 +156,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Contains("BackendTelemetryUploader: ingest PERMANENT", logContent); } + [Fact] + public async Task UploadBatch_logs_the_whole_interception_finding_at_warning() + { + // The old PERMANENT line carried the first 200 body characters, which ended before the + // platform's "Client Certificate Required" text. The INTERCEPTED line names the finding. + using var tmp = new TempDirectory(); + var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info); + + var handler = new RecordingHttpMessageHandler(); + handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage); + using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) }; + + var sut = new BackendTelemetryUploader( + httpClient: http, + baseUrl: "https://backend.test", + tenantId: "T1", + logger: logger); + + var result = await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + Assert.True(result.IsIntercepted); + + Assert.True(handler.Captured[0].TryGetHeader("X-Correlation-ID", out var corr)); + var logContent = ReadLogs(tmp.Path); + Assert.Contains("BackendTelemetryUploader: ingest INTERCEPTED", logContent); + Assert.Contains($"corr={corr}", logContent); + Assert.Contains("status=403", logContent); + Assert.Contains("reason=client_certificate_required", logContent); + Assert.Contains($"pageTitle=\"{InterceptionFixtures.PlatformPageTitle}\"", logContent); + Assert.Contains("serverCertIssuer=\"(unknown)\"", logContent); + Assert.DoesNotContain("ingest PERMANENT", logContent); + } + [Fact] public async Task UploadBatch_empty_batch_no_log_no_network() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs index 32cf0386..f06fdbcf 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/BackendTelemetryUploaderTests.cs @@ -4,7 +4,16 @@ using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry; +using AutopilotMonitor.DecisionCore.Engine; using Newtonsoft.Json.Linq; using Xunit; using AutopilotMonitor.Shared.Models; @@ -51,7 +60,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport string? model = "Latitude", string? serial = "ABC123", string? bootstrap = null, - string? version = "11.0.0") + string? version = "11.0.0", + AuthFailureTracker? tracker = null, + ServerCertificateObserver? serverCertificates = null, + NetworkMetrics? networkMetrics = null) { Http = new HttpClient(Handler) { Timeout = TimeSpan.FromSeconds(30) }; Sut = new BackendTelemetryUploader( @@ -62,7 +74,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport model: model, serialNumber: serial, bootstrapToken: bootstrap, - agentVersion: version); + agentVersion: version, + authFailureTracker: tracker, + serverCertificates: serverCertificates, + networkMetrics: networkMetrics); } public void Dispose() @@ -245,6 +260,85 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Null(result.ErrorReason); } + // ============================================================ 2xx without the correlation echo + + [Fact] + public async Task Ok_with_the_echo_is_a_success_and_not_counted() + { + var metrics = new NetworkMetrics(); + using var rig = new Rig(networkMetrics: metrics); + rig.Handler.QueueStatus(HttpStatusCode.OK); + + var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal(0, metrics.OkWithoutEchoCount); + } + + [Fact] + public async Task Ok_without_the_echo_is_still_a_success_and_only_counted() + { + // A 200 that did not come from the backend (e.g. a block page served with 200): the + // upload path behaves exactly as before — control signals are still read — and the + // only difference is the counter. + var metrics = new NetworkMetrics(); + using var tmp = new TempDirectory(); + var tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance, + new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info)); + tracker.RecordFailure(403, "agent/telemetry"); + using var rig = new Rig(tracker: tracker, networkMetrics: metrics); + rig.Handler.QueueForeign(HttpStatusCode.OK, "Example"); + rig.Handler.QueueForeign(HttpStatusCode.OK, "{\"deviceKillSignal\":true}", mediaType: "application/json"); + + var page = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + var signals = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(2) }, CancellationToken.None); + + Assert.True(page.Success); + Assert.Null(page.ErrorReason); + Assert.True(signals.Success); + Assert.True(signals.DeviceKillSignal); + Assert.Equal(0, tracker.ConsecutiveFailures); // a 2xx still resets the auth streak + Assert.Equal(2, metrics.OkWithoutEchoCount); + } + + [Fact] + public async Task Answers_other_than_2xx_never_count_as_ok_without_echo() + { + var metrics = new NetworkMetrics(); + using var rig = new Rig(networkMetrics: metrics); + rig.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage); + rig.Handler.QueueForeign(HttpStatusCode.ServiceUnavailable, "busy"); + + await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + Assert.Equal(0, metrics.OkWithoutEchoCount); + } + + [Fact] + public async Task The_count_reaches_the_agent_metrics_snapshot() + { + var metrics = new NetworkMetrics(); + using var rig = new Rig(networkMetrics: metrics); + rig.Handler.QueueForeign(HttpStatusCode.OK, "Example"); + await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + var sink = new FakeSignalIngressSink(); + using var tmp = new TempDirectory(); + var collector = new AgentSelfMetricsCollector( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(sink, SystemClock.Instance), + networkMetrics: metrics, + logger: new AgentLogger(System.IO.Path.Combine(tmp.Path, "logs"), AgentLogLevel.Info), + agentVersion: "2.0.test"); + collector.CollectSafe(); + + var snapshot = (IDictionary)Assert.Single(sink.Posted, p => + p.TypedPayload is IDictionary d && d.ContainsKey("agent_version")).TypedPayload!; + Assert.Equal(1L, snapshot["telemetry_upload_ok_without_echo"]); + } + [Theory] [InlineData(HttpStatusCode.Unauthorized)] [InlineData(HttpStatusCode.Forbidden)] @@ -261,6 +355,104 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport Assert.Contains(((int)status).ToString(), result.ErrorReason); } + // ============================================================ 401/403 origin + + private sealed class OriginRig : IDisposable + { + private readonly TempDirectory _tmp = new TempDirectory(); + + public FakeDistressReporter Distress { get; } = new FakeDistressReporter(); + public AuthFailureTracker Tracker { get; } + public Rig Upload { get; } + + public OriginRig() + { + Tracker = new AuthFailureTracker(maxFailures: 5, timeoutMinutes: 0, SystemClock.Instance, + new AgentLogger(_tmp.Path, AgentLogLevel.Info), Distress); + var serverCertificates = new ServerCertificateObserver(); + using (var cert = ServerCertificateObserverTests.SelfSigned("CN=Example Inspection CA")) + { + serverCertificates.Validate(cert, System.Net.Security.SslPolicyErrors.None); + } + Upload = new Rig(tracker: Tracker, serverCertificates: serverCertificates); + } + + public Task UploadAsync() => Upload.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None); + + public void Dispose() + { + Upload.Dispose(); + _tmp.Dispose(); + } + } + + [Fact] + public async Task Backend_403_is_an_auth_failure_that_counts_and_reports_DeviceNotRegistered() + { + using var rig = new OriginRig(); + rig.Upload.Handler.QueueStatus(HttpStatusCode.Forbidden, body: "{\"error\":\"Device validation failed\"}"); + + var result = await rig.UploadAsync(); + + Assert.True(result.IsAuthFailure); + Assert.False(result.IsIntercepted); + Assert.Equal(1, rig.Tracker.ConsecutiveFailures); + Assert.Equal(DistressErrorType.DeviceNotRegistered, Assert.Single(rig.Distress.Calls).ErrorType); + } + + [Fact] + public async Task Stopped_app_403_counts_like_an_auth_failure_without_distress() + { + using var rig = new OriginRig(); + rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage); + + var result = await rig.UploadAsync(); + + Assert.True(result.IsAuthFailure); + Assert.False(result.IsIntercepted); + Assert.Equal(1, rig.Tracker.ConsecutiveFailures); + Assert.Empty(rig.Distress.Calls); + } + + [Fact] + public async Task Certificate_required_403_is_intercepted_never_counted_and_reported_once() + { + using var rig = new OriginRig(); + rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage); + rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage); + + var result = await rig.UploadAsync(); + await rig.UploadAsync(); + + Assert.False(result.Success); + Assert.True(result.IsIntercepted); + Assert.False(result.IsAuthFailure); + Assert.False(result.IsTransient); + Assert.Equal(403, result.Interception!.StatusCode); + Assert.Equal(ResponseClassification.ClientCertificateRequired, result.Interception.ForeignReason); + Assert.Equal(InterceptionFixtures.PlatformPageTitle, result.Interception.PageTitle); + Assert.Equal("CN=Example Inspection CA", result.Interception.ServerCertIssuer); + Assert.Equal(0, rig.Tracker.ConsecutiveFailures); + var distress = Assert.Single(rig.Distress.Calls); + Assert.Equal(DistressErrorType.ClientCertificateNotReceived, distress.ErrorType); + Assert.Contains("agent/telemetry http 403 client_certificate_required", distress.Message); + Assert.Contains("CN=Example Inspection CA", distress.Message); + } + + [Fact] + public async Task Proxy_page_403_is_intercepted_as_foreign_response() + { + using var rig = new OriginRig(); + rig.Upload.Handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage); + + var result = await rig.UploadAsync(); + + Assert.True(result.IsIntercepted); + Assert.Equal(ResponseClassification.ForeignResponse, result.Interception!.ForeignReason); + Assert.Equal(InterceptionFixtures.ProxyPageTitle, result.Interception.PageTitle); + Assert.Equal(0, rig.Tracker.ConsecutiveFailures); + } + [Theory] [InlineData(HttpStatusCode.BadRequest)] // 400 [InlineData(HttpStatusCode.NotFound)] // 404 diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs index e76601ef..ac8c7ec0 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/FakeBackendTelemetryUploader.cs @@ -74,6 +74,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport return this; } + /// Queues a 403 the platform's "Client Certificate Required" page answered (an interception). + public FakeBackendTelemetryUploader QueueIntercepted(int times = 1) + { + for (int i = 0; i < times; i++) + { + _script.Enqueue(_ => UploadResult.Intercepted( + "intercepted: http 403 (client_certificate_required)", InterceptionFixtures.CertificateRequired())); + } + return this; + } + public FakeBackendTelemetryUploader QueueThrow(Exception ex) { _script.Enqueue(_ => throw ex); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/InterceptionFixtures.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/InterceptionFixtures.cs new file mode 100644 index 00000000..c1cffac0 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/InterceptionFixtures.cs @@ -0,0 +1,48 @@ +using AutopilotMonitor.Agent.V2.Core.Transport; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport +{ + /// + /// Synthetic copies of the pages an agent gets instead of a backend answer: the Azure platform + /// page when no client certificate arrived, the platform page of a stopped app, and a proxy's + /// block page. Rebuilt from the observed structure and wording; none of them is a captured + /// artifact. The style block keeps the platform marker text past the first 200 characters, where + /// the old upload log excerpt ended. + /// + internal static class InterceptionFixtures + { + public const string PlatformPageTitle = "Web App - Unavailable"; + public const string ProxyPageTitle = "Example Gateway - Notification"; + public const string InspectionIssuer = "CN=Example Inspection CA, O=Example Security Ltd, C=US"; + + private const string PlatformPageHead = + "\r\n" + + "\r\n\r\nWeb App - Unavailable\r\n" + + "\r\n\r\n\r\n
\r\n"; + + public const string CertificateRequiredPage = + PlatformPageHead + + "

Error 403 - Forbidden: Client Certificate Required.

\r\n" + + "

The web app you have attempted to reach requires a client certificate and does not accept any requests without one.

\r\n" + + "
\r\n\r\n\r\n"; + + public const string StoppedAppPage = + PlatformPageHead + + "

Error 403 - This web app is stopped.

\r\n" + + "

The web app you have attempted to reach is currently stopped and does not accept any requests.

\r\n" + + "\r\n\r\n\r\n"; + + public const string ProxyPage = + "\r\nExample Gateway - Notification\r\n" + + "

Access blocked

The request was blocked by the security policy of your organization.

\r\n"; + + /// The platform's "Client Certificate Required" page seen through an inspecting proxy. + public static ResponseClassification CertificateRequired(int statusCode = 403) => + ResponseClassification.ForForeign(statusCode, ResponseClassification.ClientCertificateRequired, PlatformPageTitle, isHtml: true, InspectionIssuer); + + public static ResponseClassification StoppedApp(int statusCode = 403) => + ResponseClassification.ForPlatformStopped(statusCode, PlatformPageTitle, isHtml: true, serverCertIssuer: null); + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/RecordingHttpMessageHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/RecordingHttpMessageHandler.cs index cf29bfa8..0ec6c7fe 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/RecordingHttpMessageHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/RecordingHttpMessageHandler.cs @@ -7,6 +7,7 @@ using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Transport; namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport { @@ -23,17 +24,33 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport public IReadOnlyList Captured => _captured; + /// + /// Queues a response from the backend: like the backend's correlation middleware, it echoes + /// the request's X-Correlation-ID — the proof of origin a 401/403 is classified by. + /// public RecordingHttpMessageHandler QueueStatus(HttpStatusCode status, string? body = null) { - _script.Enqueue(_ => + _script.Enqueue(request => { var resp = new HttpResponseMessage(status); if (body != null) resp.Content = new StringContent(body, Encoding.UTF8, "application/json"); + if (request.Headers.TryGetValues(ResponseOriginClassifier.CorrelationIdHeader, out var ids)) + resp.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, ids); return resp; }); return this; } + /// + /// Queues a response that something on the network path produced (platform page, proxy + /// page): no correlation echo. + /// + public RecordingHttpMessageHandler QueueForeign(HttpStatusCode status, string body, string mediaType = "text/html") + { + _script.Enqueue(_ => new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, mediaType) }); + return this; + } + public RecordingHttpMessageHandler QueueThrow(Exception ex) { _script.Enqueue(_ => throw ex); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ResponseOriginClassifierTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ResponseOriginClassifierTests.cs new file mode 100644 index 00000000..d6320694 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ResponseOriginClassifierTests.cs @@ -0,0 +1,173 @@ +#nullable enable +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Transport; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport +{ + /// + /// : only the echo of the request's correlation id proves + /// that a 401/403 came from the backend. Fixtures are synthetic copies of the platform and proxy + /// pages (). + /// + public sealed class ResponseOriginClassifierTests + { + private const string SentId = "0123456789abcdef0123456789abcdef"; + + private static HttpResponseMessage Response(HttpStatusCode status, string? body, string mediaType, string? echo = null) + { + var response = new HttpResponseMessage(status); + if (body != null) response.Content = new StringContent(body, Encoding.UTF8, mediaType); + if (echo != null) response.Headers.TryAddWithoutValidation(ResponseOriginClassifier.CorrelationIdHeader, echo); + return response; + } + + private static ResponseClassification Classify(HttpResponseMessage response, string? issuer = null) + { + var body = ResponseOriginClassifier.ReadBodyPrefixAsync(response).GetAwaiter().GetResult(); + return ResponseOriginClassifier.Classify(response, SentId, body, issuer); + } + + [Fact] + public void Platform_certificate_required_page_is_foreign_client_certificate_required() + { + using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html"); + + var c = Classify(response, InterceptionFixtures.InspectionIssuer); + + Assert.Equal(ResponseOrigin.Foreign, c.Origin); + Assert.True(c.IsForeign); + Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason); + Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle); + Assert.True(c.IsHtml); + Assert.Equal(403, c.StatusCode); + Assert.Equal(InterceptionFixtures.InspectionIssuer, c.ServerCertIssuer); + } + + [Fact] + public void Platform_stopped_app_page_is_platform_stopped() + { + using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.StoppedAppPage, "text/html"); + + var c = Classify(response); + + Assert.Equal(ResponseOrigin.PlatformStopped, c.Origin); + Assert.False(c.IsForeign); + Assert.Null(c.ForeignReason); + Assert.Equal(InterceptionFixtures.PlatformPageTitle, c.PageTitle); + } + + [Fact] + public void Proxy_page_is_foreign_response_with_its_title() + { + using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.ProxyPage, "text/html"); + + var c = Classify(response); + + Assert.Equal(ResponseOrigin.Foreign, c.Origin); + Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason); + Assert.Equal(InterceptionFixtures.ProxyPageTitle, c.PageTitle); + } + + [Fact] + public void Json_403_with_echoed_correlation_id_is_backend() + { + using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json", echo: SentId); + + Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin); + } + + [Fact] + public void Json_403_without_echo_is_foreign_response() + { + using var response = Response(HttpStatusCode.Forbidden, "{\"error\":\"Device validation failed\"}", "application/json"); + + var c = Classify(response); + + Assert.Equal(ResponseOrigin.Foreign, c.Origin); + Assert.Equal(ResponseClassification.ForeignResponse, c.ForeignReason); + Assert.False(c.IsHtml); + Assert.Null(c.PageTitle); + } + + [Fact] + public void Html_page_with_echo_is_backend_because_the_header_wins() + { + using var response = Response(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage, "text/html", echo: SentId); + + Assert.Equal(ResponseOrigin.Backend, Classify(response).Origin); + } + + [Fact] + public void Echo_of_another_request_id_is_not_proof() + { + using var response = Response(HttpStatusCode.Unauthorized, "{}", "application/json", echo: "ffffffffffffffffffffffffffffffff"); + + Assert.Equal(ResponseOrigin.Foreign, Classify(response).Origin); + } + + [Fact] + public void No_sent_id_can_never_be_echoed() + { + using var response = Response(HttpStatusCode.Forbidden, "{}", "application/json", echo: SentId); + + Assert.Equal(ResponseOrigin.Foreign, ResponseOriginClassifier.Classify(response, sentCorrelationId: null, bodyPrefix: "{}").Origin); + } + + [Fact] + public void Markers_match_case_insensitively_and_in_plain_text() + { + using var stopped = Response(HttpStatusCode.Forbidden, "error 403 - this web app is stopped.", "text/plain"); + using var certificate = Response(HttpStatusCode.Forbidden, "Forbidden: CLIENT CERTIFICATE REQUIRED", "text/plain"); + + Assert.Equal(ResponseOrigin.PlatformStopped, Classify(stopped).Origin); + var c = Classify(certificate); + Assert.Equal(ResponseClassification.ClientCertificateRequired, c.ForeignReason); + Assert.False(c.IsHtml); + } + + [Fact] + public void Html_served_with_generic_content_type_is_sniffed() + { + using var response = Response(HttpStatusCode.Forbidden, " Blocked", "text/plain"); + + var c = Classify(response); + + Assert.True(c.IsHtml); + Assert.Equal("Blocked", c.PageTitle); + } + + [Fact] + public void Page_title_is_capped_at_80_characters() + { + var longTitle = new string('x', 120); + using var response = Response(HttpStatusCode.Forbidden, $"{longTitle}", "text/html"); + + Assert.Equal(new string('x', 80), Classify(response).PageTitle); + } + + [Fact] + public async Task Body_prefix_is_capped_and_a_marker_beyond_it_is_not_seen() + { + var body = new string(' ', ResponseOriginClassifier.MaxBodyChars) + "Client Certificate Required"; + using var response = Response(HttpStatusCode.Forbidden, body, "text/plain"); + + var prefix = await ResponseOriginClassifier.ReadBodyPrefixAsync(response); + + Assert.Equal(ResponseOriginClassifier.MaxBodyChars, prefix!.Length); + Assert.Equal(ResponseClassification.ForeignResponse, ResponseOriginClassifier.Classify(response, SentId, prefix).ForeignReason); + } + + [Fact] + public async Task Missing_body_reads_as_null_and_classifies_as_foreign_response() + { + using var response = new HttpResponseMessage(HttpStatusCode.Forbidden); + + Assert.Null(await ResponseOriginClassifier.ReadBodyPrefixAsync(response)); + Assert.Equal(ResponseClassification.ForeignResponse, Classify(response).ForeignReason); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ServerCertificateObserverTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ServerCertificateObserverTests.cs new file mode 100644 index 00000000..f8d3383f --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/ServerCertificateObserverTests.cs @@ -0,0 +1,94 @@ +#nullable enable +using System; +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using AutopilotMonitor.Agent.V2.Core.Transport; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport +{ + /// + /// must never weaken TLS validation: the installed + /// callback returns exactly the verdict .NET Framework applies without one + /// (sslPolicyErrors == None) and only records the issuer as evidence. + /// + public sealed class ServerCertificateObserverTests + { + internal static X509Certificate2 SelfSigned(string subject) + { + using (var rsa = RSA.Create(2048)) + { + var request = new CertificateRequest(subject, rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddDays(-1), DateTimeOffset.UtcNow.AddDays(1)); + } + } + + private static Func AttachedCallback(ServerCertificateObserver observer) + { + var handler = new HttpClientHandler(); + observer.Attach(handler); + return handler.ServerCertificateCustomValidationCallback!; + } + + [Theory] + [InlineData(SslPolicyErrors.None, true)] + [InlineData(SslPolicyErrors.RemoteCertificateNotAvailable, false)] + [InlineData(SslPolicyErrors.RemoteCertificateNameMismatch, false)] + [InlineData(SslPolicyErrors.RemoteCertificateChainErrors, false)] + [InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch, false)] + [InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateChainErrors, false)] + [InlineData(SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)] + [InlineData(SslPolicyErrors.RemoteCertificateNotAvailable | SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors, false)] + public void Callback_accepts_only_a_certificate_without_policy_errors(SslPolicyErrors errors, bool expected) + { + using var certificate = SelfSigned("CN=Example Inspection CA"); + var observer = new ServerCertificateObserver(); + var callback = AttachedCallback(observer); + + Assert.Equal(expected, callback(null!, certificate, null!, errors)); + // Recorded whatever the verdict: a rejected inspection certificate is evidence too. + Assert.Equal(certificate.Issuer, observer.LastIssuer); + } + + [Fact] + public void Last_validated_connection_wins() + { + using var first = SelfSigned("CN=Example Backend CA"); + using var second = SelfSigned("CN=Example Inspection CA"); + var observer = new ServerCertificateObserver(); + var callback = AttachedCallback(observer); + + Assert.Null(observer.LastIssuer); + callback(null!, first, null!, SslPolicyErrors.None); + callback(null!, second, null!, SslPolicyErrors.None); + + Assert.Equal(second.Issuer, observer.LastIssuer); + } + + [Fact] + public void Missing_certificate_is_rejected_and_keeps_the_last_issuer() + { + using var certificate = SelfSigned("CN=Example Backend CA"); + var observer = new ServerCertificateObserver(); + var callback = AttachedCallback(observer); + callback(null!, certificate, null!, SslPolicyErrors.None); + + Assert.False(callback(null!, null!, null!, SslPolicyErrors.RemoteCertificateNotAvailable)); + Assert.Equal(certificate.Issuer, observer.LastIssuer); + } + + [Fact] + public void Long_issuer_is_capped() + { + using var certificate = SelfSigned($"CN=Example Inspection CA, O={new string('o', 200)}"); + var observer = new ServerCertificateObserver(); + + observer.Validate(certificate, SslPolicyErrors.None); + + Assert.Equal(ServerCertificateObserver.MaxIssuerLength, observer.LastIssuer!.Length); + Assert.StartsWith(observer.LastIssuer, certificate.Issuer, StringComparison.Ordinal); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorInterceptionTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorInterceptionTests.cs new file mode 100644 index 00000000..a187ef7d --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Transport/TelemetryUploadOrchestratorInterceptionTests.cs @@ -0,0 +1,282 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Transport; +using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport +{ + /// + /// The transport-local pause after an intercepted upload (a 401/403 something on the network + /// path answered): no HTTP while it holds, backoff 30 s → 15 min, a success ends it, the + /// terminal phase gets exactly one attempt despite it, and it is never an auth failure. + /// Driven by the virtual clock — the pause is a "not before", never a timer. + /// + public sealed class TelemetryUploadOrchestratorInterceptionTests + { + private static readonly DateTime Start = new DateTime(2026, 10, 12, 10, 0, 0, DateTimeKind.Utc); + + private sealed class Rig : IDisposable + { + private int _rowKey; + + public TempDirectory Tmp { get; } = new TempDirectory(); + public VirtualClock Clock { get; } = new VirtualClock(Start); + public FakeBackendTelemetryUploader Uploader { get; } = new FakeBackendTelemetryUploader(); + public TelemetryUploadOrchestrator Sut { get; } + + public Rig() + { + var spool = new TelemetrySpool(Tmp.Path, Clock); + Sut = new TelemetryUploadOrchestrator(spool, Uploader, Clock, batchSize: 10, retryBackoffs: new[] { TimeSpan.Zero }); + } + + public void Enqueue(int count = 1) + { + for (int i = 0; i < count; i++) + { + var rowKey = (++_rowKey).ToString("D10"); + Sut.Enqueue(new TelemetryItemDraft( + kind: TelemetryItemKind.Event, + partitionKey: "T1_S1", + rowKey: rowKey, + payloadJson: "{}", + isSessionScoped: true, + requiresImmediateFlush: false)); + } + } + + public void Dispose() { Sut.Dispose(); Tmp.Dispose(); } + } + + [Fact] + public async Task Intercepted_upload_retains_the_batch_and_later_drains_make_no_http_call() + { + using var rig = new Rig(); + rig.Enqueue(2); + rig.Uploader.QueueIntercepted(); + + var first = await rig.Sut.DrainAllAsync(); + + Assert.Equal(1, rig.Uploader.CallCount); + Assert.Equal(1, first.FailedBatches); + Assert.Equal(0, first.UploadedItems); + Assert.Contains("intercepted", first.LastErrorReason); + Assert.Equal(-1, rig.Sut.LastUploadedItemId); + Assert.True(rig.Sut.IsInterceptionPaused); + Assert.False(rig.Sut.IsPaused); + + // Periodic drains and immediate-flush wakeups (a new item, another drain) during the + // pause: no HTTP, and an expected state rather than another failed batch. + rig.Enqueue(); + var paused = await rig.Sut.DrainAllAsync(); + rig.Clock.Advance(TimeSpan.FromSeconds(29)); + await rig.Sut.DrainAllAsync(); + + Assert.Equal(1, rig.Uploader.CallCount); + Assert.Equal(0, paused.FailedBatches); + Assert.Contains("upload intercepted, next attempt not before", paused.LastErrorReason); + Assert.Equal(-1, rig.Sut.LastUploadedItemId); + } + + [Fact] + public async Task Backoff_runs_30s_1m_2m_5m_10m_then_15m_capped() + { + var expected = new[] { 30, 60, 120, 300, 600, 900, 900 }.Select(s => TimeSpan.FromSeconds(s)).ToArray(); + Assert.Equal(expected.Take(6), TelemetryUploadOrchestrator.InterceptionBackoffs); + + using var rig = new Rig(); + rig.Enqueue(); + rig.Uploader.QueueIntercepted(expected.Length); + + foreach (var delay in expected) + { + var calls = rig.Uploader.CallCount; + await rig.Sut.DrainAllAsync(); + Assert.Equal(calls + 1, rig.Uploader.CallCount); + + rig.Clock.Advance(delay - TimeSpan.FromSeconds(1)); + Assert.True(rig.Sut.IsInterceptionPaused); + await rig.Sut.DrainAllAsync(); + Assert.Equal(calls + 1, rig.Uploader.CallCount); + + rig.Clock.Advance(TimeSpan.FromSeconds(1)); + Assert.False(rig.Sut.IsInterceptionPaused); + } + } + + /// A clock that can be set back, as NTP does to a device that booted with a future RTC time. + private sealed class SettableClock : AutopilotMonitor.DecisionCore.Engine.IClock + { + public SettableClock(DateTime utcNow) { UtcNow = utcNow; } + public DateTime UtcNow { get; set; } + public Task Delay(TimeSpan delay, System.Threading.CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + [Fact] + public async Task Clock_set_back_past_the_pause_start_ends_the_pause_instead_of_stretching_it() + { + using var tmp = new TempDirectory(); + var clock = new SettableClock(Start); + var uploader = new FakeBackendTelemetryUploader().QueueIntercepted(2); + using var sut = new TelemetryUploadOrchestrator(new TelemetrySpool(tmp.Path, clock), uploader, clock, retryBackoffs: new[] { TimeSpan.Zero }); + sut.Enqueue(new TelemetryItemDraft(TelemetryItemKind.Event, "T1_S1", "0000000001", "{}", isSessionScoped: true)); + + await sut.DrainAllAsync(); + clock.UtcNow = Start.AddHours(-2); + + Assert.False(sut.IsInterceptionPaused); + await sut.DrainAllAsync(); + Assert.Equal(2, uploader.CallCount); + Assert.True(sut.IsInterceptionPaused); // re-armed from the corrected clock + } + + [Fact] + public async Task Successful_upload_ends_the_pause_and_resets_the_backoff() + { + using var rig = new Rig(); + rig.Enqueue(3); + rig.Uploader.QueueIntercepted(2).QueueOk().QueueIntercepted(); + + await rig.Sut.DrainAllAsync(); // intercepted, 30 s + rig.Clock.Advance(TimeSpan.FromSeconds(30)); + await rig.Sut.DrainAllAsync(); // intercepted, 1 min + rig.Clock.Advance(TimeSpan.FromMinutes(1)); + var cleared = await rig.Sut.DrainAllAsync(); // the path is clear + + Assert.Equal(3, cleared.UploadedItems); + Assert.False(rig.Sut.IsInterceptionPaused); + + rig.Enqueue(); + await rig.Sut.DrainAllAsync(); // intercepted again: back to 30 s + rig.Clock.Advance(TimeSpan.FromSeconds(29)); + Assert.True(rig.Sut.IsInterceptionPaused); + rig.Clock.Advance(TimeSpan.FromSeconds(1)); + Assert.False(rig.Sut.IsInterceptionPaused); + } + + [Fact] + public async Task Terminal_drain_makes_exactly_one_attempt_despite_the_pause() + { + using var rig = new Rig(); + rig.Enqueue(); + rig.Uploader.QueueIntercepted(3); + await rig.Sut.DrainAllAsync(); + Assert.Equal(1, rig.Uploader.CallCount); + + Assert.True(rig.Sut.GrantTerminalAttempt()); + var terminal = await rig.Sut.DrainAllAsync(); + + Assert.Equal(2, rig.Uploader.CallCount); + Assert.Equal(1, terminal.FailedBatches); + Assert.True(rig.Sut.IsInterceptionPaused); + + // One attempt per process: a later terminal drain (orchestrator.Stop) honours the pause. + rig.Sut.GrantTerminalAttempt(); + await rig.Sut.DrainAllAsync(); + Assert.Equal(2, rig.Uploader.CallCount); + } + + [Fact] + public async Task Terminal_attempt_that_gets_through_drains_everything() + { + using var rig = new Rig(); + rig.Enqueue(25); + rig.Uploader.QueueIntercepted().QueueOk(3); + await rig.Sut.DrainAllAsync(); + + rig.Sut.GrantTerminalAttempt(); + var terminal = await rig.Sut.DrainAllAsync(); + + Assert.Equal(25, terminal.UploadedItems); + Assert.Equal(4, rig.Uploader.CallCount); + Assert.Equal(24, rig.Sut.LastUploadedItemId); + Assert.False(rig.Sut.IsInterceptionPaused); + } + + [Fact] + public async Task Grant_without_a_pause_changes_nothing_until_an_interception_meets_it() + { + using var rig = new Rig(); + Assert.False(rig.Sut.GrantTerminalAttempt()); + rig.Enqueue(); + rig.Uploader.QueueOk().QueueIntercepted(3); + + var normal = await rig.Sut.DrainAllAsync(); + Assert.Equal(1, normal.UploadedItems); + + rig.Enqueue(); + await rig.Sut.DrainAllAsync(); // intercepted during the terminal phase + await rig.Sut.DrainAllAsync(); // the granted attempt + await rig.Sut.DrainAllAsync(); // pause honoured again + + Assert.Equal(3, rig.Uploader.CallCount); + } + + [Fact] + public async Task DeviceBlocked_pause_still_skips_terminal_drains() + { + using var rig = new Rig(); + rig.Enqueue(); + rig.Uploader.QueueOkWithSignals(deviceBlocked: true, unblockAt: Start.AddMinutes(10)); + await rig.Sut.DrainAllAsync(); + Assert.True(rig.Sut.IsPaused); + + rig.Enqueue(); + Assert.False(rig.Sut.GrantTerminalAttempt()); + var terminal = await rig.Sut.DrainAllAsync(); + + Assert.Equal(1, rig.Uploader.CallCount); + Assert.Contains("device blocked", terminal.LastErrorReason); + } + + [Fact] + public async Task First_interception_is_reported_once_and_never_as_a_blocked_or_auth_failure() + { + using var rig = new Rig(); + rig.Enqueue(); + rig.Uploader.QueueIntercepted(2); + var reports = new List(); + var blocked = 0; + rig.Sut.UploadIntercepted += (_, r) => reports.Add(r); + rig.Sut.UploadBlocked += (_, __) => blocked++; + + await rig.Sut.DrainAllAsync(); + rig.Clock.Advance(TimeSpan.FromSeconds(30)); + await rig.Sut.DrainAllAsync(); + + Assert.Equal(2, rig.Uploader.CallCount); + var report = Assert.Single(reports); + Assert.Equal(Start, report.FirstBlockedAtUtc); + Assert.Equal(ResponseClassification.ClientCertificateRequired, report.Interception.ForeignReason); + Assert.Equal(403, report.Interception.StatusCode); + Assert.Equal(InterceptionFixtures.InspectionIssuer, report.Interception.ServerCertIssuer); + Assert.Equal(0, blocked); + } + + [Fact] + public async Task Permanent_block_report_keeps_its_own_once_per_process_gate() + { + using var rig = new Rig(); + rig.Enqueue(); + rig.Uploader.QueueIntercepted().QueuePermanent("http 400: contract").QueuePermanent("http 400: contract"); + int intercepted = 0, blocked = 0; + rig.Sut.UploadIntercepted += (_, __) => intercepted++; + rig.Sut.UploadBlocked += (_, __) => blocked++; + + await rig.Sut.DrainAllAsync(); + rig.Clock.Advance(TimeSpan.FromSeconds(30)); + await rig.Sut.DrainAllAsync(); + await rig.Sut.DrainAllAsync(); + + Assert.Equal(3, rig.Uploader.CallCount); + Assert.Equal(1, intercepted); + Assert.Equal(1, blocked); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/AgentConfiguration.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/AgentConfiguration.cs index 6c883f87..2497f905 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/AgentConfiguration.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/AgentConfiguration.cs @@ -195,6 +195,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration /// public int AbsoluteMaxSessionHours { get; set; } = 48; + /// + /// Further boots the agent may try after a session registration was answered by a + /// 401/403 the backend did not author, before it cleans up like the emergency break. + /// Merged from remote config (also from the cache, since an intercepted start gets no + /// live config), clamped to 0..5. Default: 1. + /// + public int InterceptedRegistrationRetryBoots { get; set; } = 1; + /// /// Bootstrap token for pre-MDM auth during OOBE. /// Embedded by the OOBE bootstrapper script via --bootstrap-token CLI arg. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs index 7e071b20..c635463f 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigMerger.cs @@ -28,6 +28,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration /// public static class RemoteConfigMerger { + /// + /// Upper bound for : each + /// extra boot keeps an intercepted device's agent installed until its next boot. + /// + private const int MaxInterceptedRegistrationRetryBoots = 5; + /// /// Overwrites the relevant fields with values from /// . Remote wins unconditionally for every field that has @@ -174,6 +180,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration agentConfig.MaxAuthFailures = remoteConfig.MaxAuthFailures; agentConfig.AuthFailureTimeoutMinutes = remoteConfig.AuthFailureTimeoutMinutes; + // ---------------- Intercepted-registration buffer — null (a backend that predates the + // field) keeps the current value; anything else is clamped to + // 0..MaxInterceptedRegistrationRetryBoots. + if (remoteConfig.InterceptedRegistrationRetryBoots.HasValue) + { + var requested = remoteConfig.InterceptedRegistrationRetryBoots.Value; + var bounded = Math.Max(0, Math.Min(MaxInterceptedRegistrationRetryBoots, requested)); + if (bounded != requested) + logger?.Warning($"Remote config: InterceptedRegistrationRetryBoots {requested} is outside 0..{MaxInterceptedRegistrationRetryBoots} — using {bounded}."); + agentConfig.InterceptedRegistrationRetryBoots = bounded; + } + // ---------------- Nested CollectorConfiguration — only the two knobs that have a // V2 consumer on AgentConfiguration itself. The rest flow through // the untouched remoteConfig.Collectors to DefaultComponentFactory. diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs index 98abbf57..f1fd92aa 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs @@ -178,10 +178,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration { // Auth failures are NOT retryable — 401/403 won't go away with another // attempt; record + bail straight to cache/defaults so an operator can - // see the AuthCertificateRejected/DeviceNotRegistered distress signal - // without 100 s of needless retry delay. - _logger.Warning($"Config fetch authentication failed: {ex.Message}"); - _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.EndpointUnavailable); + // see the distress signal without 100 s of needless retry delay. A 401/403 + // from something on the network path is not counted by the tracker; it + // reports ClientCertificateNotReceived instead. + _logger.Warning(ex.IsForeignResponse + ? $"Config fetch intercepted on the network path: {ex.Message}" + : $"Config fetch authentication failed: {ex.Message}"); + _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config", ex.Classification); lastException = ex; LastFetchAuthStatusCode = ex.StatusCode; break; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/NetworkFingerprint.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/NetworkFingerprint.cs new file mode 100644 index 00000000..8ab26b5a --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/NetworkFingerprint.cs @@ -0,0 +1,48 @@ +#nullable enable +using System; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic; +using AutopilotMonitor.Agent.V2.Core.Persistence; + +namespace AutopilotMonitor.Agent.V2.Core.Monitoring +{ + /// + /// Short local fingerprint of the network a start ran on: the first hex + /// characters of SHA-256 over the active interface's connection type and default gateway, as + /// captures them. Stored beside the failed-registration + /// record and compared by the next successful registration; only that comparison (changed or + /// not) leaves the device, never a fingerprint. + /// + internal static class NetworkFingerprint + { + internal const int Length = 16; + + /// Fingerprint of the current network; null when it is unknown. Never throws. + public static string? TryCapture() + { + try + { + return Of(NetworkChangeDetector.CaptureInterfaceState()); + } + catch + { + return null; + } + } + + /// Fingerprint of a captured state; null without an active interface, type or gateway. + internal static string? Of(NetworkChangeDetector.NetworkStateSnapshot? state) + { + if (state == null || !state.HasNetwork + || string.IsNullOrEmpty(state.ConnectionType) || string.IsNullOrEmpty(state.Gateway)) + { + return null; + } + + return StartupEventGate.HashString(state.ConnectionType + "|" + state.Gateway).Substring(0, Length); + } + + /// True/false when both fingerprints are known, null when either is unknown. + public static bool? Differs(string? before, string? after) => + before == null || after == null ? (bool?)null : !string.Equals(before, after, StringComparison.Ordinal); + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/AgentSelfMetricsCollector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/AgentSelfMetricsCollector.cs index c503257a..825f448a 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/AgentSelfMetricsCollector.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/AgentSelfMetricsCollector.cs @@ -85,12 +85,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic protected override void Collect() { - // Full path writes 38 keys: agent_version + process metrics (5: cpu, ws, private, - // threads, handles) + spool stats (5) + network delta (9: requests, failures, - // bytes_up/down, avg_latency, total_up/down/requests/latency) + IME tracker health - // (9) + signal pipeline (9: 5 counters, stage, applied ordinal, wg level+score). - // cap=38 → HashHelpers.GetPrime(38)=41 buckets → no resize on the last key. - var data = new Dictionary(capacity: 38, StringComparer.Ordinal) + // Full path writes 48 keys: agent_version + process metrics (5: cpu, ws, private, + // threads, handles) + spool stats (5) + network (10: requests, failures, + // bytes_up/down, avg_latency, total_up/down/requests/latency, upload 2xx without + // echo) + IME tracker health (18) + signal pipeline (9: 5 counters, stage, applied + // ordinal, wg level+score). cap=48 → no resize on the last key. + var data = new Dictionary(capacity: 48, StringComparer.Ordinal) { { "agent_version", _agentVersion } }; @@ -160,6 +160,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic } // --- Network delta --- + // Cumulative and independent of the delta baseline: telemetry uploads answered 2xx + // without the backend's correlation echo (counted only — see BackendTelemetryUploader). + data["telemetry_upload_ok_without_echo"] = _networkMetrics.OkWithoutEchoCount; try { var currentNet = _networkMetrics.GetSnapshot(); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/NetworkChangeDetector.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/NetworkChangeDetector.cs index 52d76a57..d95d4027 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/NetworkChangeDetector.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Telemetry/Periodic/NetworkChangeDetector.cs @@ -37,6 +37,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic private const int ConnectivityCheckDelayMs = 5000; private const int ConnectivityCheckTimeoutMs = 8000; private const string Source = "NetworkChangeDetector"; + private const string WifiConnectionType = "WiFi"; /// /// Well-known MDM/Entra endpoints that must be reachable for enrollment to succeed. @@ -312,6 +313,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic // ------------------------------------------------------------------- private NetworkStateSnapshot CaptureCurrentState() + { + var snapshot = CaptureInterfaceState(_logger); + + // Collect WiFi SSID synchronously (needed for before/after comparison) + if (snapshot.ConnectionType == WifiConnectionType) + { + Guid? nicGuid = Guid.TryParse(snapshot.InterfaceId, out var parsedGuid) ? parsedGuid : (Guid?)null; + var wifiInfo = CollectWiFiInfo(nicGuid); + snapshot.WifiSsid = wifiInfo.ssid; + snapshot.WifiSignalPercent = wifiInfo.signal; + snapshot.WifiRadioType = wifiInfo.radioType; + } + + return snapshot; + } + + /// + /// Interface-level state of the active network (): + /// connection type, adapter, first IPv4 address, first gateway — no WiFi details. Shared + /// with so the registration record fingerprints the network + /// exactly as this detector sees it. Never throws: a failed read is "no network". + /// + internal static NetworkStateSnapshot CaptureInterfaceState(AgentLogger logger = null) { try { @@ -346,13 +370,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic } } - var snapshot = new NetworkStateSnapshot + return new NetworkStateSnapshot { HasNetwork = true, AdapterName = activeNic.Name, AdapterDescription = activeNic.Description, InterfaceType = activeNic.NetworkInterfaceType.ToString(), - ConnectionType = isWifi ? "WiFi" : "Ethernet", + ConnectionType = isWifi ? WifiConnectionType : "Ethernet", MacAddress = activeNic.GetPhysicalAddress().ToString(), InterfaceId = activeNic.Id, LinkSpeedMbps = activeNic.Speed / 1_000_000, @@ -360,22 +384,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic Gateway = gateway, CapturedAt = DateTime.UtcNow }; - - // Collect WiFi SSID synchronously (needed for before/after comparison) - if (isWifi) - { - Guid? nicGuid = Guid.TryParse(activeNic.Id, out var parsedGuid) ? parsedGuid : (Guid?)null; - var wifiInfo = CollectWiFiInfo(nicGuid); - snapshot.WifiSsid = wifiInfo.ssid; - snapshot.WifiSignalPercent = wifiInfo.signal; - snapshot.WifiRadioType = wifiInfo.radioType; - } - - return snapshot; } catch (Exception ex) { - _logger.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}"); + logger?.Debug($"NetworkChangeDetector: state capture failed: {ex.Message}"); return new NetworkStateSnapshot { HasNetwork = false, CapturedAt = DateTime.UtcNow }; } } @@ -434,7 +446,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic if (snapshot == null || !snapshot.HasNetwork) return "None"; - if (snapshot.ConnectionType == "WiFi" && !string.IsNullOrEmpty(snapshot.WifiSsid)) + if (snapshot.ConnectionType == WifiConnectionType && !string.IsNullOrEmpty(snapshot.WifiSsid)) return $"WiFi '{snapshot.WifiSsid}'"; return $"{snapshot.ConnectionType} ({snapshot.IpAddress ?? "no IP"})"; @@ -465,7 +477,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic if (before.ConnectionType != after.ConnectionType) return "type_change"; - if (before.ConnectionType == "WiFi" && after.ConnectionType == "WiFi" && + if (before.ConnectionType == WifiConnectionType && after.ConnectionType == WifiConnectionType && before.WifiSsid != after.WifiSsid) return "ssid_change"; @@ -479,7 +491,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic // Inner type: network state snapshot // ------------------------------------------------------------------- - private class NetworkStateSnapshot + internal sealed class NetworkStateSnapshot { public bool HasNetwork { get; set; } public string AdapterName { get; set; } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs index cbd2d650..ac14f5b9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/BackendApiClient.cs @@ -1,9 +1,11 @@ using System; using System.Linq; +using System.Net; using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.Models; using Newtonsoft.Json; @@ -40,6 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport private readonly string _bootstrapToken; private readonly string _agentVersion; private readonly Logging.AgentLogger _logger; + private readonly ServerCertificateObserver _serverCertificates; /// /// Protected ctor for testability — SessionRegistrationHelperTests.FakeApiClient @@ -48,6 +51,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// protected BackendApiClient() { } + /// + /// Observer installed on the 's handler; its issuer becomes + /// evidence on a 401/403 that did not come from the backend. Null = no evidence. + /// public BackendApiClient( HttpClient httpClient, string baseUrl, @@ -57,7 +64,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport bool useBootstrapTokenAuth, string bootstrapToken, string agentVersion, - Logging.AgentLogger logger) + Logging.AgentLogger logger, + ServerCertificateObserver serverCertificates = null) { _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("baseUrl is mandatory.", nameof(baseUrl)); @@ -69,6 +77,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport _bootstrapToken = bootstrapToken; _agentVersion = agentVersion; _logger = logger; + _serverCertificates = serverCertificates; } /// @@ -107,7 +116,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport { using (var httpRequest = new HttpRequestMessage(HttpMethod.Get, url)) { - AddSecurityHeaders(httpRequest); + var correlationId = AddSecurityHeaders(httpRequest); _logger?.Debug($"GetAgentConfigAsync: GET {url} (attempt {attempt}/{maxAttempts})"); HttpResponseMessage response; @@ -137,7 +146,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport continue; } - await ThrowOnAuthFailureAsync(response).ConfigureAwait(false); + await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false); response.EnsureSuccessStatusCode(); var responseJson = await response.Content.ReadAsStringAsync().ConfigureAwait(false); @@ -173,9 +182,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// emergency-break path passes a longer because its first /// request of a freshly booted process pays cold TLS + DNS + a possible backend /// cold start (observed 503s take well over 5 s). - /// Returns true when the HTTP round-trip completed (the endpoint always - /// answers 200), false on timeout/transport failure — the caller-side - /// anti-flood layer uses this to decide whether the report counts as delivered. + /// Returns true when the HTTP round-trip completed (the endpoint answers 200 for + /// every report it receives), false on timeout/transport failure or when a + /// 401/403 came from something other than the backend (the report never arrived) — the + /// caller-side anti-flood layer uses this to decide whether the report counts as delivered. /// Virtual for tests. /// public virtual async Task ReportAgentErrorAsync(AgentErrorReport report, TimeSpan? timeout = null) @@ -195,13 +205,25 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport { // Backend security validation requires tenant id before parsing the body. httpRequest.Headers.Add("X-Tenant-Id", report.TenantId); - AddSecurityHeaders(httpRequest); + var correlationId = AddSecurityHeaders(httpRequest); using (var cts = new CancellationTokenSource(timeout ?? TimeSpan.FromSeconds(5))) using (var response = await _httpClient.SendAsync(httpRequest, cts.Token).ConfigureAwait(false)) { - // Response status is deliberately ignored — endpoint always returns 200. - return true; + // Every status counts as delivered (unchanged) — except a 401/403 without the + // backend's correlation echo: then the report never reached the backend. + if (!IsAuthFailureStatus(response.StatusCode)) + return true; + + var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false); + var classification = ResponseOriginClassifier.Classify(response, correlationId, body); + if (!classification.IsForeign) + return true; + + _logger?.Debug( + $"ReportAgentErrorAsync: http {(int)response.StatusCode} did not come from the backend " + + $"({classification.ForeignReason}) — report not delivered."); + return false; } } } @@ -221,7 +243,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport Content = new StringContent(json, Encoding.UTF8, "application/json") }) { - AddSecurityHeaders(httpRequest); + var correlationId = AddSecurityHeaders(httpRequest); HttpResponseMessage response; try @@ -236,7 +258,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport using (response) { - await ThrowOnAuthFailureAsync(response).ConfigureAwait(false); + await ThrowOnAuthFailureAsync(response, correlationId).ConfigureAwait(false); await ThrowOnCodedConflictAsync(response).ConfigureAwait(false); response.EnsureSuccessStatusCode(); @@ -284,61 +306,76 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// Throws for 401/403 responses so callers can /// distinguish authentication failures from transient server errors. /// - /// Field case (2026-08-09): a pre-cutover agent build pointed at a - /// retired Function App; Azure answers for a stopped app with a platform-level 403 - /// carrying an HTML error page ("Web App - Unavailable"). Our backend never returns - /// HTML on 401/403 — an HTML body therefore means the response did not come from the - /// backend's auth layer at all (stopped/retired app, edge proxy, WAF). Reporting that - /// as "device is not authorized" sends troubleshooting down the wrong path, so the - /// two cases get distinct messages and the exception carries a flag. + /// Only a response that echoes this request's X-Correlation-ID is the backend's + /// verdict (). Anything else never came from the + /// backend's auth layer: the platform page of a stopped app (field case 2026-08-09: a + /// pre-cutover build pointed at a retired Function App), or the platform's "Client + /// Certificate Required" page / a proxy page when TLS inspection on the network path + /// strips the device certificate. Reporting those as "device is not authorized" sends + /// troubleshooting down the wrong path, so each origin gets its own message and the + /// exception carries the classification. /// /// - private static async Task ThrowOnAuthFailureAsync(HttpResponseMessage response) + private async Task ThrowOnAuthFailureAsync(HttpResponseMessage response, string correlationId) { - if (response.StatusCode != System.Net.HttpStatusCode.Unauthorized && - response.StatusCode != System.Net.HttpStatusCode.Forbidden) - { + if (!IsAuthFailureStatus(response.StatusCode)) return; - } - var statusText = $"{(int)response.StatusCode} {response.StatusCode}"; + var statusCode = (int)response.StatusCode; + var statusText = $"{statusCode} {response.StatusCode}"; + var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false); + var classification = ResponseOriginClassifier.Classify( + response, correlationId, body, _serverCertificates?.LastIssuer); - string body = null; - try + if (classification.Origin != ResponseOrigin.Backend) { - body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - } - catch - { - // Body is diagnostic-only; classification below treats null as non-HTML. - } - - if (LooksLikeHtmlErrorPage(response, body)) - { - var title = TryExtractHtmlTitle(body); - var titleText = title != null ? $" (\"{title}\")" : string.Empty; - var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)"; throw new BackendAuthException( - $"Backend returned {statusText} with an HTML platform error page{titleText} from '{host}'. " + - "This response did not come from the Autopilot Monitor backend — the endpoint appears stopped, " + - "retired, or intercepted by a proxy. This is NOT a device-authorization failure; an outdated " + - "agent build pointing at a stale ApiBaseUrl is the most likely cause.", - (int)response.StatusCode, - endpointUnavailable: true); + DescribeNonBackendAnswer(response, statusText, classification), + statusCode, + classification: classification); } // Backend JSON bodies may carry a machine-readable errorCode (Shared // Constants.AgentErrorCodes) that tells the caller how to react — e.g. // session_owner_mismatch → rotate the session id instead of treating this as a // certificate rejection. Absent/unparseable → null, classic auth failure. - var errorCode = TryExtractErrorCode(body); throw new BackendAuthException( $"Backend returned {statusText}. " + "The device is not authorized. Check client certificate and Autopilot device validation.", - (int)response.StatusCode, - errorCode: errorCode); + statusCode, + errorCode: TryExtractErrorCode(body), + classification: classification); } + /// + /// Message for a 401/403 the backend did not author. The cause leads because + /// RegistrationFailureSummary.LastError keeps only 256 characters; HTML answers + /// keep the phrase "HTML platform error page" plus the page title (ANALYZE-NET-001 matches both). + /// + private static string DescribeNonBackendAnswer(HttpResponseMessage response, string statusText, ResponseClassification classification) + { + var host = response.RequestMessage?.RequestUri?.Host ?? "(unknown host)"; + var titleText = classification.PageTitle != null ? $" (\"{classification.PageTitle}\")" : string.Empty; + var htmlPage = $"an HTML platform error page{titleText}"; + + if (classification.Origin == ResponseOrigin.PlatformStopped) + { + return $"Backend returned {statusText} with {(classification.IsHtml ? htmlPage : "a platform error page")} from '{host}'. " + + "This response did not come from the Autopilot Monitor backend — the endpoint appears stopped or retired. " + + "This is NOT a device-authorization failure; an outdated agent build pointing at a stale ApiBaseUrl " + + "is the most likely cause."; + } + + var cause = classification.ForeignReason == ResponseClassification.ClientCertificateRequired + ? "The device certificate did not reach the backend (TLS inspection or a proxy on the network path)" + : "Something on the network path answered instead of the backend (TLS inspection, a proxy or a security gateway)"; + var answer = classification.IsHtml ? htmlPage : "a response without the backend's correlation echo"; + return $"{cause}: {statusText} from '{host}' was {answer}. This is NOT a device-authorization failure."; + } + + private static bool IsAuthFailureStatus(HttpStatusCode statusCode) => + statusCode == HttpStatusCode.Unauthorized || statusCode == HttpStatusCode.Forbidden; + /// /// Throws for a 409 whose JSON body carries a /// machine-readable errorCode — register-session answers @@ -395,62 +432,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport } /// - /// True when a 401/403 response is an HTML page rather than a backend JSON/plain-text - /// reply. Checks the declared Content-Type first, then falls back to sniffing the body - /// prefix so a platform page served with a generic Content-Type is still recognized. + /// Adds device / version / bootstrap-token security headers and returns the request's + /// correlation id. Client certificate is sent at the TLS layer via the externally-supplied + /// HttpClient's HttpClientHandler. /// - private static bool LooksLikeHtmlErrorPage(HttpResponseMessage response, string body) - { - var mediaType = response.Content?.Headers?.ContentType?.MediaType; - if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase)) - return true; - - if (string.IsNullOrWhiteSpace(body)) - return false; - - var head = body.TrimStart(); - return head.StartsWith(" - /// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so the - /// log line names the platform page without dumping markup. Null when absent/oversized. - /// - private static string TryExtractHtmlTitle(string body) - { - if (string.IsNullOrEmpty(body)) return null; - try - { - var match = System.Text.RegularExpressions.Regex.Match( - body, @"]*>\s*(.*?)\s*", - System.Text.RegularExpressions.RegexOptions.IgnoreCase | - System.Text.RegularExpressions.RegexOptions.Singleline, - TimeSpan.FromMilliseconds(250)); - if (!match.Success) return null; - - var title = match.Groups[1].Value; - return title.Length == 0 ? null : (title.Length > 80 ? title.Substring(0, 80) : title); - } - catch - { - return null; - } - } - - /// - /// Adds device / version / bootstrap-token security headers. Client certificate is sent - /// at the TLS layer via the externally-supplied HttpClient's HttpClientHandler. - /// - private void AddSecurityHeaders(HttpRequestMessage request) + private string AddSecurityHeaders(HttpRequestMessage request) { // Per-request correlation id for end-to-end tracing (mirrors BackendTelemetryUploader). // The backend's CorrelationIdMiddleware reads X-Correlation-ID; without it the backend // mints a fresh id per inbound request and agent↔backend calls cannot be tied together. // Generated per request so each attempt (e.g. config-fetch 503 retries) is independently - // traceable. + // traceable — and so the echo on a 401/403 proves which request the backend answered. var correlationId = Guid.NewGuid().ToString("N"); - request.Headers.Add("X-Correlation-ID", correlationId); + request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId); _logger?.Debug($"BackendApiClient: {request.Method} {request.RequestUri?.AbsolutePath} (corr={correlationId})"); if (_useBootstrapTokenAuth && !string.IsNullOrEmpty(_bootstrapToken)) @@ -464,6 +458,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport request.Headers.Add("X-Device-SerialNumber", _serialNumber); if (!string.IsNullOrEmpty(_agentVersion)) request.Headers.Add("X-Agent-Version", _agentVersion); + + return correlationId; } public void Dispose() @@ -473,23 +469,36 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport } /// - /// Thrown when the backend returns 401 or 403, indicating the device is not authorized — - /// or, when is set, that the 401/403 was a platform-level - /// HTML response (stopped/retired app, edge proxy) rather than a backend auth verdict. + /// Thrown for a 401 or 403. says who answered: the backend (the device is + /// not authorized), the platform page of a stopped app (), or + /// something on the network path () — the last two say nothing + /// about this device's authorization. /// public class BackendAuthException : Exception { /// - /// HTTP status code returned by the backend (401 or 403). + /// HTTP status code of the response (401 or 403). /// Used by the DistressReporter to classify the failure type. /// public int StatusCode { get; } + /// Classification and evidence of the response; when constructed without one. + public ResponseClassification Classification { get; } + + public ResponseOrigin Origin => Classification.Origin; + /// - /// True when the 401/403 carried an HTML platform error page — the endpoint itself is - /// stopped/retired/proxied and the status says nothing about this device's authorization. + /// True only for the platform page of a stopped app: the endpoint itself is down. Counts + /// toward the auth shutdown like a backend rejection, without a distress report. /// - public bool EndpointUnavailable { get; } + public bool EndpointUnavailable => Origin == ResponseOrigin.PlatformStopped; + + /// + /// True when something other than the backend answered (the platform's "Client Certificate + /// Required" page, a proxy page): the device certificate did not reach the backend. Never + /// counts as an authentication failure. + /// + public bool IsForeignResponse => Origin == ResponseOrigin.Foreign; /// /// Machine-readable code from the backend's JSON error body (see Shared @@ -497,11 +506,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport /// public string ErrorCode { get; } - public BackendAuthException(string message, int statusCode = 0, bool endpointUnavailable = false, string errorCode = null) : base(message) + public BackendAuthException(string message, int statusCode = 0, string errorCode = null, ResponseClassification classification = null) : base(message) { StatusCode = statusCode; - EndpointUnavailable = endpointUnavailable; ErrorCode = errorCode; + Classification = classification ?? ResponseClassification.ForBackend(statusCode); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkMetrics.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkMetrics.cs index 978558b0..6448ece1 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkMetrics.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkMetrics.cs @@ -14,6 +14,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport private long _totalBytesUp; private long _totalBytesDown; private long _totalLatencyMs; + private long _okWithoutEchoCount; public void RecordRequest(long bytesUp, long bytesDown, long latencyMs, bool failed) { @@ -25,6 +26,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport Interlocked.Increment(ref _failureCount); } + /// + /// Counts a 2xx telemetry upload whose response did not echo the request's + /// X-Correlation-ID: something other than the backend may have answered (a block page + /// served with 200). Observed only — the upload still counts as a success. + /// + public void RecordOkWithoutEcho() => Interlocked.Increment(ref _okWithoutEchoCount); + + /// Cumulative count since the process started. + public long OkWithoutEchoCount => Interlocked.Read(ref _okWithoutEchoCount); + /// /// Returns an immutable snapshot of all counters for delta calculation. /// diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs index 89da3464..1e3d5ad0 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/EnrollmentOrchestrator.cs @@ -10,6 +10,7 @@ using AutopilotMonitor.Agent.V2.Core.Persistence; using AutopilotMonitor.Agent.V2.Core.Telemetry.Events; using AutopilotMonitor.Agent.V2.Core.Telemetry.Signals; using AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry; using AutopilotMonitor.DecisionCore.Classifiers; using AutopilotMonitor.DecisionCore.Engine; @@ -315,6 +316,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// public long IngressPendingSignalCount => _ingress?.PendingSignalCount ?? 0L; + /// + /// Terminal phase: grants the transport its one upload attempt despite an active + /// interception pause () and, + /// only while that pause holds, wakes the drain loop so the attempt runs inside the caller's + /// bounded drain window. Without an interception pause nothing changes. No-op before + /// . + /// + public void RequestTerminalUploadAttempt() + { + var transport = _transport; + if (transport == null) return; + if (transport.GrantTerminalAttempt()) + OnImmediateFlushRequested(this, EventArgs.Empty); + } + /// /// Typed read-model surfaces returned by /// (ARCH-F4). Set during step 14; null before Start and when @@ -384,6 +400,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration // here cannot deadlock the drain. _transport.BatchPoisoned += OnBatchPoisoned; _transport.UploadBlocked += OnUploadBlocked; + _transport.UploadIntercepted += OnUploadIntercepted; // The spool reference reaches peripheral collectors (AgentSelfMetricsCollector via // PeriodicCollectorLifecycleHost) as a CreateCollectorHosts parameter at step 14, @@ -1004,11 +1021,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration catch (AggregateException) { /* wrapped OperationCanceledException */ } catch (Exception ex) { _logger.Warning($"EnrollmentOrchestrator: drain loop join failed: {ex.Message}"); } - // 5) Terminaler Drain — finale Batches. + // 5) Terminaler Drain — finale Batches. An active interception pause still lets the + // process make its one terminal attempt here, unless DrainSpool already used it. try { using (var timeoutCts = new CancellationTokenSource(_terminalDrainTimeout)) { + _transport?.GrantTerminalAttempt(); _transport?.DrainAllAsync(timeoutCts.Token).GetAwaiter().GetResult(); } } @@ -1155,6 +1174,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration } } + /// + /// Records the first intercepted upload of the process as telemetry_upload_blocked + /// with the interception evidence. Gated once per process by the transport, independently + /// of . Queued behind the intercepted batch like every other + /// item: it reaches the backend only if the path clears in this run, and its arrival time + /// marks the end of the interception. + /// + private void OnUploadIntercepted(object? sender, InterceptionReport report) + { + var interception = report.Interception; + _logger.Warning( + $"EnrollmentOrchestrator: telemetry upload intercepted on the network path ({interception.ForeignReason}, " + + $"http {interception.StatusCode}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " + + $"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\") — uploads pause with backoff; " + + "the agent keeps observing."); + + var ingress = _ingress; + if (ingress == null) return; + try + { + new InformationalEventPost(ingress, _clock).Emit(new Shared.Models.EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.TelemetryUploadBlocked, + Severity = Shared.Models.EventSeverity.Warning, + Source = "TelemetryUploadOrchestrator", + Phase = Shared.Models.EnrollmentPhase.Unknown, + Message = "Telemetry upload intercepted on the network path: the answer did not come from the " + + "backend, so the device certificate did not reach it. Uploads pause with backoff and " + + "resume once the path clears; this marker is queued behind the intercepted batch.", + Timestamp = _clock.UtcNow, + ImmediateUpload = true, + Data = new Dictionary(StringComparer.Ordinal) + { + ["reason"] = interception.ForeignReason ?? ResponseClassification.ForeignResponse, + ["operation"] = "agent/telemetry", + ["statusCode"] = interception.StatusCode, + ["pageTitle"] = interception.PageTitle ?? string.Empty, + ["serverCertIssuer"] = interception.ServerCertIssuer ?? string.Empty, + ["firstBlockedAtUtc"] = report.FirstBlockedAtUtc.ToString("o", CultureInfo.InvariantCulture), + }, + }); + } + catch (Exception ex) + { + _logger.Error("EnrollmentOrchestrator: failed to post telemetry_upload_blocked (interception).", ex); + } + } + // ---------------------------------------------------------------- Test helpers /// diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/MtlsHttpClientFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/MtlsHttpClientFactory.cs index 9ecdf0f1..dbadf90b 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/MtlsHttpClientFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/MtlsHttpClientFactory.cs @@ -5,6 +5,7 @@ using System.Net.Http; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Transport; namespace AutopilotMonitor.Agent.V2.Core.Orchestration { @@ -40,10 +41,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// ('s _handler-Feld ist internal und in net48 /// reflection-unzuverlässig). /// + /// + /// , when supplied, observes the server certificate of + /// every new connection without changing the validation verdict. + /// /// public static HttpClientHandler CreateHandler( ICertificateResolver resolver, - AgentLogger logger) + AgentLogger logger, + ServerCertificateObserver? serverCertificates = null) { if (resolver == null) throw new ArgumentNullException(nameof(resolver)); if (logger == null) throw new ArgumentNullException(nameof(logger)); @@ -62,6 +68,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, }; handler.ClientCertificates.Add(cert); + serverCertificates?.Attach(handler); logger.Info( $"MtlsHttpClientFactory: handler ready (cert thumbprint={cert.Thumbprint})."); @@ -85,9 +92,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration ICertificateResolver resolver, AgentLogger logger, TimeSpan? timeout = null, - NetworkMetrics? metrics = null) + NetworkMetrics? metrics = null, + ServerCertificateObserver? serverCertificates = null) { - var innerHandler = CreateHandler(resolver, logger); + var innerHandler = CreateHandler(resolver, logger, serverCertificates); HttpMessageHandler pipelineHead = innerHandler; if (metrics != null) { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs index 6c5cf5c8..412faff7 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/AuthFailureTracker.cs @@ -1,7 +1,9 @@ using System; using System.Threading; +using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.DecisionCore.Engine; using AutopilotMonitor.Shared.Models; @@ -29,6 +31,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// twice about a certificate it already rejected. /// /// + /// Only a 401/403 the backend authored counts (). One + /// from something on the network path (TLS inspection, a proxy page) never counts and never + /// touches the streak — it says nothing about this device's authorization — and is reported + /// once per process as . + /// + /// /// Thread-safety: consecutive-count uses ; the first-failure timestamp /// is protected by a single monitor lock. is raised outside the /// lock to prevent handler re-entrancy deadlocks. @@ -47,6 +55,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security private DateTime? _firstFailureUtc; private int _thresholdFired; // 0/1 — Interlocked exchange makes the event single-shot private int _distressDispatched; // 0/1 per failure streak — reset by RecordSuccess + private int _foreignDistressDispatched; // 0/1 per process — never reset + private Task _foreignDistressSend; // the per-process ClientCertificateNotReceived send; null until dispatched public AuthFailureTracker( int maxFailures, @@ -90,16 +100,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// dispatch further distress signals. Emits the first /// time either ceiling is crossed; subsequent calls after termination is armed are no-ops. /// - /// - /// True when the 401/403 was a platform-level HTML response (stopped/retired app, edge - /// proxy — see ). Suppresses the - /// distress dispatch: the report would go to the very endpoint that just answered with a - /// platform error page, and its DeviceNotRegistered classification would be wrong anyway. - /// Counting and the shutdown thresholds are unaffected. Should a later failure in the - /// same streak come from a live backend, that one still dispatches the streak's report. + /// + /// Who answered (); null = the backend. + /// counts but suppresses the distress dispatch: + /// the report would go to the very endpoint that just answered with its stopped page, and + /// DeviceNotRegistered would be wrong anyway; a later failure in the same streak from a live + /// backend still dispatches the streak's report. never + /// counts — see . /// - public void RecordFailure(int statusCode, string operation, bool endpointUnavailable = false) + public void RecordFailure(int statusCode, string operation, ResponseClassification classification = null) { + if (classification?.IsForeign == true) + { + RecordForeignResponse(statusCode, operation, classification); + return; + } + if (Volatile.Read(ref _thresholdFired) == 1) return; var count = Interlocked.Increment(ref _consecutiveFailures); @@ -114,9 +130,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security // V1 parity — only one distress report per failure streak. Repeat hits use the local // log only; the backend already knows from the first report that the device is - // unauthorized. Endpoint-unavailable failures never dispatch (nothing to send to) but + // unauthorized. Stopped-endpoint failures never dispatch (nothing to send to) but // also never consume the streak's report slot. - if (endpointUnavailable) + if (classification?.Origin == ResponseOrigin.PlatformStopped) { if (_distressReporter != null && Volatile.Read(ref _distressDispatched) == 0) { @@ -180,6 +196,60 @@ namespace AutopilotMonitor.Agent.V2.Core.Security _logger.Warning($"AuthFailureTracker: ThresholdExceeded handler threw: {ex.Message}"); } } + + /// + /// A 401/403 that something on the network path authored. Never counts and never touches + /// the streak — an interception says nothing about this device's authorization, and five + /// of them must not shut a healthy agent down. The first one per process dispatches + /// : the distress channel needs + /// no client certificate, so it gets through where the request did not. Message order is + /// operation, status, sub-reason, page title, server-certificate issuer (the reporter cuts + /// at 256 characters). + /// + private void RecordForeignResponse(int statusCode, string operation, ResponseClassification classification) + { + var finding = + $"{operation} http {statusCode} {classification.ForeignReason}; " + + $"page \"{classification.PageTitle ?? "(none)"}\"; " + + $"server cert issuer \"{classification.ServerCertIssuer ?? "(unknown)"}\""; + _logger.Info($"AuthFailureTracker: answer did not come from the backend — not an authentication failure, not counted ({finding})."); + + if (_distressReporter == null || Interlocked.CompareExchange(ref _foreignDistressDispatched, 1, 0) != 0) + return; + + try + { + // Fire-and-forget like every other dispatch here, but kept: a caller about to end + // the process waits for it (WaitForPendingForeignDistress). + Volatile.Write(ref _foreignDistressSend, _distressReporter.TrySendAsync( + DistressErrorType.ClientCertificateNotReceived, finding, httpStatusCode: statusCode)); + } + catch (Exception ex) + { + _logger.Debug($"AuthFailureTracker: distress dispatch threw: {ex.Message}"); + } + } + + /// + /// Gives this process's send up + /// to to leave the machine. The dispatch is fire-and-forget; a caller + /// about to end the process (a start whose registration failed exits or cleans up right away) + /// waits here so the report is not abandoned mid-flight. Returns true when nothing is pending + /// or the send finished within the bound. Never throws. + /// + public bool WaitForPendingForeignDistress(TimeSpan bound) + { + var send = Volatile.Read(ref _foreignDistressSend); + if (send == null || send.IsCompleted) return true; + try + { + return send.Wait(bound); + } + catch (Exception) + { + return true; // faulted or cancelled: finished, nothing left to wait for + } + } } public sealed class AuthFailureThresholdEventArgs : EventArgs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionIdPersistence.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionIdPersistence.cs index f286e274..e80fa432 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionIdPersistence.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionIdPersistence.cs @@ -1,8 +1,10 @@ using System; using System.IO; +using System.Text; using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Shared.Models; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace AutopilotMonitor.Agent.V2.Core.Security { @@ -27,6 +29,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security private readonly string _registrationFailurePath; private readonly object _lockObject = new object(); + // Agent-local key in registration-failure.json; RegistrationFailureSummary has no such member. + private const string LocalNetworkFingerprintKey = "NetworkFingerprint"; + public SessionIdPersistence(string dataDirectory) { if (string.IsNullOrWhiteSpace(dataDirectory)) @@ -170,22 +175,29 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// /// Folds one failed registration run into registration-failure.json: the run - /// counter grows, the first failure time is kept, everything else describes the latest - /// run. A run that never registers exits and leaves nothing else behind — the emergency - /// break reads this file so the backend learns why the session never appeared. - /// Best-effort; never throws. + /// counters grow (an intercepted run — InterceptedRuns > 0 on + /// — also counts toward + /// ), the first failure time is kept, + /// everything else describes the latest run. A run that never registers exits and leaves + /// nothing else behind — the emergency break reads this file so the backend learns why the + /// session never appeared. (the latest run's network) + /// is stored beside the summary, never in it, so it cannot reach the wire. Written through to + /// disk (L.12): the process exits right after and the next boot decides on the record. + /// Returns the merged summary, null when it could not be written. Never throws. /// - public void RecordRegistrationFailure(RegistrationFailureSummary latestRun, AgentLogger logger = null) + public RegistrationFailureSummary RecordRegistrationFailure( + RegistrationFailureSummary latestRun, AgentLogger logger = null, string networkFingerprint = null) { - if (latestRun == null) return; + if (latestRun == null) return null; lock (_lockObject) { try { - var previous = LoadRegistrationFailureInternal(); + var previous = LoadRegistrationFailureInternal()?.Summary; var merged = new RegistrationFailureSummary { FailedRuns = (previous?.FailedRuns ?? 0) + 1, + InterceptedRuns = (previous?.InterceptedRuns ?? 0) + (latestRun.InterceptedRuns > 0 ? 1 : 0), FirstFailedAtUtc = previous?.FirstFailedAtUtc ?? latestRun.LastFailedAtUtc, LastFailedAtUtc = latestRun.LastFailedAtUtc, Outcome = latestRun.Outcome, @@ -195,18 +207,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Security AttemptWindowSeconds = latestRun.AttemptWindowSeconds, ConfigFetchOutcome = latestRun.ConfigFetchOutcome, }; - File.WriteAllText(_registrationFailurePath, JsonConvert.SerializeObject(merged)); - logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns}."); + WriteRegistrationFailure(merged, networkFingerprint); + logger?.Info($"SessionIdPersistence: recorded failed registration run #{merged.FailedRuns} ({merged.InterceptedRuns} intercepted)."); + return merged; } catch (Exception ex) { logger?.Warning($"SessionIdPersistence: RecordRegistrationFailure failed: {ex.Message}"); + return null; } } } - /// The failed-registration record since the last success, or null (none, unreadable). + /// + /// The failed-registration record since the last success, or null (none, unreadable). + /// The summary only — what the emergency break may send. + /// public RegistrationFailureSummary LoadRegistrationFailure() + { + lock (_lockObject) return LoadRegistrationFailureInternal()?.Summary; + } + + /// + /// The failed-registration record plus the agent-local network fingerprint of its latest + /// run, or null (none, unreadable). For the next successful start's comparison only. + /// + public RegistrationFailureRecord LoadRegistrationFailureRecord() { lock (_lockObject) return LoadRegistrationFailureInternal(); } @@ -252,13 +278,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Security } } - private RegistrationFailureSummary LoadRegistrationFailureInternal() + private RegistrationFailureRecord LoadRegistrationFailureInternal() { try { if (!File.Exists(_registrationFailurePath)) return null; - var record = JsonConvert.DeserializeObject(File.ReadAllText(_registrationFailurePath)); - return record != null && record.FailedRuns > 0 ? record : null; + var json = JObject.Parse(File.ReadAllText(_registrationFailurePath)); + var summary = json.ToObject(); + return summary != null && summary.FailedRuns > 0 + ? new RegistrationFailureRecord(summary, (string)json[LocalNetworkFingerprintKey]) + : null; } catch { @@ -267,6 +296,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Security } } + /// + /// The summary's own properties plus one agent-local key: the fingerprint sits next to the + /// wire type, never inside it (the summary deserializes without it). + /// + private void WriteRegistrationFailure(RegistrationFailureSummary summary, string networkFingerprint) + { + var json = JObject.FromObject(summary); + if (!string.IsNullOrEmpty(networkFingerprint)) + json[LocalNetworkFingerprintKey] = networkFingerprint; + + var bytes = Encoding.UTF8.GetBytes(json.ToString(Formatting.None)); + using (var stream = new FileStream( + _registrationFailurePath, FileMode.Create, FileAccess.Write, FileShare.None, + bufferSize: 4096, options: FileOptions.WriteThrough)) + { + stream.Write(bytes, 0, bytes.Length); + stream.Flush(flushToDisk: true); + } + } + private static void TryDelete(string path, AgentLogger logger) { try { if (File.Exists(path)) File.Delete(path); } @@ -305,4 +354,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Security } } } + + /// + /// registration-failure.json as the agent keeps it: the + /// the emergency break may send, plus the network fingerprint of the latest failed run, which + /// stays on the device — only the next successful start's comparison (changed or not) is reported. + /// + public sealed class RegistrationFailureRecord + { + public RegistrationFailureRecord(RegistrationFailureSummary summary, string networkFingerprint) + { + Summary = summary ?? throw new ArgumentNullException(nameof(summary)); + NetworkFingerprint = networkFingerprint; + } + + public RegistrationFailureSummary Summary { get; } + + /// Fingerprint of the latest failed run's network; null when it was unknown. + public string NetworkFingerprint { get; } + } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs index 2b3f2f2d..8a0296bf 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs @@ -33,7 +33,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// On (401/403) → report to the /// (which fires the first-failure distress + may trip /// the shutdown threshold) and return - /// without retrying — the backend has definitively rejected the device cert. + /// without retrying — the backend has definitively rejected the device cert. A 401/403 + /// something on the network path answered ends the same way, marked + /// . /// Exception to the above — a 403 or 409 carrying error code /// session_owner_mismatch (SESSION-OWNER-BINDING): the persisted SessionId names a /// session that is not this device's — bound to another device identity (403, typically an @@ -135,8 +137,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Security logger.Error($"Session owner mismatch reported by backend but session rotation failed: {rotateEx.Message}", rotateEx); if (ex is BackendAuthException authEx) { - authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.EndpointUnavailable); - return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message); + authFailureTracker?.RecordFailure(authEx.StatusCode, "agent/register-session", authEx.Classification); + return SessionRegistrationResult.AuthFailed(authEx.StatusCode, authEx.Message, authEx.IsForeignResponse); } return SessionRegistrationResult.Failed(ex.Message); } @@ -154,12 +156,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Security } catch (BackendAuthException ex) { - logger.Error($"Session registration authentication failed ({ex.StatusCode}): {ex.Message}"); + logger.Error(ex.IsForeignResponse + ? $"Session registration intercepted on the network path ({ex.StatusCode}): {ex.Message}" + : $"Session registration authentication failed ({ex.StatusCode}): {ex.Message}"); // Feed the central auth-failure tracker so the first-failure distress dispatches // and the shutdown threshold advances. V1 parity: no retry on auth-failure — - // backend has definitively rejected the cert/token. - authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.EndpointUnavailable); - return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message); + // backend has definitively rejected the cert/token. A 401/403 from something on + // the network path is not counted; the tracker reports it once per process as + // ClientCertificateNotReceived, and the run ends AuthFailed marked intercepted + // (the caller's buffer decides between a further boot and cleaning up). + authFailureTracker?.RecordFailure(ex.StatusCode, "agent/register-session", ex.Classification); + return SessionRegistrationResult.AuthFailed(ex.StatusCode, ex.Message, ex.IsForeignResponse); } catch (Exception ex) { @@ -274,7 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// Backend responded with Success=true. The session row is live. Succeeded = 0, - /// 401/403 from the backend — cert/token rejected. No retry was attempted. + /// + /// 401/403 — cert/token rejected by the backend, or answered by something on the network + /// path (). No retry was attempted. + /// AuthFailed = 1, /// All attempts failed with non-auth errors (network, 5xx, malformed response). @@ -299,25 +309,34 @@ namespace AutopilotMonitor.Agent.V2.Core.Security /// public string RotatedFromSessionId { get; } + /// + /// True when the run ended on a 401/403 that something on the network path answered (TLS + /// inspection, a proxy page — ), not + /// the backend. Such starts count toward the intercepted-registration buffer. + /// + public bool IsIntercepted { get; } + private SessionRegistrationResult( SessionRegistrationOutcome outcome, RegisterSessionResponse response, int httpStatusCode, string errorMessage, - string rotatedFromSessionId = null) + string rotatedFromSessionId = null, + bool isIntercepted = false) { Outcome = outcome; Response = response; HttpStatusCode = httpStatusCode; ErrorMessage = errorMessage; RotatedFromSessionId = rotatedFromSessionId; + IsIntercepted = isIntercepted; } public static SessionRegistrationResult Succeeded(RegisterSessionResponse response, string rotatedFromSessionId = null) => new SessionRegistrationResult(SessionRegistrationOutcome.Succeeded, response, 200, null, rotatedFromSessionId); - public static SessionRegistrationResult AuthFailed(int statusCode, string message) - => new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message); + public static SessionRegistrationResult AuthFailed(int statusCode, string message, bool intercepted = false) + => new SessionRegistrationResult(SessionRegistrationOutcome.AuthFailed, null, statusCode, message, isIntercepted: intercepted); public static SessionRegistrationResult Failed(string message) => new SessionRegistrationResult(SessionRegistrationOutcome.Failed, null, 0, message); diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs index 073631f3..2658068e 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs @@ -1343,6 +1343,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination return; } + // An interception pause (uploads answered by something on the network path) would + // otherwise skip this drain entirely: the terminal phase gets exactly one attempt, then + // termination goes on as usual — whatever did not go up is lost, as with any session. + try { _drain.RequestTerminalUploadAttempt(); } + catch (Exception ex) { _logger.Warning($"EnrollmentTerminationHandler: terminal upload attempt request threw: {ex.Message}"); } + // Block briefly so pending events can land before the next destructive step // (shutdown.exe, self-destruct). Two phases share the same bounded budget: // diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IDrainStatus.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IDrainStatus.cs index a2f39434..96e2c7c4 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IDrainStatus.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IDrainStatus.cs @@ -54,5 +54,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination /// is true. /// int SpoolPendingItemCount { get; } + + /// + /// Called once at the start of every DrainSpool: lets the transport make its one + /// terminal upload attempt even while an interception pause holds uploads back (production: + /// ). + /// Without an interception pause it changes nothing. + /// + void RequestTerminalUploadAttempt(); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorDrainStatus.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorDrainStatus.cs index 11d7a3ef..915e5c39 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorDrainStatus.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorDrainStatus.cs @@ -31,5 +31,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination public bool CanObserveSpool => true; public int SpoolPendingItemCount => _orchestrator.PendingItemCount; + + public void RequestTerminalUploadAttempt() => _orchestrator.RequestTerminalUploadAttempt(); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ResponseOrigin.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ResponseOrigin.cs new file mode 100644 index 00000000..1993b2e4 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ResponseOrigin.cs @@ -0,0 +1,226 @@ +#nullable enable +using System; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace AutopilotMonitor.Agent.V2.Core.Transport +{ + /// Who answered a 401/403 — see . + public enum ResponseOrigin + { + /// The backend: the response echoes the request's X-Correlation-ID. A real authorization verdict. + Backend = 0, + + /// + /// The platform page of a stopped web app (no echo, "This web app is stopped"). The endpoint + /// itself is down; counts toward the auth shutdown so a dead endpoint does not keep the agent alive. + /// + PlatformStopped = 1, + + /// + /// Anything else without the echo: the platform's "Client Certificate Required" page (the device + /// certificate did not reach the backend) or a page of a proxy on the network path. Never an + /// authorization verdict. + /// + Foreign = 2, + } + + /// + /// A classified 401/403 plus the evidence the agent reports about it (log, distress, + /// telemetry_upload_blocked). Built by . + /// + public sealed class ResponseClassification + { + /// Foreign sub-reason: the platform answered "Client Certificate Required". + public const string ClientCertificateRequired = "client_certificate_required"; + + /// Foreign sub-reason: any other answer that did not come from the backend (e.g. a proxy page). + public const string ForeignResponse = "foreign_response"; + + private ResponseClassification( + ResponseOrigin origin, + int statusCode, + string? foreignReason, + string? pageTitle, + bool isHtml, + string? serverCertIssuer) + { + Origin = origin; + StatusCode = statusCode; + ForeignReason = foreignReason; + PageTitle = pageTitle; + IsHtml = isHtml; + ServerCertIssuer = serverCertIssuer; + } + + public ResponseOrigin Origin { get; } + + /// HTTP status of the classified response (401 or 403). + public int StatusCode { get; } + + /// or for ; null otherwise. + public string? ForeignReason { get; } + + /// HTML <title> of the page (at most 80 characters), null when absent. + public string? PageTitle { get; } + + /// True when the body is an HTML page (declared text/html or sniffed). + public bool IsHtml { get; } + + /// Issuer of the server certificate of the last validated backend connection (); null when unknown. + public string? ServerCertIssuer { get; } + + public bool IsForeign => Origin == ResponseOrigin.Foreign; + + internal static ResponseClassification ForBackend(int statusCode) => + new ResponseClassification(ResponseOrigin.Backend, statusCode, null, null, isHtml: false, serverCertIssuer: null); + + internal static ResponseClassification ForPlatformStopped(int statusCode, string? pageTitle, bool isHtml, string? serverCertIssuer) => + new ResponseClassification(ResponseOrigin.PlatformStopped, statusCode, null, pageTitle, isHtml, serverCertIssuer); + + internal static ResponseClassification ForForeign(int statusCode, string foreignReason, string? pageTitle, bool isHtml, string? serverCertIssuer) => + new ResponseClassification(ResponseOrigin.Foreign, statusCode, foreignReason, pageTitle, isHtml, serverCertIssuer); + } + + /// + /// Decides whether a 401/403 came from the backend at all. The backend's correlation + /// middleware echoes the request's X-Correlation-ID on every response it writes (the + /// agent sends a fresh id per request); the Azure platform's own pages and a proxy's pages never + /// carry it. The echo is therefore the only proof of origin — body shape, content type and + /// title are evidence, not proof. Shared by every agent call site that can receive a 401/403. + /// + public static class ResponseOriginClassifier + { + /// Request header the backend echoes on every response it writes. + public const string CorrelationIdHeader = "X-Correlation-ID"; + + /// Only the head of a body is inspected; platform and proxy pages are a few KB. + public const int MaxBodyChars = 16 * 1024; + + private const int MaxTitleLength = 80; + + // Text of the platform page of a stopped web app ("Error 403 - This web app is stopped."). + private const string StoppedAppMarker = "This web app is stopped"; + + // Text of the platform page served when no client certificate arrived + // ("Error 403 - Forbidden: Client Certificate Required."). + private const string ClientCertificateRequiredMarker = "Client Certificate Required"; + + /// + /// Classifies a 401/403. is the id this request carried; + /// is the head of the body (); + /// is passed through as evidence. An echo always wins: + /// a backend page is the backend's verdict whatever it looks like. + /// + public static ResponseClassification Classify( + HttpResponseMessage response, + string? sentCorrelationId, + string? bodyPrefix, + string? serverCertIssuer = null) + { + if (response == null) throw new ArgumentNullException(nameof(response)); + + var statusCode = (int)response.StatusCode; + if (EchoesCorrelationId(response, sentCorrelationId)) + return ResponseClassification.ForBackend(statusCode); + + var body = Cap(bodyPrefix); + var isHtml = LooksLikeHtml(response, body); + var title = TryExtractHtmlTitle(body); + + if (Contains(body, StoppedAppMarker)) + return ResponseClassification.ForPlatformStopped(statusCode, title, isHtml, serverCertIssuer); + + var reason = Contains(body, ClientCertificateRequiredMarker) + ? ResponseClassification.ClientCertificateRequired + : ResponseClassification.ForeignResponse; + return ResponseClassification.ForForeign(statusCode, reason, title, isHtml, serverCertIssuer); + } + + /// + /// The first characters of the body, null when there is none or it + /// cannot be read. Never throws — the body is evidence only. + /// + public static async Task ReadBodyPrefixAsync(HttpResponseMessage response) + { + var content = response?.Content; + if (content == null) return null; + try + { + return Cap(await content.ReadAsStringAsync().ConfigureAwait(false)); + } + catch + { + return null; + } + } + + /// + /// True when carries in + /// — the backend wrote it. Also used on 2xx answers, where + /// a missing echo is only counted (BackendTelemetryUploader), never acted on. + /// + public static bool EchoesCorrelationId(HttpResponseMessage response, string? sentCorrelationId) + { + if (response == null) throw new ArgumentNullException(nameof(response)); + if (string.IsNullOrEmpty(sentCorrelationId)) return false; + if (!response.Headers.TryGetValues(CorrelationIdHeader, out var values)) return false; + + foreach (var value in values) + { + if (string.Equals(value?.Trim(), sentCorrelationId, StringComparison.Ordinal)) return true; + } + return false; + } + + /// + /// True when the body is an HTML page rather than a JSON/plain-text reply. Checks the declared + /// Content-Type first, then sniffs the body prefix so a page served with a generic + /// Content-Type is still recognized. + /// + private static bool LooksLikeHtml(HttpResponseMessage response, string? body) + { + var mediaType = response.Content?.Headers?.ContentType?.MediaType; + if (string.Equals(mediaType, "text/html", StringComparison.OrdinalIgnoreCase)) + return true; + + if (string.IsNullOrWhiteSpace(body)) + return false; + + var head = body!.TrimStart(); + return head.StartsWith(" + /// Best-effort extraction of the HTML <title> (e.g. "Web App - Unavailable") so logs and + /// reports name the page without dumping markup. Null when absent, empty or the regex times out. + /// + private static string? TryExtractHtmlTitle(string? body) + { + if (string.IsNullOrEmpty(body)) return null; + try + { + var match = Regex.Match( + body, @"]*>\s*(.*?)\s*", + RegexOptions.IgnoreCase | RegexOptions.Singleline, + TimeSpan.FromMilliseconds(250)); + if (!match.Success) return null; + + var title = match.Groups[1].Value; + return title.Length == 0 ? null : (title.Length > MaxTitleLength ? title.Substring(0, MaxTitleLength) : title); + } + catch + { + return null; + } + } + + private static bool Contains(string? body, string marker) => + body != null && body.IndexOf(marker, StringComparison.OrdinalIgnoreCase) >= 0; + + private static string? Cap(string? body) => + body != null && body.Length > MaxBodyChars ? body.Substring(0, MaxBodyChars) : body; + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ServerCertificateObserver.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ServerCertificateObserver.cs new file mode 100644 index 00000000..7db95262 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/ServerCertificateObserver.cs @@ -0,0 +1,62 @@ +#nullable enable +using System; +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Threading; + +namespace AutopilotMonitor.Agent.V2.Core.Transport +{ + /// + /// Remembers the issuer of the server certificate of the last TLS connection a backend + /// validated. Evidence for a 401/403 that did not come from the + /// backend: a TLS-inspecting proxy presents a certificate from its own CA, which names the + /// product; a Microsoft issuer means the device certificate got lost on the device itself. + /// + /// Observation only: the callback returns exactly sslPolicyErrors == None, the verdict + /// .NET Framework applies to a client connection without a callback, so validation is never + /// weakened. The callback runs during the handshake of a new connection; a reused keep-alive + /// connection is not validated again. + /// + /// + public sealed class ServerCertificateObserver + { + /// Longest issuer kept. The CN leads the distinguished name and survives the cut. + internal const int MaxIssuerLength = 128; + + private string? _lastIssuer; + + /// Issuer of the last validated server certificate, null before the first handshake. + public string? LastIssuer => Volatile.Read(ref _lastIssuer); + + /// Installs the observing validation callback on . + public void Attach(HttpClientHandler handler) + { + if (handler == null) throw new ArgumentNullException(nameof(handler)); + handler.ServerCertificateCustomValidationCallback = + (_, certificate, _, sslPolicyErrors) => Validate(certificate, sslPolicyErrors); + } + + /// Records the issuer and returns the unchanged default verdict. + internal bool Validate(X509Certificate2? certificate, SslPolicyErrors sslPolicyErrors) + { + Record(certificate); + return sslPolicyErrors == SslPolicyErrors.None; + } + + private void Record(X509Certificate2? certificate) + { + if (certificate == null) return; + try + { + var issuer = certificate.Issuer; + if (string.IsNullOrEmpty(issuer)) return; + Volatile.Write(ref _lastIssuer, issuer.Length > MaxIssuerLength ? issuer.Substring(0, MaxIssuerLength) : issuer); + } + catch + { + // Observation must never fail a handshake. + } + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs index ff7b4590..6e28dbb3 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/BackendTelemetryUploader.cs @@ -9,6 +9,7 @@ using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.Models; @@ -49,7 +50,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry private readonly string? _agentVersion; private readonly AuthFailureTracker? _authFailureTracker; private readonly Logging.AgentLogger? _logger; + private readonly ServerCertificateObserver? _serverCertificates; + private readonly NetworkMetrics? _networkMetrics; + /// + /// Observer installed on the 's handler; its issuer becomes + /// evidence on an intercepted upload. Null = no evidence. + /// + /// + /// Counts 2xx answers without the correlation echo (agent_metrics_snapshot). Null = not counted. + /// public BackendTelemetryUploader( HttpClient httpClient, string baseUrl, @@ -60,7 +70,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry string? bootstrapToken = null, string? agentVersion = null, AuthFailureTracker? authFailureTracker = null, - Logging.AgentLogger? logger = null) + Logging.AgentLogger? logger = null, + ServerCertificateObserver? serverCertificates = null, + NetworkMetrics? networkMetrics = null) { _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl)); @@ -75,6 +87,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry _agentVersion = agentVersion; _authFailureTracker = authFailureTracker; _logger = logger; + _serverCertificates = serverCertificates; + _networkMetrics = networkMetrics; } public async Task UploadBatchAsync( @@ -117,7 +131,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); request.Content.Headers.ContentEncoding.Add("gzip"); AddSecurityHeaders(request); - request.Headers.Add("X-Correlation-ID", correlationId); + request.Headers.Add(ResponseOriginClassifier.CorrelationIdHeader, correlationId); // P14: device-clock send time, one value per request. Lets the backend separate // spool delay (SentAt − OccurredUtc, both device frame) from device-vs-server @@ -154,7 +168,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry using (response) { - var result = await MapResponseAsync(response).ConfigureAwait(false); + var result = await MapResponseAsync(response, correlationId).ConfigureAwait(false); sw.Stop(); if (result.Success) @@ -165,6 +179,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry { _logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}"); } + else if (result.Interception != null) + { + // The whole page finding instead of a body excerpt: the 200-character + // excerpt used to end before the "Client Certificate Required" text. + var interception = result.Interception; + _logger?.Warning( + $"BackendTelemetryUploader: ingest INTERCEPTED — the answer did not come from the backend " + + $"(corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={interception.StatusCode}, " + + $"reason={interception.ForeignReason}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " + + $"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\")."); + } else if (result.IsSessionGone) { _logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}"); @@ -179,7 +204,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry } } - private async Task MapResponseAsync(HttpResponseMessage response) + private async Task MapResponseAsync(HttpResponseMessage response, string correlationId) { var statusCode = (int)response.StatusCode; @@ -189,22 +214,37 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // backend hiccup does not eventually shut the agent down. _authFailureTracker?.RecordSuccess(); + // A 2xx without the backend's correlation echo may be a block page served with 200. + // Counted only (agent_metrics_snapshot) and still a success: whether every real + // backend 2xx carries the echo is measured before anything is ever enforced on it. + if (!ResponseOriginClassifier.EchoesCorrelationId(response, correlationId)) + _networkMetrics?.RecordOkWithoutEcho(); + // M4.6.ε — parse backend-to-agent control signals from the 2xx response body. // Body is best-effort JSON; anything unparseable degrades cleanly to plain Ok(). return await TryReadControlSignalsAsync(response).ConfigureAwait(false); } - var reason = await TryReadReasonAsync(response).ConfigureAwait(false); + var body = await ResponseOriginClassifier.ReadBodyPrefixAsync(response).ConfigureAwait(false); + var reason = string.IsNullOrEmpty(body) ? string.Empty : (body!.Length <= 200 ? body : body.Substring(0, 200)); var shortReason = $"http {statusCode}{(string.IsNullOrEmpty(reason) ? string.Empty : ": " + reason)}"; switch (response.StatusCode) { case HttpStatusCode.Unauthorized: // 401 case HttpStatusCode.Forbidden: // 403 + { // Feed the central tracker so the agent can shut down after MaxAuthFailures - // instead of retrying a permanently-revoked certificate forever. - _authFailureTracker?.RecordFailure(statusCode, "agent/telemetry"); - return UploadResult.Unauthorized($"unauthorized: {shortReason}"); + // instead of retrying a permanently-revoked certificate forever. It counts only + // what the backend (or the platform page of a stopped endpoint) answered; an + // answer from the network path is reported once and never counted. + var classification = ResponseOriginClassifier.Classify( + response, correlationId, body, _serverCertificates?.LastIssuer); + _authFailureTracker?.RecordFailure(statusCode, "agent/telemetry", classification); + return classification.IsForeign + ? UploadResult.Intercepted($"intercepted: http {statusCode} ({classification.ForeignReason})", classification) + : UploadResult.Unauthorized($"unauthorized: {shortReason}"); + } case HttpStatusCode.RequestTimeout: // 408 case (HttpStatusCode)429: // TooManyRequests — enum available net48 SP1+, cast for safety @@ -238,20 +278,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry return UploadResult.Transient(shortReason); } - private static async Task TryReadReasonAsync(HttpResponseMessage response) - { - try - { - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - if (string.IsNullOrEmpty(body)) return string.Empty; - return body.Length <= 200 ? body : body.Substring(0, 200); - } - catch - { - return string.Empty; - } - } - /// /// M4.6.ε — parse backend-to-agent control signals from a 2xx batch-response body. Fields /// are optional: an empty body or a body without any of these fields is treated as diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/InterceptionReport.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/InterceptionReport.cs new file mode 100644 index 00000000..a01ca255 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/InterceptionReport.cs @@ -0,0 +1,25 @@ +#nullable enable +using System; + +namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry +{ + /// + /// The first intercepted upload of the process, carried by + /// so the orchestrator can surface a + /// telemetry_upload_blocked event with the evidence. + /// + public sealed class InterceptionReport + { + public InterceptionReport(ResponseClassification interception, DateTime firstBlockedAtUtc) + { + Interception = interception ?? throw new ArgumentNullException(nameof(interception)); + FirstBlockedAtUtc = firstBlockedAtUtc; + } + + /// Status, sub-reason, page title and server-certificate issuer of the intercepted upload. + public ResponseClassification Interception { get; } + + /// When the first upload was intercepted (agent clock, UTC). + public DateTime FirstBlockedAtUtc { get; } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs index fdadff70..c97e5cf4 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/TelemetryUploadOrchestrator.cs @@ -25,6 +25,24 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry TimeSpan.FromMilliseconds(1600), }; + /// + /// Pause after each consecutive intercepted upload (the last entry repeats). Every attempt + /// is a POST of the head batch; the platform or proxy rejects it before the backend sees it. + /// + public static readonly IReadOnlyList InterceptionBackoffs = new[] + { + TimeSpan.FromSeconds(30), + TimeSpan.FromMinutes(1), + TimeSpan.FromMinutes(2), + TimeSpan.FromMinutes(5), + TimeSpan.FromMinutes(10), + TimeSpan.FromMinutes(15), + }; + + private const int TerminalAttemptNotGranted = 0; + private const int TerminalAttemptGranted = 1; + private const int TerminalAttemptUsed = 2; + private readonly ITelemetrySpool _spool; private readonly IBackendTelemetryUploader _uploader; private readonly IClock _clock; @@ -35,6 +53,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // M4.6.ε — backend-to-agent control signals. private DateTime? _pausedUntilUtc; + // Interception pause — a "not before" for the next upload after an intercepted one, modelled + // on the DeviceBlocked pause: drains compare it with the clock and return without HTTP, no + // timer (L.7). One immutable window so the termination thread reads it atomically; null = + // never intercepted or cleared by a success. The backoff step and the one-shot report are + // only touched under the drain guard. + private InterceptionWindow? _interceptionWindow; + private int _interceptionBackoffStep; + private bool _interceptionReported; + + // The one upload attempt the terminal phase may make despite the interception pause. + private int _terminalAttemptState; + // P1 — current effective batch size. Shrinks (never grows) when the backend returns 413, // so an oversized batch is split down instead of discarded. Only mutated under the drain // guard. Floor 1. @@ -113,6 +143,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry /// public event EventHandler? UploadBlocked; + /// + /// Raised once per process (after the drain guard releases) for the first intercepted upload + /// (), independent of . + /// Like that event, anything the handler posts queues behind the intercepted batch and + /// reaches the backend only once the path clears in the same run. + /// + public event EventHandler? UploadIntercepted; + /// /// Raised once per process (after the drain guard releases) when the backend answered /// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never @@ -128,6 +166,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry /// public bool IsPaused => _pausedUntilUtc.HasValue && _pausedUntilUtc.Value > _clock.UtcNow; + /// + /// true while the pause after an intercepted upload has not expired. Drains return + /// without HTTP in this state — except the one attempt + /// allows. A successful upload ends the pause and resets the backoff. + /// + public bool IsInterceptionPaused + { + get + { + var window = Volatile.Read(ref _interceptionWindow); + if (window == null) return false; + + // Device clocks jump (RTC, NTP). A clock set back past the pause start ends the + // pause instead of stretching it by the jump; the next attempt re-arms it. + var now = _clock.UtcNow; + return now >= window.FromUtc && now < window.UntilUtc; + } + } + + /// + /// Grants the single upload attempt per process that the terminal phase may make despite an + /// active interception pause; the first drain that meets the pause afterwards uses it. + /// Idempotent. Without an interception pause the grant changes nothing. The DeviceBlocked + /// pause is not affected and still skips terminal drains. Returns + /// , so the caller knows whether waking the drain pays off. + /// + public bool GrantTerminalAttempt() + { + Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptGranted, TerminalAttemptNotGranted); + return IsInterceptionPaused; + } + public long LastUploadedItemId => _spool.LastUploadedItemId; public TelemetryItem Enqueue(TelemetryItemDraft draft) @@ -150,6 +220,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry List? pendingPoison = null; string? pendingBlockedReason = null; string? pendingSessionGoneReason = null; + InterceptionReport? pendingInterception = null; DrainResult result; try @@ -158,12 +229,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry int failedBatches = 0; string? lastError = null; - // P1 — shared success-side control-signal handling. Applied to BOTH the normal - // success path and the poison-survivor re-upload, so a DeviceBlocked / DeviceKill / - // AdminAction / Actions response on the survivor upload is honoured identically and - // never silently dropped. Returns true if the backend just quarantined the device. - bool CollectControlSignals(UploadResult o) + // P1 — shared success-side handling. Applied to BOTH the normal success path and + // the poison-survivor re-upload, so a DeviceBlocked / DeviceKill / AdminAction / + // Actions response on the survivor upload is honoured identically and never silently + // dropped, and either success ends an interception pause. Returns true if the + // backend just quarantined the device. + bool HandleSuccessfulUpload(UploadResult o) { + EndInterceptionPause(); ApplyControlSignals(o); // mutates _pausedUntilUtc synchronously (next IsPaused sees it) if (CarriesSignal(o)) (pendingSignals ??= new List()).Add(o); return o.DeviceBlocked; @@ -182,6 +255,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry var batch = _spool.Peek(_effectiveBatchSize); if (batch.Count == 0) break; + // Interception pause: no HTTP before the not-before passes — periodic drains and + // immediate-flush wakeups alike — except the one attempt the terminal phase is + // granted. Checked after the Peek so only a real attempt uses the grant. + if (IsInterceptionPaused && !TryUseTerminalAttempt()) + { + lastError = $"upload intercepted, next attempt not before {_interceptionWindow?.UntilUtc:O}"; + break; + } + var outcome = await TryUploadWithRetryAsync(batch, cancellationToken).ConfigureAwait(false); if (outcome.Success) @@ -192,7 +274,22 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // Stop draining immediately if the backend just quarantined the device — // the next loop iteration would see IsPaused and break anyway, but there // is no point in even Peeking another batch. - if (CollectControlSignals(outcome)) break; + if (HandleSuccessfulUpload(outcome)) break; + } + else if (outcome.Interception != null) + { + // Something on the network path answered instead of the backend (TLS + // inspection, proxy page). RETAIN (cursor stays), pause with backoff, and + // surface the first one once per process — never as an auth failure. + failedBatches++; + lastError = outcome.ErrorReason; + StartInterceptionPause(outcome.Interception); + if (!_interceptionReported) + { + _interceptionReported = true; + pendingInterception = new InterceptionReport(outcome.Interception, _clock.UtcNow); + } + break; } else if (outcome.IsPoison) { @@ -236,7 +333,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // P1 fix: honour any control signals the survivor re-upload carried // (DeviceBlocked / DeviceKill / AdminAction / Actions) — same as the // normal success path; previously these were silently dropped. - deviceBlockedBySurvivors = CollectControlSignals(survivorOutcome); + deviceBlockedBySurvivors = HandleSuccessfulUpload(survivorOutcome); } // Survivors uploaded (or none) — advance past the whole batch; the named items @@ -316,7 +413,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry // One-shot loud marker so a stuck non-transient, non-auth block is visible on // the backend instead of only in the agent log (transient = normal retry; - // auth = the session is already shutting down via the auth path). + // a backend auth rejection = the auth path shuts the agent down; an + // interception has its own one-shot report above). if (!outcome.IsTransient && !outcome.IsAuthFailure && !_uploadBlockedReported) { _uploadBlockedReported = true; @@ -362,9 +460,56 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry catch { /* handler must not abort the drain result */ } } + if (pendingInterception != null) + { + try { UploadIntercepted?.Invoke(this, pendingInterception); } + catch { /* handler must not abort the drain result */ } + } + return result; } + /// + /// Arms the interception pause with the next backoff step (the last step repeats). Called + /// under the drain guard. + /// + private void StartInterceptionPause(ResponseClassification interception) + { + var delay = InterceptionBackoffs[Math.Min(_interceptionBackoffStep, InterceptionBackoffs.Count - 1)]; + _interceptionBackoffStep = Math.Min(_interceptionBackoffStep + 1, InterceptionBackoffs.Count - 1); + var now = _clock.UtcNow; + Volatile.Write(ref _interceptionWindow, new InterceptionWindow(now, now + delay)); + + // The uploader already logged the page finding at Warning; this line adds the schedule. + _logger?.Info( + $"TelemetryUploadOrchestrator: upload intercepted ({interception.ForeignReason}, http {interception.StatusCode}) — " + + $"batch retained, next attempt not before {now + delay:O} ({delay.TotalSeconds:0}s)."); + } + + /// A successful upload ends the interception pause and resets the backoff. Called under the drain guard. + private void EndInterceptionPause() + { + if (Interlocked.Exchange(ref _interceptionWindow, null) == null) return; + + _interceptionBackoffStep = 0; + _logger?.Info("TelemetryUploadOrchestrator: upload path clear again after an interception — uploads resume."); + } + + private bool TryUseTerminalAttempt() => + Interlocked.CompareExchange(ref _terminalAttemptState, TerminalAttemptUsed, TerminalAttemptGranted) == TerminalAttemptGranted; + + private sealed class InterceptionWindow + { + public InterceptionWindow(DateTime fromUtc, DateTime untilUtc) + { + FromUtc = fromUtc; + UntilUtc = untilUtc; + } + + public DateTime FromUtc { get; } + public DateTime UntilUtc { get; } + } + /// /// Number of items at the head of whose partition key is not the /// current session's. 0 when no current key is configured (every batch counts as own). diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs index 9f138f89..a97ac37e 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Transport/Telemetry/UploadResult.cs @@ -31,7 +31,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry bool requiresSplit = false, bool isPoison = false, IReadOnlyList? poisonRowKeys = null, - bool isSessionGone = false) + bool isSessionGone = false, + ResponseClassification? interception = null) { Success = success; ErrorReason = errorReason; @@ -46,6 +47,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry IsPoison = isPoison; PoisonRowKeys = poisonRowKeys; IsSessionGone = isSessionGone; + Interception = interception; } public bool Success { get; } @@ -93,6 +95,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry /// public bool IsSessionGone { get; } + /// + /// A 401/403 that did not come from the backend (no X-Correlation-ID echo): the + /// platform's "Client Certificate Required" page or a proxy page — TLS inspection on the + /// network path. Not an auth failure: never counted by the AuthFailureTracker. The + /// orchestrator retains the batch and pauses uploads with backoff. + /// + public bool IsIntercepted => Interception != null; + + /// Status, sub-reason, page title and server-certificate issuer of the interception; null unless . + public ResponseClassification? Interception { get; } + /// /// Non-terminal quarantine signalled by the backend. The agent should stop draining /// uploads until (if set) elapses — the session stays alive so @@ -186,5 +199,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false, deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null, isSessionGone: true); + + /// + /// A 401/403 something on the network path answered (). + /// Flagged , never . + /// + public static UploadResult Intercepted(string reason, ResponseClassification interception) + { + if (interception == null) throw new ArgumentNullException(nameof(interception)); + if (!interception.IsForeign) throw new ArgumentException("Only a foreign response is an interception.", nameof(interception)); + return new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false, + deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null, + interception: interception); + } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs index 9f33125a..31613110 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.Guards.cs @@ -256,10 +256,10 @@ namespace AutopilotMonitor.Agent.V2 /// /// true when the file-based enrollment-complete.marker signals that a /// previous session already completed enrollment — written by - /// right before it fires the cleanup retry, - /// so that a next-boot observing the marker exits cleanly even if the cleanup retry - /// failed to remove the Scheduled Task. Triggers another self-destruct attempt when - /// is set. + /// (emergency break, intercepted-registration give-up, + /// config kill signal) right before it fires the cleanup retry, so that a next-boot observing the marker exits + /// cleanly even if the cleanup retry failed to remove the Scheduled Task. Triggers another + /// self-destruct attempt when is set. /// /// The HKLM\SOFTWARE\AutopilotMonitor\Deployed registry value is NOT consulted /// here: it is a bootstrap-script reentry lock (read by Install-AutopilotMonitor-v2.ps1 @@ -350,23 +350,10 @@ namespace AutopilotMonitor.Agent.V2 try { onBreakFired?.Invoke(sessionAgeHours); } catch (Exception cbEx) { logger.Debug($"Emergency-break notify callback failed: {cbEx.Message}"); } - // Write enrollment-complete.marker so the next start exits cleanly even if - // the cleanup retry below fails (Scheduled Task still alive, files locked, …). - try - { - Directory.CreateDirectory(stateDirectory); - var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName); - File.WriteAllText(markerPath, - $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)"); - } - catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); } - - if (selfDestructOnComplete) - { - TryRetryCleanup(cleanupServiceFactory, logger, "emergency_break"); - } - - persistence.Delete(logger); + EndMonitoringAtStart( + stateDirectory, persistence, selfDestructOnComplete, cleanupServiceFactory, logger, + markerText: $"Emergency break at {DateTime.UtcNow:O} (session age: {sessionAgeHours:F1}h)", + cleanupReason: "emergency_break"); logger.Info("Emergency break: cleanup completed. Agent will exit."); return true; } @@ -377,6 +364,40 @@ namespace AutopilotMonitor.Agent.V2 } } + /// + /// Ends monitoring at process start: writes enrollment-complete.marker first so the + /// next start exits cleanly even if the cleanup retry fails (Scheduled Task still alive, + /// files locked, …), runs the cleanup retry when is set + /// (the emergency break and the give-up pass SelfDestructOnComplete, the kill signal + /// forces it), and deletes the persisted session files. Shared by the session-age emergency + /// break, the intercepted-registration give-up + /// (BackendSessionRegistration.ConcludeFailedRun) and the config-channel kill signal + /// so all three leave the device in the same state. Never throws. + /// + internal static void EndMonitoringAtStart( + string stateDirectory, + SessionIdPersistence persistence, + bool runSelfDestruct, + Func cleanupServiceFactory, + AgentLogger logger, + string markerText, + string cleanupReason) + { + try + { + Directory.CreateDirectory(stateDirectory); + File.WriteAllText(Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName), markerText); + } + catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); } + + if (runSelfDestruct) + { + TryRetryCleanup(cleanupServiceFactory, logger, cleanupReason); + } + + persistence?.Delete(logger); + } + /// /// Kill-switch on the control channel: honours a DeviceKillSignal served on the /// agent/config response. Config is fetched at every process start (boot via @@ -409,22 +430,15 @@ namespace AutopilotMonitor.Agent.V2 if (consoleMode) Console.Out.WriteLine("KILL: administrator issued a remote kill signal. Cleaning up and exiting."); - // Write enrollment-complete.marker first so the next start exits cleanly even if - // the cleanup below fails (Scheduled Task still alive, files locked, …) — same - // guarantee-of-termination pattern as the session-age emergency break. - try - { - Directory.CreateDirectory(stateDirectory); - var markerPath = Path.Combine(stateDirectory, EnrollmentCompleteMarkerFileName); - File.WriteAllText(markerPath, $"Kill signal at {DateTime.UtcNow:O} (config channel)"); - } - catch (Exception ex) { logger.Warning($"Failed to write enrollment-complete.marker: {ex.Message}"); } + SessionIdPersistence persistence = null; + try { persistence = new SessionIdPersistence(dataDirectory); } + catch (Exception ex) { logger.Warning($"Failed to open the persisted session after kill: {ex.Message}"); } // forceSelfDestruct — no SelfDestructOnComplete gate, matching the telemetry-kill. - TryRetryCleanup(cleanupServiceFactory, logger, "kill_signal"); - - try { new SessionIdPersistence(dataDirectory).Delete(logger); } - catch (Exception ex) { logger.Warning($"Failed to delete persisted session after kill: {ex.Message}"); } + EndMonitoringAtStart( + stateDirectory, persistence, runSelfDestruct: true, cleanupServiceFactory, logger, + markerText: $"Kill signal at {DateTime.UtcNow:O} (config channel)", + cleanupReason: "kill_signal"); logger.Info("Kill signal handled — agent exiting."); return true; diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs index 9478dcd2..01ca614d 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs @@ -269,13 +269,17 @@ namespace AutopilotMonitor.Agent.V2 // Phase 6 (POST /api/agent/register-session with retry + outcome-based exit-code // mapping + on-failure client disposal) is encapsulated in BackendSessionRegistration. - // Exit codes: 6 = AuthFailed, 7 = anything else. See Runtime/BackendSessionRegistration.cs. + // Exit codes: 6 = AuthFailed, 7 = anything else. A registration intercepted on more + // starts than the buffer allows cleans up like the emergency break (state folder + + // cleanup factory) and still exits 6. See Runtime/BackendSessionRegistration.cs. var registration = Runtime.BackendSessionRegistration.Register( bootstrap.AgentConfig, auth, telemetry.MtlsHttpClient, GetAgentVersion(), consoleMode, logger, rotateSession: () => Runtime.BackendSessionRegistration.RotateSession( bootstrap.AgentConfig, bootstrap.SessionPersistence, auth, transportDir, logger), sessionPersistence: bootstrap.SessionPersistence, - configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString()); + configFetchOutcome: runtimeConfig.RemoteConfigService.LastFetchOutcome.ToString(), + stateDirectory: stateSubdir, + cleanupServiceFactory: bootstrap.CleanupServiceFactory); if (registration.ShouldExit) { return registration.ExitCode; diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs index 0466e1bc..603a3a6e 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs @@ -345,7 +345,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime { lifecyclePost = new InformationalEventPost(ingress, SystemClock.Instance, logger); LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, remoteConfigService, logger, - rotatedFromSessionId: registrationResult?.RotatedFromSessionId); + rotatedFromSessionId: registrationResult?.RotatedFromSessionId, + priorRegistrationFailures: registration.PriorFailures); // Wire-visible signal that the agent is running on defaults/cache // rather than the live tenant config — closes the cold-start // blind spot observed in session 8f2bef72 (2026-05-19). diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendClientFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendClientFactory.cs index cfa313fb..52f81200 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendClientFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendClientFactory.cs @@ -7,6 +7,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging; using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport; using AutopilotMonitor.Agent.V2.Core.Orchestration; using AutopilotMonitor.Agent.V2.Core.Security; +using AutopilotMonitor.Agent.V2.Core.Transport; using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry; using AutopilotMonitor.DecisionCore.Engine; using AutopilotMonitor.Shared.Models; @@ -76,7 +77,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime logger.Debug("Client certificate authentication disabled."); } - var apiHttpClient = BuildBackendApiHttpClient(networkMetrics, clientCertificate, agentVersion); + // One observer for both backend handler stacks (this one and the mTLS telemetry stack + // in BuildTelemetryClients): the issuer of the last validated server certificate is + // the evidence for a 401/403 that something on the network path answered. + var serverCertificates = new ServerCertificateObserver(); + + var apiHttpClient = BuildBackendApiHttpClient(networkMetrics, clientCertificate, agentVersion, serverCertificates); var backendApiClient = new BackendApiClient( httpClient: apiHttpClient, @@ -87,7 +93,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime useBootstrapTokenAuth: agentConfig.UseBootstrapTokenAuth, bootstrapToken: agentConfig.BootstrapToken, agentVersion: agentVersion, - logger: logger); + logger: logger, + serverCertificates: serverCertificates); // M4.6.γ — Emergency + Distress reporters. Plumbed into RemoteConfigService so // Config-fetch failures (auth vs network) flow to the correct channel. Also fires @@ -141,7 +148,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime serialNumber: hardware.SerialNumber, distressReporter: distressReporter, emergencyReporter: emergencyReporter, - authFailureTracker: authFailureTracker); + authFailureTracker: authFailureTracker, + serverCertificates: serverCertificates); } /// @@ -168,7 +176,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime mtlsHttpClient = MtlsHttpClientFactory.Create( resolver: new DefaultCertificateResolver(), logger: logger, - metrics: auth.NetworkMetrics); + metrics: auth.NetworkMetrics, + serverCertificates: auth.ServerCertificates); } catch (InvalidOperationException ex) { @@ -189,7 +198,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime bootstrapToken: agentConfig.UseBootstrapTokenAuth ? agentConfig.BootstrapToken : null, agentVersion: agentVersion, authFailureTracker: auth.AuthFailureTracker, - logger: logger); // Plan §5 Fix 5 — upload-cadence logging + logger: logger, // Plan §5 Fix 5 — upload-cadence logging + serverCertificates: auth.ServerCertificates, + networkMetrics: auth.NetworkMetrics); } catch (Exception ex) { @@ -211,14 +222,16 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// — records every SendAsync into /// the shared counter. /// — TLS, optional client cert, - /// gzip/deflate response decompression. + /// gzip/deflate response decompression, optional observing server-certificate + /// callback (; never changes the verdict). /// /// /// internal static HttpClient BuildBackendApiHttpClient( NetworkMetrics networkMetrics, X509Certificate2 clientCertificate, - string agentVersion) + string agentVersion, + ServerCertificateObserver serverCertificates = null) { var inner = new HttpClientHandler(); if (inner.SupportsAutomaticDecompression) @@ -229,6 +242,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime { inner.ClientCertificates.Add(clientCertificate); } + serverCertificates?.Attach(inner); var pipeline = new NetworkMetricsRecordingHandler(networkMetrics, inner); var client = new HttpClient(pipeline) @@ -270,6 +284,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime public EmergencyReporter EmergencyReporter { get; } public AuthFailureTracker AuthFailureTracker { get; } + /// + /// Server-certificate observer shared by the BackendApiClient handler and the mTLS + /// telemetry handler built in Phase 5. + /// + public ServerCertificateObserver ServerCertificates { get; } + public BackendAuthBundle( BackendApiClient backendApiClient, NetworkMetrics networkMetrics, @@ -279,7 +299,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime string serialNumber, DistressReporter distressReporter, EmergencyReporter emergencyReporter, - AuthFailureTracker authFailureTracker) + AuthFailureTracker authFailureTracker, + ServerCertificateObserver serverCertificates) { BackendApiClient = backendApiClient; NetworkMetrics = networkMetrics; @@ -290,6 +311,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime DistressReporter = distressReporter; EmergencyReporter = emergencyReporter; AuthFailureTracker = authFailureTracker; + ServerCertificates = serverCertificates; } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendSessionRegistration.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendSessionRegistration.cs index 6940fa17..1451815f 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendSessionRegistration.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/BackendSessionRegistration.cs @@ -5,6 +5,8 @@ using System.Net.NetworkInformation; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Configuration; using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime; using AutopilotMonitor.Agent.V2.Core.Security; using AutopilotMonitor.Shared.Models; @@ -19,10 +21,22 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// no-op and session status / phase / admin-overrides / validator reconcile all break. /// On registration failure we follow V1's rule: collectors MUST NOT start and the /// agent exits cleanly; the Scheduled Task has only a boot trigger, so the next attempt is - /// the next boot. The failed run is recorded for the emergency break report. + /// the next boot. The failed run is recorded for the emergency break report. A registration + /// that something on the network path answered gets InterceptedRegistrationRetryBoots + /// further boots, then the agent cleans up like the emergency break (). + /// A successful registration after failed starts reports them on agent_started + /// (). /// internal static class BackendSessionRegistration { + /// + /// How long a failed start waits for the pending ClientCertificateNotReceived distress before + /// it exits or cleans up — the distress reporter's own HTTP timeout. + /// + internal static readonly TimeSpan PendingDistressBound = TimeSpan.FromSeconds(5); + + /// State folder for enrollment-complete.marker on the intercepted-registration give-up. + /// Self-destruct for the give-up (run when SelfDestructOnComplete). public static SessionRegistrationOutcomeResult Register( AgentConfiguration agentConfig, BackendAuthBundle auth, @@ -32,7 +46,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime AgentLogger logger, Func rotateSession = null, SessionIdPersistence sessionPersistence = null, - string configFetchOutcome = null) + string configFetchOutcome = null, + string stateDirectory = null, + Func cleanupServiceFactory = null) { if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig)); if (auth == null) throw new ArgumentNullException(nameof(auth)); @@ -57,36 +73,41 @@ namespace AutopilotMonitor.Agent.V2.Runtime rotateSession: rotateSession) .GetAwaiter().GetResult(); - RecordRegistrationRun( - sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow, - linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, logger); - - if (registrationResult.Outcome != SessionRegistrationOutcome.Succeeded) + if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded) { - logger.Error( - $"=== SESSION REGISTRATION FAILED ({registrationResult.Outcome}: {registrationResult.ErrorMessage}) — " + - "collectors will NOT start to prevent orphaned events. ==="); - if (consoleMode) - Console.Error.WriteLine($"FATAL: session registration failed ({registrationResult.Outcome}). Agent exiting."); - try { mtlsHttpClient?.Dispose(); } catch { } - try { auth.BackendApiClient?.Dispose(); } catch { } - // Exit code differs so the diag skill can distinguish Auth vs Network in Scheduled-Task history. - return SessionRegistrationOutcomeResult.Exit( - registrationResult.Outcome == SessionRegistrationOutcome.AuthFailed ? 6 : 7); + return SessionRegistrationOutcomeResult.Continue( + registrationResult, ClearAfterSuccess(sessionPersistence, NetworkFingerprint.TryCapture, logger)); } - return SessionRegistrationOutcomeResult.Continue(registrationResult); + var record = RecordFailedRun( + sessionPersistence, registrationResult, startedUtc, DateTime.UtcNow, + linkUpAtStart, TryIsNetworkLinkUp(), configFetchOutcome, NetworkFingerprint.TryCapture(), logger); + + logger.Error( + $"=== SESSION REGISTRATION FAILED ({registrationResult.Outcome}: {registrationResult.ErrorMessage}) — " + + "collectors will NOT start to prevent orphaned events. ==="); + if (consoleMode) + Console.Error.WriteLine($"FATAL: session registration failed ({registrationResult.Outcome}). Agent exiting."); + try { mtlsHttpClient?.Dispose(); } catch { } + try { auth.BackendApiClient?.Dispose(); } catch { } + + return SessionRegistrationOutcomeResult.Exit(ConcludeFailedRun( + registrationResult, record, agentConfig, auth.AuthFailureTracker, + sessionPersistence, stateDirectory, cleanupServiceFactory, logger)); } /// Longest kept on disk and on the wire. internal const int MaxRecordedErrorLength = 256; /// - /// Keeps registration-failure.json in step with this run: a failed run is folded - /// into the record (the emergency break later reports it — for a session that never - /// registered, the only account of why), a successful one clears it. Never throws. + /// Folds a failed run into registration-failure.json (the emergency break later reports + /// it — for a session that never registered, the only account of why) and returns the record + /// including this run; null without persistence or when it could not be written. An + /// intercepted run also counts toward , + /// which only a successful registration resets. is this + /// run's network; it is kept beside the record and never leaves the device. Never throws. /// - internal static void RecordRegistrationRun( + internal static RegistrationFailureSummary RecordFailedRun( SessionIdPersistence sessionPersistence, SessionRegistrationResult registrationResult, DateTime startedUtc, @@ -94,24 +115,20 @@ namespace AutopilotMonitor.Agent.V2.Runtime bool? linkUpAtStart, bool? linkUpAtEnd, string configFetchOutcome, + string networkFingerprint, AgentLogger logger) { - if (sessionPersistence == null || registrationResult == null) return; + if (sessionPersistence == null || registrationResult == null) return null; try { - if (registrationResult.Outcome == SessionRegistrationOutcome.Succeeded) - { - sessionPersistence.ClearRegistrationFailure(logger); - return; - } - var error = registrationResult.ErrorMessage; if (error != null && error.Length > MaxRecordedErrorLength) error = error.Substring(0, MaxRecordedErrorLength); - sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary + return sessionPersistence.RecordRegistrationFailure(new RegistrationFailureSummary { FailedRuns = 1, + InterceptedRuns = registrationResult.IsIntercepted ? 1 : 0, FirstFailedAtUtc = endedUtc, LastFailedAtUtc = endedUtc, Outcome = registrationResult.Outcome.ToString(), @@ -120,14 +137,99 @@ namespace AutopilotMonitor.Agent.V2.Runtime NetworkLinkUpAtEnd = linkUpAtEnd, AttemptWindowSeconds = Math.Round(Math.Max(0, (endedUtc - startedUtc).TotalSeconds), 1), ConfigFetchOutcome = configFetchOutcome, - }, logger); + }, logger, networkFingerprint); } catch (Exception ex) { logger?.Warning($"Registration run record failed (best-effort): {ex.Message}"); + return null; } } + /// + /// A successful registration clears registration-failure.json. When earlier starts of + /// this session had failed, the cleared record comes back for this run's agent_started + /// — until now the success erased it silently and late successes were uncountable. The + /// network fingerprints are compared here ( runs + /// only when there is a record) and never leave the device. Null when there was no record. + /// Never throws. + /// + internal static PriorRegistrationFailures ClearAfterSuccess( + SessionIdPersistence sessionPersistence, + Func captureNetworkFingerprint, + AgentLogger logger) + { + if (sessionPersistence == null) return null; + try + { + var cleared = sessionPersistence.LoadRegistrationFailureRecord(); + sessionPersistence.ClearRegistrationFailure(logger); + if (cleared == null) return null; + + var prior = new PriorRegistrationFailures( + cleared.Summary.FailedRuns, + cleared.Summary.InterceptedRuns, + cleared.Summary.Outcome, + NetworkFingerprint.Differs(cleared.NetworkFingerprint, captureNetworkFingerprint?.Invoke())); + logger?.Info( + $"Session registered after {prior.FailedRuns} failed start(s) ({prior.InterceptedRuns} intercepted, " + + $"last: {prior.LastOutcome}, network changed: {prior.NetworkChanged?.ToString() ?? "unknown"})."); + return prior; + } + catch (Exception ex) + { + logger?.Warning($"Registration run record failed (best-effort): {ex.Message}"); + return null; + } + } + + /// + /// The end of a start whose registration failed. First the ClientCertificateNotReceived + /// distress, which the tracker dispatched fire-and-forget, gets up to + /// to leave the machine: this process is about to exit or + /// clean up, and nothing else waits for it. Then the intercepted-registration buffer: when + /// this start was intercepted and the intercepted starts since the last success (this one + /// included) exceed InterceptedRegistrationRetryBoots, monitoring ends exactly like + /// the 48-hour emergency break (). Every other + /// failure — a backend 401/403, a network failure, an interception within the buffer, a + /// record that could not be written — leaves everything in place for the next boot. Returns + /// the exit code: 6 for an auth failure (the give-up included: it is still one), 7 otherwise. + /// + internal static int ConcludeFailedRun( + SessionRegistrationResult registrationResult, + RegistrationFailureSummary record, + AgentConfiguration agentConfig, + AuthFailureTracker authFailureTracker, + SessionIdPersistence sessionPersistence, + string stateDirectory, + Func cleanupServiceFactory, + AgentLogger logger) + { + if (authFailureTracker != null && !authFailureTracker.WaitForPendingForeignDistress(PendingDistressBound)) + logger.Warning($"ClientCertificateNotReceived distress still in flight after {PendingDistressBound.TotalSeconds:F0}s — continuing without it."); + + var buffer = agentConfig.InterceptedRegistrationRetryBoots; + if (registrationResult.IsIntercepted && record != null && record.InterceptedRuns > buffer) + { + logger.Warning( + $"=== REGISTRATION INTERCEPTED ON {record.InterceptedRuns} START(S) SINCE THE LAST SUCCESS " + + $"(buffer: {buffer} further boot(s)) — cleaning up like the emergency break. ==="); + Program.EndMonitoringAtStart( + stateDirectory, sessionPersistence, agentConfig.SelfDestructOnComplete, cleanupServiceFactory, logger, + markerText: $"Intercepted registration at {DateTime.UtcNow:O} ({record.InterceptedRuns} intercepted start(s), buffer {buffer})", + cleanupReason: "intercepted_registration"); + } + else if (registrationResult.IsIntercepted) + { + logger.Info( + $"Registration intercepted on {record?.InterceptedRuns.ToString() ?? "an unrecorded number of"} start(s) " + + $"since the last success (buffer: {buffer} further boot(s)) — the next boot retries."); + } + + // Exit code differs so the diag skill can distinguish Auth vs Network in Scheduled-Task history. + return registrationResult.Outcome == SessionRegistrationOutcome.AuthFailed ? 6 : 7; + } + private static bool? TryIsNetworkLinkUp() { try { return NetworkInterface.GetIsNetworkAvailable(); } @@ -266,7 +368,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// Phase 6 outcome — either an early exit (V1 parity: 6 = AuthFailed, 7 = anything else) /// or a Continue payload carrying the successful /// for downstream consumers (initial-signal posting, admin-preemption handling, - /// session-started anchor, AdminAction surface in the wired telemetry response). + /// session-started anchor, AdminAction surface in the wired telemetry response) and the + /// failed starts this registration followed (, for agent_started). /// internal sealed class SessionRegistrationOutcomeResult { @@ -274,17 +377,50 @@ namespace AutopilotMonitor.Agent.V2.Runtime public int ExitCode { get; } public SessionRegistrationResult Registration { get; } - private SessionRegistrationOutcomeResult(bool shouldExit, int exitCode, SessionRegistrationResult registration) + /// The failed-registration record this success cleared; null when there was none. + public PriorRegistrationFailures PriorFailures { get; } + + private SessionRegistrationOutcomeResult( + bool shouldExit, int exitCode, SessionRegistrationResult registration, PriorRegistrationFailures priorFailures) { ShouldExit = shouldExit; ExitCode = exitCode; Registration = registration; + PriorFailures = priorFailures; } public static SessionRegistrationOutcomeResult Exit(int code) - => new SessionRegistrationOutcomeResult(true, code, null); + => new SessionRegistrationOutcomeResult(true, code, null, null); - public static SessionRegistrationOutcomeResult Continue(SessionRegistrationResult registration) - => new SessionRegistrationOutcomeResult(false, 0, registration); + public static SessionRegistrationOutcomeResult Continue( + SessionRegistrationResult registration, PriorRegistrationFailures priorFailures = null) + => new SessionRegistrationOutcomeResult(false, 0, registration, priorFailures); + } + + /// + /// What the failed starts before a successful registration left in the record it cleared, + /// reported once on that run's agent_started (priorRegistration* keys): whether a + /// further boot rescues intercepted or failed starts becomes countable. Only the fingerprint + /// comparison is carried, never a fingerprint. + /// + internal sealed class PriorRegistrationFailures + { + public PriorRegistrationFailures(int failedRuns, int interceptedRuns, string lastOutcome, bool? networkChanged) + { + FailedRuns = failedRuns; + InterceptedRuns = interceptedRuns; + LastOutcome = lastOutcome; + NetworkChanged = networkChanged; + } + + public int FailedRuns { get; } + + public int InterceptedRuns { get; } + + /// Registration outcome of the latest failed start (e.g. AuthFailed, Failed). + public string LastOutcome { get; } + + /// The latest failed start ran on a different network than this one; null when either side is unknown. + public bool? NetworkChanged { get; } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs index e5c0609d..d36ae1f9 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs @@ -35,6 +35,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime /// and the tenant-influenced runtime knobs so dashboards can classify crash-loops, /// backend-rejected sessions and forced self-destruct runs. Phase stays /// — the event is NOT a phase declaration. + /// (failed starts this run's registration + /// followed) adds the priorRegistration* keys; null leaves them out. /// public static void EmitAgentStarted( InformationalEventPost post, @@ -43,7 +45,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime string agentVersion, RemoteConfigService remoteConfigService, AgentLogger logger, - string rotatedFromSessionId = null) + string rotatedFromSessionId = null, + PriorRegistrationFailures priorRegistrationFailures = null) { try { @@ -95,6 +98,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime if (previousExit?.LastBootUtc.HasValue == true) data["previousBootUtc"] = previousExit.LastBootUtc.Value.ToString("o"); + // Earlier starts of this session failed to register and this one succeeded: the + // record the success cleared, so late registrations (and whether the network + // changed in between) are countable. Each key only when known. + if (priorRegistrationFailures != null) + { + data["priorRegistrationFailedRuns"] = priorRegistrationFailures.FailedRuns; + data["priorRegistrationInterceptedRuns"] = priorRegistrationFailures.InterceptedRuns; + if (!string.IsNullOrEmpty(priorRegistrationFailures.LastOutcome)) + data["priorRegistrationLastOutcome"] = priorRegistrationFailures.LastOutcome; + if (priorRegistrationFailures.NetworkChanged.HasValue) + data["priorRegistrationNetworkChanged"] = priorRegistrationFailures.NetworkChanged.Value; + } + post.Emit(new EnrollmentEvent { SessionId = agentConfig.SessionId, diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/AgentConfigResolverTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/AgentConfigResolverTests.cs index 8ca59792..e87bd99b 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/AgentConfigResolverTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/AgentConfigResolverTests.cs @@ -47,6 +47,7 @@ public class AgentConfigResolverTests Assert.True(r.EnableGeoLocation); Assert.Equal(5, r.MaxAuthFailures); Assert.Equal(0, r.AuthFailureTimeoutMinutes); + Assert.Equal(1, r.InterceptedRegistrationRetryBoots); // backend constant, no tenant or operator setting Assert.Equal("Info", r.LogLevel); Assert.Equal(10, r.RebootDelaySeconds); Assert.Equal(60, r.EnrollmentSummaryTimeoutSeconds); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/CorrelationIdMiddlewareTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/CorrelationIdMiddlewareTests.cs new file mode 100644 index 00000000..95313037 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/CorrelationIdMiddlewareTests.cs @@ -0,0 +1,148 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Middleware; +using Microsoft.AspNetCore.Http; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// The agent treats a 401/403 as this backend's verdict only when the response echoes the +/// X-Correlation-ID it sent; the platform's own pages and a proxy's pages carry none. The +/// middleware stamps the header onto the shared response before the rest +/// of the pipeline runs, so the stamp does not depend on the status a later step writes: a +/// middleware envelope () and a function response alike — with the +/// ASP.NET Core integration a worker HttpResponseData is a view over that same response, +/// and its header set starts from the headers already on it. +/// +public class CorrelationIdMiddlewareTests +{ + private const string HeaderName = "X-Correlation-ID"; + + /// The agent's format: a fresh Guid.ToString("N") per request. + private const string AgentCorrelationId = "7c9e6679a1b24f3e8d5c0b1a2f3e4d5c"; + + private sealed class Run + { + public required HttpContext Http { get; init; } + public required IDictionary Items { get; init; } + + public string EchoedId => Http.Response.Headers[HeaderName].ToString(); + + public string Body() + { + Http.Response.Body.Position = 0; + return new StreamReader(Http.Response.Body, Encoding.UTF8).ReadToEnd(); + } + } + + /// Runs the middleware; is whatever answers the request after it. + private static async Task InvokeAsync(string? inboundId, Func answer) + { + var http = new DefaultHttpContext(); + http.Response.Body = new MemoryStream(); + if (inboundId != null) + http.Request.Headers[HeaderName] = inboundId; + var items = new Dictionary { ["HttpRequestContext"] = http }; + var context = new Mock(); + context.SetupGet(c => c.Items).Returns(items); + + var sut = new CorrelationIdMiddleware(NullLogger.Instance); + await sut.Invoke(context.Object, ctx => answer(ctx, http)); + return new Run { Http = http, Items = items }; + } + + /// A function's answer: status and JSON body on the shared response. + private static Func FunctionAnswers(HttpStatusCode status) => (_, http) => + { + http.Response.StatusCode = (int)status; + http.Response.ContentType = "application/json"; + return http.Response.WriteAsync("{\"success\":false}"); + }; + + // ── A valid inbound id is echoed on every status ───────────────────────────── + + [Theory] + [InlineData(HttpStatusCode.OK)] // ingest 2xx + [InlineData(HttpStatusCode.Unauthorized)] // ValidateSecurityAsync → ResponseHelper.ErrorAsync + [InlineData(HttpStatusCode.Forbidden)] // the same, plus RegisterSession's and GetDiagnosticsUploadUrl's own 403 bodies + public async Task A_valid_inbound_id_is_echoed_whatever_status_the_function_answers_with(HttpStatusCode status) + { + var run = await InvokeAsync(AgentCorrelationId, FunctionAnswers(status)); + + Assert.Equal((int)status, run.Http.Response.StatusCode); + Assert.Equal(AgentCorrelationId, run.EchoedId); + } + + [Theory] + [InlineData(HttpStatusCode.Unauthorized)] + [InlineData(HttpStatusCode.Forbidden)] + public async Task A_valid_inbound_id_is_echoed_on_a_middleware_error_envelope_and_in_its_body(HttpStatusCode status) + { + var run = await InvokeAsync(AgentCorrelationId, (ctx, http) => ApiErrorWriter.WriteAsync( + http, ctx.GetCorrelationId(), status, ApiErrorWriter.DefaultCode(status), "Request rejected.")); + + Assert.Equal((int)status, run.Http.Response.StatusCode); + Assert.Equal(AgentCorrelationId, run.EchoedId); + using var body = JsonDocument.Parse(run.Body()); + Assert.Equal(AgentCorrelationId, body.RootElement.GetProperty("correlationId").GetString()); + } + + public static TheoryData InsideTheAllowList => new() + { + AgentCorrelationId, + "Az_09-mixed", + new string('a', 128), + }; + + [Theory] + [MemberData(nameof(InsideTheAllowList))] + public async Task Every_id_inside_the_allow_list_is_echoed_verbatim(string inboundId) + { + var run = await InvokeAsync(inboundId, FunctionAnswers(HttpStatusCode.Forbidden)); + + Assert.Equal(inboundId, run.EchoedId); + Assert.Equal(inboundId, run.Items["CorrelationId"]); + } + + // ── Anything else is replaced, never echoed ────────────────────────────────── + + public static TheoryData OutsideTheAllowList => new() + { + "", + "7c9e6679 a1b2", + "7c9e6679\r\nX-Forged: 1", + "7c9e6679.a1b2", + "7c9e6679-ä", + new string('a', 129), + }; + + [Theory] + [MemberData(nameof(OutsideTheAllowList))] + public async Task An_inbound_id_outside_the_allow_list_is_replaced_not_echoed(string inboundId) + { + var run = await InvokeAsync(inboundId, FunctionAnswers(HttpStatusCode.Forbidden)); + + Assert.NotEqual(inboundId, run.EchoedId); + AssertFreshId(run); + } + + [Fact] + public async Task A_request_without_an_id_gets_a_fresh_one() + { + var run = await InvokeAsync(null, FunctionAnswers(HttpStatusCode.OK)); + + AssertFreshId(run); + } + + /// A minted id is a compact GUID, and the request's logs and error bodies carry the same value. + private static void AssertFreshId(Run run) + { + Assert.Matches("^[0-9a-f]{32}$", run.EchoedId); + Assert.Equal(run.EchoedId, run.Items["CorrelationId"]); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/DistressValidationTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/DistressValidationTests.cs index 8807e793..d1e27924 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/DistressValidationTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/DistressValidationTests.cs @@ -454,6 +454,7 @@ public class DistressValidationTests [InlineData(DistressErrorType.ConfigFetchDenied)] [InlineData(DistressErrorType.SessionRegistrationDenied)] [InlineData(DistressErrorType.TpmPssUnsupported)] + [InlineData(DistressErrorType.ClientCertificateNotReceived)] public void EnumIsDefined_AllValidErrorTypes_ReturnsTrue(DistressErrorType errorType) { Assert.True(Enum.IsDefined(typeof(DistressErrorType), errorType)); @@ -462,7 +463,7 @@ public class DistressValidationTests [Theory] [InlineData(99)] [InlineData(-1)] - [InlineData(9)] + [InlineData(10)] [InlineData(999)] public void EnumIsDefined_InvalidValues_ReturnsFalse(int rawValue) { @@ -470,11 +471,11 @@ public class DistressValidationTests } [Fact] - public void DistressErrorType_HasExactly9Values() + public void DistressErrorType_HasExactly10Values() { // If someone adds a new enum value, this test forces them to also add test coverage var values = Enum.GetValues(typeof(DistressErrorType)); - Assert.Equal(9, values.Length); + Assert.Equal(10, values.Length); } // ========================================================================= diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/MiddlewarePipelineOrderTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/MiddlewarePipelineOrderTests.cs index 2f6959db..9b9dfbd0 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/MiddlewarePipelineOrderTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/MiddlewarePipelineOrderTests.cs @@ -64,6 +64,29 @@ public class MiddlewarePipelineOrderTests Assert.Equal(1, Count(src, $"UseMiddleware<{nameof(PolicyEnforcementMiddleware)}>()")); } + [Fact] + public void Correlation_id_is_stamped_before_every_middleware_that_can_answer_early() + { + // The agent treats a 401/403 as this backend's verdict only when it echoes the X-Correlation-ID + // the agent sent (CorrelationIdMiddlewareTests). A step that answers before the stamp would + // send an error the agent cannot attribute to this backend. + var src = ProgramSource(); + + var correlation = src.IndexOf($"UseMiddleware<{nameof(CorrelationIdMiddleware)}>()", System.StringComparison.Ordinal); + Assert.True(correlation >= 0, "CorrelationIdMiddleware is not registered in Program.cs"); + Assert.Equal(1, Count(src, $"UseMiddleware<{nameof(CorrelationIdMiddleware)}>()")); + + foreach (var answering in new[] + { + nameof(GlobalExceptionMiddleware), nameof(AuthenticationMiddleware), nameof(ClientAppBindingMiddleware), + nameof(PolicyEnforcementMiddleware), nameof(UserRateLimitMiddleware), nameof(McpQuotaEnforcementMiddleware), + }) + { + var at = src.IndexOf($"UseMiddleware<{answering}>()", System.StringComparison.Ordinal); + Assert.True(at < 0 || correlation < at, $"CorrelationIdMiddleware must run before {answering}"); + } + } + private static int Count(string haystack, string needle) { var count = 0; diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs index 8ee6a790..289dc97e 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/ReportAgentErrorFunctionTests.cs @@ -342,7 +342,7 @@ public class ReportAgentErrorFunctionTests await h.RunAsync(report, Read(null)); var details = Assert.Single(h.OpsEvents).Details; - Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3", details); + Assert.Contains("\"priorRegistrationFailure\":{\"failedRuns\":3,\"interceptedRuns\":2", details); Assert.Contains("\"networkLinkUpAtEnd\":false", details); Assert.Contains("\"attemptWindowSeconds\":1204.5", details); Assert.Contains("\"configFetchOutcome\":\"UsedDefaults\"", details); @@ -360,7 +360,8 @@ public class ReportAgentErrorFunctionTests h.SessionRepo.Verify(r => r.StoreEventsBatchAsync( It.Is>(e => - ((Dictionary)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3))), Times.Once); + ((Dictionary)e[0].Data["priorRegistrationFailure"])["failedRuns"]!.Equals(3) + && ((Dictionary)e[0].Data["priorRegistrationFailure"])["interceptedRuns"]!.Equals(2))), Times.Once); } [Fact] @@ -394,6 +395,20 @@ public class ReportAgentErrorFunctionTests Assert.NotSame(raw, clean); } + // Intercepted runs are a subset of the failed runs; agents before the field send none (0). + [Theory] + [InlineData(-1, 0)] + [InlineData(0, 0)] + [InlineData(2, 2)] + [InlineData(7, 3)] + public void SanitizeRegistrationFailure_clamps_intercepted_runs_to_the_failed_runs(int interceptedRuns, int expected) + { + var raw = Failure(); + raw.InterceptedRuns = interceptedRuns; + + Assert.Equal(expected, ReportAgentErrorFunction.SanitizeRegistrationFailure(raw)!.InterceptedRuns); + } + [Theory] [InlineData(0)] [InlineData(-1)] @@ -409,6 +424,7 @@ public class ReportAgentErrorFunctionTests private static RegistrationFailureSummary Failure() => new() { FailedRuns = 3, + InterceptedRuns = 2, FirstFailedAtUtc = Now.AddDays(-20), LastFailedAtUtc = Now.AddDays(-19), Outcome = "Failed", diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs index 991c1cc9..3d9fcf88 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Ingest/ReportAgentErrorFunction.cs @@ -196,6 +196,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest ["AgentTimestamp"] = report.Timestamp.ToString("O"), ["SessionAgeHours"] = report.SessionAgeHours?.ToString("0.0", CultureInfo.InvariantCulture) ?? string.Empty, ["PriorRegistrationFailedRuns"] = priorRegistration?.FailedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty, + ["PriorRegistrationInterceptedRuns"] = priorRegistration?.InterceptedRuns.ToString(CultureInfo.InvariantCulture) ?? string.Empty, ["PriorRegistrationOutcome"] = priorRegistration?.Outcome ?? string.Empty, ["PriorRegistrationLastError"] = priorRegistration?.LastError ?? string.Empty, ["PriorRegistrationLinkUpAtStart"] = FormatNullableBool(priorRegistration?.NetworkLinkUpAtStart), @@ -486,7 +487,8 @@ namespace AutopilotMonitor.Functions.Functions.Ingest /// /// Bounds the device-supplied failed-registration record before it reaches App Insights, the /// ops feed or a timeline: strings are truncated, a record without a positive run count is - /// dropped, negative windows are clamped. Returns a copy; null in, null out. + /// dropped, negative windows are clamped, the intercepted runs are clamped to 0..FailedRuns + /// (older agents omit them: 0). Returns a copy; null in, null out. /// internal static RegistrationFailureSummary? SanitizeRegistrationFailure(RegistrationFailureSummary? source) { @@ -494,6 +496,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest return new RegistrationFailureSummary { FailedRuns = source.FailedRuns, + InterceptedRuns = Math.Clamp(source.InterceptedRuns, 0, source.FailedRuns), FirstFailedAtUtc = source.FirstFailedAtUtc, LastFailedAtUtc = source.LastFailedAtUtc, Outcome = Truncate(source.Outcome, MaxRegistrationTokenLength) ?? string.Empty, @@ -548,6 +551,7 @@ namespace AutopilotMonitor.Functions.Functions.Ingest data["priorRegistrationFailure"] = new Dictionary { ["failedRuns"] = prior.FailedRuns, + ["interceptedRuns"] = prior.InterceptedRuns, ["firstFailedAtUtc"] = prior.FirstFailedAtUtc.ToString("o"), ["lastFailedAtUtc"] = prior.LastFailedAtUtc.ToString("o"), ["outcome"] = prior.Outcome, diff --git a/src/Backend/AutopilotMonitor.Functions/Services/AgentConfigResolver.cs b/src/Backend/AutopilotMonitor.Functions/Services/AgentConfigResolver.cs index 99a7c11e..6844c017 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/AgentConfigResolver.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/AgentConfigResolver.cs @@ -36,7 +36,13 @@ namespace AutopilotMonitor.Functions.Services public class AgentConfigResolver { /// Bumped when the wire shape gains a field the agent must understand. - public const int ConfigVersion = 40; // EnableDoGroupIdAutoSet (Delivery Optimization group ID from network fingerprint) + public const int ConfigVersion = 41; // InterceptedRegistrationRetryBoots (boots after an intercepted registration) + + /// + /// Served as . A code + /// constant, no tenant or operator setting: changing it is a backend deploy. + /// + public const int DefaultInterceptedRegistrationRetryBoots = 1; /// /// The agent line whose hash oracle is served when no X-Agent-Version is available @@ -127,6 +133,7 @@ namespace AutopilotMonitor.Functions.Services EnableEspContinueAnywayObservation = tenantConfig.EnableEspContinueAnywayObservation ?? false, MaxAuthFailures = tenantConfig.MaxAuthFailures ?? 5, AuthFailureTimeoutMinutes = tenantConfig.AuthFailureTimeoutMinutes ?? 0, + InterceptedRegistrationRetryBoots = DefaultInterceptedRegistrationRetryBoots, LogLevel = tenantConfig.LogLevel ?? "Info", RebootOnComplete = tenantConfig.RebootOnComplete ?? false, RebootDelaySeconds = tenantConfig.RebootDelaySeconds ?? 10, diff --git a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs index 99e58684..ccfd5295 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs @@ -1120,6 +1120,7 @@ namespace AutopilotMonitor.Functions.Services : new { failedRuns = failure.FailedRuns, + interceptedRuns = failure.InterceptedRuns, firstFailedAtUtc = failure.FirstFailedAtUtc, lastFailedAtUtc = failure.LastFailedAtUtc, outcome = failure.Outcome, diff --git a/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts index 50bbb9e5..a1635b62 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/__tests__/session-coverage.test.ts @@ -54,6 +54,40 @@ describe('buildSessionCoverage — clean session', () => { }); describe('buildSessionCoverage — observation window', () => { + it('reports starts that failed to register before the agent finally registered', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 5, { + priorRegistrationFailedRuns: '2', + priorRegistrationInterceptedRuns: 2, + priorRegistrationLastOutcome: 'AuthFailed', + priorRegistrationNetworkChanged: 'True', + }), + ]); + expect(c.window.registrationRetries).toEqual([ + { at: at(5), failedRuns: 2, interceptedRuns: 2, lastOutcome: 'AuthFailed', networkChanged: true }, + ]); + expect(c.gaps).toEqual([ + expect.stringContaining(`Before the agent start at ${at(5)}, 2 start(s) of this session failed to register, 2 of them answered by something on the network path instead of the backend; the network differed from the failed start`), + ]); + }); + + it('keeps an unknown network flag unknown and ignores starts without failed registrations', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 5, { priorRegistrationFailedRuns: 0 }), + ev('agent_started', 50, { previousExitType: 'reboot_kill', priorRegistrationFailedRuns: 1, priorRegistrationLastOutcome: 'Failed' }), + ]); + expect(c.window.registrationRetries).toEqual([ + { at: at(50), failedRuns: 1, interceptedRuns: null, lastOutcome: 'Failed', networkChanged: null }, + ]); + const retryGap = c.gaps.find((g) => g.includes('failed to register')) ?? ''; + expect(retryGap).not.toContain('network'); + }); + + it('omits registrationRetries when no start reports failed registrations', () => { + const c = buildSessionCoverage(SESSION, [ev('agent_started', 5)]); + expect(c.window).not.toHaveProperty('registrationRetries'); + }); + it('flags a missing agent_started', () => { const c = buildSessionCoverage(SESSION, [ev('esp_phase_changed', 10)]); expect(c.window.observedFrom).toBeNull(); @@ -237,6 +271,46 @@ describe('buildSessionCoverage — collectors, upload, disk', () => { const c = buildSessionCoverage(SESSION, [ev('agent_started', 0), ev('telemetry_upload_poisoned', 1, {})]); expect(c.upload.poisonedItems).toBe(1); }); + + it('reports an interception on the network path instead of a backend rejection', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('telemetry_upload_blocked', 300, { + reason: 'client_certificate_required', + operation: 'agent/telemetry', + statusCode: 403, + pageTitle: 'Web App - Unavailable', + serverCertIssuer: 'CN=Example Inspection Intermediate CA', + firstBlockedAtUtc: at(290), + }), + ]); + expect(c.upload.blocked).toBe(true); + expect(c.upload.intercepted).toEqual({ + reason: 'client_certificate_required', + firstBlockedAtUtc: at(290), + pageTitle: 'Web App - Unavailable', + serverCertIssuer: 'CN=Example Inspection Intermediate CA', + }); + expect(c.gaps).toEqual([ + expect.stringContaining(`Telemetry uploads were intercepted from ${at(290)}: the platform answered "Client Certificate Required"`), + ]); + expect(c.gaps[0]).toContain('issued by "CN=Example Inspection Intermediate CA"'); + expect(c.gaps.join(' ')).not.toContain('permanent backend rejection'); + }); + + it('names the foreign page and keeps a backend rejection as its own gap', () => { + const c = buildSessionCoverage(SESSION, [ + ev('agent_started', 0), + ev('telemetry_upload_blocked', 100, { reason: 'permanent 4xx' }), + ev('telemetry_upload_blocked', 200, { reason: 'foreign_response', pageTitle: 'Gateway - Notification' }), + ]); + // No firstBlockedAtUtc on the event: fall back to the event's own timestamp. + expect(c.upload.intercepted).toEqual({ reason: 'foreign_response', firstBlockedAtUtc: at(200), pageTitle: 'Gateway - Notification', serverCertIssuer: null }); + expect(c.gaps).toEqual([ + expect.stringContaining('blocked by a permanent backend rejection'), + expect.stringContaining('something on the network path answered instead of the backend (page "Gateway - Notification")'), + ]); + }); }); describe('buildSessionCoverage — diagnostics package', () => { diff --git a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts index 9571e391..33bbc4ed 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts @@ -247,6 +247,8 @@ export interface AgentConfigResponse { maxAuthFailures: number; /** Maximum time in minutes the agent keeps retrying after the first auth failure. 0 = disabled (no time limit, only MaxAuthFailures applies). Default: 0. */ authFailureTimeoutMinutes: number; + /** How many further boots the agent may try after a session registration was answered by a 401/403 this backend did not author (no echoed X-Correlation-ID: a proxy or the platform answered) before it cleans up like the 48-hour emergency break. 0 = clean up at the first intercepted registration. null = an older backend that does not send the field; the agent keeps its default. The agent clamps the value to 0..5 and keeps it in its config cache, because an intercepted start gets no live config. */ + interceptedRegistrationRetryBoots?: number; /** Log verbosity level for the agent. "Info" = normal messages, "Debug" = component state/decisions, "Verbose" = per-event tracing, "Trace" = full diagnostic output. Default: "Info" */ logLevel: string; /** Whether to reboot the device after enrollment completes (and cleanup/self-destruct). Default: false */ diff --git a/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts index 9de069c7..892c72af 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/session-coverage.ts @@ -49,6 +49,8 @@ export const COVERAGE_EVENT_FIELDS = 'data.overwriteRewinds,data.overwriteBytesReprocessed,data.passMaxMs,data.passGapMaxMs,' + 'data.collector,data.reason,data.errorType,' + 'data.pendingItemCount,data.pendingBytes,data.kind,data.itemCount,data.disk_free_gb,' + + 'data.pageTitle,data.serverCertIssuer,data.firstBlockedAtUtc,' + + 'data.priorRegistrationFailedRuns,data.priorRegistrationInterceptedRuns,data.priorRegistrationLastOutcome,data.priorRegistrationNetworkChanged,' + 'data.truncated,data.includedFiles,data.includedBytes,data.skippedFiles,data.skippedByReason,data.problemsByKind,' + 'data.installingNames,data.downloadingNames,' + // Scripts block; scriptType/script_type already ride in SUMMARY_EVENT_FIELDS' own list. @@ -65,6 +67,11 @@ export interface CoverageWindow { agentStarts: number; /** Distinct `previousExitType` values reported by restarts (the first start has none). */ previousExitTypes: string[]; + /** + * Starts that registered only after earlier starts of the same session had failed to register + * (`agent_started.priorRegistration*`). Those earlier boots observed nothing. + */ + registrationRetries?: RegistrationRetry[]; lateStart: { bootToAgentStartSeconds: number | null; agentUptimeSeconds: number | null; @@ -116,10 +123,40 @@ export interface CollectorDegradation { at: string | null; } +export interface RegistrationRetry { + /** The `agent_started` that finally registered. */ + at: string | null; + failedRuns: number; + /** Failed starts whose answer did not come from our backend (platform or proxy page). */ + interceptedRuns: number | null; + lastOutcome: string | null; + /** Network fingerprint (connection type + gateway) differed from the last failed start; null = unknown. */ + networkChanged: boolean | null; +} + +/** Answer reasons of `telemetry_upload_blocked` that mean "something on the network path answered, not the backend". */ +const INTERCEPTION_REASONS: ReadonlySet = new Set(['client_certificate_required', 'foreign_response']); + +export interface UploadInterception { + /** + * `client_certificate_required`: the platform answered "Client Certificate Required" — the + * device certificate did not reach the backend. `foreign_response`: a page from something else + * on the path, e.g. a proxy's block page. + */ + reason: string; + /** First such answer, agent clock. */ + firstBlockedAtUtc: string | null; + pageTitle: string | null; + /** Issuer of the server certificate the agent saw; a proxy's own CA proves TLS inspection. */ + serverCertIssuer: string | null; +} + export interface UploadCoverage { spoolPressure: boolean; poisonedItems: number; blocked: boolean; + /** Present when the block was an interception on the network path (the event arrives only once the path cleared). */ + intercepted?: UploadInterception; backpressureEpisodes: number; diskSpaceLow: boolean; } @@ -180,6 +217,13 @@ function bool(v: unknown): boolean { return String(v ?? '').toLowerCase() === 'true'; } +/** true / false / null (absent or unparseable) — for flags where "unknown" must not read as false. */ +function triBool(v: unknown): boolean | null { + if (typeof v === 'boolean') return v; + const s = String(v ?? '').trim().toLowerCase(); + return s === 'true' ? true : s === 'false' ? false : null; +} + function countMap(v: unknown): Record | undefined { if (!v || typeof v !== 'object' || Array.isArray(v)) return undefined; const out: Record = {}; @@ -285,6 +329,27 @@ export function buildSessionCoverage( const exits = previousExitTypes.length > 0 ? ` (previous exit: ${previousExitTypes.join(', ')})` : ''; gaps.push(`The agent started ${starts.length} times${exits}: signals during the restart gaps were not observed.`); } + // A start that failed to register exits before any collector runs and leaves no event of its + // own; the start that finally registers reports the failed ones (agents after 2026-10). + const registrationRetries: RegistrationRetry[] = []; + for (const s of starts) { + const failedRuns = num(s.data?.priorRegistrationFailedRuns); + if (failedRuns === null || failedRuns <= 0) continue; + registrationRetries.push({ + at: str(s.timestamp), + failedRuns, + interceptedRuns: num(s.data?.priorRegistrationInterceptedRuns), + lastOutcome: str(s.data?.priorRegistrationLastOutcome), + networkChanged: triBool(s.data?.priorRegistrationNetworkChanged), + }); + } + for (const r of registrationRetries) { + const intercepted = (r.interceptedRuns ?? 0) > 0 ? `, ${r.interceptedRuns} of them answered by something on the network path instead of the backend` : ''; + const network = r.networkChanged === true ? '; the network differed from the failed start' : r.networkChanged === false ? '; same network as the failed start' : ''; + gaps.push( + `Before the agent start at ${r.at ?? 'an unknown time'}, ${r.failedRuns} start(s) of this session failed to register${intercepted}${network}: nothing was observed during those boots.`, + ); + } // ── IME log tracker ───────────────────────────────────────────────────────── const degradedEv = first('ime_tracker_degraded'); @@ -377,13 +442,26 @@ export function buildSessionCoverage( const spool = last('spool_pressure_detected'); const poisoned = of('telemetry_upload_poisoned'); const poisonedItems = poisoned.reduce((n, e) => n + (num(e.data?.itemCount) ?? 1), 0); - const blocked = of('telemetry_upload_blocked').length > 0; + const blockedEvents = of('telemetry_upload_blocked'); + const blocked = blockedEvents.length > 0; + const isInterception = (e: Ev): boolean => INTERCEPTION_REASONS.has(str(e.data?.reason) ?? ''); + const interceptedEv = blockedEvents.find(isInterception); + const rejectedByBackend = blockedEvents.some((e) => !isInterception(e)); + const intercepted: UploadInterception | undefined = interceptedEv + ? { + reason: str(interceptedEv.data?.reason) ?? 'foreign_response', + firstBlockedAtUtc: str(interceptedEv.data?.firstBlockedAtUtc) ?? str(interceptedEv.timestamp), + pageTitle: str(interceptedEv.data?.pageTitle), + serverCertIssuer: str(interceptedEv.data?.serverCertIssuer), + } + : undefined; const backpressureEpisodes = of('ingress_backpressure').length; const disk = last('disk_space_low'); const upload: UploadCoverage = { spoolPressure: !!spool, poisonedItems, blocked, + ...(intercepted ? { intercepted } : {}), backpressureEpisodes, diskSpaceLow: !!disk, }; @@ -395,9 +473,19 @@ export function buildSessionCoverage( const kinds = [...new Set(poisoned.map((e) => str(e.data?.kind)).filter((x): x is string => !!x))]; gaps.push(`${poisonedItems} telemetry item(s) were quarantined as poison and never uploaded${kinds.length > 0 ? ` (${kinds.join(', ')})` : ''}; they remain in the spool inside the diagnostics ZIP.`); } - if (blocked) { + if (rejectedByBackend) { gaps.push('Telemetry upload was blocked by a permanent backend rejection at some point; later events reached the backend only once the block cleared.'); } + if (intercepted) { + const issuer = intercepted.serverCertIssuer ? `; the server certificate the agent saw was issued by "${intercepted.serverCertIssuer}"` : ''; + const what = + intercepted.reason === 'client_certificate_required' + ? 'the platform answered "Client Certificate Required", so the device certificate did not reach the backend (TLS inspection or a proxy on the network path)' + : `something on the network path answered instead of the backend${intercepted.pageTitle ? ` (page "${intercepted.pageTitle}")` : ''}, typically a proxy or security gateway`; + gaps.push( + `Telemetry uploads were intercepted from ${intercepted.firstBlockedAtUtc ?? 'an unknown time'}: ${what}${issuer}. Events from that period reached the backend only after the path cleared.`, + ); + } if (disk) { const free = num(disk.data?.disk_free_gb); gaps.push(`System drive ran low on space${free !== null ? ` (${free} GB free)` : ''}: spool and diagnostics writes may have failed.`); @@ -508,6 +596,7 @@ export function buildSessionCoverage( startedAtBackdatedSeconds, agentStarts: starts.length, previousExitTypes, + ...(registrationRetries.length > 0 ? { registrationRetries } : {}), lateStart, historicReplaySuppressed, }, diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 66f45a4f..947e13e2 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -694,7 +694,7 @@ namespace AutopilotMonitor.Shared public const string CollectorDegraded = "collector_degraded"; // One-shot Warning when a collector/watcher fails to arm (e.g. EventLogWatcher start error). Review MON-D1: without it a dead kernel watcher is indistinguishable from a genuine no-signal session. public const string StateQuarantineRecovered = "state_quarantine_recovered"; // Warning emitted on the start that honours a prior run's quarantine marker (3 consecutive journal-append failures). Review TRACE-H2: without it a corruption-recovery looks like a normal restart on the backend. public const string TelemetryUploadPoisoned = "telemetry_upload_poisoned"; // Warning when the drain quarantined a single oversized item (413 even on its own — the only locally-provable poison) to keep the pipeline flowing. Review TRACE-H1/P1. - public const string TelemetryUploadBlocked = "telemetry_upload_blocked"; // Warning (one-shot) when the drain is blocked by a retained unknown permanent 4xx (400 contract bug / 404 route mismatch / tenant-validation). Data is NOT discarded. LOCAL diagnostic (agent log + diag ZIP): this marker is queued behind the blocking batch and only reaches the backend after the block clears — no out-of-band bypass by design. + public const string TelemetryUploadBlocked = "telemetry_upload_blocked"; // Warning (one-shot each) when the drain is blocked by a retained unknown permanent 4xx (400 contract bug / 404 route mismatch / tenant-validation), or by an interception on the network path (data.reason client_certificate_required | foreign_response, plus operation, statusCode, pageTitle, serverCertIssuer, firstBlockedAtUtc). Data is NOT discarded. LOCAL diagnostic (agent log + diag ZIP): this marker is queued behind the blocking batch and only reaches the backend after the block clears — no out-of-band bypass by design. public const string SessionParkedWithoutDeadline = "session_parked_without_deadline"; // One-shot Warning tripwire (liveness plan PR1): a non-terminal session entered the post-AccountSetup dead-end zone (ESP final exit or advisory failure) with NO resolution-capable deadline armed. After the 2026-06-12 dead-end fixes this must never fire in production — every occurrence is a bug report for an unknown parking variant. public const string DiskSpaceLow = "disk_space_low"; // One-shot Warning (state-change-only, 1 GB hysteresis — NOT periodic) when the system drive drops below 2 GB free. Review MON-B10: disk-full was previously only visible inside Debug performance_snapshot payloads (filtered out of timelines), so a disk-starved enrollment carried no actionable threshold signal. public const string EspPolicyProviderStalled = "esp_policy_provider_stalled"; // One-shot-per-provider Warning: a registered EnrollmentStatusTracking CSP policy provider stayed incomplete >= 15 min (Setup/Apps TrackingPoliciesCreated != 1, or DevicePreparation InstallationState not in {2,3}). The ESP waits on Setup/Apps providers WITHOUT any OS timeout — e.g. pre-provisioning + co-management leaves only "ConfigMgr" registered (no "Sidecar" = IME) and the user ESP sits at "Apps (Identifying)" for days. Observational only; payload carries the full provider list. See EspPolicyProviderProbe. diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs index cf735774..0b02f197 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Config/AgentConfigResponse.cs @@ -94,6 +94,16 @@ namespace AutopilotMonitor.Shared.Models /// public int AuthFailureTimeoutMinutes { get; set; } = 0; + /// + /// How many further boots the agent may try after a session registration was answered by + /// a 401/403 this backend did not author (no echoed X-Correlation-ID: a proxy or + /// the platform answered) before it cleans up like the 48-hour emergency break. + /// 0 = clean up at the first intercepted registration. null = an older backend that does + /// not send the field; the agent keeps its default. The agent clamps the value to 0..5 and + /// keeps it in its config cache, because an intercepted start gets no live config. + /// + public int? InterceptedRegistrationRetryBoots { get; set; } + /// /// Log verbosity level for the agent. /// "Info" = normal messages, "Debug" = component state/decisions, "Verbose" = per-event tracing, "Trace" = full diagnostic output. diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DistressReport.cs b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DistressReport.cs index 6af20b44..9faa8423 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DistressReport.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/DistressReport.cs @@ -42,6 +42,16 @@ namespace AutopilotMonitor.Shared.Models /// firmware update or device replacement, not an agent change. /// TpmPssUnsupported = 8, + + /// + /// The agent got a 401/403 this backend did not author: the response does not echo the + /// request's X-Correlation-ID. It is the platform's "Client Certificate Required" + /// page or a proxy's own page, so the device certificate never reached the backend — + /// typically TLS inspection on the network path. + /// carries the operation, the status, the page title and the issuer of the server + /// certificate the agent saw. + /// + ClientCertificateNotReceived = 9, } /// diff --git a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/RegistrationFailureSummary.cs b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/RegistrationFailureSummary.cs index 7f00fb8c..2d5e15be 100644 --- a/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/RegistrationFailureSummary.cs +++ b/src/Shared/AutopilotMonitor.Shared/Models/Diagnostics/RegistrationFailureSummary.cs @@ -14,6 +14,13 @@ namespace AutopilotMonitor.Shared.Models /// Process starts whose registration failed since the last success. public int FailedRuns { get; set; } + /// + /// Of , the starts whose registration was answered by a 401/403 that + /// something on the network path authored (TLS inspection, a proxy page) rather than the + /// backend. The agent gives up and cleans up once it exceeds the configured buffer. + /// + public int InterceptedRuns { get; set; } + /// UTC time the first of these runs gave up. public DateTime FirstFailedAtUtc { get; set; } diff --git a/src/Web/autopilot-monitor-web/app/admin/components/DistressReportsSection.tsx b/src/Web/autopilot-monitor-web/app/admin/components/DistressReportsSection.tsx index 73002650..80500c6a 100644 --- a/src/Web/autopilot-monitor-web/app/admin/components/DistressReportsSection.tsx +++ b/src/Web/autopilot-monitor-web/app/admin/components/DistressReportsSection.tsx @@ -27,6 +27,8 @@ const ERROR_TYPE_LABELS: Record = { TenantRejected: { label: "Tenant Rejected", color: "bg-purple-100 text-purple-800 dark:bg-purple-900/40 dark:text-purple-300" }, ConfigFetchDenied: { label: "Config Denied", color: "bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300" }, SessionRegistrationDenied: { label: "Session Denied", color: "bg-blue-100 text-blue-800 dark:bg-blue-900/40 dark:text-blue-300" }, + TpmPssUnsupported: { label: "TPM Incompatible", color: "bg-red-100 text-red-800 dark:bg-red-900/40 dark:text-red-300" }, + ClientCertificateNotReceived: { label: "Cert Not Received", color: "bg-orange-100 text-orange-800 dark:bg-orange-900/40 dark:text-orange-300" }, }; function ErrorTypeBadge({ errorType }: { errorType: string }) { diff --git a/src/Web/autopilot-monitor-web/app/admin/tenants/sections/tenantConfigReportCatalog.ts b/src/Web/autopilot-monitor-web/app/admin/tenants/sections/tenantConfigReportCatalog.ts index f0b30384..b4be9d46 100644 --- a/src/Web/autopilot-monitor-web/app/admin/tenants/sections/tenantConfigReportCatalog.ts +++ b/src/Web/autopilot-monitor-web/app/admin/tenants/sections/tenantConfigReportCatalog.ts @@ -271,6 +271,8 @@ export const RUNTIME_FIELDS: Record