From c26867ced861e94efcf658fe2311b22711a669cb Mon Sep 17 00:00:00 2001 From: Oliver Kieselbach Date: Mon, 24 Aug 2026 11:49:37 +0200 Subject: [PATCH] fix(agent): network-link gate before initial config fetch + post-registration recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause (edu.kosh audit 2026-08-24, session ba828038): on boot-time relaunches the agent starts before the Wi-Fi link is up. The Phase 4 config fetch had no network gate (98f8114d only guarded Phase 6 registration), burned its 10/30/60s retry budget into a dead link and stranded the whole session on built-in defaults (ConfigVersion=0) — up to ~50% of that tenant's sessions, with no self-heal short of a manual rotate_config action. Not a version regression: rates varied wildly per day within the same agent build; the failed requests never reached the backend. Two complementary layers: - RemoteConfigService gates the initial fetch on the shared NetworkLinkWait poll (60s cap — longer than registration's 15s because the fetch runs first), so the retry budget starts on a live link. rotate_config stays ungated (latency-bound). - Phase 6.5: after registration proves the backend reachable, TryRecoverConfigAsync refetches once and AgentRuntimeConfig.TryRecoverAfterRegistration re-applies the live config (same merge/integrity pipeline as Phase 4). LastFetch* keeps the original failure record; remote_config_fetch_failed reports it with a recovered flag at Info severity, agent_started flags remoteConfigRecovered. Deliberately not re-run in recovery: kill-signal check (telemetry-channel control signals cover post-registration kills) and endpoint migration (next start). Co-Authored-By: Claude Fable 5 --- .../RemoteConfigServiceRetryTests.cs | 104 ++++++++++++++++++ .../LifecycleEmittersRemoteConfigTests.cs | 71 +++++++++++- .../Configuration/RemoteConfigService.cs | 83 ++++++++++++++ .../Monitoring/Transport/NetworkLinkWait.cs | 38 +++++++ .../Security/SessionRegistrationHelper.cs | 26 +---- .../AutopilotMonitor.Agent.V2/Program.cs | 9 ++ .../Runtime/AgentRuntimeConfig.cs | 74 ++++++++++++- .../Runtime/LifecycleEmitters.cs | 25 ++++- 8 files changed, 402 insertions(+), 28 deletions(-) create mode 100644 src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkLinkWait.cs diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs index bb7a260e..ed88f05c 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs @@ -128,6 +128,94 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration Assert.Equal(4, svc.LastFetchAttempts); } + // ── Network-link gate (edu.kosh audit 2026-08-24) ─────────────────── + + [Fact] + public async Task FetchConfig_InitialFetch_WaitsForNetworkLink() + { + // The initial startup fetch must gate on the link-level signal so the retry + // budget isn't burned into a NIC that is provably still down (agent relaunched + // at boot, Wi-Fi still associating). + var svc = NewService(BackendBehaviour.SuccessImmediately); + + await svc.FetchConfigAsync(retryOnTransientErrors: true); + + Assert.Equal(1, svc.NetworkLinkWaits); + } + + [Fact] + public async Task FetchConfig_RotatePath_SkipsNetworkLinkGate() + { + // rotate_config runs mid-session on a proven link and must stay latency-bounded. + var svc = NewService(BackendBehaviour.SuccessImmediately); + + await svc.FetchConfigAsync(retryOnTransientErrors: false); + + Assert.Equal(0, svc.NetworkLinkWaits); + } + + // ── Post-registration recovery ────────────────────────────────────── + + [Fact] + public async Task TryRecover_AfterFailedFetch_AppliesLiveConfig_KeepsFailureRecord() + { + // The core self-heal contract: registration proved the backend reachable, the + // recovery swaps in the live config — but the LastFetch* observability keeps + // describing the failed startup fetch so the wire event can report both. + var svc = NewService(BackendBehaviour.FailAllThenSucceed); + await svc.FetchConfigAsync(retryOnTransientErrors: true); + Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome); + + var recovered = await svc.TryRecoverConfigAsync(); + + Assert.NotNull(recovered); + Assert.Equal(42, recovered.ConfigVersion); + Assert.Equal(42, svc.CurrentConfig.ConfigVersion); + Assert.True(svc.RecoveredAfterFetchFailure); + // Startup-failure record intact: + Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome); + Assert.Equal(1 + RemoteConfigService.InitialFetchRetryBackoff.Length, svc.LastFetchAttempts); + Assert.Equal(nameof(HttpRequestException), svc.LastFetchFailureType); + } + + [Fact] + public async Task TryRecover_AfterSuccessfulFetch_IsNoOp() + { + var svc = NewService(BackendBehaviour.SuccessImmediately); + await svc.FetchConfigAsync(retryOnTransientErrors: true); + + var recovered = await svc.TryRecoverConfigAsync(); + + Assert.Null(recovered); + Assert.False(svc.RecoveredAfterFetchFailure); + } + + [Fact] + public async Task TryRecover_BeforeAnyFetch_IsNoOp() + { + var svc = NewService(BackendBehaviour.SuccessImmediately); + + var recovered = await svc.TryRecoverConfigAsync(); + + Assert.Null(recovered); + Assert.False(svc.RecoveredAfterFetchFailure); + } + + [Fact] + public async Task TryRecover_WhenBackendStillFailing_KeepsFallback_NeverThrows() + { + var svc = NewService(BackendBehaviour.AlwaysTransientFailure); + var fallback = await svc.FetchConfigAsync(retryOnTransientErrors: true); + Assert.Equal(0, fallback.ConfigVersion); + + var recovered = await svc.TryRecoverConfigAsync(); + + Assert.Null(recovered); + Assert.False(svc.RecoveredAfterFetchFailure); + Assert.Equal(0, svc.CurrentConfig.ConfigVersion); // fallback config untouched + Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome); + } + // ── Outcome state defaults ────────────────────────────────────────── [Fact] @@ -145,6 +233,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration SuccessImmediately, AlwaysTransientFailure, FailTwiceThenSucceed, + /// All 4 initial attempts fail; the NEXT call (the recovery fetch) succeeds. + FailAllThenSucceed, AlwaysAuthFailure401, AlwaysReturnNull, } @@ -177,6 +267,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration _behaviour = behaviour; } + /// Number of times the network-link gate seam was awaited. + public int NetworkLinkWaits { get; private set; } + protected override Task CallBackendAsync() { _calls++; @@ -188,6 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration if (_calls < 3) throw new HttpRequestException("simulated cold-start timeout"); return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 }); + case BackendBehaviour.FailAllThenSucceed: + if (_calls <= 1 + RemoteConfigService.InitialFetchRetryBackoff.Length) + throw new HttpRequestException("simulated dead network link"); + return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 }); case BackendBehaviour.AlwaysTransientFailure: throw new HttpRequestException("simulated network failure"); case BackendBehaviour.AlwaysAuthFailure401: @@ -202,6 +299,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration // Zero-delay so the suite finishes in ms instead of the live 10/30/60 s schedule. protected override Task DelayBetweenAttemptsAsync(TimeSpan delay) => Task.CompletedTask; + // Record + skip the real link poll so the suite is deterministic on offline CI. + protected override Task WaitForNetworkLinkAsync() + { + NetworkLinkWaits++; + return Task.CompletedTask; + } + // In-memory + per-instance cache so a prior test's successful fetch can't // bleed into a later test that expects UsedDefaults. private AgentConfigResponse? _cached; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs index ef5d78f6..0fad1fc1 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs @@ -182,6 +182,73 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program Assert.Equal("FromCache", data["outcome"]); } + // ── Post-registration recovery (Phase 6.5, edu.kosh audit 2026-08-24) ─ + + [Fact] + public void EmitFetchFailedIfAny_AfterRecovery_MarksRecoveredAndDemotesToInfo() + { + using var rig = new EmitterRig(); + // Startup fetch failed (network link not up yet), then the post-registration + // recovery fetched ConfigVersion 39. The event must keep the original failure + // details AND mark the heal. + rig.RemoteConfig.SetFakeOutcome( + RemoteConfigFetchOutcome.UsedDefaults, + configVersion: 39, // CurrentConfig now holds the recovered config + attempts: 4, + failureType: "HttpRequestException", + failureMessage: "simulated dead network link", + recovered: true); + + LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny( + rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger); + + var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed"); + Assert.Equal("UsedDefaults", data["outcome"]); // startup failure record intact + Assert.Equal(4, data["attempts"]); + Assert.Equal(true, data["recovered"]); + Assert.Equal(39, data["recoveredConfigVersion"]); + Assert.Equal(nameof(EventSeverity.Info), rig.Sink.Posted[0].Payload!["severity"]); + } + + [Fact] + public void EmitFetchFailedIfAny_WithoutRecovery_StaysWarning() + { + using var rig = new EmitterRig(); + rig.RemoteConfig.SetFakeOutcome( + RemoteConfigFetchOutcome.UsedDefaults, + configVersion: 0, + attempts: 4, + failureType: "HttpRequestException", + failureMessage: "still down"); + + LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny( + rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger); + + var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed"); + Assert.False(data.ContainsKey("recovered")); + Assert.False(data.ContainsKey("recoveredConfigVersion")); + Assert.Equal(nameof(EventSeverity.Warning), rig.Sink.Posted[0].Payload!["severity"]); + } + + [Fact] + public void EmitAgentStarted_AfterRecovery_FlagsLiveConfig() + { + using var rig = new EmitterRig(); + rig.RemoteConfig.SetFakeOutcome( + RemoteConfigFetchOutcome.UsedDefaults, + configVersion: 39, + recovered: true); + + LifecycleEmitters.EmitAgentStarted( + rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger); + + var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started"); + Assert.Equal(39, data["configVersion"]); + Assert.Equal(true, data["remoteConfigFetched"]); // running on a live-fetched config + Assert.Equal("UsedDefaults", data["remoteConfigOutcome"]); // startup outcome stays honest + Assert.Equal(true, data["remoteConfigRecovered"]); + } + // ── Helpers ───────────────────────────────────────────────────────── private static Dictionary AssertSingleEmit( @@ -258,8 +325,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program int attempts = 1, string? failureType = null, string? failureMessage = null, - int? authStatusCode = null) + int? authStatusCode = null, + bool recovered = false) { + typeof(RemoteConfigService).GetProperty(nameof(RecoveredAfterFetchFailure))?.SetValue(this, recovered); // CurrentConfig is set via the protected SetConfig path — easiest: // a real FetchConfigAsync replacement that just stashes a canned config. typeof(RemoteConfigService).GetMethod("SetConfig", diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs index 00fc1334..6bf29db0 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs @@ -42,6 +42,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration TimeSpan.FromSeconds(60), }; + /// + /// Upper bound for the pre-fetch network-link wait on the initial startup fetch. + /// Deliberately longer than the registration gate's 15s: the fetch runs FIRST after a + /// boot-time relaunch, when Wi-Fi may still be associating (802.1X, OOBE), and a fetch + /// burned into a dead link strands the whole session on built-in defaults — edu.kosh + /// audit 2026-08-24: ~50% of sessions lost their tenant config exactly this way. + /// + internal static readonly TimeSpan InitialFetchNetworkLinkWaitMax = TimeSpan.FromSeconds(60); + /// /// Gets the current configuration (thread-safe) /// @@ -76,6 +85,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration /// HTTP status for auth failures (401/403), null otherwise. public int? LastFetchAuthStatusCode { get; private set; } + /// + /// True once replaced a failed startup fetch with a + /// live config. The LastFetch* properties intentionally keep describing the failed + /// startup fetch so the wire event can report both the original failure and the recovery. + /// + public bool RecoveredAfterFetchFailure { get; private set; } + public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null) { _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); @@ -116,6 +132,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration LastFetchFailureMessage = null; LastFetchAuthStatusCode = null; + // Initial startup fetch only: give a still-associating NIC a chance to come up + // before the first attempt. Without this the retry budget (10/30/60s) is burned + // into a provably dead link — the agent starts at boot, Wi-Fi associates minutes + // later, and the session runs on defaults although the backend was never the + // problem. The rotate_config path (retryOnTransientErrors:false) runs mid-session + // on a proven link and must stay latency-bounded, so it skips the gate. + if (retryOnTransientErrors) + { + try { await WaitForNetworkLinkAsync().ConfigureAwait(false); } + catch (Exception ex) { _logger.Debug($"Pre-fetch network wait failed ({ex.Message}) — fetching anyway."); } + } + var maxAttempts = retryOnTransientErrors ? InitialFetchRetryBackoff.Length + 1 : 1; Exception lastException = null; @@ -211,6 +239,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration return defaults; } + /// + /// Single-attempt recovery fetch for a session whose startup fetch failed. Called right + /// after session registration succeeded — registration proves the backend is reachable, + /// so the config that Phase 4 could not get is one cheap call away instead of the whole + /// session being stranded on cache/defaults (only a manual rotate_config action + /// could rescue it before this existed). + /// + /// Unlike this does NOT reset the LastFetch* + /// observability state — those keep describing the failed startup fetch so + /// remote_config_fetch_failed can report the original failure AND the recovery. + /// Returns the recovered config, or null when there is nothing to recover (startup fetch + /// succeeded / never ran) or the recovery attempt itself failed. Never throws. + /// + /// + public async Task TryRecoverConfigAsync() + { + if (LastFetchOutcome != RemoteConfigFetchOutcome.FromCache && + LastFetchOutcome != RemoteConfigFetchOutcome.UsedDefaults) + return null; + + try + { + _logger.Info("Retrying remote config fetch after successful session registration..."); + var config = await CallBackendAsync().ConfigureAwait(false); + if (config == null) + { + _logger.Warning("Config recovery fetch returned empty response — keeping fallback config."); + return null; + } + + _logger.Info($"Remote config recovered: ConfigVersion={config.ConfigVersion}, " + + $"GatherRules={config.GatherRules?.Count ?? 0}"); + LogCollectorSettings(config.Collectors); + SetConfig(config); + CacheConfig(config); + _authFailureTracker?.RecordSuccess(); + RecoveredAfterFetchFailure = true; + return config; + } + catch (Exception ex) + { + _logger.Warning($"Config recovery fetch failed ({ex.Message}) — keeping fallback config."); + return null; + } + } + private void SetConfig(AgentConfigResponse config) { lock (_configLock) @@ -358,6 +432,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration protected virtual Task DelayBetweenAttemptsAsync(TimeSpan delay) => Task.Delay(delay); + /// + /// Awaits the pre-fetch network-link gate (initial startup fetch only). Production + /// polls the link-level signal via bounded by + /// ; tests subclass to record the call + /// and return immediately. + /// + protected virtual Task WaitForNetworkLinkAsync() + => NetworkLinkWait.WaitAsync(_logger, InitialFetchNetworkLinkWaitMax, "Remote config fetch"); + public void Dispose() { // Nothing to dispose diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkLinkWait.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkLinkWait.cs new file mode 100644 index 00000000..f553378c --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/NetworkLinkWait.cs @@ -0,0 +1,38 @@ +using System; +using System.Net.NetworkInformation; +using System.Threading.Tasks; +using AutopilotMonitor.Agent.V2.Core.Logging; + +namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport +{ + /// + /// Boot-time NIC grace shared by every first-contact backend call (remote-config fetch, + /// session registration, emergency-break reporter pattern): polls the cheap link-level + /// signal once per second for at most . It cannot prove backend + /// reachability — the callers' retry loops handle that — it only keeps retry budgets from + /// being burned into a link that is provably still down (BootTrigger relaunch after a + /// mid-enrollment reboot, Wi-Fi still associating; tenant aebdce78 audits 2026-08-23/24). + /// Free on the normal path — a live link returns immediately. Probe errors end the wait, + /// never the caller's operation. + /// + internal static class NetworkLinkWait + { + internal static async Task WaitAsync(AgentLogger logger, TimeSpan maxWait, string context) + { + if (NetworkInterface.GetIsNetworkAvailable()) return; + + logger?.Info($"{context}: no network link yet — waiting up to {maxWait.TotalSeconds:F0}s."); + var deadline = DateTime.UtcNow + maxWait; + while (DateTime.UtcNow < deadline) + { + await Task.Delay(1000).ConfigureAwait(false); + if (NetworkInterface.GetIsNetworkAvailable()) + { + logger?.Info($"{context}: network link is up."); + return; + } + } + logger?.Warning($"{context}: still no network link after {maxWait.TotalSeconds:F0}s — attempting anyway."); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs index ed8065a0..dbc0c971 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Security/SessionRegistrationHelper.cs @@ -1,5 +1,4 @@ using System; -using System.Net.NetworkInformation; using System.Threading.Tasks; using AutopilotMonitor.Agent.V2.Core.Configuration; using AutopilotMonitor.Agent.V2.Core.Logging; @@ -165,29 +164,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Security => Task.Delay(((int)Math.Pow(2, attempt)) * 1000); /// - /// Boot-time NIC grace (same pattern as the emergency-break reporter): polls the cheap - /// link-level signal once per second for at most . It - /// cannot prove backend reachability — the retry loop handles that — it only keeps the - /// first attempts from being burned into a link that is provably still down. Probe + /// Boot-time NIC grace before the first registration attempt — shared polling loop in + /// , bounded by . Probe /// errors end the wait, never the registration. /// - internal static async Task WaitForNetworkLinkAsync(AgentLogger logger) - { - if (NetworkInterface.GetIsNetworkAvailable()) return; - - logger?.Info($"Session registration: no network link yet — waiting up to {NetworkLinkWaitMax.TotalSeconds:F0}s."); - var deadline = DateTime.UtcNow + NetworkLinkWaitMax; - while (DateTime.UtcNow < deadline) - { - await Task.Delay(1000).ConfigureAwait(false); - if (NetworkInterface.GetIsNetworkAvailable()) - { - logger?.Info("Session registration: network link is up."); - return; - } - } - logger?.Warning($"Session registration: still no network link after {NetworkLinkWaitMax.TotalSeconds:F0}s — attempting anyway."); - } + internal static Task WaitForNetworkLinkAsync(AgentLogger logger) + => NetworkLinkWait.WaitAsync(logger, NetworkLinkWaitMax, "Session registration"); private static SessionRegistration BuildRegistration( AgentConfiguration agentConfig, diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs index d4127782..448bf310 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Program.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Program.cs @@ -274,6 +274,15 @@ namespace AutopilotMonitor.Agent.V2 { return registration.ExitCode; } + + // Phase 6.5 (config self-heal): when the Phase 4 fetch ran before the network link + // was up, registration has just proven the backend reachable — retry the fetch once + // so the session runs on the live tenant config instead of cache/defaults. + // See AgentRuntimeConfig.TryRecoverAfterRegistration for what is deliberately + // NOT re-run here (kill-signal check, endpoint migration). + Runtime.AgentRuntimeConfig.TryRecoverAfterRegistration( + bootstrap.AgentConfig, auth, runtimeConfig, GetAgentVersion(), consoleMode, logger); + // Phase 7+8 (orchestrator + shutdown lifecycle + onIngressReady wiring + // initial signal posts + analyzer/probes startup + shutdown.Wait + finally) // is encapsulated in AgentRuntimeHost. See Runtime/AgentRuntimeHost.cs. diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs index 59badec9..c39d86b1 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs @@ -69,6 +69,63 @@ namespace AutopilotMonitor.Agent.V2.Runtime } } + var configMergeResult = ApplyFetchedConfig(agentConfig, remoteConfig, auth, agentVersion, consoleMode, logger); + + return new RuntimeConfigBundle(remoteConfigService, remoteConfig, configMergeResult, pendingMigrationTarget: null); + } + + /// + /// Phase 6.5 — config self-heal. When the Phase 4 fetch ran before the network link was + /// up (agent relaunched at boot, Wi-Fi still associating — edu.kosh audit 2026-08-24: + /// ~50% of sessions ran on ConfigVersion=0 exactly this way), a successful session + /// registration has just proven the backend reachable. Retry the fetch once and apply + /// the live tenant config instead of stranding the whole session on cache/defaults. + /// + /// Deliberately NOT re-run here: the kill-signal check (post-registration kills arrive + /// via the telemetry-channel control signals on every upload response) and endpoint + /// migration (registration already succeeded against the current base URL; a migration + /// signal is honoured at the next agent start). Never throws — recovery is strictly + /// best-effort on top of an already-degraded-but-working fallback. + /// + /// + public static void TryRecoverAfterRegistration( + AgentConfiguration agentConfig, + BackendAuthBundle auth, + RuntimeConfigBundle bundle, + string agentVersion, + bool consoleMode, + AgentLogger logger) + { + try + { + var service = bundle?.RemoteConfigService; + if (service == null) return; + + var recovered = service.TryRecoverConfigAsync().GetAwaiter().GetResult(); + if (recovered == null) return; + + var mergeResult = ApplyFetchedConfig(agentConfig, recovered, auth, agentVersion, consoleMode, logger); + bundle.ApplyRecovered(recovered, mergeResult); + } + catch (Exception ex) + { + logger.Warning($"Post-registration config recovery failed: {ex.Message}"); + } + } + + /// + /// Applies a live-fetched config to the runtime: merge, tracker ceilings, log level, + /// integrity SHA + check, bootstrap-config cleanup. Shared by the Phase 4 startup path + /// and the post-registration recovery path so both apply identical semantics. + /// + private static RemoteConfigMergeResult ApplyFetchedConfig( + AgentConfiguration agentConfig, + AgentConfigResponse remoteConfig, + BackendAuthBundle auth, + string agentVersion, + bool consoleMode, + AgentLogger logger) + { // Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that // downstream consumers (CleanupService, SummaryDialogLauncher, StartupEnvironmentProbes, // DiagnosticsPackageService, EnrollmentTerminationHandler, the logger, the watchdog) @@ -91,7 +148,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime TryCleanupBootstrapConfig(agentConfig, agentVersion, logger); - return new RuntimeConfigBundle(remoteConfigService, remoteConfig, configMergeResult, pendingMigrationTarget: null); + return configMergeResult; } /// @@ -175,10 +232,21 @@ namespace AutopilotMonitor.Agent.V2.Runtime internal sealed class RuntimeConfigBundle { public RemoteConfigService RemoteConfigService { get; } - public AgentConfigResponse RemoteConfig { get; } - public RemoteConfigMergeResult MergeResult { get; } + public AgentConfigResponse RemoteConfig { get; private set; } + public RemoteConfigMergeResult MergeResult { get; private set; } public string PendingMigrationTarget { get; } + /// + /// Swaps in the config recovered by + /// so Phase 7+8 (orchestrator, collectors, gather rules, IME patterns, the + /// unrestricted-mode audit) run on the live tenant config instead of the fallback. + /// + public void ApplyRecovered(AgentConfigResponse recoveredConfig, RemoteConfigMergeResult mergeResult) + { + RemoteConfig = recoveredConfig; + MergeResult = mergeResult; + } + public RuntimeConfigBundle( RemoteConfigService remoteConfigService, AgentConfigResponse remoteConfig, diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs index 021235d5..72aa1582 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs @@ -52,6 +52,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime // prior to this, ConfigVersion=0 was only deducible by inferring from // the absence of tenant-controlled knobs in the DataJson. var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted; + var recovered = remoteConfigService?.RecoveredAfterFetchFailure == true; var configVersion = remoteConfigService?.CurrentConfig?.ConfigVersion ?? 0; var data = new Dictionary @@ -66,8 +67,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime { "diagnosticsUploadMode", agentConfig.DiagnosticsUploadMode ?? "Off" }, { "previousExitType", previousExit?.ExitType ?? "unknown" }, { "unrestrictedMode", agentConfig.UnrestrictedMode }, + // remoteConfigFetched answers "is the agent running on a live-fetched + // config" — true also when the post-registration recovery replaced a + // failed startup fetch (remoteConfigOutcome then still names the startup + // failure; remoteConfigRecovered marks the heal). { "configVersion", configVersion }, - { "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded }, + { "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded || recovered }, { "remoteConfigOutcome", outcome.ToString() }, // WinRT OOBE state sampled at this start. Together with previousExitType // this makes late starts provable: "completed" on a first_run means OOBE @@ -75,6 +80,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime { "oobeStateAtAgentStart", OobeStateReader.Read() }, }; + if (recovered) + data["remoteConfigRecovered"] = true; + if (!string.IsNullOrEmpty(previousExit?.CrashExceptionType)) data["previousCrashException"] = previousExit.CrashExceptionType; @@ -135,12 +143,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime ? $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); using cached config." : $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); running with built-in defaults (ConfigVersion=0)."; + // Post-registration recovery (Phase 6.5) replaced the fallback with a live + // fetch before this emitter ran: keep the failure details (the network-late + // condition is worth surfacing) but mark the heal and demote to Info — the + // session is NOT degraded. + var recovered = remoteConfigService.RecoveredAfterFetchFailure; + if (recovered) + { + var recoveredVersion = remoteConfigService.CurrentConfig?.ConfigVersion ?? 0; + data["recovered"] = true; + data["recoveredConfigVersion"] = recoveredVersion; + msg += $" Recovered after session registration (ConfigVersion={recoveredVersion})."; + } + post.Emit(new EnrollmentEvent { SessionId = agentConfig.SessionId, TenantId = agentConfig.TenantId, EventType = SharedConstants.EventTypes.RemoteConfigFetchFailed, - Severity = EventSeverity.Warning, + Severity = recovered ? EventSeverity.Info : EventSeverity.Warning, Source = "Agent", Phase = EnrollmentPhase.Unknown, Message = msg,