mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 20:53:06 +00:00
fix(agent): network-link gate before initial config fetch + post-registration recovery
Root cause (edu.kosh audit 2026-08-24, session ba828038): on boot-time relaunches
the agent starts before the Wi-Fi link is up. The Phase 4 config fetch had no
network gate (98f8114d only guarded Phase 6 registration), burned its 10/30/60s
retry budget into a dead link and stranded the whole session on built-in defaults
(ConfigVersion=0) — up to ~50% of that tenant's sessions, with no self-heal short
of a manual rotate_config action. Not a version regression: rates varied wildly
per day within the same agent build; the failed requests never reached the backend.
Two complementary layers:
- RemoteConfigService gates the initial fetch on the shared NetworkLinkWait poll
(60s cap — longer than registration's 15s because the fetch runs first), so the
retry budget starts on a live link. rotate_config stays ungated (latency-bound).
- Phase 6.5: after registration proves the backend reachable, TryRecoverConfigAsync
refetches once and AgentRuntimeConfig.TryRecoverAfterRegistration re-applies the
live config (same merge/integrity pipeline as Phase 4). LastFetch* keeps the
original failure record; remote_config_fetch_failed reports it with a recovered
flag at Info severity, agent_started flags remoteConfigRecovered.
Deliberately not re-run in recovery: kill-signal check (telemetry-channel control
signals cover post-registration kills) and endpoint migration (next start).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+104
@@ -128,6 +128,94 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
Assert.Equal(4, svc.LastFetchAttempts);
|
||||
}
|
||||
|
||||
// ── Network-link gate (edu.kosh audit 2026-08-24) ───────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_InitialFetch_WaitsForNetworkLink()
|
||||
{
|
||||
// The initial startup fetch must gate on the link-level signal so the retry
|
||||
// budget isn't burned into a NIC that is provably still down (agent relaunched
|
||||
// at boot, Wi-Fi still associating).
|
||||
var svc = NewService(BackendBehaviour.SuccessImmediately);
|
||||
|
||||
await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
|
||||
Assert.Equal(1, svc.NetworkLinkWaits);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchConfig_RotatePath_SkipsNetworkLinkGate()
|
||||
{
|
||||
// rotate_config runs mid-session on a proven link and must stay latency-bounded.
|
||||
var svc = NewService(BackendBehaviour.SuccessImmediately);
|
||||
|
||||
await svc.FetchConfigAsync(retryOnTransientErrors: false);
|
||||
|
||||
Assert.Equal(0, svc.NetworkLinkWaits);
|
||||
}
|
||||
|
||||
// ── Post-registration recovery ──────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task TryRecover_AfterFailedFetch_AppliesLiveConfig_KeepsFailureRecord()
|
||||
{
|
||||
// The core self-heal contract: registration proved the backend reachable, the
|
||||
// recovery swaps in the live config — but the LastFetch* observability keeps
|
||||
// describing the failed startup fetch so the wire event can report both.
|
||||
var svc = NewService(BackendBehaviour.FailAllThenSucceed);
|
||||
await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
|
||||
|
||||
var recovered = await svc.TryRecoverConfigAsync();
|
||||
|
||||
Assert.NotNull(recovered);
|
||||
Assert.Equal(42, recovered.ConfigVersion);
|
||||
Assert.Equal(42, svc.CurrentConfig.ConfigVersion);
|
||||
Assert.True(svc.RecoveredAfterFetchFailure);
|
||||
// Startup-failure record intact:
|
||||
Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
|
||||
Assert.Equal(1 + RemoteConfigService.InitialFetchRetryBackoff.Length, svc.LastFetchAttempts);
|
||||
Assert.Equal(nameof(HttpRequestException), svc.LastFetchFailureType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryRecover_AfterSuccessfulFetch_IsNoOp()
|
||||
{
|
||||
var svc = NewService(BackendBehaviour.SuccessImmediately);
|
||||
await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
|
||||
var recovered = await svc.TryRecoverConfigAsync();
|
||||
|
||||
Assert.Null(recovered);
|
||||
Assert.False(svc.RecoveredAfterFetchFailure);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryRecover_BeforeAnyFetch_IsNoOp()
|
||||
{
|
||||
var svc = NewService(BackendBehaviour.SuccessImmediately);
|
||||
|
||||
var recovered = await svc.TryRecoverConfigAsync();
|
||||
|
||||
Assert.Null(recovered);
|
||||
Assert.False(svc.RecoveredAfterFetchFailure);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TryRecover_WhenBackendStillFailing_KeepsFallback_NeverThrows()
|
||||
{
|
||||
var svc = NewService(BackendBehaviour.AlwaysTransientFailure);
|
||||
var fallback = await svc.FetchConfigAsync(retryOnTransientErrors: true);
|
||||
Assert.Equal(0, fallback.ConfigVersion);
|
||||
|
||||
var recovered = await svc.TryRecoverConfigAsync();
|
||||
|
||||
Assert.Null(recovered);
|
||||
Assert.False(svc.RecoveredAfterFetchFailure);
|
||||
Assert.Equal(0, svc.CurrentConfig.ConfigVersion); // fallback config untouched
|
||||
Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
|
||||
}
|
||||
|
||||
// ── Outcome state defaults ──────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
@@ -145,6 +233,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
SuccessImmediately,
|
||||
AlwaysTransientFailure,
|
||||
FailTwiceThenSucceed,
|
||||
/// <summary>All 4 initial attempts fail; the NEXT call (the recovery fetch) succeeds.</summary>
|
||||
FailAllThenSucceed,
|
||||
AlwaysAuthFailure401,
|
||||
AlwaysReturnNull,
|
||||
}
|
||||
@@ -177,6 +267,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
_behaviour = behaviour;
|
||||
}
|
||||
|
||||
/// <summary>Number of times the network-link gate seam was awaited.</summary>
|
||||
public int NetworkLinkWaits { get; private set; }
|
||||
|
||||
protected override Task<AgentConfigResponse> CallBackendAsync()
|
||||
{
|
||||
_calls++;
|
||||
@@ -188,6 +281,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
if (_calls < 3)
|
||||
throw new HttpRequestException("simulated cold-start timeout");
|
||||
return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
|
||||
case BackendBehaviour.FailAllThenSucceed:
|
||||
if (_calls <= 1 + RemoteConfigService.InitialFetchRetryBackoff.Length)
|
||||
throw new HttpRequestException("simulated dead network link");
|
||||
return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
|
||||
case BackendBehaviour.AlwaysTransientFailure:
|
||||
throw new HttpRequestException("simulated network failure");
|
||||
case BackendBehaviour.AlwaysAuthFailure401:
|
||||
@@ -202,6 +299,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
|
||||
// Zero-delay so the suite finishes in ms instead of the live 10/30/60 s schedule.
|
||||
protected override Task DelayBetweenAttemptsAsync(TimeSpan delay) => Task.CompletedTask;
|
||||
|
||||
// Record + skip the real link poll so the suite is deterministic on offline CI.
|
||||
protected override Task WaitForNetworkLinkAsync()
|
||||
{
|
||||
NetworkLinkWaits++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
// In-memory + per-instance cache so a prior test's successful fetch can't
|
||||
// bleed into a later test that expects UsedDefaults.
|
||||
private AgentConfigResponse? _cached;
|
||||
|
||||
+70
-1
@@ -182,6 +182,73 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
Assert.Equal("FromCache", data["outcome"]);
|
||||
}
|
||||
|
||||
// ── Post-registration recovery (Phase 6.5, edu.kosh audit 2026-08-24) ─
|
||||
|
||||
[Fact]
|
||||
public void EmitFetchFailedIfAny_AfterRecovery_MarksRecoveredAndDemotesToInfo()
|
||||
{
|
||||
using var rig = new EmitterRig();
|
||||
// Startup fetch failed (network link not up yet), then the post-registration
|
||||
// recovery fetched ConfigVersion 39. The event must keep the original failure
|
||||
// details AND mark the heal.
|
||||
rig.RemoteConfig.SetFakeOutcome(
|
||||
RemoteConfigFetchOutcome.UsedDefaults,
|
||||
configVersion: 39, // CurrentConfig now holds the recovered config
|
||||
attempts: 4,
|
||||
failureType: "HttpRequestException",
|
||||
failureMessage: "simulated dead network link",
|
||||
recovered: true);
|
||||
|
||||
LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
|
||||
rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
|
||||
|
||||
var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
|
||||
Assert.Equal("UsedDefaults", data["outcome"]); // startup failure record intact
|
||||
Assert.Equal(4, data["attempts"]);
|
||||
Assert.Equal(true, data["recovered"]);
|
||||
Assert.Equal(39, data["recoveredConfigVersion"]);
|
||||
Assert.Equal(nameof(EventSeverity.Info), rig.Sink.Posted[0].Payload!["severity"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EmitFetchFailedIfAny_WithoutRecovery_StaysWarning()
|
||||
{
|
||||
using var rig = new EmitterRig();
|
||||
rig.RemoteConfig.SetFakeOutcome(
|
||||
RemoteConfigFetchOutcome.UsedDefaults,
|
||||
configVersion: 0,
|
||||
attempts: 4,
|
||||
failureType: "HttpRequestException",
|
||||
failureMessage: "still down");
|
||||
|
||||
LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
|
||||
rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
|
||||
|
||||
var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
|
||||
Assert.False(data.ContainsKey("recovered"));
|
||||
Assert.False(data.ContainsKey("recoveredConfigVersion"));
|
||||
Assert.Equal(nameof(EventSeverity.Warning), rig.Sink.Posted[0].Payload!["severity"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EmitAgentStarted_AfterRecovery_FlagsLiveConfig()
|
||||
{
|
||||
using var rig = new EmitterRig();
|
||||
rig.RemoteConfig.SetFakeOutcome(
|
||||
RemoteConfigFetchOutcome.UsedDefaults,
|
||||
configVersion: 39,
|
||||
recovered: true);
|
||||
|
||||
LifecycleEmitters.EmitAgentStarted(
|
||||
rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
|
||||
|
||||
var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
|
||||
Assert.Equal(39, data["configVersion"]);
|
||||
Assert.Equal(true, data["remoteConfigFetched"]); // running on a live-fetched config
|
||||
Assert.Equal("UsedDefaults", data["remoteConfigOutcome"]); // startup outcome stays honest
|
||||
Assert.Equal(true, data["remoteConfigRecovered"]);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
private static Dictionary<string, object> AssertSingleEmit(
|
||||
@@ -258,8 +325,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
int attempts = 1,
|
||||
string? failureType = null,
|
||||
string? failureMessage = null,
|
||||
int? authStatusCode = null)
|
||||
int? authStatusCode = null,
|
||||
bool recovered = false)
|
||||
{
|
||||
typeof(RemoteConfigService).GetProperty(nameof(RecoveredAfterFetchFailure))?.SetValue(this, recovered);
|
||||
// CurrentConfig is set via the protected SetConfig path — easiest:
|
||||
// a real FetchConfigAsync replacement that just stashes a canned config.
|
||||
typeof(RemoteConfigService).GetMethod("SetConfig",
|
||||
|
||||
@@ -42,6 +42,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
TimeSpan.FromSeconds(60),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Upper bound for the pre-fetch network-link wait on the initial startup fetch.
|
||||
/// Deliberately longer than the registration gate's 15s: the fetch runs FIRST after a
|
||||
/// boot-time relaunch, when Wi-Fi may still be associating (802.1X, OOBE), and a fetch
|
||||
/// burned into a dead link strands the whole session on built-in defaults — edu.kosh
|
||||
/// audit 2026-08-24: ~50% of sessions lost their tenant config exactly this way.
|
||||
/// </summary>
|
||||
internal static readonly TimeSpan InitialFetchNetworkLinkWaitMax = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the current configuration (thread-safe)
|
||||
/// </summary>
|
||||
@@ -76,6 +85,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// <summary>HTTP status for auth failures (401/403), null otherwise.</summary>
|
||||
public int? LastFetchAuthStatusCode { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// True once <see cref="TryRecoverConfigAsync"/> replaced a failed startup fetch with a
|
||||
/// live config. The <c>LastFetch*</c> properties intentionally keep describing the failed
|
||||
/// startup fetch so the wire event can report both the original failure and the recovery.
|
||||
/// </summary>
|
||||
public bool RecoveredAfterFetchFailure { get; private set; }
|
||||
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
{
|
||||
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
|
||||
@@ -116,6 +132,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
LastFetchFailureMessage = null;
|
||||
LastFetchAuthStatusCode = null;
|
||||
|
||||
// Initial startup fetch only: give a still-associating NIC a chance to come up
|
||||
// before the first attempt. Without this the retry budget (10/30/60s) is burned
|
||||
// into a provably dead link — the agent starts at boot, Wi-Fi associates minutes
|
||||
// later, and the session runs on defaults although the backend was never the
|
||||
// problem. The rotate_config path (retryOnTransientErrors:false) runs mid-session
|
||||
// on a proven link and must stay latency-bounded, so it skips the gate.
|
||||
if (retryOnTransientErrors)
|
||||
{
|
||||
try { await WaitForNetworkLinkAsync().ConfigureAwait(false); }
|
||||
catch (Exception ex) { _logger.Debug($"Pre-fetch network wait failed ({ex.Message}) — fetching anyway."); }
|
||||
}
|
||||
|
||||
var maxAttempts = retryOnTransientErrors ? InitialFetchRetryBackoff.Length + 1 : 1;
|
||||
Exception lastException = null;
|
||||
|
||||
@@ -211,6 +239,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
return defaults;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single-attempt recovery fetch for a session whose startup fetch failed. Called right
|
||||
/// after session registration succeeded — registration proves the backend is reachable,
|
||||
/// so the config that Phase 4 could not get is one cheap call away instead of the whole
|
||||
/// session being stranded on cache/defaults (only a manual <c>rotate_config</c> action
|
||||
/// could rescue it before this existed).
|
||||
/// <para>
|
||||
/// Unlike <see cref="FetchConfigAsync"/> this does NOT reset the <c>LastFetch*</c>
|
||||
/// observability state — those keep describing the failed startup fetch so
|
||||
/// <c>remote_config_fetch_failed</c> can report the original failure AND the recovery.
|
||||
/// Returns the recovered config, or null when there is nothing to recover (startup fetch
|
||||
/// succeeded / never ran) or the recovery attempt itself failed. Never throws.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public async Task<AgentConfigResponse> TryRecoverConfigAsync()
|
||||
{
|
||||
if (LastFetchOutcome != RemoteConfigFetchOutcome.FromCache &&
|
||||
LastFetchOutcome != RemoteConfigFetchOutcome.UsedDefaults)
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
_logger.Info("Retrying remote config fetch after successful session registration...");
|
||||
var config = await CallBackendAsync().ConfigureAwait(false);
|
||||
if (config == null)
|
||||
{
|
||||
_logger.Warning("Config recovery fetch returned empty response — keeping fallback config.");
|
||||
return null;
|
||||
}
|
||||
|
||||
_logger.Info($"Remote config recovered: ConfigVersion={config.ConfigVersion}, " +
|
||||
$"GatherRules={config.GatherRules?.Count ?? 0}");
|
||||
LogCollectorSettings(config.Collectors);
|
||||
SetConfig(config);
|
||||
CacheConfig(config);
|
||||
_authFailureTracker?.RecordSuccess();
|
||||
RecoveredAfterFetchFailure = true;
|
||||
return config;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"Config recovery fetch failed ({ex.Message}) — keeping fallback config.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private void SetConfig(AgentConfigResponse config)
|
||||
{
|
||||
lock (_configLock)
|
||||
@@ -358,6 +432,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
protected virtual Task DelayBetweenAttemptsAsync(TimeSpan delay)
|
||||
=> Task.Delay(delay);
|
||||
|
||||
/// <summary>
|
||||
/// Awaits the pre-fetch network-link gate (initial startup fetch only). Production
|
||||
/// polls the link-level signal via <see cref="NetworkLinkWait"/> bounded by
|
||||
/// <see cref="InitialFetchNetworkLinkWaitMax"/>; tests subclass to record the call
|
||||
/// and return immediately.
|
||||
/// </summary>
|
||||
protected virtual Task WaitForNetworkLinkAsync()
|
||||
=> NetworkLinkWait.WaitAsync(_logger, InitialFetchNetworkLinkWaitMax, "Remote config fetch");
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
// Nothing to dispose
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
using System;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace shared by every first-contact backend call (remote-config fetch,
|
||||
/// session registration, emergency-break reporter pattern): polls the cheap link-level
|
||||
/// signal once per second for at most <paramref name="maxWait"/>. It cannot prove backend
|
||||
/// reachability — the callers' retry loops handle that — it only keeps retry budgets from
|
||||
/// being burned into a link that is provably still down (BootTrigger relaunch after a
|
||||
/// mid-enrollment reboot, Wi-Fi still associating; tenant aebdce78 audits 2026-08-23/24).
|
||||
/// Free on the normal path — a live link returns immediately. Probe errors end the wait,
|
||||
/// never the caller's operation.
|
||||
/// </summary>
|
||||
internal static class NetworkLinkWait
|
||||
{
|
||||
internal static async Task WaitAsync(AgentLogger logger, TimeSpan maxWait, string context)
|
||||
{
|
||||
if (NetworkInterface.GetIsNetworkAvailable()) return;
|
||||
|
||||
logger?.Info($"{context}: no network link yet — waiting up to {maxWait.TotalSeconds:F0}s.");
|
||||
var deadline = DateTime.UtcNow + maxWait;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(1000).ConfigureAwait(false);
|
||||
if (NetworkInterface.GetIsNetworkAvailable())
|
||||
{
|
||||
logger?.Info($"{context}: network link is up.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
logger?.Warning($"{context}: still no network link after {maxWait.TotalSeconds:F0}s — attempting anyway.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,4 @@
|
||||
using System;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
@@ -165,29 +164,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Security
|
||||
=> Task.Delay(((int)Math.Pow(2, attempt)) * 1000);
|
||||
|
||||
/// <summary>
|
||||
/// Boot-time NIC grace (same pattern as the emergency-break reporter): polls the cheap
|
||||
/// link-level signal once per second for at most <see cref="NetworkLinkWaitMax"/>. It
|
||||
/// cannot prove backend reachability — the retry loop handles that — it only keeps the
|
||||
/// first attempts from being burned into a link that is provably still down. Probe
|
||||
/// Boot-time NIC grace before the first registration attempt — shared polling loop in
|
||||
/// <see cref="NetworkLinkWait"/>, bounded by <see cref="NetworkLinkWaitMax"/>. Probe
|
||||
/// errors end the wait, never the registration.
|
||||
/// </summary>
|
||||
internal static async Task WaitForNetworkLinkAsync(AgentLogger logger)
|
||||
{
|
||||
if (NetworkInterface.GetIsNetworkAvailable()) return;
|
||||
|
||||
logger?.Info($"Session registration: no network link yet — waiting up to {NetworkLinkWaitMax.TotalSeconds:F0}s.");
|
||||
var deadline = DateTime.UtcNow + NetworkLinkWaitMax;
|
||||
while (DateTime.UtcNow < deadline)
|
||||
{
|
||||
await Task.Delay(1000).ConfigureAwait(false);
|
||||
if (NetworkInterface.GetIsNetworkAvailable())
|
||||
{
|
||||
logger?.Info("Session registration: network link is up.");
|
||||
return;
|
||||
}
|
||||
}
|
||||
logger?.Warning($"Session registration: still no network link after {NetworkLinkWaitMax.TotalSeconds:F0}s — attempting anyway.");
|
||||
}
|
||||
internal static Task WaitForNetworkLinkAsync(AgentLogger logger)
|
||||
=> NetworkLinkWait.WaitAsync(logger, NetworkLinkWaitMax, "Session registration");
|
||||
|
||||
private static SessionRegistration BuildRegistration(
|
||||
AgentConfiguration agentConfig,
|
||||
|
||||
@@ -274,6 +274,15 @@ namespace AutopilotMonitor.Agent.V2
|
||||
{
|
||||
return registration.ExitCode;
|
||||
}
|
||||
|
||||
// Phase 6.5 (config self-heal): when the Phase 4 fetch ran before the network link
|
||||
// was up, registration has just proven the backend reachable — retry the fetch once
|
||||
// so the session runs on the live tenant config instead of cache/defaults.
|
||||
// See AgentRuntimeConfig.TryRecoverAfterRegistration for what is deliberately
|
||||
// NOT re-run here (kill-signal check, endpoint migration).
|
||||
Runtime.AgentRuntimeConfig.TryRecoverAfterRegistration(
|
||||
bootstrap.AgentConfig, auth, runtimeConfig, GetAgentVersion(), consoleMode, logger);
|
||||
|
||||
// Phase 7+8 (orchestrator + shutdown lifecycle + onIngressReady wiring +
|
||||
// initial signal posts + analyzer/probes startup + shutdown.Wait + finally)
|
||||
// is encapsulated in AgentRuntimeHost. See Runtime/AgentRuntimeHost.cs.
|
||||
|
||||
@@ -69,6 +69,63 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
var configMergeResult = ApplyFetchedConfig(agentConfig, remoteConfig, auth, agentVersion, consoleMode, logger);
|
||||
|
||||
return new RuntimeConfigBundle(remoteConfigService, remoteConfig, configMergeResult, pendingMigrationTarget: null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Phase 6.5 — config self-heal. When the Phase 4 fetch ran before the network link was
|
||||
/// up (agent relaunched at boot, Wi-Fi still associating — edu.kosh audit 2026-08-24:
|
||||
/// ~50% of sessions ran on ConfigVersion=0 exactly this way), a successful session
|
||||
/// registration has just proven the backend reachable. Retry the fetch once and apply
|
||||
/// the live tenant config instead of stranding the whole session on cache/defaults.
|
||||
/// <para>
|
||||
/// Deliberately NOT re-run here: the kill-signal check (post-registration kills arrive
|
||||
/// via the telemetry-channel control signals on every upload response) and endpoint
|
||||
/// migration (registration already succeeded against the current base URL; a migration
|
||||
/// signal is honoured at the next agent start). Never throws — recovery is strictly
|
||||
/// best-effort on top of an already-degraded-but-working fallback.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static void TryRecoverAfterRegistration(
|
||||
AgentConfiguration agentConfig,
|
||||
BackendAuthBundle auth,
|
||||
RuntimeConfigBundle bundle,
|
||||
string agentVersion,
|
||||
bool consoleMode,
|
||||
AgentLogger logger)
|
||||
{
|
||||
try
|
||||
{
|
||||
var service = bundle?.RemoteConfigService;
|
||||
if (service == null) return;
|
||||
|
||||
var recovered = service.TryRecoverConfigAsync().GetAwaiter().GetResult();
|
||||
if (recovered == null) return;
|
||||
|
||||
var mergeResult = ApplyFetchedConfig(agentConfig, recovered, auth, agentVersion, consoleMode, logger);
|
||||
bundle.ApplyRecovered(recovered, mergeResult);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Warning($"Post-registration config recovery failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a live-fetched config to the runtime: merge, tracker ceilings, log level,
|
||||
/// integrity SHA + check, bootstrap-config cleanup. Shared by the Phase 4 startup path
|
||||
/// and the post-registration recovery path so both apply identical semantics.
|
||||
/// </summary>
|
||||
private static RemoteConfigMergeResult ApplyFetchedConfig(
|
||||
AgentConfiguration agentConfig,
|
||||
AgentConfigResponse remoteConfig,
|
||||
BackendAuthBundle auth,
|
||||
string agentVersion,
|
||||
bool consoleMode,
|
||||
AgentLogger logger)
|
||||
{
|
||||
// Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
|
||||
// downstream consumers (CleanupService, SummaryDialogLauncher, StartupEnvironmentProbes,
|
||||
// DiagnosticsPackageService, EnrollmentTerminationHandler, the logger, the watchdog)
|
||||
@@ -91,7 +148,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
|
||||
TryCleanupBootstrapConfig(agentConfig, agentVersion, logger);
|
||||
|
||||
return new RuntimeConfigBundle(remoteConfigService, remoteConfig, configMergeResult, pendingMigrationTarget: null);
|
||||
return configMergeResult;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -175,10 +232,21 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
internal sealed class RuntimeConfigBundle
|
||||
{
|
||||
public RemoteConfigService RemoteConfigService { get; }
|
||||
public AgentConfigResponse RemoteConfig { get; }
|
||||
public RemoteConfigMergeResult MergeResult { get; }
|
||||
public AgentConfigResponse RemoteConfig { get; private set; }
|
||||
public RemoteConfigMergeResult MergeResult { get; private set; }
|
||||
public string PendingMigrationTarget { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Swaps in the config recovered by <see cref="AgentRuntimeConfig.TryRecoverAfterRegistration"/>
|
||||
/// so Phase 7+8 (orchestrator, collectors, gather rules, IME patterns, the
|
||||
/// unrestricted-mode audit) run on the live tenant config instead of the fallback.
|
||||
/// </summary>
|
||||
public void ApplyRecovered(AgentConfigResponse recoveredConfig, RemoteConfigMergeResult mergeResult)
|
||||
{
|
||||
RemoteConfig = recoveredConfig;
|
||||
MergeResult = mergeResult;
|
||||
}
|
||||
|
||||
public RuntimeConfigBundle(
|
||||
RemoteConfigService remoteConfigService,
|
||||
AgentConfigResponse remoteConfig,
|
||||
|
||||
@@ -52,6 +52,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// prior to this, ConfigVersion=0 was only deducible by inferring from
|
||||
// the absence of tenant-controlled knobs in the DataJson.
|
||||
var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted;
|
||||
var recovered = remoteConfigService?.RecoveredAfterFetchFailure == true;
|
||||
var configVersion = remoteConfigService?.CurrentConfig?.ConfigVersion ?? 0;
|
||||
|
||||
var data = new Dictionary<string, object>
|
||||
@@ -66,8 +67,12 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{ "diagnosticsUploadMode", agentConfig.DiagnosticsUploadMode ?? "Off" },
|
||||
{ "previousExitType", previousExit?.ExitType ?? "unknown" },
|
||||
{ "unrestrictedMode", agentConfig.UnrestrictedMode },
|
||||
// remoteConfigFetched answers "is the agent running on a live-fetched
|
||||
// config" — true also when the post-registration recovery replaced a
|
||||
// failed startup fetch (remoteConfigOutcome then still names the startup
|
||||
// failure; remoteConfigRecovered marks the heal).
|
||||
{ "configVersion", configVersion },
|
||||
{ "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded },
|
||||
{ "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded || recovered },
|
||||
{ "remoteConfigOutcome", outcome.ToString() },
|
||||
// WinRT OOBE state sampled at this start. Together with previousExitType
|
||||
// this makes late starts provable: "completed" on a first_run means OOBE
|
||||
@@ -75,6 +80,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{ "oobeStateAtAgentStart", OobeStateReader.Read() },
|
||||
};
|
||||
|
||||
if (recovered)
|
||||
data["remoteConfigRecovered"] = true;
|
||||
|
||||
if (!string.IsNullOrEmpty(previousExit?.CrashExceptionType))
|
||||
data["previousCrashException"] = previousExit.CrashExceptionType;
|
||||
|
||||
@@ -135,12 +143,25 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
? $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); using cached config."
|
||||
: $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); running with built-in defaults (ConfigVersion=0).";
|
||||
|
||||
// Post-registration recovery (Phase 6.5) replaced the fallback with a live
|
||||
// fetch before this emitter ran: keep the failure details (the network-late
|
||||
// condition is worth surfacing) but mark the heal and demote to Info — the
|
||||
// session is NOT degraded.
|
||||
var recovered = remoteConfigService.RecoveredAfterFetchFailure;
|
||||
if (recovered)
|
||||
{
|
||||
var recoveredVersion = remoteConfigService.CurrentConfig?.ConfigVersion ?? 0;
|
||||
data["recovered"] = true;
|
||||
data["recoveredConfigVersion"] = recoveredVersion;
|
||||
msg += $" Recovered after session registration (ConfigVersion={recoveredVersion}).";
|
||||
}
|
||||
|
||||
post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = agentConfig.SessionId,
|
||||
TenantId = agentConfig.TenantId,
|
||||
EventType = SharedConstants.EventTypes.RemoteConfigFetchFailed,
|
||||
Severity = EventSeverity.Warning,
|
||||
Severity = recovered ? EventSeverity.Info : EventSeverity.Warning,
|
||||
Source = "Agent",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = msg,
|
||||
|
||||
Reference in New Issue
Block a user