diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs
new file mode 100644
index 00000000..363770da
--- /dev/null
+++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs
@@ -0,0 +1,212 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+using AutopilotMonitor.Agent.V2.Core.Configuration;
+using AutopilotMonitor.Agent.V2.Core.Logging;
+using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
+using AutopilotMonitor.Agent.V2.Core.Security;
+using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
+using AutopilotMonitor.Shared.Models;
+using Xunit;
+
+namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
+{
+ ///
+ /// Pins the §observability retry + outcome-state contract on
+ /// . Motivating incident: session
+ /// 8f2bef72 (2026-05-19) where a Function App cold-start after a deploy
+ /// stalled the single-shot GetAgentConfig for 60s, the client gave up,
+ /// the agent silently fell back to ConfigVersion=0 defaults, and
+ /// the entire enrollment ran the wrong policy (Mode=Off, SelfDestruct=true)
+ /// with no wire-visible signal of the fallback.
+ ///
+ /// Tests override the two protected seams (CallBackendAsync +
+ /// DelayBetweenAttemptsAsync) so the retry suite runs in
+ /// milliseconds instead of the live 10/30/60-second cadence.
+ ///
+ ///
+ public class RemoteConfigServiceRetryTests : IDisposable
+ {
+ private readonly TempDirectory _tmp = new TempDirectory();
+ private readonly AgentLogger _logger;
+
+ public RemoteConfigServiceRetryTests()
+ {
+ _logger = new AgentLogger(Path.Combine(_tmp.Path, "logs"), AgentLogLevel.Info);
+ }
+
+ public void Dispose() => _tmp.Dispose();
+
+ // ── Retry behaviour ─────────────────────────────────────────────────
+
+ [Fact]
+ public async Task FetchConfig_SucceedsOnFirstAttempt_NoRetry()
+ {
+ var svc = NewService(BackendBehaviour.SuccessImmediately);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+ Assert.NotNull(result);
+ Assert.Equal(42, result.ConfigVersion);
+ Assert.Equal(RemoteConfigFetchOutcome.Succeeded, svc.LastFetchOutcome);
+ Assert.Equal(1, svc.LastFetchAttempts);
+ Assert.Null(svc.LastFetchFailureType);
+ }
+
+ [Fact]
+ public async Task FetchConfig_TransientFailure_RetriesUpToCap()
+ {
+ // Always-fail transient → 4 attempts total (1 initial + 3 retries from
+ // InitialFetchRetryBackoff.Length=3). Then fall back to defaults.
+ var svc = NewService(BackendBehaviour.AlwaysTransientFailure);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+ Assert.NotNull(result);
+ Assert.Equal(0, result.ConfigVersion); // built-in defaults
+ Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+ Assert.Equal(1 + RemoteConfigService.InitialFetchRetryBackoff.Length, svc.LastFetchAttempts);
+ Assert.Equal(nameof(HttpRequestException), svc.LastFetchFailureType);
+ Assert.NotEmpty(svc.LastFetchFailureMessage);
+ }
+
+ [Fact]
+ public async Task FetchConfig_TransientThenSuccess_Returns_LiveConfig()
+ {
+ // Reproduces the cold-start pattern: first call times out, second succeeds.
+ // The whole reason this retry mechanism exists.
+ var svc = NewService(BackendBehaviour.FailTwiceThenSucceed);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+ Assert.Equal(42, result.ConfigVersion);
+ Assert.Equal(RemoteConfigFetchOutcome.Succeeded, svc.LastFetchOutcome);
+ Assert.Equal(3, svc.LastFetchAttempts);
+ }
+
+ [Fact]
+ public async Task FetchConfig_AuthFailure_BailsImmediately_NoRetry()
+ {
+ // 401/403 won't change with a retry — failing fast is correct. Verifies the
+ // retry loop's only short-circuit branch + that the status code is captured.
+ var svc = NewService(BackendBehaviour.AlwaysAuthFailure401);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+ Assert.NotNull(result); // fell back to defaults
+ Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+ Assert.Equal(1, svc.LastFetchAttempts);
+ Assert.Equal(nameof(BackendAuthException), svc.LastFetchFailureType);
+ Assert.Equal(401, svc.LastFetchAuthStatusCode);
+ }
+
+ [Fact]
+ public async Task FetchConfig_RetryDisabled_OneAttemptOnly()
+ {
+ // ServerControlPlane's rotate_config path uses single-shot to avoid blocking
+ // the action handler for 100 s on a flaky backend.
+ var svc = NewService(BackendBehaviour.AlwaysTransientFailure);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: false);
+
+ Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+ Assert.Equal(1, svc.LastFetchAttempts);
+ }
+
+ [Fact]
+ public async Task FetchConfig_EmptyResponse_TreatedAsTransient()
+ {
+ // A backend bug that returns null instead of throwing — fall through to
+ // retry so a transient null doesn't strand the agent on defaults.
+ var svc = NewService(BackendBehaviour.AlwaysReturnNull);
+
+ var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+ Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+ // 1 initial + 3 retries = 4 attempts on always-null
+ Assert.Equal(4, svc.LastFetchAttempts);
+ }
+
+ // ── Outcome state defaults ──────────────────────────────────────────
+
+ [Fact]
+ public void NotAttempted_BeforeFirstFetch()
+ {
+ var svc = NewService(BackendBehaviour.SuccessImmediately);
+ Assert.Equal(RemoteConfigFetchOutcome.NotAttempted, svc.LastFetchOutcome);
+ Assert.Equal(0, svc.LastFetchAttempts);
+ }
+
+ // ── Helpers ─────────────────────────────────────────────────────────
+
+ private enum BackendBehaviour
+ {
+ SuccessImmediately,
+ AlwaysTransientFailure,
+ FailTwiceThenSucceed,
+ AlwaysAuthFailure401,
+ AlwaysReturnNull,
+ }
+
+ private FakeRemoteConfigService NewService(BackendBehaviour behaviour)
+ {
+ // Pass a throwaway BackendApiClient — the fake overrides CallBackendAsync
+ // before it ever hits the wire.
+ var apiClient = new BackendApiClient(
+ httpClient: new HttpClient(),
+ baseUrl: "http://localhost",
+ manufacturer: string.Empty,
+ model: string.Empty,
+ serialNumber: string.Empty,
+ useBootstrapTokenAuth: false,
+ bootstrapToken: null,
+ agentVersion: "0.0.0",
+ logger: _logger);
+ return new FakeRemoteConfigService(apiClient, _logger, behaviour);
+ }
+
+ private sealed class FakeRemoteConfigService : RemoteConfigService
+ {
+ private readonly BackendBehaviour _behaviour;
+ private int _calls;
+
+ public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour)
+ : base(apiClient, "00000000-0000-0000-0000-000000000001", logger)
+ {
+ _behaviour = behaviour;
+ }
+
+ protected override Task CallBackendAsync()
+ {
+ _calls++;
+ switch (_behaviour)
+ {
+ case BackendBehaviour.SuccessImmediately:
+ return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
+ case BackendBehaviour.FailTwiceThenSucceed:
+ if (_calls < 3)
+ throw new HttpRequestException("simulated cold-start timeout");
+ return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
+ case BackendBehaviour.AlwaysTransientFailure:
+ throw new HttpRequestException("simulated network failure");
+ case BackendBehaviour.AlwaysAuthFailure401:
+ throw new BackendAuthException("Unauthorized", 401);
+ case BackendBehaviour.AlwaysReturnNull:
+ return Task.FromResult(null);
+ default:
+ throw new InvalidOperationException("unknown behaviour");
+ }
+ }
+
+ // Zero-delay so the suite finishes in ms instead of the live 10/30/60 s schedule.
+ protected override Task DelayBetweenAttemptsAsync(TimeSpan delay) => Task.CompletedTask;
+
+ // In-memory + per-instance cache so a prior test's successful fetch can't
+ // bleed into a later test that expects UsedDefaults.
+ private AgentConfigResponse _cached;
+ protected override void CacheConfig(AgentConfigResponse config) { _cached = config; }
+ protected override AgentConfigResponse LoadCachedConfig() => _cached;
+ }
+ }
+}
diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs
new file mode 100644
index 00000000..896bd3f2
--- /dev/null
+++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs
@@ -0,0 +1,279 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using AutopilotMonitor.Agent.V2.Core.Configuration;
+using AutopilotMonitor.Agent.V2.Core.Logging;
+using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
+using AutopilotMonitor.Agent.V2.Core.Orchestration;
+using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
+using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
+using AutopilotMonitor.Agent.V2.Runtime;
+using AutopilotMonitor.DecisionCore.Engine;
+using AutopilotMonitor.DecisionCore.Signals;
+using AutopilotMonitor.Shared.Models;
+using Xunit;
+using ProgramAlias = AutopilotMonitor.Agent.V2.Program;
+
+namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
+{
+ ///
+ /// Pins the §observability contract on
+ /// (must carry configVersion + remoteConfigFetched) and
+ /// (must emit a
+ /// dedicated remote_config_fetch_failed wire event when the agent fell back
+ /// to cache or built-in defaults; must NOT emit on success). Motivating incident:
+ /// session 8f2bef72 (2026-05-19) where the only on-the-wire trace of a degraded
+ /// agent run was the absence of expected events — invisible to operators.
+ ///
+ public class LifecycleEmittersRemoteConfigTests
+ {
+ // ── agent_started carries configVersion + remoteConfigFetched ─────────
+
+ [Fact]
+ public void EmitAgentStarted_OnSuccessfulFetch_FlagsRemoteConfigTrue()
+ {
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.Succeeded, configVersion: 28);
+
+ LifecycleEmitters.EmitAgentStarted(
+ rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+ Assert.Equal(28, data["configVersion"]);
+ Assert.Equal(true, data["remoteConfigFetched"]);
+ Assert.Equal("Succeeded", data["remoteConfigOutcome"]);
+ }
+
+ [Fact]
+ public void EmitAgentStarted_OnFallbackToDefaults_FlagsRemoteConfigFalse()
+ {
+ using var rig = new EmitterRig();
+ // The motivating incident: cold-start → all retries fail → defaults applied.
+ // The wire trace MUST make this visible at a glance.
+ rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.UsedDefaults, configVersion: 0);
+
+ LifecycleEmitters.EmitAgentStarted(
+ rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+ Assert.Equal(0, data["configVersion"]);
+ Assert.Equal(false, data["remoteConfigFetched"]);
+ Assert.Equal("UsedDefaults", data["remoteConfigOutcome"]);
+ }
+
+ [Fact]
+ public void EmitAgentStarted_OnFallbackToCache_FlagsCacheOutcome()
+ {
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.FromCache, configVersion: 27);
+
+ LifecycleEmitters.EmitAgentStarted(
+ rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+ Assert.Equal(27, data["configVersion"]); // stale-but-real version, not 0
+ Assert.Equal(false, data["remoteConfigFetched"]); // cache != fresh fetch
+ Assert.Equal("FromCache", data["remoteConfigOutcome"]);
+ }
+
+ [Fact]
+ public void EmitAgentStarted_TolerantOfNullRemoteConfigService()
+ {
+ // Defensive: the agent must always emit agent_started even if the
+ // RemoteConfigService construction failed before the lifecycle hook ran.
+ using var rig = new EmitterRig();
+
+ LifecycleEmitters.EmitAgentStarted(
+ rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", remoteConfigService: null, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+ Assert.Equal(0, data["configVersion"]);
+ Assert.Equal(false, data["remoteConfigFetched"]);
+ Assert.Equal(nameof(RemoteConfigFetchOutcome.NotAttempted), data["remoteConfigOutcome"]);
+ }
+
+ // ── remote_config_fetch_failed event ─────────────────────────────────
+
+ [Fact]
+ public void EmitFetchFailedIfAny_OnSuccess_DoesNotEmit()
+ {
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.Succeeded, configVersion: 28);
+
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+ rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+ Assert.Empty(rig.Sink.Posted);
+ }
+
+ [Fact]
+ public void EmitFetchFailedIfAny_OnNotAttempted_DoesNotEmit()
+ {
+ using var rig = new EmitterRig();
+ // No SetFakeOutcome → default = NotAttempted. Test the defensive branch
+ // in case the emitter is wired before any fetch happens.
+
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+ rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+ Assert.Empty(rig.Sink.Posted);
+ }
+
+ [Fact]
+ public void EmitFetchFailedIfAny_OnDefaultsFallback_EmitsWithDetails()
+ {
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(
+ RemoteConfigFetchOutcome.UsedDefaults,
+ configVersion: 0,
+ attempts: 4,
+ failureType: "HttpRequestException",
+ failureMessage: "simulated cold-start timeout");
+
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+ rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+ Assert.Equal("UsedDefaults", data["outcome"]);
+ Assert.Equal(4, data["attempts"]);
+ Assert.Equal("HttpRequestException", data["failureType"]);
+ Assert.Equal("simulated cold-start timeout", data["failureMessage"]);
+ Assert.False(data.ContainsKey("authStatusCode")); // not an auth failure
+ }
+
+ [Fact]
+ public void EmitFetchFailedIfAny_OnAuthFailure_IncludesStatusCode()
+ {
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(
+ RemoteConfigFetchOutcome.UsedDefaults,
+ configVersion: 0,
+ attempts: 1, // auth bails immediately, no retry
+ failureType: "BackendAuthException",
+ failureMessage: "Unauthorized",
+ authStatusCode: 401);
+
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+ rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+ Assert.Equal(1, data["attempts"]);
+ Assert.Equal(401, data["authStatusCode"]);
+ }
+
+ [Fact]
+ public void EmitFetchFailedIfAny_OnCacheFallback_Emits()
+ {
+ // Cache fallback IS a degraded mode — the tenant might have flipped a
+ // setting that the agent isn't seeing because the live fetch failed.
+ // Worth a wire event so operators can correlate.
+ using var rig = new EmitterRig();
+ rig.RemoteConfig.SetFakeOutcome(
+ RemoteConfigFetchOutcome.FromCache,
+ configVersion: 27,
+ attempts: 4,
+ failureType: "HttpRequestException",
+ failureMessage: "transient error");
+
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+ rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+ var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+ Assert.Equal("FromCache", data["outcome"]);
+ }
+
+ // ── Helpers ─────────────────────────────────────────────────────────
+
+ private static Dictionary AssertSingleEmit(
+ FakeSignalIngressSink sink, string expectedEventType)
+ {
+ Assert.Single(sink.Posted);
+ var posted = sink.Posted[0];
+ Assert.Equal(DecisionSignalKind.InformationalEvent, posted.Kind);
+ Assert.NotNull(posted.Payload);
+ Assert.Equal(expectedEventType, posted.Payload!["eventType"]);
+ // typedPayload carries the Data dict object-preserving (per InformationalEventPost.cs).
+ Assert.IsType>(posted.TypedPayload);
+ return (Dictionary)posted.TypedPayload!;
+ }
+
+ private sealed class EmitterRig : System.IDisposable
+ {
+ public TempDirectory Tmp { get; } = new TempDirectory();
+ public AgentLogger Logger { get; }
+ public FakeSignalIngressSink Sink { get; }
+ public InformationalEventPost Post { get; }
+ public FakeRemoteConfigService RemoteConfig { get; }
+
+ public AgentConfiguration AgentConfig { get; } = new AgentConfiguration
+ {
+ ApiBaseUrl = "https://example.invalid",
+ SessionId = "S1",
+ TenantId = "00000000-0000-0000-0000-000000000001",
+ AgentMaxLifetimeMinutes = 360,
+ MaxAuthFailures = 3,
+ AuthFailureTimeoutMinutes = 30,
+ DiagnosticsUploadMode = "Off",
+ CommandLineArgs = string.Empty,
+ };
+
+ public ProgramAlias.PreviousExitSummary PreviousExit { get; }
+ = new ProgramAlias.PreviousExitSummary { ExitType = "first_run" };
+
+ public EmitterRig()
+ {
+ Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
+ Sink = new FakeSignalIngressSink();
+ Post = new InformationalEventPost(Sink, SystemClock.Instance, Logger);
+
+ var apiClient = new BackendApiClient(
+ httpClient: new System.Net.Http.HttpClient(),
+ baseUrl: "http://localhost",
+ manufacturer: string.Empty,
+ model: string.Empty,
+ serialNumber: string.Empty,
+ useBootstrapTokenAuth: false,
+ bootstrapToken: null,
+ agentVersion: "0.0.0",
+ logger: Logger);
+ RemoteConfig = new FakeRemoteConfigService(apiClient, Logger);
+ }
+
+ public void Dispose() => Tmp.Dispose();
+ }
+
+ ///
+ /// Subclass with a setter for the outcome state so tests can pin emit behaviour
+ /// without going through the full FetchConfigAsync retry path (that's covered
+ /// separately in RemoteConfigServiceRetryTests).
+ ///
+ private sealed class FakeRemoteConfigService : RemoteConfigService
+ {
+ public FakeRemoteConfigService(BackendApiClient api, AgentLogger logger)
+ : base(api, "00000000-0000-0000-0000-000000000001", logger) { }
+
+ public void SetFakeOutcome(
+ RemoteConfigFetchOutcome outcome,
+ int configVersion = 0,
+ int attempts = 1,
+ string failureType = null,
+ string failureMessage = null,
+ int? authStatusCode = null)
+ {
+ // CurrentConfig is set via the protected SetConfig path — easiest:
+ // a real FetchConfigAsync replacement that just stashes a canned config.
+ typeof(RemoteConfigService).GetMethod("SetConfig",
+ System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
+ ?.Invoke(this, new object[] { new AgentConfigResponse { ConfigVersion = configVersion } });
+
+ // The public outcome props have private setters; use reflection so the
+ // test can control them without dirtying the production API surface.
+ typeof(RemoteConfigService).GetProperty(nameof(LastFetchOutcome))?.SetValue(this, outcome);
+ typeof(RemoteConfigService).GetProperty(nameof(LastFetchAttempts))?.SetValue(this, attempts);
+ typeof(RemoteConfigService).GetProperty(nameof(LastFetchFailureType))?.SetValue(this, failureType);
+ typeof(RemoteConfigService).GetProperty(nameof(LastFetchFailureMessage))?.SetValue(this, failureMessage);
+ typeof(RemoteConfigService).GetProperty(nameof(LastFetchAuthStatusCode))?.SetValue(this, authStatusCode);
+ }
+ }
+ }
+}
diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs
index d5770b01..602abe4b 100644
--- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs
+++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs
@@ -27,6 +27,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
private AgentConfigResponse _currentConfig;
private readonly object _configLock = new object();
+ ///
+ /// Linear-backoff schedule (seconds) for the initial call
+ /// when retries are enabled. Sized for the worst real-world cold-start pattern: deploy
+ /// triggers ~30-60s Function App spin-up, the first agent fetch hits it, times out, then
+ /// the second attempt 10s later usually succeeds. The 60s third attempt covers double
+ /// cold-starts (Function App + downstream Table/Blob warm-up). Exposed internal so the
+ /// xUnit suite can assert the cadence without hard-coding a copy of the array.
+ ///
+ internal static readonly TimeSpan[] InitialFetchRetryBackoff =
+ {
+ TimeSpan.FromSeconds(10),
+ TimeSpan.FromSeconds(30),
+ TimeSpan.FromSeconds(60),
+ };
+
///
/// Gets the current configuration (thread-safe)
///
@@ -41,6 +56,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
}
}
+ ///
+ /// Outcome of the most recent call. Observability
+ /// surface so the agent-startup pipeline can emit a wire-visible event when the
+ /// fetch fell back — closes the historical blind spot where a cold-start-induced
+ /// timeout silently degraded the agent to built-in defaults with no backend trace.
+ ///
+ public RemoteConfigFetchOutcome LastFetchOutcome { get; private set; } = RemoteConfigFetchOutcome.NotAttempted;
+
+ /// Number of HTTP attempts the last fetch consumed (1 on first-shot success, up to .Length+1 with retries).
+ public int LastFetchAttempts { get; private set; }
+
+ /// Type name of the last terminal exception (e.g. TaskCanceledException, BackendAuthException); null when the fetch succeeded.
+ public string LastFetchFailureType { get; private set; }
+
+ /// One-line human-readable failure description (Exception.Message, truncated to 512 chars).
+ public string LastFetchFailureMessage { get; private set; }
+
+ /// HTTP status for auth failures (401/403), null otherwise.
+ public int? LastFetchAuthStatusCode { get; private set; }
+
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
{
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
@@ -55,60 +90,116 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
}
///
- /// Fetches the initial configuration from the backend
- /// Falls back to cached config if the API is unreachable
+ /// Fetches the initial configuration from the backend. On transient failures
+ /// (network/timeout/5xx — anything that isn't )
+ /// retries per when
+ /// is true. Falls back to cached config, then to defaults. Auth failures are
+ /// NEVER retried (a 401/403 won't change without an operator action).
+ ///
+ /// Records the terminal outcome in //
+ /// / so the startup pipeline can
+ /// emit a wire-visible remote_config_fetch_failed event when the fallback path
+ /// is taken — closes the blind spot where a cold-start-induced timeout silently
+ /// degraded the agent to defaults with no backend trace.
+ ///
///
- public async Task FetchConfigAsync()
+ ///
+ /// True for the initial agent-startup fetch (cold-start tolerance is essential).
+ /// False for the live ServerAction rotate_config path (a single-attempt
+ /// best-effort suffices; retries would block the action handler for ~100 s).
+ ///
+ public async Task FetchConfigAsync(bool retryOnTransientErrors = false)
{
- try
+ LastFetchOutcome = RemoteConfigFetchOutcome.NotAttempted;
+ LastFetchAttempts = 0;
+ LastFetchFailureType = null;
+ LastFetchFailureMessage = null;
+ LastFetchAuthStatusCode = null;
+
+ var maxAttempts = retryOnTransientErrors ? InitialFetchRetryBackoff.Length + 1 : 1;
+ Exception lastException = null;
+
+ for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
- _logger.Info("Fetching remote configuration from backend...");
- var config = await _apiClient.GetAgentConfigAsync(_tenantId);
-
- if (config != null)
+ LastFetchAttempts = attempt;
+ try
{
- _logger.Info($"Remote config fetched: ConfigVersion={config.ConfigVersion}, " +
- $"GatherRules={config.GatherRules?.Count ?? 0}");
+ if (attempt == 1)
+ _logger.Info("Fetching remote configuration from backend...");
+ else
+ _logger.Info($"Fetching remote configuration from backend (attempt {attempt}/{maxAttempts})...");
- LogCollectorSettings(config.Collectors);
- SetConfig(config);
- CacheConfig(config);
- _authFailureTracker?.RecordSuccess();
- return config;
+ var config = await CallBackendAsync();
+
+ if (config != null)
+ {
+ _logger.Info($"Remote config fetched: ConfigVersion={config.ConfigVersion}, " +
+ $"GatherRules={config.GatherRules?.Count ?? 0}");
+
+ LogCollectorSettings(config.Collectors);
+ SetConfig(config);
+ CacheConfig(config);
+ _authFailureTracker?.RecordSuccess();
+ LastFetchOutcome = RemoteConfigFetchOutcome.Succeeded;
+ return config;
+ }
+
+ _logger.Warning("Remote config fetch returned empty response");
+ lastException = new InvalidOperationException("Remote config fetch returned null");
+ // Empty response is treated as a transient error — fall through to retry.
+ }
+ catch (BackendAuthException ex)
+ {
+ // Auth failures are NOT retryable — 401/403 won't go away with another
+ // attempt; record + bail straight to cache/defaults so an operator can
+ // see the AuthCertificateRejected/DeviceNotRegistered distress signal
+ // without 100 s of needless retry delay.
+ _logger.Warning($"Config fetch authentication failed: {ex.Message}");
+ _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
+ lastException = ex;
+ LastFetchAuthStatusCode = ex.StatusCode;
+ break;
+ }
+ catch (Exception ex)
+ {
+ _logger.Warning($"Failed to fetch remote config (attempt {attempt}/{maxAttempts}): {ex.Message}");
+ lastException = ex;
+
+ // Non-auth failure on FINAL attempt → use authenticated emergency channel.
+ // Earlier attempts are still in the retry budget; don't spam.
+ if (attempt == maxAttempts && _emergencyReporter != null)
+ {
+ _ = _emergencyReporter.TrySendAsync(
+ AgentErrorType.ConfigFetchFailed,
+ ex.Message);
+ }
}
- _logger.Warning("Remote config fetch returned empty response");
- }
- catch (BackendAuthException ex)
- {
- _logger.Warning($"Config fetch authentication failed: {ex.Message}");
-
- // V1 parity — AuthFailureTracker is the single dispatch point for auth-failure
- // distress signals (see HandleAuthFailure in EventUploadOrchestrator). It sends
- // the first-failure report via its constructor-injected DistressReporter with
- // the correct DistressErrorType per status code (401 → AuthCertificateRejected,
- // 403 → DeviceNotRegistered) and suppresses duplicates on subsequent failures.
- _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
- }
- catch (Exception ex)
- {
- _logger.Warning($"Failed to fetch remote config: {ex.Message}");
-
- // Non-auth failure → use authenticated emergency channel
- if (_emergencyReporter != null)
+ // Sleep before the next retry — only if there IS a next attempt.
+ if (attempt < maxAttempts)
{
- _ = _emergencyReporter.TrySendAsync(
- AgentErrorType.ConfigFetchFailed,
- ex.Message);
+ var delay = InitialFetchRetryBackoff[attempt - 1];
+ _logger.Info($"Retrying remote config fetch in {delay.TotalSeconds:0}s...");
+ try { await DelayBetweenAttemptsAsync(delay); }
+ catch (TaskCanceledException) { break; }
}
}
+ // Capture failure context for the observability event.
+ if (lastException != null)
+ {
+ LastFetchFailureType = lastException.GetType().Name;
+ var msg = lastException.Message ?? string.Empty;
+ LastFetchFailureMessage = msg.Length > 512 ? msg.Substring(0, 512) : msg;
+ }
+
// Fall back to cached config
var cached = LoadCachedConfig();
if (cached != null)
{
_logger.Info("Using cached remote configuration");
SetConfig(cached);
+ LastFetchOutcome = RemoteConfigFetchOutcome.FromCache;
return cached;
}
@@ -116,6 +207,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
_logger.Info("No cached config available, using defaults (all optional collectors disabled)");
var defaults = CreateDefaultConfig();
SetConfig(defaults);
+ LastFetchOutcome = RemoteConfigFetchOutcome.UsedDefaults;
return defaults;
}
@@ -135,7 +227,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
_logger.Info($" Performance: {(collectors.EnablePerformanceCollector ? "ON" : "OFF")} (interval: {collectors.PerformanceIntervalSeconds}s)");
}
- private void CacheConfig(AgentConfigResponse config)
+ // Marked protected virtual so xUnit can swap to in-memory caching and avoid
+ // %ProgramData%\AutopilotMonitor\Config\remote-config.json bleeding state
+ // between parallel tests (a successful test would otherwise plant a cache that
+ // makes subsequent "fail + UsedDefaults" tests resolve to FromCache instead).
+ protected virtual void CacheConfig(AgentConfigResponse config)
{
try
{
@@ -158,7 +254,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
}
}
- private AgentConfigResponse LoadCachedConfig()
+ // protected virtual — see CacheConfig for rationale.
+ protected virtual AgentConfigResponse LoadCachedConfig()
{
try
{
@@ -197,9 +294,53 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
};
}
+ // ============================================================ Test seams ====
+ // Two minimal hooks the xUnit suite overrides so it can pin the retry contract
+ // (success-on-N-th-attempt + auth-fail-bails-immediately + outcome state) without
+ // standing up a real HttpClient OR waiting the 10/30/60 s of the live backoff.
+
+ ///
+ /// Performs the actual GetAgentConfig HTTP call. Production hits the
+ /// injected ; tests subclass to return canned
+ /// responses / throw canned exceptions and assert the retry+outcome machinery
+ /// around it.
+ ///
+ protected virtual Task CallBackendAsync()
+ => _apiClient.GetAgentConfigAsync(_tenantId);
+
+ ///
+ /// Awaits the per-attempt backoff. Production sleeps for the supplied
+ /// entry; tests subclass to return
+ /// immediately so a 3-attempt retry suite finishes in milliseconds instead of
+ /// 100 seconds.
+ ///
+ protected virtual Task DelayBetweenAttemptsAsync(TimeSpan delay)
+ => Task.Delay(delay);
+
public void Dispose()
{
// Nothing to dispose
}
}
+
+ ///
+ /// Terminal classification of the most recent
+ /// call. Surfaces what config the agent ended up running with — successful live fetch,
+ /// stale cache fall-back, or built-in defaults — so the startup pipeline can emit a
+ /// wire-visible event when the fallback path was taken.
+ ///
+ public enum RemoteConfigFetchOutcome
+ {
+ /// FetchConfigAsync has not been called yet on this instance.
+ NotAttempted = 0,
+
+ /// Live backend fetch returned a non-null config; agent runs with fresh tenant settings.
+ Succeeded = 1,
+
+ /// Live fetch failed; previously-cached config was loaded from disk and applied.
+ FromCache = 2,
+
+ /// Live fetch failed AND no cache available; agent runs with built-in defaults (ConfigVersion=0).
+ UsedDefaults = 3,
+ }
}
diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs
index 4d559906..17cb8ae8 100644
--- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs
+++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs
@@ -41,7 +41,13 @@ namespace AutopilotMonitor.Agent.V2.Runtime
var remoteConfigService = new RemoteConfigService(
auth.BackendApiClient, agentConfig.TenantId, logger,
auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
- var remoteConfig = remoteConfigService.FetchConfigAsync().GetAwaiter().GetResult();
+ // retryOnTransientErrors:true is essential for the initial fetch — a Function App
+ // cold-start (typically 30-60 s after a deploy) will time out a single-shot call and
+ // silently strand the agent on built-in defaults for the entire session. Auth
+ // failures still bail immediately (they won't change with a retry).
+ var remoteConfig = remoteConfigService
+ .FetchConfigAsync(retryOnTransientErrors: true)
+ .GetAwaiter().GetResult();
// Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
// downstream consumers (CleanupService, SummaryDialogLauncher, StartupEnvironmentProbes,
diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs
index 42454128..f41af269 100644
--- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs
+++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs
@@ -297,7 +297,11 @@ namespace AutopilotMonitor.Agent.V2.Runtime
orchestrator.Start(ingress =>
{
lifecyclePost = new InformationalEventPost(ingress, SystemClock.Instance, logger);
- LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, logger);
+ LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, remoteConfigService, logger);
+ // Wire-visible signal that the agent is running on defaults/cache
+ // rather than the live tenant config — closes the cold-start
+ // blind spot observed in session 8f2bef72 (2026-05-19).
+ LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(lifecyclePost, agentConfig, remoteConfigService, logger);
LifecycleEmitters.EmitVersionCheckIfAny(lifecyclePost, agentConfig, logger);
LifecycleEmitters.EmitUnrestrictedModeAuditIfChanged(lifecyclePost, agentConfig, configMergeResult, logger);
diff --git a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs
index b56067fe..80181bac 100644
--- a/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs
+++ b/src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs
@@ -40,10 +40,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
AgentConfiguration agentConfig,
Program.PreviousExitSummary previousExit,
string agentVersion,
+ RemoteConfigService remoteConfigService,
AgentLogger logger)
{
try
{
+ // configVersion + remoteConfigFetched expose whether the agent is running
+ // with fresh tenant settings (Succeeded), a stale cache (FromCache), or
+ // built-in defaults (UsedDefaults). Closes the historical blind spot —
+ // prior to this, ConfigVersion=0 was only deducible by inferring from
+ // the absence of tenant-controlled knobs in the DataJson.
+ var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted;
+ var configVersion = remoteConfigService?.CurrentConfig?.ConfigVersion ?? 0;
+
var data = new Dictionary
{
{ "agentVersion", agentVersion },
@@ -56,6 +65,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
{ "diagnosticsUploadMode", agentConfig.DiagnosticsUploadMode ?? "Off" },
{ "previousExitType", previousExit?.ExitType ?? "unknown" },
{ "unrestrictedMode", agentConfig.UnrestrictedMode },
+ { "configVersion", configVersion },
+ { "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded },
+ { "remoteConfigOutcome", outcome.ToString() },
};
if (!string.IsNullOrEmpty(previousExit?.CrashExceptionType))
@@ -83,6 +95,60 @@ namespace AutopilotMonitor.Agent.V2.Runtime
}
}
+ ///
+ /// Wire-visible counterpart to the local "Failed to fetch remote config" warning
+ /// log: when the initial call
+ /// fell back to cache or built-in defaults, emit a dedicated event so the backend
+ /// timeline shows the agent didn't get fresh tenant config — without it, the only
+ /// trace was a Warning in the local agent log (gone after a SelfDestruct cleanup).
+ /// No-op when the fetch succeeded.
+ ///
+ public static void EmitRemoteConfigFetchFailedIfAny(
+ InformationalEventPost post,
+ AgentConfiguration agentConfig,
+ RemoteConfigService remoteConfigService,
+ AgentLogger logger)
+ {
+ try
+ {
+ var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted;
+ if (outcome == RemoteConfigFetchOutcome.Succeeded || outcome == RemoteConfigFetchOutcome.NotAttempted)
+ return;
+
+ var data = new Dictionary
+ {
+ { "outcome", outcome.ToString() },
+ { "attempts", remoteConfigService.LastFetchAttempts },
+ { "failureType", remoteConfigService.LastFetchFailureType ?? string.Empty },
+ { "failureMessage", remoteConfigService.LastFetchFailureMessage ?? string.Empty },
+ };
+
+ if (remoteConfigService.LastFetchAuthStatusCode.HasValue)
+ data["authStatusCode"] = remoteConfigService.LastFetchAuthStatusCode.Value;
+
+ var msg = outcome == RemoteConfigFetchOutcome.FromCache
+ ? $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); using cached config."
+ : $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); running with built-in defaults (ConfigVersion=0).";
+
+ post.Emit(new EnrollmentEvent
+ {
+ SessionId = agentConfig.SessionId,
+ TenantId = agentConfig.TenantId,
+ EventType = "remote_config_fetch_failed",
+ Severity = EventSeverity.Warning,
+ Source = "Agent",
+ Phase = EnrollmentPhase.Unknown,
+ Message = msg,
+ Data = data,
+ ImmediateUpload = true,
+ });
+ }
+ catch (Exception ex)
+ {
+ logger.Warning($"remote_config_fetch_failed emission failed: {ex.Message}");
+ }
+ }
+
public static void EmitVersionCheckIfAny(
InformationalEventPost post,
AgentConfiguration agentConfig,
diff --git a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt
index cc34bb39..5fa2986d 100644
--- a/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt
+++ b/src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt
@@ -1 +1 @@
-805
+815