fix(agent/v2): retry remote config fetch + wire-visible fallback observability

Motivating incident — session 8f2bef72 (2026-05-19, OK's VM test): a Function
App cold-start triggered by a fresh deploy stalled the agent's single-shot
GetAgentConfig call for ~60 s; the client gave up, RemoteConfigService
silently fell back to built-in defaults (ConfigVersion=0,
SelfDestructOnComplete=true, DiagnosticsUploadMode=Off), and the entire
enrollment ran the wrong policy with no wire-visible trace of the degraded
mode. ReportAgentError (the emergency channel) was also racing the cold
start and lost to 499.

Three fixes:

1. Retry FetchConfigAsync on transient errors (3 attempts; 10/30/60 s linear
   backoff). BackendAuthException still bails immediately — 401/403 won't
   change with a retry and an operator needs the AuthCertificateRejected /
   DeviceNotRegistered distress signal fast. EmergencyReporter only fires on
   the final attempt to give the function time to warm. Live ServerAction
   rotate_config keeps single-shot so it doesn't block the action handler
   for ~100 s.

2. agent_started DataJson now carries configVersion + remoteConfigFetched +
   remoteConfigOutcome (Succeeded / FromCache / UsedDefaults / NotAttempted).
   Operators can tell at a glance whether the agent is running on fresh
   tenant config vs. defaults vs. stale cache without inferring from
   absence of expected tenant-controlled knobs.

3. New remote_config_fetch_failed Warning event emitted right after
   agent_started when the fallback path was taken. Carries attempts,
   failureType, failureMessage (truncated 512), and authStatusCode when
   relevant. Closes the blind spot — the only prior trace was a Warning
   in the local agent log which is gone after a SelfDestruct cleanup.

V2 Agent suite: 1092/1092 (+16: 7 retry behaviour + 9 emitter shape).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Configuration/RemoteConfigServiceRetryTests.cs

@@ -0,0 +1,212 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+using AutopilotMonitor.Agent.V2.Core.Configuration;
+using AutopilotMonitor.Agent.V2.Core.Logging;
+using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
+using AutopilotMonitor.Agent.V2.Core.Security;
+using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
+using AutopilotMonitor.Shared.Models;
+using Xunit;
+
+namespace AutopilotMonitor.Agent.V2.Core.Tests.Configuration
+{
+    /// <summary>
+    /// Pins the §observability retry + outcome-state contract on
+    /// <see cref="RemoteConfigService"/>. Motivating incident: session
+    /// 8f2bef72 (2026-05-19) where a Function App cold-start after a deploy
+    /// stalled the single-shot GetAgentConfig for 60s, the client gave up,
+    /// the agent silently fell back to <c>ConfigVersion=0</c> defaults, and
+    /// the entire enrollment ran the wrong policy (Mode=Off, SelfDestruct=true)
+    /// with no wire-visible signal of the fallback.
+    /// <para>
+    /// Tests override the two protected seams (<c>CallBackendAsync</c> +
+    /// <c>DelayBetweenAttemptsAsync</c>) so the retry suite runs in
+    /// milliseconds instead of the live 10/30/60-second cadence.
+    /// </para>
+    /// </summary>
+    public class RemoteConfigServiceRetryTests : IDisposable
+    {
+        private readonly TempDirectory _tmp = new TempDirectory();
+        private readonly AgentLogger _logger;
+
+        public RemoteConfigServiceRetryTests()
+        {
+            _logger = new AgentLogger(Path.Combine(_tmp.Path, "logs"), AgentLogLevel.Info);
+        }
+
+        public void Dispose() => _tmp.Dispose();
+
+        // ── Retry behaviour ─────────────────────────────────────────────────
+
+        [Fact]
+        public async Task FetchConfig_SucceedsOnFirstAttempt_NoRetry()
+        {
+            var svc = NewService(BackendBehaviour.SuccessImmediately);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+            Assert.NotNull(result);
+            Assert.Equal(42, result.ConfigVersion);
+            Assert.Equal(RemoteConfigFetchOutcome.Succeeded, svc.LastFetchOutcome);
+            Assert.Equal(1, svc.LastFetchAttempts);
+            Assert.Null(svc.LastFetchFailureType);
+        }
+
+        [Fact]
+        public async Task FetchConfig_TransientFailure_RetriesUpToCap()
+        {
+            // Always-fail transient → 4 attempts total (1 initial + 3 retries from
+            // InitialFetchRetryBackoff.Length=3). Then fall back to defaults.
+            var svc = NewService(BackendBehaviour.AlwaysTransientFailure);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+            Assert.NotNull(result);
+            Assert.Equal(0, result.ConfigVersion); // built-in defaults
+            Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+            Assert.Equal(1 + RemoteConfigService.InitialFetchRetryBackoff.Length, svc.LastFetchAttempts);
+            Assert.Equal(nameof(HttpRequestException), svc.LastFetchFailureType);
+            Assert.NotEmpty(svc.LastFetchFailureMessage);
+        }
+
+        [Fact]
+        public async Task FetchConfig_TransientThenSuccess_Returns_LiveConfig()
+        {
+            // Reproduces the cold-start pattern: first call times out, second succeeds.
+            // The whole reason this retry mechanism exists.
+            var svc = NewService(BackendBehaviour.FailTwiceThenSucceed);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+            Assert.Equal(42, result.ConfigVersion);
+            Assert.Equal(RemoteConfigFetchOutcome.Succeeded, svc.LastFetchOutcome);
+            Assert.Equal(3, svc.LastFetchAttempts);
+        }
+
+        [Fact]
+        public async Task FetchConfig_AuthFailure_BailsImmediately_NoRetry()
+        {
+            // 401/403 won't change with a retry — failing fast is correct. Verifies the
+            // retry loop's only short-circuit branch + that the status code is captured.
+            var svc = NewService(BackendBehaviour.AlwaysAuthFailure401);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+            Assert.NotNull(result); // fell back to defaults
+            Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+            Assert.Equal(1, svc.LastFetchAttempts);
+            Assert.Equal(nameof(BackendAuthException), svc.LastFetchFailureType);
+            Assert.Equal(401, svc.LastFetchAuthStatusCode);
+        }
+
+        [Fact]
+        public async Task FetchConfig_RetryDisabled_OneAttemptOnly()
+        {
+            // ServerControlPlane's rotate_config path uses single-shot to avoid blocking
+            // the action handler for 100 s on a flaky backend.
+            var svc = NewService(BackendBehaviour.AlwaysTransientFailure);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: false);
+
+            Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+            Assert.Equal(1, svc.LastFetchAttempts);
+        }
+
+        [Fact]
+        public async Task FetchConfig_EmptyResponse_TreatedAsTransient()
+        {
+            // A backend bug that returns null instead of throwing — fall through to
+            // retry so a transient null doesn't strand the agent on defaults.
+            var svc = NewService(BackendBehaviour.AlwaysReturnNull);
+
+            var result = await svc.FetchConfigAsync(retryOnTransientErrors: true);
+
+            Assert.Equal(RemoteConfigFetchOutcome.UsedDefaults, svc.LastFetchOutcome);
+            // 1 initial + 3 retries = 4 attempts on always-null
+            Assert.Equal(4, svc.LastFetchAttempts);
+        }
+
+        // ── Outcome state defaults ──────────────────────────────────────────
+
+        [Fact]
+        public void NotAttempted_BeforeFirstFetch()
+        {
+            var svc = NewService(BackendBehaviour.SuccessImmediately);
+            Assert.Equal(RemoteConfigFetchOutcome.NotAttempted, svc.LastFetchOutcome);
+            Assert.Equal(0, svc.LastFetchAttempts);
+        }
+
+        // ── Helpers ─────────────────────────────────────────────────────────
+
+        private enum BackendBehaviour
+        {
+            SuccessImmediately,
+            AlwaysTransientFailure,
+            FailTwiceThenSucceed,
+            AlwaysAuthFailure401,
+            AlwaysReturnNull,
+        }
+
+        private FakeRemoteConfigService NewService(BackendBehaviour behaviour)
+        {
+            // Pass a throwaway BackendApiClient — the fake overrides CallBackendAsync
+            // before it ever hits the wire.
+            var apiClient = new BackendApiClient(
+                httpClient: new HttpClient(),
+                baseUrl: "http://localhost",
+                manufacturer: string.Empty,
+                model: string.Empty,
+                serialNumber: string.Empty,
+                useBootstrapTokenAuth: false,
+                bootstrapToken: null,
+                agentVersion: "0.0.0",
+                logger: _logger);
+            return new FakeRemoteConfigService(apiClient, _logger, behaviour);
+        }
+
+        private sealed class FakeRemoteConfigService : RemoteConfigService
+        {
+            private readonly BackendBehaviour _behaviour;
+            private int _calls;
+
+            public FakeRemoteConfigService(BackendApiClient apiClient, AgentLogger logger, BackendBehaviour behaviour)
+                : base(apiClient, "00000000-0000-0000-0000-000000000001", logger)
+            {
+                _behaviour = behaviour;
+            }
+
+            protected override Task<AgentConfigResponse> CallBackendAsync()
+            {
+                _calls++;
+                switch (_behaviour)
+                {
+                    case BackendBehaviour.SuccessImmediately:
+                        return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
+                    case BackendBehaviour.FailTwiceThenSucceed:
+                        if (_calls < 3)
+                            throw new HttpRequestException("simulated cold-start timeout");
+                        return Task.FromResult(new AgentConfigResponse { ConfigVersion = 42 });
+                    case BackendBehaviour.AlwaysTransientFailure:
+                        throw new HttpRequestException("simulated network failure");
+                    case BackendBehaviour.AlwaysAuthFailure401:
+                        throw new BackendAuthException("Unauthorized", 401);
+                    case BackendBehaviour.AlwaysReturnNull:
+                        return Task.FromResult<AgentConfigResponse>(null);
+                    default:
+                        throw new InvalidOperationException("unknown behaviour");
+                }
+            }
+
+            // Zero-delay so the suite finishes in ms instead of the live 10/30/60 s schedule.
+            protected override Task DelayBetweenAttemptsAsync(TimeSpan delay) => Task.CompletedTask;
+
+            // In-memory + per-instance cache so a prior test's successful fetch can't
+            // bleed into a later test that expects UsedDefaults.
+            private AgentConfigResponse _cached;
+            protected override void CacheConfig(AgentConfigResponse config) { _cached = config; }
+            protected override AgentConfigResponse LoadCachedConfig() => _cached;
+        }
+    }
+}

src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Program/LifecycleEmittersRemoteConfigTests.cs

@@ -0,0 +1,279 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using AutopilotMonitor.Agent.V2.Core.Configuration;
+using AutopilotMonitor.Agent.V2.Core.Logging;
+using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
+using AutopilotMonitor.Agent.V2.Core.Orchestration;
+using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
+using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
+using AutopilotMonitor.Agent.V2.Runtime;
+using AutopilotMonitor.DecisionCore.Engine;
+using AutopilotMonitor.DecisionCore.Signals;
+using AutopilotMonitor.Shared.Models;
+using Xunit;
+using ProgramAlias = AutopilotMonitor.Agent.V2.Program;
+
+namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
+{
+    /// <summary>
+    /// Pins the §observability contract on <see cref="LifecycleEmitters.EmitAgentStarted"/>
+    /// (must carry <c>configVersion</c> + <c>remoteConfigFetched</c>) and
+    /// <see cref="LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny"/> (must emit a
+    /// dedicated <c>remote_config_fetch_failed</c> wire event when the agent fell back
+    /// to cache or built-in defaults; must NOT emit on success). Motivating incident:
+    /// session 8f2bef72 (2026-05-19) where the only on-the-wire trace of a degraded
+    /// agent run was the absence of expected events — invisible to operators.
+    /// </summary>
+    public class LifecycleEmittersRemoteConfigTests
+    {
+        // ── agent_started carries configVersion + remoteConfigFetched ─────────
+
+        [Fact]
+        public void EmitAgentStarted_OnSuccessfulFetch_FlagsRemoteConfigTrue()
+        {
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.Succeeded, configVersion: 28);
+
+            LifecycleEmitters.EmitAgentStarted(
+                rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+            Assert.Equal(28, data["configVersion"]);
+            Assert.Equal(true, data["remoteConfigFetched"]);
+            Assert.Equal("Succeeded", data["remoteConfigOutcome"]);
+        }
+
+        [Fact]
+        public void EmitAgentStarted_OnFallbackToDefaults_FlagsRemoteConfigFalse()
+        {
+            using var rig = new EmitterRig();
+            // The motivating incident: cold-start → all retries fail → defaults applied.
+            // The wire trace MUST make this visible at a glance.
+            rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.UsedDefaults, configVersion: 0);
+
+            LifecycleEmitters.EmitAgentStarted(
+                rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+            Assert.Equal(0, data["configVersion"]);
+            Assert.Equal(false, data["remoteConfigFetched"]);
+            Assert.Equal("UsedDefaults", data["remoteConfigOutcome"]);
+        }
+
+        [Fact]
+        public void EmitAgentStarted_OnFallbackToCache_FlagsCacheOutcome()
+        {
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.FromCache, configVersion: 27);
+
+            LifecycleEmitters.EmitAgentStarted(
+                rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+            Assert.Equal(27, data["configVersion"]); // stale-but-real version, not 0
+            Assert.Equal(false, data["remoteConfigFetched"]); // cache != fresh fetch
+            Assert.Equal("FromCache", data["remoteConfigOutcome"]);
+        }
+
+        [Fact]
+        public void EmitAgentStarted_TolerantOfNullRemoteConfigService()
+        {
+            // Defensive: the agent must always emit agent_started even if the
+            // RemoteConfigService construction failed before the lifecycle hook ran.
+            using var rig = new EmitterRig();
+
+            LifecycleEmitters.EmitAgentStarted(
+                rig.Post, rig.AgentConfig, rig.PreviousExit, "2.0.806", remoteConfigService: null, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "agent_started");
+            Assert.Equal(0, data["configVersion"]);
+            Assert.Equal(false, data["remoteConfigFetched"]);
+            Assert.Equal(nameof(RemoteConfigFetchOutcome.NotAttempted), data["remoteConfigOutcome"]);
+        }
+
+        // ── remote_config_fetch_failed event ─────────────────────────────────
+
+        [Fact]
+        public void EmitFetchFailedIfAny_OnSuccess_DoesNotEmit()
+        {
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(RemoteConfigFetchOutcome.Succeeded, configVersion: 28);
+
+            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+                rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+            Assert.Empty(rig.Sink.Posted);
+        }
+
+        [Fact]
+        public void EmitFetchFailedIfAny_OnNotAttempted_DoesNotEmit()
+        {
+            using var rig = new EmitterRig();
+            // No SetFakeOutcome → default = NotAttempted. Test the defensive branch
+            // in case the emitter is wired before any fetch happens.
+
+            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+                rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+            Assert.Empty(rig.Sink.Posted);
+        }
+
+        [Fact]
+        public void EmitFetchFailedIfAny_OnDefaultsFallback_EmitsWithDetails()
+        {
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(
+                RemoteConfigFetchOutcome.UsedDefaults,
+                configVersion: 0,
+                attempts: 4,
+                failureType: "HttpRequestException",
+                failureMessage: "simulated cold-start timeout");
+
+            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+                rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+            Assert.Equal("UsedDefaults", data["outcome"]);
+            Assert.Equal(4, data["attempts"]);
+            Assert.Equal("HttpRequestException", data["failureType"]);
+            Assert.Equal("simulated cold-start timeout", data["failureMessage"]);
+            Assert.False(data.ContainsKey("authStatusCode")); // not an auth failure
+        }
+
+        [Fact]
+        public void EmitFetchFailedIfAny_OnAuthFailure_IncludesStatusCode()
+        {
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(
+                RemoteConfigFetchOutcome.UsedDefaults,
+                configVersion: 0,
+                attempts: 1, // auth bails immediately, no retry
+                failureType: "BackendAuthException",
+                failureMessage: "Unauthorized",
+                authStatusCode: 401);
+
+            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+                rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+            Assert.Equal(1, data["attempts"]);
+            Assert.Equal(401, data["authStatusCode"]);
+        }
+
+        [Fact]
+        public void EmitFetchFailedIfAny_OnCacheFallback_Emits()
+        {
+            // Cache fallback IS a degraded mode — the tenant might have flipped a
+            // setting that the agent isn't seeing because the live fetch failed.
+            // Worth a wire event so operators can correlate.
+            using var rig = new EmitterRig();
+            rig.RemoteConfig.SetFakeOutcome(
+                RemoteConfigFetchOutcome.FromCache,
+                configVersion: 27,
+                attempts: 4,
+                failureType: "HttpRequestException",
+                failureMessage: "transient error");
+
+            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(
+                rig.Post, rig.AgentConfig, rig.RemoteConfig, rig.Logger);
+
+            var data = AssertSingleEmit(rig.Sink, expectedEventType: "remote_config_fetch_failed");
+            Assert.Equal("FromCache", data["outcome"]);
+        }
+
+        // ── Helpers ─────────────────────────────────────────────────────────
+
+        private static Dictionary<string, object> AssertSingleEmit(
+            FakeSignalIngressSink sink, string expectedEventType)
+        {
+            Assert.Single(sink.Posted);
+            var posted = sink.Posted[0];
+            Assert.Equal(DecisionSignalKind.InformationalEvent, posted.Kind);
+            Assert.NotNull(posted.Payload);
+            Assert.Equal(expectedEventType, posted.Payload!["eventType"]);
+            // typedPayload carries the Data dict object-preserving (per InformationalEventPost.cs).
+            Assert.IsType<Dictionary<string, object>>(posted.TypedPayload);
+            return (Dictionary<string, object>)posted.TypedPayload!;
+        }
+
+        private sealed class EmitterRig : System.IDisposable
+        {
+            public TempDirectory Tmp { get; } = new TempDirectory();
+            public AgentLogger Logger { get; }
+            public FakeSignalIngressSink Sink { get; }
+            public InformationalEventPost Post { get; }
+            public FakeRemoteConfigService RemoteConfig { get; }
+
+            public AgentConfiguration AgentConfig { get; } = new AgentConfiguration
+            {
+                ApiBaseUrl = "https://example.invalid",
+                SessionId = "S1",
+                TenantId = "00000000-0000-0000-0000-000000000001",
+                AgentMaxLifetimeMinutes = 360,
+                MaxAuthFailures = 3,
+                AuthFailureTimeoutMinutes = 30,
+                DiagnosticsUploadMode = "Off",
+                CommandLineArgs = string.Empty,
+            };
+
+            public ProgramAlias.PreviousExitSummary PreviousExit { get; }
+                = new ProgramAlias.PreviousExitSummary { ExitType = "first_run" };
+
+            public EmitterRig()
+            {
+                Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
+                Sink = new FakeSignalIngressSink();
+                Post = new InformationalEventPost(Sink, SystemClock.Instance, Logger);
+
+                var apiClient = new BackendApiClient(
+                    httpClient: new System.Net.Http.HttpClient(),
+                    baseUrl: "http://localhost",
+                    manufacturer: string.Empty,
+                    model: string.Empty,
+                    serialNumber: string.Empty,
+                    useBootstrapTokenAuth: false,
+                    bootstrapToken: null,
+                    agentVersion: "0.0.0",
+                    logger: Logger);
+                RemoteConfig = new FakeRemoteConfigService(apiClient, Logger);
+            }
+
+            public void Dispose() => Tmp.Dispose();
+        }
+
+        /// <summary>
+        /// Subclass with a setter for the outcome state so tests can pin emit behaviour
+        /// without going through the full FetchConfigAsync retry path (that's covered
+        /// separately in <c>RemoteConfigServiceRetryTests</c>).
+        /// </summary>
+        private sealed class FakeRemoteConfigService : RemoteConfigService
+        {
+            public FakeRemoteConfigService(BackendApiClient api, AgentLogger logger)
+                : base(api, "00000000-0000-0000-0000-000000000001", logger) { }
+
+            public void SetFakeOutcome(
+                RemoteConfigFetchOutcome outcome,
+                int configVersion = 0,
+                int attempts = 1,
+                string failureType = null,
+                string failureMessage = null,
+                int? authStatusCode = null)
+            {
+                // CurrentConfig is set via the protected SetConfig path — easiest:
+                // a real FetchConfigAsync replacement that just stashes a canned config.
+                typeof(RemoteConfigService).GetMethod("SetConfig",
+                    System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance)
+                    ?.Invoke(this, new object[] { new AgentConfigResponse { ConfigVersion = configVersion } });
+
+                // The public outcome props have private setters; use reflection so the
+                // test can control them without dirtying the production API surface.
+                typeof(RemoteConfigService).GetProperty(nameof(LastFetchOutcome))?.SetValue(this, outcome);
+                typeof(RemoteConfigService).GetProperty(nameof(LastFetchAttempts))?.SetValue(this, attempts);
+                typeof(RemoteConfigService).GetProperty(nameof(LastFetchFailureType))?.SetValue(this, failureType);
+                typeof(RemoteConfigService).GetProperty(nameof(LastFetchFailureMessage))?.SetValue(this, failureMessage);
+                typeof(RemoteConfigService).GetProperty(nameof(LastFetchAuthStatusCode))?.SetValue(this, authStatusCode);
+            }
+        }
+    }
+}

src/Agent/AutopilotMonitor.Agent.V2.Core/Configuration/RemoteConfigService.cs

@@ -27,6 +27,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
         private AgentConfigResponse _currentConfig;
         private readonly object _configLock = new object();
 
+        /// <summary>
+        /// Linear-backoff schedule (seconds) for the initial <see cref="FetchConfigAsync"/> call
+        /// when retries are enabled. Sized for the worst real-world cold-start pattern: deploy
+        /// triggers ~30-60s Function App spin-up, the first agent fetch hits it, times out, then
+        /// the second attempt 10s later usually succeeds. The 60s third attempt covers double
+        /// cold-starts (Function App + downstream Table/Blob warm-up). Exposed internal so the
+        /// xUnit suite can assert the cadence without hard-coding a copy of the array.
+        /// </summary>
+        internal static readonly TimeSpan[] InitialFetchRetryBackoff =
+        {
+            TimeSpan.FromSeconds(10),
+            TimeSpan.FromSeconds(30),
+            TimeSpan.FromSeconds(60),
+        };
+
         /// <summary>
         /// Gets the current configuration (thread-safe)
         /// </summary>
@@ -41,6 +56,26 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
             }
         }
 
+        /// <summary>
+        /// Outcome of the most recent <see cref="FetchConfigAsync"/> call. Observability
+        /// surface so the agent-startup pipeline can emit a wire-visible event when the
+        /// fetch fell back — closes the historical blind spot where a cold-start-induced
+        /// timeout silently degraded the agent to built-in defaults with no backend trace.
+        /// </summary>
+        public RemoteConfigFetchOutcome LastFetchOutcome { get; private set; } = RemoteConfigFetchOutcome.NotAttempted;
+
+        /// <summary>Number of HTTP attempts the last fetch consumed (1 on first-shot success, up to <see cref="InitialFetchRetryBackoff"/>.Length+1 with retries).</summary>
+        public int LastFetchAttempts { get; private set; }
+
+        /// <summary>Type name of the last terminal exception (e.g. <c>TaskCanceledException</c>, <c>BackendAuthException</c>); null when the fetch succeeded.</summary>
+        public string LastFetchFailureType { get; private set; }
+
+        /// <summary>One-line human-readable failure description (Exception.Message, truncated to 512 chars).</summary>
+        public string LastFetchFailureMessage { get; private set; }
+
+        /// <summary>HTTP status for auth failures (401/403), null otherwise.</summary>
+        public int? LastFetchAuthStatusCode { get; private set; }
+
         public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
         {
             _apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
@@ -55,60 +90,116 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
         }
 
         /// <summary>
-        /// Fetches the initial configuration from the backend
-        /// Falls back to cached config if the API is unreachable
+        /// Fetches the initial configuration from the backend. On transient failures
+        /// (network/timeout/5xx — anything that isn't <see cref="BackendAuthException"/>)
+        /// retries per <see cref="InitialFetchRetryBackoff"/> when <paramref name="retryOnTransientErrors"/>
+        /// is true. Falls back to cached config, then to defaults. Auth failures are
+        /// NEVER retried (a 401/403 won't change without an operator action).
+        /// <para>
+        /// Records the terminal outcome in <see cref="LastFetchOutcome"/>/<see cref="LastFetchFailureType"/>/
+        /// <see cref="LastFetchFailureMessage"/>/<see cref="LastFetchAttempts"/> so the startup pipeline can
+        /// emit a wire-visible <c>remote_config_fetch_failed</c> event when the fallback path
+        /// is taken — closes the blind spot where a cold-start-induced timeout silently
+        /// degraded the agent to defaults with no backend trace.
+        /// </para>
         /// </summary>
-        public async Task<AgentConfigResponse> FetchConfigAsync()
+        /// <param name="retryOnTransientErrors">
+        /// True for the initial agent-startup fetch (cold-start tolerance is essential).
+        /// False for the live ServerAction <c>rotate_config</c> path (a single-attempt
+        /// best-effort suffices; retries would block the action handler for ~100 s).
+        /// </param>
+        public async Task<AgentConfigResponse> FetchConfigAsync(bool retryOnTransientErrors = false)
         {
-            try
+            LastFetchOutcome = RemoteConfigFetchOutcome.NotAttempted;
+            LastFetchAttempts = 0;
+            LastFetchFailureType = null;
+            LastFetchFailureMessage = null;
+            LastFetchAuthStatusCode = null;
+
+            var maxAttempts = retryOnTransientErrors ? InitialFetchRetryBackoff.Length + 1 : 1;
+            Exception lastException = null;
+
+            for (var attempt = 1; attempt <= maxAttempts; attempt++)
             {
-                _logger.Info("Fetching remote configuration from backend...");
-                var config = await _apiClient.GetAgentConfigAsync(_tenantId);
-
-                if (config != null)
+                LastFetchAttempts = attempt;
+                try
                 {
-                    _logger.Info($"Remote config fetched: ConfigVersion={config.ConfigVersion}, " +
-                                 $"GatherRules={config.GatherRules?.Count ?? 0}");
+                    if (attempt == 1)
+                        _logger.Info("Fetching remote configuration from backend...");
+                    else
+                        _logger.Info($"Fetching remote configuration from backend (attempt {attempt}/{maxAttempts})...");
 
-                    LogCollectorSettings(config.Collectors);
-                    SetConfig(config);
-                    CacheConfig(config);
-                    _authFailureTracker?.RecordSuccess();
-                    return config;
+                    var config = await CallBackendAsync();
+
+                    if (config != null)
+                    {
+                        _logger.Info($"Remote config fetched: ConfigVersion={config.ConfigVersion}, " +
+                                     $"GatherRules={config.GatherRules?.Count ?? 0}");
+
+                        LogCollectorSettings(config.Collectors);
+                        SetConfig(config);
+                        CacheConfig(config);
+                        _authFailureTracker?.RecordSuccess();
+                        LastFetchOutcome = RemoteConfigFetchOutcome.Succeeded;
+                        return config;
+                    }
+
+                    _logger.Warning("Remote config fetch returned empty response");
+                    lastException = new InvalidOperationException("Remote config fetch returned null");
+                    // Empty response is treated as a transient error — fall through to retry.
+                }
+                catch (BackendAuthException ex)
+                {
+                    // Auth failures are NOT retryable — 401/403 won't go away with another
+                    // attempt; record + bail straight to cache/defaults so an operator can
+                    // see the AuthCertificateRejected/DeviceNotRegistered distress signal
+                    // without 100 s of needless retry delay.
+                    _logger.Warning($"Config fetch authentication failed: {ex.Message}");
+                    _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
+                    lastException = ex;
+                    LastFetchAuthStatusCode = ex.StatusCode;
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    _logger.Warning($"Failed to fetch remote config (attempt {attempt}/{maxAttempts}): {ex.Message}");
+                    lastException = ex;
+
+                    // Non-auth failure on FINAL attempt → use authenticated emergency channel.
+                    // Earlier attempts are still in the retry budget; don't spam.
+                    if (attempt == maxAttempts && _emergencyReporter != null)
+                    {
+                        _ = _emergencyReporter.TrySendAsync(
+                            AgentErrorType.ConfigFetchFailed,
+                            ex.Message);
+                    }
                 }
 
-                _logger.Warning("Remote config fetch returned empty response");
-            }
-            catch (BackendAuthException ex)
-            {
-                _logger.Warning($"Config fetch authentication failed: {ex.Message}");
-
-                // V1 parity — AuthFailureTracker is the single dispatch point for auth-failure
-                // distress signals (see HandleAuthFailure in EventUploadOrchestrator). It sends
-                // the first-failure report via its constructor-injected DistressReporter with
-                // the correct DistressErrorType per status code (401 → AuthCertificateRejected,
-                // 403 → DeviceNotRegistered) and suppresses duplicates on subsequent failures.
-                _authFailureTracker?.RecordFailure(ex.StatusCode, "agent/config");
-            }
-            catch (Exception ex)
-            {
-                _logger.Warning($"Failed to fetch remote config: {ex.Message}");
-
-                // Non-auth failure → use authenticated emergency channel
-                if (_emergencyReporter != null)
+                // Sleep before the next retry — only if there IS a next attempt.
+                if (attempt < maxAttempts)
                 {
-                    _ = _emergencyReporter.TrySendAsync(
-                        AgentErrorType.ConfigFetchFailed,
-                        ex.Message);
+                    var delay = InitialFetchRetryBackoff[attempt - 1];
+                    _logger.Info($"Retrying remote config fetch in {delay.TotalSeconds:0}s...");
+                    try { await DelayBetweenAttemptsAsync(delay); }
+                    catch (TaskCanceledException) { break; }
                 }
             }
 
+            // Capture failure context for the observability event.
+            if (lastException != null)
+            {
+                LastFetchFailureType = lastException.GetType().Name;
+                var msg = lastException.Message ?? string.Empty;
+                LastFetchFailureMessage = msg.Length > 512 ? msg.Substring(0, 512) : msg;
+            }
+
             // Fall back to cached config
             var cached = LoadCachedConfig();
             if (cached != null)
             {
                 _logger.Info("Using cached remote configuration");
                 SetConfig(cached);
+                LastFetchOutcome = RemoteConfigFetchOutcome.FromCache;
                 return cached;
             }
 
@@ -116,6 +207,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
             _logger.Info("No cached config available, using defaults (all optional collectors disabled)");
             var defaults = CreateDefaultConfig();
             SetConfig(defaults);
+            LastFetchOutcome = RemoteConfigFetchOutcome.UsedDefaults;
             return defaults;
         }
 
@@ -135,7 +227,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
             _logger.Info($"    Performance: {(collectors.EnablePerformanceCollector ? "ON" : "OFF")} (interval: {collectors.PerformanceIntervalSeconds}s)");
         }
 
-        private void CacheConfig(AgentConfigResponse config)
+        // Marked protected virtual so xUnit can swap to in-memory caching and avoid
+        // %ProgramData%\AutopilotMonitor\Config\remote-config.json bleeding state
+        // between parallel tests (a successful test would otherwise plant a cache that
+        // makes subsequent "fail + UsedDefaults" tests resolve to FromCache instead).
+        protected virtual void CacheConfig(AgentConfigResponse config)
         {
             try
             {
@@ -158,7 +254,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
             }
         }
 
-        private AgentConfigResponse LoadCachedConfig()
+        // protected virtual — see CacheConfig for rationale.
+        protected virtual AgentConfigResponse LoadCachedConfig()
         {
             try
             {
@@ -197,9 +294,53 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
             };
         }
 
+        // ============================================================ Test seams ====
+        // Two minimal hooks the xUnit suite overrides so it can pin the retry contract
+        // (success-on-N-th-attempt + auth-fail-bails-immediately + outcome state) without
+        // standing up a real HttpClient OR waiting the 10/30/60 s of the live backoff.
+
+        /// <summary>
+        /// Performs the actual <c>GetAgentConfig</c> HTTP call. Production hits the
+        /// injected <see cref="BackendApiClient"/>; tests subclass to return canned
+        /// responses / throw canned exceptions and assert the retry+outcome machinery
+        /// around it.
+        /// </summary>
+        protected virtual Task<AgentConfigResponse> CallBackendAsync()
+            => _apiClient.GetAgentConfigAsync(_tenantId);
+
+        /// <summary>
+        /// Awaits the per-attempt backoff. Production sleeps for the supplied
+        /// <see cref="InitialFetchRetryBackoff"/> entry; tests subclass to return
+        /// immediately so a 3-attempt retry suite finishes in milliseconds instead of
+        /// 100 seconds.
+        /// </summary>
+        protected virtual Task DelayBetweenAttemptsAsync(TimeSpan delay)
+            => Task.Delay(delay);
+
         public void Dispose()
         {
             // Nothing to dispose
         }
     }
+
+    /// <summary>
+    /// Terminal classification of the most recent <see cref="RemoteConfigService.FetchConfigAsync"/>
+    /// call. Surfaces what config the agent ended up running with — successful live fetch,
+    /// stale cache fall-back, or built-in defaults — so the startup pipeline can emit a
+    /// wire-visible event when the fallback path was taken.
+    /// </summary>
+    public enum RemoteConfigFetchOutcome
+    {
+        /// <summary>FetchConfigAsync has not been called yet on this instance.</summary>
+        NotAttempted = 0,
+
+        /// <summary>Live backend fetch returned a non-null config; agent runs with fresh tenant settings.</summary>
+        Succeeded = 1,
+
+        /// <summary>Live fetch failed; previously-cached config was loaded from disk and applied.</summary>
+        FromCache = 2,
+
+        /// <summary>Live fetch failed AND no cache available; agent runs with built-in defaults (ConfigVersion=0).</summary>
+        UsedDefaults = 3,
+    }
 }

src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeConfig.cs

@@ -41,7 +41,13 @@ namespace AutopilotMonitor.Agent.V2.Runtime
             var remoteConfigService = new RemoteConfigService(
                 auth.BackendApiClient, agentConfig.TenantId, logger,
                 auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
-            var remoteConfig = remoteConfigService.FetchConfigAsync().GetAwaiter().GetResult();
+            // retryOnTransientErrors:true is essential for the initial fetch — a Function App
+            // cold-start (typically 30-60 s after a deploy) will time out a single-shot call and
+            // silently strand the agent on built-in defaults for the entire session. Auth
+            // failures still bail immediately (they won't change with a retry).
+            var remoteConfig = remoteConfigService
+                .FetchConfigAsync(retryOnTransientErrors: true)
+                .GetAwaiter().GetResult();
 
             // Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
             // downstream consumers (CleanupService, SummaryDialogLauncher, StartupEnvironmentProbes,

src/Agent/AutopilotMonitor.Agent.V2/Runtime/AgentRuntimeHost.cs

@@ -297,7 +297,11 @@ namespace AutopilotMonitor.Agent.V2.Runtime
                         orchestrator.Start(ingress =>
                         {
                             lifecyclePost = new InformationalEventPost(ingress, SystemClock.Instance, logger);
-                            LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, logger);
+                            LifecycleEmitters.EmitAgentStarted(lifecyclePost, agentConfig, previousExit, agentVersion, remoteConfigService, logger);
+                            // Wire-visible signal that the agent is running on defaults/cache
+                            // rather than the live tenant config — closes the cold-start
+                            // blind spot observed in session 8f2bef72 (2026-05-19).
+                            LifecycleEmitters.EmitRemoteConfigFetchFailedIfAny(lifecyclePost, agentConfig, remoteConfigService, logger);
                             LifecycleEmitters.EmitVersionCheckIfAny(lifecyclePost, agentConfig, logger);
                             LifecycleEmitters.EmitUnrestrictedModeAuditIfChanged(lifecyclePost, agentConfig, configMergeResult, logger);
 

src/Agent/AutopilotMonitor.Agent.V2/Runtime/LifecycleEmitters.cs

@@ -40,10 +40,19 @@ namespace AutopilotMonitor.Agent.V2.Runtime
             AgentConfiguration agentConfig,
             Program.PreviousExitSummary previousExit,
             string agentVersion,
+            RemoteConfigService remoteConfigService,
             AgentLogger logger)
         {
             try
             {
+                // configVersion + remoteConfigFetched expose whether the agent is running
+                // with fresh tenant settings (Succeeded), a stale cache (FromCache), or
+                // built-in defaults (UsedDefaults). Closes the historical blind spot —
+                // prior to this, ConfigVersion=0 was only deducible by inferring from
+                // the absence of tenant-controlled knobs in the DataJson.
+                var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted;
+                var configVersion = remoteConfigService?.CurrentConfig?.ConfigVersion ?? 0;
+
                 var data = new Dictionary<string, object>
                 {
                     { "agentVersion", agentVersion },
@@ -56,6 +65,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
                     { "diagnosticsUploadMode", agentConfig.DiagnosticsUploadMode ?? "Off" },
                     { "previousExitType", previousExit?.ExitType ?? "unknown" },
                     { "unrestrictedMode", agentConfig.UnrestrictedMode },
+                    { "configVersion", configVersion },
+                    { "remoteConfigFetched", outcome == RemoteConfigFetchOutcome.Succeeded },
+                    { "remoteConfigOutcome", outcome.ToString() },
                 };
 
                 if (!string.IsNullOrEmpty(previousExit?.CrashExceptionType))
@@ -83,6 +95,60 @@ namespace AutopilotMonitor.Agent.V2.Runtime
             }
         }
 
+        /// <summary>
+        /// Wire-visible counterpart to the local "Failed to fetch remote config" warning
+        /// log: when the initial <see cref="RemoteConfigService.FetchConfigAsync"/> call
+        /// fell back to cache or built-in defaults, emit a dedicated event so the backend
+        /// timeline shows the agent didn't get fresh tenant config — without it, the only
+        /// trace was a Warning in the local agent log (gone after a SelfDestruct cleanup).
+        /// No-op when the fetch succeeded.
+        /// </summary>
+        public static void EmitRemoteConfigFetchFailedIfAny(
+            InformationalEventPost post,
+            AgentConfiguration agentConfig,
+            RemoteConfigService remoteConfigService,
+            AgentLogger logger)
+        {
+            try
+            {
+                var outcome = remoteConfigService?.LastFetchOutcome ?? RemoteConfigFetchOutcome.NotAttempted;
+                if (outcome == RemoteConfigFetchOutcome.Succeeded || outcome == RemoteConfigFetchOutcome.NotAttempted)
+                    return;
+
+                var data = new Dictionary<string, object>
+                {
+                    { "outcome", outcome.ToString() },
+                    { "attempts", remoteConfigService.LastFetchAttempts },
+                    { "failureType", remoteConfigService.LastFetchFailureType ?? string.Empty },
+                    { "failureMessage", remoteConfigService.LastFetchFailureMessage ?? string.Empty },
+                };
+
+                if (remoteConfigService.LastFetchAuthStatusCode.HasValue)
+                    data["authStatusCode"] = remoteConfigService.LastFetchAuthStatusCode.Value;
+
+                var msg = outcome == RemoteConfigFetchOutcome.FromCache
+                    ? $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); using cached config."
+                    : $"Remote config fetch failed after {remoteConfigService.LastFetchAttempts} attempt(s); running with built-in defaults (ConfigVersion=0).";
+
+                post.Emit(new EnrollmentEvent
+                {
+                    SessionId = agentConfig.SessionId,
+                    TenantId = agentConfig.TenantId,
+                    EventType = "remote_config_fetch_failed",
+                    Severity = EventSeverity.Warning,
+                    Source = "Agent",
+                    Phase = EnrollmentPhase.Unknown,
+                    Message = msg,
+                    Data = data,
+                    ImmediateUpload = true,
+                });
+            }
+            catch (Exception ex)
+            {
+                logger.Warning($"remote_config_fetch_failed emission failed: {ex.Message}");
+            }
+        }
+
         public static void EmitVersionCheckIfAny(
             InformationalEventPost post,
             AgentConfiguration agentConfig,

src/Agent/AutopilotMonitor.Agent.V2/buildcounter.txt

@@ -1 +1 @@
-805
+815