mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 21:19:38 +00:00
6d607c9045
Root cause (tenant aebdce78 audit 2026-08-24, session ba828038): on boot-time
relaunches the agent starts before the Wi-Fi link is up. The Phase 4 config fetch
had no network gate (98f8114d only guarded Phase 6 registration), burned its
10/30/60s retry budget into a dead link and stranded the whole session on built-in
defaults (ConfigVersion=0) — up to ~50% of that tenant's sessions, with no
self-heal short of a manual rotate_config action. Not a version regression: rates
varied wildly per day within the same agent build; the failed requests never
reached the backend.
Two complementary layers:
- RemoteConfigService gates the initial fetch on the shared NetworkLinkWait poll
(60s cap — longer than registration's 15s because the fetch runs first), so the
retry budget starts on a live link. rotate_config stays ungated (latency-bound).
- Phase 6.5: after registration proves the backend reachable, TryRecoverConfigAsync
refetches once and AgentRuntimeConfig.TryRecoverAfterRegistration re-applies the
live config (same merge/integrity pipeline as Phase 4). LastFetch* keeps the
original failure record; remote_config_fetch_failed reports it with a recovered
flag at Info severity, agent_started flags remoteConfigRecovered.
Deliberately not re-run in recovery: kill-signal check (telemetry-channel control
signals cover post-registration kills) and endpoint migration (next start).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
263 lines
13 KiB
C#
263 lines
13 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Reflection;
|
|
using System.Threading.Tasks;
|
|
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
|
using AutopilotMonitor.Agent.V2.Core.Logging;
|
|
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
|
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
|
using AutopilotMonitor.Agent.V2.Core.Security;
|
|
using AutopilotMonitor.Shared.Models;
|
|
|
|
namespace AutopilotMonitor.Agent.V2.Runtime
|
|
{
|
|
/// <summary>
|
|
/// Phase 4 of <see cref="Program"/>'s <c>RunAgent</c>: fetch the tenant's
|
|
/// <see cref="AgentConfigResponse"/>, merge it into the runtime
|
|
/// <see cref="AgentConfiguration"/> via <see cref="RemoteConfigMerger"/>, refresh tracker
|
|
/// ceilings + log level, propagate the backend-expected SHA, run the post-config binary-
|
|
/// integrity check, and best-effort delete the persisted bootstrap-config (H-2 mitigation
|
|
/// once the MDM cert proves it can authenticate).
|
|
/// <para>
|
|
/// No early-exit paths — Phase 4 always returns a populated bundle. Failures inside the
|
|
/// integrity check or bootstrap-config-cleanup are surfaced as fire-and-forget distress
|
|
/// reports / log entries; the agent must keep starting on a stale config rather than
|
|
/// abort, matching V1 behaviour.
|
|
/// </para>
|
|
/// </summary>
|
|
internal static class AgentRuntimeConfig
|
|
{
|
|
public static RuntimeConfigBundle Resolve(
|
|
AgentConfiguration agentConfig,
|
|
BackendAuthBundle auth,
|
|
string agentVersion,
|
|
bool consoleMode,
|
|
AgentLogger logger,
|
|
bool allowEndpointMigration = true)
|
|
{
|
|
if (agentConfig == null) throw new ArgumentNullException(nameof(agentConfig));
|
|
if (auth == null) throw new ArgumentNullException(nameof(auth));
|
|
if (logger == null) throw new ArgumentNullException(nameof(logger));
|
|
|
|
var remoteConfigService = new RemoteConfigService(
|
|
auth.BackendApiClient, agentConfig.TenantId, logger,
|
|
auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
|
|
// retryOnTransientErrors:true is essential for the initial fetch — a Function App
|
|
// cold-start (typically 30-60 s after a deploy) will time out a single-shot call and
|
|
// silently strand the agent on built-in defaults for the entire session. Auth
|
|
// failures still bail immediately (they won't change with a retry).
|
|
var remoteConfig = remoteConfigService
|
|
.FetchConfigAsync(retryOnTransientErrors: true)
|
|
.GetAwaiter().GetResult();
|
|
|
|
// Endpoint migration (config-channel re-home): decided BEFORE merge / binary-
|
|
// integrity / bootstrap-cleanup so none of those side effects run against a config
|
|
// from the backend being abandoned — a migration stub may advertise stale agent
|
|
// hashes, and a spurious IntegrityCheckFailed + force-update against them would be
|
|
// actively harmful. RunAgent rebuilds Phase 3 clients on the new base URL and calls
|
|
// Resolve again with allowEndpointMigration:false (single hop — the new backend's
|
|
// config cannot chain another migration).
|
|
if (allowEndpointMigration)
|
|
{
|
|
var migrationTarget = EndpointMigration.ResolveTarget(
|
|
remoteConfig, remoteConfigService.LastFetchOutcome, agentConfig.ApiBaseUrl, logger);
|
|
if (migrationTarget != null)
|
|
{
|
|
return new RuntimeConfigBundle(
|
|
remoteConfigService, remoteConfig, mergeResult: null,
|
|
pendingMigrationTarget: migrationTarget);
|
|
}
|
|
}
|
|
|
|
var configMergeResult = ApplyFetchedConfig(agentConfig, remoteConfig, auth, agentVersion, consoleMode, logger);
|
|
|
|
return new RuntimeConfigBundle(remoteConfigService, remoteConfig, configMergeResult, pendingMigrationTarget: null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Phase 6.5 — config self-heal. When the Phase 4 fetch ran before the network link was
|
|
/// up (agent relaunched at boot, Wi-Fi still associating — tenant aebdce78 audit 2026-08-24:
|
|
/// ~50% of sessions ran on ConfigVersion=0 exactly this way), a successful session
|
|
/// registration has just proven the backend reachable. Retry the fetch once and apply
|
|
/// the live tenant config instead of stranding the whole session on cache/defaults.
|
|
/// <para>
|
|
/// Deliberately NOT re-run here: the kill-signal check (post-registration kills arrive
|
|
/// via the telemetry-channel control signals on every upload response) and endpoint
|
|
/// migration (registration already succeeded against the current base URL; a migration
|
|
/// signal is honoured at the next agent start). Never throws — recovery is strictly
|
|
/// best-effort on top of an already-degraded-but-working fallback.
|
|
/// </para>
|
|
/// </summary>
|
|
public static void TryRecoverAfterRegistration(
|
|
AgentConfiguration agentConfig,
|
|
BackendAuthBundle auth,
|
|
RuntimeConfigBundle bundle,
|
|
string agentVersion,
|
|
bool consoleMode,
|
|
AgentLogger logger)
|
|
{
|
|
try
|
|
{
|
|
var service = bundle?.RemoteConfigService;
|
|
if (service == null) return;
|
|
|
|
var recovered = service.TryRecoverConfigAsync().GetAwaiter().GetResult();
|
|
if (recovered == null) return;
|
|
|
|
var mergeResult = ApplyFetchedConfig(agentConfig, recovered, auth, agentVersion, consoleMode, logger);
|
|
bundle.ApplyRecovered(recovered, mergeResult);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.Warning($"Post-registration config recovery failed: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Applies a live-fetched config to the runtime: merge, tracker ceilings, log level,
|
|
/// integrity SHA + check, bootstrap-config cleanup. Shared by the Phase 4 startup path
|
|
/// and the post-registration recovery path so both apply identical semantics.
|
|
/// </summary>
|
|
private static RemoteConfigMergeResult ApplyFetchedConfig(
|
|
AgentConfiguration agentConfig,
|
|
AgentConfigResponse remoteConfig,
|
|
BackendAuthBundle auth,
|
|
string agentVersion,
|
|
bool consoleMode,
|
|
AgentLogger logger)
|
|
{
|
|
// Project remote tenant-controlled knobs onto the runtime AgentConfiguration so that
|
|
// downstream consumers (CleanupService, SummaryDialogLauncher, StartupEnvironmentProbes,
|
|
// DiagnosticsPackageService, EnrollmentTerminationHandler, the logger, the watchdog)
|
|
// actually respect the tenant admin settings. V1 parity — remote wins
|
|
// unconditionally for every knob that has a 1:1 mapping. CLI flags seed the initial
|
|
// AgentConfiguration in BuildAgentConfiguration and then yield to tenant policy.
|
|
var configMergeResult = RemoteConfigMerger.Merge(agentConfig, remoteConfig, logger);
|
|
|
|
// Refresh tracker ceilings with the tenant-specific values we just merged in.
|
|
auth.AuthFailureTracker.UpdateLimits(agentConfig.MaxAuthFailures, agentConfig.AuthFailureTimeoutMinutes);
|
|
|
|
logger.SetLogLevel(agentConfig.LogLevel);
|
|
|
|
// Propagate the backend-expected SHA so the runtime hash-mismatch trigger has the
|
|
// up-to-date integrity hash. Also refresh AllowAgentDowngrade.
|
|
if (!string.IsNullOrEmpty(remoteConfig.LatestAgentSha256))
|
|
SelfUpdater.BackendExpectedSha256 = remoteConfig.LatestAgentSha256;
|
|
|
|
VerifyBinaryIntegrity(remoteConfig, auth.EmergencyReporter, agentVersion, consoleMode, logger);
|
|
|
|
TryCleanupBootstrapConfig(agentConfig, agentVersion, logger);
|
|
|
|
return configMergeResult;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Post-config binary-integrity check: verify the running EXE's SHA-256 against the
|
|
/// value advertised by the backend. V1 parity — on mismatch we (a) emit an
|
|
/// IntegrityCheckFailed emergency report and (b) fire the runtime self-update trigger
|
|
/// so the agent auto-heals (SelfUpdater.CheckAndApplyUpdateAsync force-update path).
|
|
/// The trigger is single-shot per process (Interlocked guard inside the verifier).
|
|
/// </summary>
|
|
private static void VerifyBinaryIntegrity(
|
|
AgentConfigResponse remoteConfig,
|
|
EmergencyReporter emergencyReporter,
|
|
string agentVersion,
|
|
bool consoleMode,
|
|
AgentLogger logger)
|
|
{
|
|
var agentDirForTrigger = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) ?? string.Empty;
|
|
Func<string, bool, Task> runtimeSelfUpdateTrigger = (zipHash, downgrade) =>
|
|
{
|
|
if (!string.IsNullOrEmpty(zipHash))
|
|
SelfUpdater.BackendExpectedSha256 = zipHash;
|
|
|
|
return SelfUpdater.CheckAndApplyUpdateAsync(
|
|
currentVersion: agentVersion,
|
|
agentDir: agentDirForTrigger,
|
|
consoleMode: consoleMode,
|
|
forceUpdate: true,
|
|
triggerReason: "runtime_hash_mismatch",
|
|
downloadTimeoutMsOverride: 60000,
|
|
allowDowngrade: downgrade);
|
|
};
|
|
|
|
var integrityResult = BinaryIntegrityVerifier.Check(
|
|
expectedSha256: remoteConfig.LatestAgentExeSha256,
|
|
logger: logger,
|
|
runtimeSelfUpdateTrigger: runtimeSelfUpdateTrigger,
|
|
zipHash: remoteConfig.LatestAgentSha256,
|
|
allowDowngrade: remoteConfig.AllowAgentDowngrade);
|
|
if (integrityResult.IsMismatch)
|
|
{
|
|
_ = emergencyReporter.TrySendAsync(
|
|
AgentErrorType.IntegrityCheckFailed,
|
|
$"Running exe SHA-256 differs from backend-advertised hash. actual={integrityResult.ActualSha256}, expected={integrityResult.ExpectedSha256}");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// H-2 mitigation: delete the persisted bootstrap-config.json once the MDM cert
|
|
/// proves it can authenticate. Non-blocking — any failure leaves the file for retry.
|
|
/// </summary>
|
|
private static void TryCleanupBootstrapConfig(
|
|
AgentConfiguration agentConfig,
|
|
string agentVersion,
|
|
AgentLogger logger)
|
|
{
|
|
try
|
|
{
|
|
BootstrapConfigCleanup
|
|
.TryDeleteIfCertReadyAsync(agentConfig, logger, agentVersion)
|
|
.GetAwaiter().GetResult();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
logger.Debug($"BootstrapConfigCleanup outer exception: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Phase 4 outcome — the live <see cref="RemoteConfigService"/> (used later by the
|
|
/// <c>rotate_config</c> ServerAction callback), the fetched <see cref="AgentConfigResponse"/>
|
|
/// and the <see cref="RemoteConfigMergeResult"/> diff snapshot (used by the
|
|
/// <c>EmitUnrestrictedModeAuditIfChanged</c> lifecycle hook).
|
|
/// <para>
|
|
/// When <see cref="PendingMigrationTarget"/> is non-null the bundle is a partial result:
|
|
/// the fetch found a live endpoint-migration signal and Resolve returned before merge /
|
|
/// integrity / bootstrap-cleanup (<see cref="MergeResult"/> is null). RunAgent must
|
|
/// rebuild the backend clients on the target URL and Resolve again.
|
|
/// </para>
|
|
/// </summary>
|
|
internal sealed class RuntimeConfigBundle
|
|
{
|
|
public RemoteConfigService RemoteConfigService { get; }
|
|
public AgentConfigResponse RemoteConfig { get; private set; }
|
|
public RemoteConfigMergeResult MergeResult { get; private set; }
|
|
public string PendingMigrationTarget { get; }
|
|
|
|
/// <summary>
|
|
/// Swaps in the config recovered by <see cref="AgentRuntimeConfig.TryRecoverAfterRegistration"/>
|
|
/// so Phase 7+8 (orchestrator, collectors, gather rules, IME patterns, the
|
|
/// unrestricted-mode audit) run on the live tenant config instead of the fallback.
|
|
/// </summary>
|
|
public void ApplyRecovered(AgentConfigResponse recoveredConfig, RemoteConfigMergeResult mergeResult)
|
|
{
|
|
RemoteConfig = recoveredConfig;
|
|
MergeResult = mergeResult;
|
|
}
|
|
|
|
public RuntimeConfigBundle(
|
|
RemoteConfigService remoteConfigService,
|
|
AgentConfigResponse remoteConfig,
|
|
RemoteConfigMergeResult mergeResult,
|
|
string pendingMigrationTarget = null)
|
|
{
|
|
RemoteConfigService = remoteConfigService;
|
|
RemoteConfig = remoteConfig;
|
|
MergeResult = mergeResult;
|
|
PendingMigrationTarget = pendingMigrationTarget;
|
|
}
|
|
}
|
|
}
|