feat(agent-v2): app_install_starved names the app starving the AccountSetup gate (PR3)

The gate-starvation dead-ends (caa6cf50, 1ec8f4c6) were ultimately caused
by a single required user-targeted app that never started installing.
Until now the session just hung anonymously; this makes the starving app
a named, actionable customer hint.

New ImeLogTracker.GetStarvedUserEspApps(): required (Install/Uninstall
intent) apps in the current AccountSetup phase with no download/install
activity ever observed, not terminal, not failed. Apps that are alive
(Downloading/Installing) stay excluded; error apps belong to the failure
path. Same conservative thread-safety contract as AreUserEspAppsSettled.

Two timer-free trigger points, one-shot per appId:
- Live: when the ESP page exits and the user-apps-settled probe reports
  false, EspAndHelloTracker emits app_install_starved (Warning) per
  starved app; dedupe via a lock-guarded HashSet exposed as
  StarvedAppsReported.
- Terminal: EnrollmentTerminationHandler sweeps leftovers on every
  outcome (before app_tracking_summary, skipped on WhiteGlove Part 1) and
  consults the live path dedupe set via the read-model surface, so no
  app is ever reported twice.

Wiring: ImeLogHost bridge -> DefaultComponentFactory probe ->
EspAndHelloTracker; CollectorSurfaces + IAppTrackingReadModel +
OrchestratorAppTrackingReadModel expose the starved list and the
already-reported set to the termination handler.

New event type synced to MCP resource catalog (app_events) and Web
gather-rules eventTypes.

V2.Core 1365/1365, MCP vitest 220, Web 458, full solution build clean.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-06-12 15:53:57 +02:00
parent eb3af41a96
commit ab7bd1afa9
15 changed files with 682 additions and 5 deletions
@@ -0,0 +1,174 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
{
/// <summary>
/// Liveness plan PR3 — live-path <c>app_install_starved</c> emission. When the ESP page
/// exits while the user-apps-settled probe reports false, the coordinator names the
/// starved app(s) (one-shot Warning per appId) instead of leaving the operator with an
/// anonymous stall. The dedupe set is exposed via
/// <see cref="EspAndHelloTracker.StarvedAppsReported"/> for the termination sweep.
/// </summary>
public sealed class EspAndHelloTrackerStarvedAppsTests
{
private static readonly DateTime Fixed = new DateTime(2026, 6, 12, 8, 0, 2, DateTimeKind.Utc);
private sealed class Fixture : IDisposable
{
public TempDirectory Tmp { get; } = new TempDirectory();
public AgentLogger Logger { get; }
public FakeSignalIngressSink Ingress { get; } = new FakeSignalIngressSink();
public FakeSignalIngressSink TrackerPostSink { get; } = new FakeSignalIngressSink();
public VirtualClock Clock { get; } = new VirtualClock(Fixed);
public Fixture() { Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Debug); }
public EspAndHelloTracker BuildCoordinator(
Func<bool> settledProbe,
Func<IReadOnlyList<AppPackageState>> starvedProbe)
{
return new EspAndHelloTracker(
sessionId: "S1",
tenantId: "T1",
post: new InformationalEventPost(TrackerPostSink, Clock),
logger: Logger,
skipConfigProbe: () => ((bool?)false, (bool?)false),
accountSetupActivityProbe: () => true,
userEspAppsSettledProbe: settledProbe,
starvedUserEspAppsProbe: starvedProbe);
}
public List<FakeSignalIngressSink.PostedSignal> StarvedPosts() =>
TrackerPostSink.Posted.Where(p =>
p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == "app_install_starved").ToList();
public void Dispose() { Tmp.Dispose(); }
}
private static AppPackageState StarvedApp(string id, string? name = null)
{
var pkg = new AppPackageState(id, listPos: 0);
pkg.UpdateIntent(AppIntent.Install);
if (name != null) pkg.UpdateName(name);
return pkg;
}
[Fact]
public void Unsettled_exit_emits_warning_per_starved_app()
{
using var f = new Fixture();
using var coordinator = f.BuildCoordinator(
() => false,
() => new[] { StarvedApp("app-1", "Contoso Backgrounds") });
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
var post = Assert.Single(f.StarvedPosts());
Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]);
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(post.TypedPayload);
Assert.Equal("app-1", data["appId"]);
Assert.Equal("Contoso Backgrounds", data["appName"]);
Assert.Equal("esp_exited_user_apps_not_settled", data["trigger"]);
Assert.Contains("app-1", coordinator.StarvedAppsReported);
}
[Fact]
public void Repeated_exits_emit_once_per_appId()
{
using var f = new Fixture();
using var coordinator = f.BuildCoordinator(
() => false,
() => new[] { StarvedApp("app-1") });
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5));
Assert.Single(f.StarvedPosts());
}
[Fact]
public void New_starved_app_on_later_exit_is_reported_additively()
{
using var f = new Fixture();
var starved = new List<AppPackageState> { StarvedApp("app-1") };
using var coordinator = f.BuildCoordinator(() => false, () => starved);
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
starved.Add(StarvedApp("app-2"));
coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5));
var posts = f.StarvedPosts();
Assert.Equal(2, posts.Count);
Assert.Equal(2, coordinator.StarvedAppsReported.Count);
}
[Fact]
public void Settled_exit_emits_no_starved_events()
{
// When the probe settles, the synthesis path runs instead — starved apps and a
// settled gate are mutually exclusive by the IME probe's construction anyway.
using var f = new Fixture();
using var coordinator = f.BuildCoordinator(
() => true,
() => new[] { StarvedApp("app-1") });
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
Assert.Empty(f.StarvedPosts());
}
[Fact]
public void Throwing_starved_probe_is_swallowed()
{
using var f = new Fixture();
using var coordinator = f.BuildCoordinator(
() => false,
() => throw new InvalidOperationException("probe boom"));
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
var posted = Assert.Single(f.Ingress.Posted);
Assert.Equal(DecisionSignalKind.EspExiting, posted.Kind);
Assert.Empty(f.StarvedPosts());
}
[Fact]
public void Default_probe_emits_nothing()
{
using var f = new Fixture();
using var coordinator = new EspAndHelloTracker(
sessionId: "S1",
tenantId: "T1",
post: new InformationalEventPost(f.TrackerPostSink, f.Clock),
logger: f.Logger,
skipConfigProbe: () => ((bool?)false, (bool?)false),
accountSetupActivityProbe: () => true,
userEspAppsSettledProbe: () => false);
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
coordinator.TriggerEspExitedForTest(Fixed);
Assert.Empty(f.StarvedPosts());
Assert.Empty(coordinator.StarvedAppsReported);
}
}
}
@@ -0,0 +1,150 @@
using System.Collections.Generic;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Shared.Models;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
{
/// <summary>
/// Liveness plan PR3 — <see cref="ImeLogTracker.GetStarvedUserEspApps"/> semantics. A
/// starved app is REQUIRED (Install/Uninstall intent), tracked in the current AccountSetup
/// phase, has never shown download/install activity, and is neither terminal nor failed.
/// Apps that are alive (Downloading/Installing or any prior progress) and error-state apps
/// (owned by the failure path) must never be reported.
/// </summary>
public sealed class ImeLogTrackerStarvedUserEspAppsTests
{
private static ImeLogTracker BuildTracker(TempDirectory tmp) =>
new ImeLogTracker(
logFolder: tmp.Path,
patterns: new List<ImeLogPattern>(),
logger: new AgentLogger(tmp.Path, AgentLogLevel.Info));
private static AppPackageState AddApp(
ImeLogTracker tracker,
string id,
AppIntent intent,
AppInstallationState state)
{
var pkg = tracker.PackageStates.GetPackage(id, createIfNotFound: true);
pkg.UpdateIntent(intent);
if (state == AppInstallationState.Installed)
{
pkg.UpdateState(AppInstallationState.Installing);
}
if (state != AppInstallationState.Unknown)
{
pkg.UpdateState(state);
}
return pkg;
}
[Fact]
public void Pending_required_app_is_starved()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
var starved = tracker.GetStarvedUserEspApps();
var app = Assert.Single(starved);
Assert.Equal("app-pending", app.Id);
}
[Fact]
public void InProgress_without_download_activity_is_starved()
{
// IME announced the app (policy line → InProgress) but enforcement never started —
// exactly the field shape that starved the gate (user-targeted app on "pending").
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-announced", AppIntent.Install, AppInstallationState.InProgress);
var starved = tracker.GetStarvedUserEspApps();
var app = Assert.Single(starved);
Assert.Equal("app-announced", app.Id);
}
[Fact]
public void Downloading_and_Installing_apps_are_alive_not_starved()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-downloading", AppIntent.Install, AppInstallationState.Downloading);
AddApp(tracker, "app-installing", AppIntent.Install, AppInstallationState.Installing);
Assert.Empty(tracker.GetStarvedUserEspApps());
}
[Fact]
public void Terminal_and_error_apps_are_not_starved()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed);
AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped);
// Error apps belong to the failure path (app_install_failed names them already).
AddApp(tracker, "app-failed", AppIntent.Install, AppInstallationState.Error);
Assert.Empty(tracker.GetStarvedUserEspApps());
}
[Fact]
public void Non_required_apps_are_not_starved()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-available", AppIntent.Available, AppInstallationState.Unknown);
AddApp(tracker, "app-unknown-intent", AppIntent.Unknown, AppInstallationState.Unknown);
Assert.Empty(tracker.GetStarvedUserEspApps());
}
[Fact]
public void Outside_AccountSetup_phase_returns_empty()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("DeviceSetup");
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
Assert.Empty(tracker.GetStarvedUserEspApps());
}
[Fact]
public void Phase_never_detected_returns_empty()
{
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
Assert.Empty(tracker.GetStarvedUserEspApps());
}
[Fact]
public void Mixed_list_reports_only_the_starved_app()
{
// The field shape: one app installed, one skipped, one never started.
using var tmp = new TempDirectory();
var tracker = BuildTracker(tmp);
tracker.SeedCurrentPhaseForTesting("AccountSetup");
AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed);
AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped);
AddApp(tracker, "app-starving", AppIntent.Install, AppInstallationState.Unknown);
var starved = tracker.GetStarvedUserEspApps();
var app = Assert.Single(starved);
Assert.Equal("app-starving", app.Id);
}
}
}
@@ -159,6 +159,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
// non-Apps gating test assigns a snapshot whose FailedSubcategory is NOT
// "Apps".
public EspTerminalFailureSnapshot? LastEspTerminalFailureOverride { get; set; } = null;
// Liveness plan PR3 — starved-apps surfaces for the terminal sweep tests.
public IReadOnlyList<AppPackageState>? StarvedAppsOverride { get; set; } = null;
public IReadOnlyCollection<string> StarvedAlreadyReportedOverride { get; set; } = Array.Empty<string>();
public List<string> TerminationActionLog { get; } = new List<string>();
public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero;
// Shutdown-gap closure (2026-05-15) — when set, the constructed handler shares
@@ -223,6 +227,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
}
public EspTerminalFailureSnapshot? LastEspTerminalFailure => _rig.LastEspTerminalFailureOverride;
public IReadOnlyList<AppPackageState>? GetStarvedUserEspApps() => _rig.StarvedAppsOverride;
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported => _rig.StarvedAlreadyReportedOverride;
}
/// <summary>
@@ -1079,6 +1087,77 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
Assert.Empty(rig.PromotionCalls);
}
// ----------------------------------------------------------------
// Liveness plan PR3: app_install_starved terminal sweep
// ----------------------------------------------------------------
private static AppPackageState BuildStarvedApp(string id, string name)
{
var pkg = new AppPackageState(id, listPos: 0);
pkg.UpdateIntent(AppIntent.Install);
pkg.UpdateName(name);
return pkg;
}
[Fact]
public void StarvedApps_at_termination_emit_one_warning_per_app()
{
using var rig = new Rig();
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.EspAccountSetup));
Assert.Contains("app_install_starved", rig.EmittedEventTypes);
var post = rig.Ingress.Posted.Single(p =>
p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == "app_install_starved");
Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]);
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(post.TypedPayload);
Assert.Equal("app-starving", data["appId"]);
Assert.Equal("termination", data["trigger"]);
Assert.Equal("TimedOut", data["terminationOutcome"]);
}
[Fact]
public void StarvedApps_already_reported_by_live_path_are_skipped()
{
using var rig = new Rig();
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
rig.StarvedAlreadyReportedOverride = new[] { "app-starving" };
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed));
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
}
[Fact]
public void No_starved_apps_emits_nothing()
{
using var rig = new Rig();
rig.StarvedAppsOverride = null; // no IME surface
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
}
[Fact]
public void StarvedApps_skipped_on_whiteglove_part1()
{
// Part 1 has no user session — user apps haven't run yet; Part 2 is where they land.
using var rig = new Rig();
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
rig.Build().Handle(sender: null!,
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed));
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
}
[Fact]
public void PromoteActiveInstalls_skipped_on_effect_infrastructure_failure()
{
@@ -290,6 +290,47 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
}
}
/// <summary>
/// Liveness plan PR3 — returns the required user-ESP apps that are STARVING the
/// AccountSetup apps gate: tracked in the current AccountSetup phase, Install/Uninstall
/// intent, no download/install activity ever observed, not terminal and not failed.
/// These are the apps whose enforcement IME never started (e.g. a user-targeted app
/// stuck on "pending") — the actual blocker behind the gate-starvation dead-ends.
/// <para>
/// Deliberately excludes apps that are alive (<see cref="AppPackageState.DownloadingOrInstallingSeen"/>
/// covers Downloading/Installing and anything that ever progressed) and error-state apps
/// (the failure path already names those). Same thread-safety contract as
/// <see cref="AreUserEspAppsSettled"/>: snapshot copy, any race-induced exception
/// returns an empty list.
/// </para>
/// </summary>
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps()
{
try
{
if (!string.Equals(_lastEspPhaseDetected, "AccountSetup", StringComparison.OrdinalIgnoreCase))
return Array.Empty<AppPackageState>();
var apps = new List<AppPackageState>(_packageStates);
if (apps.Count == 0) return Array.Empty<AppPackageState>();
var starved = new List<AppPackageState>();
foreach (var pkg in apps)
{
if (pkg == null || !pkg.IsRequired) continue;
if (pkg.IsCompleted || pkg.IsError) continue;
if (pkg.DownloadingOrInstallingSeen) continue;
starved.Add(pkg);
}
return starved;
}
catch (Exception ex)
{
_logger.Debug($"ImeLogTracker: GetStarvedUserEspApps probe threw: {ex.Message}");
return Array.Empty<AppPackageState>();
}
}
/// <summary>
/// Promotes every app currently in <see cref="AppInstallationState.Installing"/> to
/// <see cref="AppInstallationState.Error"/>, stamping <paramref name="failureType"/>
@@ -40,11 +40,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
private readonly Func<(bool? skipUser, bool? skipDevice)> _skipConfigProbe;
private readonly Func<bool> _accountSetupActivityProbe;
private readonly Func<bool> _userEspAppsSettledProbe;
private readonly Func<System.Collections.Generic.IReadOnlyList<Ime.AppPackageState>> _starvedUserEspAppsProbe;
// Session caa6cf50 gate-starvation fix (2026-06-11) — fire-once guard for the
// user-ESP-apps-settled AccountSetup synthesis (see MaybeSynthesizeAccountSetupComplete).
private bool _userAppsSettledSynthesisFired;
// Liveness plan PR3 — one-shot-per-appId dedupe for app_install_starved emissions.
// Written from the Shell-Core watcher thread (esp_exited path), read at termination by
// the EnrollmentTerminationHandler via StarvedAppsReported — guarded by _starvedLock.
private readonly System.Collections.Generic.HashSet<string> _starvedAppsReported =
new System.Collections.Generic.HashSet<string>(StringComparer.OrdinalIgnoreCase);
private readonly object _starvedLock = new object();
private HelloTracker _helloTracker;
private ShellCoreTracker _shellCoreTracker;
private ProvisioningStatusTracker _provisioningTracker;
@@ -195,7 +203,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
int[] modernDeploymentHarmlessEventIds = null,
Func<(bool? skipUser, bool? skipDevice)> skipConfigProbe = null,
Func<bool> accountSetupActivityProbe = null,
Func<bool> userEspAppsSettledProbe = null)
Func<bool> userEspAppsSettledProbe = null,
Func<System.Collections.Generic.IReadOnlyList<Ime.AppPackageState>> starvedUserEspAppsProbe = null)
{
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
@@ -218,6 +227,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
// probe (wired to ImeLogHost.AreUserEspAppsSettled by DefaultComponentFactory).
// Defaults to "not settled" so single-tracker wiring scenarios keep prior behaviour.
_userEspAppsSettledProbe = userEspAppsSettledProbe ?? (() => false);
// Liveness plan PR3: IME-side starved-apps probe (wired to
// ImeLogHost.GetStarvedUserEspApps by DefaultComponentFactory). Defaults to empty
// so single-tracker wiring scenarios never emit app_install_starved.
_starvedUserEspAppsProbe = starvedUserEspAppsProbe
?? (() => Array.Empty<Ime.AppPackageState>());
}
/// <summary>
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
/// live (esp_exited) path. Read by the <c>EnrollmentTerminationHandler</c> terminal
/// sweep so the two emission paths never double-report an app. Snapshot copy.
/// </summary>
public System.Collections.Generic.IReadOnlyCollection<string> StarvedAppsReported
{
get
{
lock (_starvedLock)
{
return new System.Collections.Generic.List<string>(_starvedAppsReported);
}
}
}
// =====================================================================
@@ -618,7 +648,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_logger.Debug($"EspAndHelloTracker: user-ESP-apps-settled probe threw: {ex.Message}");
return;
}
if (!settled) return;
if (!settled)
{
// Liveness plan PR3: the ESP page is gone but the user-apps gate is NOT settled
// — name the app(s) that never started installing instead of leaving the
// operator with an anonymous "session hangs in AccountSetup". One-shot per
// appId; apps that are alive (Downloading/Installing) or failed are excluded
// by the probe itself.
EmitStarvedUserEspApps(trigger: "esp_exited_user_apps_not_settled");
return;
}
_userAppsSettledSynthesisFired = true;
_logger.Warning(
@@ -650,6 +689,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (user-apps-settled synthesis)", ex); }
}
/// <summary>
/// Liveness plan PR3 — emit a one-shot <c>app_install_starved</c> Warning per starved
/// user-ESP app (required, never started installing, not failed). Best-effort: probe or
/// emit failures are logged and swallowed; a failed emit keeps the appId out of the
/// dedupe set so the termination sweep can retry.
/// </summary>
private void EmitStarvedUserEspApps(string trigger)
{
System.Collections.Generic.IReadOnlyList<Ime.AppPackageState> starved;
try { starved = _starvedUserEspAppsProbe(); }
catch (Exception ex)
{
_logger.Debug($"EspAndHelloTracker: starved-apps probe threw: {ex.Message}");
return;
}
if (starved == null || starved.Count == 0) return;
foreach (var app in starved)
{
if (app?.Id == null) continue;
lock (_starvedLock)
{
if (!_starvedAppsReported.Add(app.Id)) continue;
}
try
{
var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
_logger.Warning(
$"EspAndHelloTracker: required user-ESP app '{name}' ({app.Id}) never started installing " +
$"(state={app.InstallationState}) — starving the AccountSetup apps gate (trigger={trigger}).");
_post.Emit(new EnrollmentEvent
{
SessionId = _sessionId,
TenantId = _tenantId,
EventType = Constants.EventTypes.AppInstallStarved,
Severity = EventSeverity.Warning,
Source = "EspAndHelloTracker",
Phase = EnrollmentPhase.Unknown,
Message = $"Required app '{name}' never started installing while the ESP AccountSetup " +
"apps gate waited on it — the app is starving the enrollment completion.",
Data = new Dictionary<string, object>
{
{ "appId", app.Id },
{ "appName", name },
{ "state", app.InstallationState.ToString() },
{ "intent", app.Intent.ToString() },
{ "targeted", app.Targeted.ToString() },
{ "trigger", trigger },
},
ImmediateUpload = true,
});
}
catch (Exception ex)
{
_logger.Error($"EspAndHelloTracker: app_install_starved emit failed for '{app.Id}'", ex);
lock (_starvedLock) { _starvedAppsReported.Remove(app.Id); }
}
}
}
// Test seam for the user-apps-settled synthesis — drives OnEspExited with the live
// handler signature, mirroring TriggerEspExitedForTest's contract.
internal bool UserAppsSettledSynthesisFiredForTest => _userAppsSettledSynthesisFired;
@@ -113,5 +113,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
/// was observed.
/// </summary>
public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => EspAndHelloHost?.LastEspTerminalFailure;
/// <summary>
/// Liveness plan PR3 — required user-ESP apps that never started installing (current
/// AccountSetup phase). Empty when no IME host exists.
/// </summary>
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps() =>
_imeLogHost?.GetStarvedUserEspApps() ?? Array.Empty<AppPackageState>();
/// <summary>
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
/// live (esp_exited) path. Consulted by the termination handler so the terminal sweep
/// never double-reports an app. Empty when no ESP/Hello host exists.
/// </summary>
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported =>
EspAndHelloHost?.StarvedAppsReported ?? Array.Empty<string>();
}
}
@@ -121,7 +121,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
modernDeploymentBackfillLookbackMinutes: collectors.ModernDeploymentBackfillLookbackMinutes,
modernDeploymentHarmlessEventIds: collectors.ModernDeploymentHarmlessEventIds,
stateDirectory: _stateDirectory,
userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true);
userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true,
// Liveness plan PR3: starved-apps probe over the same lazily-assigned IME host.
starvedUserEspAppsProbe: () =>
imeLogHostRef?.GetStarvedUserEspApps()
?? Array.Empty<Monitoring.Enrollment.Ime.AppPackageState>());
hosts.Add(espAndHelloHost);
// Hybrid User-Driven completion-gap fix (2026-05-01): the AadJoinHost notifies
@@ -48,6 +48,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
/// </summary>
public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => _tracker.LastEspTerminalFailure;
/// <summary>
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
/// live path. Read by <c>CollectorSurfaces</c> for the termination handler's dedupe.
/// </summary>
public System.Collections.Generic.IReadOnlyCollection<string> StarvedAppsReported => _tracker.StarvedAppsReported;
public EspAndHelloHost(
string sessionId,
string tenantId,
@@ -61,7 +67,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
int modernDeploymentBackfillLookbackMinutes,
int[]? modernDeploymentHarmlessEventIds,
string stateDirectory,
Func<bool>? userEspAppsSettledProbe = null)
Func<bool>? userEspAppsSettledProbe = null,
Func<System.Collections.Generic.IReadOnlyList<Monitoring.Enrollment.Ime.AppPackageState>>? starvedUserEspAppsProbe = null)
{
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
if (clock == null) throw new ArgumentNullException(nameof(clock));
@@ -78,7 +85,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
modernDeploymentBackfillLookbackMinutes: modernDeploymentBackfillLookbackMinutes,
stateDirectory: stateDirectory,
modernDeploymentHarmlessEventIds: modernDeploymentHarmlessEventIds,
userEspAppsSettledProbe: userEspAppsSettledProbe);
userEspAppsSettledProbe: userEspAppsSettledProbe,
starvedUserEspAppsProbe: starvedUserEspAppsProbe);
_tracker.WhiteGloveCompleted += OnTrackerWhiteGloveCompleted;
_tracker.DeviceSetupProvisioningComplete += OnTrackerDeviceSetupProvisioningComplete;
@@ -60,6 +60,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
/// </summary>
public bool AreUserEspAppsSettled() => _tracker.AreUserEspAppsSettled();
/// <summary>
/// Liveness plan PR3 — bridge for the <c>EspAndHelloTracker</c> starved-apps probe and
/// the <c>EnrollmentTerminationHandler</c> terminal sweep. See
/// <see cref="ImeLogTracker.GetStarvedUserEspApps"/>.
/// </summary>
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps() => _tracker.GetStarvedUserEspApps();
/// <summary>
/// c117946b debrief (2026-05-12) — bridge for the <c>EnrollmentTerminationHandler</c>
/// pre-hook to promote apps still in <see cref="AppInstallationState.Installing"/>
@@ -181,6 +181,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
// the new Error counts via the `likelyStuckNames` bucket.
MaybePromoteActiveInstallsAsStuck(state, args);
// Liveness plan PR3 — leftover starved apps (required, never started
// installing) at termination get their one-shot app_install_starved
// BEFORE the summary so both reach the backend in the terminal flush.
// Runs on every outcome; the live (esp_exited) path's dedupe set is
// consulted so no app is reported twice. Promoted likely-stuck apps are
// Error by now and excluded by the probe itself.
EmitStarvedUserEspApps(args);
EmitAppTrackingSummary();
}
@@ -402,6 +410,65 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
}
}
/// <summary>
/// Liveness plan PR3 — terminal sweep for <c>app_install_starved</c>. Emits a one-shot
/// Warning per required user-ESP app that never started installing and was not already
/// reported by the live (esp_exited) path in <c>EspAndHelloTracker</c>. Best-effort:
/// any accessor / emit failure is logged and swallowed.
/// </summary>
private void EmitStarvedUserEspApps(EnrollmentTerminatedEventArgs args)
{
if (_post == null) return;
try
{
var starved = _appTracking.GetStarvedUserEspApps();
if (starved == null || starved.Count == 0) return;
var alreadyReported = _appTracking.StarvedUserEspAppsAlreadyReported
?? (IReadOnlyCollection<string>)Array.Empty<string>();
var reported = new HashSet<string>(alreadyReported, StringComparer.OrdinalIgnoreCase);
foreach (var app in starved)
{
if (app?.Id == null || !reported.Add(app.Id)) continue;
var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
_logger.Warning(
$"EnrollmentTerminationHandler: required user-ESP app '{name}' ({app.Id}) never started " +
$"installing (state={app.InstallationState}) — reporting app_install_starved at termination " +
$"(outcome={args.Outcome}).");
_post.Emit(new EnrollmentEvent
{
SessionId = _configuration.SessionId,
TenantId = _configuration.TenantId,
EventType = Constants.EventTypes.AppInstallStarved,
Severity = EventSeverity.Warning,
Source = "EnrollmentTerminationHandler",
Phase = EnrollmentPhase.Unknown,
Message = $"Required app '{name}' never started installing while the ESP AccountSetup " +
"apps gate waited on it — the app starved the enrollment completion.",
Data = new Dictionary<string, object>
{
{ "appId", app.Id },
{ "appName", name },
{ "state", app.InstallationState.ToString() },
{ "intent", app.Intent.ToString() },
{ "targeted", app.Targeted.ToString() },
{ "trigger", "termination" },
{ "terminationOutcome", args.Outcome.ToString() },
},
ImmediateUpload = true,
});
}
}
catch (Exception ex)
{
_logger.Warning($"EnrollmentTerminationHandler: app_install_starved terminal sweep failed: {ex.Message}");
}
}
/// <summary>
/// c117946b debrief (2026-05-12) — pre-hook for <see cref="EmitAppTrackingSummary"/>.
/// On the terminal-ESP-Apps-failure path, promote every app still in
@@ -73,5 +73,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
/// mis-classify in-flight installs. Null = no HRESULT context → fallback.
/// </summary>
EspTerminalFailureSnapshot? LastEspTerminalFailure { get; }
/// <summary>
/// Liveness plan PR3 — required user-ESP apps that never started installing while the
/// AccountSetup apps gate waited on them (no download/install activity, not failed,
/// not terminal). The termination handler emits a one-shot <c>app_install_starved</c>
/// Warning per leftover app on every terminal outcome. May return null / empty when no
/// IME surface exists.
/// </summary>
IReadOnlyList<AppPackageState>? GetStarvedUserEspApps();
/// <summary>
/// Liveness plan PR3 — appIds the live (esp_exited) path already reported via
/// <c>app_install_starved</c>. The terminal sweep skips these so the two emission
/// paths never double-report an app.
/// </summary>
IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported { get; }
}
}
@@ -39,5 +39,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
public EspTerminalFailureSnapshot? LastEspTerminalFailure =>
_orchestrator.CollectorSurfaces?.LastEspTerminalFailure;
public IReadOnlyList<AppPackageState>? GetStarvedUserEspApps() =>
_orchestrator.CollectorSurfaces?.GetStarvedUserEspApps();
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported =>
_orchestrator.CollectorSurfaces?.StarvedUserEspAppsAlreadyReported ?? Array.Empty<string>();
}
}
@@ -60,6 +60,7 @@ export const EVENT_TYPES_CATALOG = {
'app_install_failed',
'app_download_started',
'app_install_skipped',
'app_install_starved',
'download_progress',
'do_telemetry',
'all_apps_completed',
@@ -247,6 +247,12 @@ namespace AutopilotMonitor.Shared
public const string AppInstallFailed = "app_install_failed";
public const string AppDownloadStarted = "app_download_started";
public const string AppInstallSkipped = "app_install_skipped";
// Liveness plan PR3 — one-shot Warning per app naming a required user-ESP app that
// NEVER started installing (no download/install activity seen) while the ESP
// AccountSetup apps gate waited on it. Emitted when the user-apps-settled probe
// returns false after the ESP page exited, and again at termination for leftovers.
// This is the actionable customer hint behind "session hangs in AccountSetup".
public const string AppInstallStarved = "app_install_starved";
public const string NetworkStateChange = "network_state_change";
public const string NetworkConnectivityCheck = "network_connectivity_check";
public const string ErrorDetected = "error_detected";
@@ -80,6 +80,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [
description: "An application installation failed." },
{ value: "app_install_skipped", label: "app_install_skipped", category: "app",
description: "An application installation was skipped." },
{ value: "app_install_starved", label: "app_install_starved", category: "app",
description: "A required user-ESP app never started installing while the AccountSetup apps gate waited on it (one-shot per app)." },
{ value: "app_download_started", label: "app_download_started", category: "app",
description: "An application download has started." },
{ value: "download_progress", label: "download_progress", category: "app",