diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerStarvedAppsTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerStarvedAppsTests.cs new file mode 100644 index 00000000..7810b11e --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/EspAndHelloTrackerStarvedAppsTests.cs @@ -0,0 +1,174 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals; +using AutopilotMonitor.Agent.V2.Core.Orchestration; +using AutopilotMonitor.Agent.V2.Core.SignalAdapters; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring +{ + /// + /// Liveness plan PR3 — live-path app_install_starved emission. When the ESP page + /// exits while the user-apps-settled probe reports false, the coordinator names the + /// starved app(s) (one-shot Warning per appId) instead of leaving the operator with an + /// anonymous stall. The dedupe set is exposed via + /// for the termination sweep. + /// + public sealed class EspAndHelloTrackerStarvedAppsTests + { + private static readonly DateTime Fixed = new DateTime(2026, 6, 12, 8, 0, 2, DateTimeKind.Utc); + + private sealed class Fixture : IDisposable + { + public TempDirectory Tmp { get; } = new TempDirectory(); + public AgentLogger Logger { get; } + public FakeSignalIngressSink Ingress { get; } = new FakeSignalIngressSink(); + public FakeSignalIngressSink TrackerPostSink { get; } = new FakeSignalIngressSink(); + public VirtualClock Clock { get; } = new VirtualClock(Fixed); + + public Fixture() { Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Debug); } + + public EspAndHelloTracker BuildCoordinator( + Func settledProbe, + Func> starvedProbe) + { + return new EspAndHelloTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(TrackerPostSink, Clock), + logger: Logger, + skipConfigProbe: () => ((bool?)false, (bool?)false), + accountSetupActivityProbe: () => true, + userEspAppsSettledProbe: settledProbe, + starvedUserEspAppsProbe: starvedProbe); + } + + public List StarvedPosts() => + TrackerPostSink.Posted.Where(p => + p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == "app_install_starved").ToList(); + + public void Dispose() { Tmp.Dispose(); } + } + + private static AppPackageState StarvedApp(string id, string? name = null) + { + var pkg = new AppPackageState(id, listPos: 0); + pkg.UpdateIntent(AppIntent.Install); + if (name != null) pkg.UpdateName(name); + return pkg; + } + + [Fact] + public void Unsettled_exit_emits_warning_per_starved_app() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator( + () => false, + () => new[] { StarvedApp("app-1", "Contoso Backgrounds") }); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + + var post = Assert.Single(f.StarvedPosts()); + Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]); + var data = Assert.IsAssignableFrom>(post.TypedPayload); + Assert.Equal("app-1", data["appId"]); + Assert.Equal("Contoso Backgrounds", data["appName"]); + Assert.Equal("esp_exited_user_apps_not_settled", data["trigger"]); + Assert.Contains("app-1", coordinator.StarvedAppsReported); + } + + [Fact] + public void Repeated_exits_emit_once_per_appId() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator( + () => false, + () => new[] { StarvedApp("app-1") }); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5)); + + Assert.Single(f.StarvedPosts()); + } + + [Fact] + public void New_starved_app_on_later_exit_is_reported_additively() + { + using var f = new Fixture(); + var starved = new List { StarvedApp("app-1") }; + using var coordinator = f.BuildCoordinator(() => false, () => starved); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + starved.Add(StarvedApp("app-2")); + coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5)); + + var posts = f.StarvedPosts(); + Assert.Equal(2, posts.Count); + Assert.Equal(2, coordinator.StarvedAppsReported.Count); + } + + [Fact] + public void Settled_exit_emits_no_starved_events() + { + // When the probe settles, the synthesis path runs instead — starved apps and a + // settled gate are mutually exclusive by the IME probe's construction anyway. + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator( + () => true, + () => new[] { StarvedApp("app-1") }); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + + Assert.Empty(f.StarvedPosts()); + } + + [Fact] + public void Throwing_starved_probe_is_swallowed() + { + using var f = new Fixture(); + using var coordinator = f.BuildCoordinator( + () => false, + () => throw new InvalidOperationException("probe boom")); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + + var posted = Assert.Single(f.Ingress.Posted); + Assert.Equal(DecisionSignalKind.EspExiting, posted.Kind); + Assert.Empty(f.StarvedPosts()); + } + + [Fact] + public void Default_probe_emits_nothing() + { + using var f = new Fixture(); + using var coordinator = new EspAndHelloTracker( + sessionId: "S1", + tenantId: "T1", + post: new InformationalEventPost(f.TrackerPostSink, f.Clock), + logger: f.Logger, + skipConfigProbe: () => ((bool?)false, (bool?)false), + accountSetupActivityProbe: () => true, + userEspAppsSettledProbe: () => false); + using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock); + + coordinator.TriggerEspExitedForTest(Fixed); + + Assert.Empty(f.StarvedPosts()); + Assert.Empty(coordinator.StarvedAppsReported); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/ImeLogTrackerStarvedUserEspAppsTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/ImeLogTrackerStarvedUserEspAppsTests.cs new file mode 100644 index 00000000..8ba1ee74 --- /dev/null +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Ime/ImeLogTrackerStarvedUserEspAppsTests.cs @@ -0,0 +1,150 @@ +using System.Collections.Generic; +using AutopilotMonitor.Agent.V2.Core.Logging; +using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime; +using AutopilotMonitor.Agent.V2.Core.Tests.Harness; +using AutopilotMonitor.Shared.Models; +using Xunit; + +namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime +{ + /// + /// Liveness plan PR3 — semantics. A + /// starved app is REQUIRED (Install/Uninstall intent), tracked in the current AccountSetup + /// phase, has never shown download/install activity, and is neither terminal nor failed. + /// Apps that are alive (Downloading/Installing or any prior progress) and error-state apps + /// (owned by the failure path) must never be reported. + /// + public sealed class ImeLogTrackerStarvedUserEspAppsTests + { + private static ImeLogTracker BuildTracker(TempDirectory tmp) => + new ImeLogTracker( + logFolder: tmp.Path, + patterns: new List(), + logger: new AgentLogger(tmp.Path, AgentLogLevel.Info)); + + private static AppPackageState AddApp( + ImeLogTracker tracker, + string id, + AppIntent intent, + AppInstallationState state) + { + var pkg = tracker.PackageStates.GetPackage(id, createIfNotFound: true); + pkg.UpdateIntent(intent); + if (state == AppInstallationState.Installed) + { + pkg.UpdateState(AppInstallationState.Installing); + } + if (state != AppInstallationState.Unknown) + { + pkg.UpdateState(state); + } + return pkg; + } + + [Fact] + public void Pending_required_app_is_starved() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown); + + var starved = tracker.GetStarvedUserEspApps(); + + var app = Assert.Single(starved); + Assert.Equal("app-pending", app.Id); + } + + [Fact] + public void InProgress_without_download_activity_is_starved() + { + // IME announced the app (policy line → InProgress) but enforcement never started — + // exactly the field shape that starved the gate (user-targeted app on "pending"). + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-announced", AppIntent.Install, AppInstallationState.InProgress); + + var starved = tracker.GetStarvedUserEspApps(); + + var app = Assert.Single(starved); + Assert.Equal("app-announced", app.Id); + } + + [Fact] + public void Downloading_and_Installing_apps_are_alive_not_starved() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-downloading", AppIntent.Install, AppInstallationState.Downloading); + AddApp(tracker, "app-installing", AppIntent.Install, AppInstallationState.Installing); + + Assert.Empty(tracker.GetStarvedUserEspApps()); + } + + [Fact] + public void Terminal_and_error_apps_are_not_starved() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed); + AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped); + // Error apps belong to the failure path (app_install_failed names them already). + AddApp(tracker, "app-failed", AppIntent.Install, AppInstallationState.Error); + + Assert.Empty(tracker.GetStarvedUserEspApps()); + } + + [Fact] + public void Non_required_apps_are_not_starved() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-available", AppIntent.Available, AppInstallationState.Unknown); + AddApp(tracker, "app-unknown-intent", AppIntent.Unknown, AppInstallationState.Unknown); + + Assert.Empty(tracker.GetStarvedUserEspApps()); + } + + [Fact] + public void Outside_AccountSetup_phase_returns_empty() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("DeviceSetup"); + AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown); + + Assert.Empty(tracker.GetStarvedUserEspApps()); + } + + [Fact] + public void Phase_never_detected_returns_empty() + { + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown); + + Assert.Empty(tracker.GetStarvedUserEspApps()); + } + + [Fact] + public void Mixed_list_reports_only_the_starved_app() + { + // The field shape: one app installed, one skipped, one never started. + using var tmp = new TempDirectory(); + var tracker = BuildTracker(tmp); + tracker.SeedCurrentPhaseForTesting("AccountSetup"); + AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed); + AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped); + AddApp(tracker, "app-starving", AppIntent.Install, AppInstallationState.Unknown); + + var starved = tracker.GetStarvedUserEspApps(); + + var app = Assert.Single(starved); + Assert.Equal("app-starving", app.Id); + } + } +} diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs index 7c2a7702..9649c4b5 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Termination/EnrollmentTerminationHandlerTests.cs @@ -159,6 +159,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination // non-Apps gating test assigns a snapshot whose FailedSubcategory is NOT // "Apps". public EspTerminalFailureSnapshot? LastEspTerminalFailureOverride { get; set; } = null; + + // Liveness plan PR3 — starved-apps surfaces for the terminal sweep tests. + public IReadOnlyList? StarvedAppsOverride { get; set; } = null; + public IReadOnlyCollection StarvedAlreadyReportedOverride { get; set; } = Array.Empty(); public List TerminationActionLog { get; } = new List(); public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero; // Shutdown-gap closure (2026-05-15) — when set, the constructed handler shares @@ -223,6 +227,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination } public EspTerminalFailureSnapshot? LastEspTerminalFailure => _rig.LastEspTerminalFailureOverride; + + public IReadOnlyList? GetStarvedUserEspApps() => _rig.StarvedAppsOverride; + + public IReadOnlyCollection StarvedUserEspAppsAlreadyReported => _rig.StarvedAlreadyReportedOverride; } /// @@ -1079,6 +1087,77 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination Assert.Empty(rig.PromotionCalls); } + // ---------------------------------------------------------------- + // Liveness plan PR3: app_install_starved terminal sweep + // ---------------------------------------------------------------- + + private static AppPackageState BuildStarvedApp(string id, string name) + { + var pkg = new AppPackageState(id, listPos: 0); + pkg.UpdateIntent(AppIntent.Install); + pkg.UpdateName(name); + return pkg; + } + + [Fact] + public void StarvedApps_at_termination_emit_one_warning_per_app() + { + using var rig = new Rig(); + rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") }; + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.EspAccountSetup)); + + Assert.Contains("app_install_starved", rig.EmittedEventTypes); + var post = rig.Ingress.Posted.Single(p => + p.Payload != null + && p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et) + && et == "app_install_starved"); + Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]); + var data = Assert.IsAssignableFrom>(post.TypedPayload); + Assert.Equal("app-starving", data["appId"]); + Assert.Equal("termination", data["trigger"]); + Assert.Equal("TimedOut", data["terminationOutcome"]); + } + + [Fact] + public void StarvedApps_already_reported_by_live_path_are_skipped() + { + using var rig = new Rig(); + rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") }; + rig.StarvedAlreadyReportedOverride = new[] { "app-starving" }; + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed)); + + Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes); + } + + [Fact] + public void No_starved_apps_emits_nothing() + { + using var rig = new Rig(); + rig.StarvedAppsOverride = null; // no IME surface + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed)); + + Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes); + } + + [Fact] + public void StarvedApps_skipped_on_whiteglove_part1() + { + // Part 1 has no user session — user apps haven't run yet; Part 2 is where they land. + using var rig = new Rig(); + rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") }; + + rig.Build().Handle(sender: null!, + Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed)); + + Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes); + } + [Fact] public void PromoteActiveInstalls_skipped_on_effect_infrastructure_failure() { diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs index 6ccf7ad6..8f113750 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/Ime/ImeLogTracker.cs @@ -290,6 +290,47 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime } } + /// + /// Liveness plan PR3 — returns the required user-ESP apps that are STARVING the + /// AccountSetup apps gate: tracked in the current AccountSetup phase, Install/Uninstall + /// intent, no download/install activity ever observed, not terminal and not failed. + /// These are the apps whose enforcement IME never started (e.g. a user-targeted app + /// stuck on "pending") — the actual blocker behind the gate-starvation dead-ends. + /// + /// Deliberately excludes apps that are alive ( + /// covers Downloading/Installing and anything that ever progressed) and error-state apps + /// (the failure path already names those). Same thread-safety contract as + /// : snapshot copy, any race-induced exception + /// returns an empty list. + /// + /// + public IReadOnlyList GetStarvedUserEspApps() + { + try + { + if (!string.Equals(_lastEspPhaseDetected, "AccountSetup", StringComparison.OrdinalIgnoreCase)) + return Array.Empty(); + + var apps = new List(_packageStates); + if (apps.Count == 0) return Array.Empty(); + + var starved = new List(); + foreach (var pkg in apps) + { + if (pkg == null || !pkg.IsRequired) continue; + if (pkg.IsCompleted || pkg.IsError) continue; + if (pkg.DownloadingOrInstallingSeen) continue; + starved.Add(pkg); + } + return starved; + } + catch (Exception ex) + { + _logger.Debug($"ImeLogTracker: GetStarvedUserEspApps probe threw: {ex.Message}"); + return Array.Empty(); + } + } + /// /// Promotes every app currently in to /// , stamping diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs index 68c08046..43f5b788 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Enrollment/SystemSignals/EspAndHelloTracker.cs @@ -40,11 +40,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals private readonly Func<(bool? skipUser, bool? skipDevice)> _skipConfigProbe; private readonly Func _accountSetupActivityProbe; private readonly Func _userEspAppsSettledProbe; + private readonly Func> _starvedUserEspAppsProbe; // Session caa6cf50 gate-starvation fix (2026-06-11) — fire-once guard for the // user-ESP-apps-settled AccountSetup synthesis (see MaybeSynthesizeAccountSetupComplete). private bool _userAppsSettledSynthesisFired; + // Liveness plan PR3 — one-shot-per-appId dedupe for app_install_starved emissions. + // Written from the Shell-Core watcher thread (esp_exited path), read at termination by + // the EnrollmentTerminationHandler via StarvedAppsReported — guarded by _starvedLock. + private readonly System.Collections.Generic.HashSet _starvedAppsReported = + new System.Collections.Generic.HashSet(StringComparer.OrdinalIgnoreCase); + private readonly object _starvedLock = new object(); + private HelloTracker _helloTracker; private ShellCoreTracker _shellCoreTracker; private ProvisioningStatusTracker _provisioningTracker; @@ -195,7 +203,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals int[] modernDeploymentHarmlessEventIds = null, Func<(bool? skipUser, bool? skipDevice)> skipConfigProbe = null, Func accountSetupActivityProbe = null, - Func userEspAppsSettledProbe = null) + Func userEspAppsSettledProbe = null, + Func> starvedUserEspAppsProbe = null) { _sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId)); _tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId)); @@ -218,6 +227,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals // probe (wired to ImeLogHost.AreUserEspAppsSettled by DefaultComponentFactory). // Defaults to "not settled" so single-tracker wiring scenarios keep prior behaviour. _userEspAppsSettledProbe = userEspAppsSettledProbe ?? (() => false); + // Liveness plan PR3: IME-side starved-apps probe (wired to + // ImeLogHost.GetStarvedUserEspApps by DefaultComponentFactory). Defaults to empty + // so single-tracker wiring scenarios never emit app_install_starved. + _starvedUserEspAppsProbe = starvedUserEspAppsProbe + ?? (() => Array.Empty()); + } + + /// + /// Liveness plan PR3 — appIds already reported via app_install_starved on the + /// live (esp_exited) path. Read by the EnrollmentTerminationHandler terminal + /// sweep so the two emission paths never double-report an app. Snapshot copy. + /// + public System.Collections.Generic.IReadOnlyCollection StarvedAppsReported + { + get + { + lock (_starvedLock) + { + return new System.Collections.Generic.List(_starvedAppsReported); + } + } } // ===================================================================== @@ -618,7 +648,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals _logger.Debug($"EspAndHelloTracker: user-ESP-apps-settled probe threw: {ex.Message}"); return; } - if (!settled) return; + if (!settled) + { + // Liveness plan PR3: the ESP page is gone but the user-apps gate is NOT settled + // — name the app(s) that never started installing instead of leaving the + // operator with an anonymous "session hangs in AccountSetup". One-shot per + // appId; apps that are alive (Downloading/Installing) or failed are excluded + // by the probe itself. + EmitStarvedUserEspApps(trigger: "esp_exited_user_apps_not_settled"); + return; + } _userAppsSettledSynthesisFired = true; _logger.Warning( @@ -650,6 +689,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (user-apps-settled synthesis)", ex); } } + /// + /// Liveness plan PR3 — emit a one-shot app_install_starved Warning per starved + /// user-ESP app (required, never started installing, not failed). Best-effort: probe or + /// emit failures are logged and swallowed; a failed emit keeps the appId out of the + /// dedupe set so the termination sweep can retry. + /// + private void EmitStarvedUserEspApps(string trigger) + { + System.Collections.Generic.IReadOnlyList starved; + try { starved = _starvedUserEspAppsProbe(); } + catch (Exception ex) + { + _logger.Debug($"EspAndHelloTracker: starved-apps probe threw: {ex.Message}"); + return; + } + if (starved == null || starved.Count == 0) return; + + foreach (var app in starved) + { + if (app?.Id == null) continue; + lock (_starvedLock) + { + if (!_starvedAppsReported.Add(app.Id)) continue; + } + + try + { + var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name; + _logger.Warning( + $"EspAndHelloTracker: required user-ESP app '{name}' ({app.Id}) never started installing " + + $"(state={app.InstallationState}) — starving the AccountSetup apps gate (trigger={trigger})."); + + _post.Emit(new EnrollmentEvent + { + SessionId = _sessionId, + TenantId = _tenantId, + EventType = Constants.EventTypes.AppInstallStarved, + Severity = EventSeverity.Warning, + Source = "EspAndHelloTracker", + Phase = EnrollmentPhase.Unknown, + Message = $"Required app '{name}' never started installing while the ESP AccountSetup " + + "apps gate waited on it — the app is starving the enrollment completion.", + Data = new Dictionary + { + { "appId", app.Id }, + { "appName", name }, + { "state", app.InstallationState.ToString() }, + { "intent", app.Intent.ToString() }, + { "targeted", app.Targeted.ToString() }, + { "trigger", trigger }, + }, + ImmediateUpload = true, + }); + } + catch (Exception ex) + { + _logger.Error($"EspAndHelloTracker: app_install_starved emit failed for '{app.Id}'", ex); + lock (_starvedLock) { _starvedAppsReported.Remove(app.Id); } + } + } + } + // Test seam for the user-apps-settled synthesis — drives OnEspExited with the live // handler signature, mirroring TriggerEspExitedForTest's contract. internal bool UserAppsSettledSynthesisFiredForTest => _userAppsSettledSynthesisFired; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/CollectorSurfaces.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/CollectorSurfaces.cs index a0af0869..978e1a70 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/CollectorSurfaces.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/CollectorSurfaces.cs @@ -113,5 +113,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// was observed. /// public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => EspAndHelloHost?.LastEspTerminalFailure; + + /// + /// Liveness plan PR3 — required user-ESP apps that never started installing (current + /// AccountSetup phase). Empty when no IME host exists. + /// + public IReadOnlyList GetStarvedUserEspApps() => + _imeLogHost?.GetStarvedUserEspApps() ?? Array.Empty(); + + /// + /// Liveness plan PR3 — appIds already reported via app_install_starved on the + /// live (esp_exited) path. Consulted by the termination handler so the terminal sweep + /// never double-reports an app. Empty when no ESP/Hello host exists. + /// + public IReadOnlyCollection StarvedUserEspAppsAlreadyReported => + EspAndHelloHost?.StarvedAppsReported ?? Array.Empty(); } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs index b0d34f0b..e68f49d2 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/DefaultComponentFactory.cs @@ -121,7 +121,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration modernDeploymentBackfillLookbackMinutes: collectors.ModernDeploymentBackfillLookbackMinutes, modernDeploymentHarmlessEventIds: collectors.ModernDeploymentHarmlessEventIds, stateDirectory: _stateDirectory, - userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true); + userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true, + // Liveness plan PR3: starved-apps probe over the same lazily-assigned IME host. + starvedUserEspAppsProbe: () => + imeLogHostRef?.GetStarvedUserEspApps() + ?? Array.Empty()); hosts.Add(espAndHelloHost); // Hybrid User-Driven completion-gap fix (2026-05-01): the AadJoinHost notifies diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspAndHelloHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspAndHelloHost.cs index e90f5885..3a7ff1ed 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspAndHelloHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/EspAndHelloHost.cs @@ -48,6 +48,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => _tracker.LastEspTerminalFailure; + /// + /// Liveness plan PR3 — appIds already reported via app_install_starved on the + /// live path. Read by CollectorSurfaces for the termination handler's dedupe. + /// + public System.Collections.Generic.IReadOnlyCollection StarvedAppsReported => _tracker.StarvedAppsReported; + public EspAndHelloHost( string sessionId, string tenantId, @@ -61,7 +67,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration int modernDeploymentBackfillLookbackMinutes, int[]? modernDeploymentHarmlessEventIds, string stateDirectory, - Func? userEspAppsSettledProbe = null) + Func? userEspAppsSettledProbe = null, + Func>? starvedUserEspAppsProbe = null) { if (ingress == null) throw new ArgumentNullException(nameof(ingress)); if (clock == null) throw new ArgumentNullException(nameof(clock)); @@ -78,7 +85,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration modernDeploymentBackfillLookbackMinutes: modernDeploymentBackfillLookbackMinutes, stateDirectory: stateDirectory, modernDeploymentHarmlessEventIds: modernDeploymentHarmlessEventIds, - userEspAppsSettledProbe: userEspAppsSettledProbe); + userEspAppsSettledProbe: userEspAppsSettledProbe, + starvedUserEspAppsProbe: starvedUserEspAppsProbe); _tracker.WhiteGloveCompleted += OnTrackerWhiteGloveCompleted; _tracker.DeviceSetupProvisioningComplete += OnTrackerDeviceSetupProvisioningComplete; diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/ImeLogHost.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/ImeLogHost.cs index 08df1ff6..8bf8bbeb 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/ImeLogHost.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Orchestration/Hosts/ImeLogHost.cs @@ -60,6 +60,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration /// public bool AreUserEspAppsSettled() => _tracker.AreUserEspAppsSettled(); + /// + /// Liveness plan PR3 — bridge for the EspAndHelloTracker starved-apps probe and + /// the EnrollmentTerminationHandler terminal sweep. See + /// . + /// + public IReadOnlyList GetStarvedUserEspApps() => _tracker.GetStarvedUserEspApps(); + /// /// c117946b debrief (2026-05-12) — bridge for the EnrollmentTerminationHandler /// pre-hook to promote apps still in diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs index d3c1c497..03812ba5 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/EnrollmentTerminationHandler.cs @@ -181,6 +181,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination // the new Error counts via the `likelyStuckNames` bucket. MaybePromoteActiveInstallsAsStuck(state, args); + // Liveness plan PR3 — leftover starved apps (required, never started + // installing) at termination get their one-shot app_install_starved + // BEFORE the summary so both reach the backend in the terminal flush. + // Runs on every outcome; the live (esp_exited) path's dedupe set is + // consulted so no app is reported twice. Promoted likely-stuck apps are + // Error by now and excluded by the probe itself. + EmitStarvedUserEspApps(args); + EmitAppTrackingSummary(); } @@ -402,6 +410,65 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination } } + /// + /// Liveness plan PR3 — terminal sweep for app_install_starved. Emits a one-shot + /// Warning per required user-ESP app that never started installing and was not already + /// reported by the live (esp_exited) path in EspAndHelloTracker. Best-effort: + /// any accessor / emit failure is logged and swallowed. + /// + private void EmitStarvedUserEspApps(EnrollmentTerminatedEventArgs args) + { + if (_post == null) return; + + try + { + var starved = _appTracking.GetStarvedUserEspApps(); + if (starved == null || starved.Count == 0) return; + + var alreadyReported = _appTracking.StarvedUserEspAppsAlreadyReported + ?? (IReadOnlyCollection)Array.Empty(); + var reported = new HashSet(alreadyReported, StringComparer.OrdinalIgnoreCase); + + foreach (var app in starved) + { + if (app?.Id == null || !reported.Add(app.Id)) continue; + + var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name; + _logger.Warning( + $"EnrollmentTerminationHandler: required user-ESP app '{name}' ({app.Id}) never started " + + $"installing (state={app.InstallationState}) — reporting app_install_starved at termination " + + $"(outcome={args.Outcome})."); + + _post.Emit(new EnrollmentEvent + { + SessionId = _configuration.SessionId, + TenantId = _configuration.TenantId, + EventType = Constants.EventTypes.AppInstallStarved, + Severity = EventSeverity.Warning, + Source = "EnrollmentTerminationHandler", + Phase = EnrollmentPhase.Unknown, + Message = $"Required app '{name}' never started installing while the ESP AccountSetup " + + "apps gate waited on it — the app starved the enrollment completion.", + Data = new Dictionary + { + { "appId", app.Id }, + { "appName", name }, + { "state", app.InstallationState.ToString() }, + { "intent", app.Intent.ToString() }, + { "targeted", app.Targeted.ToString() }, + { "trigger", "termination" }, + { "terminationOutcome", args.Outcome.ToString() }, + }, + ImmediateUpload = true, + }); + } + } + catch (Exception ex) + { + _logger.Warning($"EnrollmentTerminationHandler: app_install_starved terminal sweep failed: {ex.Message}"); + } + } + /// /// c117946b debrief (2026-05-12) — pre-hook for . /// On the terminal-ESP-Apps-failure path, promote every app still in diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IAppTrackingReadModel.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IAppTrackingReadModel.cs index f534423a..7e8df6b8 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IAppTrackingReadModel.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/IAppTrackingReadModel.cs @@ -73,5 +73,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination /// mis-classify in-flight installs. Null = no HRESULT context → fallback. /// EspTerminalFailureSnapshot? LastEspTerminalFailure { get; } + + /// + /// Liveness plan PR3 — required user-ESP apps that never started installing while the + /// AccountSetup apps gate waited on them (no download/install activity, not failed, + /// not terminal). The termination handler emits a one-shot app_install_starved + /// Warning per leftover app on every terminal outcome. May return null / empty when no + /// IME surface exists. + /// + IReadOnlyList? GetStarvedUserEspApps(); + + /// + /// Liveness plan PR3 — appIds the live (esp_exited) path already reported via + /// app_install_starved. The terminal sweep skips these so the two emission + /// paths never double-report an app. + /// + IReadOnlyCollection StarvedUserEspAppsAlreadyReported { get; } } } diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorAppTrackingReadModel.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorAppTrackingReadModel.cs index 79426297..e7698451 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorAppTrackingReadModel.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/Termination/OrchestratorAppTrackingReadModel.cs @@ -39,5 +39,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination public EspTerminalFailureSnapshot? LastEspTerminalFailure => _orchestrator.CollectorSurfaces?.LastEspTerminalFailure; + + public IReadOnlyList? GetStarvedUserEspApps() => + _orchestrator.CollectorSurfaces?.GetStarvedUserEspApps(); + + public IReadOnlyCollection StarvedUserEspAppsAlreadyReported => + _orchestrator.CollectorSurfaces?.StarvedUserEspAppsAlreadyReported ?? Array.Empty(); } } diff --git a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts index 78477d9f..3492042a 100644 --- a/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts +++ b/src/McpServer/autopilot-monitor-mcp/src/resource-catalog.ts @@ -60,6 +60,7 @@ export const EVENT_TYPES_CATALOG = { 'app_install_failed', 'app_download_started', 'app_install_skipped', + 'app_install_starved', 'download_progress', 'do_telemetry', 'all_apps_completed', diff --git a/src/Shared/AutopilotMonitor.Shared/Constants.cs b/src/Shared/AutopilotMonitor.Shared/Constants.cs index 5c071af3..0e460b83 100644 --- a/src/Shared/AutopilotMonitor.Shared/Constants.cs +++ b/src/Shared/AutopilotMonitor.Shared/Constants.cs @@ -247,6 +247,12 @@ namespace AutopilotMonitor.Shared public const string AppInstallFailed = "app_install_failed"; public const string AppDownloadStarted = "app_download_started"; public const string AppInstallSkipped = "app_install_skipped"; + // Liveness plan PR3 — one-shot Warning per app naming a required user-ESP app that + // NEVER started installing (no download/install activity seen) while the ESP + // AccountSetup apps gate waited on it. Emitted when the user-apps-settled probe + // returns false after the ESP page exited, and again at termination for leftovers. + // This is the actionable customer hint behind "session hangs in AccountSetup". + public const string AppInstallStarved = "app_install_starved"; public const string NetworkStateChange = "network_state_change"; public const string NetworkConnectivityCheck = "network_connectivity_check"; public const string ErrorDetected = "error_detected"; diff --git a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts index fd0ed995..e86672de 100644 --- a/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts +++ b/src/Web/autopilot-monitor-web/app/gather-rules/eventTypes.ts @@ -80,6 +80,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [ description: "An application installation failed." }, { value: "app_install_skipped", label: "app_install_skipped", category: "app", description: "An application installation was skipped." }, + { value: "app_install_starved", label: "app_install_starved", category: "app", + description: "A required user-ESP app never started installing while the AccountSetup apps gate waited on it (one-shot per app)." }, { value: "app_download_started", label: "app_download_started", category: "app", description: "An application download has started." }, { value: "download_progress", label: "download_progress", category: "app",