mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(agent-v2): app_install_starved names the app starving the AccountSetup gate (PR3)
The gate-starvation dead-ends (caa6cf50, 1ec8f4c6) were ultimately caused by a single required user-targeted app that never started installing. Until now the session just hung anonymously; this makes the starving app a named, actionable customer hint. New ImeLogTracker.GetStarvedUserEspApps(): required (Install/Uninstall intent) apps in the current AccountSetup phase with no download/install activity ever observed, not terminal, not failed. Apps that are alive (Downloading/Installing) stay excluded; error apps belong to the failure path. Same conservative thread-safety contract as AreUserEspAppsSettled. Two timer-free trigger points, one-shot per appId: - Live: when the ESP page exits and the user-apps-settled probe reports false, EspAndHelloTracker emits app_install_starved (Warning) per starved app; dedupe via a lock-guarded HashSet exposed as StarvedAppsReported. - Terminal: EnrollmentTerminationHandler sweeps leftovers on every outcome (before app_tracking_summary, skipped on WhiteGlove Part 1) and consults the live path dedupe set via the read-model surface, so no app is ever reported twice. Wiring: ImeLogHost bridge -> DefaultComponentFactory probe -> EspAndHelloTracker; CollectorSurfaces + IAppTrackingReadModel + OrchestratorAppTrackingReadModel expose the starved list and the already-reported set to the termination handler. New event type synced to MCP resource catalog (app_events) and Web gather-rules eventTypes. V2.Core 1365/1365, MCP vitest 220, Web 458, full solution build clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+174
@@ -0,0 +1,174 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
|
||||
{
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — live-path <c>app_install_starved</c> emission. When the ESP page
|
||||
/// exits while the user-apps-settled probe reports false, the coordinator names the
|
||||
/// starved app(s) (one-shot Warning per appId) instead of leaving the operator with an
|
||||
/// anonymous stall. The dedupe set is exposed via
|
||||
/// <see cref="EspAndHelloTracker.StarvedAppsReported"/> for the termination sweep.
|
||||
/// </summary>
|
||||
public sealed class EspAndHelloTrackerStarvedAppsTests
|
||||
{
|
||||
private static readonly DateTime Fixed = new DateTime(2026, 6, 12, 8, 0, 2, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Fixture : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public AgentLogger Logger { get; }
|
||||
public FakeSignalIngressSink Ingress { get; } = new FakeSignalIngressSink();
|
||||
public FakeSignalIngressSink TrackerPostSink { get; } = new FakeSignalIngressSink();
|
||||
public VirtualClock Clock { get; } = new VirtualClock(Fixed);
|
||||
|
||||
public Fixture() { Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Debug); }
|
||||
|
||||
public EspAndHelloTracker BuildCoordinator(
|
||||
Func<bool> settledProbe,
|
||||
Func<IReadOnlyList<AppPackageState>> starvedProbe)
|
||||
{
|
||||
return new EspAndHelloTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(TrackerPostSink, Clock),
|
||||
logger: Logger,
|
||||
skipConfigProbe: () => ((bool?)false, (bool?)false),
|
||||
accountSetupActivityProbe: () => true,
|
||||
userEspAppsSettledProbe: settledProbe,
|
||||
starvedUserEspAppsProbe: starvedProbe);
|
||||
}
|
||||
|
||||
public List<FakeSignalIngressSink.PostedSignal> StarvedPosts() =>
|
||||
TrackerPostSink.Posted.Where(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "app_install_starved").ToList();
|
||||
|
||||
public void Dispose() { Tmp.Dispose(); }
|
||||
}
|
||||
|
||||
private static AppPackageState StarvedApp(string id, string? name = null)
|
||||
{
|
||||
var pkg = new AppPackageState(id, listPos: 0);
|
||||
pkg.UpdateIntent(AppIntent.Install);
|
||||
if (name != null) pkg.UpdateName(name);
|
||||
return pkg;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unsettled_exit_emits_warning_per_starved_app()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => false,
|
||||
() => new[] { StarvedApp("app-1", "Contoso Backgrounds") });
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
|
||||
var post = Assert.Single(f.StarvedPosts());
|
||||
Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]);
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(post.TypedPayload);
|
||||
Assert.Equal("app-1", data["appId"]);
|
||||
Assert.Equal("Contoso Backgrounds", data["appName"]);
|
||||
Assert.Equal("esp_exited_user_apps_not_settled", data["trigger"]);
|
||||
Assert.Contains("app-1", coordinator.StarvedAppsReported);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Repeated_exits_emit_once_per_appId()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => false,
|
||||
() => new[] { StarvedApp("app-1") });
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5));
|
||||
|
||||
Assert.Single(f.StarvedPosts());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void New_starved_app_on_later_exit_is_reported_additively()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
var starved = new List<AppPackageState> { StarvedApp("app-1") };
|
||||
using var coordinator = f.BuildCoordinator(() => false, () => starved);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
starved.Add(StarvedApp("app-2"));
|
||||
coordinator.TriggerEspExitedForTest(Fixed.AddMinutes(5));
|
||||
|
||||
var posts = f.StarvedPosts();
|
||||
Assert.Equal(2, posts.Count);
|
||||
Assert.Equal(2, coordinator.StarvedAppsReported.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Settled_exit_emits_no_starved_events()
|
||||
{
|
||||
// When the probe settles, the synthesis path runs instead — starved apps and a
|
||||
// settled gate are mutually exclusive by the IME probe's construction anyway.
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => true,
|
||||
() => new[] { StarvedApp("app-1") });
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
|
||||
Assert.Empty(f.StarvedPosts());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Throwing_starved_probe_is_swallowed()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var coordinator = f.BuildCoordinator(
|
||||
() => false,
|
||||
() => throw new InvalidOperationException("probe boom"));
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
|
||||
var posted = Assert.Single(f.Ingress.Posted);
|
||||
Assert.Equal(DecisionSignalKind.EspExiting, posted.Kind);
|
||||
Assert.Empty(f.StarvedPosts());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Default_probe_emits_nothing()
|
||||
{
|
||||
using var f = new Fixture();
|
||||
using var coordinator = new EspAndHelloTracker(
|
||||
sessionId: "S1",
|
||||
tenantId: "T1",
|
||||
post: new InformationalEventPost(f.TrackerPostSink, f.Clock),
|
||||
logger: f.Logger,
|
||||
skipConfigProbe: () => ((bool?)false, (bool?)false),
|
||||
accountSetupActivityProbe: () => true,
|
||||
userEspAppsSettledProbe: () => false);
|
||||
using var adapter = new EspAndHelloTrackerAdapter(coordinator, f.Ingress, f.Clock);
|
||||
|
||||
coordinator.TriggerEspExitedForTest(Fixed);
|
||||
|
||||
Assert.Empty(f.StarvedPosts());
|
||||
Assert.Empty(coordinator.StarvedAppsReported);
|
||||
}
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
using System.Collections.Generic;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Ime
|
||||
{
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — <see cref="ImeLogTracker.GetStarvedUserEspApps"/> semantics. A
|
||||
/// starved app is REQUIRED (Install/Uninstall intent), tracked in the current AccountSetup
|
||||
/// phase, has never shown download/install activity, and is neither terminal nor failed.
|
||||
/// Apps that are alive (Downloading/Installing or any prior progress) and error-state apps
|
||||
/// (owned by the failure path) must never be reported.
|
||||
/// </summary>
|
||||
public sealed class ImeLogTrackerStarvedUserEspAppsTests
|
||||
{
|
||||
private static ImeLogTracker BuildTracker(TempDirectory tmp) =>
|
||||
new ImeLogTracker(
|
||||
logFolder: tmp.Path,
|
||||
patterns: new List<ImeLogPattern>(),
|
||||
logger: new AgentLogger(tmp.Path, AgentLogLevel.Info));
|
||||
|
||||
private static AppPackageState AddApp(
|
||||
ImeLogTracker tracker,
|
||||
string id,
|
||||
AppIntent intent,
|
||||
AppInstallationState state)
|
||||
{
|
||||
var pkg = tracker.PackageStates.GetPackage(id, createIfNotFound: true);
|
||||
pkg.UpdateIntent(intent);
|
||||
if (state == AppInstallationState.Installed)
|
||||
{
|
||||
pkg.UpdateState(AppInstallationState.Installing);
|
||||
}
|
||||
if (state != AppInstallationState.Unknown)
|
||||
{
|
||||
pkg.UpdateState(state);
|
||||
}
|
||||
return pkg;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Pending_required_app_is_starved()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
|
||||
|
||||
var starved = tracker.GetStarvedUserEspApps();
|
||||
|
||||
var app = Assert.Single(starved);
|
||||
Assert.Equal("app-pending", app.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void InProgress_without_download_activity_is_starved()
|
||||
{
|
||||
// IME announced the app (policy line → InProgress) but enforcement never started —
|
||||
// exactly the field shape that starved the gate (user-targeted app on "pending").
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-announced", AppIntent.Install, AppInstallationState.InProgress);
|
||||
|
||||
var starved = tracker.GetStarvedUserEspApps();
|
||||
|
||||
var app = Assert.Single(starved);
|
||||
Assert.Equal("app-announced", app.Id);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Downloading_and_Installing_apps_are_alive_not_starved()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-downloading", AppIntent.Install, AppInstallationState.Downloading);
|
||||
AddApp(tracker, "app-installing", AppIntent.Install, AppInstallationState.Installing);
|
||||
|
||||
Assert.Empty(tracker.GetStarvedUserEspApps());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Terminal_and_error_apps_are_not_starved()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed);
|
||||
AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped);
|
||||
// Error apps belong to the failure path (app_install_failed names them already).
|
||||
AddApp(tracker, "app-failed", AppIntent.Install, AppInstallationState.Error);
|
||||
|
||||
Assert.Empty(tracker.GetStarvedUserEspApps());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Non_required_apps_are_not_starved()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-available", AppIntent.Available, AppInstallationState.Unknown);
|
||||
AddApp(tracker, "app-unknown-intent", AppIntent.Unknown, AppInstallationState.Unknown);
|
||||
|
||||
Assert.Empty(tracker.GetStarvedUserEspApps());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Outside_AccountSetup_phase_returns_empty()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("DeviceSetup");
|
||||
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
|
||||
|
||||
Assert.Empty(tracker.GetStarvedUserEspApps());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Phase_never_detected_returns_empty()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
AddApp(tracker, "app-pending", AppIntent.Install, AppInstallationState.Unknown);
|
||||
|
||||
Assert.Empty(tracker.GetStarvedUserEspApps());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Mixed_list_reports_only_the_starved_app()
|
||||
{
|
||||
// The field shape: one app installed, one skipped, one never started.
|
||||
using var tmp = new TempDirectory();
|
||||
var tracker = BuildTracker(tmp);
|
||||
tracker.SeedCurrentPhaseForTesting("AccountSetup");
|
||||
AddApp(tracker, "app-installed", AppIntent.Install, AppInstallationState.Installed);
|
||||
AddApp(tracker, "app-skipped", AppIntent.Install, AppInstallationState.Skipped);
|
||||
AddApp(tracker, "app-starving", AppIntent.Install, AppInstallationState.Unknown);
|
||||
|
||||
var starved = tracker.GetStarvedUserEspApps();
|
||||
|
||||
var app = Assert.Single(starved);
|
||||
Assert.Equal("app-starving", app.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
+79
@@ -159,6 +159,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
// non-Apps gating test assigns a snapshot whose FailedSubcategory is NOT
|
||||
// "Apps".
|
||||
public EspTerminalFailureSnapshot? LastEspTerminalFailureOverride { get; set; } = null;
|
||||
|
||||
// Liveness plan PR3 — starved-apps surfaces for the terminal sweep tests.
|
||||
public IReadOnlyList<AppPackageState>? StarvedAppsOverride { get; set; } = null;
|
||||
public IReadOnlyCollection<string> StarvedAlreadyReportedOverride { get; set; } = Array.Empty<string>();
|
||||
public List<string> TerminationActionLog { get; } = new List<string>();
|
||||
public TimeSpan SpoolDrainPeriodOverride { get; set; } = TimeSpan.Zero;
|
||||
// Shutdown-gap closure (2026-05-15) — when set, the constructed handler shares
|
||||
@@ -223,6 +227,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
}
|
||||
|
||||
public EspTerminalFailureSnapshot? LastEspTerminalFailure => _rig.LastEspTerminalFailureOverride;
|
||||
|
||||
public IReadOnlyList<AppPackageState>? GetStarvedUserEspApps() => _rig.StarvedAppsOverride;
|
||||
|
||||
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported => _rig.StarvedAlreadyReportedOverride;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -1079,6 +1087,77 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Empty(rig.PromotionCalls);
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------
|
||||
// Liveness plan PR3: app_install_starved terminal sweep
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
private static AppPackageState BuildStarvedApp(string id, string name)
|
||||
{
|
||||
var pkg = new AppPackageState(id, listPos: 0);
|
||||
pkg.UpdateIntent(AppIntent.Install);
|
||||
pkg.UpdateName(name);
|
||||
return pkg;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StarvedApps_at_termination_emit_one_warning_per_app()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.MaxLifetimeExceeded, EnrollmentTerminationOutcome.TimedOut, SessionStage.EspAccountSetup));
|
||||
|
||||
Assert.Contains("app_install_starved", rig.EmittedEventTypes);
|
||||
var post = rig.Ingress.Posted.Single(p =>
|
||||
p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
|
||||
&& et == "app_install_starved");
|
||||
Assert.Equal("Warning", post.Payload![SignalPayloadKeys.Severity]);
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(post.TypedPayload);
|
||||
Assert.Equal("app-starving", data["appId"]);
|
||||
Assert.Equal("termination", data["trigger"]);
|
||||
Assert.Equal("TimedOut", data["terminationOutcome"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StarvedApps_already_reported_by_live_path_are_skipped()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
|
||||
rig.StarvedAlreadyReportedOverride = new[] { "app-starving" };
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Failed, SessionStage.Failed));
|
||||
|
||||
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_starved_apps_emits_nothing()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
rig.StarvedAppsOverride = null; // no IME surface
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.Completed));
|
||||
|
||||
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void StarvedApps_skipped_on_whiteglove_part1()
|
||||
{
|
||||
// Part 1 has no user session — user apps haven't run yet; Part 2 is where they land.
|
||||
using var rig = new Rig();
|
||||
rig.StarvedAppsOverride = new[] { BuildStarvedApp("app-starving", "Contoso Backgrounds") };
|
||||
|
||||
rig.Build().Handle(sender: null!,
|
||||
Args(EnrollmentTerminationReason.DecisionTerminalStage, EnrollmentTerminationOutcome.Succeeded, SessionStage.WhiteGloveSealed));
|
||||
|
||||
Assert.DoesNotContain("app_install_starved", rig.EmittedEventTypes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PromoteActiveInstalls_skipped_on_effect_infrastructure_failure()
|
||||
{
|
||||
|
||||
@@ -290,6 +290,47 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — returns the required user-ESP apps that are STARVING the
|
||||
/// AccountSetup apps gate: tracked in the current AccountSetup phase, Install/Uninstall
|
||||
/// intent, no download/install activity ever observed, not terminal and not failed.
|
||||
/// These are the apps whose enforcement IME never started (e.g. a user-targeted app
|
||||
/// stuck on "pending") — the actual blocker behind the gate-starvation dead-ends.
|
||||
/// <para>
|
||||
/// Deliberately excludes apps that are alive (<see cref="AppPackageState.DownloadingOrInstallingSeen"/>
|
||||
/// covers Downloading/Installing and anything that ever progressed) and error-state apps
|
||||
/// (the failure path already names those). Same thread-safety contract as
|
||||
/// <see cref="AreUserEspAppsSettled"/>: snapshot copy, any race-induced exception
|
||||
/// returns an empty list.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!string.Equals(_lastEspPhaseDetected, "AccountSetup", StringComparison.OrdinalIgnoreCase))
|
||||
return Array.Empty<AppPackageState>();
|
||||
|
||||
var apps = new List<AppPackageState>(_packageStates);
|
||||
if (apps.Count == 0) return Array.Empty<AppPackageState>();
|
||||
|
||||
var starved = new List<AppPackageState>();
|
||||
foreach (var pkg in apps)
|
||||
{
|
||||
if (pkg == null || !pkg.IsRequired) continue;
|
||||
if (pkg.IsCompleted || pkg.IsError) continue;
|
||||
if (pkg.DownloadingOrInstallingSeen) continue;
|
||||
starved.Add(pkg);
|
||||
}
|
||||
return starved;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"ImeLogTracker: GetStarvedUserEspApps probe threw: {ex.Message}");
|
||||
return Array.Empty<AppPackageState>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Promotes every app currently in <see cref="AppInstallationState.Installing"/> to
|
||||
/// <see cref="AppInstallationState.Error"/>, stamping <paramref name="failureType"/>
|
||||
|
||||
+103
-2
@@ -40,11 +40,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
private readonly Func<(bool? skipUser, bool? skipDevice)> _skipConfigProbe;
|
||||
private readonly Func<bool> _accountSetupActivityProbe;
|
||||
private readonly Func<bool> _userEspAppsSettledProbe;
|
||||
private readonly Func<System.Collections.Generic.IReadOnlyList<Ime.AppPackageState>> _starvedUserEspAppsProbe;
|
||||
|
||||
// Session caa6cf50 gate-starvation fix (2026-06-11) — fire-once guard for the
|
||||
// user-ESP-apps-settled AccountSetup synthesis (see MaybeSynthesizeAccountSetupComplete).
|
||||
private bool _userAppsSettledSynthesisFired;
|
||||
|
||||
// Liveness plan PR3 — one-shot-per-appId dedupe for app_install_starved emissions.
|
||||
// Written from the Shell-Core watcher thread (esp_exited path), read at termination by
|
||||
// the EnrollmentTerminationHandler via StarvedAppsReported — guarded by _starvedLock.
|
||||
private readonly System.Collections.Generic.HashSet<string> _starvedAppsReported =
|
||||
new System.Collections.Generic.HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly object _starvedLock = new object();
|
||||
|
||||
private HelloTracker _helloTracker;
|
||||
private ShellCoreTracker _shellCoreTracker;
|
||||
private ProvisioningStatusTracker _provisioningTracker;
|
||||
@@ -195,7 +203,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
int[] modernDeploymentHarmlessEventIds = null,
|
||||
Func<(bool? skipUser, bool? skipDevice)> skipConfigProbe = null,
|
||||
Func<bool> accountSetupActivityProbe = null,
|
||||
Func<bool> userEspAppsSettledProbe = null)
|
||||
Func<bool> userEspAppsSettledProbe = null,
|
||||
Func<System.Collections.Generic.IReadOnlyList<Ime.AppPackageState>> starvedUserEspAppsProbe = null)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
@@ -218,6 +227,27 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// probe (wired to ImeLogHost.AreUserEspAppsSettled by DefaultComponentFactory).
|
||||
// Defaults to "not settled" so single-tracker wiring scenarios keep prior behaviour.
|
||||
_userEspAppsSettledProbe = userEspAppsSettledProbe ?? (() => false);
|
||||
// Liveness plan PR3: IME-side starved-apps probe (wired to
|
||||
// ImeLogHost.GetStarvedUserEspApps by DefaultComponentFactory). Defaults to empty
|
||||
// so single-tracker wiring scenarios never emit app_install_starved.
|
||||
_starvedUserEspAppsProbe = starvedUserEspAppsProbe
|
||||
?? (() => Array.Empty<Ime.AppPackageState>());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
|
||||
/// live (esp_exited) path. Read by the <c>EnrollmentTerminationHandler</c> terminal
|
||||
/// sweep so the two emission paths never double-report an app. Snapshot copy.
|
||||
/// </summary>
|
||||
public System.Collections.Generic.IReadOnlyCollection<string> StarvedAppsReported
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (_starvedLock)
|
||||
{
|
||||
return new System.Collections.Generic.List<string>(_starvedAppsReported);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =====================================================================
|
||||
@@ -618,7 +648,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
_logger.Debug($"EspAndHelloTracker: user-ESP-apps-settled probe threw: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
if (!settled) return;
|
||||
if (!settled)
|
||||
{
|
||||
// Liveness plan PR3: the ESP page is gone but the user-apps gate is NOT settled
|
||||
// — name the app(s) that never started installing instead of leaving the
|
||||
// operator with an anonymous "session hangs in AccountSetup". One-shot per
|
||||
// appId; apps that are alive (Downloading/Installing) or failed are excluded
|
||||
// by the probe itself.
|
||||
EmitStarvedUserEspApps(trigger: "esp_exited_user_apps_not_settled");
|
||||
return;
|
||||
}
|
||||
|
||||
_userAppsSettledSynthesisFired = true;
|
||||
_logger.Warning(
|
||||
@@ -650,6 +689,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
catch (Exception ex) { _logger.Error("AccountSetupProvisioningComplete handler failed (user-apps-settled synthesis)", ex); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — emit a one-shot <c>app_install_starved</c> Warning per starved
|
||||
/// user-ESP app (required, never started installing, not failed). Best-effort: probe or
|
||||
/// emit failures are logged and swallowed; a failed emit keeps the appId out of the
|
||||
/// dedupe set so the termination sweep can retry.
|
||||
/// </summary>
|
||||
private void EmitStarvedUserEspApps(string trigger)
|
||||
{
|
||||
System.Collections.Generic.IReadOnlyList<Ime.AppPackageState> starved;
|
||||
try { starved = _starvedUserEspAppsProbe(); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"EspAndHelloTracker: starved-apps probe threw: {ex.Message}");
|
||||
return;
|
||||
}
|
||||
if (starved == null || starved.Count == 0) return;
|
||||
|
||||
foreach (var app in starved)
|
||||
{
|
||||
if (app?.Id == null) continue;
|
||||
lock (_starvedLock)
|
||||
{
|
||||
if (!_starvedAppsReported.Add(app.Id)) continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
|
||||
_logger.Warning(
|
||||
$"EspAndHelloTracker: required user-ESP app '{name}' ({app.Id}) never started installing " +
|
||||
$"(state={app.InstallationState}) — starving the AccountSetup apps gate (trigger={trigger}).");
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _sessionId,
|
||||
TenantId = _tenantId,
|
||||
EventType = Constants.EventTypes.AppInstallStarved,
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "EspAndHelloTracker",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Required app '{name}' never started installing while the ESP AccountSetup " +
|
||||
"apps gate waited on it — the app is starving the enrollment completion.",
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "appId", app.Id },
|
||||
{ "appName", name },
|
||||
{ "state", app.InstallationState.ToString() },
|
||||
{ "intent", app.Intent.ToString() },
|
||||
{ "targeted", app.Targeted.ToString() },
|
||||
{ "trigger", trigger },
|
||||
},
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error($"EspAndHelloTracker: app_install_starved emit failed for '{app.Id}'", ex);
|
||||
lock (_starvedLock) { _starvedAppsReported.Remove(app.Id); }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test seam for the user-apps-settled synthesis — drives OnEspExited with the live
|
||||
// handler signature, mirroring TriggerEspExitedForTest's contract.
|
||||
internal bool UserAppsSettledSynthesisFiredForTest => _userAppsSettledSynthesisFired;
|
||||
|
||||
@@ -113,5 +113,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// was observed.
|
||||
/// </summary>
|
||||
public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => EspAndHelloHost?.LastEspTerminalFailure;
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — required user-ESP apps that never started installing (current
|
||||
/// AccountSetup phase). Empty when no IME host exists.
|
||||
/// </summary>
|
||||
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps() =>
|
||||
_imeLogHost?.GetStarvedUserEspApps() ?? Array.Empty<AppPackageState>();
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
|
||||
/// live (esp_exited) path. Consulted by the termination handler so the terminal sweep
|
||||
/// never double-reports an app. Empty when no ESP/Hello host exists.
|
||||
/// </summary>
|
||||
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported =>
|
||||
EspAndHelloHost?.StarvedAppsReported ?? Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,7 +121,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
modernDeploymentBackfillLookbackMinutes: collectors.ModernDeploymentBackfillLookbackMinutes,
|
||||
modernDeploymentHarmlessEventIds: collectors.ModernDeploymentHarmlessEventIds,
|
||||
stateDirectory: _stateDirectory,
|
||||
userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true);
|
||||
userEspAppsSettledProbe: () => imeLogHostRef?.AreUserEspAppsSettled() == true,
|
||||
// Liveness plan PR3: starved-apps probe over the same lazily-assigned IME host.
|
||||
starvedUserEspAppsProbe: () =>
|
||||
imeLogHostRef?.GetStarvedUserEspApps()
|
||||
?? Array.Empty<Monitoring.Enrollment.Ime.AppPackageState>());
|
||||
hosts.Add(espAndHelloHost);
|
||||
|
||||
// Hybrid User-Driven completion-gap fix (2026-05-01): the AadJoinHost notifies
|
||||
|
||||
@@ -48,6 +48,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public Termination.EspTerminalFailureSnapshot? LastEspTerminalFailure => _tracker.LastEspTerminalFailure;
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — appIds already reported via <c>app_install_starved</c> on the
|
||||
/// live path. Read by <c>CollectorSurfaces</c> for the termination handler's dedupe.
|
||||
/// </summary>
|
||||
public System.Collections.Generic.IReadOnlyCollection<string> StarvedAppsReported => _tracker.StarvedAppsReported;
|
||||
|
||||
public EspAndHelloHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
@@ -61,7 +67,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
int modernDeploymentBackfillLookbackMinutes,
|
||||
int[]? modernDeploymentHarmlessEventIds,
|
||||
string stateDirectory,
|
||||
Func<bool>? userEspAppsSettledProbe = null)
|
||||
Func<bool>? userEspAppsSettledProbe = null,
|
||||
Func<System.Collections.Generic.IReadOnlyList<Monitoring.Enrollment.Ime.AppPackageState>>? starvedUserEspAppsProbe = null)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
@@ -78,7 +85,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
modernDeploymentBackfillLookbackMinutes: modernDeploymentBackfillLookbackMinutes,
|
||||
stateDirectory: stateDirectory,
|
||||
modernDeploymentHarmlessEventIds: modernDeploymentHarmlessEventIds,
|
||||
userEspAppsSettledProbe: userEspAppsSettledProbe);
|
||||
userEspAppsSettledProbe: userEspAppsSettledProbe,
|
||||
starvedUserEspAppsProbe: starvedUserEspAppsProbe);
|
||||
|
||||
_tracker.WhiteGloveCompleted += OnTrackerWhiteGloveCompleted;
|
||||
_tracker.DeviceSetupProvisioningComplete += OnTrackerDeviceSetupProvisioningComplete;
|
||||
|
||||
@@ -60,6 +60,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public bool AreUserEspAppsSettled() => _tracker.AreUserEspAppsSettled();
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — bridge for the <c>EspAndHelloTracker</c> starved-apps probe and
|
||||
/// the <c>EnrollmentTerminationHandler</c> terminal sweep. See
|
||||
/// <see cref="ImeLogTracker.GetStarvedUserEspApps"/>.
|
||||
/// </summary>
|
||||
public IReadOnlyList<AppPackageState> GetStarvedUserEspApps() => _tracker.GetStarvedUserEspApps();
|
||||
|
||||
/// <summary>
|
||||
/// c117946b debrief (2026-05-12) — bridge for the <c>EnrollmentTerminationHandler</c>
|
||||
/// pre-hook to promote apps still in <see cref="AppInstallationState.Installing"/>
|
||||
|
||||
@@ -181,6 +181,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// the new Error counts via the `likelyStuckNames` bucket.
|
||||
MaybePromoteActiveInstallsAsStuck(state, args);
|
||||
|
||||
// Liveness plan PR3 — leftover starved apps (required, never started
|
||||
// installing) at termination get their one-shot app_install_starved
|
||||
// BEFORE the summary so both reach the backend in the terminal flush.
|
||||
// Runs on every outcome; the live (esp_exited) path's dedupe set is
|
||||
// consulted so no app is reported twice. Promoted likely-stuck apps are
|
||||
// Error by now and excluded by the probe itself.
|
||||
EmitStarvedUserEspApps(args);
|
||||
|
||||
EmitAppTrackingSummary();
|
||||
}
|
||||
|
||||
@@ -402,6 +410,65 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — terminal sweep for <c>app_install_starved</c>. Emits a one-shot
|
||||
/// Warning per required user-ESP app that never started installing and was not already
|
||||
/// reported by the live (esp_exited) path in <c>EspAndHelloTracker</c>. Best-effort:
|
||||
/// any accessor / emit failure is logged and swallowed.
|
||||
/// </summary>
|
||||
private void EmitStarvedUserEspApps(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (_post == null) return;
|
||||
|
||||
try
|
||||
{
|
||||
var starved = _appTracking.GetStarvedUserEspApps();
|
||||
if (starved == null || starved.Count == 0) return;
|
||||
|
||||
var alreadyReported = _appTracking.StarvedUserEspAppsAlreadyReported
|
||||
?? (IReadOnlyCollection<string>)Array.Empty<string>();
|
||||
var reported = new HashSet<string>(alreadyReported, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var app in starved)
|
||||
{
|
||||
if (app?.Id == null || !reported.Add(app.Id)) continue;
|
||||
|
||||
var name = string.IsNullOrEmpty(app.Name) ? app.Id : app.Name;
|
||||
_logger.Warning(
|
||||
$"EnrollmentTerminationHandler: required user-ESP app '{name}' ({app.Id}) never started " +
|
||||
$"installing (state={app.InstallationState}) — reporting app_install_starved at termination " +
|
||||
$"(outcome={args.Outcome}).");
|
||||
|
||||
_post.Emit(new EnrollmentEvent
|
||||
{
|
||||
SessionId = _configuration.SessionId,
|
||||
TenantId = _configuration.TenantId,
|
||||
EventType = Constants.EventTypes.AppInstallStarved,
|
||||
Severity = EventSeverity.Warning,
|
||||
Source = "EnrollmentTerminationHandler",
|
||||
Phase = EnrollmentPhase.Unknown,
|
||||
Message = $"Required app '{name}' never started installing while the ESP AccountSetup " +
|
||||
"apps gate waited on it — the app starved the enrollment completion.",
|
||||
Data = new Dictionary<string, object>
|
||||
{
|
||||
{ "appId", app.Id },
|
||||
{ "appName", name },
|
||||
{ "state", app.InstallationState.ToString() },
|
||||
{ "intent", app.Intent.ToString() },
|
||||
{ "targeted", app.Targeted.ToString() },
|
||||
{ "trigger", "termination" },
|
||||
{ "terminationOutcome", args.Outcome.ToString() },
|
||||
},
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Warning($"EnrollmentTerminationHandler: app_install_starved terminal sweep failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// c117946b debrief (2026-05-12) — pre-hook for <see cref="EmitAppTrackingSummary"/>.
|
||||
/// On the terminal-ESP-Apps-failure path, promote every app still in
|
||||
|
||||
@@ -73,5 +73,21 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// mis-classify in-flight installs. Null = no HRESULT context → fallback.
|
||||
/// </summary>
|
||||
EspTerminalFailureSnapshot? LastEspTerminalFailure { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — required user-ESP apps that never started installing while the
|
||||
/// AccountSetup apps gate waited on them (no download/install activity, not failed,
|
||||
/// not terminal). The termination handler emits a one-shot <c>app_install_starved</c>
|
||||
/// Warning per leftover app on every terminal outcome. May return null / empty when no
|
||||
/// IME surface exists.
|
||||
/// </summary>
|
||||
IReadOnlyList<AppPackageState>? GetStarvedUserEspApps();
|
||||
|
||||
/// <summary>
|
||||
/// Liveness plan PR3 — appIds the live (esp_exited) path already reported via
|
||||
/// <c>app_install_starved</c>. The terminal sweep skips these so the two emission
|
||||
/// paths never double-report an app.
|
||||
/// </summary>
|
||||
IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported { get; }
|
||||
}
|
||||
}
|
||||
|
||||
+6
@@ -39,5 +39,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
public EspTerminalFailureSnapshot? LastEspTerminalFailure =>
|
||||
_orchestrator.CollectorSurfaces?.LastEspTerminalFailure;
|
||||
|
||||
public IReadOnlyList<AppPackageState>? GetStarvedUserEspApps() =>
|
||||
_orchestrator.CollectorSurfaces?.GetStarvedUserEspApps();
|
||||
|
||||
public IReadOnlyCollection<string> StarvedUserEspAppsAlreadyReported =>
|
||||
_orchestrator.CollectorSurfaces?.StarvedUserEspAppsAlreadyReported ?? Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,6 +60,7 @@ export const EVENT_TYPES_CATALOG = {
|
||||
'app_install_failed',
|
||||
'app_download_started',
|
||||
'app_install_skipped',
|
||||
'app_install_starved',
|
||||
'download_progress',
|
||||
'do_telemetry',
|
||||
'all_apps_completed',
|
||||
|
||||
@@ -247,6 +247,12 @@ namespace AutopilotMonitor.Shared
|
||||
public const string AppInstallFailed = "app_install_failed";
|
||||
public const string AppDownloadStarted = "app_download_started";
|
||||
public const string AppInstallSkipped = "app_install_skipped";
|
||||
// Liveness plan PR3 — one-shot Warning per app naming a required user-ESP app that
|
||||
// NEVER started installing (no download/install activity seen) while the ESP
|
||||
// AccountSetup apps gate waited on it. Emitted when the user-apps-settled probe
|
||||
// returns false after the ESP page exited, and again at termination for leftovers.
|
||||
// This is the actionable customer hint behind "session hangs in AccountSetup".
|
||||
public const string AppInstallStarved = "app_install_starved";
|
||||
public const string NetworkStateChange = "network_state_change";
|
||||
public const string NetworkConnectivityCheck = "network_connectivity_check";
|
||||
public const string ErrorDetected = "error_detected";
|
||||
|
||||
@@ -80,6 +80,8 @@ export const KNOWN_EVENT_TYPES: EventTypeEntry[] = [
|
||||
description: "An application installation failed." },
|
||||
{ value: "app_install_skipped", label: "app_install_skipped", category: "app",
|
||||
description: "An application installation was skipped." },
|
||||
{ value: "app_install_starved", label: "app_install_starved", category: "app",
|
||||
description: "A required user-ESP app never started installing while the AccountSetup apps gate waited on it (one-shot per app)." },
|
||||
{ value: "app_download_started", label: "app_download_started", category: "app",
|
||||
description: "An application download has started." },
|
||||
{ value: "download_progress", label: "download_progress", category: "app",
|
||||
|
||||
Reference in New Issue
Block a user