feat(v2): close Hybrid User-Driven completion gaps

Session e58bcfdb stuck 5h on Hybrid User-Driven enrollment because
the foouser OOBE shell triggered desktop_arrived prematurely and no
Hello/AAD-real-user signals ever followed.

Agent (V2):
- DesktopArrivalDetector excludes foouser/autopilot placeholders
  (UPN prefix + exact bare username; autopilotadmin etc. stay through)
- AadJoinWatcherAdapter dual-emits placeholder + real-user join as
  informational events for Events-table consumers
- DesktopArrivalDetector resets on placeholder->real-user transition
  via AadJoinHost.onRealUserJoined wiring
- New single-shot HybridLoginPendingDetector (10m, gated on
  reboot_kill + isHybridJoin, cancels on real-user join)

Backend:
- New FailureSnapshotBuilder writes per-session diagnostic snapshot
  on 5h-timeout (last ESP phase, desktop arrival, Hello policy,
  AAD-join state, missing canonical signals); aad_user_join slot
  satisfied by either Part 1 or Part 2 join event
- Sessions.FailureSnapshotJson column with Store/Map round-trip
- UpdateSessionStatusAsync extended with optional snapshot param

Web:
- New collapsible Failure Snapshot block in SessionInfoCard

Tests: V2.Core 920, Backend 862, DecisionCore 252, V1.Core 530
(regression-clean), Web vitest 208.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-02 10:56:10 +02:00
parent ac3b3d82db
commit fb31a2d6d1
23 changed files with 1798 additions and 40 deletions
@@ -0,0 +1,178 @@
#nullable enable
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
{
/// <summary>
/// Unit tests for <see cref="DesktopArrivalDetector.IsExcludedUser"/>. Hybrid-User-Driven
/// session diagnosis (e58bcfdb-…, 2026-05-01) showed the detector treated the fooUser
/// OOBE shell as a real user desktop, firing DesktopArrived 6 s after agent start —
/// before the Hybrid reboot to the AD account ever happened. The exclusion now covers
/// the foouser@/autopilot@ Autopilot provisioning placeholders in both UPN and
/// DOMAIN\User shapes.
/// </summary>
public sealed class DesktopArrivalDetectorTests
{
// ---------------- Existing exclusions stay covered ----------------
[Theory]
[InlineData("SYSTEM")]
[InlineData("system")]
[InlineData("LOCAL SERVICE")]
[InlineData("NETWORK SERVICE")]
[InlineData("DefaultUser0")]
[InlineData("DefaultUser1")]
[InlineData("defaultuser0")]
[InlineData("DefaultUser42")]
public void System_and_default_users_are_excluded(string user)
{
Assert.True(DesktopArrivalDetector.IsExcludedUser(user));
}
[Theory]
[InlineData("NT AUTHORITY\\SYSTEM")]
[InlineData("WORKGROUP\\DefaultUser0")]
[InlineData("CONTOSO\\DefaultUser1")]
public void Domain_qualified_system_users_are_excluded(string user)
{
Assert.True(DesktopArrivalDetector.IsExcludedUser(user));
}
// ---------------- Real users are NOT excluded ----------------
[Theory]
[InlineData("alice")]
[InlineData("CONTOSO\\alice")]
[InlineData("alice@contoso.com")]
[InlineData("bob.smith@fabrikam.com")]
// Codex review 2026-05-01 (Finding 3): the bare-username matcher was tightened
// from prefix-match to exact-match, so the following real account shapes that
// *start with* "autopilot" or "foouser" must now stay through the gate. Reused
// by UserProfileResolver — incorrectly excluding these would resolve the wrong
// home directory.
[InlineData("CONTOSO\\autopilotadmin")]
[InlineData("CONTOSO\\autopilot.admin")]
[InlineData("FABRIKAM\\foouserservice")]
[InlineData("autopilotadmin")]
[InlineData("foouserservice")]
public void Real_users_are_not_excluded(string user)
{
Assert.False(DesktopArrivalDetector.IsExcludedUser(user));
}
// ---------------- New: Autopilot placeholder UPN form ----------------
[Theory]
[InlineData("foouser@fabrikam.onmicrosoft.com")]
[InlineData("FooUser@fabrikam.onmicrosoft.com")]
[InlineData("FOOUSER@example.com")]
[InlineData("autopilot@contoso.com")]
[InlineData("Autopilot@contoso.com")]
[InlineData("AUTOPILOT@contoso.com")]
public void Autopilot_placeholder_upn_is_excluded(string upn)
{
// Hybrid User-Driven OOBE shell runs explorer.exe under foouser@<tenant>;
// matching this prevents premature DesktopArrived firing on the foo desktop.
// UPN form is delegated to AadJoinInfo.IsPlaceholderUserEmail (prefix-match
// on the local-part, so any tenant domain works).
Assert.True(DesktopArrivalDetector.IsExcludedUser(upn));
}
// ---------------- New: Domain-qualified placeholder (exact bare-name match) ----------------
[Theory]
[InlineData("AzureAD\\foouser")]
[InlineData("WORKGROUP\\autopilot")]
[InlineData("AzureAD\\FooUser")] // case-insensitive
[InlineData("AzureAD\\AUTOPILOT")] // case-insensitive
[InlineData("foouser")] // bare username (no domain prefix)
[InlineData("autopilot")] // bare username (no domain prefix)
public void Domain_qualified_placeholder_is_excluded(string user)
{
// WMI Win32_Process.GetOwner sometimes returns DOMAIN\foouser instead of the
// UPN form. The bare-username match is now EXACT (Finding 3) — no prefix.
Assert.True(DesktopArrivalDetector.IsExcludedUser(user));
}
// ---------------- Edge cases ----------------
[Theory]
[InlineData(null)]
[InlineData("")]
public void Null_or_empty_is_excluded(string? user)
{
Assert.True(DesktopArrivalDetector.IsExcludedUser(user!));
}
[Fact]
public void Real_user_email_with_foouser_substring_is_NOT_excluded()
{
// Defense against false positives — UPN match goes through
// AadJoinInfo.IsPlaceholderUserEmail which anchors the local-part at start.
// "realfoouser@…" and "not-autopilot@…" must NOT trigger.
Assert.False(DesktopArrivalDetector.IsExcludedUser("realfoouser@contoso.com"));
Assert.False(DesktopArrivalDetector.IsExcludedUser("not-autopilot@contoso.com"));
}
[Fact]
public void Real_user_with_default_substring_anywhere_is_NOT_excluded()
{
Assert.False(DesktopArrivalDetector.IsExcludedUser("MyDefaultUser"));
// "DefaultUser" prefix-match: this WOULD trigger because the bare-username
// DefaultUser* code path is still prefix-based (Windows generates
// DefaultUser0/1/2/...). Documenting current behavior.
Assert.True(DesktopArrivalDetector.IsExcludedUser("DefaultUserBob"));
}
// ============================================================================
// ResetForRealUserSwitch (Pkt 5 — placeholder→real-user transition)
// ============================================================================
[Fact]
public void ResetForRealUserSwitch_does_not_throw_when_detector_is_stopped()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
using var detector = new DesktopArrivalDetector(logger);
// Detector has not been Started — Reset must be safe regardless. The Hybrid
// reset path may fire before any explicit Start in test fixtures, and we
// never want host-wiring to crash the agent.
detector.ResetForRealUserSwitch();
}
[Fact]
public void ResetForRealUserSwitch_is_idempotent()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
using var detector = new DesktopArrivalDetector(logger);
// Calling reset multiple times back-to-back must not leak timer instances or
// throw. The composition root currently invokes it once per real-user join,
// but the contract is still safe-to-repeat.
detector.ResetForRealUserSwitch();
detector.ResetForRealUserSwitch();
detector.ResetForRealUserSwitch();
}
[Fact]
public void ResetForRealUserSwitch_after_Stop_restarts_polling_path()
{
using var tmp = new TempDirectory();
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
using var detector = new DesktopArrivalDetector(logger);
detector.Start();
detector.Stop();
// After Stop the polling timer is null. Reset must reinstate it without
// requiring a fresh Start call (the Hybrid reboot transition shouldn't have
// to rebuild the host).
detector.ResetForRealUserSwitch();
}
}
}
@@ -0,0 +1,253 @@
#nullable enable
using System;
using System.Linq;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring
{
/// <summary>
/// Tests for the single-shot Hybrid User-Driven login-pending detector. Single-shot
/// semantics, conditional emission, and cancel-on-real-user are the contracts that
/// matter — the detector exists explicitly to NOT add periodicity / timer sprawl.
/// </summary>
public sealed class HybridLoginPendingDetectorTests
{
private static readonly DateTime Fixed = new DateTime(2026, 5, 1, 14, 11, 0, DateTimeKind.Utc);
private sealed class Fixture : IDisposable
{
public TempDirectory Tmp { get; } = new TempDirectory();
public AgentLogger Logger { get; }
public AadJoinWatcher Watcher { get; }
public FakeSignalIngressSink Ingress { get; } = new FakeSignalIngressSink();
public VirtualClock Clock { get; } = new VirtualClock(Fixed);
public InformationalEventPost Post { get; }
// Adapter wires the watcher's AadUserJoined into a real AadUserJoinedLate signal —
// tests need it so TriggerFromTest("...") fires the watcher event the detector
// is subscribed to.
public AadJoinWatcherAdapter Adapter { get; }
public Fixture()
{
Logger = new AgentLogger(Tmp.Path, AgentLogLevel.Info);
Watcher = new AadJoinWatcher(Logger);
Post = new InformationalEventPost(Ingress, Clock);
Adapter = new AadJoinWatcherAdapter(Watcher, Ingress, Clock);
}
public void Dispose()
{
Adapter.Dispose();
Watcher.Dispose();
Tmp.Dispose();
}
}
private static FakeSignalIngressSink.PostedSignal? FindHybridLoginPending(FakeSignalIngressSink ingress) =>
ingress.Posted.FirstOrDefault(p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == SharedEventTypes.HybridLoginPending);
// ============================================================================
// Single-shot emission path
// ============================================================================
[Fact]
public void Arm_then_TriggerFromTest_emits_hybrid_login_pending_with_warning_severity()
{
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger, TimeSpan.FromMinutes(10));
detector.Arm();
detector.TriggerFromTest();
var info = FindHybridLoginPending(f.Ingress);
Assert.NotNull(info);
Assert.Equal("HybridLoginPendingDetector", info!.Payload![SignalPayloadKeys.Source]);
Assert.Equal("Warning", info.Payload[SignalPayloadKeys.Severity]);
Assert.Equal("true", info.Payload[SignalPayloadKeys.ImmediateUpload]);
Assert.Equal("10", info.Payload["delayMinutes"]);
Assert.Equal("timer_fired", info.Payload["reason"]);
Assert.Equal("true", info.Payload["isHybridJoin"]);
Assert.True(detector.HasFiredForTest);
}
[Fact]
public void TriggerFromTest_without_Arm_emits_nothing()
{
// Codex review 2026-05-01: contract tightened — emission requires a deliberate
// Arm(). The earlier behavior (emit-regardless) had a misleading test name and
// a fragile contract; production was safe only because Arm() was the sole timer
// creator. Now the guard is explicit so any future code path that bypasses Arm()
// can't silently produce a hybrid_login_pending event.
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.TriggerFromTest();
Assert.Null(FindHybridLoginPending(f.Ingress));
Assert.False(detector.HasFiredForTest);
}
[Fact]
public void Arm_is_idempotent()
{
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
detector.Arm();
detector.Arm();
// No hybrid_login_pending fires from arming alone — arming just starts a timer.
Assert.Null(FindHybridLoginPending(f.Ingress));
Assert.True(detector.IsArmedForTest);
}
[Fact]
public void Multiple_TriggerFromTest_calls_emit_only_once()
{
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
detector.TriggerFromTest();
detector.TriggerFromTest();
detector.TriggerFromTest();
// Single-shot — exactly one informational event regardless of how many times
// the timer logic is re-entered.
var matches = f.Ingress.Posted.Where(p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == SharedEventTypes.HybridLoginPending).ToList();
Assert.Single(matches);
}
// ============================================================================
// Cancel-on-real-user path
// ============================================================================
[Fact]
public void RealUserJoined_before_TriggerFromTest_cancels_emission()
{
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
detector.TriggerRealUserJoinedFromTest();
detector.TriggerFromTest();
Assert.True(detector.IsCancelledByRealUserForTest);
Assert.False(detector.HasFiredForTest);
Assert.Null(FindHybridLoginPending(f.Ingress));
}
[Fact]
public void RealUserJoined_before_Arm_short_circuits_Arm()
{
// The watcher could in principle see a real user before Arm() is called
// (composition-time race). The detector subscribes in its constructor so the
// _cancelledByRealUser flag is set early and Arm() respects it.
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.TriggerRealUserJoinedFromTest();
detector.Arm();
detector.TriggerFromTest();
Assert.True(detector.IsCancelledByRealUserForTest);
Assert.False(detector.IsArmedForTest);
Assert.False(detector.HasFiredForTest);
Assert.Null(FindHybridLoginPending(f.Ingress));
}
[Fact]
public void Placeholder_user_does_not_cancel_emission()
{
// Placeholder is the EXPECTED state at the time the detector arms — only the
// appearance of a REAL AAD user (not foouser/autopilot) should cancel.
// The detector subscribes to AadUserJoined (the real-user event) only — the
// placeholder pathway never fires that event, so we don't need to simulate it
// here; just verifying the timer fires through unchanged.
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
// No TriggerRealUserJoinedFromTest() — only a placeholder appeared in JoinInfo.
detector.TriggerFromTest();
Assert.False(detector.IsCancelledByRealUserForTest);
Assert.True(detector.HasFiredForTest);
Assert.NotNull(FindHybridLoginPending(f.Ingress));
}
// ============================================================================
// Lifecycle / Dispose
// ============================================================================
[Fact]
public void Dispose_after_Arm_does_not_throw()
{
using var f = new Fixture();
var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
detector.Dispose();
// No emission expected — dispose ate the timer before it fired.
Assert.Null(FindHybridLoginPending(f.Ingress));
}
[Fact]
public void TriggerFromTest_after_Dispose_does_not_emit()
{
using var f = new Fixture();
var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger);
detector.Arm();
detector.Dispose();
detector.TriggerFromTest();
Assert.Null(FindHybridLoginPending(f.Ingress));
}
[Fact]
public void Ctor_null_args_throw()
{
using var f = new Fixture();
Assert.Throws<ArgumentNullException>(() => new HybridLoginPendingDetector(null!, f.Post, f.Logger));
Assert.Throws<ArgumentNullException>(() => new HybridLoginPendingDetector(f.Watcher, null!, f.Logger));
Assert.Throws<ArgumentNullException>(() => new HybridLoginPendingDetector(f.Watcher, f.Post, null!));
}
// ============================================================================
// Custom delay
// ============================================================================
[Fact]
public void Custom_delay_is_reflected_in_payload()
{
using var f = new Fixture();
using var detector = new HybridLoginPendingDetector(f.Watcher, f.Post, f.Logger, TimeSpan.FromMinutes(15));
detector.Arm();
detector.TriggerFromTest();
var info = FindHybridLoginPending(f.Ingress);
Assert.NotNull(info);
Assert.Equal("15", info!.Payload!["delayMinutes"]);
}
}
}
@@ -5,7 +5,9 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.SignalAdapters;
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using SharedEventTypes = AutopilotMonitor.Shared.Constants.EventTypes;
using Xunit;
namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
@@ -43,15 +45,42 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
adapter.TriggerFromTest("alice@contoso.com", "abcd1234");
var posted = Assert.Single(f.Ingress.Posted);
Assert.Equal(DecisionSignalKind.AadUserJoinedLate, posted.Kind);
Assert.Equal("AadJoinWatcher", posted.SourceOrigin);
Assert.Equal(EvidenceKind.Derived, posted.Evidence.Kind);
Assert.Equal("contoso.com", posted.Payload!["userDomain"]);
Assert.Equal("true", posted.Payload["hasThumbprint"]);
// Codex review 2026-05-01: dual-emission — decision signal + informational event.
var decisionSignal = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
Assert.Equal("AadJoinWatcher", decisionSignal.SourceOrigin);
Assert.Equal(EvidenceKind.Derived, decisionSignal.Evidence.Kind);
Assert.Equal("contoso.com", decisionSignal.Payload!["userDomain"]);
Assert.Equal("true", decisionSignal.Payload["hasThumbprint"]);
// PII guard: full email MUST NOT appear in payload.
Assert.DoesNotContain("alice@", string.Concat(posted.Payload.Values));
// PII guard: full email MUST NOT appear in either rail's payload.
Assert.DoesNotContain("alice@", string.Concat(decisionSignal.Payload.Values));
}
[Fact]
public void TriggerFromTest_also_emits_aad_user_joined_late_informational_event()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
adapter.TriggerFromTest("alice@contoso.com", "abcd1234");
// FailureSnapshotBuilder reads the Events table — without this rail it cannot
// see that the real AAD user was ever observed (HandleAadUserJoinedLateV1 is
// observation-only, no timeline effect). Snapshot would always say
// aadJoinState=placeholder/unknown.
var info = f.Ingress.Posted.Single(p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == SharedEventTypes.AadUserJoinedLate);
Assert.Equal("AadJoinWatcher", info.Payload![SignalPayloadKeys.Source]);
Assert.Equal("true", info.Payload[SignalPayloadKeys.ImmediateUpload]);
Assert.Equal("Info", info.Payload[SignalPayloadKeys.Severity]);
Assert.Equal("contoso.com", info.Payload["userDomain"]);
Assert.Equal("true", info.Payload["hasThumbprint"]);
// PII guard: the message must not glue local-part to domain.
Assert.DoesNotContain("alice@", info.Payload[SignalPayloadKeys.Message]);
}
[Fact]
@@ -62,9 +91,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
adapter.TriggerFromTest("bob@tailspin.com", "x");
var posted = f.Ingress.Posted.Single();
Assert.Equal(DecisionSignalKind.UserAadSignInComplete, posted.Kind);
Assert.Contains("Post-reboot", posted.Evidence.Summary);
var decisionSignal = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.UserAadSignInComplete);
Assert.Contains("Post-reboot", decisionSignal.Evidence.Summary);
// Part 2 mode also dual-emits the corresponding info event.
var info = f.Ingress.Posted.Single(p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == SharedEventTypes.UserAadSignInComplete);
Assert.Equal("tailspin.com", info.Payload!["userDomain"]);
}
[Fact]
@@ -75,7 +111,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
adapter.TriggerFromTest("user@example.org", "");
Assert.Equal("false", f.Ingress.Posted[0].Payload!["hasThumbprint"]);
var decision = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
Assert.Equal("false", decision.Payload!["hasThumbprint"]);
}
[Fact]
@@ -86,11 +123,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
adapter.TriggerFromTest("no-at-sign", "x");
Assert.Equal("unknown", f.Ingress.Posted[0].Payload!["userDomain"]);
var decision = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
Assert.Equal("unknown", decision.Payload!["userDomain"]);
}
[Fact]
public void Duplicate_trigger_is_deduplicated()
public void Duplicate_trigger_is_deduplicated_on_both_rails()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
@@ -98,7 +136,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
adapter.TriggerFromTest("alice@contoso.com", "x");
adapter.TriggerFromTest("alice@contoso.com", "x");
Assert.Single(f.Ingress.Posted);
// Exactly one decision signal + one informational event for the real-user
// path (fire-once semantics on _fired).
Assert.Single(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
Assert.Single(
f.Ingress.Posted,
p => p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et)
&& et == SharedEventTypes.AadUserJoinedLate);
Assert.Equal(2, f.Ingress.Posted.Count);
}
[Fact]
@@ -109,5 +156,130 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
Assert.Throws<ArgumentNullException>(() => new AadJoinWatcherAdapter(f.Watcher, null!, f.Clock));
Assert.Throws<ArgumentNullException>(() => new AadJoinWatcherAdapter(f.Watcher, f.Ingress, null!));
}
// ============================================================================
// Placeholder path (Hybrid completion-gap fix, 2026-05-01)
// ============================================================================
[Fact]
public void TriggerPlaceholderFromTest_emits_aad_placeholder_user_detected_informational_event()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
adapter.TriggerPlaceholderFromTest("foouser@fabrikam.onmicrosoft.com");
var info = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.InformationalEvent);
Assert.Equal(SharedEventTypes.AadPlaceholderUserDetected, info.Payload![SignalPayloadKeys.EventType]);
Assert.Equal("AadJoinWatcher", info.Payload[SignalPayloadKeys.Source]);
Assert.Equal("true", info.Payload[SignalPayloadKeys.ImmediateUpload]);
Assert.Equal("foouser", info.Payload["placeholderType"]);
Assert.Equal("fabrikam.onmicrosoft.com", info.Payload["userDomain"]);
Assert.Contains("registryKey", info.Payload.Keys);
// PII guard: the local part of the email must not appear (only domain).
Assert.DoesNotContain("foouser@", info.Payload[SignalPayloadKeys.Message]);
}
[Fact]
public void TriggerPlaceholderFromTest_classifies_autopilot_placeholder()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
adapter.TriggerPlaceholderFromTest("autopilot@contoso.com");
var info = f.Ingress.Posted.Single(p => p.Kind == DecisionSignalKind.InformationalEvent);
Assert.Equal("autopilot", info.Payload!["placeholderType"]);
Assert.Equal("contoso.com", info.Payload["userDomain"]);
}
[Fact]
public void Duplicate_placeholder_trigger_is_deduplicated()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
adapter.TriggerPlaceholderFromTest("foouser@example.com");
adapter.TriggerPlaceholderFromTest("foouser@example.com");
adapter.TriggerPlaceholderFromTest("foouser@example.com");
Assert.Single(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.InformationalEvent);
}
[Fact]
public void Placeholder_and_real_user_emit_separate_signals_independently()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(f.Watcher, f.Ingress, f.Clock);
adapter.TriggerPlaceholderFromTest("foouser@fabrikam.com");
adapter.TriggerFromTest("alice@fabrikam.com", "abcd");
// 3 posts total after Codex review 2026-05-01 dual-emission:
// - placeholder informational event
// - real-user decision signal (AadUserJoinedLate)
// - real-user informational event (aad_user_joined_late)
Assert.Equal(3, f.Ingress.Posted.Count);
Assert.Single(f.Ingress.Posted, p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et1)
&& et1 == SharedEventTypes.AadPlaceholderUserDetected);
Assert.Single(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
Assert.Single(f.Ingress.Posted, p =>
p.Kind == DecisionSignalKind.InformationalEvent
&& p.Payload != null
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var et2)
&& et2 == SharedEventTypes.AadUserJoinedLate);
}
// ============================================================================
// onRealUserJoined callback (Pkt 5 — DesktopArrivalDetector reset wiring)
// ============================================================================
[Fact]
public void Real_user_join_invokes_onRealUserJoined_callback_exactly_once()
{
using var f = new Fixture();
int invocations = 0;
using var adapter = new AadJoinWatcherAdapter(
f.Watcher, f.Ingress, f.Clock,
onRealUserJoined: () => invocations++);
adapter.TriggerFromTest("alice@contoso.com", "x");
adapter.TriggerFromTest("alice@contoso.com", "x"); // dedup — callback must not fire twice
Assert.Equal(1, invocations);
}
[Fact]
public void Placeholder_does_not_invoke_onRealUserJoined_callback()
{
using var f = new Fixture();
int invocations = 0;
using var adapter = new AadJoinWatcherAdapter(
f.Watcher, f.Ingress, f.Clock,
onRealUserJoined: () => invocations++);
adapter.TriggerPlaceholderFromTest("foouser@example.com");
Assert.Equal(0, invocations);
}
[Fact]
public void Throwing_onRealUserJoined_callback_does_not_break_signal_post()
{
using var f = new Fixture();
using var adapter = new AadJoinWatcherAdapter(
f.Watcher, f.Ingress, f.Clock,
onRealUserJoined: () => throw new InvalidOperationException("boom"));
// The signal post completes BEFORE the callback fires — even if the callback
// throws, the decision signal must already be on the bus.
adapter.TriggerFromTest("alice@contoso.com", "x");
Assert.Single(f.Ingress.Posted, p => p.Kind == DecisionSignalKind.AadUserJoinedLate);
}
}
}
@@ -67,6 +67,38 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
_pollingTimer = null;
}
/// <summary>
/// Resets desktop-arrival tracking after a placeholder→real-user transition (Hybrid
/// User-Driven completion-gap fix, 2026-05-01). Used by the composition root when
/// <see cref="AadJoinWatcher.AadUserJoined"/> fires for a real user — the previous
/// fooUser desktop the detector observed is invalidated, and polling restarts so the
/// AD-user desktop after the Hybrid reboot is detected as the actual real desktop.
/// <para>
/// Idempotent: safe to call multiple times. No-op if the detector already fired
/// after the reset (a subsequent real-user join after a real-user desktop is also
/// idempotent because the polling timer is restarted regardless and the next match
/// against IsExcludedUser will short-circuit on the still-valid desktop).
/// </para>
/// </summary>
public void ResetForRealUserSwitch()
{
_logger.Info("DesktopArrivalDetector: reset for real-user switch (placeholder→real user transition)");
// Drop any prior arrival state — subsequent polls must re-evaluate the current
// explorer.exe owner against IsExcludedUser anew.
_desktopArrived = false;
_excludedUserTraced = false;
// Restart polling. Dispose any existing timer first to avoid leaking ticks
// from an already-running schedule.
_pollingTimer?.Dispose();
_pollingTimer = new Timer(
PollForDesktop,
null,
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(PollingIntervalSeconds));
}
private void PollForDesktop(object state)
{
if (_desktopArrived)
@@ -179,14 +211,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
/// <summary>
/// Returns true if the user name matches any excluded system/service account.
/// Handles both "User" and "DOMAIN\User" formats.
/// Also matches the pattern DefaultUser* (case-insensitive).
/// Handles "User", "DOMAIN\User", and UPN ("user@domain") formats.
/// Also matches the patterns DefaultUser* and the Autopilot provisioning placeholders
/// foouser@* / autopilot@* (case-insensitive). The placeholder match prevents the
/// fooUser OOBE shell on Hybrid User-Driven enrollments from being treated as a
/// real user desktop.
/// </summary>
internal static bool IsExcludedUser(string fullUserName)
{
if (string.IsNullOrEmpty(fullUserName))
return true;
// UPN form (user@domain) — delegate to the same placeholder oracle the
// AadJoinWatcher uses, so the foouser@/autopilot@ list stays in one place.
if (fullUserName.IndexOf('@') >= 0
&& AadJoinInfo.IsPlaceholderUserEmail(fullUserName))
return true;
// Extract just the username part (after backslash if present)
var userName = fullUserName;
var backslashIndex = fullUserName.LastIndexOf('\\');
@@ -204,6 +245,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
if (userName.StartsWith("DefaultUser", StringComparison.OrdinalIgnoreCase))
return true;
// DOMAIN\foouser / DOMAIN\autopilot — the WMI GetOwner call sometimes returns a
// synthetic local-machine domain instead of the UPN. EXACT match the bare
// username here (not prefix), so legitimate real accounts like
// CONTOSO\autopilotadmin or DOMAIN\foouserservice stay through the gate.
// Codex review 2026-05-01 (Finding 3): the previous prefix match was too broad
// and reused via UserProfileResolver, which would have resolved to the wrong
// user profile for any account starting with "autopilot" or "foouser".
// The UPN form (foouser@*, autopilot@*) is still handled above by
// AadJoinInfo.IsPlaceholderUserEmail — that path is what covers the
// real-world Autopilot placeholders (foouser@<tenant>.onmicrosoft.com).
if (string.Equals(userName, "foouser", StringComparison.OrdinalIgnoreCase)
|| string.Equals(userName, "autopilot", StringComparison.OrdinalIgnoreCase))
return true;
return false;
}
@@ -0,0 +1,157 @@
#nullable enable
using System;
using System.Collections.Generic;
using System.Threading;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Shared.Models;
using SharedConstants = AutopilotMonitor.Shared.Constants;
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
{
/// <summary>
/// Single-shot detector for the Hybrid User-Driven completion-gap (2026-05-01 trigger:
/// session e58bcfdb-3e68-4f23-a3c2-437429ca9e78). After a Hybrid AAD-Join reboot the
/// agent restarts and waits for the user to log in with their real AD/AAD account
/// (which replaces the foouser/autopilot placeholder in JoinInfo). When that login is
/// overdue the agent today goes silent until the backend 5-h watchdog fires; this
/// detector emits a <c>hybrid_login_pending</c> warning event after a short single-shot
/// timer so the operator sees an explicit "stuck waiting for AD login" signal in the
/// timeline.
/// <para>
/// Conditions checked at <see cref="Arm"/>: composition root must have already
/// confirmed a) post-reboot, b) <c>isHybridJoin == true</c>. The detector itself only
/// owns the timer + <see cref="AadJoinWatcher.AadUserJoined"/> cancel-path. No polling.
/// No periodicity. Fires at most once per agent process.
/// </para>
/// </summary>
internal sealed class HybridLoginPendingDetector : IDisposable
{
internal const int DefaultDelayMinutes = 10;
internal const string SourceLabel = "HybridLoginPendingDetector";
private readonly AadJoinWatcher _watcher;
private readonly InformationalEventPost _post;
private readonly AgentLogger _logger;
private readonly TimeSpan _delay;
private readonly object _lock = new object();
private Timer? _timer;
private bool _armed;
private bool _fired;
private bool _cancelledByRealUser;
private int _disposed;
internal HybridLoginPendingDetector(
AadJoinWatcher watcher,
InformationalEventPost post,
AgentLogger logger,
TimeSpan? delay = null)
{
_watcher = watcher ?? throw new ArgumentNullException(nameof(watcher));
_post = post ?? throw new ArgumentNullException(nameof(post));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_delay = delay ?? TimeSpan.FromMinutes(DefaultDelayMinutes);
// Subscribe up-front so a real-user join that arrives BEFORE Arm() is still
// recorded — _cancelledByRealUser short-circuits any later Arm() call.
_watcher.AadUserJoined += OnAadUserJoined;
}
/// <summary>
/// Starts the single-shot timer. Idempotent — repeated calls after the first arm
/// are no-ops. Has no effect if the watcher already saw a real AAD user (the
/// arming was racy and the cancel won), or if the detector already fired.
/// </summary>
public void Arm()
{
lock (_lock)
{
if (_disposed != 0) return;
if (_armed || _fired) return;
if (_cancelledByRealUser)
{
_logger.Info("HybridLoginPendingDetector: arm requested but real AAD user already joined — skipped");
return;
}
_armed = true;
_logger.Info(
$"HybridLoginPendingDetector: armed — will emit hybrid_login_pending in {_delay.TotalMinutes:F0} min if real AAD user has not joined by then");
_timer = new Timer(OnTimer, null, _delay, Timeout.InfiniteTimeSpan);
}
}
public void Dispose()
{
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
try { _watcher.AadUserJoined -= OnAadUserJoined; } catch { }
lock (_lock)
{
_timer?.Dispose();
_timer = null;
}
}
// Test seam — bypasses the timer schedule. Same lock + state guards as the real path.
internal void TriggerFromTest() => OnTimer(null);
// Test seam — synthesises an AadUserJoined arrival without going through the
// watcher's private event-raise path. The watcher's events can only be invoked from
// inside the watcher class, so production unit tests have no way to observe the
// detector's cancel-on-real-user behavior otherwise.
internal void TriggerRealUserJoinedFromTest() =>
OnAadUserJoined(this, new AadUserJoinedEventArgs("test@example.com", "test-thumbprint"));
internal bool IsArmedForTest { get { lock (_lock) { return _armed; } } }
internal bool IsCancelledByRealUserForTest { get { lock (_lock) { return _cancelledByRealUser; } } }
internal bool HasFiredForTest { get { lock (_lock) { return _fired; } } }
private void OnTimer(object? state) => EmitInternal(reason: "timer_fired");
private void OnAadUserJoined(object sender, AadUserJoinedEventArgs e)
{
lock (_lock)
{
_cancelledByRealUser = true;
if (!_armed || _fired) return;
_timer?.Dispose();
_timer = null;
_logger.Info("HybridLoginPendingDetector: cancelled — real AAD user joined before timeout");
}
}
private void EmitInternal(string reason)
{
lock (_lock)
{
// _armed guard (Codex review 2026-05-01): production is safe because the
// timer is only created in Arm(), but explicit guard makes the contract
// unambiguous — emission requires a deliberate Arm(), full stop.
if (!_armed || _fired || _cancelledByRealUser || _disposed != 0) return;
_fired = true;
_timer?.Dispose();
_timer = null;
}
var data = new Dictionary<string, string>(StringComparer.Ordinal)
{
["delayMinutes"] = ((int)_delay.TotalMinutes).ToString(),
["reason"] = reason,
["isHybridJoin"] = "true", // armed only when composition root verified this
};
_post.Emit(
eventType: SharedConstants.EventTypes.HybridLoginPending,
source: SourceLabel,
message: $"Hybrid AAD Join: {(int)_delay.TotalMinutes} min after reboot still no real AD user — login overdue (placeholder still active in JoinInfo)",
severity: EventSeverity.Warning,
immediateUpload: true,
data: data);
_logger.Warning("HybridLoginPendingDetector: emitted hybrid_login_pending warning");
}
}
}
@@ -69,6 +69,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
private ImeLogHost? _imeLogHost;
private EspAndHelloHost? _espAndHelloHost;
private AadJoinHost? _aadJoinHost;
private Transport.Telemetry.ITelemetrySpool? _telemetrySpool;
/// <summary>
@@ -84,6 +85,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
/// </summary>
internal EspAndHelloHost? EspAndHelloHost => _espAndHelloHost;
/// <summary>
/// Exposes the <see cref="AadJoinHost"/> so the runtime host can arm the Hybrid
/// User-Driven login-pending detector after observing a reboot-kill on a Hybrid-AAD
/// device (2026-05-01 completion-gap fix). Returns <c>null</c> before
/// <see cref="CreateCollectorHosts"/> has been called.
/// </summary>
internal AadJoinHost? AadJoinHost => _aadJoinHost;
/// <summary>
/// Exposes the IME tracker's package-state list to peripheral consumers such as the
/// <c>FinalStatusBuilder</c> in M4.6.β. Returns <c>null</c> before
@@ -178,9 +187,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
stateDirectory: _stateDirectory);
hosts.Add(_espAndHelloHost);
hosts.Add(new DesktopArrivalHost(logger, ingress, clock));
// Hybrid User-Driven completion-gap fix (2026-05-01): the AadJoinHost notifies
// the DesktopArrivalHost when a real AAD user replaces the foouser/autopilot
// placeholder, so the detector resets and re-evaluates after the Hybrid reboot
// instead of staying latched on the foo desktop. Order matters — DesktopArrivalHost
// must exist before AadJoinHost so the callback target is available.
var desktopArrivalHost = new DesktopArrivalHost(logger, ingress, clock);
hosts.Add(desktopArrivalHost);
hosts.Add(new AadJoinHost(logger, ingress, clock));
_aadJoinHost = new AadJoinHost(
logger, ingress, clock,
onRealUserJoined: desktopArrivalHost.RequestResetForRealUserSwitch);
hosts.Add(_aadJoinHost);
// Single-rail refactor (plan §5.8) — DeviceInfoCollector existed in V2.Core but had
// no host so the Device-Details UI block was empty in V2 sessions (V1-Parity Issue #2).
@@ -1,4 +1,5 @@
#nullable enable
using System;
using System.Threading;
using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
@@ -13,20 +14,43 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
private readonly AadJoinWatcher _watcher;
private readonly AadJoinWatcherAdapter _adapter;
private readonly HybridLoginPendingDetector _hybridLoginPendingDetector;
private int _disposed;
public AadJoinHost(AgentLogger logger, ISignalIngressSink ingress, IClock clock)
public AadJoinHost(
AgentLogger logger,
ISignalIngressSink ingress,
IClock clock,
Action? onRealUserJoined = null)
{
_watcher = new AadJoinWatcher(logger);
_adapter = new AadJoinWatcherAdapter(_watcher, ingress, clock);
_adapter = new AadJoinWatcherAdapter(_watcher, ingress, clock, onRealUserJoined: onRealUserJoined);
_hybridLoginPendingDetector = new HybridLoginPendingDetector(
watcher: _watcher,
post: new InformationalEventPost(ingress, clock),
logger: logger);
}
public void Start() => _watcher.Start();
public void Stop() => _watcher.Stop();
/// <summary>
/// Composition-root entry point for the Hybrid User-Driven completion-gap fix
/// (2026-05-01). Only call when both prerequisites hold: a) the prior agent process
/// was killed by an OS reboot (<c>previousExitType=reboot_kill</c>), b) the
/// Autopilot profile is Hybrid AAD Join (<c>isHybridJoin=true</c>). The detector
/// itself enforces single-shot semantics + cancel-on-real-user — repeated calls or
/// late calls after a real user joined are safe no-ops.
/// </summary>
public void ArmHybridLoginPendingDetector() => _hybridLoginPendingDetector.Arm();
// Test seam — exposes the detector for unit tests in the V2.Core.Tests project.
internal HybridLoginPendingDetector HybridLoginPendingDetectorForTest => _hybridLoginPendingDetector;
public void Dispose()
{
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
try { _hybridLoginPendingDetector.Dispose(); } catch { }
try { _adapter.Dispose(); } catch { }
try { _watcher.Dispose(); } catch { }
}
@@ -24,6 +24,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
public void Start() => _detector.Start();
public void Stop() => _detector.Stop();
/// <summary>
/// Resets desktop-arrival tracking after a placeholder→real-user transition
/// (Hybrid User-Driven completion-gap fix, 2026-05-01). Wired by the composition
/// root through <see cref="AadJoinHost"/>'s <c>onRealUserJoined</c> callback so the
/// fooUser desktop the detector observed during foo-OOBE is invalidated and polling
/// restarts to detect the AD-user desktop after the Hybrid reboot.
/// </summary>
public void RequestResetForRealUserSwitch() => _detector.ResetForRealUserSwitch();
public void Dispose()
{
if (Interlocked.Exchange(ref _disposed, 1) == 1) return;
@@ -5,50 +5,70 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.DecisionCore.Engine;
using AutopilotMonitor.DecisionCore.Signals;
using AutopilotMonitor.Shared.Models;
using SharedConstants = AutopilotMonitor.Shared.Constants;
namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
{
/// <summary>
/// Adapter for <see cref="AadJoinWatcher"/> → <see cref="DecisionSignalKind.AadUserJoinedLate"/>
/// (Part 1) or <see cref="DecisionSignalKind.UserAadSignInComplete"/> (Part 2). Plan §2.1a / §2.2.
/// <para>
/// Only the <c>AadUserJoined</c>-Event (real user, non-placeholder) ist Decision-relevant
/// und wird als Signal emittiert. <c>PlaceholderUserDetected</c> ist diagnostisch
/// (transienter Provisioning-Account) und wird aktuell nicht als Decision-Signal
/// weitergereicht — bei Bedarf später als Event-Timeline-Entry emittierbar.
/// </para>
/// Adapter for <see cref="AadJoinWatcher"/>:
/// <list type="bullet">
/// <item><description><see cref="AadJoinWatcher.AadUserJoined"/> →
/// <see cref="DecisionSignalKind.AadUserJoinedLate"/> (Part 1) or
/// <see cref="DecisionSignalKind.UserAadSignInComplete"/> (Part 2). Plan §2.1a / §2.2.</description></item>
/// <item><description><see cref="AadJoinWatcher.PlaceholderUserDetected"/> → emits a
/// <c>aad_placeholder_user_detected</c> informational event for backend timeline
/// visibility (Hybrid User-Driven completion-gap fix, 2026-05-01). NOT a decision signal —
/// the engine has no <see cref="DecisionSignalKind"/> for placeholder presence and the
/// placeholder is expected to be replaced by a real user.</description></item>
/// </list>
/// </summary>
internal sealed class AadJoinWatcherAdapter : IDisposable
{
private const string SourceLabel = "AadJoinWatcher";
private readonly AadJoinWatcher _watcher;
private readonly ISignalIngressSink _ingress;
private readonly IClock _clock;
private readonly InformationalEventPost _post;
private readonly Action? _onRealUserJoined;
private readonly bool _part2Mode;
private bool _fired;
private bool _placeholderFired;
public AadJoinWatcherAdapter(
AadJoinWatcher watcher,
ISignalIngressSink ingress,
IClock clock,
bool part2Mode = false)
bool part2Mode = false,
Action? onRealUserJoined = null)
{
_watcher = watcher ?? throw new ArgumentNullException(nameof(watcher));
_ingress = ingress ?? throw new ArgumentNullException(nameof(ingress));
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
_post = new InformationalEventPost(ingress, clock);
_part2Mode = part2Mode;
_onRealUserJoined = onRealUserJoined;
_watcher.AadUserJoined += OnAadUserJoined;
_watcher.PlaceholderUserDetected += OnPlaceholderUserDetected;
}
public void Dispose()
{
_watcher.AadUserJoined -= OnAadUserJoined;
_watcher.PlaceholderUserDetected -= OnPlaceholderUserDetected;
}
private void OnAadUserJoined(object sender, AadUserJoinedEventArgs e) => EmitInternal(e.UserEmail, e.Thumbprint);
private void OnPlaceholderUserDetected(object sender, AadPlaceholderUserDetectedEventArgs e) =>
EmitPlaceholderInternal(e.UserEmail);
internal void TriggerFromTest(string userEmail, string thumbprint) => EmitInternal(userEmail, thumbprint);
internal void TriggerPlaceholderFromTest(string userEmail) => EmitPlaceholderInternal(userEmail);
private void EmitInternal(string userEmail, string thumbprint)
{
if (_fired) return;
@@ -67,7 +87,7 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
_ingress.Post(
kind: kind,
occurredAtUtc: _clock.UtcNow,
sourceOrigin: "AadJoinWatcher",
sourceOrigin: SourceLabel,
evidence: new Evidence(
kind: EvidenceKind.Derived,
identifier: "aad-join-watcher-v1",
@@ -79,6 +99,61 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
["registryKey"] = @"HKLM\SYSTEM\CurrentControlSet\Control\CloudDomainJoin\JoinInfo",
}),
payload: payload);
// Codex review 2026-05-01 — also dual-emit as informational event.
// HandleAadUserJoinedLateV1 / Part 2 handler in DecisionEngine.Classic is
// observation-only (no timeline effect), so without this dual-emission the
// backend Events table never sees the real-user join. FailureSnapshotBuilder
// (and any other event-stream consumer) needs this anchor to correctly classify
// aadJoinState=real_user. Same PII-safe payload shape as the decision signal.
var infoEventType = _part2Mode
? SharedConstants.EventTypes.UserAadSignInComplete
: SharedConstants.EventTypes.AadUserJoinedLate;
var infoMessage = _part2Mode
? $"Post-reboot AAD user sign-in detected (domain={payload["userDomain"]})"
: $"Late AAD user join detected (domain={payload["userDomain"]})";
_post.Emit(
eventType: infoEventType,
source: SourceLabel,
message: infoMessage,
severity: EventSeverity.Info,
immediateUpload: true,
data: payload,
occurredAtUtc: _clock.UtcNow);
// Pkt 5 — notify the composition root that a real AAD user has appeared, so the
// DesktopArrivalDetector can be reset for the AD-user-after-Hybrid-reboot case.
// Fire AFTER the decision signal so reducer-state ordering remains the canonical source.
try { _onRealUserJoined?.Invoke(); }
catch { /* callback is best-effort — never let host wiring break the signal post */ }
}
private void EmitPlaceholderInternal(string userEmail)
{
if (_placeholderFired) return;
_placeholderFired = true;
// Hybrid User-Driven OOBE: foouser@/autopilot@ is the transient provisioning account.
// Emit a backend-timeline event so operators can see "Foo phase active" — gives the
// diagnose path an explicit anchor when the real-user join is overdue.
var placeholderType = ClassifyPlaceholder(userEmail);
var data = new Dictionary<string, string>(StringComparer.Ordinal)
{
["userDomain"] = ExtractDomain(userEmail),
["placeholderType"] = placeholderType,
["registryKey"] = @"HKLM\SYSTEM\CurrentControlSet\Control\CloudDomainJoin\JoinInfo",
};
// PII-friendly message: placeholderType + domain reported separately so the
// local-part of the email is never glued to the domain in the message string.
_post.Emit(
eventType: SharedConstants.EventTypes.AadPlaceholderUserDetected,
source: SourceLabel,
message: $"Autopilot provisioning placeholder detected (type={placeholderType}, domain={data["userDomain"]}) — waiting for real AAD user join",
severity: EventSeverity.Info,
immediateUpload: true,
data: data,
occurredAtUtc: _clock.UtcNow);
}
private static string ExtractDomain(string email)
@@ -87,5 +162,13 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
var at = email.IndexOf('@');
return at >= 0 && at + 1 < email.Length ? email.Substring(at + 1) : "unknown";
}
private static string ClassifyPlaceholder(string email)
{
if (string.IsNullOrEmpty(email)) return "unknown";
if (email.StartsWith("foouser@", StringComparison.OrdinalIgnoreCase)) return "foouser";
if (email.StartsWith("autopilot@", StringComparison.OrdinalIgnoreCase)) return "autopilot";
return "unknown";
}
}
}
@@ -6,6 +6,7 @@ using AutopilotMonitor.Agent.V2.Core.Logging;
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
using AutopilotMonitor.Agent.V2.Core.Orchestration;
using AutopilotMonitor.Agent.V2.Core.Runtime;
using AutopilotMonitor.Agent.V2.Core.Security;
using AutopilotMonitor.Agent.V2.Core.Termination;
using AutopilotMonitor.DecisionCore.Classifiers;
using AutopilotMonitor.DecisionCore.Engine;
@@ -333,6 +334,22 @@ namespace AutopilotMonitor.Agent.V2.Runtime
if (string.Equals(previousExit?.ExitType, "reboot_kill", StringComparison.OrdinalIgnoreCase))
{
LifecycleEmitters.PostSystemRebootObserved(orchestrator.IngressSink, previousExit, logger);
// Hybrid User-Driven completion-gap fix (2026-05-01): the reboot is the
// expected switch from foouser/autopilot OOBE to the real AD account.
// If the user never logs in we go silent until the 5-h backend watchdog;
// arm a single-shot 10-min detector that emits hybrid_login_pending
// once if the AAD user join is overdue. Cancelled when AadJoinWatcher
// sees the real user. Only armed for actual Hybrid devices — non-Hybrid
// reboots don't have the placeholder→real-user flow that motivates this.
if (!isWhiteGloveResume && EnrollmentRegistryDetector.DetectHybridJoin())
{
try { componentFactory.AadJoinHost?.ArmHybridLoginPendingDetector(); }
catch (Exception ex)
{
logger.Warning($"ArmHybridLoginPendingDetector threw: {ex.Message}");
}
}
}
// V2 race-fix follow-up (10c8e0bf review, 2026-04-27) —
@@ -1 +1 @@
520
541
@@ -1 +1 @@
1235
1240
@@ -0,0 +1,350 @@
using System;
using System.Collections.Generic;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json.Linq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Tests for the failure-state snapshot built when the maintenance 5h-timeout sweep
/// graduates a session to Failed (Hybrid User-Driven completion-gap fix, 2026-05-01).
/// The snapshot is the single point of post-mortem context for stuck sessions, so the
/// shape contract has to be stable: schemaVersion, missingSignals list, and the lifecycle
/// anchors all need to round-trip cleanly. Tests focus on the diagnostic value: would an
/// operator looking at the snapshot for session e58bcfdb-… understand what went wrong?
/// </summary>
public class FailureSnapshotBuilderTests
{
private static readonly DateTime Now = new DateTime(2026, 5, 1, 18, 45, 31, DateTimeKind.Utc);
private static EnrollmentEvent Event(string type, DateTime ts, string? message = null,
Dictionary<string, object>? data = null) => new()
{
EventType = type,
Timestamp = ts,
Source = "test",
Message = message ?? type,
Data = data!,
};
[Fact]
public void Build_returns_null_for_empty_event_list()
{
Assert.Null(FailureSnapshotBuilder.Build(null, Now));
Assert.Null(FailureSnapshotBuilder.Build(Array.Empty<EnrollmentEvent>(), Now));
}
[Fact]
public void Build_carries_schemaVersion_and_generation_metadata()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("agent_started", Now.AddHours(-5)),
}, Now);
Assert.NotNull(json);
var obj = JObject.Parse(json!);
Assert.Equal(FailureSnapshotBuilder.CurrentSchemaVersion, (int)obj["schemaVersion"]!);
Assert.NotNull(obj["generatedAtUtc"]);
Assert.Equal(1, (int)obj["eventCount"]!);
}
[Fact]
public void Build_extracts_last_esp_phase_from_esp_phase_changed_events()
{
var t0 = Now.AddHours(-5);
var json = FailureSnapshotBuilder.Build(new[]
{
Event("esp_phase_changed", t0.AddSeconds(1),
data: new Dictionary<string, object> { ["espPhase"] = "DeviceSetup" }),
Event("esp_phase_changed", t0.AddSeconds(3),
data: new Dictionary<string, object> { ["espPhase"] = "AccountSetup" }),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("AccountSetup", (string?)obj["lastEspPhase"]);
Assert.NotNull(obj["lastEspPhaseAtUtc"]);
}
[Fact]
public void Build_records_desktop_arrival_with_timestamp()
{
var t = Now.AddHours(-5).AddSeconds(7);
var json = FailureSnapshotBuilder.Build(new[]
{
Event("desktop_arrived", t),
}, Now);
var obj = JObject.Parse(json!);
Assert.True((bool)obj["desktopArrived"]!);
Assert.NotNull(obj["desktopArrivedAtUtc"]);
}
[Fact]
public void Build_records_hello_policy_disabled_state()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("hello_policy_detected", Now.AddHours(-4),
data: new Dictionary<string, object> { ["helloEnabled"] = "false" }),
}, Now);
var obj = JObject.Parse(json!);
Assert.True((bool)obj["helloPolicyDetected"]!);
Assert.False((bool)obj["helloPolicyEnabled"]!);
}
// ============================================================================
// AAD join-state classification
// ============================================================================
[Fact]
public void Build_classifies_placeholder_state_from_aad_placeholder_event()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("aad_placeholder_user_detected", Now.AddHours(-4),
data: new Dictionary<string, object> { ["placeholderType"] = "foouser" }),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("placeholder", (string?)obj["aadJoinState"]);
}
[Fact]
public void Build_classifies_real_user_state_from_aad_user_joined_late()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("aad_user_joined_late", Now.AddHours(-3)),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("real_user", (string?)obj["aadJoinState"]);
}
[Fact]
public void Build_aad_user_join_slot_is_satisfied_by_part_1_event()
{
// Codex review 2026-05-01 (Finding 1): the dual-emitted informational event
// is what makes this work. Before the fix, AadJoinWatcherAdapter only posted
// a DecisionSignal — HandleAadUserJoinedLateV1 emits no timeline effect, so
// the Events table never held an aad_user_joined_late row. The snapshot then
// always marked it missing AND classified aadJoinState as placeholder/unknown
// even on healthy real-user joins.
var json = FailureSnapshotBuilder.Build(new[]
{
Event("aad_user_joined_late", Now.AddHours(-3)),
}, Now);
var obj = JObject.Parse(json!);
var missing = obj["missingSignals"]!.ToObject<List<string>>()!;
Assert.DoesNotContain("aad_user_join", missing); // slot satisfied
Assert.DoesNotContain("aad_user_joined_late", missing); // literal must never appear
}
[Fact]
public void Build_aad_user_join_slot_is_satisfied_by_part_2_event()
{
// Codex review 2026-05-01 (Finding 4): Part 2 (post-reboot user sign-in) emits
// user_aad_signin_complete instead of aad_user_joined_late. Both must satisfy
// the same conceptual slot — analog to hello_terminal admitting any of the
// four Hello terminal event types. Without the new branch, every healthy Part-2
// session would falsely list aad_user_joined_late as missing.
var json = FailureSnapshotBuilder.Build(new[]
{
Event("user_aad_signin_complete", Now.AddHours(-3)),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("real_user", (string?)obj["aadJoinState"]);
var missing = obj["missingSignals"]!.ToObject<List<string>>()!;
Assert.DoesNotContain("aad_user_join", missing); // slot satisfied
Assert.DoesNotContain("aad_user_joined_late", missing); // literal must never appear
Assert.DoesNotContain("user_aad_signin_complete", missing); // ditto
}
[Fact]
public void Build_aad_user_join_slot_is_listed_missing_when_neither_part_event_present()
{
// The trigger session e58bcfdb saw no real-user join at all — the slot must
// be reported missing. Bare event list (just agent_started) keeps the test
// focused on the slot logic; the e58bcfdb-replay test below covers the
// realistic full anchor set.
var json = FailureSnapshotBuilder.Build(new[]
{
Event("agent_started", Now.AddHours(-5)),
}, Now);
var obj = JObject.Parse(json!);
var missing = obj["missingSignals"]!.ToObject<List<string>>()!;
Assert.Contains("aad_user_join", missing);
}
[Fact]
public void Build_falls_back_to_aad_join_status_isFooUser_when_only_that_is_present()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("aad_join_status", Now.AddHours(-4),
data: new Dictionary<string, object> { ["isFooUser"] = "true" }),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("placeholder", (string?)obj["aadJoinState"]);
}
[Fact]
public void Build_real_user_event_overrides_earlier_placeholder()
{
// The temporal pattern in Hybrid User-Driven: foo desktop appears (placeholder),
// reboot, real user signs in (real_user). Snapshot should report the LATEST state.
var json = FailureSnapshotBuilder.Build(new[]
{
Event("aad_placeholder_user_detected", Now.AddHours(-5)),
Event("aad_user_joined_late", Now.AddHours(-3)),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("real_user", (string?)obj["aadJoinState"]);
}
// ============================================================================
// Reboot + Hybrid context
// ============================================================================
[Fact]
public void Build_records_reboot_observation_and_hybrid_flag()
{
var json = FailureSnapshotBuilder.Build(new[]
{
Event("autopilot_profile", Now.AddHours(-5),
data: new Dictionary<string, object> { ["isHybridJoin"] = "true", ["enrollmentType"] = "v1" }),
Event("system_reboot_detected", Now.AddHours(-4)),
}, Now);
var obj = JObject.Parse(json!);
Assert.True((bool)obj["rebootObserved"]!);
Assert.True((bool)obj["isHybridJoin"]!);
Assert.Equal("v1", (string?)obj["enrollmentType"]);
}
// ============================================================================
// Missing-signal computation — the diagnostic core of the snapshot
// ============================================================================
[Fact]
public void Build_lists_missing_signals_for_e58bcfdb_style_session()
{
// Replays the canonical anchors from session e58bcfdb-… (the trigger session for
// this fix). Confirms the snapshot would have surfaced exactly the gaps we
// identified in the diagnosis: no esp_exiting, no hello terminal, no
// aad_user_joined_late, no enrollment_complete.
var t0 = new DateTime(2026, 5, 1, 13, 45, 31, DateTimeKind.Utc);
var nowAtTimeout = t0.AddHours(5);
var events = new[]
{
Event("agent_started", t0),
Event("autopilot_profile", t0.AddSeconds(1),
data: new Dictionary<string, object> { ["isHybridJoin"] = "true", ["enrollmentType"] = "v1" }),
Event("esp_phase_changed", t0.AddSeconds(2),
data: new Dictionary<string, object> { ["espPhase"] = "DeviceSetup" }),
Event("aad_join_status", t0.AddSeconds(2),
data: new Dictionary<string, object> { ["isFooUser"] = "true" }),
Event("esp_phase_changed", t0.AddSeconds(3),
data: new Dictionary<string, object> { ["espPhase"] = "AccountSetup" }),
Event("desktop_arrived", t0.AddSeconds(6)),
Event("system_reboot_detected", t0.AddMinutes(25).AddSeconds(53)),
};
var json = FailureSnapshotBuilder.Build(events, nowAtTimeout);
var obj = JObject.Parse(json!);
var missing = obj["missingSignals"]!.ToObject<List<string>>()!;
Assert.Contains("esp_exiting", missing);
Assert.Contains("hello_policy_detected", missing);
Assert.Contains("hello_terminal", missing);
Assert.Contains("aad_user_join", missing); // conceptual slot — Part 1 OR Part 2
Assert.Contains("enrollment_complete", missing);
// Things the session DID see should NOT be in the missing list.
Assert.DoesNotContain("esp_phase_changed", missing);
Assert.DoesNotContain("desktop_arrived", missing);
// Slot label is the conceptual name; the literal Part 1 / Part 2 event-type
// names must NOT appear directly in missingSignals (that would double-count).
Assert.DoesNotContain("aad_user_joined_late", missing);
Assert.DoesNotContain("user_aad_signin_complete", missing);
// High-level diagnostic fields render correctly.
Assert.Equal("AccountSetup", (string?)obj["lastEspPhase"]);
Assert.True((bool)obj["desktopArrived"]!);
Assert.True((bool)obj["rebootObserved"]!);
Assert.True((bool)obj["isHybridJoin"]!);
Assert.Equal("placeholder", (string?)obj["aadJoinState"]);
}
[Fact]
public void Build_treats_any_hello_terminal_as_satisfying_the_slot()
{
// hello_terminal is the conceptual slot — any of {completed, failed, blocked, skipped}
// satisfies it. Tested with hello_skipped which is the most common Hybrid path.
var json = FailureSnapshotBuilder.Build(new[]
{
Event("hello_skipped", Now.AddHours(-4)),
}, Now);
var obj = JObject.Parse(json!);
var missing = obj["missingSignals"]!.ToObject<List<string>>()!;
Assert.DoesNotContain("hello_terminal", missing);
}
// ============================================================================
// Timing / stale silence
// ============================================================================
[Fact]
public void Build_computes_silenceMinutes_from_last_event_to_now()
{
var t = Now.AddHours(-3); // last event was 3h ago
var json = FailureSnapshotBuilder.Build(new[] { Event("agent_started", t) }, Now);
var obj = JObject.Parse(json!);
var silence = (int)obj["silenceMinutes"]!;
Assert.InRange(silence, 179, 181);
}
[Fact]
public void Build_handles_unsorted_input_via_secondary_sort()
{
// The contract says events come sorted, but the builder defensively re-sorts.
var t0 = Now.AddHours(-5);
var json = FailureSnapshotBuilder.Build(new[]
{
Event("esp_phase_changed", t0.AddSeconds(3),
data: new Dictionary<string, object> { ["espPhase"] = "AccountSetup" }),
Event("esp_phase_changed", t0.AddSeconds(1),
data: new Dictionary<string, object> { ["espPhase"] = "DeviceSetup" }),
}, Now);
var obj = JObject.Parse(json!);
Assert.Equal("AccountSetup", (string?)obj["lastEspPhase"]);
}
[Fact]
public void Build_tolerates_events_with_null_or_missing_data()
{
// Events stored before the Data field was added still have to parse cleanly.
var json = FailureSnapshotBuilder.Build(new[]
{
new EnrollmentEvent { EventType = "agent_started", Timestamp = Now.AddHours(-5), Source = "test", Data = null! },
}, Now);
Assert.NotNull(json);
var obj = JObject.Parse(json!);
Assert.Equal(1, (int)obj["eventCount"]!);
}
}
@@ -49,11 +49,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null,
bool? isUserDriven = null, DateTime? resumedAt = null,
DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false,
string? failureSource = null, string? adminMarkedAction = null)
string? failureSource = null, string? adminMarkedAction = null,
string? failureSnapshotJson = null)
=> _storage.UpdateSessionStatusAsync(tenantId, sessionId, status,
currentPhase, failureReason, completedAt, earliestEventTimestamp,
latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt,
stalledAt, clearStalledAt, clearFailureReason, failureSource, adminMarkedAction);
stalledAt, clearStalledAt, clearFailureReason, failureSource, adminMarkedAction,
failureSnapshotJson);
public Task<bool> QueueServerActionAsync(string tenantId, string sessionId, ServerAction action)
=> _storage.QueueServerActionAsync(tenantId, sessionId, action);
@@ -0,0 +1,234 @@
using System;
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Shared.Models;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services
{
/// <summary>
/// Builds a compact JSON snapshot of "what we last knew about the session" at the
/// moment the maintenance watchdog times it out. Hybrid User-Driven completion-gap
/// fix (2026-05-01 trigger: session e58bcfdb-3e68-4f23-a3c2-437429ca9e78). Today's
/// 5h-timeout failure reason is just the literal string "Session timed out after 5
/// hours" — that gives operators no insight into where the session was stuck. This
/// builder walks the session's events and extracts the canonical lifecycle anchors
/// (last ESP phase, desktop arrival, Hello policy, AAD-join state, network state,
/// reboot observation) plus a list of MISSING signals so the snapshot answers
/// "why did this not complete?" at a glance.
/// <para>
/// The shape is intentionally flat and small (≤ ~1 KB JSON typical) so it fits in a
/// single Sessions-table column without strangling Storage costs. Schema is versioned
/// so the Web UI can render forward-compatibly.
/// </para>
/// </summary>
public static class FailureSnapshotBuilder
{
public const int CurrentSchemaVersion = 1;
// Canonical signals we expect to have seen by the time a healthy session completes.
// Missing entries are surfaced in the snapshot under "missingSignals". Some entries
// are conceptual slots that admit multiple concrete event types (see special-case
// branches in Build): "hello_terminal" is satisfied by any Hello terminal event,
// and "aad_user_join" by either the Part 1 (aad_user_joined_late) or Part 2
// (user_aad_signin_complete) real-user event.
private static readonly string[] CanonicalCompletionSignals =
{
"esp_phase_changed",
"esp_exiting",
"desktop_arrived",
"hello_policy_detected",
"hello_provisioning_completed", // OR _failed / _blocked / _skipped — emitted as "hello_terminal" slot
"aad_user_joined_late", // OR user_aad_signin_complete — emitted as "aad_user_join" slot
"enrollment_complete",
};
/// <summary>
/// Build a snapshot from a session's chronological event list. Returns
/// <c>null</c> when the input is empty (caller should write nothing rather than
/// the literal "{}"). All inputs are tolerated — missing/malformed event data is
/// recorded as "unknown" rather than causing the maintenance pass to throw.
/// </summary>
public static string? Build(IReadOnlyList<EnrollmentEvent>? events, DateTime nowUtc)
{
if (events == null || events.Count == 0)
return null;
// Events arrive sorted by Sequence (ascending) per ISessionRepository contract.
// Defensive copy + secondary sort by Timestamp keeps the snapshot stable even
// if the caller passes an unsorted list.
var ordered = events.OrderBy(e => e.Timestamp).ToList();
var seenTypes = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var evt in ordered) seenTypes.Add(evt.EventType ?? "");
// ---- Last ESP phase (from esp_phase_changed events) ------------------------
string? lastEspPhase = null;
DateTime? lastEspPhaseAt = null;
foreach (var evt in ordered)
{
if (string.Equals(evt.EventType, "esp_phase_changed", StringComparison.OrdinalIgnoreCase))
{
lastEspPhase = TryGetDataString(evt, "espPhase") ?? lastEspPhase;
lastEspPhaseAt = evt.Timestamp;
}
}
// ---- Desktop arrival -------------------------------------------------------
var desktopArrivedEvt = ordered.FirstOrDefault(e =>
string.Equals(e.EventType, "desktop_arrived", StringComparison.OrdinalIgnoreCase));
var desktopArrived = desktopArrivedEvt != null;
DateTime? desktopArrivedAt = desktopArrivedEvt?.Timestamp;
// ---- Hello policy ----------------------------------------------------------
var helloPolicyEvt = ordered.FirstOrDefault(e =>
string.Equals(e.EventType, "hello_policy_detected", StringComparison.OrdinalIgnoreCase));
bool? helloPolicyEnabled = null;
if (helloPolicyEvt != null)
{
var raw = TryGetDataString(helloPolicyEvt, "helloEnabled");
if (bool.TryParse(raw, out var parsed)) helloPolicyEnabled = parsed;
}
// ---- AAD join state — placeholder vs real user vs unknown ------------------
// Sources (newest wins):
// - aad_placeholder_user_detected → placeholder
// - aad_user_joined_late → real_user (Part 1 path)
// - user_aad_signin_complete → real_user (Part 2 path; future)
// - aad_join_status → fallback heuristic via isFooUser flag
string aadJoinState = "unknown";
DateTime? aadJoinStateAt = null;
foreach (var evt in ordered)
{
var et = evt.EventType ?? "";
if (string.Equals(et, "aad_placeholder_user_detected", StringComparison.OrdinalIgnoreCase))
{
aadJoinState = "placeholder";
aadJoinStateAt = evt.Timestamp;
}
else if (string.Equals(et, "aad_user_joined_late", StringComparison.OrdinalIgnoreCase)
|| string.Equals(et, "user_aad_signin_complete", StringComparison.OrdinalIgnoreCase))
{
aadJoinState = "real_user";
aadJoinStateAt = evt.Timestamp;
}
else if (aadJoinState == "unknown"
&& string.Equals(et, "aad_join_status", StringComparison.OrdinalIgnoreCase))
{
var isFoo = TryGetDataString(evt, "isFooUser");
if (bool.TryParse(isFoo, out var foo))
{
aadJoinState = foo ? "placeholder" : "real_user";
aadJoinStateAt = evt.Timestamp;
}
}
}
// ---- Reboot observed -------------------------------------------------------
var rebootEvt = ordered.FirstOrDefault(e =>
string.Equals(e.EventType, "system_reboot_detected", StringComparison.OrdinalIgnoreCase));
var rebootObserved = rebootEvt != null;
// ---- Hybrid-join hint from autopilot_profile / enrollment_type_detected ----
bool? isHybridJoin = null;
string? enrollmentType = null;
foreach (var evt in ordered)
{
var et = evt.EventType ?? "";
if (string.Equals(et, "autopilot_profile", StringComparison.OrdinalIgnoreCase)
|| string.Equals(et, "enrollment_facts_observed", StringComparison.OrdinalIgnoreCase))
{
var hybridRaw = TryGetDataString(evt, "isHybridJoin");
if (bool.TryParse(hybridRaw, out var h)) isHybridJoin = h;
enrollmentType ??= TryGetDataString(evt, "enrollmentType");
}
else if (enrollmentType == null
&& string.Equals(et, "enrollment_type_detected", StringComparison.OrdinalIgnoreCase))
{
enrollmentType = TryGetDataString(evt, "enrollmentType");
}
}
// ---- Last network state ----------------------------------------------------
string? lastNetworkState = null;
for (var i = ordered.Count - 1; i >= 0; i--)
{
if (string.Equals(ordered[i].EventType, "network_state_change", StringComparison.OrdinalIgnoreCase))
{
lastNetworkState = ordered[i].Message;
break;
}
}
// ---- Timing facts ----------------------------------------------------------
var lastEventAt = ordered[ordered.Count - 1].Timestamp;
var silenceMinutes = (int)Math.Max(0, (nowUtc - lastEventAt).TotalMinutes);
// ---- Missing canonical signals --------------------------------------------
var missing = new List<string>();
foreach (var sig in CanonicalCompletionSignals)
{
if (sig == "hello_provisioning_completed")
{
// Treat any Hello terminal as satisfying this slot.
if (!seenTypes.Contains("hello_provisioning_completed")
&& !seenTypes.Contains("hello_provisioning_failed")
&& !seenTypes.Contains("hello_provisioning_blocked")
&& !seenTypes.Contains("hello_skipped"))
{
missing.Add("hello_terminal");
}
}
else if (sig == "aad_user_joined_late")
{
// Codex review 2026-05-01: Part 1 (Classic / WhiteGlove) emits
// aad_user_joined_late; Part 2 (post-reboot user sign-in) emits
// user_aad_signin_complete. Both satisfy the same conceptual slot.
// Without this branch, every healthy Part-2 session would falsely
// list aad_user_joined_late as missing even though aadJoinState
// is correctly classified as real_user above.
if (!seenTypes.Contains("aad_user_joined_late")
&& !seenTypes.Contains("user_aad_signin_complete"))
{
missing.Add("aad_user_join");
}
}
else if (!seenTypes.Contains(sig))
{
missing.Add(sig);
}
}
var snapshot = new
{
schemaVersion = CurrentSchemaVersion,
generatedAtUtc = nowUtc.ToString("o"),
eventCount = ordered.Count,
lastEventAtUtc = lastEventAt.ToString("o"),
silenceMinutes,
lastEspPhase,
lastEspPhaseAtUtc = lastEspPhaseAt?.ToString("o"),
desktopArrived,
desktopArrivedAtUtc = desktopArrivedAt?.ToString("o"),
helloPolicyDetected = helloPolicyEvt != null,
helloPolicyEnabled,
aadJoinState,
aadJoinStateAtUtc = aadJoinStateAt?.ToString("o"),
rebootObserved,
isHybridJoin,
enrollmentType,
lastNetworkState,
missingSignals = missing,
};
return JsonConvert.SerializeObject(snapshot, Formatting.None);
}
private static string? TryGetDataString(EnrollmentEvent evt, string key)
{
if (evt?.Data == null) return null;
if (!evt.Data.TryGetValue(key, out var raw) || raw == null) return null;
return raw.ToString();
}
}
}
@@ -274,11 +274,31 @@ namespace AutopilotMonitor.Functions.Services
foreach (var session in stalledSessions)
{
// Hybrid User-Driven completion-gap fix (2026-05-01): before
// graduating the session to terminal Failed, build a compact
// snapshot of "what we last knew" so operators don't have to
// scroll through hundreds of events to reconstruct where the
// session was stuck. Best-effort — a snapshot read failure
// must never block the timeout transition itself.
string? snapshotJson = null;
try
{
var sessionEvents = await _sessionRepo.GetSessionEventsAsync(
session.TenantId, session.SessionId, maxResults: 1000);
snapshotJson = FailureSnapshotBuilder.Build(sessionEvents, now);
}
catch (Exception snapEx)
{
_logger.LogWarning(snapEx,
$"Failed to build failure snapshot for session {session.SessionId}; proceeding with timeout transition without snapshot");
}
await _sessionRepo.UpdateSessionStatusAsync(
session.TenantId,
session.SessionId,
SessionStatus.Failed,
failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)"
failureReason: $"Session timed out after {timeoutHours} hours (started at {session.StartedAt:yyyy-MM-dd HH:mm:ss} UTC)",
failureSnapshotJson: snapshotJson
);
sessionCount++;
}
@@ -249,7 +249,8 @@ namespace AutopilotMonitor.Functions.Services
GeoLoc = entity.GetString("GeoLoc") ?? string.Empty,
PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0,
RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0,
ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false
ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false,
FailureSnapshotJson = entity.GetString("FailureSnapshotJson") ?? string.Empty
};
}
@@ -963,7 +964,7 @@ namespace AutopilotMonitor.Functions.Services
/// back to the session's LastEventAt and only to UtcNow as a last resort, so DurationSeconds
/// reflects when the session went silent rather than when the operator clicked the button.
/// </param>
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null)
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null)
{
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
@@ -1109,6 +1110,16 @@ namespace AutopilotMonitor.Functions.Services
update["FailureReason"] = failureReason;
}
// Persist the failure-state snapshot when provided. Only the maintenance
// 5h-timeout path supplies this today (Hybrid User-Driven completion-gap
// fix, 2026-05-01); other terminal paths leave the field untouched, which
// is fine — null/empty means "no snapshot available" and the UI falls back
// to the existing FailureReason rendering.
if (status == SessionStatus.Failed && !string.IsNullOrEmpty(failureSnapshotJson))
{
update["FailureSnapshotJson"] = failureSnapshotJson;
}
// Record the origin of a Failed status (agent / rule:<id> / manual). When the caller
// passes null we leave the existing value untouched — concurrent updates from different
// sources shouldn't stomp on each other's attribution.
@@ -1307,6 +1318,11 @@ namespace AutopilotMonitor.Functions.Services
if (status == SessionStatus.Failed && !string.IsNullOrEmpty(failureReason))
forceUpdate["FailureReason"] = failureReason;
// Mirror the FailureSnapshotJson from the regular path so the
// force-merge fallback never silently drops the snapshot.
if (status == SessionStatus.Failed && !string.IsNullOrEmpty(failureSnapshotJson))
forceUpdate["FailureSnapshotJson"] = failureSnapshotJson;
if (isPreProvisioned.HasValue)
forceUpdate["IsPreProvisioned"] = isPreProvisioned.Value;
@@ -1862,7 +1878,8 @@ namespace AutopilotMonitor.Functions.Services
GeoLoc = entity.GetString("GeoLoc") ?? string.Empty,
PlatformScriptCount = SafeGetInt32(entity, "PlatformScriptCount") ?? 0,
RemediationScriptCount = SafeGetInt32(entity, "RemediationScriptCount") ?? 0,
ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false
ExcessiveEventsAlerted = entity.GetBoolean("ExcessiveEventsAlerted") ?? false,
FailureSnapshotJson = entity.GetString("FailureSnapshotJson") ?? string.Empty
};
}
@@ -231,6 +231,19 @@ namespace AutopilotMonitor.Shared
// TEMPORARY: shadow SM rollout verification — remove when CompletionStateMachine is promoted to primary
public const string ShadowDiscrepancy = "shadow_discrepancy";
// Hybrid User-Driven enrollment observability (V2 — Hybrid completion gaps, 2026-05-01).
// Pure observability — never consumed by the DecisionEngine, no DecisionSignalKind wired up.
public const string AadPlaceholderUserDetected = "aad_placeholder_user_detected"; // foouser@/autopilot@ first appearance in JoinInfo
public const string HybridLoginPending = "hybrid_login_pending"; // Hybrid: 10 min after reboot still placeholder, real AD login overdue
// Real-user join — companion to AadPlaceholderUserDetected. The DecisionEngine consumes
// these as DecisionSignalKind only (HandleAadUserJoinedLateV1 is observation-only and
// emits no timeline effect), so the AadJoinWatcherAdapter dual-emits them as
// informational events too. Without this the FailureSnapshotBuilder cannot tell that
// the real AAD user was ever observed (Codex review 2026-05-01).
public const string AadUserJoinedLate = "aad_user_joined_late"; // Real user replaces placeholder mid-session (Part 1 path)
public const string UserAadSignInComplete = "user_aad_signin_complete"; // Real user signs in post-reboot (Part 2 path)
}
// -----------------------------------------------------------------------
@@ -27,7 +27,8 @@ namespace AutopilotMonitor.Shared.DataAccess
DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null,
bool? isUserDriven = null, DateTime? resumedAt = null,
DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false,
string? failureSource = null, string? adminMarkedAction = null);
string? failureSource = null, string? adminMarkedAction = null,
string? failureSnapshotJson = null);
Task IncrementSessionEventCountAsync(
string tenantId, string sessionId, int increment,
DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null,
@@ -298,6 +298,18 @@ namespace AutopilotMonitor.Shared.Models
/// on long-dead sessions.
/// </summary>
public DateTime? PendingActionsQueuedAt { get; set; }
/// <summary>
/// Compact JSON snapshot of "last known session state" written by the maintenance
/// 5h-timeout sweep when a session graduates to terminal Failed (Hybrid User-Driven
/// completion-gap fix, 2026-05-01). Captures the canonical lifecycle anchors —
/// last ESP phase, desktop arrival, Hello policy, AAD-join state, missing signals —
/// so operators don't have to scroll through hundreds of events to reconstruct
/// where a stuck session was when the watchdog fired. Empty / null on
/// healthy-completion paths and on sessions that predate the field. Built by
/// <see cref="AutopilotMonitor.Functions.Services.FailureSnapshotBuilder"/>.
/// </summary>
public string FailureSnapshotJson { get; set; } = string.Empty;
}
/// <summary>
@@ -0,0 +1,133 @@
"use client";
import { useState } from "react";
/**
* Renders the maintenance-built failure snapshot when present (Hybrid User-Driven
* completion-gap fix, 2026-05-01). The backend writes the JSON only on the 5h-timeout
* graduation path, so this block is hidden for healthy completions and for sessions
* that predate the field. The shape is best-effort — unknown fields are tolerated;
* legacy snapshots without `schemaVersion` simply render the fields they do carry.
*/
interface FailureSnapshot {
schemaVersion?: number;
generatedAtUtc?: string;
eventCount?: number;
lastEventAtUtc?: string;
silenceMinutes?: number;
lastEspPhase?: string | null;
lastEspPhaseAtUtc?: string | null;
desktopArrived?: boolean;
desktopArrivedAtUtc?: string | null;
helloPolicyDetected?: boolean;
helloPolicyEnabled?: boolean | null;
aadJoinState?: string;
aadJoinStateAtUtc?: string | null;
rebootObserved?: boolean;
isHybridJoin?: boolean | null;
enrollmentType?: string | null;
lastNetworkState?: string | null;
missingSignals?: string[];
}
export default function FailureSnapshotBlock({ failureSnapshotJson }: { failureSnapshotJson?: string }) {
const [expanded, setExpanded] = useState(false);
if (!failureSnapshotJson || failureSnapshotJson.trim().length === 0) return null;
let snapshot: FailureSnapshot | null = null;
try {
snapshot = JSON.parse(failureSnapshotJson) as FailureSnapshot;
} catch {
// Invalid JSON — render the raw string in a debug fallback so operators can still
// copy/paste it into a JSON formatter offline. No exception bubbles up to break
// the surrounding card.
return (
<div className="mt-4 p-3 rounded-lg bg-rose-50 border border-rose-200 text-sm text-rose-800">
<strong>Failure Snapshot (raw):</strong>
<pre className="mt-2 text-xs font-mono whitespace-pre-wrap break-all">{failureSnapshotJson}</pre>
</div>
);
}
const missing = snapshot.missingSignals ?? [];
return (
<div className="mt-4 rounded-lg bg-rose-50 border border-rose-200 text-sm text-rose-900">
<button
type="button"
className="w-full flex items-center justify-between px-3 py-2 text-left font-semibold"
onClick={() => setExpanded((e) => !e)}
aria-expanded={expanded}
>
<span className="flex items-center gap-2">
<svg className="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L4.082 16.5c-.77.833.192 2.5 1.732 2.5z" />
</svg>
Failure Snapshot {snapshot.silenceMinutes !== undefined && <span className="font-normal">(agent went silent {snapshot.silenceMinutes} min before timeout)</span>}
</span>
<svg className={`w-4 h-4 transition-transform ${expanded ? "rotate-180" : ""}`} fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M19 9l-7 7-7-7" />
</svg>
</button>
{expanded && (
<div className="px-3 pb-3 pt-1 space-y-2">
<div className="grid grid-cols-1 sm:grid-cols-2 gap-x-4 gap-y-1.5">
<SnapshotItem label="Last ESP phase" value={snapshot.lastEspPhase ?? "—"} sub={snapshot.lastEspPhaseAtUtc ?? undefined} />
<SnapshotItem label="Desktop arrived" value={fmtBool(snapshot.desktopArrived)} sub={snapshot.desktopArrivedAtUtc ?? undefined} />
<SnapshotItem label="Hello policy" value={fmtHelloPolicy(snapshot)} />
<SnapshotItem label="AAD join state" value={snapshot.aadJoinState ?? "unknown"} sub={snapshot.aadJoinStateAtUtc ?? undefined} />
<SnapshotItem label="Reboot observed" value={fmtBool(snapshot.rebootObserved)} />
<SnapshotItem label="Hybrid AAD join" value={snapshot.isHybridJoin === null || snapshot.isHybridJoin === undefined ? "unknown" : fmtBool(snapshot.isHybridJoin)} />
<SnapshotItem label="Enrollment type" value={snapshot.enrollmentType ?? "unknown"} />
<SnapshotItem label="Event count" value={snapshot.eventCount?.toString() ?? "—"} />
{snapshot.lastNetworkState && (
<SnapshotItem label="Last network state" value={snapshot.lastNetworkState} className="sm:col-span-2" />
)}
</div>
{missing.length > 0 && (
<div className="pt-2">
<div className="text-xs font-semibold text-rose-800 mb-1">Missing canonical signals at timeout:</div>
<div className="flex flex-wrap gap-1.5">
{missing.map((sig) => (
<span key={sig} className="inline-flex items-center px-2 py-0.5 text-xs font-mono rounded bg-white border border-rose-300 text-rose-800">
{sig}
</span>
))}
</div>
</div>
)}
{snapshot.generatedAtUtc && (
<div className="text-xs text-rose-700 pt-1">
Snapshot generated {new Date(snapshot.generatedAtUtc).toLocaleString()}
{snapshot.schemaVersion !== undefined && <> · schema v{snapshot.schemaVersion}</>}
</div>
)}
</div>
)}
</div>
);
}
function SnapshotItem({ label, value, sub, className }: { label: string; value: React.ReactNode; sub?: string; className?: string }) {
return (
<div className={className}>
<div className="text-xs font-semibold text-rose-800">{label}</div>
<div className="text-sm text-rose-900 break-words">{value}</div>
{sub && <div className="text-[11px] text-rose-700 font-mono">{sub}</div>}
</div>
);
}
function fmtBool(v: boolean | undefined | null) {
if (v === true) return "yes";
if (v === false) return "no";
return "unknown";
}
function fmtHelloPolicy(s: FailureSnapshot): string {
if (!s.helloPolicyDetected) return "not detected";
if (s.helloPolicyEnabled === true) return "detected (enabled)";
if (s.helloPolicyEnabled === false) return "detected (disabled)";
return "detected";
}
@@ -2,6 +2,7 @@
import { useState } from "react";
import { Session } from "@/types";
import FailureSnapshotBlock from "./FailureSnapshotBlock";
interface SessionInfoCardProps {
session: Session;
@@ -81,6 +82,7 @@ export default function SessionInfoCard({ session, enrollmentDuration, displaySt
</span>
</div>
)}
<FailureSnapshotBlock failureSnapshotJson={session.failureSnapshotJson} />
</div>
);
}
@@ -33,4 +33,12 @@ export interface Session {
geoCountry?: string;
geoRegion?: string;
geoCity?: string;
/**
* Compact JSON snapshot of "last known session state" written by the maintenance
* 5h-timeout sweep when a session graduates to terminal Failed (Hybrid User-Driven
* completion-gap fix, 2026-05-01). Empty for healthy completions and sessions that
* predate the field. The detail page renders a collapsible "Failure Snapshot" block
* when populated.
*/
failureSnapshotJson?: string;
}