diff --git a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs index b08c0e58..4eed9e46 100644 --- a/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs +++ b/src/Agent/AutopilotMonitor.Agent.V2.Core/SignalAdapters/ImeLogTrackerAdapter.cs @@ -953,7 +953,17 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters var exitSuffix = script.ExitCode.HasValue ? $" (exit: {script.ExitCode.Value.ToString(culture)})" : string.Empty; - var msg = $"{label} {shortId}: {messageCore}{exitSuffix}"; + // stderr is the only rule that can classify a Success/exit-0 script as script_failed + // (rule 2 above). Without naming that in the message, the event reads as a + // contradiction — "script_failed: Success (exit: 0)" (misclassification audit + // 2026-07-16). Keep the visibility rule, make the message say why. + var stderrSuffix = isFailure + && hasStderr + && !string.Equals(script.Result, "Failed", StringComparison.OrdinalIgnoreCase) + && (!script.ExitCode.HasValue || script.ExitCode.Value == 0) + ? " — stderr output detected" + : string.Empty; + var msg = $"{label} {shortId}: {messageCore}{exitSuffix}{stderrSuffix}"; _post.Emit( eventType: eventType, diff --git a/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryCompletionHelloNeverObservedTests.cs b/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryCompletionHelloNeverObservedTests.cs new file mode 100644 index 00000000..8686c3e4 --- /dev/null +++ b/src/Agent/AutopilotMonitor.DecisionCore.Tests/AdvisoryCompletionHelloNeverObservedTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using AutopilotMonitor.DecisionCore.Engine; +using AutopilotMonitor.DecisionCore.Signals; +using AutopilotMonitor.DecisionCore.State; +using Xunit; + +namespace AutopilotMonitor.DecisionCore.Tests +{ + /// + /// Misclassification audit 2026-07-16 (sessions 2dac8298 / 7635ab18): the ESP page exits + /// normally, the DAD-validated real-user desktop arrives, but the Hello policy read never + /// commits (no PassportForWork registry value → no hello_policy_detected ever) and no + /// wizard is ever seen. The hello_resolution prerequisite is then structurally + /// unresolvable and the esp-exit advisory window used to hard-fail a provably finished + /// enrollment with esp_exit_without_completion_evidence. + /// Coverage: + /// + /// Hello-never-observed at advisory fire → promote to AwaitingHello + arm + /// hello_safety (NOT Failed), trigger suffix HelloNeverObservedPromote. + /// hello_safety fire then completes through Finalizing with synthetic + /// HelloOutcome=Timeout. + /// Policy read committed (enabled) → fallthrough still fails (no behavior change). + /// No desktop → fallthrough still fails. + /// Advisory-anchor variant (real ESP terminal failure) is exempt — still fails. + /// + /// + public sealed class AdvisoryCompletionHelloNeverObservedTests + { + private static readonly DateTime T0 = new DateTime(2026, 7, 15, 8, 0, 0, DateTimeKind.Utc); + + private static DecisionSignal MakeSignal( + long ordinal, + DecisionSignalKind kind, + DateTime occurredAtUtc, + IReadOnlyDictionary? payload = null) + { + return new DecisionSignal( + sessionSignalOrdinal: ordinal, + sessionTraceOrdinal: ordinal, + kind: kind, + kindSchemaVersion: 1, + occurredAtUtc: occurredAtUtc, + sourceOrigin: "test", + evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"), + payload: payload); + } + + private static DecisionSignal DeadlineFired(long ordinal, DateTime occurredAtUtc, string deadlineName) => + MakeSignal(ordinal, DecisionSignalKind.DeadlineFired, occurredAtUtc, + new Dictionary { [SignalPayloadKeys.Deadline] = deadlineName }); + + private static ActiveDeadline? FindDeadline(DecisionState state, string name) => + state.Deadlines.FirstOrDefault(d => d.Name == name); + + /// + /// Session-2dac8298 shape: normal ESP run, AccountSetup entered, genuine final exit + /// (arms the esp-exit advisory window), real-user desktop shortly after — and NO + /// hello_policy_detected, NO wizard, NO IME user-session completion for the whole run. + /// + private static DecisionState SetupHelloNeverObservedSession( + DecisionEngine engine, + bool desktopArrives = true, + bool helloPolicyEnabledRead = false, + bool allowContinueAnyway = false) + { + var state = DecisionState.CreateInitial("sess-2dac8298", "tenant-047b2e1f", T0); + state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState; + state = engine.Reduce(state, MakeSignal( + 5, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1), + new Dictionary + { + [SignalPayloadKeys.SkipUserEsp] = "false", + [SignalPayloadKeys.SkipDeviceEsp] = "false", + [SignalPayloadKeys.EspAllowContinueAnyway] = allowContinueAnyway ? "true" : "false", + })).NewState; + if (helloPolicyEnabledRead) + { + state = engine.Reduce(state, MakeSignal( + 8, DecisionSignalKind.HelloPolicyDetected, T0.AddMinutes(1), + new Dictionary { [SignalPayloadKeys.HelloEnabled] = "true" })).NewState; + } + state = engine.Reduce(state, MakeSignal( + 10, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2), + new Dictionary { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState; + state = engine.Reduce(state, MakeSignal( + 20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(8), + new Dictionary { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState; + // Genuine post-AccountSetup final exit — arms the esp-exit advisory window + // (30 min) because AccountSetup was entered and no promotion arm applies. + state = engine.Reduce(state, MakeSignal( + 30, DecisionSignalKind.EspExiting, T0.AddMinutes(10))).NewState; + if (desktopArrives) + { + state = engine.Reduce(state, MakeSignal( + 40, DecisionSignalKind.DesktopArrived, T0.AddMinutes(10.5))).NewState; + } + return state; + } + + [Fact] + public void Setup_ArmsEspExitAdvisoryWindow() + { + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine); + + Assert.NotNull(FindDeadline(state, DeadlineNames.AdvisoryCompletion)); + Assert.Null(state.HelloPolicyEnabled); + Assert.Null(state.HelloResolvedUtc); + Assert.Null(state.Outcome); + } + + [Fact] + public void DeadlineFired_HelloNeverObserved_PromotesToAwaitingHello_NotFailed() + { + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine); + + var fireAt = T0.AddMinutes(40); + var step = engine.Reduce(state, DeadlineFired(50, fireAt, DeadlineNames.AdvisoryCompletion)); + + Assert.Equal(SessionStage.AwaitingHello, step.NewState.Stage); + Assert.Null(step.NewState.Outcome); + Assert.Equal( + $"DeadlineFired:{DeadlineNames.AdvisoryCompletion}:HelloNeverObservedPromote", + step.Transition.Trigger); + + // hello_safety armed in state + scheduled as an effect; advisory window retired. + var safety = FindDeadline(step.NewState, DeadlineNames.HelloSafety); + Assert.NotNull(safety); + Assert.Null(FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion)); + var scheduleEffect = step.Effects.Single(e => e.Kind == DecisionEffectKind.ScheduleDeadline); + Assert.Equal(DeadlineNames.HelloSafety, scheduleEffect.Deadline!.Name); + } + + [Fact] + public void HelloSafetyFired_AfterPromote_CompletesThroughFinalizing() + { + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine); + state = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion)).NewState; + + var step = engine.Reduce(state, DeadlineFired(60, T0.AddMinutes(45), DeadlineNames.HelloSafety)); + + Assert.Equal(SessionStage.Finalizing, step.NewState.Stage); + Assert.NotNull(step.NewState.HelloResolvedUtc); + Assert.Equal("Timeout", step.NewState.HelloOutcome!.Value); + } + + [Fact] + public void DeadlineFired_PolicyReadEnabled_NoWizard_StillFails() + { + // A COMMITTED enabled policy read means Hello genuinely never finished — the + // never-observed promote must not apply and the fallthrough failure stands. + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine, helloPolicyEnabledRead: true); + + var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion)); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome); + var failEffect = step.Effects.Single(e => e.Kind == DecisionEffectKind.EmitEventTimelineEntry); + Assert.Equal("esp_exit_without_completion_evidence", failEffect.Parameters!["reason"]); + } + + [Fact] + public void DeadlineFired_NoDesktop_StillFails() + { + // Without the DAD-validated desktop there is no user-presence evidence at all — + // the never-observed promote must not apply. + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine, desktopArrives: false); + + var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion)); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome); + } + + [Fact] + public void DeadlineFired_AdvisoryAnchorVariant_HelloNeverObserved_StillFails() + { + // The advisory variant is anchored to a REAL ESP terminal failure — a missing + // Hello policy read must not soften that evidence. Only the esp-exit variant + // (normal page close) qualifies for the never-observed promote. + var engine = new DecisionEngine(); + var state = SetupHelloNeverObservedSession(engine, allowContinueAnyway: true); + state = engine.Reduce(state, MakeSignal( + 45, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(11), + new Dictionary + { + ["failureType"] = "Provisioning_AccountSetup_Apps_Failed", + ["errorCode"] = "0x87d1041c", + ["failedSubcategory"] = "Apps", + ["category"] = "AccountSetup", + })).NewState; + Assert.NotNull(state.EspAdvisoryFailureRecordedUtc); + + var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(41), DeadlineNames.AdvisoryCompletion)); + + Assert.Equal(SessionStage.Failed, step.NewState.Stage); + Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs index 195b51a1..89eabb98 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/EnrollmentTimeoutClassifierTests.cs @@ -368,6 +368,63 @@ public class EnrollmentTimeoutClassifierTests Assert.Equal(SessionStatus.Incomplete, status); } + // -------- Misclassification audit 2026-07-16 -------- + + [Fact] + public void ExtractRollup_agent_timeout_enrollment_failed_is_not_explicit_failure() + { + // The max-lifetime watchdog's enrollment_failed(failureType=agent_timeout) is "the agent + // gave up waiting", not a failure verdict — it must not poison rule 1 (tenant a53e67ec). + var timeout = Evt("enrollment_failed", + data: new Dictionary { ["failureType"] = "agent_timeout" }); + Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { timeout }).HasExplicitFailure); + + // Any OTHER failureType stays an explicit failure. + var genuine = Evt("enrollment_failed", + data: new Dictionary { ["failureType"] = "esp_terminal" }); + Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { genuine }).HasExplicitFailure); + } + + [Fact] + public void Classify_agent_timeout_failed_session_classifies_honestly() + { + // WG Part-2 shape (session 1506ce9f): DeviceSetup 4/4, no user phase, watchdog fired at + // 6h — honest verdict is AwaitingUser (within grace), never Failed. + var timeout = Evt("enrollment_failed", + data: new Dictionary { ["failureType"] = "agent_timeout" }); + var (status, _) = Classify(new[] { Esp(DeviceSetup44), timeout }, hoursSinceStart: 6); + Assert.Equal(SessionStatus.AwaitingUser, status); + } + + [Fact] + public void ExtractRollup_account_0ofN_observation_records_total() + { + // A "0 of 5" rollup was previously dropped entirely (the strongest-observation fold only + // kept n > 0), making the Incomplete reason read "0/0" — session 08ddbeec. + var r = EnrollmentTimeoutClassifier.ExtractRollup(new[] { Esp(DeviceSetup44), Esp(AccountSetup05) }); + Assert.Equal(0, r.AccountSetupSucceededCount); + Assert.Equal(5, r.AccountSetupTotal); + Assert.False(r.AccountSetupAllSucceeded); + } + + [Fact] + public void Classify_incomplete_reason_shows_observed_account_rollup() + { + var (status, reason) = Classify(new[] { Esp(DeviceSetup44), Esp(AccountSetup05) }, hoursSinceStart: 80, grace: 72); + Assert.Equal(SessionStatus.Incomplete, status); + Assert.Contains("0/5", reason); + Assert.DoesNotContain("0/0", reason); + } + + [Fact] + public void Classify_incomplete_reason_names_never_observed_account_rollup() + { + // No AccountSetup rollup at all — the reason must say so instead of a fabricated "0/0". + var (status, reason) = Classify(new[] { Esp(DeviceSetup44) }, hoursSinceStart: 80, grace: 72); + Assert.Equal(SessionStatus.Incomplete, status); + Assert.Contains("Account Setup progress never observed", reason); + } + // -------- ResolveGraceHours -------- [Theory] diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs index b8c53918..12139f55 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/IngestMaxLifetimeShutdownTests.cs @@ -78,4 +78,66 @@ public class IngestMaxLifetimeShutdownTests { Assert.False(EventIngestProcessor.IsMaxLifetimeAgentShutdown(null)); } + + // -------- IsAgentTimeoutFailure (misclassification audit 2026-07-16) -------- + // The V1-parity companion: enrollment_failed(failureType=agent_timeout) emitted by + // LifecycleEmitters.CreateMaxLifetimeEmitter. Same semantics as the shutdown shape — + // detected events route through the honest classifier instead of a hard Failed. + + private static EnrollmentEvent Failure(string? failureType, string eventType = "enrollment_failed") + { + var evt = new EnrollmentEvent + { + EventType = eventType, + Message = "Agent max lifetime expired (360 min) — enrollment did not complete in time", + }; + if (failureType != null) + { + evt.Data = new Dictionary + { + ["failureType"] = failureType, + ["failureSource"] = "max_lifetime_timer", + }; + } + return evt; + } + + [Fact] + public void AgentTimeoutFailure_IsDetected() + { + Assert.True(EventIngestProcessor.IsAgentTimeoutFailure(Failure("agent_timeout"))); + } + + [Fact] + public void AgentTimeoutFailure_IsDetected_CaseInsensitively() + { + Assert.True(EventIngestProcessor.IsAgentTimeoutFailure(Failure("Agent_Timeout"))); + } + + [Theory] + [InlineData("esp_terminal")] + [InlineData("app_install_failed")] + public void OtherFailureTypes_AreNotDetected(string failureType) + { + Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(Failure(failureType))); + } + + [Fact] + public void MissingFailureTypeData_IsNotDetected() + { + Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(Failure(failureType: null))); + } + + [Fact] + public void OtherEventTypes_AreNotDetected_EvenWithAgentTimeoutType() + { + Assert.False(EventIngestProcessor.IsAgentTimeoutFailure( + Failure("agent_timeout", eventType: "agent_shutting_down"))); + } + + [Fact] + public void NullEvent_IsNotDetected_ForAgentTimeout() + { + Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(null)); + } } diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs index 905b336c..7c945f85 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/OffboardingSessionEnumeratorTests.cs @@ -114,6 +114,8 @@ public class OffboardingSessionEnumeratorTests public Task> GetUsageWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException(); public Task> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException(); public Task> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException(); + public Task> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) => throw new NotImplementedException(); + public Task> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException(); public Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException(); public Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException(); public Task> GetAllTenantIdsAsync() => throw new NotImplementedException(); diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/SerialNumberHeuristicsTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/SerialNumberHeuristicsTests.cs new file mode 100644 index 00000000..89f729cc --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions.Tests/SerialNumberHeuristicsTests.cs @@ -0,0 +1,40 @@ +using AutopilotMonitor.Functions.Helpers; +using Xunit; + +namespace AutopilotMonitor.Functions.Tests; + +/// +/// Guards the registration supersede pass (misclassification audit 2026-07-16): OEM +/// placeholder serials would cross-match unrelated devices, so they must never be treated +/// as a device identity. +/// +public class SerialNumberHeuristicsTests +{ + [Theory] + [InlineData("PF4DF1EN")] + [InlineData("5CG5504Y5L")] + [InlineData(" PF4DF1EN ")] // trimmed + [InlineData("VMware-56 4d 8a")] + public void RealSerials_AreUsable(string serial) + { + Assert.True(SerialNumberHeuristics.IsUsableSerialNumber(serial)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [InlineData("0")] + [InlineData("abc")] // too short to identify a device + [InlineData("None")] + [InlineData("UNKNOWN")] + [InlineData("Default string")] + [InlineData("To be filled by O.E.M.")] + [InlineData("System Serial Number")] + [InlineData("Not Specified")] + [InlineData("n/a")] + public void PlaceholderOrEmptySerials_AreNotUsable(string? serial) + { + Assert.False(SerialNumberHeuristics.IsUsableSerialNumber(serial)); + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs index 3aad2dd9..45583de3 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableMaintenanceRepository.cs @@ -72,6 +72,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => _storage.GetExcessiveDataSendersAsync(tenantId, windowCutoff, maxSessionWindowHours); + public Task> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) + => _storage.GetLegacyTimeoutFailedSessionsAsync(tenantId, maxResults); + + public Task> GetSessionsLeanAsync(string tenantId) + => _storage.GetSessionsLeanAsync(tenantId); + public Task> GetAllTenantIdsAsync() => _storage.GetAllTenantIdsAsync(); diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs index b4c97eb1..088c41df 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableSessionRepository.cs @@ -36,6 +36,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage public Task FindSessionTenantIdAsync(string sessionId) => _storage.FindSessionTenantIdAsync(sessionId); + public Task> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber) + => _storage.GetOpenSessionsForDeviceAsync(tenantId, serialNumber); + public Task> GetSessionsAsync(string tenantId, int? days = null) => _storage.GetSessionsAsync(tenantId, days); @@ -62,13 +65,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, - string? failureSnapshotJson = null) + string? failureSnapshotJson = null, bool allowTerminalReclassification = false) { var transitioned = await _storage.UpdateSessionStatusAsync(tenantId, sessionId, status, currentPhase, failureReason, completedAt, earliestEventTimestamp, latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt, stalledAt, clearStalledAt, clearFailureReason, failureSource, adminMarkedAction, - failureSnapshotJson); + failureSnapshotJson, allowTerminalReclassification); // Reconcile the authoritative EventCount + RebootCount whenever a session actually // transitions to a terminal state through ANY caller — ingest, admin mark diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Admin/ReclassifyLegacySessionsFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/ReclassifyLegacySessionsFunction.cs new file mode 100644 index 00000000..28195df8 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Admin/ReclassifyLegacySessionsFunction.cs @@ -0,0 +1,117 @@ +using System.Net; +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Functions.Security; +using AutopilotMonitor.Functions.Services; +using Microsoft.Azure.Functions.Worker; +using Microsoft.Azure.Functions.Worker.Http; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Functions.Admin +{ + /// + /// Global-Admin trigger for the one-time retro-reconciliation of misclassified historical + /// sessions (misclassification audit 2026-07-16). DRY-RUN BY DEFAULT — a real run requires + /// the explicit dryRun=false query parameter, so the operator always sees the counts + /// and samples before anything is written. + /// + public class ReclassifyLegacySessionsFunction + { + private readonly ILogger _logger; + private readonly LegacyReclassificationService _service; + + private const int DefaultMaxSessions = 200; + private const int MaxMaxSessions = 2000; + + public ReclassifyLegacySessionsFunction( + ILogger logger, + LegacyReclassificationService service) + { + _logger = logger; + _service = service; + } + + /// + /// POST /api/maintenance/reclassify-legacy + /// Query parameters: + /// - mode: "legacy_timeouts" (default) — re-classify pre-classifier "Session timed out" Failed rows; + /// "pending_orphans" — resolve Pending rows superseded by a later session of the same device. + /// - dryRun: "false" to actually write; anything else (or absent) reports only. + /// - tenantId: optional single-tenant scope (GUID); absent = all tenants. + /// - maxSessions: per-invocation examination cap (default 200, max 2000). Re-run to continue. + /// + [Function("ReclassifyLegacySessions")] + public async Task Run( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/reclassify-legacy")] HttpRequestData req, + FunctionContext context) + { + // Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware + var userEmail = TenantHelper.GetUserIdentifier(req); + + var mode = (req.Query["mode"] ?? LegacyReclassificationService.ModeLegacyTimeouts).Trim().ToLowerInvariant(); + var dryRun = !string.Equals(req.Query["dryRun"], "false", StringComparison.OrdinalIgnoreCase); + var tenantIdScope = req.Query["tenantId"]; + + if (!string.IsNullOrEmpty(tenantIdScope) && !Guid.TryParse(tenantIdScope, out _)) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new { error = "tenantId must be a GUID" }); + return bad; + } + + var maxSessions = DefaultMaxSessions; + if (!string.IsNullOrEmpty(req.Query["maxSessions"])) + { + if (!int.TryParse(req.Query["maxSessions"], out maxSessions) || maxSessions < 1) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new { error = $"maxSessions must be a positive integer (max {MaxMaxSessions})" }); + return bad; + } + maxSessions = Math.Min(maxSessions, MaxMaxSessions); + } + + _logger.LogInformation( + "Legacy reclassification requested by {User}: mode={Mode} dryRun={DryRun} tenantScope={TenantScope} maxSessions={MaxSessions}", + userEmail, mode, dryRun, tenantIdScope ?? "(all)", maxSessions); + + try + { + var result = mode switch + { + LegacyReclassificationService.ModeLegacyTimeouts => + await _service.ReclassifyLegacyTimeoutsAsync(tenantIdScope, dryRun, maxSessions, userEmail), + LegacyReclassificationService.ModePendingOrphans => + await _service.ResolvePendingOrphansAsync(tenantIdScope, dryRun, maxSessions, userEmail), + _ => null, + }; + + if (result == null) + { + var bad = req.CreateResponse(HttpStatusCode.BadRequest); + await bad.WriteAsJsonAsync(new + { + error = $"Unknown mode '{mode}'. Valid: {LegacyReclassificationService.ModeLegacyTimeouts}, {LegacyReclassificationService.ModePendingOrphans}" + }); + return bad; + } + + var response = req.CreateResponse(HttpStatusCode.OK); + await response.WriteAsJsonAsync(new + { + success = true, + result, + triggeredBy = userEmail, + triggeredAt = DateTime.UtcNow, + }); + return response; + } + catch (Exception ex) + { + _logger.LogError(ex, "Legacy reclassification run failed"); + var error = req.CreateResponse(HttpStatusCode.InternalServerError); + await error.WriteAsJsonAsync(new { success = false, message = "Internal server error" }); + return error; + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RegisterSessionFunction.cs b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RegisterSessionFunction.cs index ae55b6a8..a32c0746 100644 --- a/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RegisterSessionFunction.cs +++ b/src/Backend/AutopilotMonitor.Functions/Functions/Sessions/RegisterSessionFunction.cs @@ -184,6 +184,16 @@ namespace AutopilotMonitor.Functions.Functions.Sessions // Retrieve the stored session to include full data in SignalR message var session = await _sessionRepo.GetSessionAsync(registration.TenantId, registration.SessionId); + // Supersede pass (misclassification audit 2026-07-16): when a device registers a FRESH + // session while older non-terminal sessions of the same device still exist (e.g. + // WhiteGlove Part 2 running under a new session id leaves the Part-1 row Pending + // forever, or a wiped/re-enrolled device abandons its old InProgress run), resolve + // those predecessors as Incomplete("Superseded by ..."). One physical device can only + // ever run one enrollment. Best-effort — never blocks the registration. + var supersededMessages = isFreshRegistration + ? await SupersedeOrphanedPredecessorsAsync(registration) + : Array.Empty(); + // AdminAction on re-register is only the portal button signal. SessionSummary.AdminMarkedAction // is set exclusively by MarkSessionSucceededFunction / MarkSessionFailedFunction. An // agent that restarts after its own completion must NOT receive a phantom AdminAction — @@ -246,9 +256,79 @@ namespace AutopilotMonitor.Functions.Functions.Sessions { HttpResponse = response, SignalRMessages = new[] { tenantMessage, globalAdminMessage } + .Concat(supersededMessages) + .ToArray() }; } + /// + /// Resolves older non-terminal sessions (Pending/InProgress/Stalled/AwaitingUser) of the + /// same device to Incomplete("Superseded by session …") when a fresh registration proves + /// the device has moved on. Returns portal-live SignalR updates for each resolved row + /// (same "newevents" sessionUpdate shape the ingest path uses). Fail-soft: any error + /// returns an empty array and the registration proceeds untouched. + /// + private async Task SupersedeOrphanedPredecessorsAsync(SessionRegistration registration) + { + try + { + if (!Helpers.SerialNumberHeuristics.IsUsableSerialNumber(registration.SerialNumber)) + return Array.Empty(); + + var openSessions = await _sessionRepo.GetOpenSessionsForDeviceAsync( + registration.TenantId, registration.SerialNumber.Trim()); + if (openSessions.Count == 0) + return Array.Empty(); + + var messages = new List(); + foreach (var stale in openSessions) + { + if (string.Equals(stale.SessionId, registration.SessionId, StringComparison.OrdinalIgnoreCase)) + continue; + // Only strictly older runs — a clock-skewed or concurrent row is left alone. + if (stale.StartedAt >= registration.StartedAt) + continue; + + var reason = $"Superseded by session {registration.SessionId}: the device registered a new " + + $"enrollment session at {registration.StartedAt:yyyy-MM-dd HH:mm} UTC before this one " + + "reached a terminal state."; + var transitioned = await _sessionRepo.UpdateSessionStatusAsync( + registration.TenantId, stale.SessionId, SessionStatus.Incomplete, + failureReason: reason, failureSource: "superseded_by_reregistration"); + if (!transitioned) + continue; + + _logger.LogInformation( + "Superseded stale {StaleStatus} session {StaleSessionId} (started {StaleStartedAt:yyyy-MM-dd}) with new session {SessionId} for device serial (tenant {TenantId})", + stale.Status, stale.SessionId, stale.StartedAt, registration.SessionId, registration.TenantId); + + messages.Add(new SignalRMessageAction("newevents") + { + GroupName = $"tenant-{registration.TenantId}", + Arguments = new object[] { new { + sessionId = stale.SessionId, + tenantId = registration.TenantId, + eventCount = 0, + sessionUpdate = new + { + Status = SessionStatus.Incomplete, + FailureReason = reason, + } + } } + }); + } + + return messages.ToArray(); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "Supersede pass failed for tenant {TenantId} session {SessionId}; registration proceeds", + registration.TenantId, registration.SessionId); + return Array.Empty(); + } + } + /// /// Fire-and-forget enrollment-started webhook. Loads tenant config, respects the /// opt-in WebhookNotifyOnStart/TeamsNotifyOnStart toggle, and dispatches diff --git a/src/Backend/AutopilotMonitor.Functions/Helpers/SerialNumberHeuristics.cs b/src/Backend/AutopilotMonitor.Functions/Helpers/SerialNumberHeuristics.cs new file mode 100644 index 00000000..5f818108 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Helpers/SerialNumberHeuristics.cs @@ -0,0 +1,25 @@ +namespace AutopilotMonitor.Functions.Helpers +{ + /// + /// Decides whether a reported serial number identifies a real physical device. OEM + /// placeholder strings (VM defaults, unset BIOS fields) would cross-match unrelated + /// devices and must never drive device-identity logic such as the registration + /// supersede pass (misclassification audit 2026-07-16). + /// + public static class SerialNumberHeuristics + { + private static readonly HashSet PlaceholderSerials = new(StringComparer.OrdinalIgnoreCase) + { + "0", "none", "unknown", "default string", "to be filled by o.e.m.", + "system serial number", "not specified", "not applicable", "n/a", "na", "invalid", + }; + + public static bool IsUsableSerialNumber(string? serialNumber) + { + var trimmed = serialNumber?.Trim(); + return !string.IsNullOrEmpty(trimmed) + && trimmed.Length >= 4 + && !PlaceholderSerials.Contains(trimmed); + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Program.cs b/src/Backend/AutopilotMonitor.Functions/Program.cs index 4f61c7fd..bb269403 100644 --- a/src/Backend/AutopilotMonitor.Functions/Program.cs +++ b/src/Backend/AutopilotMonitor.Functions/Program.cs @@ -251,6 +251,7 @@ builder.Services.AddSingleton(); builder.Services.AddHttpClient() .AddPolicyHandler((sp, _) => sp.GetRequiredService().ExternalDataApi); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton< AutopilotMonitor.Functions.Services.Monitoring.IAzureMonitorMetricsReader, AutopilotMonitor.Functions.Services.Monitoring.AzureMonitorMetricsReader>(); diff --git a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs index e312aca8..92b5b39b 100644 --- a/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs +++ b/src/Backend/AutopilotMonitor.Functions/Security/EndpointAccessPolicyCatalog.cs @@ -396,6 +396,9 @@ public static class EndpointAccessPolicyCatalog new("POST", "versions/block", EndpointPolicy.GlobalAdminOnly), new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly), new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly), + // One-time retro-reconcile of misclassified historical sessions (audit 2026-07-16) — + // mutating, dry-run by default, GA only. + new("POST", "maintenance/reclassify-legacy", EndpointPolicy.GlobalAdminOnly), // ── Critical-Table Backup (plan §PR1+§PR2) ───────────────────────────── new("POST", "global/backups/trigger", EndpointPolicy.GlobalAdminOnly), diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs b/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs index 149c5d15..069bf82f 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EnrollmentTimeoutClassifier.cs @@ -124,7 +124,16 @@ namespace AutopilotMonitor.Functions.Services var type = evt.EventType ?? string.Empty; if (Eq(type, "enrollment_failed") || Eq(type, "esp_failure")) - hasFailure = true; + { + // The agent's max-lifetime watchdog emits enrollment_failed with + // failureType=agent_timeout — that is "the agent gave up waiting", not an + // enrollment failure verdict. Counting it as an explicit failure would send + // every timed-out-but-actually-provisioned session through rule 1 straight to + // Failed, defeating the honest classification this class exists for + // (misclassification audit 2026-07-16, tenant a53e67ec cluster). + if (!Eq(TryGetDataString(evt, "failureType") ?? string.Empty, "agent_timeout")) + hasFailure = true; + } else if (Eq(type, "enrollment_complete") || Eq(type, "whiteglove_complete")) hasComplete = true; else if (Eq(type, "agent_emergency_break")) @@ -189,7 +198,11 @@ namespace AutopilotMonitor.Functions.Services else if (Eq(cat, AccountSetup)) { // Keep the strongest AccountSetup observation (highest completion count). + // A "0 of M" observation must still record the total — dropping it entirely + // made the Incomplete reason read "0/0" although a rollup WAS observed + // (misclassification audit 2026-07-16, session 08ddbeec). if (n > acctBestN) { acctBestN = n; acctBestM = total; } + else if (acctBestM == 0 && total > 0) { acctBestM = total; } } } @@ -297,9 +310,13 @@ namespace AutopilotMonitor.Functions.Services $"Device Setup completed; awaiting user / Account Setup phase — agent silent, within {graceHours}h grace{acct}"); } + // "0/0" would suggest a parsed rollup that never existed — when no Account Setup + // rollup was ever observed, say so instead (misclassification audit 2026-07-16). + var acctDetail = rollup.AccountSetupTotal > 0 + ? $"last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal}" + : "Account Setup progress never observed"; return (SessionStatus.Incomplete, - $"No completion signal within {graceHours}h grace after Device Setup completed " + - $"(last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal})"); + $"No completion signal within {graceHours}h grace after Device Setup completed ({acctDetail})"); } // 6. Silent before Device Setup completed, with no explicit failure → unknown, not a failure. diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs index 759bd088..84e5a9a1 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Classification.cs @@ -62,6 +62,14 @@ namespace AutopilotMonitor.Functions.Services case "session_stalled": classification.SessionStalledEvent = evt; break; + case "esp_provisioning_status": + case "desktop_arrived": + case "hello_provisioning_completed": + case "hello_skipped": + // Completion-evidence kinds the EnrollmentTimeoutClassifier feeds on — + // gate for the late-telemetry reconcile of Incomplete/AwaitingUser sessions. + classification.HasCompletionEvidenceCandidate = true; + break; case "agent_shutting_down": if (IsMaxLifetimeAgentShutdown(evt)) classification.AgentMaxLifetimeShutdownEvent = evt; @@ -111,6 +119,176 @@ namespace AutopilotMonitor.Functions.Services return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase); } + /// + /// True for an enrollment_failed event whose Data.failureType is + /// agent_timeout — the V1-parity companion of the max-lifetime watchdog shutdown + /// (LifecycleEmitters.CreateMaxLifetimeEmitter). Semantically identical to + /// : "the agent gave up waiting", not a failure + /// verdict, so it must be classified honestly instead of hard-failed. + /// + internal static bool IsAgentTimeoutFailure(EnrollmentEvent? evt) + { + if (evt == null || !string.Equals(evt.EventType, "enrollment_failed", StringComparison.Ordinal)) + return false; + var failureType = evt.Data != null && evt.Data.ContainsKey("failureType") + ? evt.Data["failureType"]?.ToString() + : null; + return string.Equals(failureType, "agent_timeout", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Shared verdict path for both max-lifetime shapes (enrollment_failed/agent_timeout and + /// agent_shutting_down/max_lifetime): classify the session honestly via + /// instead of hard-failing it. The watchdog + /// firing only proves the agent stopped waiting — the enrollment itself may be fully + /// provisioned with the user simply absent (AwaitingUser), silently dead (Incomplete), + /// or even provably finished (Succeeded reconcile). Mirrors the maintenance sweep's + /// stage-2 classification (misclassification audit 2026-07-16). + /// Returns (transitioned, failureReason) where failureReason is only set for the + /// terminal non-success outcomes so downstream failure notifications stay accurate. + /// + private async Task<(bool transitioned, string? failureReason)> ApplyMaxLifetimeVerdictAsync( + IngestEventsRequest request, string sessionPrefix, EventClassification c, + EnrollmentEvent triggerEvent, SessionStatus? preFetchedStatus) + { + var session = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId); + var currentStatus = session?.Status ?? preFetchedStatus; + + // Pending (WhiteGlove sealed) sessions are deliberately long-lived and resume via + // re-registration — a watchdog artifact must never terminalize them. + if (currentStatus == SessionStatus.Pending) + { + _logger.LogInformation( + "{SessionPrefix} max_lifetime verdict skipped — session is Pending (WhiteGlove)", sessionPrefix); + return (false, null); + } + + // Full event read (not just this batch): the ESP rollups / desktop / Hello facts the + // classifier needs usually predate the final watchdog batch. Rare path (once per + // max-lifetime session), so the extra read is acceptable. Best-effort — with no + // events the classifier degrades to rule 6 (Incomplete), still more honest than Failed. + List sessionEvents = new(); + try + { + sessionEvents = await _sessionRepo.GetSessionEventsAsync( + request.TenantId, request.SessionId, maxResults: 1000); + } + catch (Exception readEx) + { + _logger.LogWarning(readEx, + "{SessionPrefix} failed to read events for max_lifetime classification; proceeding with batch only", sessionPrefix); + sessionEvents = new List(); + } + + int? tenantGraceHours = null, absoluteMaxHours = null; + try + { + var config = await _configService.GetConfigurationAsync(request.TenantId); + tenantGraceHours = config?.SessionGraceHours; + absoluteMaxHours = config?.AbsoluteMaxSessionHours; + } + catch (Exception cfgEx) + { + _logger.LogWarning(cfgEx, "{SessionPrefix} failed to read tenant config for max_lifetime classification; using defaults", sessionPrefix); + } + var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours); + + var now = DateTime.UtcNow; + // WhiteGlove Part 2: measure the grace window from the resume, not the weeks-old + // Part-1 start (same anchor the maintenance sweep uses). + var effectiveStart = session?.ResumedAt ?? session?.StartedAt ?? triggerEvent.Timestamp; + + var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents); + var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession( + rollup, effectiveStart, now, graceHours, session?.LastEventAt ?? c.LatestEventTimestamp); + + // Keep the max-lifetime trigger visible: the Succeeded reasons already carry their own + // silence-transparency clause, everything else gets the watchdog context appended. + if (targetStatus != SessionStatus.Succeeded) + reason += " Verdict triggered by the agent's max-lifetime watchdog shutdown."; + + if (targetStatus == currentStatus) + { + _logger.LogInformation("{SessionPrefix} max_lifetime verdict {Status} equals current status — no transition", sessionPrefix, targetStatus); + return (false, null); + } + + var isTerminalNonSuccess = targetStatus == SessionStatus.Failed || targetStatus == SessionStatus.Incomplete; + + string? snapshotJson = null; + if (isTerminalNonSuccess) + { + try { snapshotJson = FailureSnapshotBuilder.Build(sessionEvents, now); } + catch (Exception snapEx) + { + _logger.LogWarning(snapEx, "{SessionPrefix} failed to build failure snapshot for max_lifetime verdict", sessionPrefix); + } + } + + var transitioned = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, targetStatus, triggerEvent.Phase, reason, + completedAt: isTerminalNonSuccess ? triggerEvent.Timestamp : (DateTime?)null, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, + failureSource: isTerminalNonSuccess ? "max_lifetime_watchdog" : null, + failureSnapshotJson: snapshotJson); + + _logger.LogWarning("{SessionPrefix} Status: {Status} via max_lifetime honest classification - {Reason} (transitioned={Transitioned})", + sessionPrefix, targetStatus, reason, transitioned); + + return (transitioned, isTerminalNonSuccess ? reason : null); + } + + /// + /// Re-runs the honest timeout classification for an Incomplete/AwaitingUser session after + /// straggler telemetry arrived, applying ONLY a Succeeded verdict (heal). Everything else + /// is a no-op: the existing verdict stands until evidence proves success. Best-effort — + /// a failure here must never break the ingest. + /// + private async Task TryLateTelemetryReconcileAsync( + IngestEventsRequest request, string sessionPrefix, EventClassification c) + { + try + { + var sessionEvents = await _sessionRepo.GetSessionEventsAsync( + request.TenantId, request.SessionId, maxResults: 1000); + var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents); + + var session = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId); + if (session == null) return; + + int? tenantGraceHours = null, absoluteMaxHours = null; + try + { + var config = await _configService.GetConfigurationAsync(request.TenantId); + tenantGraceHours = config?.SessionGraceHours; + absoluteMaxHours = config?.AbsoluteMaxSessionHours; + } + catch { /* defaults below */ } + var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours); + + var now = DateTime.UtcNow; + var effectiveStart = session.ResumedAt ?? session.StartedAt; + var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession( + rollup, effectiveStart, now, graceHours, session.LastEventAt ?? c.LatestEventTimestamp); + + if (targetStatus != SessionStatus.Succeeded) + return; + + var healed = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.Succeeded, + failureReason: reason, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp); + if (healed) + _logger.LogInformation( + "{SessionPrefix} Status: Succeeded (late-telemetry reconcile healed {Previous}) - {Reason}", + sessionPrefix, session.Status, reason); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "{SessionPrefix} late-telemetry reconcile failed; existing verdict stands", sessionPrefix); + } + } + private static bool IsPeriodicOrStallEvent(string? eventType) => eventType switch { "performance_snapshot" => true, @@ -142,15 +320,29 @@ namespace AutopilotMonitor.Functions.Services } else if (c.FailureEvent != null) { - failureReason = c.FailureEvent.Data?.ContainsKey("errorCode") == true - ? $"{c.FailureEvent.Message} ({c.FailureEvent.Data["errorCode"]})" - : c.FailureEvent.Message; + if (IsAgentTimeoutFailure(c.FailureEvent)) + { + // enrollment_failed(failureType=agent_timeout) is the max-lifetime watchdog + // giving up, NOT an enrollment failure verdict. Hard-failing here misdeclared + // fully provisioned WhiteGlove Part-2 sessions whose user simply never logged + // in (misclassification audit 2026-07-16, tenant a53e67ec: honest verdict + // AwaitingUser). Route through the same honest classification the maintenance + // sweep uses instead. + (statusTransitioned, failureReason) = await ApplyMaxLifetimeVerdictAsync( + request, sessionPrefix, c, c.FailureEvent, preFetchedStatus); + } + else + { + failureReason = c.FailureEvent.Data?.ContainsKey("errorCode") == true + ? $"{c.FailureEvent.Message} ({c.FailureEvent.Data["errorCode"]})" + : c.FailureEvent.Message; - statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync( - request.TenantId, request.SessionId, SessionStatus.Failed, c.FailureEvent.Phase, failureReason, - completedAt: c.FailureEvent.Timestamp, - earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp); - _logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned); + statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync( + request.TenantId, request.SessionId, SessionStatus.Failed, c.FailureEvent.Phase, failureReason, + completedAt: c.FailureEvent.Timestamp, + earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp); + _logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned); + } } else if (c.EspFailureEvent != null) { @@ -217,32 +409,14 @@ namespace AutopilotMonitor.Functions.Services // shutdown event is the last one the session ever sends, so without this // mapping the session stays InProgress forever. Any genuine terminal event // in the same batch wins via the else-if chain above; an already-terminal - // session is protected by the repository's idempotency guard. Pending - // (WhiteGlove) sessions are skipped — they are deliberately long-lived and - // resume via re-registration. - var currentStatus = preFetchedStatus - ?? (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status; - if (currentStatus == SessionStatus.Pending) - { - _logger.LogInformation( - "{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix); - } - else - { - var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true - ? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null; - failureReason = string.IsNullOrEmpty(uptimeMinutes) - ? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)." - : $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown)."; - - statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync( - request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason, - completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp, - earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp, - failureSource: "max_lifetime_watchdog"); - _logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})", - sessionPrefix, statusTransitioned); - } + // session is protected by the repository's idempotency guard. Instead of + // hard-failing, the session is classified honestly (AwaitingUser/Incomplete/ + // Succeeded/Failed) — the watchdog giving up is not an enrollment failure + // verdict (misclassification audit 2026-07-16). Pending (WhiteGlove) sessions + // are skipped inside the helper — they are deliberately long-lived and resume + // via re-registration. + (statusTransitioned, failureReason) = await ApplyMaxLifetimeVerdictAsync( + request, sessionPrefix, c, c.AgentMaxLifetimeShutdownEvent, preFetchedStatus); } if (c.WhiteGloveStartedEvent != null) @@ -283,6 +457,19 @@ namespace AutopilotMonitor.Functions.Services if (healed) _logger.LogInformation("{SessionPrefix} Status: InProgress (healed from Stalled by new real event)", sessionPrefix); } + else if (c.HasCompletionEvidenceCandidate + && (currentStatus == SessionStatus.Incomplete || currentStatus == SessionStatus.AwaitingUser)) + { + // Late-telemetry reconcile (misclassification audit 2026-07-16, session + // 357cefe7): a sweep verdict is based on the events known AT THAT TIME. When a + // straggler upload later delivers completion evidence (ESP rollups, desktop + // arrival, a positive Hello terminal) for an Incomplete/AwaitingUser session, + // re-run the honest classifier and heal to Succeeded if the evidence now proves + // it. Only the Succeeded verdict is applied — the transition guard allows + // Incomplete→Succeeded, and re-terminalizing with a different reason is noise. + // The batch pre-scan flag keeps this off the hot path for unrelated uploads. + await TryLateTelemetryReconcileAsync(request, sessionPrefix, c); + } } return (statusTransitioned, whiteGloveStatusTransitioned, failureReason); @@ -326,5 +513,14 @@ namespace AutopilotMonitor.Functions.Services /// with an authoritative distinct count at the terminal transition. /// public int RebootCount { get; set; } + + /// + /// True when this batch carries at least one event kind the + /// feeds on (ESP rollups, desktop arrival, + /// positive Hello terminals). Gates the late-telemetry reconcile for + /// Incomplete/AwaitingUser sessions so unrelated straggler uploads never trigger a + /// full event read (misclassification audit 2026-07-16). + /// + public bool HasCompletionEvidenceCandidate { get; set; } } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Notifications.cs b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Notifications.cs index e3a332ef..3bcf353e 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Notifications.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/EventIngestProcessor.Notifications.cs @@ -38,7 +38,12 @@ namespace AutopilotMonitor.Functions.Services ? $"https://portal.autopilotmonitor.com/sessions/{request.SessionId}" : null; - if (statusTransitioned && (c.CompletionEvent != null || c.FailureEvent != null)) + // A failure alert requires an actual failure-ish verdict: an agent_timeout + // enrollment_failed can honestly classify to AwaitingUser or even Succeeded + // (ApplyMaxLifetimeVerdictAsync), in which case failureReason stays null and no + // failure notification must go out for a session that did not fail. + var failureVerdictApplies = c.FailureEvent != null && failureReason != null; + if (statusTransitioned && (c.CompletionEvent != null || failureVerdictApplies)) { var isSuccess = c.CompletionEvent != null; var targets = isSuccess ? successChannels : failureChannels; diff --git a/src/Backend/AutopilotMonitor.Functions/Services/LegacyReclassificationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/LegacyReclassificationService.cs new file mode 100644 index 00000000..27c920e4 --- /dev/null +++ b/src/Backend/AutopilotMonitor.Functions/Services/LegacyReclassificationService.cs @@ -0,0 +1,288 @@ +using AutopilotMonitor.Functions.Helpers; +using AutopilotMonitor.Shared.DataAccess; +using AutopilotMonitor.Shared.Models; +using Microsoft.Extensions.Logging; + +namespace AutopilotMonitor.Functions.Services +{ + /// + /// One-time admin retro-reconciliation of misclassified historical sessions + /// (misclassification audit 2026-07-16). Two modes, both dry-run capable: + /// + /// legacy_timeouts — Failed rows carrying the pre-classifier blanket + /// "Session timed out after ..." verdict are re-run through + /// ; sessions the classifier would NOT fail + /// graduate to their honest verdict (Incomplete, or Succeeded on completion evidence). + /// The audit sample projected ~85–90% of these as misdeclared. + /// pending_orphans — Pending (WhiteGlove Part 1) rows whose device provably + /// registered a LATER session (Part 2 under a fresh session id, or re-provisioning) are + /// resolved as Incomplete("Superseded by ..."). Forward-looking coverage lives in the + /// registration supersede pass; this mode clears the existing backlog. + /// + /// Hand-marked sessions (AdminMarkedAction) are never touched. + /// + public class LegacyReclassificationService + { + private readonly ILogger _logger; + private readonly ISessionRepository _sessionRepo; + private readonly IMaintenanceRepository _maintenanceRepo; + private readonly TenantConfigurationService _configService; + + public const string ModeLegacyTimeouts = "legacy_timeouts"; + public const string ModePendingOrphans = "pending_orphans"; + private const int MaxSamplesInResponse = 100; + + public LegacyReclassificationService( + ILogger logger, + ISessionRepository sessionRepo, + IMaintenanceRepository maintenanceRepo, + TenantConfigurationService configService) + { + _logger = logger; + _sessionRepo = sessionRepo; + _maintenanceRepo = maintenanceRepo; + _configService = configService; + } + + public sealed class ReclassificationSample + { + public string TenantId { get; set; } = string.Empty; + public string SessionId { get; set; } = string.Empty; + public string OldStatus { get; set; } = string.Empty; + public string NewStatus { get; set; } = string.Empty; + public string Reason { get; set; } = string.Empty; + } + + public sealed class ReclassificationResult + { + public string Mode { get; set; } = string.Empty; + public bool DryRun { get; set; } + public int TenantsExamined { get; set; } + public int SessionsExamined { get; set; } + public int WouldChange { get; set; } + public int Changed { get; set; } + public int ToSucceeded { get; set; } + public int ToIncomplete { get; set; } + public int KeptFailed { get; set; } + public int Skipped { get; set; } + public int Errors { get; set; } + /// True when the maxSessions cap stopped the run before the backlog was exhausted — re-run to continue. + public bool CapReached { get; set; } + public List Samples { get; } = new(); + } + + public async Task ReclassifyLegacyTimeoutsAsync( + string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy) + { + var result = new ReclassificationResult { Mode = ModeLegacyTimeouts, DryRun = dryRun }; + var now = DateTime.UtcNow; + + foreach (var tenantId in await ResolveTenantsAsync(tenantIdScope)) + { + if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; } + result.TenantsExamined++; + + int? tenantGraceHours = null, absoluteMaxHours = null; + try + { + var config = await _configService.GetConfigurationAsync(tenantId); + tenantGraceHours = config?.SessionGraceHours; + absoluteMaxHours = config?.AbsoluteMaxSessionHours; + } + catch { /* defaults */ } + var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours); + + var candidates = await _maintenanceRepo.GetLegacyTimeoutFailedSessionsAsync( + tenantId, maxSessions - result.SessionsExamined + 1); + if (candidates.Count > maxSessions - result.SessionsExamined) + { + result.CapReached = true; + candidates = candidates.Take(maxSessions - result.SessionsExamined).ToList(); + } + + foreach (var session in candidates) + { + result.SessionsExamined++; + try + { + var events = await _sessionRepo.GetSessionEventsAsync(tenantId, session.SessionId, maxResults: 1000); + var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events); + var effectiveStart = session.ResumedAt ?? session.StartedAt; + var (target, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession( + rollup, effectiveStart, now, graceHours, session.LastEventAt); + + if (target == SessionStatus.Failed) + { + result.KeptFailed++; + continue; + } + // These sessions are weeks old — the grace window has long expired, so the + // classifier cannot return AwaitingUser; guard anyway so a surprise + // non-terminal verdict is skipped rather than applied. + if (target != SessionStatus.Succeeded && target != SessionStatus.Incomplete) + { + result.Skipped++; + continue; + } + + if (target != SessionStatus.Succeeded) + reason += " Retro-reclassified from the legacy blanket timeout verdict."; + + RecordSample(result, tenantId, session, target, reason, oldStatus: "Failed"); + + if (dryRun) + { + result.WouldChange++; + continue; + } + + var transitioned = await _sessionRepo.UpdateSessionStatusAsync( + tenantId, session.SessionId, target, + failureReason: reason, + failureSource: target == SessionStatus.Incomplete ? "legacy_timeout_retro_reconcile" : null, + allowTerminalReclassification: true); + if (transitioned) + { + result.Changed++; + if (target == SessionStatus.Succeeded) result.ToSucceeded++; else result.ToIncomplete++; + } + else + { + result.Skipped++; + } + } + catch (Exception ex) + { + result.Errors++; + _logger.LogWarning(ex, "Legacy timeout reclassification failed for session {SessionId} (tenant {TenantId})", + session.SessionId, tenantId); + } + } + } + + await WriteAuditAsync(result, tenantIdScope, triggeredBy); + return result; + } + + public async Task ResolvePendingOrphansAsync( + string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy) + { + var result = new ReclassificationResult { Mode = ModePendingOrphans, DryRun = dryRun }; + + foreach (var tenantId in await ResolveTenantsAsync(tenantIdScope)) + { + if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; } + result.TenantsExamined++; + + var lean = await _maintenanceRepo.GetSessionsLeanAsync(tenantId); + var bySerial = lean + .Where(s => SerialNumberHeuristics.IsUsableSerialNumber(s.SerialNumber)) + .GroupBy(s => s.SerialNumber!.Trim(), StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase); + + foreach (var pending in lean.Where(s => s.Status == SessionStatus.Pending)) + { + if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; } + if (!SerialNumberHeuristics.IsUsableSerialNumber(pending.SerialNumber)) + continue; + + result.SessionsExamined++; + try + { + var successor = bySerial[pending.SerialNumber!.Trim()] + .Where(s => !string.Equals(s.SessionId, pending.SessionId, StringComparison.OrdinalIgnoreCase) + && s.StartedAt > pending.StartedAt) + .OrderByDescending(s => s.StartedAt) + .FirstOrDefault(); + if (successor == null) + { + result.Skipped++; + continue; + } + + var reason = $"Superseded by session {successor.SessionId}: the device registered a new " + + $"enrollment session at {successor.StartedAt:yyyy-MM-dd HH:mm} UTC before this one " + + "reached a terminal state."; + RecordSample(result, tenantId, pending, SessionStatus.Incomplete, reason, oldStatus: "Pending"); + + if (dryRun) + { + result.WouldChange++; + continue; + } + + // Pending is non-terminal — the standard transition guard covers this write. + var transitioned = await _sessionRepo.UpdateSessionStatusAsync( + tenantId, pending.SessionId, SessionStatus.Incomplete, + failureReason: reason, failureSource: "superseded_by_reregistration"); + if (transitioned) { result.Changed++; result.ToIncomplete++; } + else result.Skipped++; + } + catch (Exception ex) + { + result.Errors++; + _logger.LogWarning(ex, "Pending-orphan resolution failed for session {SessionId} (tenant {TenantId})", + pending.SessionId, tenantId); + } + } + } + + await WriteAuditAsync(result, tenantIdScope, triggeredBy); + return result; + } + + private async Task> ResolveTenantsAsync(string? tenantIdScope) + { + if (!string.IsNullOrEmpty(tenantIdScope)) + return new List { tenantIdScope }; + return await _maintenanceRepo.GetAllTenantIdsAsync(); + } + + private static void RecordSample( + ReclassificationResult result, string tenantId, SessionSummary session, + SessionStatus target, string reason, string oldStatus) + { + if (result.Samples.Count >= MaxSamplesInResponse) return; + result.Samples.Add(new ReclassificationSample + { + TenantId = tenantId, + SessionId = session.SessionId, + OldStatus = oldStatus, + NewStatus = target.ToString(), + Reason = reason, + }); + } + + private async Task WriteAuditAsync(ReclassificationResult result, string? tenantIdScope, string triggeredBy) + { + // Dry-runs leave no audit trail — nothing changed. Real runs record the counts under + // the scoped tenant (or the platform marker tenant scope "00000000-...": audit rows + // require a tenant partition, so an all-tenant run logs one entry per touched tenant + // would be noisy — a single summary entry under the scope keeps it queryable). + if (result.DryRun || result.Changed == 0) return; + try + { + await _maintenanceRepo.LogAuditEntryAsync( + tenantIdScope ?? Guid.Empty.ToString(), + "LegacyReclassification", + "Session", + $"{result.Changed} sessions", + triggeredBy, + new Dictionary + { + { "Mode", result.Mode }, + { "SessionsExamined", result.SessionsExamined.ToString() }, + { "Changed", result.Changed.ToString() }, + { "ToSucceeded", result.ToSucceeded.ToString() }, + { "ToIncomplete", result.ToIncomplete.ToString() }, + { "KeptFailed", result.KeptFailed.ToString() }, + { "Errors", result.Errors.ToString() }, + }); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to write audit entry for legacy reclassification run"); + } + } + } +} diff --git a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs index 584f0678..6def2870 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/MaintenanceService.cs @@ -256,9 +256,11 @@ namespace AutopilotMonitor.Functions.Services /// Two-stage sweep for stuck sessions: /// 1. **Agent-silent Stalled marker** (2h fixed): Sessions still InProgress but with no events /// for more than 2h are marked as Stalled (non-terminal). Picks up agents that cannot emit - /// session_stalled themselves (bluescreen, network loss, power off). Excludes WhiteGlove. - /// 2. **Session timeout Failed** (5h default): Sessions that exceed the full SessionTimeoutHours - /// window (InProgress or Stalled) graduate to terminal Failed state. + /// session_stalled themselves (bluescreen, network loss, power off). Sealed WhiteGlove + /// devices are protected by their Pending status; resumed Part-2 runs participate normally + /// with their window anchored to ResumedAt (misclassification audit 2026-07-16). + /// 2. **Session timeout** (5h default): Sessions that exceed the full SessionTimeoutHours + /// window (InProgress or Stalled) are reclassified honestly via EnrollmentTimeoutClassifier. /// Both stages run in the same 2h maintenance pass so no new timers are introduced /// (preserving Container App scale-to-zero). /// @@ -341,10 +343,27 @@ namespace AutopilotMonitor.Functions.Services foreach (var session in stalledSessions) { + // WhiteGlove Part 2: the sweep window is measured from the resume, not from + // the weeks-old Part-1 StartedAt — otherwise a freshly resumed Part 2 would + // be instantly "past every window" and terminalized while still live + // (misclassification audit 2026-07-16). ResumedAt > StartedAt always holds + // when set; plain non-WG sessions have no ResumedAt and keep StartedAt. + var effectiveStart = session.ResumedAt ?? session.StartedAt; + if (effectiveStart > cutoffTime) + continue; // resumed run not yet past the timeout window — stage 1 owns it + + // Silence guard: this is a *silent-session* sweep. A session whose agent + // reported within the silence window is provably alive (long enrollments can + // legitimately exceed SessionTimeoutHours) and must never be terminalized here + // — without this, an actively-installing 6h enrollment could be classified + // Incomplete mid-run purely because it outlived the 5h window. + if (session.LastEventAt.HasValue && session.LastEventAt.Value > silenceCutoff) + continue; + // Fast path: an AwaitingUser session still inside the grace window needs no // work and — crucially — no event read (grace is purely time-based). A late // completion is picked up by the ingest path, so we don't re-scan every pass. - var elapsedHours = (now - session.StartedAt).TotalHours; + var elapsedHours = (now - effectiveStart).TotalHours; if (session.Status == SessionStatus.AwaitingUser && elapsedHours < graceHours) continue; @@ -375,7 +394,7 @@ namespace AutopilotMonitor.Functions.Services // Device Setup → Incomplete. Never Failed without an explicit failure signal. var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents); var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession( - rollup, session.StartedAt, now, graceHours, session.LastEventAt); + rollup, effectiveStart, now, graceHours, session.LastEventAt); // No-op if the verdict equals the current (non-terminal) state. if (targetStatus == session.Status) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs index a4b75eb5..c1a128a9 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Maintenance.cs @@ -614,12 +614,15 @@ namespace AutopilotMonitor.Functions.Services // EnrollmentTimeoutClassifier. AwaitingUser is included too so it keeps being // re-evaluated each pass and graduates to Incomplete once SessionGraceHours elapses // (tasks/enrollment-status-reclassification.md). - // IsPreProvisioned ne true → WhiteGlove sessions are intentionally long-lived and - // must never be timed out, even after months in storage. + // WhiteGlove: sealed devices are protected by STATUS (Pending is not in this set), + // not by IsPreProvisioned — a resumed Part-2 session is InProgress and must be + // sweepable like any other run, otherwise it lingers as InProgress forever when the + // agent dies mid-Part-2 (misclassification audit 2026-07-16). The caller anchors the + // timeout window to ResumedAt for these rows so a freshly resumed Part 2 with a + // weeks-old StartedAt is not terminalized while it is still live. var filter = $"PartitionKey eq '{tenantId}' " + $"and (Status eq 'InProgress' or Status eq 'Stalled' or Status eq 'AwaitingUser') " + - $"and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z' " + - $"and IsPreProvisioned ne true"; + $"and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z'"; var query = tableClient.QueryAsync(filter: filter); var sessions = new List(); @@ -644,10 +647,15 @@ namespace AutopilotMonitor.Functions.Services /// themselves (bluescreen, network loss, power off). /// /// Filter criteria: - /// - Status eq 'InProgress' — do not re-mark already-Stalled sessions + /// - Status eq 'InProgress' — do not re-mark already-Stalled sessions; sealed WhiteGlove + /// sessions are Pending and therefore never match (the former extra + /// "IsPreProvisioned ne true" clause also excluded resumed Part-2 runs, which then could + /// never be marked Stalled — misclassification audit 2026-07-16) /// - LastEventAt < silenceCutoff — at least the configured window of agent silence - /// - StartedAt ge hardCutoff — do not catch sessions that will be picked up by the 5h timeout sweep - /// - IsPreProvisioned ne true — exclude sealed WhiteGlove sessions (intentionally long-lived) + /// - StartedAt ge hardCutoff OR ResumedAt ge hardCutoff — do not catch sessions that will + /// be picked up by the 5h timeout sweep. The ResumedAt alternative keeps a WhiteGlove + /// Part-2 run (weeks-old StartedAt, fresh ResumedAt) in THIS stage instead of stage 2; + /// rows without ResumedAt fail that comparison server-side and fall back to StartedAt. /// public async Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) { @@ -662,8 +670,7 @@ namespace AutopilotMonitor.Functions.Services var filter = $"PartitionKey eq '{tenantId}' " + $"and Status eq 'InProgress' " + $"and LastEventAt lt datetime'{silenceCutoffStr}Z' " + - $"and StartedAt ge datetime'{hardCutoffStr}Z' " + - $"and IsPreProvisioned ne true"; + $"and (StartedAt ge datetime'{hardCutoffStr}Z' or ResumedAt ge datetime'{hardCutoffStr}Z')"; var query = tableClient.QueryAsync(filter: filter); @@ -682,6 +689,76 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Failed sessions carrying the pre-classifier blanket "Session timed out after ..." + /// FailureReason — candidates for the one-time admin retro-reconcile + /// (misclassification audit 2026-07-16). The prefix is matched server-side via an + /// OData string range (ge prefix, lt prefix-with-last-char-incremented). + /// + public async Task> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + // Prefix range for "Session timed out after": upper bound increments the final + // 'r' to 's'. Matches exactly the legacy sweep's reason format + // "Session timed out after {N} hours (started at ... UTC)". + var filter = $"PartitionKey eq '{tenantId}' " + + $"and Status eq 'Failed' " + + $"and FailureReason ge 'Session timed out after' " + + $"and FailureReason lt 'Session timed out aftes'"; + + var sessions = new List(); + await foreach (var entity in tableClient.QueryAsync(filter: filter)) + { + sessions.Add(MapToSessionSummary(entity)); + if (sessions.Count >= maxResults) + break; + } + return sessions; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to get legacy timeout-failed sessions for tenant {tenantId}"); + return new List(); + } + } + + /// + /// Narrow projection of every session row in the tenant partition — one scan feeding the + /// in-memory Pending-orphan matching (misclassification audit 2026-07-16). Fields outside + /// the projection come back as defaults; callers must not read them. + /// + public async Task> GetSessionsLeanAsync(string tenantId) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + var select = new[] + { + "PartitionKey", "RowKey", "Status", "StartedAt", "ResumedAt", "LastEventAt", + "SerialNumber", "IsPreProvisioned" + }; + + var sessions = new List(); + await foreach (var entity in tableClient.QueryAsync( + filter: $"PartitionKey eq '{tenantId}'", select: select)) + { + sessions.Add(MapToSessionSummary(entity)); + } + return sessions; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to get lean session projection for tenant {tenantId}"); + return new List(); + } + } + /// /// Gets sessions where the device has been actively sending data for longer than /// . diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs index 612ce165..1bb54bfb 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Sessions.cs @@ -1353,6 +1353,46 @@ namespace AutopilotMonitor.Functions.Services } } + /// + /// Open (non-terminal) sessions of the same physical device within one tenant: Pending, + /// InProgress, Stalled or AwaitingUser rows matching the SerialNumber. Server-side + /// filtered partition query with a narrow projection — SerialNumber is not a key column, + /// so this scans the tenant partition, which is acceptable for the once-per-registration + /// supersede pass (misclassification audit 2026-07-16). Fail-soft: returns an empty list + /// on storage errors so registration is never blocked. + /// + public async Task> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber) + { + SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); + if (string.IsNullOrWhiteSpace(serialNumber)) + return new List(); + + try + { + var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions); + var filter = $"PartitionKey eq '{tenantId}' " + + $"and SerialNumber eq '{ODataSanitizer.EscapeValue(serialNumber)}' " + + $"and (Status eq 'Pending' or Status eq 'InProgress' or Status eq 'Stalled' or Status eq 'AwaitingUser')"; + var select = new[] + { + "PartitionKey", "RowKey", "Status", "StartedAt", "ResumedAt", "LastEventAt", + "SerialNumber", "DeviceName", "IsPreProvisioned", "Model" + }; + + var sessions = new List(); + await foreach (var entity in tableClient.QueryAsync(filter: filter, select: select)) + { + sessions.Add(MapToSessionSummary(entity)); + } + return sessions; + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to get open sessions for device serial in tenant {tenantId}"); + return new List(); + } + } + /// /// Finds the tenantId for a session by scanning SessionsIndex. /// Used for Global Admin cross-tenant session lookup when tenantId is unknown. @@ -1452,7 +1492,7 @@ namespace AutopilotMonitor.Functions.Services : null; } - public async Task UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null) + public async Task UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null, bool allowTerminalReclassification = false) { SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId)); SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId)); @@ -1477,7 +1517,25 @@ namespace AutopilotMonitor.Functions.Services // overwrite an existing terminal. Non-terminal incoming statuses fall through to their // own guards below. var existingStatusStr = session.GetString("Status"); - if (!IsTerminalTransitionAllowed(existingStatusStr, status)) + // allowTerminalReclassification (admin retro-reconcile ONLY, misclassification + // audit 2026-07-16): permits rewriting an existing terminal verdict — e.g. a + // legacy pre-classifier "Session timed out" Failed graduating to Incomplete. + // Hand-marked sessions stay untouchable (checked below), and an idempotent + // same-status write is still refused. + if (allowTerminalReclassification) + { + if (existingStatusStr == status.ToString()) + { + _logger.LogInformation($"Session {sessionId}: reclassification to identical status '{status}' — no-op"); + return false; + } + if (!string.IsNullOrEmpty(session.GetString("AdminMarkedAction"))) + { + _logger.LogInformation($"Session {sessionId}: admin-marked terminal — reclassification refused"); + return false; + } + } + else if (!IsTerminalTransitionAllowed(existingStatusStr, status)) { _logger.LogInformation($"Session {sessionId}: transition '{existingStatusStr}' → '{status}' not allowed (terminal/reconcile guard), skipping"); return false; @@ -1753,7 +1811,18 @@ namespace AutopilotMonitor.Functions.Services // our retries (e.g. an Incomplete verdict stomping a concurrently-written // Succeeded/Failed, or a second Incomplete overwriting the first). var freshStatusStr = freshSession.GetString("Status"); - if (!IsTerminalTransitionAllowed(freshStatusStr, status)) + // Mirror the normal path's override semantics (admin retro-reconcile): + // same-status and admin-marked rows stay refused even under override. + if (allowTerminalReclassification) + { + if (freshStatusStr == status.ToString() + || !string.IsNullOrEmpty(freshSession.GetString("AdminMarkedAction"))) + { + _logger.LogInformation($"Session {sessionId}: reclassification force write refused (same status or admin-marked)"); + return false; + } + } + else if (!IsTerminalTransitionAllowed(freshStatusStr, status)) { _logger.LogInformation($"Session {sessionId}: transition '{freshStatusStr}' → '{status}' not allowed (terminal/reconcile guard), skipping force write"); return false; diff --git a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs index 4a3fe6d9..0a219cf3 100644 --- a/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs +++ b/src/Shared/AutopilotMonitor.DecisionCore/Engine/DecisionEngine.Edge.cs @@ -676,6 +676,64 @@ namespace AutopilotMonitor.DecisionCore.Engine return new DecisionStep(rearmedState, rearmedTransition, rearmedEffects); } + // Hello-never-observed promote (misclassification audit 2026-07-16, sessions + // 2dac8298/7635ab18) — esp-exit variant only. When the Hello policy read never + // committed (HelloPolicyEnabled == null: no PassportForWork value ever appeared, + // so the tracker never emitted hello_policy_detected) AND no wizard was ever + // seen, the hello_resolution prerequisite is STRUCTURALLY unresolvable: no + // policy fact activates the disabled stand-in, and hello_safety was never armed + // because the AwaitingHello promotion path was never taken. The DAD-validated + // real-user desktop has been up for the whole 30-min window with no wizard and + // no enforcement progress (the re-arm guard above ran first) — failing now would + // declare a provably finished enrollment failed. Promote to AwaitingHello with + // the safety deadline instead of completing outright: a genuinely late wizard + // start still blocks completion (HelloWizardStarted re-arms the wait), otherwise + // the safety timeout resolves Hello synthetically and the session completes + // through Finalizing. Deliberately does NOT require the IME user-session gate — + // the FP sessions never saw an IME user-session completion (that gate guards + // completion evidence, not user presence; presence is the DAD-validated desktop). + if (!hasAdvisoryAnchor + && desktopArrived + && state.HelloResolvedUtc == null + && state.HelloPolicyEnabled == null + && state.HelloWizardStartedUtc == null) + { + var neverObservedSafetyDueAt = EffectiveDeadlineBase(state, signal).Add(s_helloSafetyWindow); + var neverObservedSafety = new ActiveDeadline( + name: DeadlineNames.HelloSafety, + dueAtUtc: neverObservedSafetyDueAt, + firesSignalKind: DecisionSignalKind.DeadlineFired, + firesPayload: new Dictionary + { + [SignalPayloadKeys.Deadline] = DeadlineNames.HelloSafety, + }); + + builder = builder + .WithStage(SessionStage.AwaitingHello) + .AddDeadline(neverObservedSafety); + + var neverObservedTrigger = + $"DeadlineFired:{DeadlineNames.AdvisoryCompletion}:HelloNeverObservedPromote"; + var neverObservedWaitingEffect = BuildCompletionWaitingEffect( + state, builder, signal, trigger: neverObservedTrigger); + + var neverObservedState = builder.Build(); + var neverObservedTransition = BuildTakenTransition( + before: state, + signal: signal, + toStage: SessionStage.AwaitingHello, + nextStepIndex: nextStep, + trigger: neverObservedTrigger); + + var neverObservedEffects = new List(2) + { + new DecisionEffect(DecisionEffectKind.ScheduleDeadline, deadline: neverObservedSafety), + }; + if (neverObservedWaitingEffect != null) neverObservedEffects.Add(neverObservedWaitingEffect); + + return new DecisionStep(neverObservedState, neverObservedTransition, neverObservedEffects.ToArray()); + } + // Conjunction not met — the session is failed. The two arming variants carry // distinct context (see method doc): the advisory variant un-defangs the original // ESP failure; the esp-exit variant never had an ESP failure, so it gets its own diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs index 0db35a24..5bdefd01 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/IMaintenanceRepository.cs @@ -84,6 +84,22 @@ namespace AutopilotMonitor.Shared.DataAccess Task> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff); Task> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours); + // --- Legacy Reclassification (misclassification audit 2026-07-16) --- + /// + /// Failed sessions whose FailureReason carries the pre-classifier blanket + /// "Session timed out after ..." verdict — candidates for the one-time admin + /// retro-reconcile through EnrollmentTimeoutClassifier. Server-side prefix + /// range filter, capped at . + /// + Task> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults); + + /// + /// Narrow projection of every session row in the tenant partition (id, status, timing, + /// serial). Used by the Pending-orphan resolution to match superseding sessions of the + /// same device in memory with one scan instead of one scan per Pending row. + /// + Task> GetSessionsLeanAsync(string tenantId); + // --- Tenant Discovery --- Task> GetAllTenantIdsAsync(); diff --git a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs index 396dc868..9465e990 100644 --- a/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs +++ b/src/Shared/AutopilotMonitor.Shared/DataAccess/ISessionRepository.cs @@ -81,7 +81,7 @@ namespace AutopilotMonitor.Shared.DataAccess bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, - string? failureSnapshotJson = null); + string? failureSnapshotJson = null, bool allowTerminalReclassification = false); /// /// Increments per-session counters via read-modify-write. Returns the post-merge session /// snapshot (the RMW read with the applied increments) so hot-path callers can skip a @@ -140,6 +140,16 @@ namespace AutopilotMonitor.Shared.DataAccess /// Task MarkExcessiveEventsAutoActionedAsync(string tenantId, string sessionId); + /// + /// Open (non-terminal: Pending / InProgress / Stalled / AwaitingUser) sessions of the same + /// physical device, identified by SerialNumber within the tenant partition. Narrow + /// projection — used by the registration supersede pass to resolve orphaned predecessor + /// sessions when a device re-registers under a new session id + /// (misclassification audit 2026-07-16: WhiteGlove Part 2 under a fresh id left the + /// Part-1 row Pending forever). + /// + Task> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber); + // --- IME Version History --- Task RecordImeVersionAsync(string version, string tenantId, string sessionId); Task> GetImeVersionHistoryAsync(); diff --git a/src/Web/autopilot-monitor-web/app/sla/page.tsx b/src/Web/autopilot-monitor-web/app/sla/page.tsx index 7f88a6d6..9846f626 100644 --- a/src/Web/autopilot-monitor-web/app/sla/page.tsx +++ b/src/Web/autopilot-monitor-web/app/sla/page.tsx @@ -102,6 +102,7 @@ function formatDuration(seconds: number): string { const statusLabels: Record = { 0: "InProgress", 1: "Pending", 2: "Stalled", 3: "Succeeded", 4: "Failed", 5: "Unknown", + 6: "AwaitingUser", 7: "Incomplete", // appended 2026-07-08 — ordinals must match SessionStatus }; export default function SlaPage() {