mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
fix(status): honest verdicts across all misclassification audit findings
Misclassification audit 2026-07-16 (1000 non-succeeded sessions, 22 deep
dives) surfaced six systematic misdeclaration sources. This implements
all of them:
1. DecisionCore: HelloNeverObservedPromote in the esp-exit advisory
handler — when the Hello policy read never committed and no wizard
was ever seen, hello_resolution is structurally unresolvable; promote
to AwaitingHello + hello_safety instead of hard-failing a provably
finished enrollment (sessions 2dac8298/7635ab18). Placed after the
enforcement-progress re-arm so active installs still win; advisory-
anchor variant stays exempt.
2. Ingest: both max-lifetime shapes (enrollment_failed/agent_timeout and
agent_shutting_down/max_lifetime) now route through the
EnrollmentTimeoutClassifier instead of hard Failed (tenant a53e67ec:
fully provisioned WG Part 2, user never logged in => AwaitingUser).
agent_timeout no longer counts as HasExplicitFailure; failure
webhooks fire only on actual failure verdicts.
3. Maintenance sweep: dropped the too-broad "IsPreProvisioned ne true"
exclusion (sealed devices are protected by their Pending status);
resumed WG Part-2 runs sweep normally with the window anchored to
ResumedAt. New stage-2 silence guard also fixes a latent bug where an
actively-reporting >5h enrollment could be terminalized mid-run.
4. Registration supersede: a fresh registration resolves older
non-terminal sessions of the same device (serial, placeholder-guarded)
as Incomplete("Superseded by ...") with live SignalR updates — closes
the forever-Pending orphan gap when WG Part 2 runs under a new id.
5. Admin retro-reconcile: POST /api/maintenance/reclassify-legacy
(GlobalAdminOnly, DRY-RUN BY DEFAULT) re-runs the classifier over the
pre-classifier "Session timed out" Failed backlog (~495 rows, sample
projected ~87% misdeclared) and resolves Pending orphans. Requires the
new opt-in allowTerminalReclassification override in the status
writer (admin-marked rows stay untouchable).
6. Polish: "0 of N" AccountSetup rollups keep their total (no more
fabricated "0/0" reasons, "never observed" wording when absent);
late-telemetry reconcile heals Incomplete/AwaitingUser to Succeeded
when straggler evidence proves completion; SLA page knows
AwaitingUser/Incomplete ordinals; script_failed on Success/exit-0 now
names the stderr rule in its message.
Tests: backend 2965, DecisionCore 474 (+6 new), agent 1777, web 557 —
all green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -953,7 +953,17 @@ namespace AutopilotMonitor.Agent.V2.Core.SignalAdapters
|
||||
var exitSuffix = script.ExitCode.HasValue
|
||||
? $" (exit: {script.ExitCode.Value.ToString(culture)})"
|
||||
: string.Empty;
|
||||
var msg = $"{label} {shortId}: {messageCore}{exitSuffix}";
|
||||
// stderr is the only rule that can classify a Success/exit-0 script as script_failed
|
||||
// (rule 2 above). Without naming that in the message, the event reads as a
|
||||
// contradiction — "script_failed: Success (exit: 0)" (misclassification audit
|
||||
// 2026-07-16). Keep the visibility rule, make the message say why.
|
||||
var stderrSuffix = isFailure
|
||||
&& hasStderr
|
||||
&& !string.Equals(script.Result, "Failed", StringComparison.OrdinalIgnoreCase)
|
||||
&& (!script.ExitCode.HasValue || script.ExitCode.Value == 0)
|
||||
? " — stderr output detected"
|
||||
: string.Empty;
|
||||
var msg = $"{label} {shortId}: {messageCore}{exitSuffix}{stderrSuffix}";
|
||||
|
||||
_post.Emit(
|
||||
eventType: eventType,
|
||||
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.DecisionCore.State;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.DecisionCore.Tests
|
||||
{
|
||||
/// <summary>
|
||||
/// Misclassification audit 2026-07-16 (sessions 2dac8298 / 7635ab18): the ESP page exits
|
||||
/// normally, the DAD-validated real-user desktop arrives, but the Hello policy read never
|
||||
/// commits (no PassportForWork registry value → no hello_policy_detected ever) and no
|
||||
/// wizard is ever seen. The <c>hello_resolution</c> prerequisite is then structurally
|
||||
/// unresolvable and the esp-exit advisory window used to hard-fail a provably finished
|
||||
/// enrollment with <c>esp_exit_without_completion_evidence</c>.
|
||||
/// Coverage:
|
||||
/// <list type="bullet">
|
||||
/// <item>Hello-never-observed at advisory fire → promote to AwaitingHello + arm
|
||||
/// hello_safety (NOT Failed), trigger suffix <c>HelloNeverObservedPromote</c>.</item>
|
||||
/// <item>hello_safety fire then completes through Finalizing with synthetic
|
||||
/// HelloOutcome=Timeout.</item>
|
||||
/// <item>Policy read committed (enabled) → fallthrough still fails (no behavior change).</item>
|
||||
/// <item>No desktop → fallthrough still fails.</item>
|
||||
/// <item>Advisory-anchor variant (real ESP terminal failure) is exempt — still fails.</item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public sealed class AdvisoryCompletionHelloNeverObservedTests
|
||||
{
|
||||
private static readonly DateTime T0 = new DateTime(2026, 7, 15, 8, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static DecisionSignal MakeSignal(
|
||||
long ordinal,
|
||||
DecisionSignalKind kind,
|
||||
DateTime occurredAtUtc,
|
||||
IReadOnlyDictionary<string, string>? payload = null)
|
||||
{
|
||||
return new DecisionSignal(
|
||||
sessionSignalOrdinal: ordinal,
|
||||
sessionTraceOrdinal: ordinal,
|
||||
kind: kind,
|
||||
kindSchemaVersion: 1,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
sourceOrigin: "test",
|
||||
evidence: new Evidence(EvidenceKind.Synthetic, $"{kind}-{ordinal}", "test"),
|
||||
payload: payload);
|
||||
}
|
||||
|
||||
private static DecisionSignal DeadlineFired(long ordinal, DateTime occurredAtUtc, string deadlineName) =>
|
||||
MakeSignal(ordinal, DecisionSignalKind.DeadlineFired, occurredAtUtc,
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.Deadline] = deadlineName });
|
||||
|
||||
private static ActiveDeadline? FindDeadline(DecisionState state, string name) =>
|
||||
state.Deadlines.FirstOrDefault(d => d.Name == name);
|
||||
|
||||
/// <summary>
|
||||
/// Session-2dac8298 shape: normal ESP run, AccountSetup entered, genuine final exit
|
||||
/// (arms the esp-exit advisory window), real-user desktop shortly after — and NO
|
||||
/// hello_policy_detected, NO wizard, NO IME user-session completion for the whole run.
|
||||
/// </summary>
|
||||
private static DecisionState SetupHelloNeverObservedSession(
|
||||
DecisionEngine engine,
|
||||
bool desktopArrives = true,
|
||||
bool helloPolicyEnabledRead = false,
|
||||
bool allowContinueAnyway = false)
|
||||
{
|
||||
var state = DecisionState.CreateInitial("sess-2dac8298", "tenant-047b2e1f", T0);
|
||||
state = engine.Reduce(state, MakeSignal(0, DecisionSignalKind.SessionStarted, T0)).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
5, DecisionSignalKind.EspConfigDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.SkipUserEsp] = "false",
|
||||
[SignalPayloadKeys.SkipDeviceEsp] = "false",
|
||||
[SignalPayloadKeys.EspAllowContinueAnyway] = allowContinueAnyway ? "true" : "false",
|
||||
})).NewState;
|
||||
if (helloPolicyEnabledRead)
|
||||
{
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
8, DecisionSignalKind.HelloPolicyDetected, T0.AddMinutes(1),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.HelloEnabled] = "true" })).NewState;
|
||||
}
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
10, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(2),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "DeviceSetup" })).NewState;
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
20, DecisionSignalKind.EspPhaseChanged, T0.AddMinutes(8),
|
||||
new Dictionary<string, string> { [SignalPayloadKeys.EspPhase] = "AccountSetup" })).NewState;
|
||||
// Genuine post-AccountSetup final exit — arms the esp-exit advisory window
|
||||
// (30 min) because AccountSetup was entered and no promotion arm applies.
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
30, DecisionSignalKind.EspExiting, T0.AddMinutes(10))).NewState;
|
||||
if (desktopArrives)
|
||||
{
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
40, DecisionSignalKind.DesktopArrived, T0.AddMinutes(10.5))).NewState;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Setup_ArmsEspExitAdvisoryWindow()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine);
|
||||
|
||||
Assert.NotNull(FindDeadline(state, DeadlineNames.AdvisoryCompletion));
|
||||
Assert.Null(state.HelloPolicyEnabled);
|
||||
Assert.Null(state.HelloResolvedUtc);
|
||||
Assert.Null(state.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFired_HelloNeverObserved_PromotesToAwaitingHello_NotFailed()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine);
|
||||
|
||||
var fireAt = T0.AddMinutes(40);
|
||||
var step = engine.Reduce(state, DeadlineFired(50, fireAt, DeadlineNames.AdvisoryCompletion));
|
||||
|
||||
Assert.Equal(SessionStage.AwaitingHello, step.NewState.Stage);
|
||||
Assert.Null(step.NewState.Outcome);
|
||||
Assert.Equal(
|
||||
$"DeadlineFired:{DeadlineNames.AdvisoryCompletion}:HelloNeverObservedPromote",
|
||||
step.Transition.Trigger);
|
||||
|
||||
// hello_safety armed in state + scheduled as an effect; advisory window retired.
|
||||
var safety = FindDeadline(step.NewState, DeadlineNames.HelloSafety);
|
||||
Assert.NotNull(safety);
|
||||
Assert.Null(FindDeadline(step.NewState, DeadlineNames.AdvisoryCompletion));
|
||||
var scheduleEffect = step.Effects.Single(e => e.Kind == DecisionEffectKind.ScheduleDeadline);
|
||||
Assert.Equal(DeadlineNames.HelloSafety, scheduleEffect.Deadline!.Name);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void HelloSafetyFired_AfterPromote_CompletesThroughFinalizing()
|
||||
{
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine);
|
||||
state = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion)).NewState;
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(60, T0.AddMinutes(45), DeadlineNames.HelloSafety));
|
||||
|
||||
Assert.Equal(SessionStage.Finalizing, step.NewState.Stage);
|
||||
Assert.NotNull(step.NewState.HelloResolvedUtc);
|
||||
Assert.Equal("Timeout", step.NewState.HelloOutcome!.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFired_PolicyReadEnabled_NoWizard_StillFails()
|
||||
{
|
||||
// A COMMITTED enabled policy read means Hello genuinely never finished — the
|
||||
// never-observed promote must not apply and the fallthrough failure stands.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine, helloPolicyEnabledRead: true);
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
var failEffect = step.Effects.Single(e => e.Kind == DecisionEffectKind.EmitEventTimelineEntry);
|
||||
Assert.Equal("esp_exit_without_completion_evidence", failEffect.Parameters!["reason"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFired_NoDesktop_StillFails()
|
||||
{
|
||||
// Without the DAD-validated desktop there is no user-presence evidence at all —
|
||||
// the never-observed promote must not apply.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine, desktopArrives: false);
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(40), DeadlineNames.AdvisoryCompletion));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DeadlineFired_AdvisoryAnchorVariant_HelloNeverObserved_StillFails()
|
||||
{
|
||||
// The advisory variant is anchored to a REAL ESP terminal failure — a missing
|
||||
// Hello policy read must not soften that evidence. Only the esp-exit variant
|
||||
// (normal page close) qualifies for the never-observed promote.
|
||||
var engine = new DecisionEngine();
|
||||
var state = SetupHelloNeverObservedSession(engine, allowContinueAnyway: true);
|
||||
state = engine.Reduce(state, MakeSignal(
|
||||
45, DecisionSignalKind.EspTerminalFailure, T0.AddMinutes(11),
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["failureType"] = "Provisioning_AccountSetup_Apps_Failed",
|
||||
["errorCode"] = "0x87d1041c",
|
||||
["failedSubcategory"] = "Apps",
|
||||
["category"] = "AccountSetup",
|
||||
})).NewState;
|
||||
Assert.NotNull(state.EspAdvisoryFailureRecordedUtc);
|
||||
|
||||
var step = engine.Reduce(state, DeadlineFired(50, T0.AddMinutes(41), DeadlineNames.AdvisoryCompletion));
|
||||
|
||||
Assert.Equal(SessionStage.Failed, step.NewState.Stage);
|
||||
Assert.Equal(SessionOutcome.EnrollmentFailed, step.NewState.Outcome);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -368,6 +368,63 @@ public class EnrollmentTimeoutClassifierTests
|
||||
Assert.Equal(SessionStatus.Incomplete, status);
|
||||
}
|
||||
|
||||
// -------- Misclassification audit 2026-07-16 --------
|
||||
|
||||
[Fact]
|
||||
public void ExtractRollup_agent_timeout_enrollment_failed_is_not_explicit_failure()
|
||||
{
|
||||
// The max-lifetime watchdog's enrollment_failed(failureType=agent_timeout) is "the agent
|
||||
// gave up waiting", not a failure verdict — it must not poison rule 1 (tenant a53e67ec).
|
||||
var timeout = Evt("enrollment_failed",
|
||||
data: new Dictionary<string, object> { ["failureType"] = "agent_timeout" });
|
||||
Assert.False(EnrollmentTimeoutClassifier.ExtractRollup(new[] { timeout }).HasExplicitFailure);
|
||||
|
||||
// Any OTHER failureType stays an explicit failure.
|
||||
var genuine = Evt("enrollment_failed",
|
||||
data: new Dictionary<string, object> { ["failureType"] = "esp_terminal" });
|
||||
Assert.True(EnrollmentTimeoutClassifier.ExtractRollup(new[] { genuine }).HasExplicitFailure);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_agent_timeout_failed_session_classifies_honestly()
|
||||
{
|
||||
// WG Part-2 shape (session 1506ce9f): DeviceSetup 4/4, no user phase, watchdog fired at
|
||||
// 6h — honest verdict is AwaitingUser (within grace), never Failed.
|
||||
var timeout = Evt("enrollment_failed",
|
||||
data: new Dictionary<string, object> { ["failureType"] = "agent_timeout" });
|
||||
var (status, _) = Classify(new[] { Esp(DeviceSetup44), timeout }, hoursSinceStart: 6);
|
||||
Assert.Equal(SessionStatus.AwaitingUser, status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ExtractRollup_account_0ofN_observation_records_total()
|
||||
{
|
||||
// A "0 of 5" rollup was previously dropped entirely (the strongest-observation fold only
|
||||
// kept n > 0), making the Incomplete reason read "0/0" — session 08ddbeec.
|
||||
var r = EnrollmentTimeoutClassifier.ExtractRollup(new[] { Esp(DeviceSetup44), Esp(AccountSetup05) });
|
||||
Assert.Equal(0, r.AccountSetupSucceededCount);
|
||||
Assert.Equal(5, r.AccountSetupTotal);
|
||||
Assert.False(r.AccountSetupAllSucceeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_incomplete_reason_shows_observed_account_rollup()
|
||||
{
|
||||
var (status, reason) = Classify(new[] { Esp(DeviceSetup44), Esp(AccountSetup05) }, hoursSinceStart: 80, grace: 72);
|
||||
Assert.Equal(SessionStatus.Incomplete, status);
|
||||
Assert.Contains("0/5", reason);
|
||||
Assert.DoesNotContain("0/0", reason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Classify_incomplete_reason_names_never_observed_account_rollup()
|
||||
{
|
||||
// No AccountSetup rollup at all — the reason must say so instead of a fabricated "0/0".
|
||||
var (status, reason) = Classify(new[] { Esp(DeviceSetup44) }, hoursSinceStart: 80, grace: 72);
|
||||
Assert.Equal(SessionStatus.Incomplete, status);
|
||||
Assert.Contains("Account Setup progress never observed", reason);
|
||||
}
|
||||
|
||||
// -------- ResolveGraceHours --------
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -78,4 +78,66 @@ public class IngestMaxLifetimeShutdownTests
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsMaxLifetimeAgentShutdown(null));
|
||||
}
|
||||
|
||||
// -------- IsAgentTimeoutFailure (misclassification audit 2026-07-16) --------
|
||||
// The V1-parity companion: enrollment_failed(failureType=agent_timeout) emitted by
|
||||
// LifecycleEmitters.CreateMaxLifetimeEmitter. Same semantics as the shutdown shape —
|
||||
// detected events route through the honest classifier instead of a hard Failed.
|
||||
|
||||
private static EnrollmentEvent Failure(string? failureType, string eventType = "enrollment_failed")
|
||||
{
|
||||
var evt = new EnrollmentEvent
|
||||
{
|
||||
EventType = eventType,
|
||||
Message = "Agent max lifetime expired (360 min) — enrollment did not complete in time",
|
||||
};
|
||||
if (failureType != null)
|
||||
{
|
||||
evt.Data = new Dictionary<string, object>
|
||||
{
|
||||
["failureType"] = failureType,
|
||||
["failureSource"] = "max_lifetime_timer",
|
||||
};
|
||||
}
|
||||
return evt;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AgentTimeoutFailure_IsDetected()
|
||||
{
|
||||
Assert.True(EventIngestProcessor.IsAgentTimeoutFailure(Failure("agent_timeout")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AgentTimeoutFailure_IsDetected_CaseInsensitively()
|
||||
{
|
||||
Assert.True(EventIngestProcessor.IsAgentTimeoutFailure(Failure("Agent_Timeout")));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("esp_terminal")]
|
||||
[InlineData("app_install_failed")]
|
||||
public void OtherFailureTypes_AreNotDetected(string failureType)
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(Failure(failureType)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingFailureTypeData_IsNotDetected()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(Failure(failureType: null)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void OtherEventTypes_AreNotDetected_EvenWithAgentTimeoutType()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(
|
||||
Failure("agent_timeout", eventType: "agent_shutting_down")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void NullEvent_IsNotDetected_ForAgentTimeout()
|
||||
{
|
||||
Assert.False(EventIngestProcessor.IsAgentTimeoutFailure(null));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,6 +114,8 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<List<SessionSummary>> GetUsageWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetGeoWindowSessionsAsync(DateTime startDate, DateTime endDate, string? tenantId = null) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetStalledSessionsAsync(string tenantId, DateTime cutoffTime) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Guards the registration supersede pass (misclassification audit 2026-07-16): OEM
|
||||
/// placeholder serials would cross-match unrelated devices, so they must never be treated
|
||||
/// as a device identity.
|
||||
/// </summary>
|
||||
public class SerialNumberHeuristicsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData("PF4DF1EN")]
|
||||
[InlineData("5CG5504Y5L")]
|
||||
[InlineData(" PF4DF1EN ")] // trimmed
|
||||
[InlineData("VMware-56 4d 8a")]
|
||||
public void RealSerials_AreUsable(string serial)
|
||||
{
|
||||
Assert.True(SerialNumberHeuristics.IsUsableSerialNumber(serial));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("0")]
|
||||
[InlineData("abc")] // too short to identify a device
|
||||
[InlineData("None")]
|
||||
[InlineData("UNKNOWN")]
|
||||
[InlineData("Default string")]
|
||||
[InlineData("To be filled by O.E.M.")]
|
||||
[InlineData("System Serial Number")]
|
||||
[InlineData("Not Specified")]
|
||||
[InlineData("n/a")]
|
||||
public void PlaceholderOrEmptySerials_AreNotUsable(string? serial)
|
||||
{
|
||||
Assert.False(SerialNumberHeuristics.IsUsableSerialNumber(serial));
|
||||
}
|
||||
}
|
||||
+6
@@ -72,6 +72,12 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours)
|
||||
=> _storage.GetExcessiveDataSendersAsync(tenantId, windowCutoff, maxSessionWindowHours);
|
||||
|
||||
public Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults)
|
||||
=> _storage.GetLegacyTimeoutFailedSessionsAsync(tenantId, maxResults);
|
||||
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId)
|
||||
=> _storage.GetSessionsLeanAsync(tenantId);
|
||||
|
||||
public Task<List<string>> GetAllTenantIdsAsync()
|
||||
=> _storage.GetAllTenantIdsAsync();
|
||||
|
||||
|
||||
+5
-2
@@ -36,6 +36,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
public Task<string?> FindSessionTenantIdAsync(string sessionId)
|
||||
=> _storage.FindSessionTenantIdAsync(sessionId);
|
||||
|
||||
public Task<List<SessionSummary>> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber)
|
||||
=> _storage.GetOpenSessionsForDeviceAsync(tenantId, serialNumber);
|
||||
|
||||
public Task<List<SessionSummary>> GetSessionsAsync(string tenantId, int? days = null)
|
||||
=> _storage.GetSessionsAsync(tenantId, days);
|
||||
|
||||
@@ -62,13 +65,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
bool? isUserDriven = null, DateTime? resumedAt = null,
|
||||
DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false,
|
||||
string? failureSource = null, string? adminMarkedAction = null,
|
||||
string? failureSnapshotJson = null)
|
||||
string? failureSnapshotJson = null, bool allowTerminalReclassification = false)
|
||||
{
|
||||
var transitioned = await _storage.UpdateSessionStatusAsync(tenantId, sessionId, status,
|
||||
currentPhase, failureReason, completedAt, earliestEventTimestamp,
|
||||
latestEventTimestamp, isPreProvisioned, isUserDriven, resumedAt,
|
||||
stalledAt, clearStalledAt, clearFailureReason, failureSource, adminMarkedAction,
|
||||
failureSnapshotJson);
|
||||
failureSnapshotJson, allowTerminalReclassification);
|
||||
|
||||
// Reconcile the authoritative EventCount + RebootCount whenever a session actually
|
||||
// transitions to a terminal state through ANY caller — ingest, admin mark
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
using System.Net;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
{
|
||||
/// <summary>
|
||||
/// Global-Admin trigger for the one-time retro-reconciliation of misclassified historical
|
||||
/// sessions (misclassification audit 2026-07-16). DRY-RUN BY DEFAULT — a real run requires
|
||||
/// the explicit <c>dryRun=false</c> query parameter, so the operator always sees the counts
|
||||
/// and samples before anything is written.
|
||||
/// </summary>
|
||||
public class ReclassifyLegacySessionsFunction
|
||||
{
|
||||
private readonly ILogger<ReclassifyLegacySessionsFunction> _logger;
|
||||
private readonly LegacyReclassificationService _service;
|
||||
|
||||
private const int DefaultMaxSessions = 200;
|
||||
private const int MaxMaxSessions = 2000;
|
||||
|
||||
public ReclassifyLegacySessionsFunction(
|
||||
ILogger<ReclassifyLegacySessionsFunction> logger,
|
||||
LegacyReclassificationService service)
|
||||
{
|
||||
_logger = logger;
|
||||
_service = service;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// POST /api/maintenance/reclassify-legacy
|
||||
/// Query parameters:
|
||||
/// - mode: "legacy_timeouts" (default) — re-classify pre-classifier "Session timed out" Failed rows;
|
||||
/// "pending_orphans" — resolve Pending rows superseded by a later session of the same device.
|
||||
/// - dryRun: "false" to actually write; anything else (or absent) reports only.
|
||||
/// - tenantId: optional single-tenant scope (GUID); absent = all tenants.
|
||||
/// - maxSessions: per-invocation examination cap (default 200, max 2000). Re-run to continue.
|
||||
/// </summary>
|
||||
[Function("ReclassifyLegacySessions")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "maintenance/reclassify-legacy")] HttpRequestData req,
|
||||
FunctionContext context)
|
||||
{
|
||||
// Authentication + GlobalAdminOnly authorization enforced by PolicyEnforcementMiddleware
|
||||
var userEmail = TenantHelper.GetUserIdentifier(req);
|
||||
|
||||
var mode = (req.Query["mode"] ?? LegacyReclassificationService.ModeLegacyTimeouts).Trim().ToLowerInvariant();
|
||||
var dryRun = !string.Equals(req.Query["dryRun"], "false", StringComparison.OrdinalIgnoreCase);
|
||||
var tenantIdScope = req.Query["tenantId"];
|
||||
|
||||
if (!string.IsNullOrEmpty(tenantIdScope) && !Guid.TryParse(tenantIdScope, out _))
|
||||
{
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new { error = "tenantId must be a GUID" });
|
||||
return bad;
|
||||
}
|
||||
|
||||
var maxSessions = DefaultMaxSessions;
|
||||
if (!string.IsNullOrEmpty(req.Query["maxSessions"]))
|
||||
{
|
||||
if (!int.TryParse(req.Query["maxSessions"], out maxSessions) || maxSessions < 1)
|
||||
{
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new { error = $"maxSessions must be a positive integer (max {MaxMaxSessions})" });
|
||||
return bad;
|
||||
}
|
||||
maxSessions = Math.Min(maxSessions, MaxMaxSessions);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Legacy reclassification requested by {User}: mode={Mode} dryRun={DryRun} tenantScope={TenantScope} maxSessions={MaxSessions}",
|
||||
userEmail, mode, dryRun, tenantIdScope ?? "(all)", maxSessions);
|
||||
|
||||
try
|
||||
{
|
||||
var result = mode switch
|
||||
{
|
||||
LegacyReclassificationService.ModeLegacyTimeouts =>
|
||||
await _service.ReclassifyLegacyTimeoutsAsync(tenantIdScope, dryRun, maxSessions, userEmail),
|
||||
LegacyReclassificationService.ModePendingOrphans =>
|
||||
await _service.ResolvePendingOrphansAsync(tenantIdScope, dryRun, maxSessions, userEmail),
|
||||
_ => null,
|
||||
};
|
||||
|
||||
if (result == null)
|
||||
{
|
||||
var bad = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await bad.WriteAsJsonAsync(new
|
||||
{
|
||||
error = $"Unknown mode '{mode}'. Valid: {LegacyReclassificationService.ModeLegacyTimeouts}, {LegacyReclassificationService.ModePendingOrphans}"
|
||||
});
|
||||
return bad;
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new
|
||||
{
|
||||
success = true,
|
||||
result,
|
||||
triggeredBy = userEmail,
|
||||
triggeredAt = DateTime.UtcNow,
|
||||
});
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Legacy reclassification run failed");
|
||||
var error = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await error.WriteAsJsonAsync(new { success = false, message = "Internal server error" });
|
||||
return error;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -184,6 +184,16 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
// Retrieve the stored session to include full data in SignalR message
|
||||
var session = await _sessionRepo.GetSessionAsync(registration.TenantId, registration.SessionId);
|
||||
|
||||
// Supersede pass (misclassification audit 2026-07-16): when a device registers a FRESH
|
||||
// session while older non-terminal sessions of the same device still exist (e.g.
|
||||
// WhiteGlove Part 2 running under a new session id leaves the Part-1 row Pending
|
||||
// forever, or a wiped/re-enrolled device abandons its old InProgress run), resolve
|
||||
// those predecessors as Incomplete("Superseded by ..."). One physical device can only
|
||||
// ever run one enrollment. Best-effort — never blocks the registration.
|
||||
var supersededMessages = isFreshRegistration
|
||||
? await SupersedeOrphanedPredecessorsAsync(registration)
|
||||
: Array.Empty<SignalRMessageAction>();
|
||||
|
||||
// AdminAction on re-register is only the portal button signal. SessionSummary.AdminMarkedAction
|
||||
// is set exclusively by MarkSessionSucceededFunction / MarkSessionFailedFunction. An
|
||||
// agent that restarts after its own completion must NOT receive a phantom AdminAction —
|
||||
@@ -246,9 +256,79 @@ namespace AutopilotMonitor.Functions.Functions.Sessions
|
||||
{
|
||||
HttpResponse = response,
|
||||
SignalRMessages = new[] { tenantMessage, globalAdminMessage }
|
||||
.Concat(supersededMessages)
|
||||
.ToArray()
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves older non-terminal sessions (Pending/InProgress/Stalled/AwaitingUser) of the
|
||||
/// same device to Incomplete("Superseded by session …") when a fresh registration proves
|
||||
/// the device has moved on. Returns portal-live SignalR updates for each resolved row
|
||||
/// (same "newevents" sessionUpdate shape the ingest path uses). Fail-soft: any error
|
||||
/// returns an empty array and the registration proceeds untouched.
|
||||
/// </summary>
|
||||
private async Task<SignalRMessageAction[]> SupersedeOrphanedPredecessorsAsync(SessionRegistration registration)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!Helpers.SerialNumberHeuristics.IsUsableSerialNumber(registration.SerialNumber))
|
||||
return Array.Empty<SignalRMessageAction>();
|
||||
|
||||
var openSessions = await _sessionRepo.GetOpenSessionsForDeviceAsync(
|
||||
registration.TenantId, registration.SerialNumber.Trim());
|
||||
if (openSessions.Count == 0)
|
||||
return Array.Empty<SignalRMessageAction>();
|
||||
|
||||
var messages = new List<SignalRMessageAction>();
|
||||
foreach (var stale in openSessions)
|
||||
{
|
||||
if (string.Equals(stale.SessionId, registration.SessionId, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
// Only strictly older runs — a clock-skewed or concurrent row is left alone.
|
||||
if (stale.StartedAt >= registration.StartedAt)
|
||||
continue;
|
||||
|
||||
var reason = $"Superseded by session {registration.SessionId}: the device registered a new " +
|
||||
$"enrollment session at {registration.StartedAt:yyyy-MM-dd HH:mm} UTC before this one " +
|
||||
"reached a terminal state.";
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
registration.TenantId, stale.SessionId, SessionStatus.Incomplete,
|
||||
failureReason: reason, failureSource: "superseded_by_reregistration");
|
||||
if (!transitioned)
|
||||
continue;
|
||||
|
||||
_logger.LogInformation(
|
||||
"Superseded stale {StaleStatus} session {StaleSessionId} (started {StaleStartedAt:yyyy-MM-dd}) with new session {SessionId} for device serial (tenant {TenantId})",
|
||||
stale.Status, stale.SessionId, stale.StartedAt, registration.SessionId, registration.TenantId);
|
||||
|
||||
messages.Add(new SignalRMessageAction("newevents")
|
||||
{
|
||||
GroupName = $"tenant-{registration.TenantId}",
|
||||
Arguments = new object[] { new {
|
||||
sessionId = stale.SessionId,
|
||||
tenantId = registration.TenantId,
|
||||
eventCount = 0,
|
||||
sessionUpdate = new
|
||||
{
|
||||
Status = SessionStatus.Incomplete,
|
||||
FailureReason = reason,
|
||||
}
|
||||
} }
|
||||
});
|
||||
}
|
||||
|
||||
return messages.ToArray();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Supersede pass failed for tenant {TenantId} session {SessionId}; registration proceeds",
|
||||
registration.TenantId, registration.SessionId);
|
||||
return Array.Empty<SignalRMessageAction>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fire-and-forget enrollment-started webhook. Loads tenant config, respects the
|
||||
/// opt-in <c>WebhookNotifyOnStart</c>/<c>TeamsNotifyOnStart</c> toggle, and dispatches
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
namespace AutopilotMonitor.Functions.Helpers
|
||||
{
|
||||
/// <summary>
|
||||
/// Decides whether a reported serial number identifies a real physical device. OEM
|
||||
/// placeholder strings (VM defaults, unset BIOS fields) would cross-match unrelated
|
||||
/// devices and must never drive device-identity logic such as the registration
|
||||
/// supersede pass (misclassification audit 2026-07-16).
|
||||
/// </summary>
|
||||
public static class SerialNumberHeuristics
|
||||
{
|
||||
private static readonly HashSet<string> PlaceholderSerials = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"0", "none", "unknown", "default string", "to be filled by o.e.m.",
|
||||
"system serial number", "not specified", "not applicable", "n/a", "na", "invalid",
|
||||
};
|
||||
|
||||
public static bool IsUsableSerialNumber(string? serialNumber)
|
||||
{
|
||||
var trimmed = serialNumber?.Trim();
|
||||
return !string.IsNullOrEmpty(trimmed)
|
||||
&& trimmed.Length >= 4
|
||||
&& !PlaceholderSerials.Contains(trimmed);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,6 +251,7 @@ builder.Services.AddSingleton<ImeLogPatternService>();
|
||||
builder.Services.AddHttpClient<GitHubRuleRepository>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
|
||||
builder.Services.AddSingleton<MaintenanceService>();
|
||||
builder.Services.AddSingleton<LegacyReclassificationService>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Services.Monitoring.IAzureMonitorMetricsReader,
|
||||
AutopilotMonitor.Functions.Services.Monitoring.AzureMonitorMetricsReader>();
|
||||
|
||||
@@ -396,6 +396,9 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("POST", "versions/block", EndpointPolicy.GlobalAdminOnly),
|
||||
new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly),
|
||||
new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
// One-time retro-reconcile of misclassified historical sessions (audit 2026-07-16) —
|
||||
// mutating, dry-run by default, GA only.
|
||||
new("POST", "maintenance/reclassify-legacy", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
// ── Critical-Table Backup (plan §PR1+§PR2) ─────────────────────────────
|
||||
new("POST", "global/backups/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
@@ -124,7 +124,16 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var type = evt.EventType ?? string.Empty;
|
||||
|
||||
if (Eq(type, "enrollment_failed") || Eq(type, "esp_failure"))
|
||||
hasFailure = true;
|
||||
{
|
||||
// The agent's max-lifetime watchdog emits enrollment_failed with
|
||||
// failureType=agent_timeout — that is "the agent gave up waiting", not an
|
||||
// enrollment failure verdict. Counting it as an explicit failure would send
|
||||
// every timed-out-but-actually-provisioned session through rule 1 straight to
|
||||
// Failed, defeating the honest classification this class exists for
|
||||
// (misclassification audit 2026-07-16, tenant a53e67ec cluster).
|
||||
if (!Eq(TryGetDataString(evt, "failureType") ?? string.Empty, "agent_timeout"))
|
||||
hasFailure = true;
|
||||
}
|
||||
else if (Eq(type, "enrollment_complete") || Eq(type, "whiteglove_complete"))
|
||||
hasComplete = true;
|
||||
else if (Eq(type, "agent_emergency_break"))
|
||||
@@ -189,7 +198,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
else if (Eq(cat, AccountSetup))
|
||||
{
|
||||
// Keep the strongest AccountSetup observation (highest completion count).
|
||||
// A "0 of M" observation must still record the total — dropping it entirely
|
||||
// made the Incomplete reason read "0/0" although a rollup WAS observed
|
||||
// (misclassification audit 2026-07-16, session 08ddbeec).
|
||||
if (n > acctBestN) { acctBestN = n; acctBestM = total; }
|
||||
else if (acctBestM == 0 && total > 0) { acctBestM = total; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,9 +310,13 @@ namespace AutopilotMonitor.Functions.Services
|
||||
$"Device Setup completed; awaiting user / Account Setup phase — agent silent, within {graceHours}h grace{acct}");
|
||||
}
|
||||
|
||||
// "0/0" would suggest a parsed rollup that never existed — when no Account Setup
|
||||
// rollup was ever observed, say so instead (misclassification audit 2026-07-16).
|
||||
var acctDetail = rollup.AccountSetupTotal > 0
|
||||
? $"last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal}"
|
||||
: "Account Setup progress never observed";
|
||||
return (SessionStatus.Incomplete,
|
||||
$"No completion signal within {graceHours}h grace after Device Setup completed " +
|
||||
$"(last Account Setup {rollup.AccountSetupSucceededCount}/{rollup.AccountSetupTotal})");
|
||||
$"No completion signal within {graceHours}h grace after Device Setup completed ({acctDetail})");
|
||||
}
|
||||
|
||||
// 6. Silent before Device Setup completed, with no explicit failure → unknown, not a failure.
|
||||
|
||||
+230
-34
@@ -62,6 +62,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
case "session_stalled":
|
||||
classification.SessionStalledEvent = evt;
|
||||
break;
|
||||
case "esp_provisioning_status":
|
||||
case "desktop_arrived":
|
||||
case "hello_provisioning_completed":
|
||||
case "hello_skipped":
|
||||
// Completion-evidence kinds the EnrollmentTimeoutClassifier feeds on —
|
||||
// gate for the late-telemetry reconcile of Incomplete/AwaitingUser sessions.
|
||||
classification.HasCompletionEvidenceCandidate = true;
|
||||
break;
|
||||
case "agent_shutting_down":
|
||||
if (IsMaxLifetimeAgentShutdown(evt))
|
||||
classification.AgentMaxLifetimeShutdownEvent = evt;
|
||||
@@ -111,6 +119,176 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return string.Equals(reason, "max_lifetime", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True for an <c>enrollment_failed</c> event whose <c>Data.failureType</c> is
|
||||
/// <c>agent_timeout</c> — the V1-parity companion of the max-lifetime watchdog shutdown
|
||||
/// (LifecycleEmitters.CreateMaxLifetimeEmitter). Semantically identical to
|
||||
/// <see cref="IsMaxLifetimeAgentShutdown"/>: "the agent gave up waiting", not a failure
|
||||
/// verdict, so it must be classified honestly instead of hard-failed.
|
||||
/// </summary>
|
||||
internal static bool IsAgentTimeoutFailure(EnrollmentEvent? evt)
|
||||
{
|
||||
if (evt == null || !string.Equals(evt.EventType, "enrollment_failed", StringComparison.Ordinal))
|
||||
return false;
|
||||
var failureType = evt.Data != null && evt.Data.ContainsKey("failureType")
|
||||
? evt.Data["failureType"]?.ToString()
|
||||
: null;
|
||||
return string.Equals(failureType, "agent_timeout", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Shared verdict path for both max-lifetime shapes (enrollment_failed/agent_timeout and
|
||||
/// agent_shutting_down/max_lifetime): classify the session honestly via
|
||||
/// <see cref="EnrollmentTimeoutClassifier"/> instead of hard-failing it. The watchdog
|
||||
/// firing only proves the agent stopped waiting — the enrollment itself may be fully
|
||||
/// provisioned with the user simply absent (AwaitingUser), silently dead (Incomplete),
|
||||
/// or even provably finished (Succeeded reconcile). Mirrors the maintenance sweep's
|
||||
/// stage-2 classification (misclassification audit 2026-07-16).
|
||||
/// Returns (transitioned, failureReason) where failureReason is only set for the
|
||||
/// terminal non-success outcomes so downstream failure notifications stay accurate.
|
||||
/// </summary>
|
||||
private async Task<(bool transitioned, string? failureReason)> ApplyMaxLifetimeVerdictAsync(
|
||||
IngestEventsRequest request, string sessionPrefix, EventClassification c,
|
||||
EnrollmentEvent triggerEvent, SessionStatus? preFetchedStatus)
|
||||
{
|
||||
var session = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId);
|
||||
var currentStatus = session?.Status ?? preFetchedStatus;
|
||||
|
||||
// Pending (WhiteGlove sealed) sessions are deliberately long-lived and resume via
|
||||
// re-registration — a watchdog artifact must never terminalize them.
|
||||
if (currentStatus == SessionStatus.Pending)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"{SessionPrefix} max_lifetime verdict skipped — session is Pending (WhiteGlove)", sessionPrefix);
|
||||
return (false, null);
|
||||
}
|
||||
|
||||
// Full event read (not just this batch): the ESP rollups / desktop / Hello facts the
|
||||
// classifier needs usually predate the final watchdog batch. Rare path (once per
|
||||
// max-lifetime session), so the extra read is acceptable. Best-effort — with no
|
||||
// events the classifier degrades to rule 6 (Incomplete), still more honest than Failed.
|
||||
List<EnrollmentEvent> sessionEvents = new();
|
||||
try
|
||||
{
|
||||
sessionEvents = await _sessionRepo.GetSessionEventsAsync(
|
||||
request.TenantId, request.SessionId, maxResults: 1000);
|
||||
}
|
||||
catch (Exception readEx)
|
||||
{
|
||||
_logger.LogWarning(readEx,
|
||||
"{SessionPrefix} failed to read events for max_lifetime classification; proceeding with batch only", sessionPrefix);
|
||||
sessionEvents = new List<EnrollmentEvent>();
|
||||
}
|
||||
|
||||
int? tenantGraceHours = null, absoluteMaxHours = null;
|
||||
try
|
||||
{
|
||||
var config = await _configService.GetConfigurationAsync(request.TenantId);
|
||||
tenantGraceHours = config?.SessionGraceHours;
|
||||
absoluteMaxHours = config?.AbsoluteMaxSessionHours;
|
||||
}
|
||||
catch (Exception cfgEx)
|
||||
{
|
||||
_logger.LogWarning(cfgEx, "{SessionPrefix} failed to read tenant config for max_lifetime classification; using defaults", sessionPrefix);
|
||||
}
|
||||
var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
// WhiteGlove Part 2: measure the grace window from the resume, not the weeks-old
|
||||
// Part-1 start (same anchor the maintenance sweep uses).
|
||||
var effectiveStart = session?.ResumedAt ?? session?.StartedAt ?? triggerEvent.Timestamp;
|
||||
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents);
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session?.LastEventAt ?? c.LatestEventTimestamp);
|
||||
|
||||
// Keep the max-lifetime trigger visible: the Succeeded reasons already carry their own
|
||||
// silence-transparency clause, everything else gets the watchdog context appended.
|
||||
if (targetStatus != SessionStatus.Succeeded)
|
||||
reason += " Verdict triggered by the agent's max-lifetime watchdog shutdown.";
|
||||
|
||||
if (targetStatus == currentStatus)
|
||||
{
|
||||
_logger.LogInformation("{SessionPrefix} max_lifetime verdict {Status} equals current status — no transition", sessionPrefix, targetStatus);
|
||||
return (false, null);
|
||||
}
|
||||
|
||||
var isTerminalNonSuccess = targetStatus == SessionStatus.Failed || targetStatus == SessionStatus.Incomplete;
|
||||
|
||||
string? snapshotJson = null;
|
||||
if (isTerminalNonSuccess)
|
||||
{
|
||||
try { snapshotJson = FailureSnapshotBuilder.Build(sessionEvents, now); }
|
||||
catch (Exception snapEx)
|
||||
{
|
||||
_logger.LogWarning(snapEx, "{SessionPrefix} failed to build failure snapshot for max_lifetime verdict", sessionPrefix);
|
||||
}
|
||||
}
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, targetStatus, triggerEvent.Phase, reason,
|
||||
completedAt: isTerminalNonSuccess ? triggerEvent.Timestamp : (DateTime?)null,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureSource: isTerminalNonSuccess ? "max_lifetime_watchdog" : null,
|
||||
failureSnapshotJson: snapshotJson);
|
||||
|
||||
_logger.LogWarning("{SessionPrefix} Status: {Status} via max_lifetime honest classification - {Reason} (transitioned={Transitioned})",
|
||||
sessionPrefix, targetStatus, reason, transitioned);
|
||||
|
||||
return (transitioned, isTerminalNonSuccess ? reason : null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Re-runs the honest timeout classification for an Incomplete/AwaitingUser session after
|
||||
/// straggler telemetry arrived, applying ONLY a Succeeded verdict (heal). Everything else
|
||||
/// is a no-op: the existing verdict stands until evidence proves success. Best-effort —
|
||||
/// a failure here must never break the ingest.
|
||||
/// </summary>
|
||||
private async Task TryLateTelemetryReconcileAsync(
|
||||
IngestEventsRequest request, string sessionPrefix, EventClassification c)
|
||||
{
|
||||
try
|
||||
{
|
||||
var sessionEvents = await _sessionRepo.GetSessionEventsAsync(
|
||||
request.TenantId, request.SessionId, maxResults: 1000);
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents);
|
||||
|
||||
var session = await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId);
|
||||
if (session == null) return;
|
||||
|
||||
int? tenantGraceHours = null, absoluteMaxHours = null;
|
||||
try
|
||||
{
|
||||
var config = await _configService.GetConfigurationAsync(request.TenantId);
|
||||
tenantGraceHours = config?.SessionGraceHours;
|
||||
absoluteMaxHours = config?.AbsoluteMaxSessionHours;
|
||||
}
|
||||
catch { /* defaults below */ }
|
||||
var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt ?? c.LatestEventTimestamp);
|
||||
|
||||
if (targetStatus != SessionStatus.Succeeded)
|
||||
return;
|
||||
|
||||
var healed = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Succeeded,
|
||||
failureReason: reason,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
if (healed)
|
||||
_logger.LogInformation(
|
||||
"{SessionPrefix} Status: Succeeded (late-telemetry reconcile healed {Previous}) - {Reason}",
|
||||
sessionPrefix, session.Status, reason);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "{SessionPrefix} late-telemetry reconcile failed; existing verdict stands", sessionPrefix);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsPeriodicOrStallEvent(string? eventType) => eventType switch
|
||||
{
|
||||
"performance_snapshot" => true,
|
||||
@@ -142,15 +320,29 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
else if (c.FailureEvent != null)
|
||||
{
|
||||
failureReason = c.FailureEvent.Data?.ContainsKey("errorCode") == true
|
||||
? $"{c.FailureEvent.Message} ({c.FailureEvent.Data["errorCode"]})"
|
||||
: c.FailureEvent.Message;
|
||||
if (IsAgentTimeoutFailure(c.FailureEvent))
|
||||
{
|
||||
// enrollment_failed(failureType=agent_timeout) is the max-lifetime watchdog
|
||||
// giving up, NOT an enrollment failure verdict. Hard-failing here misdeclared
|
||||
// fully provisioned WhiteGlove Part-2 sessions whose user simply never logged
|
||||
// in (misclassification audit 2026-07-16, tenant a53e67ec: honest verdict
|
||||
// AwaitingUser). Route through the same honest classification the maintenance
|
||||
// sweep uses instead.
|
||||
(statusTransitioned, failureReason) = await ApplyMaxLifetimeVerdictAsync(
|
||||
request, sessionPrefix, c, c.FailureEvent, preFetchedStatus);
|
||||
}
|
||||
else
|
||||
{
|
||||
failureReason = c.FailureEvent.Data?.ContainsKey("errorCode") == true
|
||||
? $"{c.FailureEvent.Message} ({c.FailureEvent.Data["errorCode"]})"
|
||||
: c.FailureEvent.Message;
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.FailureEvent.Phase, failureReason,
|
||||
completedAt: c.FailureEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned);
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.FailureEvent.Phase, failureReason,
|
||||
completedAt: c.FailureEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp);
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed - {FailureReason} (transitioned={Transitioned})", sessionPrefix, failureReason, statusTransitioned);
|
||||
}
|
||||
}
|
||||
else if (c.EspFailureEvent != null)
|
||||
{
|
||||
@@ -217,32 +409,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// shutdown event is the last one the session ever sends, so without this
|
||||
// mapping the session stays InProgress forever. Any genuine terminal event
|
||||
// in the same batch wins via the else-if chain above; an already-terminal
|
||||
// session is protected by the repository's idempotency guard. Pending
|
||||
// (WhiteGlove) sessions are skipped — they are deliberately long-lived and
|
||||
// resume via re-registration.
|
||||
var currentStatus = preFetchedStatus
|
||||
?? (await _sessionRepo.GetSessionAsync(request.TenantId, request.SessionId))?.Status;
|
||||
if (currentStatus == SessionStatus.Pending)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"{SessionPrefix} max_lifetime shutdown ignored — session is Pending (WhiteGlove)", sessionPrefix);
|
||||
}
|
||||
else
|
||||
{
|
||||
var uptimeMinutes = c.AgentMaxLifetimeShutdownEvent.Data?.ContainsKey("uptimeMinutes") == true
|
||||
? c.AgentMaxLifetimeShutdownEvent.Data["uptimeMinutes"]?.ToString() : null;
|
||||
failureReason = string.IsNullOrEmpty(uptimeMinutes)
|
||||
? "Agent reached its maximum lifetime without a terminal enrollment verdict (max_lifetime watchdog shutdown)."
|
||||
: $"Agent reached its maximum lifetime ({uptimeMinutes} min) without a terminal enrollment verdict (max_lifetime watchdog shutdown).";
|
||||
|
||||
statusTransitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
request.TenantId, request.SessionId, SessionStatus.Failed, c.AgentMaxLifetimeShutdownEvent.Phase, failureReason,
|
||||
completedAt: c.AgentMaxLifetimeShutdownEvent.Timestamp,
|
||||
earliestEventTimestamp: c.EarliestEventTimestamp, latestEventTimestamp: c.LatestEventTimestamp,
|
||||
failureSource: "max_lifetime_watchdog");
|
||||
_logger.LogWarning("{SessionPrefix} Status: Failed via max_lifetime shutdown mapping (transitioned={Transitioned})",
|
||||
sessionPrefix, statusTransitioned);
|
||||
}
|
||||
// session is protected by the repository's idempotency guard. Instead of
|
||||
// hard-failing, the session is classified honestly (AwaitingUser/Incomplete/
|
||||
// Succeeded/Failed) — the watchdog giving up is not an enrollment failure
|
||||
// verdict (misclassification audit 2026-07-16). Pending (WhiteGlove) sessions
|
||||
// are skipped inside the helper — they are deliberately long-lived and resume
|
||||
// via re-registration.
|
||||
(statusTransitioned, failureReason) = await ApplyMaxLifetimeVerdictAsync(
|
||||
request, sessionPrefix, c, c.AgentMaxLifetimeShutdownEvent, preFetchedStatus);
|
||||
}
|
||||
|
||||
if (c.WhiteGloveStartedEvent != null)
|
||||
@@ -283,6 +457,19 @@ namespace AutopilotMonitor.Functions.Services
|
||||
if (healed)
|
||||
_logger.LogInformation("{SessionPrefix} Status: InProgress (healed from Stalled by new real event)", sessionPrefix);
|
||||
}
|
||||
else if (c.HasCompletionEvidenceCandidate
|
||||
&& (currentStatus == SessionStatus.Incomplete || currentStatus == SessionStatus.AwaitingUser))
|
||||
{
|
||||
// Late-telemetry reconcile (misclassification audit 2026-07-16, session
|
||||
// 357cefe7): a sweep verdict is based on the events known AT THAT TIME. When a
|
||||
// straggler upload later delivers completion evidence (ESP rollups, desktop
|
||||
// arrival, a positive Hello terminal) for an Incomplete/AwaitingUser session,
|
||||
// re-run the honest classifier and heal to Succeeded if the evidence now proves
|
||||
// it. Only the Succeeded verdict is applied — the transition guard allows
|
||||
// Incomplete→Succeeded, and re-terminalizing with a different reason is noise.
|
||||
// The batch pre-scan flag keeps this off the hot path for unrelated uploads.
|
||||
await TryLateTelemetryReconcileAsync(request, sessionPrefix, c);
|
||||
}
|
||||
}
|
||||
|
||||
return (statusTransitioned, whiteGloveStatusTransitioned, failureReason);
|
||||
@@ -326,5 +513,14 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// with an authoritative distinct count at the terminal transition.
|
||||
/// </summary>
|
||||
public int RebootCount { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True when this batch carries at least one event kind the
|
||||
/// <see cref="EnrollmentTimeoutClassifier"/> feeds on (ESP rollups, desktop arrival,
|
||||
/// positive Hello terminals). Gates the late-telemetry reconcile for
|
||||
/// Incomplete/AwaitingUser sessions so unrelated straggler uploads never trigger a
|
||||
/// full event read (misclassification audit 2026-07-16).
|
||||
/// </summary>
|
||||
public bool HasCompletionEvidenceCandidate { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
+6
-1
@@ -38,7 +38,12 @@ namespace AutopilotMonitor.Functions.Services
|
||||
? $"https://portal.autopilotmonitor.com/sessions/{request.SessionId}"
|
||||
: null;
|
||||
|
||||
if (statusTransitioned && (c.CompletionEvent != null || c.FailureEvent != null))
|
||||
// A failure alert requires an actual failure-ish verdict: an agent_timeout
|
||||
// enrollment_failed can honestly classify to AwaitingUser or even Succeeded
|
||||
// (ApplyMaxLifetimeVerdictAsync), in which case failureReason stays null and no
|
||||
// failure notification must go out for a session that did not fail.
|
||||
var failureVerdictApplies = c.FailureEvent != null && failureReason != null;
|
||||
if (statusTransitioned && (c.CompletionEvent != null || failureVerdictApplies))
|
||||
{
|
||||
var isSuccess = c.CompletionEvent != null;
|
||||
var targets = isSuccess ? successChannels : failureChannels;
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// One-time admin retro-reconciliation of misclassified historical sessions
|
||||
/// (misclassification audit 2026-07-16). Two modes, both dry-run capable:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>legacy_timeouts</b> — Failed rows carrying the pre-classifier blanket
|
||||
/// "Session timed out after ..." verdict are re-run through
|
||||
/// <see cref="EnrollmentTimeoutClassifier"/>; sessions the classifier would NOT fail
|
||||
/// graduate to their honest verdict (Incomplete, or Succeeded on completion evidence).
|
||||
/// The audit sample projected ~85–90% of these as misdeclared.</item>
|
||||
/// <item><b>pending_orphans</b> — Pending (WhiteGlove Part 1) rows whose device provably
|
||||
/// registered a LATER session (Part 2 under a fresh session id, or re-provisioning) are
|
||||
/// resolved as Incomplete("Superseded by ..."). Forward-looking coverage lives in the
|
||||
/// registration supersede pass; this mode clears the existing backlog.</item>
|
||||
/// </list>
|
||||
/// Hand-marked sessions (AdminMarkedAction) are never touched.
|
||||
/// </summary>
|
||||
public class LegacyReclassificationService
|
||||
{
|
||||
private readonly ILogger<LegacyReclassificationService> _logger;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TenantConfigurationService _configService;
|
||||
|
||||
public const string ModeLegacyTimeouts = "legacy_timeouts";
|
||||
public const string ModePendingOrphans = "pending_orphans";
|
||||
private const int MaxSamplesInResponse = 100;
|
||||
|
||||
public LegacyReclassificationService(
|
||||
ILogger<LegacyReclassificationService> logger,
|
||||
ISessionRepository sessionRepo,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TenantConfigurationService configService)
|
||||
{
|
||||
_logger = logger;
|
||||
_sessionRepo = sessionRepo;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_configService = configService;
|
||||
}
|
||||
|
||||
public sealed class ReclassificationSample
|
||||
{
|
||||
public string TenantId { get; set; } = string.Empty;
|
||||
public string SessionId { get; set; } = string.Empty;
|
||||
public string OldStatus { get; set; } = string.Empty;
|
||||
public string NewStatus { get; set; } = string.Empty;
|
||||
public string Reason { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class ReclassificationResult
|
||||
{
|
||||
public string Mode { get; set; } = string.Empty;
|
||||
public bool DryRun { get; set; }
|
||||
public int TenantsExamined { get; set; }
|
||||
public int SessionsExamined { get; set; }
|
||||
public int WouldChange { get; set; }
|
||||
public int Changed { get; set; }
|
||||
public int ToSucceeded { get; set; }
|
||||
public int ToIncomplete { get; set; }
|
||||
public int KeptFailed { get; set; }
|
||||
public int Skipped { get; set; }
|
||||
public int Errors { get; set; }
|
||||
/// <summary>True when the maxSessions cap stopped the run before the backlog was exhausted — re-run to continue.</summary>
|
||||
public bool CapReached { get; set; }
|
||||
public List<ReclassificationSample> Samples { get; } = new();
|
||||
}
|
||||
|
||||
public async Task<ReclassificationResult> ReclassifyLegacyTimeoutsAsync(
|
||||
string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy)
|
||||
{
|
||||
var result = new ReclassificationResult { Mode = ModeLegacyTimeouts, DryRun = dryRun };
|
||||
var now = DateTime.UtcNow;
|
||||
|
||||
foreach (var tenantId in await ResolveTenantsAsync(tenantIdScope))
|
||||
{
|
||||
if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; }
|
||||
result.TenantsExamined++;
|
||||
|
||||
int? tenantGraceHours = null, absoluteMaxHours = null;
|
||||
try
|
||||
{
|
||||
var config = await _configService.GetConfigurationAsync(tenantId);
|
||||
tenantGraceHours = config?.SessionGraceHours;
|
||||
absoluteMaxHours = config?.AbsoluteMaxSessionHours;
|
||||
}
|
||||
catch { /* defaults */ }
|
||||
var graceHours = EnrollmentTimeoutClassifier.ResolveGraceHours(tenantGraceHours, absoluteMaxHours);
|
||||
|
||||
var candidates = await _maintenanceRepo.GetLegacyTimeoutFailedSessionsAsync(
|
||||
tenantId, maxSessions - result.SessionsExamined + 1);
|
||||
if (candidates.Count > maxSessions - result.SessionsExamined)
|
||||
{
|
||||
result.CapReached = true;
|
||||
candidates = candidates.Take(maxSessions - result.SessionsExamined).ToList();
|
||||
}
|
||||
|
||||
foreach (var session in candidates)
|
||||
{
|
||||
result.SessionsExamined++;
|
||||
try
|
||||
{
|
||||
var events = await _sessionRepo.GetSessionEventsAsync(tenantId, session.SessionId, maxResults: 1000);
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(events);
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
var (target, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt);
|
||||
|
||||
if (target == SessionStatus.Failed)
|
||||
{
|
||||
result.KeptFailed++;
|
||||
continue;
|
||||
}
|
||||
// These sessions are weeks old — the grace window has long expired, so the
|
||||
// classifier cannot return AwaitingUser; guard anyway so a surprise
|
||||
// non-terminal verdict is skipped rather than applied.
|
||||
if (target != SessionStatus.Succeeded && target != SessionStatus.Incomplete)
|
||||
{
|
||||
result.Skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (target != SessionStatus.Succeeded)
|
||||
reason += " Retro-reclassified from the legacy blanket timeout verdict.";
|
||||
|
||||
RecordSample(result, tenantId, session, target, reason, oldStatus: "Failed");
|
||||
|
||||
if (dryRun)
|
||||
{
|
||||
result.WouldChange++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, session.SessionId, target,
|
||||
failureReason: reason,
|
||||
failureSource: target == SessionStatus.Incomplete ? "legacy_timeout_retro_reconcile" : null,
|
||||
allowTerminalReclassification: true);
|
||||
if (transitioned)
|
||||
{
|
||||
result.Changed++;
|
||||
if (target == SessionStatus.Succeeded) result.ToSucceeded++; else result.ToIncomplete++;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Skipped++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors++;
|
||||
_logger.LogWarning(ex, "Legacy timeout reclassification failed for session {SessionId} (tenant {TenantId})",
|
||||
session.SessionId, tenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await WriteAuditAsync(result, tenantIdScope, triggeredBy);
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<ReclassificationResult> ResolvePendingOrphansAsync(
|
||||
string? tenantIdScope, bool dryRun, int maxSessions, string triggeredBy)
|
||||
{
|
||||
var result = new ReclassificationResult { Mode = ModePendingOrphans, DryRun = dryRun };
|
||||
|
||||
foreach (var tenantId in await ResolveTenantsAsync(tenantIdScope))
|
||||
{
|
||||
if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; }
|
||||
result.TenantsExamined++;
|
||||
|
||||
var lean = await _maintenanceRepo.GetSessionsLeanAsync(tenantId);
|
||||
var bySerial = lean
|
||||
.Where(s => SerialNumberHeuristics.IsUsableSerialNumber(s.SerialNumber))
|
||||
.GroupBy(s => s.SerialNumber!.Trim(), StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var pending in lean.Where(s => s.Status == SessionStatus.Pending))
|
||||
{
|
||||
if (result.SessionsExamined >= maxSessions) { result.CapReached = true; break; }
|
||||
if (!SerialNumberHeuristics.IsUsableSerialNumber(pending.SerialNumber))
|
||||
continue;
|
||||
|
||||
result.SessionsExamined++;
|
||||
try
|
||||
{
|
||||
var successor = bySerial[pending.SerialNumber!.Trim()]
|
||||
.Where(s => !string.Equals(s.SessionId, pending.SessionId, StringComparison.OrdinalIgnoreCase)
|
||||
&& s.StartedAt > pending.StartedAt)
|
||||
.OrderByDescending(s => s.StartedAt)
|
||||
.FirstOrDefault();
|
||||
if (successor == null)
|
||||
{
|
||||
result.Skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var reason = $"Superseded by session {successor.SessionId}: the device registered a new " +
|
||||
$"enrollment session at {successor.StartedAt:yyyy-MM-dd HH:mm} UTC before this one " +
|
||||
"reached a terminal state.";
|
||||
RecordSample(result, tenantId, pending, SessionStatus.Incomplete, reason, oldStatus: "Pending");
|
||||
|
||||
if (dryRun)
|
||||
{
|
||||
result.WouldChange++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Pending is non-terminal — the standard transition guard covers this write.
|
||||
var transitioned = await _sessionRepo.UpdateSessionStatusAsync(
|
||||
tenantId, pending.SessionId, SessionStatus.Incomplete,
|
||||
failureReason: reason, failureSource: "superseded_by_reregistration");
|
||||
if (transitioned) { result.Changed++; result.ToIncomplete++; }
|
||||
else result.Skipped++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
result.Errors++;
|
||||
_logger.LogWarning(ex, "Pending-orphan resolution failed for session {SessionId} (tenant {TenantId})",
|
||||
pending.SessionId, tenantId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await WriteAuditAsync(result, tenantIdScope, triggeredBy);
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<List<string>> ResolveTenantsAsync(string? tenantIdScope)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(tenantIdScope))
|
||||
return new List<string> { tenantIdScope };
|
||||
return await _maintenanceRepo.GetAllTenantIdsAsync();
|
||||
}
|
||||
|
||||
private static void RecordSample(
|
||||
ReclassificationResult result, string tenantId, SessionSummary session,
|
||||
SessionStatus target, string reason, string oldStatus)
|
||||
{
|
||||
if (result.Samples.Count >= MaxSamplesInResponse) return;
|
||||
result.Samples.Add(new ReclassificationSample
|
||||
{
|
||||
TenantId = tenantId,
|
||||
SessionId = session.SessionId,
|
||||
OldStatus = oldStatus,
|
||||
NewStatus = target.ToString(),
|
||||
Reason = reason,
|
||||
});
|
||||
}
|
||||
|
||||
private async Task WriteAuditAsync(ReclassificationResult result, string? tenantIdScope, string triggeredBy)
|
||||
{
|
||||
// Dry-runs leave no audit trail — nothing changed. Real runs record the counts under
|
||||
// the scoped tenant (or the platform marker tenant scope "00000000-...": audit rows
|
||||
// require a tenant partition, so an all-tenant run logs one entry per touched tenant
|
||||
// would be noisy — a single summary entry under the scope keeps it queryable).
|
||||
if (result.DryRun || result.Changed == 0) return;
|
||||
try
|
||||
{
|
||||
await _maintenanceRepo.LogAuditEntryAsync(
|
||||
tenantIdScope ?? Guid.Empty.ToString(),
|
||||
"LegacyReclassification",
|
||||
"Session",
|
||||
$"{result.Changed} sessions",
|
||||
triggeredBy,
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
{ "Mode", result.Mode },
|
||||
{ "SessionsExamined", result.SessionsExamined.ToString() },
|
||||
{ "Changed", result.Changed.ToString() },
|
||||
{ "ToSucceeded", result.ToSucceeded.ToString() },
|
||||
{ "ToIncomplete", result.ToIncomplete.ToString() },
|
||||
{ "KeptFailed", result.KeptFailed.ToString() },
|
||||
{ "Errors", result.Errors.ToString() },
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to write audit entry for legacy reclassification run");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -256,9 +256,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Two-stage sweep for stuck sessions:
|
||||
/// 1. **Agent-silent Stalled marker** (2h fixed): Sessions still InProgress but with no events
|
||||
/// for more than 2h are marked as Stalled (non-terminal). Picks up agents that cannot emit
|
||||
/// session_stalled themselves (bluescreen, network loss, power off). Excludes WhiteGlove.
|
||||
/// 2. **Session timeout Failed** (5h default): Sessions that exceed the full SessionTimeoutHours
|
||||
/// window (InProgress or Stalled) graduate to terminal Failed state.
|
||||
/// session_stalled themselves (bluescreen, network loss, power off). Sealed WhiteGlove
|
||||
/// devices are protected by their Pending status; resumed Part-2 runs participate normally
|
||||
/// with their window anchored to ResumedAt (misclassification audit 2026-07-16).
|
||||
/// 2. **Session timeout** (5h default): Sessions that exceed the full SessionTimeoutHours
|
||||
/// window (InProgress or Stalled) are reclassified honestly via EnrollmentTimeoutClassifier.
|
||||
/// Both stages run in the same 2h maintenance pass so no new timers are introduced
|
||||
/// (preserving Container App scale-to-zero).
|
||||
/// </summary>
|
||||
@@ -341,10 +343,27 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
foreach (var session in stalledSessions)
|
||||
{
|
||||
// WhiteGlove Part 2: the sweep window is measured from the resume, not from
|
||||
// the weeks-old Part-1 StartedAt — otherwise a freshly resumed Part 2 would
|
||||
// be instantly "past every window" and terminalized while still live
|
||||
// (misclassification audit 2026-07-16). ResumedAt > StartedAt always holds
|
||||
// when set; plain non-WG sessions have no ResumedAt and keep StartedAt.
|
||||
var effectiveStart = session.ResumedAt ?? session.StartedAt;
|
||||
if (effectiveStart > cutoffTime)
|
||||
continue; // resumed run not yet past the timeout window — stage 1 owns it
|
||||
|
||||
// Silence guard: this is a *silent-session* sweep. A session whose agent
|
||||
// reported within the silence window is provably alive (long enrollments can
|
||||
// legitimately exceed SessionTimeoutHours) and must never be terminalized here
|
||||
// — without this, an actively-installing 6h enrollment could be classified
|
||||
// Incomplete mid-run purely because it outlived the 5h window.
|
||||
if (session.LastEventAt.HasValue && session.LastEventAt.Value > silenceCutoff)
|
||||
continue;
|
||||
|
||||
// Fast path: an AwaitingUser session still inside the grace window needs no
|
||||
// work and — crucially — no event read (grace is purely time-based). A late
|
||||
// completion is picked up by the ingest path, so we don't re-scan every pass.
|
||||
var elapsedHours = (now - session.StartedAt).TotalHours;
|
||||
var elapsedHours = (now - effectiveStart).TotalHours;
|
||||
if (session.Status == SessionStatus.AwaitingUser && elapsedHours < graceHours)
|
||||
continue;
|
||||
|
||||
@@ -375,7 +394,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// Device Setup → Incomplete. Never Failed without an explicit failure signal.
|
||||
var rollup = EnrollmentTimeoutClassifier.ExtractRollup(sessionEvents);
|
||||
var (targetStatus, reason) = EnrollmentTimeoutClassifier.ClassifyTimedOutSession(
|
||||
rollup, session.StartedAt, now, graceHours, session.LastEventAt);
|
||||
rollup, effectiveStart, now, graceHours, session.LastEventAt);
|
||||
|
||||
// No-op if the verdict equals the current (non-terminal) state.
|
||||
if (targetStatus == session.Status)
|
||||
|
||||
@@ -614,12 +614,15 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// EnrollmentTimeoutClassifier. AwaitingUser is included too so it keeps being
|
||||
// re-evaluated each pass and graduates to Incomplete once SessionGraceHours elapses
|
||||
// (tasks/enrollment-status-reclassification.md).
|
||||
// IsPreProvisioned ne true → WhiteGlove sessions are intentionally long-lived and
|
||||
// must never be timed out, even after months in storage.
|
||||
// WhiteGlove: sealed devices are protected by STATUS (Pending is not in this set),
|
||||
// not by IsPreProvisioned — a resumed Part-2 session is InProgress and must be
|
||||
// sweepable like any other run, otherwise it lingers as InProgress forever when the
|
||||
// agent dies mid-Part-2 (misclassification audit 2026-07-16). The caller anchors the
|
||||
// timeout window to ResumedAt for these rows so a freshly resumed Part 2 with a
|
||||
// weeks-old StartedAt is not terminalized while it is still live.
|
||||
var filter = $"PartitionKey eq '{tenantId}' " +
|
||||
$"and (Status eq 'InProgress' or Status eq 'Stalled' or Status eq 'AwaitingUser') " +
|
||||
$"and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z' " +
|
||||
$"and IsPreProvisioned ne true";
|
||||
$"and StartedAt lt datetime'{cutoffTime:yyyy-MM-ddTHH:mm:ss}Z'";
|
||||
var query = tableClient.QueryAsync<TableEntity>(filter: filter);
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
@@ -644,10 +647,15 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// themselves (bluescreen, network loss, power off).
|
||||
///
|
||||
/// Filter criteria:
|
||||
/// - Status eq 'InProgress' — do not re-mark already-Stalled sessions
|
||||
/// - Status eq 'InProgress' — do not re-mark already-Stalled sessions; sealed WhiteGlove
|
||||
/// sessions are Pending and therefore never match (the former extra
|
||||
/// "IsPreProvisioned ne true" clause also excluded resumed Part-2 runs, which then could
|
||||
/// never be marked Stalled — misclassification audit 2026-07-16)
|
||||
/// - LastEventAt < silenceCutoff — at least the configured window of agent silence
|
||||
/// - StartedAt ge hardCutoff — do not catch sessions that will be picked up by the 5h timeout sweep
|
||||
/// - IsPreProvisioned ne true — exclude sealed WhiteGlove sessions (intentionally long-lived)
|
||||
/// - StartedAt ge hardCutoff OR ResumedAt ge hardCutoff — do not catch sessions that will
|
||||
/// be picked up by the 5h timeout sweep. The ResumedAt alternative keeps a WhiteGlove
|
||||
/// Part-2 run (weeks-old StartedAt, fresh ResumedAt) in THIS stage instead of stage 2;
|
||||
/// rows without ResumedAt fail that comparison server-side and fall back to StartedAt.
|
||||
/// </summary>
|
||||
public async Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff)
|
||||
{
|
||||
@@ -662,8 +670,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
var filter = $"PartitionKey eq '{tenantId}' " +
|
||||
$"and Status eq 'InProgress' " +
|
||||
$"and LastEventAt lt datetime'{silenceCutoffStr}Z' " +
|
||||
$"and StartedAt ge datetime'{hardCutoffStr}Z' " +
|
||||
$"and IsPreProvisioned ne true";
|
||||
$"and (StartedAt ge datetime'{hardCutoffStr}Z' or ResumedAt ge datetime'{hardCutoffStr}Z')";
|
||||
|
||||
var query = tableClient.QueryAsync<TableEntity>(filter: filter);
|
||||
|
||||
@@ -682,6 +689,76 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Failed sessions carrying the pre-classifier blanket "Session timed out after ..."
|
||||
/// FailureReason — candidates for the one-time admin retro-reconcile
|
||||
/// (misclassification audit 2026-07-16). The prefix is matched server-side via an
|
||||
/// OData string range (ge prefix, lt prefix-with-last-char-incremented).
|
||||
/// </summary>
|
||||
public async Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
// Prefix range for "Session timed out after": upper bound increments the final
|
||||
// 'r' to 's'. Matches exactly the legacy sweep's reason format
|
||||
// "Session timed out after {N} hours (started at ... UTC)".
|
||||
var filter = $"PartitionKey eq '{tenantId}' " +
|
||||
$"and Status eq 'Failed' " +
|
||||
$"and FailureReason ge 'Session timed out after' " +
|
||||
$"and FailureReason lt 'Session timed out aftes'";
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(filter: filter))
|
||||
{
|
||||
sessions.Add(MapToSessionSummary(entity));
|
||||
if (sessions.Count >= maxResults)
|
||||
break;
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Failed to get legacy timeout-failed sessions for tenant {tenantId}");
|
||||
return new List<SessionSummary>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Narrow projection of every session row in the tenant partition — one scan feeding the
|
||||
/// in-memory Pending-orphan matching (misclassification audit 2026-07-16). Fields outside
|
||||
/// the projection come back as defaults; callers must not read them.
|
||||
/// </summary>
|
||||
public async Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
var select = new[]
|
||||
{
|
||||
"PartitionKey", "RowKey", "Status", "StartedAt", "ResumedAt", "LastEventAt",
|
||||
"SerialNumber", "IsPreProvisioned"
|
||||
};
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
|
||||
filter: $"PartitionKey eq '{tenantId}'", select: select))
|
||||
{
|
||||
sessions.Add(MapToSessionSummary(entity));
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Failed to get lean session projection for tenant {tenantId}");
|
||||
return new List<SessionSummary>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets sessions where the device has been actively sending data for longer than
|
||||
/// <paramref name="maxSessionWindowHours"/>.
|
||||
|
||||
@@ -1353,6 +1353,46 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Open (non-terminal) sessions of the same physical device within one tenant: Pending,
|
||||
/// InProgress, Stalled or AwaitingUser rows matching the SerialNumber. Server-side
|
||||
/// filtered partition query with a narrow projection — SerialNumber is not a key column,
|
||||
/// so this scans the tenant partition, which is acceptable for the once-per-registration
|
||||
/// supersede pass (misclassification audit 2026-07-16). Fail-soft: returns an empty list
|
||||
/// on storage errors so registration is never blocked.
|
||||
/// </summary>
|
||||
public async Task<List<SessionSummary>> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
if (string.IsNullOrWhiteSpace(serialNumber))
|
||||
return new List<SessionSummary>();
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.Sessions);
|
||||
var filter = $"PartitionKey eq '{tenantId}' " +
|
||||
$"and SerialNumber eq '{ODataSanitizer.EscapeValue(serialNumber)}' " +
|
||||
$"and (Status eq 'Pending' or Status eq 'InProgress' or Status eq 'Stalled' or Status eq 'AwaitingUser')";
|
||||
var select = new[]
|
||||
{
|
||||
"PartitionKey", "RowKey", "Status", "StartedAt", "ResumedAt", "LastEventAt",
|
||||
"SerialNumber", "DeviceName", "IsPreProvisioned", "Model"
|
||||
};
|
||||
|
||||
var sessions = new List<SessionSummary>();
|
||||
await foreach (var entity in tableClient.QueryAsync<TableEntity>(filter: filter, select: select))
|
||||
{
|
||||
sessions.Add(MapToSessionSummary(entity));
|
||||
}
|
||||
return sessions;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, $"Failed to get open sessions for device serial in tenant {tenantId}");
|
||||
return new List<SessionSummary>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the tenantId for a session by scanning SessionsIndex.
|
||||
/// Used for Global Admin cross-tenant session lookup when tenantId is unknown.
|
||||
@@ -1452,7 +1492,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
: null;
|
||||
}
|
||||
|
||||
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null)
|
||||
public async Task<bool> UpdateSessionStatusAsync(string tenantId, string sessionId, SessionStatus status, EnrollmentPhase? currentPhase = null, string? failureReason = null, DateTime? completedAt = null, DateTime? earliestEventTimestamp = null, DateTime? latestEventTimestamp = null, bool? isPreProvisioned = null, bool? isUserDriven = null, DateTime? resumedAt = null, DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false, string? failureSource = null, string? adminMarkedAction = null, string? failureSnapshotJson = null, bool allowTerminalReclassification = false)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
SecurityValidator.EnsureValidGuid(sessionId, nameof(sessionId));
|
||||
@@ -1477,7 +1517,25 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// overwrite an existing terminal. Non-terminal incoming statuses fall through to their
|
||||
// own guards below.
|
||||
var existingStatusStr = session.GetString("Status");
|
||||
if (!IsTerminalTransitionAllowed(existingStatusStr, status))
|
||||
// allowTerminalReclassification (admin retro-reconcile ONLY, misclassification
|
||||
// audit 2026-07-16): permits rewriting an existing terminal verdict — e.g. a
|
||||
// legacy pre-classifier "Session timed out" Failed graduating to Incomplete.
|
||||
// Hand-marked sessions stay untouchable (checked below), and an idempotent
|
||||
// same-status write is still refused.
|
||||
if (allowTerminalReclassification)
|
||||
{
|
||||
if (existingStatusStr == status.ToString())
|
||||
{
|
||||
_logger.LogInformation($"Session {sessionId}: reclassification to identical status '{status}' — no-op");
|
||||
return false;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(session.GetString("AdminMarkedAction")))
|
||||
{
|
||||
_logger.LogInformation($"Session {sessionId}: admin-marked terminal — reclassification refused");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (!IsTerminalTransitionAllowed(existingStatusStr, status))
|
||||
{
|
||||
_logger.LogInformation($"Session {sessionId}: transition '{existingStatusStr}' → '{status}' not allowed (terminal/reconcile guard), skipping");
|
||||
return false;
|
||||
@@ -1753,7 +1811,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
// our retries (e.g. an Incomplete verdict stomping a concurrently-written
|
||||
// Succeeded/Failed, or a second Incomplete overwriting the first).
|
||||
var freshStatusStr = freshSession.GetString("Status");
|
||||
if (!IsTerminalTransitionAllowed(freshStatusStr, status))
|
||||
// Mirror the normal path's override semantics (admin retro-reconcile):
|
||||
// same-status and admin-marked rows stay refused even under override.
|
||||
if (allowTerminalReclassification)
|
||||
{
|
||||
if (freshStatusStr == status.ToString()
|
||||
|| !string.IsNullOrEmpty(freshSession.GetString("AdminMarkedAction")))
|
||||
{
|
||||
_logger.LogInformation($"Session {sessionId}: reclassification force write refused (same status or admin-marked)");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (!IsTerminalTransitionAllowed(freshStatusStr, status))
|
||||
{
|
||||
_logger.LogInformation($"Session {sessionId}: transition '{freshStatusStr}' → '{status}' not allowed (terminal/reconcile guard), skipping force write");
|
||||
return false;
|
||||
|
||||
@@ -676,6 +676,64 @@ namespace AutopilotMonitor.DecisionCore.Engine
|
||||
return new DecisionStep(rearmedState, rearmedTransition, rearmedEffects);
|
||||
}
|
||||
|
||||
// Hello-never-observed promote (misclassification audit 2026-07-16, sessions
|
||||
// 2dac8298/7635ab18) — esp-exit variant only. When the Hello policy read never
|
||||
// committed (HelloPolicyEnabled == null: no PassportForWork value ever appeared,
|
||||
// so the tracker never emitted hello_policy_detected) AND no wizard was ever
|
||||
// seen, the hello_resolution prerequisite is STRUCTURALLY unresolvable: no
|
||||
// policy fact activates the disabled stand-in, and hello_safety was never armed
|
||||
// because the AwaitingHello promotion path was never taken. The DAD-validated
|
||||
// real-user desktop has been up for the whole 30-min window with no wizard and
|
||||
// no enforcement progress (the re-arm guard above ran first) — failing now would
|
||||
// declare a provably finished enrollment failed. Promote to AwaitingHello with
|
||||
// the safety deadline instead of completing outright: a genuinely late wizard
|
||||
// start still blocks completion (HelloWizardStarted re-arms the wait), otherwise
|
||||
// the safety timeout resolves Hello synthetically and the session completes
|
||||
// through Finalizing. Deliberately does NOT require the IME user-session gate —
|
||||
// the FP sessions never saw an IME user-session completion (that gate guards
|
||||
// completion evidence, not user presence; presence is the DAD-validated desktop).
|
||||
if (!hasAdvisoryAnchor
|
||||
&& desktopArrived
|
||||
&& state.HelloResolvedUtc == null
|
||||
&& state.HelloPolicyEnabled == null
|
||||
&& state.HelloWizardStartedUtc == null)
|
||||
{
|
||||
var neverObservedSafetyDueAt = EffectiveDeadlineBase(state, signal).Add(s_helloSafetyWindow);
|
||||
var neverObservedSafety = new ActiveDeadline(
|
||||
name: DeadlineNames.HelloSafety,
|
||||
dueAtUtc: neverObservedSafetyDueAt,
|
||||
firesSignalKind: DecisionSignalKind.DeadlineFired,
|
||||
firesPayload: new Dictionary<string, string>
|
||||
{
|
||||
[SignalPayloadKeys.Deadline] = DeadlineNames.HelloSafety,
|
||||
});
|
||||
|
||||
builder = builder
|
||||
.WithStage(SessionStage.AwaitingHello)
|
||||
.AddDeadline(neverObservedSafety);
|
||||
|
||||
var neverObservedTrigger =
|
||||
$"DeadlineFired:{DeadlineNames.AdvisoryCompletion}:HelloNeverObservedPromote";
|
||||
var neverObservedWaitingEffect = BuildCompletionWaitingEffect(
|
||||
state, builder, signal, trigger: neverObservedTrigger);
|
||||
|
||||
var neverObservedState = builder.Build();
|
||||
var neverObservedTransition = BuildTakenTransition(
|
||||
before: state,
|
||||
signal: signal,
|
||||
toStage: SessionStage.AwaitingHello,
|
||||
nextStepIndex: nextStep,
|
||||
trigger: neverObservedTrigger);
|
||||
|
||||
var neverObservedEffects = new List<DecisionEffect>(2)
|
||||
{
|
||||
new DecisionEffect(DecisionEffectKind.ScheduleDeadline, deadline: neverObservedSafety),
|
||||
};
|
||||
if (neverObservedWaitingEffect != null) neverObservedEffects.Add(neverObservedWaitingEffect);
|
||||
|
||||
return new DecisionStep(neverObservedState, neverObservedTransition, neverObservedEffects.ToArray());
|
||||
}
|
||||
|
||||
// Conjunction not met — the session is failed. The two arming variants carry
|
||||
// distinct context (see method doc): the advisory variant un-defangs the original
|
||||
// ESP failure; the esp-exit variant never had an ESP failure, so it gets its own
|
||||
|
||||
@@ -84,6 +84,22 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff);
|
||||
Task<List<SessionSummary>> GetExcessiveDataSendersAsync(string tenantId, DateTime windowCutoff, int maxSessionWindowHours);
|
||||
|
||||
// --- Legacy Reclassification (misclassification audit 2026-07-16) ---
|
||||
/// <summary>
|
||||
/// Failed sessions whose FailureReason carries the pre-classifier blanket
|
||||
/// "Session timed out after ..." verdict — candidates for the one-time admin
|
||||
/// retro-reconcile through <c>EnrollmentTimeoutClassifier</c>. Server-side prefix
|
||||
/// range filter, capped at <paramref name="maxResults"/>.
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetLegacyTimeoutFailedSessionsAsync(string tenantId, int maxResults);
|
||||
|
||||
/// <summary>
|
||||
/// Narrow projection of every session row in the tenant partition (id, status, timing,
|
||||
/// serial). Used by the Pending-orphan resolution to match superseding sessions of the
|
||||
/// same device in memory with one scan instead of one scan per Pending row.
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId);
|
||||
|
||||
// --- Tenant Discovery ---
|
||||
Task<List<string>> GetAllTenantIdsAsync();
|
||||
|
||||
|
||||
@@ -81,7 +81,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
bool? isUserDriven = null, DateTime? resumedAt = null,
|
||||
DateTime? stalledAt = null, bool clearStalledAt = false, bool clearFailureReason = false,
|
||||
string? failureSource = null, string? adminMarkedAction = null,
|
||||
string? failureSnapshotJson = null);
|
||||
string? failureSnapshotJson = null, bool allowTerminalReclassification = false);
|
||||
/// <summary>
|
||||
/// Increments per-session counters via read-modify-write. Returns the post-merge session
|
||||
/// snapshot (the RMW read with the applied increments) so hot-path callers can skip a
|
||||
@@ -140,6 +140,16 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// </summary>
|
||||
Task MarkExcessiveEventsAutoActionedAsync(string tenantId, string sessionId);
|
||||
|
||||
/// <summary>
|
||||
/// Open (non-terminal: Pending / InProgress / Stalled / AwaitingUser) sessions of the same
|
||||
/// physical device, identified by SerialNumber within the tenant partition. Narrow
|
||||
/// projection — used by the registration supersede pass to resolve orphaned predecessor
|
||||
/// sessions when a device re-registers under a new session id
|
||||
/// (misclassification audit 2026-07-16: WhiteGlove Part 2 under a fresh id left the
|
||||
/// Part-1 row Pending forever).
|
||||
/// </summary>
|
||||
Task<List<SessionSummary>> GetOpenSessionsForDeviceAsync(string tenantId, string serialNumber);
|
||||
|
||||
// --- IME Version History ---
|
||||
Task<bool> RecordImeVersionAsync(string version, string tenantId, string sessionId);
|
||||
Task<List<ImeVersionHistoryEntry>> GetImeVersionHistoryAsync();
|
||||
|
||||
@@ -102,6 +102,7 @@ function formatDuration(seconds: number): string {
|
||||
|
||||
const statusLabels: Record<number, string> = {
|
||||
0: "InProgress", 1: "Pending", 2: "Stalled", 3: "Succeeded", 4: "Failed", 5: "Unknown",
|
||||
6: "AwaitingUser", 7: "Incomplete", // appended 2026-07-08 — ordinals must match SessionStatus
|
||||
};
|
||||
|
||||
export default function SlaPage() {
|
||||
|
||||
Reference in New Issue
Block a user