using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services
{
///
/// Service for recording operational events into the OpsEvents table.
/// Provides typed helper methods for each event category so callers
/// don't need to construct OpsEventEntry manually.
/// All writes are fire-and-forget safe — failures are logged but never thrown.
///
public class OpsEventService
{
private readonly IOpsEventRepository _repository;
private readonly ILogger _logger;
private readonly OpsAlertDispatchService _alertDispatch;
public OpsEventService(IOpsEventRepository repository, ILogger logger,
OpsAlertDispatchService alertDispatch)
{
_repository = repository;
_logger = logger;
_alertDispatch = alertDispatch;
}
// ── Consent ────────────────────────────────────────────────────────────
public Task RecordConsentFlowStartedAsync(string tenantId, string userId, string redirectUri)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowStarted, OpsEventSeverity.Info,
$"Admin consent flow started by {userId}",
tenantId, userId, new { redirectUri });
public Task RecordConsentFlowSuccessAsync(string tenantId, string userId, string trigger)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowSuccess, OpsEventSeverity.Info,
$"Admin consent confirmed for {trigger} by {userId}",
tenantId, userId, new { trigger });
public Task RecordConsentFlowFailedAsync(string tenantId, string userId, string error, string errorDescription)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowFailed, OpsEventSeverity.Error,
$"Admin consent failed: {error}",
tenantId, userId, new { error, errorDescription });
public Task RecordConsentRedirectUriMismatchAsync(string tenantId, string userId, string redirectUri, string redirectPath)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentRedirectUriMismatch, OpsEventSeverity.Critical,
$"Redirect URI path '{redirectPath}' not in registered paths — consent will fail with AADSTS50011",
tenantId, userId, new { redirectUri, redirectPath });
public Task RecordAppHomingFlippedAsync(string tenantId, string userId, string oldApp, string newApp, string reason, bool forced)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.AppHomingFlipped, OpsEventSeverity.Info,
$"App-reg homing flipped {oldApp} -> {newApp} by {userId} ({reason}{(forced ? ", FORCED" : "")})",
tenantId, userId, new { oldApp, newApp, reason, forced });
public Task RecordAppHomingFlippedWithEntraRolesAsync(string tenantId, string userId, string oldApp, string newApp)
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.AppHomingFlippedWithEntraRoles, OpsEventSeverity.Warning,
$"App-reg homing flipped {oldApp} -> {newApp} for a tenant with Entra app roles — re-assign roles on the new enterprise app",
tenantId, userId, new { oldApp, newApp });
// ── Maintenance ────────────────────────────────────────────────────────
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
null, triggeredBy, new { durationMs });
public Task RecordMaintenanceFailedAsync(string error, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceFailed, OpsEventSeverity.Error,
$"Maintenance failed: {error}",
null, triggeredBy, new { error });
///
/// Early-warning signal: a maintenance run finished but took longer than the soft threshold.
/// The 2h Maintenance timer shares the host's 60min functionTimeout; a run that is climbing
/// toward that ceiling (e.g. a large first-time retention backlog) surfaces here as Warning so
/// operators are alerted before a future run is hard-aborted (which would emit no event at all).
///
public Task RecordMaintenanceLongRunningAsync(int durationMs, int thresholdMinutes, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceLongRunning, OpsEventSeverity.Warning,
$"Maintenance took {durationMs}ms (> {thresholdMinutes}min soft threshold; host aborts at 60min) — triggered by {triggeredBy}",
null, triggeredBy, new { durationMs, thresholdMinutes });
///
/// Heartbeat of the hourly SessionSweep timer (the :30 interleave of the stalled-session
/// sweep — the 2h Maintenance chain still runs the same sweep at minute 0). Emitted every tick,
/// counts included, so a dead timer is detectable and dashboards can separate the interleave
/// from full maintenance runs.
///
public Task RecordSessionSweepCompletedAsync(int stalledMarked, int timedOut, int durationMs)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionSweepCompleted, OpsEventSeverity.Info,
$"Hourly session sweep completed in {durationMs}ms — {stalledMarked} marked Stalled, {timedOut} terminalized",
null, "System.SessionSweep", new { stalledMarked, timedOut, durationMs });
public Task RecordSessionSweepFailedAsync(string error)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionSweepFailed, OpsEventSeverity.Error,
$"Hourly session sweep failed: {error}",
null, "System.SessionSweep", new { error });
public Task RecordOpsEventCleanupAsync(int deletedCount, int retentionDays)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OpsEventCleanup, OpsEventSeverity.Info,
$"Cleaned up {deletedCount} ops events older than {retentionDays} days",
null, "System.Maintenance", new { deletedCount, retentionDays });
///
/// Records the orphan-cleanup result. carries the
/// per-session breakdown (which tenant + session lost lingering events and how many),
/// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped
/// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits;
/// the full count always survives in orphanSessions and a detailsTruncated
/// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while
/// its events lingered past the 24h grace — the tenant itself usually still exists.
///
public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted,
IReadOnlyList? cleanedOrphans = null)
{
const int maxDetailRows = 50;
var orphanList = cleanedOrphans ?? Array.Empty();
var detail = orphanList
.OrderByDescending(o => o.EventCount)
.Take(maxDetailRows)
.Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount })
.ToList();
return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanEventsCleaned, OpsEventSeverity.Warning,
$"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions",
null, "System.Maintenance",
new
{
orphanSessions,
totalEventsDeleted,
orphans = detail,
detailsTruncated = orphanList.Count > maxDetailRows
});
}
// ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
// Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning,
// LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped
// (+ StrandedQueued / Poisoned from the GCs and the cascade worker). Each is also
// listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register).
///
/// Run started (timer tick or manual trigger). Emitted after the maintenance lease was
/// acquired, so a lease-skip never masquerades as an active run — the Session Cleanup
/// UI banner treats "latest Started newer than latest Completed/Failed" as run-active.
///
public Task RecordSessionDeletionMaintenanceStartedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceStarted, OpsEventSeverity.Info,
$"SessionDeletionMaintenance run started (triggered by {triggeredBy})",
null, "System.Maintenance", new { triggeredBy });
///
/// The retention fanout stopped cleanly at the run-budget deadline. Not an error: the
/// remaining backlog is picked up by the next run (12h cadence) or a manual trigger.
/// Paired with a Completed event whose details carry abortedByBudget=true.
///
public Task RecordSessionDeletionMaintenanceBudgetExceededAsync(int budgetMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceBudgetExceeded, OpsEventSeverity.Warning,
$"SessionDeletionMaintenance stopped at the {budgetMinutes}min run budget — tenants={tenantsProcessed} enqueued={sessionsEnqueued}; remaining backlog resumes on the next run",
null, "System.Maintenance", new { budgetMinutes, tenantsProcessed, sessionsEnqueued });
///
/// A run (timer or manual) was skipped because another run holds the session-deletion
/// maintenance lease. Mirrors RecordCriticalTableBackupSkippedLockedAsync.
///
public Task RecordSessionDeletionMaintenanceSkippedLockedAsync(string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceSkippedLocked, OpsEventSeverity.Info,
$"SessionDeletionMaintenance skipped — another run holds the maintenance lease (triggeredBy={triggeredBy})",
null, "System.Maintenance", new { reason = "lease held by another run", triggeredBy });
/// Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal.
public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceLongRunning, OpsEventSeverity.Warning,
$"SessionDeletionMaintenance still running after {elapsedMinutes}min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})",
null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued });
/// Watchdog: maintenance run still in flight 60 minutes after start. Error-level escalation in case the operator missed the 30min warning.
public Task RecordSessionDeletionMaintenanceLongRunningSevereAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceLongRunningSevere, OpsEventSeverity.Error,
$"SessionDeletionMaintenance has been running for {elapsedMinutes}min — Azure Functions host will abort at 60min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})",
null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued });
/// Unhandled exception path. Re-thrown after this audit so the Azure Functions runtime records the failure.
public Task RecordSessionDeletionMaintenanceFailedAsync(string exceptionType, string message, string stackPreview)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceFailed, OpsEventSeverity.Error,
$"SessionDeletionMaintenance failed: {exceptionType}: {message}",
null, "System.Maintenance", new { exceptionType, message, stackPreview });
/// Stale Queued state detected (no worker pickup) — operator must inspect the manifest + progress blobs. No auto-clear.
public Task RecordSessionDeletionStrandedQueuedAsync(string tenantId, string sessionId, DateTime queuedSince, string manifestId)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionStrandedQueued, OpsEventSeverity.Warning,
$"Session {sessionId} stuck in DeletionState=Queued since {queuedSince:o} (manifestId={manifestId})",
tenantId, "System.Maintenance", new { tenantId, sessionId, queuedSince = queuedSince.ToString("o"), manifestId });
///
/// Cascade max-dequeue exhaustion (PR-B follow-up): the worker has moved the envelope to
/// the poison queue and CAS-transitioned the Sessions row to .
/// Replaces the prior deletion_poisoned tenant audit — tenant admins see only the
/// lifecycle endpoints (deletion_started, deletion_completed, deletion_restored),
/// while operators get this OpsEvent for the Session Cleanup admin page + Telegram routing.
///
/// Codex follow-ups F4 + F2: /
/// / /
/// are populated from fields the handler writes before
/// throwing. Worker reads the progress blob in its poison path and passes whatever is
/// present (all are nullable so a worker that pre-dates the progress-schema bump still
/// emits a useful event, just without root-cause data).
///
///
/// is the verifier's observed count, not
/// the true total: CascadeVerificationService caps at
/// rows per table
/// and short-circuits after the first failing table. Operators reading this number should
/// treat it as a lower bound, especially when it equals the cap.
///
///
/// is a small (≤
/// ) preview that
/// fits under the OpsEvents table's 4096-char Details truncation. The full progress-blob
/// sample (up to
/// entries) is available via the Session Cleanup admin page's stored-manifest modal.
///
///
public Task RecordSessionDeletionPoisonedAsync(
string tenantId, string sessionId, string manifestId, string reason, string messageId, int dequeueCount,
string? failureType = null, string? failureMessage = null,
int? observedResidualCount = null, string? residualSamplePreviewJson = null)
{
var cause = !string.IsNullOrEmpty(failureType)
? $" — cause: {failureType}{(string.IsNullOrEmpty(failureMessage) ? "" : $" ({failureMessage})")}"
: string.Empty;
return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionPoisoned, OpsEventSeverity.Error,
$"Session {sessionId} cascade poisoned after {dequeueCount} attempts (manifestId={manifestId}){cause}",
tenantId, "System.Maintenance",
new
{
tenantId, sessionId, manifestId, reason, messageId, dequeueCount,
failureType, failureMessage, observedResidualCount,
residualSamplePreviewJson,
});
}
///
/// Successful end of a
/// run — records the per-block totals so dashboards can fold the cadence into the timeline.
/// PR6 follow-up F3: replaces the prior LogAuditEntryAsync(null!, ...) call, which
/// silently failed because the AuditLogs schema requires a non-null PartitionKey (tenantId).
///
public Task RecordSessionDeletionMaintenanceCompletedAsync(
bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued,
int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared,
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceCompleted, OpsEventSeverity.Info,
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}",
null, "System.Maintenance", new {
killSwitchActive, tenantsProcessed, sessionsEnqueued,
sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared,
strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget,
});
///
/// Records that the retention fanout half of a
/// run was skipped because the global kill-switch was active at entry. The three GCs
/// (manifest TTL sweep, stale-Preparing, stranded-Queued) still ran — see the totals on
/// the paired event.
///
public Task RecordSessionDeletionMaintenanceFanoutSkippedAsync(
int blobsTtlGced, int preparingRowsCleared, int strandedQueuedDetected)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceFanoutSkipped, OpsEventSeverity.Info,
$"SessionDeletionMaintenance fanout skipped (kill-switch active) — GCs ran: blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected}",
null, "System.Maintenance", new { reason = "SessionDeletionKillSwitch", blobsTtlGced, preparingRowsCleared, strandedQueuedDetected });
// ── Security ───────────────────────────────────────────────────────────
public Task RecordDeviceBlockedAsync(string tenantId, string serialNumber, string reason, string blockedBy)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.DeviceBlocked, OpsEventSeverity.Warning,
$"Device {serialNumber} blocked: {reason}",
tenantId, blockedBy, new { serialNumber, reason });
// ExcessiveDataBlocked was removed 2026-07-22 together with the time-window detector that
// raised it — it blocked on session span alone and fired on ordinary overnight enrollments.
// Automatic blocks now come only from the event-count path below.
public Task RecordVersionBlockedAsync(string pattern, string blockedBy)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.VersionBlocked, OpsEventSeverity.Warning,
$"Agent version pattern '{pattern}' blocked",
null, blockedBy, new { pattern });
///
/// SESSION-OWNER-BINDING-SHADOW: a validated agent request targeted a session that is bound
/// to a different device identity (see SessionOwnershipPolicy). Stage 1 records only —
/// the request went through. Throttled by SessionOwnerBindingObserver to one event per
/// session+outcome per hour. serialMatch=true on MismatchCert is the
/// re-enroll-without-wipe shape (same device, new certificate identity, old session id);
/// serialMatch=false is a foreign device naming somebody else's session.
///
///
/// A device/bootstrap-authenticated registration named a sessionId whose SessionTenantLookup
/// row belongs to ANOTHER tenant (first-writer-wins claim in StoreSessionAsync).
/// Session ids are random GUIDs — this is a forged registration or an agent bug, never a
/// legitimate collision. The registration was refused with 409. Recorded against the
/// requesting tenant; the owning tenant is only in the details.
///
public Task RecordSessionTenantConflictAsync(
string requestedTenantId, string sessionId, string owningTenantId, string? certificateThumbprint, string? agentVersion, string endpoint)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SessionTenantConflict, OpsEventSeverity.Warning,
$"Registration refused on {endpoint}: session {sessionId} is owned by another tenant (409)",
requestedTenantId, "System.SessionTenantLookup",
new { sessionId, owningTenantId, certificateThumbprint, agentVersion, endpoint });
public Task RecordSessionOwnerMismatchAsync(
string tenantId, string sessionId, string outcome, string callerKind, string ownerKind,
bool serialMatch, string? agentVersion, string endpoint)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SessionOwnerMismatch, OpsEventSeverity.Warning,
$"Session owner mismatch ({outcome}) on {endpoint}: {callerKind} caller vs {ownerKind}-owned session {sessionId} (serialMatch={serialMatch.ToString().ToLowerInvariant()}, shadow - allowed)",
tenantId, "System.SessionOwnerBinding",
new { sessionId, outcome, callerKind, ownerKind, serialMatch, agentVersion, endpoint, enforced = false });
///
/// Assume-breach signal for the Global-Admin-only surface: an authenticated caller was refused
/// (403) on a GlobalAdminOnly route. Fired from the policy middleware's deny path via
/// (throttled 1h per caller+path, per instance). The MCP
/// routes a non-GA attempt at a GA-only tool through the same path (access-probe route), so
/// one event type covers both the direct-HTTP probe and the tool-catalog probe.
/// Critical unless the caller is a Global Reader (a known operator, not an intruder) — the
/// severity split lets one rule push Critical to Telegram while Reader slips stay on the page.
/// Recorded against the caller's HOME tenant (JWT tid); the identity is in the details.
///
public Task RecordPrivilegedRouteDeniedAsync(PrivilegedDenial d)
{
var isReader = d.CallerRole == Constants.GlobalRoles.GlobalReader;
var severity = isReader ? OpsEventSeverity.Warning : OpsEventSeverity.Critical;
var via = d.ClientSource ?? "direct";
var tool = string.IsNullOrEmpty(d.McpToolName) ? string.Empty : $", tool {d.McpToolName}";
return WriteAsync(OpsEventCategory.Security, OpsEventTypes.PrivilegedRouteDenied, severity,
$"Privileged route denied: {d.Method} {d.Path} for {d.Upn ?? "(no upn)"} (role={d.CallerRole}, via {via}{tool})",
d.TenantId, d.Upn ?? "System.PolicyEnforcement",
new
{
method = d.Method, path = d.Path, statusCode = d.StatusCode, reason = d.Reason, policy = d.Policy,
upn = d.Upn, oid = d.ObjectId, tid = d.TenantId, callerRole = d.CallerRole,
clientSource = d.ClientSource, mcpToolName = d.McpToolName, correlationId = d.CorrelationId,
});
}
///
/// Fired by when a Kill signal was actually SERVED to
/// an agent (as opposed to DeviceBlocked/VersionBlocked, which fire when the admin
/// creates the rule). This is the delivery confirmation operators wire Telegram rules
/// on — throttled at the evaluator (24h per tenant+serial+pattern) because a kill-blind
/// old agent keeps hitting the endpoint every few seconds. Details carry
/// serialNumber so the Ops Events detail modal's Block/Kill shortcuts deep-link.
///
public Task RecordKillSignalDeliveredAsync(
string tenantId, string? serialNumber, string? agentVersion, string? matchedPattern,
string trigger, string channel)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.KillSignalDelivered, OpsEventSeverity.Warning,
trigger == "version"
? $"Kill signal delivered via {channel} to agent {agentVersion ?? "?"} on device {serialNumber ?? "?"} (pattern: {matchedPattern})"
: $"Kill signal delivered via {channel} to device {serialNumber ?? "?"}",
tenantId, "System.KillSwitch",
new { serialNumber, agentVersion, matchedPattern, trigger, channel });
public Task RecordEmbeddedCertExpiringSoonAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpiringSoon, OpsEventSeverity.Warning,
$"Newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - source a successor PEM and embed it before {notAfterUtc:yyyy-MM-dd}",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertExpiringUrgentAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpiringUrgent, OpsEventSeverity.Error,
$"URGENT: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is in the bundle - agent mTLS will break without rotation",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertExpiredAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpired, OpsEventSeverity.Critical,
daysUntilExpiry < 0
? $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' EXPIRED {-daysUntilExpiry}d ago ({notAfterUtc:u}) and no successor is embedded - agent mTLS validation broken"
: $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is embedded",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertBundleEmptyAsync()
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertBundleEmpty, OpsEventSeverity.Critical,
"No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients",
null, "System.Maintenance", new { });
public Task RecordSignalRConnectionsHighAsync(int observed, int limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRConnectionsHigh, OpsEventSeverity.Warning,
$"SignalR concurrent connections at {percent}% of plan limit ({observed}/{limit}) - watch for 429s; consider adding units before saturation",
null, "System.Monitoring",
new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
public Task RecordSignalRConnectionsCriticalAsync(int observed, int limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRConnectionsCritical, OpsEventSeverity.Error,
$"CRITICAL: SignalR concurrent connections at {percent}% of plan limit ({observed}/{limit}) - new client connections will be 429'd at 100%; add units now",
null, "System.Monitoring",
new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
public Task RecordSignalRMessagesHighAsync(long observed, long limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRMessagesHigh, OpsEventSeverity.Warning,
$"SignalR daily message count at {percent}% of included plan quota ({observed}/{limit}) - resets at 00:00 UTC; overage is billed per extra million messages",
null, "System.Monitoring",
new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
public Task RecordSignalRMessagesCriticalAsync(long observed, long limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRMessagesCritical, OpsEventSeverity.Error,
$"CRITICAL: SignalR daily message count at {percent}% of included plan quota ({observed}/{limit}) - overage beyond 100% is billed per extra million messages; review traffic or add units",
null, "System.Monitoring",
new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
public Task RecordPoisonQueueBacklogHighAsync(string queueName, long count, int threshold)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.PoisonQueueBacklogHigh, OpsEventSeverity.Warning,
$"Poison queue '{queueName}' backlog at {count} message(s) (threshold: {threshold}) — async worker handler failing repeatedly; inspect dead-letter contents",
null, "System.Maintenance",
new { queueName, count, threshold });
public Task RecordPoisonQueueBacklogCriticalAsync(string queueName, long count, int threshold)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.PoisonQueueBacklogCritical, OpsEventSeverity.Error,
$"CRITICAL: poison queue '{queueName}' backlog at {count} messages (threshold: {threshold}) — sustained handler failure, downstream work is silently dropping",
null, "System.Maintenance",
new { queueName, count, threshold });
// ── Critical-Table Backup ─────────────────────────────────────────────
/// Backup run finished with all tables successfully captured. Info-level — visible in the timeline, not alertable by default.
public Task RecordCriticalTableBackupCompletedAsync(string backupId, int tableCount, int durationMs, string container, string manifestBlobName, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupCompleted, OpsEventSeverity.Info,
$"Critical-table backup {backupId} completed: {tableCount} tables, {durationMs}ms (triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { backupId, tableCount, durationMs, container, manifestBlobName, triggeredBy });
/// Backup run wrote a manifest but at least one table Failed or Skipped. Warning-level — operator should inspect manifest perTableFailures.
public Task RecordCriticalTableBackupPartialAsync(string backupId, int totalTables, int failedOrSkipped, int durationMs, string container, string manifestBlobName, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupPartial, OpsEventSeverity.Warning,
$"Critical-table backup {backupId} PARTIAL: {failedOrSkipped}/{totalTables} tables failed or skipped, manifest written ({durationMs}ms, triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { backupId, totalTables, failedOrSkipped, durationMs, container, manifestBlobName, triggeredBy });
/// Backup run never produced a valid manifest (fatal exception, storage outage). Error-level. Queue-path emits this AFTER 5x retry + poison-move; timer emits immediately.
public Task RecordCriticalTableBackupFailedAsync(string? backupId, string errorMessage, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupFailed, OpsEventSeverity.Error,
$"Critical-table backup FAILED (backupId={backupId ?? "n/a"}, triggeredBy={triggeredBy}): {errorMessage}",
null, "System.Maintenance",
new { backupId, errorMessage, triggeredBy });
/// Backup or restore was skipped because the maintenance lease was already held by another job. Info-level — not a failure.
public Task RecordCriticalTableBackupSkippedLockedAsync(string reason, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupSkippedLocked, OpsEventSeverity.Info,
$"Critical-table backup skipped — {reason} (triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { reason, triggeredBy });
///
/// A GA operator restored a single row from a backup (plan §PR2). Warning-level
/// because the restore overwrote (or created) live data via ETag-CAS — operators
/// frequently want a Telegram ping for this event so a parallel admin sees the
/// audit trail in near-real-time. Payload carries the canonical
/// { container, blobName } only — no SAS URL, per plan §Medium #6.
///
public Task RecordBackupRowRestoredAsync(
string backupId, string tableName, string partitionKey, string rowKey, string actor, string outcome)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.BackupRowRestored, OpsEventSeverity.Warning,
$"Critical-table row restored: {tableName} (pk='{partitionKey}', rk='{rowKey}') from backup {backupId} by {actor} → {outcome}",
null, actor,
new
{
backupId,
tableName,
partitionKey,
rowKey,
outcome,
container = AutopilotMonitor.Shared.Constants.BlobContainers.CriticalTableBackups,
blobName = $"{backupId}/{tableName}.ndjson",
});
// ── Tenant ─────────────────────────────────────────────────────────────
///
/// Fired when a departing admin submits free-form feedback in the offboarding
/// drain-barrier banner. Information-tier — not actionable in itself, but the kind
/// of signal product wants to be notified about so feedback gets read promptly.
///
public Task RecordOffboardingFeedbackReceivedAsync(
string tenantId, string submittedBy, string? domainName, string historyRowKey)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
var details = new Dictionary
{
["domainName"] = domainName,
["historyRowKey"] = historyRowKey,
};
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.OffboardingFeedbackReceived, OpsEventSeverity.Info,
$"Offboarding feedback received from {tenantLabel}",
tenantId, submittedBy, details);
}
public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary deletedCounts, string? domainName = null)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
// Wrap deletedCounts + domainName so the OpsEvents details panel still surfaces the
// per-table counts AND the domain (needed because Table Storage is gone by emit time).
var details = new Dictionary
{
["domainName"] = domainName,
["deletedCounts"] = deletedCounts,
};
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantOffboarded, OpsEventSeverity.Warning,
$"Tenant {tenantLabel} offboarded — all data deleted",
tenantId, performedBy, details);
}
///
/// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch
/// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion,
/// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is
/// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2.
///
public Task RecordTenantOffboardingFailedAsync(
string tenantId, string performedBy, string failedPhase, string errorMessage,
int retryCount, string? domainName = null)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantOffboardingFailed, OpsEventSeverity.Error,
$"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}",
tenantId, performedBy,
new { domainName, failedPhase, errorMessage, retryCount });
}
///
/// Fired by the tenant-auto-approve queue worker when a new signup was activated
/// automatically (AutoApproveNewTenants enabled). Info-tier audit + Telegram-routable
/// signal so the operator sees which tenants entered without a manual approval click.
/// Dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES
/// (memory feedback_ops_event_types_dual_register).
///
public Task RecordTenantAutoApprovedAsync(string tenantId, string? domainName, string signupUpn)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantAutoApproved, OpsEventSeverity.Info,
$"Tenant {tenantLabel} was auto-activated after signup (signup by {signupUpn})",
tenantId, "System (auto-approve)", new { domainName, signupUpn });
}
// ── Activation welcome mail ──
// The mail is a courtesy side effect that fails soft, which used to make every failure
// mode invisible: the "no address" and "provider rejected" branches only wrote
// LogInformation, and worker application logs never reach Application Insights below
// Warning. Between 2026-08-21 and 2026-08-26 that hid a total stop of the welcome mail.
// These three types are the visible record. All dual-registered in
// OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register).
/// The provider accepted the activation welcome mail. Info-tier confirmation.
public Task RecordWelcomeEmailSentAsync(string tenantId, string? domainName, string toEmail, string addressSource)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailSent, OpsEventSeverity.Info,
$"Welcome email sent to {toEmail} for tenant {tenantLabel} (address from {addressSource})",
tenantId, "System.Activation", new { domainName, toEmail, addressSource });
}
///
/// The tenant was activated but no welcome mail went out, because no address could be
/// resolved (neither the activation-page address nor the tenant contact address).
/// Warning-tier: nothing is broken technically, but a customer was onboarded in silence.
///
public Task RecordWelcomeEmailSkippedAsync(string tenantId, string? domainName, string reason)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailSkipped, OpsEventSeverity.Warning,
$"No welcome email for tenant {tenantLabel} — {reason}",
tenantId, "System.Activation", new { domainName, reason });
}
///
/// An address was resolved and handed to the provider, which did not accept it (or the
/// send threw). Error-tier: this one IS a defect — provider key, rejection or outage.
///
public Task RecordWelcomeEmailFailedAsync(string tenantId, string? domainName, string toEmail, string reason)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailFailed, OpsEventSeverity.Error,
$"Welcome email to {toEmail} failed for tenant {tenantLabel} — {reason}",
tenantId, "System.Activation", new { domainName, toEmail, reason });
}
// ── Tenant trial lifecycle (informational — enforcement is read-time) ──
// Both types are dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES
// (memory feedback_ops_event_types_dual_register). Dispatched by TrialExpirySweepFunction.
///
/// A Pro trial was started — the conversion moment. Fired from BOTH plan write paths:
/// the tenant-admin self-service POST and a GA grant/extension via PATCH plan
/// ( tells them apart). Info-tier, but the event a
/// sales/support channel is typically bound to, so the payload carries who to contact
/// (address + company name) rather than only the tenant GUID.
///
public Task RecordTenantTrialStartedAsync(
string tenantId, string? domainName, string? contactEmail, string? companyName,
DateTime? trialStartedUtc, DateTime? trialExpiresUtc, string grantedBy, bool selfService)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
var expiryNote = trialExpiresUtc is DateTime expiry
? $"until {expiry:yyyy-MM-dd HH:mm}Z"
: "with no end date set";
var origin = selfService ? "self-service" : "granted by an operator";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialStarted, OpsEventSeverity.Info,
$"Pro trial started for tenant {tenantLabel} {expiryNote} ({origin}, by {grantedBy})",
tenantId, grantedBy,
new { domainName, contactEmail, companyName, trialStartedUtc, trialExpiresUtc, grantedBy, selfService });
}
/// Heads-up: a Pro trial ends within the next few days. Info-tier visibility signal.
public Task RecordTenantTrialExpiringAsync(string tenantId, string? domainName, DateTime trialExpiresUtc, int daysLeft)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialExpiring, OpsEventSeverity.Info,
$"Pro trial for tenant {tenantLabel} expires in {daysLeft} day(s) ({trialExpiresUtc:yyyy-MM-dd HH:mm}Z)",
tenantId, "System.TrialSweep", new { domainName, trialExpiresUtc, daysLeft });
}
///
/// A Pro trial expired within the last sweep window — the tenant silently degraded
/// to Community at read time (retention cap, rate limits, MSP delegation, MCP plan).
/// Warning-tier: a conversion moment an operator likely wants a Telegram ping for.
///
public Task RecordTenantTrialExpiredAsync(string tenantId, string? domainName, DateTime trialExpiredUtc)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialExpired, OpsEventSeverity.Warning,
$"Pro trial for tenant {tenantLabel} expired ({trialExpiredUtc:yyyy-MM-dd HH:mm}Z) — tenant is now Community",
tenantId, "System.TrialSweep", new { domainName, trialExpiredUtc });
}
// ── Plan downgrade / retention grace (informational — enforcement is read-time) ──
// All three types are dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES
// (memory feedback_ops_event_types_dual_register). Downgraded is dispatched by
// PlanManagementFunction, the grace events by TrialExpirySweepFunction.
///
/// A GA plan mutation dropped the tenant's EFFECTIVE edition Pro → Community. The
/// retention downgrade grace period starts now; data older than the Community cap is
/// deleted only after it ends. Warning-tier: the business moment (e.g. non-payment)
/// an operator wants a ping for.
///
public Task RecordTenantPlanDowngradedAsync(
string tenantId, string? domainName, string caller, DateTime? retentionGraceEndsUtc, int storedRetentionDays)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
var graceNote = retentionGraceEndsUtc is DateTime graceEnd
? $"retention grace until {graceEnd:yyyy-MM-dd HH:mm}Z"
: "no retention grace applicable";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantPlanDowngraded, OpsEventSeverity.Warning,
$"Tenant {tenantLabel} was downgraded Pro → Community by {caller} — {graceNote} (stored retention {storedRetentionDays}d)",
tenantId, caller, new { domainName, retentionGraceEndsUtc, storedRetentionDays });
}
///
/// Heads-up: a downgraded tenant's retention grace ends within the next few days —
/// after that, the sweep enforces the Community cap and deletes the older data.
/// Emitted only for tenants that actually have data at risk (stored retention above
/// the Community cap). Re-fired daily during the heads-up window (no dedup state).
///
public Task RecordTenantRetentionGraceExpiringAsync(
string tenantId, string? domainName, DateTime graceEndsUtc, int daysLeft, int storedRetentionDays)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantRetentionGraceExpiring, OpsEventSeverity.Warning,
$"Retention grace for downgraded tenant {tenantLabel} ends in {daysLeft} day(s) ({graceEndsUtc:yyyy-MM-dd HH:mm}Z) — " +
$"data older than the Community cap will then be deleted (stored retention {storedRetentionDays}d)",
tenantId, "System.TrialSweep", new { domainName, graceEndsUtc, daysLeft, storedRetentionDays });
}
///
/// A downgraded tenant's retention grace ended within the last sweep window — from now
/// on the retention sweep enforces the Community cap and starts deleting the older data.
/// Last call for a GA intervention (re-upgrade / retention decision).
///
public Task RecordTenantRetentionGraceEndedAsync(
string tenantId, string? domainName, DateTime graceEndedUtc, int storedRetentionDays)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantRetentionGraceEnded, OpsEventSeverity.Warning,
$"Retention grace for downgraded tenant {tenantLabel} ended ({graceEndedUtc:yyyy-MM-dd HH:mm}Z) — " +
$"the Community retention cap is now enforced and older data will be deleted (stored retention {storedRetentionDays}d)",
tenantId, "System.TrialSweep", new { domainName, graceEndedUtc, storedRetentionDays });
}
///
/// F3 regression radar (insights spec §F3): an analyze rule's 7-day hit rate rose ≥2×
/// over its 28-day baseline with disjoint Wilson intervals. Fired ONCE per episode (the
/// notification-tracker row is the dedup); the message carries the full numbers so the
/// event is verifiable without a portal round-trip. Dimension wording is correlation
/// only — never causal. Dual-registered in the web OPS_EVENT_TYPES catalog
/// (memory: feedback_ops_event_types_dual_register).
///
public Task RecordRuleFrequencyRegressionAsync(
string tenantId, string ruleId, string ruleTitle,
int windowFireCount, int windowSessionCount, double windowRatePct,
int baselineFireCount, int baselineSessionCount, double baselineRatePct,
double? lift, string? dimensionSummary)
=> WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.RuleFrequencyRegression, OpsEventSeverity.Warning,
$"Rule '{ruleTitle}' ({ruleId}) fired in {windowFireCount}/{windowSessionCount} sessions ({windowRatePct}%) over 7d " +
$"vs {baselineRatePct}% baseline ({baselineFireCount}/{baselineSessionCount} over 28d)" +
(lift.HasValue ? $" — lift {lift.Value}x" : " — new signal (no baseline fires)") +
(string.IsNullOrEmpty(dimensionSummary) ? string.Empty : $". {dimensionSummary}"),
tenantId, "System.Maintenance",
new
{
ruleId,
windowFireCount,
windowSessionCount,
windowRatePct,
baselineFireCount,
baselineSessionCount,
baselineRatePct,
lift,
});
///
/// App-version duration regression radar: an app's newest version installs with a median
/// duration ≥2× (and ≥5 min absolute) over the previous version's median. Fired ONCE per
/// (app, version) episode (the notification-tracker row is the dedup); the message
/// carries the full numbers so the event is verifiable without a portal round-trip.
/// Dual-registered in the web OPS_EVENT_TYPES catalog
/// (memory: feedback_ops_event_types_dual_register).
///
public Task RecordAppVersionDurationRegressionAsync(
string tenantId, string appName, string currentVersion, string previousVersion,
int currentMedianSeconds, int previousMedianSeconds,
int currentMeasuredCount, int previousMeasuredCount, double lift)
=> WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.AppVersionDurationRegression, OpsEventSeverity.Warning,
$"App '{appName}' median install duration rose from {Math.Round(previousMedianSeconds / 60.0, 1)} to " +
$"{Math.Round(currentMedianSeconds / 60.0, 1)} min after version {currentVersion} " +
$"({currentMeasuredCount} measured installs vs {previousMeasuredCount} on {previousVersion}) — lift {lift}x",
tenantId, "System.Maintenance",
new
{
appName,
currentVersion,
previousVersion,
currentMedianSeconds,
previousMedianSeconds,
currentMeasuredCount,
previousMeasuredCount,
lift,
});
///
/// Verdict-calibration drift radar (internal/docs/backend/verdict-calibration.md): a verdict
/// path's share of sessions doubled over its 28d baseline, the silence share
/// (sweep+maxlife — agent went quiet, backend had to decide) doubled, or the pure
/// fallthrough rule r6 decides ≥20 % of classifier verdicts. Operator-only diagnostic —
/// there is deliberately NO tenant bell. Fired ONCE per (kind, path, status) episode
/// (tracker-deduped). Dual-registered in the web OPS_EVENT_TYPES catalog
/// (memory: feedback_ops_event_types_dual_register).
///
public Task RecordVerdictCalibrationDriftAsync(
string tenantId, string kind, string verdictPath, string status,
int windowHitCount, int windowSessionCount, double windowRatePct,
int baselineHitCount, int baselineSessionCount, double baselineRatePct,
double? lift, string? dimensionSummary)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.VerdictCalibrationDrift, OpsEventSeverity.Warning,
$"Verdict calibration [{kind}] {verdictPath}/{status}: {windowHitCount}/{windowSessionCount} ({windowRatePct}%) over 7d " +
$"vs {baselineRatePct}% baseline ({baselineHitCount}/{baselineSessionCount} over 28d)" +
(lift.HasValue ? $" — lift {lift.Value}x" : string.Empty) +
(string.IsNullOrEmpty(dimensionSummary) ? string.Empty : $". {dimensionSummary}"),
tenantId, "System.Maintenance",
new
{
kind,
verdictPath,
status,
windowHitCount,
windowSessionCount,
windowRatePct,
baselineHitCount,
baselineSessionCount,
baselineRatePct,
lift,
});
///
/// A tenant admin turned on-demand diagnostics upload on or off — the capability behind
/// the session-detail Collect Logs button (portal Settings, the Collect Logs quick-config
/// dialog, MCP update_tenant_config, or a config revert). Fires only on the flip, not on
/// every diagnostics edit ().
/// Info-tier. The Collect Logs quick-config dialog (source portal-collect-logs)
/// gets its own type CollectLogsQuickConfigEnabled so a rule can push on exactly
/// "someone clicked Collect Logs and switched on Hosted upload" without also firing on
/// Settings-page edits; every other enable path is DiagnosticsUploadEnabled.
/// Dual-register per memory feedback_ops_event_types_dual_register.
///
public Task RecordDiagnosticsUploadConfigChangedAsync(
string tenantId, string? domainName, DiagnosticsUploadConfigChange change, string changedBy, string source)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
var quickConfig = string.Equals(source, Functions.Config.UpdateTenantConfigurationFunction.CollectLogsSource, StringComparison.Ordinal);
if (change.Enabled && quickConfig)
{
return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.CollectLogsQuickConfigEnabled, OpsEventSeverity.Info,
$"Collect Logs: {changedBy} enabled diagnostics upload for tenant {tenantLabel} via the session quick-config dialog (destination {change.Destination ?? "?"}, mode {change.Mode})",
tenantId, changedBy, new { domainName, change.Destination, change.Mode, source });
}
return change.Enabled
? WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.DiagnosticsUploadEnabled, OpsEventSeverity.Info,
$"Diagnostics upload enabled for tenant {tenantLabel} (destination {change.Destination ?? "?"}, mode {change.Mode}) via {source} by {changedBy}",
tenantId, changedBy, new { domainName, change.Destination, change.Mode, source })
: WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.DiagnosticsUploadDisabled, OpsEventSeverity.Info,
$"Diagnostics upload disabled for tenant {tenantLabel} (mode {change.Mode}) via {source} by {changedBy}",
tenantId, changedBy, new { domainName, change.Destination, change.Mode, source });
}
// ── Agent ──────────────────────────────────────────────────────────────
///
/// A portal user queued a ServerAction for a live session (Collect Logs =
/// request_diagnostics, quick-config = rotate_config, admin
/// terminate_session). Rule-engine-queued actions do NOT pass through here —
/// this is the "who uses the on-demand controls, and how often" signal.
/// Dual-register per memory feedback_ops_event_types_dual_register.
///
public Task RecordSessionActionQueuedAsync(string tenantId, string sessionId, string actionType, string? reason, string queuedBy)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.SessionActionQueued, OpsEventSeverity.Info,
$"Server action '{actionType}' queued for session {sessionId} by {queuedBy}: {reason}",
tenantId, queuedBy, new { sessionId, actionType, reason });
public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.SessionTimeouts, OpsEventSeverity.Info,
$"{sessionCount} session(s) timed out after {timeoutHours}h",
tenantId, "System.Maintenance", new { sessionCount, timeoutHours });
///
/// An agent reported its absolute session-age emergency break (48h cap) over the
/// emergency channel — it cleaned itself up and exited on a session that never reached
/// a terminal state. This is the "are we silently losing agents?" signal; emitted by
/// once per session (guarded by
/// the timeline-event idempotency check). Severity is decided by the caller from the
/// session status at break time (ReportAgentErrorFunction.ClassifyEmergencyBreak):
/// Warning when the session was still open (the break IS the verdict — Telegram-rule
/// worthy), Info when the sweep had already terminalized it and the break is a late
/// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269
/// breaks in three days were such late cleanups (kiosk devices stored for weeks); as
/// Warnings they buried the real ones. lands in the
/// details as sessionStatusAtBreak so the feed shows which case it was.
///
public Task RecordAgentEmergencyBreakAsync(
string tenantId, string sessionId, string? agentVersion, string message,
string severity, SessionStatus? statusAtBreak, bool lateCleanup)
{
var statusText = statusAtBreak?.ToString() ?? "unknown";
var context = lateCleanup
? $"late cleanup — session already {statusText} when the break arrived"
: $"session still {statusText} when the break arrived";
return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity,
$"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}",
tenantId, "System.EmergencyChannel",
new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup });
}
///
/// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend
/// advertises for its version — the binary in the field is not the published build
/// (tamper, stale blob, or a build that never came from a committed tree). Session
/// e9753578 (2026-08-20) carried exactly this report, but it landed only in App
/// Insights and was invisible to every product surface while the field damage was
/// mis-attributed to committed code for hours. Warning-tier so operators can wire a
/// Telegram rule; the message carries both hashes verbatim.
///
public Task RecordAgentBinaryIntegrityMismatchAsync(string tenantId, string? sessionId, string? agentVersion, string message)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentBinaryIntegrityMismatch, OpsEventSeverity.Warning,
$"Agent binary integrity mismatch on session {sessionId ?? "?"} (agent {agentVersion ?? "?"}): {message}",
tenantId, "System.EmergencyChannel", new { sessionId, agentVersion });
///
/// The CMTrace time-skew tripwire fired: a terminal session's IME-derived event
/// timestamps diverge from its other events by a clean 15-minute-grid multiple —
/// the signature of a timezone mis-conversion the per-line self-anchoring
/// (internal/docs/agent/cmtrace-time-resolution.md) failed to catch. Goal state: this event
/// NEVER fires; any occurrence is a real anchoring regression or a detector bug,
/// both actionable. Emitted by once per session
/// (gated on the terminal status transition). The message carries the full numbers
/// so the event is verifiable without a portal round-trip.
/// Dual-register per memory feedback_ops_event_types_dual_register.
///
public Task RecordCmTraceTimeSkewRegressionAsync(string tenantId, string sessionId, string? agentVersion, string message, object details)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.CmTraceTimeSkewRegression, OpsEventSeverity.Warning,
$"CMTrace time-skew tripwire on session {sessionId} (agent {agentVersion ?? "?"}): {message}",
tenantId, "System.Ingest", details);
public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ExcessiveSessionEvents, OpsEventSeverity.Warning,
$"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug",
tenantId, "System.Maintenance", new { sessionId, eventCount, threshold });
///
/// Fired when maintenance auto-blocks or auto-kills a device after its session crossed
/// .
/// Critical-severity so operators can wire a Telegram rule independent of the warn-tier
/// ExcessiveSessionEvents rule. Details carry the resolved serialNumber
/// so the Ops Events detail modal's Block/Kill shortcuts deep-link correctly.
///
public Task RecordExcessiveSessionEventsAutoActionedAsync(
string tenantId, string sessionId, string serialNumber, int eventCount, int threshold,
string action, int durationHours)
=> WriteAsync(OpsEventCategory.Security, OpsEventTypes.ExcessiveSessionEventsAutoActioned, OpsEventSeverity.Critical,
$"Auto-{action.ToLowerInvariant()} device {serialNumber} for session {sessionId} ({eventCount} events ≥ {threshold}, {durationHours}h)",
tenantId, "System.Maintenance",
new { sessionId, serialNumber, eventCount, threshold, action, durationHours });
public Task RecordNewImeVersionDetectedAsync(string version, string tenantId, string sessionId)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.NewImeVersionDetected, OpsEventSeverity.Warning,
$"New IME agent version detected: {version}",
tenantId, "System.Ingest", new { version, sessionId });
///
/// An IME log pattern that matches in ≥80 % of sessions on the baseline IME version has
/// matched in NONE of ≥25 reporting sessions on — Microsoft
/// probably changed the log wording. Once per version×pattern (ImePatternStats.DriftFlaggedAt).
/// Dual-registered in the web OPS_EVENT_TYPES list (OpsAlertRulesSection.tsx).
///
public Task RecordImePatternDriftSuspectedAsync(string version, string patternId, string baselineVersion, double baselineRate, int sessions)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ImePatternDriftSuspected, OpsEventSeverity.Warning,
$"IME pattern {patternId} never matched in {sessions} sessions on IME {version} (baseline {baselineVersion}: {baselineRate:P0} of sessions) — log wording may have changed",
null, "System.Ingest", new { version, patternId, baselineVersion, baselineRate, sessions });
public Task RecordBlobStorageMissingAsync(string missingItem, int statusCode)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.BlobStorageMissing, OpsEventSeverity.Critical,
$"Agent blob storage check failed: {missingItem} is missing or unreachable (HTTP {statusCode})",
null, "System.Maintenance", new { missingItem, statusCode });
public Task RecordBlobStorageUnreachableAsync(string error)
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.BlobStorageUnreachable, OpsEventSeverity.Critical,
$"Agent blob storage unreachable: {error}",
null, "System.Maintenance", new { error });
// ── SLA ────────────────────────────────────────────────────────────────
public Task RecordSlaBreachNotificationAsync(string tenantId, string breachType,
double currentRate, double targetRate, int totalSessions, int failedSessions)
{
// Duration breaches carry the P95 in minutes, every other type a percentage — same units the
// customer-facing alert (NotificationAlertBuilder.BuildSlaBreachAlert) prints.
var unit = breachType == Shared.Models.SlaBreachType.Duration ? " min" : "%";
return WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaBreachNotification, OpsEventSeverity.Warning,
$"SLA breach notification sent for tenant {tenantId}: {breachType} {currentRate:F1}{unit} (target {targetRate:F1}{unit})",
tenantId, "System.SlaEvaluation",
new { breachType, currentRate, targetRate, totalSessions, failedSessions });
}
public Task RecordSlaConsecutiveFailuresAsync(string tenantId, int count, string? lastDevice, string? lastReason)
=> WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaConsecutiveFailures, OpsEventSeverity.Error,
$"Consecutive failure alert for tenant {tenantId}: {count} failures in a row",
tenantId, "System.SlaEvaluation",
new { count, lastDevice, lastReason });
public Task RecordSlaEvaluationCompletedAsync(int tenantsEvaluated, int breachesDetected, int notificationsSent, int durationMs)
=> WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaEvaluationCompleted, OpsEventSeverity.Info,
$"SLA evaluation: {tenantsEvaluated} tenants checked, {breachesDetected} breaches, {notificationsSent} notifications sent",
null, "System.SlaEvaluation",
new { tenantsEvaluated, breachesDetected, notificationsSent, durationMs });
// ── Platform (Azure Monitor) ───────────────────────────────────────────
///
/// Records an Azure Monitor alert notification delivered via the ops alert webhook
/// (AzureMonitorAlertWebhookFunction). severity is already mapped onto the ops scale;
/// azureSeverity keeps the original Sev0–Sev4 value for the details payload.
///
public Task RecordAzureMonitorAlertAsync(string alertRule, string severity, string monitorCondition,
string? description, string? azureSeverity, string? monitoringService, string? targetResource,
double? metricValue)
{
var suffix = string.IsNullOrWhiteSpace(description) ? string.Empty : $": {description}";
return WriteAsync(OpsEventCategory.Platform, OpsEventTypes.AzureMonitorAlert, severity,
$"Azure Monitor alert '{alertRule}' {monitorCondition}{suffix}",
null, "System.AzureMonitor",
new { alertRule, monitorCondition, azureSeverity, monitoringService, targetResource, metricValue });
}
// ── Core write method ──────────────────────────────────────────────────
private async Task WriteAsync(string category, string eventType, string severity,
string message, string? tenantId, string? userId, object? details)
{
try
{
var entry = new OpsEventEntry
{
Category = category,
EventType = eventType,
Severity = severity,
TenantId = tenantId,
UserId = userId,
Message = message,
Details = details != null ? JsonSerializer.Serialize(details) : null,
Timestamp = DateTime.UtcNow,
};
await _repository.SaveOpsEventAsync(entry);
// Fire-and-forget: dispatch alerts to the channels the rules target.
// DispatchAsync has its own top-level try/catch so unobserved exceptions are safe.
// The serialized details ride along: an outbound channel that only knows category,
// event and tenant GUID forces the reader back into the portal for every alert.
_ = _alertDispatch.DispatchAsync(category, eventType, severity, message, tenantId, entry.Details);
}
catch (Exception ex)
{
// Never throw from ops event recording — it must not break the calling flow
_logger.LogWarning(ex, "Failed to record ops event {Category}/{EventType}", category, eventType);
}
}
}
}