using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; using System.Threading.Tasks; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.DataAccess; using AutopilotMonitor.Shared.Models; using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Services { /// /// Service for recording operational events into the OpsEvents table. /// Provides typed helper methods for each event category so callers /// don't need to construct OpsEventEntry manually. /// All writes are fire-and-forget safe — failures are logged but never thrown. /// public class OpsEventService { private readonly IOpsEventRepository _repository; private readonly ILogger _logger; private readonly OpsAlertDispatchService _alertDispatch; public OpsEventService(IOpsEventRepository repository, ILogger logger, OpsAlertDispatchService alertDispatch) { _repository = repository; _logger = logger; _alertDispatch = alertDispatch; } // ── Consent ──────────────────────────────────────────────────────────── public Task RecordConsentFlowStartedAsync(string tenantId, string userId, string redirectUri) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowStarted, OpsEventSeverity.Info, $"Admin consent flow started by {userId}", tenantId, userId, new { redirectUri }); public Task RecordConsentFlowSuccessAsync(string tenantId, string userId, string trigger) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowSuccess, OpsEventSeverity.Info, $"Admin consent confirmed for {trigger} by {userId}", tenantId, userId, new { trigger }); public Task RecordConsentFlowFailedAsync(string tenantId, string userId, string error, string errorDescription) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentFlowFailed, OpsEventSeverity.Error, $"Admin consent failed: {error}", tenantId, userId, new { error, errorDescription }); public Task RecordConsentRedirectUriMismatchAsync(string tenantId, string userId, string redirectUri, string redirectPath) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.ConsentRedirectUriMismatch, OpsEventSeverity.Critical, $"Redirect URI path '{redirectPath}' not in registered paths — consent will fail with AADSTS50011", tenantId, userId, new { redirectUri, redirectPath }); public Task RecordAppHomingFlippedAsync(string tenantId, string userId, string oldApp, string newApp, string reason, bool forced) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.AppHomingFlipped, OpsEventSeverity.Info, $"App-reg homing flipped {oldApp} -> {newApp} by {userId} ({reason}{(forced ? ", FORCED" : "")})", tenantId, userId, new { oldApp, newApp, reason, forced }); public Task RecordAppHomingFlippedWithEntraRolesAsync(string tenantId, string userId, string oldApp, string newApp) => WriteAsync(OpsEventCategory.Consent, OpsEventTypes.AppHomingFlippedWithEntraRoles, OpsEventSeverity.Warning, $"App-reg homing flipped {oldApp} -> {newApp} for a tenant with Entra app roles — re-assign roles on the new enterprise app", tenantId, userId, new { oldApp, newApp }); // ── Maintenance ──────────────────────────────────────────────────────── public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info, $"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})", null, triggeredBy, new { durationMs }); public Task RecordMaintenanceFailedAsync(string error, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceFailed, OpsEventSeverity.Error, $"Maintenance failed: {error}", null, triggeredBy, new { error }); /// /// Early-warning signal: a maintenance run finished but took longer than the soft threshold. /// The 2h Maintenance timer shares the host's 60min functionTimeout; a run that is climbing /// toward that ceiling (e.g. a large first-time retention backlog) surfaces here as Warning so /// operators are alerted before a future run is hard-aborted (which would emit no event at all). /// public Task RecordMaintenanceLongRunningAsync(int durationMs, int thresholdMinutes, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceLongRunning, OpsEventSeverity.Warning, $"Maintenance took {durationMs}ms (> {thresholdMinutes}min soft threshold; host aborts at 60min) — triggered by {triggeredBy}", null, triggeredBy, new { durationMs, thresholdMinutes }); /// /// Heartbeat of the hourly SessionSweep timer (the :30 interleave of the stalled-session /// sweep — the 2h Maintenance chain still runs the same sweep at minute 0). Emitted every tick, /// counts included, so a dead timer is detectable and dashboards can separate the interleave /// from full maintenance runs. /// public Task RecordSessionSweepCompletedAsync(int stalledMarked, int timedOut, int durationMs) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionSweepCompleted, OpsEventSeverity.Info, $"Hourly session sweep completed in {durationMs}ms — {stalledMarked} marked Stalled, {timedOut} terminalized", null, "System.SessionSweep", new { stalledMarked, timedOut, durationMs }); public Task RecordSessionSweepFailedAsync(string error) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionSweepFailed, OpsEventSeverity.Error, $"Hourly session sweep failed: {error}", null, "System.SessionSweep", new { error }); public Task RecordOpsEventCleanupAsync(int deletedCount, int retentionDays) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OpsEventCleanup, OpsEventSeverity.Info, $"Cleaned up {deletedCount} ops events older than {retentionDays} days", null, "System.Maintenance", new { deletedCount, retentionDays }); /// /// Records the orphan-cleanup result. carries the /// per-session breakdown (which tenant + session lost lingering events and how many), /// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped /// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits; /// the full count always survives in orphanSessions and a detailsTruncated /// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while /// its events lingered past the 24h grace — the tenant itself usually still exists. /// public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted, IReadOnlyList? cleanedOrphans = null) { const int maxDetailRows = 50; var orphanList = cleanedOrphans ?? Array.Empty(); var detail = orphanList .OrderByDescending(o => o.EventCount) .Take(maxDetailRows) .Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount }) .ToList(); return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.OrphanEventsCleaned, OpsEventSeverity.Warning, $"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions", null, "System.Maintenance", new { orphanSessions, totalEventsDeleted, orphans = detail, detailsTruncated = orphanList.Count > maxDetailRows }); } // ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ───────────────── // Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning, // LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped // (+ StrandedQueued / Poisoned from the GCs and the cascade worker). Each is also // listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register). /// /// Run started (timer tick or manual trigger). Emitted after the maintenance lease was /// acquired, so a lease-skip never masquerades as an active run — the Session Cleanup /// UI banner treats "latest Started newer than latest Completed/Failed" as run-active. /// public Task RecordSessionDeletionMaintenanceStartedAsync(string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceStarted, OpsEventSeverity.Info, $"SessionDeletionMaintenance run started (triggered by {triggeredBy})", null, "System.Maintenance", new { triggeredBy }); /// /// The retention fanout stopped cleanly at the run-budget deadline. Not an error: the /// remaining backlog is picked up by the next run (12h cadence) or a manual trigger. /// Paired with a Completed event whose details carry abortedByBudget=true. /// public Task RecordSessionDeletionMaintenanceBudgetExceededAsync(int budgetMinutes, int tenantsProcessed, int sessionsEnqueued) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceBudgetExceeded, OpsEventSeverity.Warning, $"SessionDeletionMaintenance stopped at the {budgetMinutes}min run budget — tenants={tenantsProcessed} enqueued={sessionsEnqueued}; remaining backlog resumes on the next run", null, "System.Maintenance", new { budgetMinutes, tenantsProcessed, sessionsEnqueued }); /// /// A run (timer or manual) was skipped because another run holds the session-deletion /// maintenance lease. Mirrors RecordCriticalTableBackupSkippedLockedAsync. /// public Task RecordSessionDeletionMaintenanceSkippedLockedAsync(string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceSkippedLocked, OpsEventSeverity.Info, $"SessionDeletionMaintenance skipped — another run holds the maintenance lease (triggeredBy={triggeredBy})", null, "System.Maintenance", new { reason = "lease held by another run", triggeredBy }); /// Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal. public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceLongRunning, OpsEventSeverity.Warning, $"SessionDeletionMaintenance still running after {elapsedMinutes}min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})", null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued }); /// Watchdog: maintenance run still in flight 60 minutes after start. Error-level escalation in case the operator missed the 30min warning. public Task RecordSessionDeletionMaintenanceLongRunningSevereAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceLongRunningSevere, OpsEventSeverity.Error, $"SessionDeletionMaintenance has been running for {elapsedMinutes}min — Azure Functions host will abort at 60min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})", null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued }); /// Unhandled exception path. Re-thrown after this audit so the Azure Functions runtime records the failure. public Task RecordSessionDeletionMaintenanceFailedAsync(string exceptionType, string message, string stackPreview) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceFailed, OpsEventSeverity.Error, $"SessionDeletionMaintenance failed: {exceptionType}: {message}", null, "System.Maintenance", new { exceptionType, message, stackPreview }); /// Stale Queued state detected (no worker pickup) — operator must inspect the manifest + progress blobs. No auto-clear. public Task RecordSessionDeletionStrandedQueuedAsync(string tenantId, string sessionId, DateTime queuedSince, string manifestId) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionStrandedQueued, OpsEventSeverity.Warning, $"Session {sessionId} stuck in DeletionState=Queued since {queuedSince:o} (manifestId={manifestId})", tenantId, "System.Maintenance", new { tenantId, sessionId, queuedSince = queuedSince.ToString("o"), manifestId }); /// /// Cascade max-dequeue exhaustion (PR-B follow-up): the worker has moved the envelope to /// the poison queue and CAS-transitioned the Sessions row to . /// Replaces the prior deletion_poisoned tenant audit — tenant admins see only the /// lifecycle endpoints (deletion_started, deletion_completed, deletion_restored), /// while operators get this OpsEvent for the Session Cleanup admin page + Telegram routing. /// /// Codex follow-ups F4 + F2: / /// / / /// are populated from fields the handler writes before /// throwing. Worker reads the progress blob in its poison path and passes whatever is /// present (all are nullable so a worker that pre-dates the progress-schema bump still /// emits a useful event, just without root-cause data). /// /// /// is the verifier's observed count, not /// the true total: CascadeVerificationService caps at /// rows per table /// and short-circuits after the first failing table. Operators reading this number should /// treat it as a lower bound, especially when it equals the cap. /// /// /// is a small (≤ /// ) preview that /// fits under the OpsEvents table's 4096-char Details truncation. The full progress-blob /// sample (up to /// entries) is available via the Session Cleanup admin page's stored-manifest modal. /// /// public Task RecordSessionDeletionPoisonedAsync( string tenantId, string sessionId, string manifestId, string reason, string messageId, int dequeueCount, string? failureType = null, string? failureMessage = null, int? observedResidualCount = null, string? residualSamplePreviewJson = null) { var cause = !string.IsNullOrEmpty(failureType) ? $" — cause: {failureType}{(string.IsNullOrEmpty(failureMessage) ? "" : $" ({failureMessage})")}" : string.Empty; return WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionPoisoned, OpsEventSeverity.Error, $"Session {sessionId} cascade poisoned after {dequeueCount} attempts (manifestId={manifestId}){cause}", tenantId, "System.Maintenance", new { tenantId, sessionId, manifestId, reason, messageId, dequeueCount, failureType, failureMessage, observedResidualCount, residualSamplePreviewJson, }); } /// /// Successful end of a /// run — records the per-block totals so dashboards can fold the cadence into the timeline. /// PR6 follow-up F3: replaces the prior LogAuditEntryAsync(null!, ...) call, which /// silently failed because the AuditLogs schema requires a non-null PartitionKey (tenantId). /// public Task RecordSessionDeletionMaintenanceCompletedAsync( bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued, int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared, int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch, bool abortedByBudget) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceCompleted, OpsEventSeverity.Info, $"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive} abortedByBudget={abortedByBudget}", null, "System.Maintenance", new { killSwitchActive, tenantsProcessed, sessionsEnqueued, sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared, strandedQueuedDetected, durationMs, abortedByKillSwitch, abortedByBudget, }); /// /// Records that the retention fanout half of a /// run was skipped because the global kill-switch was active at entry. The three GCs /// (manifest TTL sweep, stale-Preparing, stranded-Queued) still ran — see the totals on /// the paired event. /// public Task RecordSessionDeletionMaintenanceFanoutSkippedAsync( int blobsTtlGced, int preparingRowsCleared, int strandedQueuedDetected) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.SessionDeletionMaintenanceFanoutSkipped, OpsEventSeverity.Info, $"SessionDeletionMaintenance fanout skipped (kill-switch active) — GCs ran: blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected}", null, "System.Maintenance", new { reason = "SessionDeletionKillSwitch", blobsTtlGced, preparingRowsCleared, strandedQueuedDetected }); // ── Security ─────────────────────────────────────────────────────────── public Task RecordDeviceBlockedAsync(string tenantId, string serialNumber, string reason, string blockedBy) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.DeviceBlocked, OpsEventSeverity.Warning, $"Device {serialNumber} blocked: {reason}", tenantId, blockedBy, new { serialNumber, reason }); // ExcessiveDataBlocked was removed 2026-07-22 together with the time-window detector that // raised it — it blocked on session span alone and fired on ordinary overnight enrollments. // Automatic blocks now come only from the event-count path below. public Task RecordVersionBlockedAsync(string pattern, string blockedBy) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.VersionBlocked, OpsEventSeverity.Warning, $"Agent version pattern '{pattern}' blocked", null, blockedBy, new { pattern }); /// /// SESSION-OWNER-BINDING-SHADOW: a validated agent request targeted a session that is bound /// to a different device identity (see SessionOwnershipPolicy). Stage 1 records only — /// the request went through. Throttled by SessionOwnerBindingObserver to one event per /// session+outcome per hour. serialMatch=true on MismatchCert is the /// re-enroll-without-wipe shape (same device, new certificate identity, old session id); /// serialMatch=false is a foreign device naming somebody else's session. /// /// /// A device/bootstrap-authenticated registration named a sessionId whose SessionTenantLookup /// row belongs to ANOTHER tenant (first-writer-wins claim in StoreSessionAsync). /// Session ids are random GUIDs — this is a forged registration or an agent bug, never a /// legitimate collision. The registration was refused with 409. Recorded against the /// requesting tenant; the owning tenant is only in the details. /// public Task RecordSessionTenantConflictAsync( string requestedTenantId, string sessionId, string owningTenantId, string? certificateThumbprint, string? agentVersion, string endpoint) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SessionTenantConflict, OpsEventSeverity.Warning, $"Registration refused on {endpoint}: session {sessionId} is owned by another tenant (409)", requestedTenantId, "System.SessionTenantLookup", new { sessionId, owningTenantId, certificateThumbprint, agentVersion, endpoint }); public Task RecordSessionOwnerMismatchAsync( string tenantId, string sessionId, string outcome, string callerKind, string ownerKind, bool serialMatch, string? agentVersion, string endpoint) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SessionOwnerMismatch, OpsEventSeverity.Warning, $"Session owner mismatch ({outcome}) on {endpoint}: {callerKind} caller vs {ownerKind}-owned session {sessionId} (serialMatch={serialMatch.ToString().ToLowerInvariant()}, shadow - allowed)", tenantId, "System.SessionOwnerBinding", new { sessionId, outcome, callerKind, ownerKind, serialMatch, agentVersion, endpoint, enforced = false }); /// /// Assume-breach signal for the Global-Admin-only surface: an authenticated caller was refused /// (403) on a GlobalAdminOnly route. Fired from the policy middleware's deny path via /// (throttled 1h per caller+path, per instance). The MCP /// routes a non-GA attempt at a GA-only tool through the same path (access-probe route), so /// one event type covers both the direct-HTTP probe and the tool-catalog probe. /// Critical unless the caller is a Global Reader (a known operator, not an intruder) — the /// severity split lets one rule push Critical to Telegram while Reader slips stay on the page. /// Recorded against the caller's HOME tenant (JWT tid); the identity is in the details. /// public Task RecordPrivilegedRouteDeniedAsync(PrivilegedDenial d) { var isReader = d.CallerRole == Constants.GlobalRoles.GlobalReader; var severity = isReader ? OpsEventSeverity.Warning : OpsEventSeverity.Critical; var via = d.ClientSource ?? "direct"; var tool = string.IsNullOrEmpty(d.McpToolName) ? string.Empty : $", tool {d.McpToolName}"; return WriteAsync(OpsEventCategory.Security, OpsEventTypes.PrivilegedRouteDenied, severity, $"Privileged route denied: {d.Method} {d.Path} for {d.Upn ?? "(no upn)"} (role={d.CallerRole}, via {via}{tool})", d.TenantId, d.Upn ?? "System.PolicyEnforcement", new { method = d.Method, path = d.Path, statusCode = d.StatusCode, reason = d.Reason, policy = d.Policy, upn = d.Upn, oid = d.ObjectId, tid = d.TenantId, callerRole = d.CallerRole, clientSource = d.ClientSource, mcpToolName = d.McpToolName, correlationId = d.CorrelationId, }); } /// /// Fired by when a Kill signal was actually SERVED to /// an agent (as opposed to DeviceBlocked/VersionBlocked, which fire when the admin /// creates the rule). This is the delivery confirmation operators wire Telegram rules /// on — throttled at the evaluator (24h per tenant+serial+pattern) because a kill-blind /// old agent keeps hitting the endpoint every few seconds. Details carry /// serialNumber so the Ops Events detail modal's Block/Kill shortcuts deep-link. /// public Task RecordKillSignalDeliveredAsync( string tenantId, string? serialNumber, string? agentVersion, string? matchedPattern, string trigger, string channel) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.KillSignalDelivered, OpsEventSeverity.Warning, trigger == "version" ? $"Kill signal delivered via {channel} to agent {agentVersion ?? "?"} on device {serialNumber ?? "?"} (pattern: {matchedPattern})" : $"Kill signal delivered via {channel} to device {serialNumber ?? "?"}", tenantId, "System.KillSwitch", new { serialNumber, agentVersion, matchedPattern, trigger, channel }); public Task RecordEmbeddedCertExpiringSoonAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpiringSoon, OpsEventSeverity.Warning, $"Newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - source a successor PEM and embed it before {notAfterUtc:yyyy-MM-dd}", null, "System.Maintenance", new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); public Task RecordEmbeddedCertExpiringUrgentAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpiringUrgent, OpsEventSeverity.Error, $"URGENT: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is in the bundle - agent mTLS will break without rotation", null, "System.Maintenance", new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); public Task RecordEmbeddedCertExpiredAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertExpired, OpsEventSeverity.Critical, daysUntilExpiry < 0 ? $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' EXPIRED {-daysUntilExpiry}d ago ({notAfterUtc:u}) and no successor is embedded - agent mTLS validation broken" : $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is embedded", null, "System.Maintenance", new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry }); public Task RecordEmbeddedCertBundleEmptyAsync() => WriteAsync(OpsEventCategory.Security, OpsEventTypes.EmbeddedCertBundleEmpty, OpsEventSeverity.Critical, "No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients", null, "System.Maintenance", new { }); public Task RecordSignalRConnectionsHighAsync(int observed, int limit, int percent, string resourceId) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRConnectionsHigh, OpsEventSeverity.Warning, $"SignalR concurrent connections at {percent}% of plan limit ({observed}/{limit}) - watch for 429s; consider adding units before saturation", null, "System.Monitoring", new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId }); public Task RecordSignalRConnectionsCriticalAsync(int observed, int limit, int percent, string resourceId) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRConnectionsCritical, OpsEventSeverity.Error, $"CRITICAL: SignalR concurrent connections at {percent}% of plan limit ({observed}/{limit}) - new client connections will be 429'd at 100%; add units now", null, "System.Monitoring", new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId }); public Task RecordSignalRMessagesHighAsync(long observed, long limit, int percent, string resourceId) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRMessagesHigh, OpsEventSeverity.Warning, $"SignalR daily message count at {percent}% of included plan quota ({observed}/{limit}) - resets at 00:00 UTC; overage is billed per extra million messages", null, "System.Monitoring", new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId }); public Task RecordSignalRMessagesCriticalAsync(long observed, long limit, int percent, string resourceId) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.SignalRMessagesCritical, OpsEventSeverity.Error, $"CRITICAL: SignalR daily message count at {percent}% of included plan quota ({observed}/{limit}) - overage beyond 100% is billed per extra million messages; review traffic or add units", null, "System.Monitoring", new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId }); public Task RecordPoisonQueueBacklogHighAsync(string queueName, long count, int threshold) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.PoisonQueueBacklogHigh, OpsEventSeverity.Warning, $"Poison queue '{queueName}' backlog at {count} message(s) (threshold: {threshold}) — async worker handler failing repeatedly; inspect dead-letter contents", null, "System.Maintenance", new { queueName, count, threshold }); public Task RecordPoisonQueueBacklogCriticalAsync(string queueName, long count, int threshold) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.PoisonQueueBacklogCritical, OpsEventSeverity.Error, $"CRITICAL: poison queue '{queueName}' backlog at {count} messages (threshold: {threshold}) — sustained handler failure, downstream work is silently dropping", null, "System.Maintenance", new { queueName, count, threshold }); // ── Critical-Table Backup ───────────────────────────────────────────── /// Backup run finished with all tables successfully captured. Info-level — visible in the timeline, not alertable by default. public Task RecordCriticalTableBackupCompletedAsync(string backupId, int tableCount, int durationMs, string container, string manifestBlobName, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupCompleted, OpsEventSeverity.Info, $"Critical-table backup {backupId} completed: {tableCount} tables, {durationMs}ms (triggeredBy={triggeredBy})", null, "System.Maintenance", new { backupId, tableCount, durationMs, container, manifestBlobName, triggeredBy }); /// Backup run wrote a manifest but at least one table Failed or Skipped. Warning-level — operator should inspect manifest perTableFailures. public Task RecordCriticalTableBackupPartialAsync(string backupId, int totalTables, int failedOrSkipped, int durationMs, string container, string manifestBlobName, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupPartial, OpsEventSeverity.Warning, $"Critical-table backup {backupId} PARTIAL: {failedOrSkipped}/{totalTables} tables failed or skipped, manifest written ({durationMs}ms, triggeredBy={triggeredBy})", null, "System.Maintenance", new { backupId, totalTables, failedOrSkipped, durationMs, container, manifestBlobName, triggeredBy }); /// Backup run never produced a valid manifest (fatal exception, storage outage). Error-level. Queue-path emits this AFTER 5x retry + poison-move; timer emits immediately. public Task RecordCriticalTableBackupFailedAsync(string? backupId, string errorMessage, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupFailed, OpsEventSeverity.Error, $"Critical-table backup FAILED (backupId={backupId ?? "n/a"}, triggeredBy={triggeredBy}): {errorMessage}", null, "System.Maintenance", new { backupId, errorMessage, triggeredBy }); /// Backup or restore was skipped because the maintenance lease was already held by another job. Info-level — not a failure. public Task RecordCriticalTableBackupSkippedLockedAsync(string reason, string triggeredBy) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CriticalTableBackupSkippedLocked, OpsEventSeverity.Info, $"Critical-table backup skipped — {reason} (triggeredBy={triggeredBy})", null, "System.Maintenance", new { reason, triggeredBy }); /// /// A GA operator restored a single row from a backup (plan §PR2). Warning-level /// because the restore overwrote (or created) live data via ETag-CAS — operators /// frequently want a Telegram ping for this event so a parallel admin sees the /// audit trail in near-real-time. Payload carries the canonical /// { container, blobName } only — no SAS URL, per plan §Medium #6. /// public Task RecordBackupRowRestoredAsync( string backupId, string tableName, string partitionKey, string rowKey, string actor, string outcome) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.BackupRowRestored, OpsEventSeverity.Warning, $"Critical-table row restored: {tableName} (pk='{partitionKey}', rk='{rowKey}') from backup {backupId} by {actor} → {outcome}", null, actor, new { backupId, tableName, partitionKey, rowKey, outcome, container = AutopilotMonitor.Shared.Constants.BlobContainers.CriticalTableBackups, blobName = $"{backupId}/{tableName}.ndjson", }); // ── Tenant ───────────────────────────────────────────────────────────── /// /// Fired when a departing admin submits free-form feedback in the offboarding /// drain-barrier banner. Information-tier — not actionable in itself, but the kind /// of signal product wants to be notified about so feedback gets read promptly. /// public Task RecordOffboardingFeedbackReceivedAsync( string tenantId, string submittedBy, string? domainName, string historyRowKey) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; var details = new Dictionary { ["domainName"] = domainName, ["historyRowKey"] = historyRowKey, }; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.OffboardingFeedbackReceived, OpsEventSeverity.Info, $"Offboarding feedback received from {tenantLabel}", tenantId, submittedBy, details); } public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary deletedCounts, string? domainName = null) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; // Wrap deletedCounts + domainName so the OpsEvents details panel still surfaces the // per-table counts AND the domain (needed because Table Storage is gone by emit time). var details = new Dictionary { ["domainName"] = domainName, ["deletedCounts"] = deletedCounts, }; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantOffboarded, OpsEventSeverity.Warning, $"Tenant {tenantLabel} offboarded — all data deleted", tenantId, performedBy, details); } /// /// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch /// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion, /// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is /// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2. /// public Task RecordTenantOffboardingFailedAsync( string tenantId, string performedBy, string failedPhase, string errorMessage, int retryCount, string? domainName = null) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantOffboardingFailed, OpsEventSeverity.Error, $"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}", tenantId, performedBy, new { domainName, failedPhase, errorMessage, retryCount }); } /// /// Fired by the tenant-auto-approve queue worker when a new signup was activated /// automatically (AutoApproveNewTenants enabled). Info-tier audit + Telegram-routable /// signal so the operator sees which tenants entered without a manual approval click. /// Dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES /// (memory feedback_ops_event_types_dual_register). /// public Task RecordTenantAutoApprovedAsync(string tenantId, string? domainName, string signupUpn) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantAutoApproved, OpsEventSeverity.Info, $"Tenant {tenantLabel} was auto-activated after signup (signup by {signupUpn})", tenantId, "System (auto-approve)", new { domainName, signupUpn }); } // ── Activation welcome mail ── // The mail is a courtesy side effect that fails soft, which used to make every failure // mode invisible: the "no address" and "provider rejected" branches only wrote // LogInformation, and worker application logs never reach Application Insights below // Warning. Between 2026-08-21 and 2026-08-26 that hid a total stop of the welcome mail. // These three types are the visible record. All dual-registered in // OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register). /// The provider accepted the activation welcome mail. Info-tier confirmation. public Task RecordWelcomeEmailSentAsync(string tenantId, string? domainName, string toEmail, string addressSource) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailSent, OpsEventSeverity.Info, $"Welcome email sent to {toEmail} for tenant {tenantLabel} (address from {addressSource})", tenantId, "System.Activation", new { domainName, toEmail, addressSource }); } /// /// The tenant was activated but no welcome mail went out, because no address could be /// resolved (neither the activation-page address nor the tenant contact address). /// Warning-tier: nothing is broken technically, but a customer was onboarded in silence. /// public Task RecordWelcomeEmailSkippedAsync(string tenantId, string? domainName, string reason) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailSkipped, OpsEventSeverity.Warning, $"No welcome email for tenant {tenantLabel} — {reason}", tenantId, "System.Activation", new { domainName, reason }); } /// /// An address was resolved and handed to the provider, which did not accept it (or the /// send threw). Error-tier: this one IS a defect — provider key, rejection or outage. /// public Task RecordWelcomeEmailFailedAsync(string tenantId, string? domainName, string toEmail, string reason) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.WelcomeEmailFailed, OpsEventSeverity.Error, $"Welcome email to {toEmail} failed for tenant {tenantLabel} — {reason}", tenantId, "System.Activation", new { domainName, toEmail, reason }); } // ── Tenant trial lifecycle (informational — enforcement is read-time) ── // Both types are dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES // (memory feedback_ops_event_types_dual_register). Dispatched by TrialExpirySweepFunction. /// /// A Pro trial was started — the conversion moment. Fired from BOTH plan write paths: /// the tenant-admin self-service POST and a GA grant/extension via PATCH plan /// ( tells them apart). Info-tier, but the event a /// sales/support channel is typically bound to, so the payload carries who to contact /// (address + company name) rather than only the tenant GUID. /// public Task RecordTenantTrialStartedAsync( string tenantId, string? domainName, string? contactEmail, string? companyName, DateTime? trialStartedUtc, DateTime? trialExpiresUtc, string grantedBy, bool selfService) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; var expiryNote = trialExpiresUtc is DateTime expiry ? $"until {expiry:yyyy-MM-dd HH:mm}Z" : "with no end date set"; var origin = selfService ? "self-service" : "granted by an operator"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialStarted, OpsEventSeverity.Info, $"Pro trial started for tenant {tenantLabel} {expiryNote} ({origin}, by {grantedBy})", tenantId, grantedBy, new { domainName, contactEmail, companyName, trialStartedUtc, trialExpiresUtc, grantedBy, selfService }); } /// Heads-up: a Pro trial ends within the next few days. Info-tier visibility signal. public Task RecordTenantTrialExpiringAsync(string tenantId, string? domainName, DateTime trialExpiresUtc, int daysLeft) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialExpiring, OpsEventSeverity.Info, $"Pro trial for tenant {tenantLabel} expires in {daysLeft} day(s) ({trialExpiresUtc:yyyy-MM-dd HH:mm}Z)", tenantId, "System.TrialSweep", new { domainName, trialExpiresUtc, daysLeft }); } /// /// A Pro trial expired within the last sweep window — the tenant silently degraded /// to Community at read time (retention cap, rate limits, MSP delegation, MCP plan). /// Warning-tier: a conversion moment an operator likely wants a Telegram ping for. /// public Task RecordTenantTrialExpiredAsync(string tenantId, string? domainName, DateTime trialExpiredUtc) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantTrialExpired, OpsEventSeverity.Warning, $"Pro trial for tenant {tenantLabel} expired ({trialExpiredUtc:yyyy-MM-dd HH:mm}Z) — tenant is now Community", tenantId, "System.TrialSweep", new { domainName, trialExpiredUtc }); } // ── Plan downgrade / retention grace (informational — enforcement is read-time) ── // All three types are dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES // (memory feedback_ops_event_types_dual_register). Downgraded is dispatched by // PlanManagementFunction, the grace events by TrialExpirySweepFunction. /// /// A GA plan mutation dropped the tenant's EFFECTIVE edition Pro → Community. The /// retention downgrade grace period starts now; data older than the Community cap is /// deleted only after it ends. Warning-tier: the business moment (e.g. non-payment) /// an operator wants a ping for. /// public Task RecordTenantPlanDowngradedAsync( string tenantId, string? domainName, string caller, DateTime? retentionGraceEndsUtc, int storedRetentionDays) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; var graceNote = retentionGraceEndsUtc is DateTime graceEnd ? $"retention grace until {graceEnd:yyyy-MM-dd HH:mm}Z" : "no retention grace applicable"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantPlanDowngraded, OpsEventSeverity.Warning, $"Tenant {tenantLabel} was downgraded Pro → Community by {caller} — {graceNote} (stored retention {storedRetentionDays}d)", tenantId, caller, new { domainName, retentionGraceEndsUtc, storedRetentionDays }); } /// /// Heads-up: a downgraded tenant's retention grace ends within the next few days — /// after that, the sweep enforces the Community cap and deletes the older data. /// Emitted only for tenants that actually have data at risk (stored retention above /// the Community cap). Re-fired daily during the heads-up window (no dedup state). /// public Task RecordTenantRetentionGraceExpiringAsync( string tenantId, string? domainName, DateTime graceEndsUtc, int daysLeft, int storedRetentionDays) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantRetentionGraceExpiring, OpsEventSeverity.Warning, $"Retention grace for downgraded tenant {tenantLabel} ends in {daysLeft} day(s) ({graceEndsUtc:yyyy-MM-dd HH:mm}Z) — " + $"data older than the Community cap will then be deleted (stored retention {storedRetentionDays}d)", tenantId, "System.TrialSweep", new { domainName, graceEndsUtc, daysLeft, storedRetentionDays }); } /// /// A downgraded tenant's retention grace ended within the last sweep window — from now /// on the retention sweep enforces the Community cap and starts deleting the older data. /// Last call for a GA intervention (re-upgrade / retention decision). /// public Task RecordTenantRetentionGraceEndedAsync( string tenantId, string? domainName, DateTime graceEndedUtc, int storedRetentionDays) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantRetentionGraceEnded, OpsEventSeverity.Warning, $"Retention grace for downgraded tenant {tenantLabel} ended ({graceEndedUtc:yyyy-MM-dd HH:mm}Z) — " + $"the Community retention cap is now enforced and older data will be deleted (stored retention {storedRetentionDays}d)", tenantId, "System.TrialSweep", new { domainName, graceEndedUtc, storedRetentionDays }); } /// /// F3 regression radar (insights spec §F3): an analyze rule's 7-day hit rate rose ≥2× /// over its 28-day baseline with disjoint Wilson intervals. Fired ONCE per episode (the /// notification-tracker row is the dedup); the message carries the full numbers so the /// event is verifiable without a portal round-trip. Dimension wording is correlation /// only — never causal. Dual-registered in the web OPS_EVENT_TYPES catalog /// (memory: feedback_ops_event_types_dual_register). /// public Task RecordRuleFrequencyRegressionAsync( string tenantId, string ruleId, string ruleTitle, int windowFireCount, int windowSessionCount, double windowRatePct, int baselineFireCount, int baselineSessionCount, double baselineRatePct, double? lift, string? dimensionSummary) => WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.RuleFrequencyRegression, OpsEventSeverity.Warning, $"Rule '{ruleTitle}' ({ruleId}) fired in {windowFireCount}/{windowSessionCount} sessions ({windowRatePct}%) over 7d " + $"vs {baselineRatePct}% baseline ({baselineFireCount}/{baselineSessionCount} over 28d)" + (lift.HasValue ? $" — lift {lift.Value}x" : " — new signal (no baseline fires)") + (string.IsNullOrEmpty(dimensionSummary) ? string.Empty : $". {dimensionSummary}"), tenantId, "System.Maintenance", new { ruleId, windowFireCount, windowSessionCount, windowRatePct, baselineFireCount, baselineSessionCount, baselineRatePct, lift, }); /// /// App-version duration regression radar: an app's newest version installs with a median /// duration ≥2× (and ≥5 min absolute) over the previous version's median. Fired ONCE per /// (app, version) episode (the notification-tracker row is the dedup); the message /// carries the full numbers so the event is verifiable without a portal round-trip. /// Dual-registered in the web OPS_EVENT_TYPES catalog /// (memory: feedback_ops_event_types_dual_register). /// public Task RecordAppVersionDurationRegressionAsync( string tenantId, string appName, string currentVersion, string previousVersion, int currentMedianSeconds, int previousMedianSeconds, int currentMeasuredCount, int previousMeasuredCount, double lift) => WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.AppVersionDurationRegression, OpsEventSeverity.Warning, $"App '{appName}' median install duration rose from {Math.Round(previousMedianSeconds / 60.0, 1)} to " + $"{Math.Round(currentMedianSeconds / 60.0, 1)} min after version {currentVersion} " + $"({currentMeasuredCount} measured installs vs {previousMeasuredCount} on {previousVersion}) — lift {lift}x", tenantId, "System.Maintenance", new { appName, currentVersion, previousVersion, currentMedianSeconds, previousMedianSeconds, currentMeasuredCount, previousMeasuredCount, lift, }); /// /// Verdict-calibration drift radar (internal/docs/backend/verdict-calibration.md): a verdict /// path's share of sessions doubled over its 28d baseline, the silence share /// (sweep+maxlife — agent went quiet, backend had to decide) doubled, or the pure /// fallthrough rule r6 decides ≥20 % of classifier verdicts. Operator-only diagnostic — /// there is deliberately NO tenant bell. Fired ONCE per (kind, path, status) episode /// (tracker-deduped). Dual-registered in the web OPS_EVENT_TYPES catalog /// (memory: feedback_ops_event_types_dual_register). /// public Task RecordVerdictCalibrationDriftAsync( string tenantId, string kind, string verdictPath, string status, int windowHitCount, int windowSessionCount, double windowRatePct, int baselineHitCount, int baselineSessionCount, double baselineRatePct, double? lift, string? dimensionSummary) => WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.VerdictCalibrationDrift, OpsEventSeverity.Warning, $"Verdict calibration [{kind}] {verdictPath}/{status}: {windowHitCount}/{windowSessionCount} ({windowRatePct}%) over 7d " + $"vs {baselineRatePct}% baseline ({baselineHitCount}/{baselineSessionCount} over 28d)" + (lift.HasValue ? $" — lift {lift.Value}x" : string.Empty) + (string.IsNullOrEmpty(dimensionSummary) ? string.Empty : $". {dimensionSummary}"), tenantId, "System.Maintenance", new { kind, verdictPath, status, windowHitCount, windowSessionCount, windowRatePct, baselineHitCount, baselineSessionCount, baselineRatePct, lift, }); /// /// A tenant admin turned on-demand diagnostics upload on or off — the capability behind /// the session-detail Collect Logs button (portal Settings, the Collect Logs quick-config /// dialog, MCP update_tenant_config, or a config revert). Fires only on the flip, not on /// every diagnostics edit (). /// Info-tier. The Collect Logs quick-config dialog (source portal-collect-logs) /// gets its own type CollectLogsQuickConfigEnabled so a rule can push on exactly /// "someone clicked Collect Logs and switched on Hosted upload" without also firing on /// Settings-page edits; every other enable path is DiagnosticsUploadEnabled. /// Dual-register per memory feedback_ops_event_types_dual_register. /// public Task RecordDiagnosticsUploadConfigChangedAsync( string tenantId, string? domainName, DiagnosticsUploadConfigChange change, string changedBy, string source) { var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})"; var quickConfig = string.Equals(source, Functions.Config.UpdateTenantConfigurationFunction.CollectLogsSource, StringComparison.Ordinal); if (change.Enabled && quickConfig) { return WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.CollectLogsQuickConfigEnabled, OpsEventSeverity.Info, $"Collect Logs: {changedBy} enabled diagnostics upload for tenant {tenantLabel} via the session quick-config dialog (destination {change.Destination ?? "?"}, mode {change.Mode})", tenantId, changedBy, new { domainName, change.Destination, change.Mode, source }); } return change.Enabled ? WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.DiagnosticsUploadEnabled, OpsEventSeverity.Info, $"Diagnostics upload enabled for tenant {tenantLabel} (destination {change.Destination ?? "?"}, mode {change.Mode}) via {source} by {changedBy}", tenantId, changedBy, new { domainName, change.Destination, change.Mode, source }) : WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.DiagnosticsUploadDisabled, OpsEventSeverity.Info, $"Diagnostics upload disabled for tenant {tenantLabel} (mode {change.Mode}) via {source} by {changedBy}", tenantId, changedBy, new { domainName, change.Destination, change.Mode, source }); } // ── Agent ────────────────────────────────────────────────────────────── /// /// A portal user queued a ServerAction for a live session (Collect Logs = /// request_diagnostics, quick-config = rotate_config, admin /// terminate_session). Rule-engine-queued actions do NOT pass through here — /// this is the "who uses the on-demand controls, and how often" signal. /// Dual-register per memory feedback_ops_event_types_dual_register. /// public Task RecordSessionActionQueuedAsync(string tenantId, string sessionId, string actionType, string? reason, string queuedBy) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.SessionActionQueued, OpsEventSeverity.Info, $"Server action '{actionType}' queued for session {sessionId} by {queuedBy}: {reason}", tenantId, queuedBy, new { sessionId, actionType, reason }); public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.SessionTimeouts, OpsEventSeverity.Info, $"{sessionCount} session(s) timed out after {timeoutHours}h", tenantId, "System.Maintenance", new { sessionCount, timeoutHours }); /// /// An agent reported its absolute session-age emergency break (48h cap) over the /// emergency channel — it cleaned itself up and exited on a session that never reached /// a terminal state. This is the "are we silently losing agents?" signal; emitted by /// once per session (guarded by /// the timeline-event idempotency check). Severity is decided by the caller from the /// session status at break time (ReportAgentErrorFunction.ClassifyEmergencyBreak): /// Warning when the session was still open (the break IS the verdict — Telegram-rule /// worthy), Info when the sweep had already terminalized it and the break is a late /// cleanup after the device was powered on again. The 2026-09-03 audit found 224 of 269 /// breaks in three days were such late cleanups (kiosk devices stored for weeks); as /// Warnings they buried the real ones. lands in the /// details as sessionStatusAtBreak so the feed shows which case it was. /// public Task RecordAgentEmergencyBreakAsync( string tenantId, string sessionId, string? agentVersion, string message, string severity, SessionStatus? statusAtBreak, bool lateCleanup) { var statusText = statusAtBreak?.ToString() ?? "unknown"; var context = lateCleanup ? $"late cleanup — session already {statusText} when the break arrived" : $"session still {statusText} when the break arrived"; return WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentEmergencyBreak, severity, $"Agent emergency break on session {sessionId} (agent {agentVersion ?? "?"}, {context}): {message}", tenantId, "System.EmergencyChannel", new { sessionId, agentVersion, sessionStatusAtBreak = statusText, lateCleanup }); } /// /// An agent reported that its RUNNING exe's SHA-256 differs from the hash the backend /// advertises for its version — the binary in the field is not the published build /// (tamper, stale blob, or a build that never came from a committed tree). Session /// e9753578 (2026-08-20) carried exactly this report, but it landed only in App /// Insights and was invisible to every product surface while the field damage was /// mis-attributed to committed code for hours. Warning-tier so operators can wire a /// Telegram rule; the message carries both hashes verbatim. /// public Task RecordAgentBinaryIntegrityMismatchAsync(string tenantId, string? sessionId, string? agentVersion, string message) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentBinaryIntegrityMismatch, OpsEventSeverity.Warning, $"Agent binary integrity mismatch on session {sessionId ?? "?"} (agent {agentVersion ?? "?"}): {message}", tenantId, "System.EmergencyChannel", new { sessionId, agentVersion }); /// /// The CMTrace time-skew tripwire fired: a terminal session's IME-derived event /// timestamps diverge from its other events by a clean 15-minute-grid multiple — /// the signature of a timezone mis-conversion the per-line self-anchoring /// (internal/docs/agent/cmtrace-time-resolution.md) failed to catch. Goal state: this event /// NEVER fires; any occurrence is a real anchoring regression or a detector bug, /// both actionable. Emitted by once per session /// (gated on the terminal status transition). The message carries the full numbers /// so the event is verifiable without a portal round-trip. /// Dual-register per memory feedback_ops_event_types_dual_register. /// public Task RecordCmTraceTimeSkewRegressionAsync(string tenantId, string sessionId, string? agentVersion, string message, object details) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.CmTraceTimeSkewRegression, OpsEventSeverity.Warning, $"CMTrace time-skew tripwire on session {sessionId} (agent {agentVersion ?? "?"}): {message}", tenantId, "System.Ingest", details); public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ExcessiveSessionEvents, OpsEventSeverity.Warning, $"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug", tenantId, "System.Maintenance", new { sessionId, eventCount, threshold }); /// /// Fired when maintenance auto-blocks or auto-kills a device after its session crossed /// . /// Critical-severity so operators can wire a Telegram rule independent of the warn-tier /// ExcessiveSessionEvents rule. Details carry the resolved serialNumber /// so the Ops Events detail modal's Block/Kill shortcuts deep-link correctly. /// public Task RecordExcessiveSessionEventsAutoActionedAsync( string tenantId, string sessionId, string serialNumber, int eventCount, int threshold, string action, int durationHours) => WriteAsync(OpsEventCategory.Security, OpsEventTypes.ExcessiveSessionEventsAutoActioned, OpsEventSeverity.Critical, $"Auto-{action.ToLowerInvariant()} device {serialNumber} for session {sessionId} ({eventCount} events ≥ {threshold}, {durationHours}h)", tenantId, "System.Maintenance", new { sessionId, serialNumber, eventCount, threshold, action, durationHours }); public Task RecordNewImeVersionDetectedAsync(string version, string tenantId, string sessionId) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.NewImeVersionDetected, OpsEventSeverity.Warning, $"New IME agent version detected: {version}", tenantId, "System.Ingest", new { version, sessionId }); /// /// An IME log pattern that matches in ≥80 % of sessions on the baseline IME version has /// matched in NONE of ≥25 reporting sessions on — Microsoft /// probably changed the log wording. Once per version×pattern (ImePatternStats.DriftFlaggedAt). /// Dual-registered in the web OPS_EVENT_TYPES list (OpsAlertRulesSection.tsx). /// public Task RecordImePatternDriftSuspectedAsync(string version, string patternId, string baselineVersion, double baselineRate, int sessions) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.ImePatternDriftSuspected, OpsEventSeverity.Warning, $"IME pattern {patternId} never matched in {sessions} sessions on IME {version} (baseline {baselineVersion}: {baselineRate:P0} of sessions) — log wording may have changed", null, "System.Ingest", new { version, patternId, baselineVersion, baselineRate, sessions }); public Task RecordBlobStorageMissingAsync(string missingItem, int statusCode) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.BlobStorageMissing, OpsEventSeverity.Critical, $"Agent blob storage check failed: {missingItem} is missing or unreachable (HTTP {statusCode})", null, "System.Maintenance", new { missingItem, statusCode }); public Task RecordBlobStorageUnreachableAsync(string error) => WriteAsync(OpsEventCategory.Agent, OpsEventTypes.BlobStorageUnreachable, OpsEventSeverity.Critical, $"Agent blob storage unreachable: {error}", null, "System.Maintenance", new { error }); // ── SLA ──────────────────────────────────────────────────────────────── public Task RecordSlaBreachNotificationAsync(string tenantId, string breachType, double currentRate, double targetRate, int totalSessions, int failedSessions) { // Duration breaches carry the P95 in minutes, every other type a percentage — same units the // customer-facing alert (NotificationAlertBuilder.BuildSlaBreachAlert) prints. var unit = breachType == Shared.Models.SlaBreachType.Duration ? " min" : "%"; return WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaBreachNotification, OpsEventSeverity.Warning, $"SLA breach notification sent for tenant {tenantId}: {breachType} {currentRate:F1}{unit} (target {targetRate:F1}{unit})", tenantId, "System.SlaEvaluation", new { breachType, currentRate, targetRate, totalSessions, failedSessions }); } public Task RecordSlaConsecutiveFailuresAsync(string tenantId, int count, string? lastDevice, string? lastReason) => WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaConsecutiveFailures, OpsEventSeverity.Error, $"Consecutive failure alert for tenant {tenantId}: {count} failures in a row", tenantId, "System.SlaEvaluation", new { count, lastDevice, lastReason }); public Task RecordSlaEvaluationCompletedAsync(int tenantsEvaluated, int breachesDetected, int notificationsSent, int durationMs) => WriteAsync(OpsEventCategory.Sla, OpsEventTypes.SlaEvaluationCompleted, OpsEventSeverity.Info, $"SLA evaluation: {tenantsEvaluated} tenants checked, {breachesDetected} breaches, {notificationsSent} notifications sent", null, "System.SlaEvaluation", new { tenantsEvaluated, breachesDetected, notificationsSent, durationMs }); // ── Platform (Azure Monitor) ─────────────────────────────────────────── /// /// Records an Azure Monitor alert notification delivered via the ops alert webhook /// (AzureMonitorAlertWebhookFunction). severity is already mapped onto the ops scale; /// azureSeverity keeps the original Sev0–Sev4 value for the details payload. /// public Task RecordAzureMonitorAlertAsync(string alertRule, string severity, string monitorCondition, string? description, string? azureSeverity, string? monitoringService, string? targetResource, double? metricValue) { var suffix = string.IsNullOrWhiteSpace(description) ? string.Empty : $": {description}"; return WriteAsync(OpsEventCategory.Platform, OpsEventTypes.AzureMonitorAlert, severity, $"Azure Monitor alert '{alertRule}' {monitorCondition}{suffix}", null, "System.AzureMonitor", new { alertRule, monitorCondition, azureSeverity, monitoringService, targetResource, metricValue }); } // ── Core write method ────────────────────────────────────────────────── private async Task WriteAsync(string category, string eventType, string severity, string message, string? tenantId, string? userId, object? details) { try { var entry = new OpsEventEntry { Category = category, EventType = eventType, Severity = severity, TenantId = tenantId, UserId = userId, Message = message, Details = details != null ? JsonSerializer.Serialize(details) : null, Timestamp = DateTime.UtcNow, }; await _repository.SaveOpsEventAsync(entry); // Fire-and-forget: dispatch alerts to the channels the rules target. // DispatchAsync has its own top-level try/catch so unobserved exceptions are safe. // The serialized details ride along: an outbound channel that only knows category, // event and tenant GUID forces the reader back into the portal for every alert. _ = _alertDispatch.DispatchAsync(category, eventType, severity, message, tenantId, entry.Details); } catch (Exception ex) { // Never throw from ops event recording — it must not break the calling flow _logger.LogWarning(ex, "Failed to record ops event {Category}/{EventType}", category, eventType); } } } }