Files
AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Services/OpsEventService.cs
T
Oliver Kieselbach d8e8cd15fd feat(entitlements): Community/Enterprise editions with self-service trial
Fail-closed entitlement layer (FeatureEntitlementCatalog + TenantEntitlementService): edition resolves at read time (PlanTier == 'enterprise' OR active trial; legacy free/pro => Community, no migration).

- Plan/trial endpoints: PATCH config/{tid}/plan (GA: tier + grant/extend/end trial, audited, cache-safe) and POST config/{tid}/trial (tenant-admin self-service 30d, once per tenant, 409 after). Generic PUT preserves plan/trial fields for all callers. Fixes SetPlanTier writing past the config cache without audit/UpdatedBy.
- Rate limits: Enterprise floor 150/min for BOTH user (portal/JWT) and device (agent/cert) paths - floor raises the global default, an explicit per-tenant override still wins; MSP users follow their home tenant.
- Retention cap: Community 90d / Enterprise 365d, clamped in the retention fanout (warn log) + non-GA write validation 7..cap (only on changed values; 0=infinite stays GA-only).
- Delegated admin (MSP): Enterprise gate on the delegated admin's HOME tenant (JWT tid, fail-closed on unknown) - managed target tenants may be any edition. Grant-time target check removed; inert grants are suppressed at resolve time (single choke point in GetScopeAsync, applied after the UPN-keyed cache).
- MCP quota: McpQuotaService + McpQuotaEnforcementMiddleware (X-Client-Source: mcp only, GA exempt) enforce daily/monthly budgets - per-user plan override > tenant edition plan; limits from SectionUsagePlans with catalog fallbacks (community 100/3000, enterprise 1000/20000); 429 with X-MCP-Quota-* headers + Retry-After; fail-open on counter reads, fail-closed on plan resolution. /api/metrics/mcp-usage/me reports effective plan + counters.
- Trial visibility: TrialExpirySweepFunction (daily 03:30 UTC, informational) emits TenantTrialExpired (Warning, 24h look-back) + TenantTrialExpiring (Info, <=3d heads-up); dual-registered in web OPS_EVENT_TYPES.
- Web: EditionBadge + one-time trial CTA in settings header, Plan & Trial card in the GA tenant edit modal (own PATCH save), retention cap hints from entitlements, pure lib/edition.ts (vitest), feature-flags surface edition/trial/entitlements; MCP TENANT_SAFE_FIELDS + trial fields.

Verified: solution build clean; backend 2653/2653 tests; web tsc + 516/516 vitest; MCP server tsc + tests green.
2026-07-07 13:06:04 +02:00

555 lines
38 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services
{
/// <summary>
/// Service for recording operational events into the OpsEvents table.
/// Provides typed helper methods for each event category so callers
/// don't need to construct OpsEventEntry manually.
/// All writes are fire-and-forget safe — failures are logged but never thrown.
/// </summary>
public class OpsEventService
{
private readonly IOpsEventRepository _repository;
private readonly ILogger<OpsEventService> _logger;
private readonly OpsAlertDispatchService _alertDispatch;
public OpsEventService(IOpsEventRepository repository, ILogger<OpsEventService> logger,
OpsAlertDispatchService alertDispatch)
{
_repository = repository;
_logger = logger;
_alertDispatch = alertDispatch;
}
// ── Consent ────────────────────────────────────────────────────────────
public Task RecordConsentFlowStartedAsync(string tenantId, string userId, string redirectUri)
=> WriteAsync(OpsEventCategory.Consent, "ConsentFlowStarted", OpsEventSeverity.Info,
$"Admin consent flow started by {userId}",
tenantId, userId, new { redirectUri });
public Task RecordConsentFlowSuccessAsync(string tenantId, string userId, string trigger)
=> WriteAsync(OpsEventCategory.Consent, "ConsentFlowSuccess", OpsEventSeverity.Info,
$"Admin consent confirmed for {trigger} by {userId}",
tenantId, userId, new { trigger });
public Task RecordConsentFlowFailedAsync(string tenantId, string userId, string error, string errorDescription)
=> WriteAsync(OpsEventCategory.Consent, "ConsentFlowFailed", OpsEventSeverity.Error,
$"Admin consent failed: {error}",
tenantId, userId, new { error, errorDescription });
public Task RecordConsentRedirectUriMismatchAsync(string tenantId, string userId, string redirectUri, string redirectPath)
=> WriteAsync(OpsEventCategory.Consent, "ConsentRedirectUriMismatch", OpsEventSeverity.Critical,
$"Redirect URI path '{redirectPath}' not in registered paths — consent will fail with AADSTS50011",
tenantId, userId, new { redirectUri, redirectPath });
// ── Maintenance ────────────────────────────────────────────────────────
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "MaintenanceCompleted", OpsEventSeverity.Info,
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
null, triggeredBy, new { durationMs });
public Task RecordMaintenanceFailedAsync(string error, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "MaintenanceFailed", OpsEventSeverity.Error,
$"Maintenance failed: {error}",
null, triggeredBy, new { error });
/// <summary>
/// Early-warning signal: a maintenance run finished but took longer than the soft threshold.
/// The 2h Maintenance timer shares the host's 60min functionTimeout; a run that is climbing
/// toward that ceiling (e.g. a large first-time retention backlog) surfaces here as Warning so
/// operators are alerted before a future run is hard-aborted (which would emit no event at all).
/// </summary>
public Task RecordMaintenanceLongRunningAsync(int durationMs, int thresholdMinutes, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "MaintenanceLongRunning", OpsEventSeverity.Warning,
$"Maintenance took {durationMs}ms (> {thresholdMinutes}min soft threshold; host aborts at 60min) — triggered by {triggeredBy}",
null, triggeredBy, new { durationMs, thresholdMinutes });
public Task RecordOpsEventCleanupAsync(int deletedCount, int retentionDays)
=> WriteAsync(OpsEventCategory.Maintenance, "OpsEventCleanup", OpsEventSeverity.Info,
$"Cleaned up {deletedCount} ops events older than {retentionDays} days",
null, "System.Maintenance", new { deletedCount, retentionDays });
/// <summary>
/// Records the orphan-cleanup result. <paramref name="cleanedOrphans"/> carries the
/// per-session breakdown (which tenant + session lost lingering events and how many),
/// so the ops dashboard shows *what* was cleaned, not just a count. The list is capped
/// to keep the OpsEvents Table row under the per-property (32 KB) / entity (~1 MB) limits;
/// the full count always survives in <c>orphanSessions</c> and a <c>detailsTruncated</c>
/// flag flags any clipping. Note: an "orphan" is a session row that no longer exists while
/// its events lingered past the 24h grace — the tenant itself usually still exists.
/// </summary>
public Task RecordOrphanEventsCleanedAsync(int orphanSessions, int totalEventsDeleted,
IReadOnlyList<OrphanedEventSession>? cleanedOrphans = null)
{
const int maxDetailRows = 50;
var orphanList = cleanedOrphans ?? Array.Empty<OrphanedEventSession>();
var detail = orphanList
.OrderByDescending(o => o.EventCount)
.Take(maxDetailRows)
.Select(o => new { tenantId = o.TenantId, sessionId = o.SessionId, eventCount = o.EventCount })
.ToList();
return WriteAsync(OpsEventCategory.Maintenance, "OrphanEventsCleaned", OpsEventSeverity.Warning,
$"Cleaned {totalEventsDeleted} orphaned events across {orphanSessions} sessions",
null, "System.Maintenance",
new
{
orphanSessions,
totalEventsDeleted,
orphans = detail,
detailsTruncated = orphanList.Count > maxDetailRows
});
}
// ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
// Four event types dispatched by SessionDeletionMaintenanceFunction. Each is also
// listed in OpsAlertRulesSection.tsx OPS_EVENT_TYPES (memory feedback_ops_event_types_dual_register).
/// <summary>Watchdog: maintenance run still in flight 30 minutes after start. Warning-level early signal.</summary>
public Task RecordSessionDeletionMaintenanceLongRunningAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceLongRunning", OpsEventSeverity.Warning,
$"SessionDeletionMaintenance still running after {elapsedMinutes}min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})",
null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued });
/// <summary>Watchdog: maintenance run still in flight 60 minutes after start. Error-level escalation in case the operator missed the 30min warning.</summary>
public Task RecordSessionDeletionMaintenanceLongRunningSevereAsync(int elapsedMinutes, int tenantsProcessed, int sessionsEnqueued)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceLongRunningSevere", OpsEventSeverity.Error,
$"SessionDeletionMaintenance has been running for {elapsedMinutes}min — Azure Functions host will abort at 60min (tenants={tenantsProcessed}, enqueued={sessionsEnqueued})",
null, "System.Maintenance", new { elapsedMinutes, tenantsProcessed, sessionsEnqueued });
/// <summary>Unhandled exception path. Re-thrown after this audit so the Azure Functions runtime records the failure.</summary>
public Task RecordSessionDeletionMaintenanceFailedAsync(string exceptionType, string message, string stackPreview)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceFailed", OpsEventSeverity.Error,
$"SessionDeletionMaintenance failed: {exceptionType}: {message}",
null, "System.Maintenance", new { exceptionType, message, stackPreview });
/// <summary>Stale Queued state detected (no worker pickup) — operator must inspect the manifest + progress blobs. No auto-clear.</summary>
public Task RecordSessionDeletionStrandedQueuedAsync(string tenantId, string sessionId, DateTime queuedSince, string manifestId)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionStrandedQueued", OpsEventSeverity.Warning,
$"Session {sessionId} stuck in DeletionState=Queued since {queuedSince:o} (manifestId={manifestId})",
tenantId, "System.Maintenance", new { tenantId, sessionId, queuedSince = queuedSince.ToString("o"), manifestId });
/// <summary>
/// Cascade max-dequeue exhaustion (PR-B follow-up): the worker has moved the envelope to
/// the poison queue and CAS-transitioned the Sessions row to <see cref="SessionDeletionState.Poisoned"/>.
/// Replaces the prior <c>deletion_poisoned</c> tenant audit — tenant admins see only the
/// lifecycle endpoints (<c>deletion_started</c>, <c>deletion_completed</c>, <c>deletion_restored</c>),
/// while operators get this OpsEvent for the Session Cleanup admin page + Telegram routing.
/// <para>
/// Codex follow-ups F4 + F2: <paramref name="failureType"/> / <paramref name="failureMessage"/>
/// / <paramref name="observedResidualCount"/> / <paramref name="residualSamplePreviewJson"/>
/// are populated from <see cref="DeletionProgress"/> fields the handler writes before
/// throwing. Worker reads the progress blob in its poison path and passes whatever is
/// present (all are nullable so a worker that pre-dates the progress-schema bump still
/// emits a useful event, just without root-cause data).
/// </para>
/// <para>
/// <paramref name="observedResidualCount"/> is the verifier's <b>observed</b> count, not
/// the true total: <c>CascadeVerificationService</c> caps at
/// <see cref="DeletionProgressConstants.VerificationResidualSampleSize"/> rows per table
/// and short-circuits after the first failing table. Operators reading this number should
/// treat it as a lower bound, especially when it equals the cap.
/// </para>
/// <para>
/// <paramref name="residualSamplePreviewJson"/> is a small (≤
/// <see cref="DeletionProgressConstants.OpsEventResidualSamplePreviewSize"/>) preview that
/// fits under the OpsEvents table's 4096-char Details truncation. The full progress-blob
/// sample (up to <see cref="DeletionProgressConstants.VerificationResidualSampleSize"/>
/// entries) is available via the Session Cleanup admin page's stored-manifest modal.
/// </para>
/// </summary>
public Task RecordSessionDeletionPoisonedAsync(
string tenantId, string sessionId, string manifestId, string reason, string messageId, int dequeueCount,
string? failureType = null, string? failureMessage = null,
int? observedResidualCount = null, string? residualSamplePreviewJson = null)
{
var cause = !string.IsNullOrEmpty(failureType)
? $" — cause: {failureType}{(string.IsNullOrEmpty(failureMessage) ? "" : $" ({failureMessage})")}"
: string.Empty;
return WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionPoisoned", OpsEventSeverity.Error,
$"Session {sessionId} cascade poisoned after {dequeueCount} attempts (manifestId={manifestId}){cause}",
tenantId, "System.Maintenance",
new
{
tenantId, sessionId, manifestId, reason, messageId, dequeueCount,
failureType, failureMessage, observedResidualCount,
residualSamplePreviewJson,
});
}
/// <summary>
/// Successful end of a <see cref="Functions.Maintenance.SessionDeletionMaintenanceFunction"/>
/// run — records the per-block totals so dashboards can fold the cadence into the timeline.
/// PR6 follow-up F3: replaces the prior <c>LogAuditEntryAsync(null!, ...)</c> call, which
/// silently failed because the AuditLogs schema requires a non-null PartitionKey (tenantId).
/// </summary>
public Task RecordSessionDeletionMaintenanceCompletedAsync(
bool killSwitchActive, int tenantsProcessed, int sessionsEnqueued,
int sessionsSkipped, int rateLimitedTenants, int blobsTtlGced, int preparingRowsCleared,
int strandedQueuedDetected, int durationMs, bool abortedByKillSwitch)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceCompleted", OpsEventSeverity.Info,
$"SessionDeletionMaintenance completed in {durationMs}ms — tenants={tenantsProcessed} enqueued={sessionsEnqueued} skipped={sessionsSkipped} blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected} killSwitch={killSwitchActive}",
null, "System.Maintenance", new {
killSwitchActive, tenantsProcessed, sessionsEnqueued,
sessionsSkipped, rateLimitedTenants, blobsTtlGced, preparingRowsCleared,
strandedQueuedDetected, durationMs, abortedByKillSwitch,
});
/// <summary>
/// Records that the retention fanout half of a <see cref="Functions.Maintenance.SessionDeletionMaintenanceFunction"/>
/// run was skipped because the global kill-switch was active at entry. The three GCs
/// (manifest TTL sweep, stale-Preparing, stranded-Queued) still ran — see the totals on
/// the paired <see cref="RecordSessionDeletionMaintenanceCompletedAsync"/> event.
/// </summary>
public Task RecordSessionDeletionMaintenanceFanoutSkippedAsync(
int blobsTtlGced, int preparingRowsCleared, int strandedQueuedDetected)
=> WriteAsync(OpsEventCategory.Maintenance, "SessionDeletionMaintenanceFanoutSkipped", OpsEventSeverity.Info,
$"SessionDeletionMaintenance fanout skipped (kill-switch active) — GCs ran: blobsTtlGced={blobsTtlGced} preparingCleared={preparingRowsCleared} stranded={strandedQueuedDetected}",
null, "System.Maintenance", new { reason = "SessionDeletionKillSwitch", blobsTtlGced, preparingRowsCleared, strandedQueuedDetected });
// ── Security ───────────────────────────────────────────────────────────
public Task RecordDeviceBlockedAsync(string tenantId, string serialNumber, string reason, string blockedBy)
=> WriteAsync(OpsEventCategory.Security, "DeviceBlocked", OpsEventSeverity.Warning,
$"Device {serialNumber} blocked: {reason}",
tenantId, blockedBy, new { serialNumber, reason });
public Task RecordExcessiveDataBlockedAsync(string tenantId, int devicesBlocked, int windowHours)
=> WriteAsync(OpsEventCategory.Security, "ExcessiveDataBlocked", OpsEventSeverity.Warning,
$"{devicesBlocked} device(s) auto-blocked for excessive data (>{windowHours}h window)",
tenantId, "System.Maintenance", new { devicesBlocked, windowHours });
public Task RecordVersionBlockedAsync(string pattern, string blockedBy)
=> WriteAsync(OpsEventCategory.Security, "VersionBlocked", OpsEventSeverity.Warning,
$"Agent version pattern '{pattern}' blocked",
null, blockedBy, new { pattern });
/// <summary>
/// Fired by <see cref="KillSwitchEvaluator"/> when a Kill signal was actually SERVED to
/// an agent (as opposed to DeviceBlocked/VersionBlocked, which fire when the admin
/// creates the rule). This is the delivery confirmation operators wire Telegram rules
/// on — throttled at the evaluator (24h per tenant+serial+pattern) because a kill-blind
/// old agent keeps hitting the endpoint every few seconds. Details carry
/// <c>serialNumber</c> so the Ops Events detail modal's Block/Kill shortcuts deep-link.
/// </summary>
public Task RecordKillSignalDeliveredAsync(
string tenantId, string? serialNumber, string? agentVersion, string? matchedPattern,
string trigger, string channel)
=> WriteAsync(OpsEventCategory.Security, "KillSignalDelivered", OpsEventSeverity.Warning,
trigger == "version"
? $"Kill signal delivered via {channel} to agent {agentVersion ?? "?"} on device {serialNumber ?? "?"} (pattern: {matchedPattern})"
: $"Kill signal delivered via {channel} to device {serialNumber ?? "?"}",
tenantId, "System.KillSwitch",
new { serialNumber, agentVersion, matchedPattern, trigger, channel });
public Task RecordEmbeddedCertExpiringSoonAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringSoon", OpsEventSeverity.Warning,
$"Newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) - source a successor PEM and embed it before {notAfterUtc:yyyy-MM-dd}",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertExpiringUrgentAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpiringUrgent", OpsEventSeverity.Error,
$"URGENT: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is in the bundle - agent mTLS will break without rotation",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertExpiredAsync(string role, string subject, string thumbprint, DateTime notAfterUtc, int daysUntilExpiry)
=> WriteAsync(OpsEventCategory.Security, "EmbeddedCertExpired", OpsEventSeverity.Critical,
daysUntilExpiry < 0
? $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' EXPIRED {-daysUntilExpiry}d ago ({notAfterUtc:u}) and no successor is embedded - agent mTLS validation broken"
: $"CRITICAL: newest embedded Intune {role.ToLowerInvariant()} '{subject}' expires in {daysUntilExpiry}d ({notAfterUtc:u}) and no successor is embedded",
null, "System.Maintenance",
new { role, subject, thumbprint, notAfterUtc = notAfterUtc.ToString("u"), daysUntilExpiry });
public Task RecordEmbeddedCertBundleEmptyAsync()
=> WriteAsync(OpsEventCategory.Security, "EmbeddedCertBundleEmpty", OpsEventSeverity.Critical,
"No embedded Intune root certificates loaded - agent mTLS validator is failing closed for ALL clients",
null, "System.Maintenance", new { });
public Task RecordSignalRConnectionsHighAsync(int observed, int limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, "SignalRConnectionsHigh", OpsEventSeverity.Warning,
$"SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - watch for 429s; consider scaling to Standard before saturation",
null, "System.Monitoring",
new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
public Task RecordSignalRConnectionsCriticalAsync(int observed, int limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, "SignalRConnectionsCritical", OpsEventSeverity.Error,
$"CRITICAL: SignalR concurrent connections at {percent}% of free-tier limit ({observed}/{limit}) - new client connections will be 429'd at 100%; scale to Standard now",
null, "System.Monitoring",
new { metric = "ConnectionCount", aggregation = "Maximum", windowMinutes = 60, observed, limit, percent, resourceId });
public Task RecordSignalRMessagesHighAsync(long observed, long limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, "SignalRMessagesHigh", OpsEventSeverity.Warning,
$"SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - quota resets at 00:00 UTC; consider scaling if pattern persists",
null, "System.Monitoring",
new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
public Task RecordSignalRMessagesCriticalAsync(long observed, long limit, int percent, string resourceId)
=> WriteAsync(OpsEventCategory.Security, "SignalRMessagesCritical", OpsEventSeverity.Error,
$"CRITICAL: SignalR daily message count at {percent}% of free-tier limit ({observed}/{limit}) - hub will throttle at 100% until 00:00 UTC reset; scale to Standard now",
null, "System.Monitoring",
new { metric = "MessageCount", aggregation = "Total", windowDay = "UTC", observed, limit, percent, resourceId });
public Task RecordPoisonQueueBacklogHighAsync(string queueName, long count, int threshold)
=> WriteAsync(OpsEventCategory.Security, "PoisonQueueBacklogHigh", OpsEventSeverity.Warning,
$"Poison queue '{queueName}' backlog at {count} message(s) (threshold: {threshold}) — async worker handler failing repeatedly; inspect dead-letter contents",
null, "System.Maintenance",
new { queueName, count, threshold });
public Task RecordPoisonQueueBacklogCriticalAsync(string queueName, long count, int threshold)
=> WriteAsync(OpsEventCategory.Security, "PoisonQueueBacklogCritical", OpsEventSeverity.Error,
$"CRITICAL: poison queue '{queueName}' backlog at {count} messages (threshold: {threshold}) — sustained handler failure, downstream work is silently dropping",
null, "System.Maintenance",
new { queueName, count, threshold });
// ── Critical-Table Backup ─────────────────────────────────────────────
/// <summary>Backup run finished with all tables successfully captured. Info-level — visible in the timeline, not alertable by default.</summary>
public Task RecordCriticalTableBackupCompletedAsync(string backupId, int tableCount, int durationMs, string container, string manifestBlobName, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "CriticalTableBackupCompleted", OpsEventSeverity.Info,
$"Critical-table backup {backupId} completed: {tableCount} tables, {durationMs}ms (triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { backupId, tableCount, durationMs, container, manifestBlobName, triggeredBy });
/// <summary>Backup run wrote a manifest but at least one table Failed or Skipped. Warning-level — operator should inspect manifest perTableFailures.</summary>
public Task RecordCriticalTableBackupPartialAsync(string backupId, int totalTables, int failedOrSkipped, int durationMs, string container, string manifestBlobName, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "CriticalTableBackupPartial", OpsEventSeverity.Warning,
$"Critical-table backup {backupId} PARTIAL: {failedOrSkipped}/{totalTables} tables failed or skipped, manifest written ({durationMs}ms, triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { backupId, totalTables, failedOrSkipped, durationMs, container, manifestBlobName, triggeredBy });
/// <summary>Backup run never produced a valid manifest (fatal exception, storage outage). Error-level. Queue-path emits this AFTER 5x retry + poison-move; timer emits immediately.</summary>
public Task RecordCriticalTableBackupFailedAsync(string? backupId, string errorMessage, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "CriticalTableBackupFailed", OpsEventSeverity.Error,
$"Critical-table backup FAILED (backupId={backupId ?? "n/a"}, triggeredBy={triggeredBy}): {errorMessage}",
null, "System.Maintenance",
new { backupId, errorMessage, triggeredBy });
/// <summary>Backup or restore was skipped because the maintenance lease was already held by another job. Info-level — not a failure.</summary>
public Task RecordCriticalTableBackupSkippedLockedAsync(string reason, string triggeredBy)
=> WriteAsync(OpsEventCategory.Maintenance, "CriticalTableBackupSkippedLocked", OpsEventSeverity.Info,
$"Critical-table backup skipped — {reason} (triggeredBy={triggeredBy})",
null, "System.Maintenance",
new { reason, triggeredBy });
/// <summary>
/// A GA operator restored a single row from a backup (plan §PR2). Warning-level
/// because the restore overwrote (or created) live data via ETag-CAS — operators
/// frequently want a Telegram ping for this event so a parallel admin sees the
/// audit trail in near-real-time. Payload carries the canonical
/// <c>{ container, blobName }</c> only — no SAS URL, per plan §Medium #6.
/// </summary>
public Task RecordBackupRowRestoredAsync(
string backupId, string tableName, string partitionKey, string rowKey, string actor, string outcome)
=> WriteAsync(OpsEventCategory.Maintenance, "BackupRowRestored", OpsEventSeverity.Warning,
$"Critical-table row restored: {tableName} (pk='{partitionKey}', rk='{rowKey}') from backup {backupId} by {actor} → {outcome}",
null, actor,
new
{
backupId,
tableName,
partitionKey,
rowKey,
outcome,
container = AutopilotMonitor.Shared.Constants.BlobContainers.CriticalTableBackups,
blobName = $"{backupId}/{tableName}.ndjson",
});
// ── Tenant ─────────────────────────────────────────────────────────────
/// <summary>
/// Fired when a departing admin submits free-form feedback in the offboarding
/// drain-barrier banner. Information-tier — not actionable in itself, but the kind
/// of signal product wants to be notified about so feedback gets read promptly.
/// </summary>
public Task RecordOffboardingFeedbackReceivedAsync(
string tenantId, string submittedBy, string? domainName, string historyRowKey)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
var details = new Dictionary<string, object?>
{
["domainName"] = domainName,
["historyRowKey"] = historyRowKey,
};
return WriteAsync(OpsEventCategory.Tenant, "OffboardingFeedbackReceived", OpsEventSeverity.Info,
$"Offboarding feedback received from {tenantLabel}",
tenantId, submittedBy, details);
}
public Task RecordTenantOffboardedAsync(string tenantId, string performedBy, Dictionary<string, int> deletedCounts, string? domainName = null)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
// Wrap deletedCounts + domainName so the OpsEvents details panel still surfaces the
// per-table counts AND the domain (needed because Table Storage is gone by emit time).
var details = new Dictionary<string, object?>
{
["domainName"] = domainName,
["deletedCounts"] = deletedCounts,
};
return WriteAsync(OpsEventCategory.Tenant, "TenantOffboarded", OpsEventSeverity.Warning,
$"Tenant {tenantLabel} offboarded — all data deleted",
tenantId, performedBy, details);
}
/// <summary>
/// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch
/// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion,
/// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is
/// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2.
/// </summary>
public Task RecordTenantOffboardingFailedAsync(
string tenantId, string performedBy, string failedPhase, string errorMessage,
int retryCount, string? domainName = null)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName)
? tenantId
: $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, "TenantOffboardingFailed", OpsEventSeverity.Error,
$"Tenant {tenantLabel} offboarding failed at phase '{failedPhase}': {errorMessage}",
tenantId, performedBy,
new { domainName, failedPhase, errorMessage, retryCount });
}
// ── Tenant trial lifecycle (informational — enforcement is read-time) ──
// Both types are dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES
// (memory feedback_ops_event_types_dual_register). Dispatched by TrialExpirySweepFunction.
/// <summary>Heads-up: an Enterprise trial ends within the next few days. Info-tier visibility signal.</summary>
public Task RecordTenantTrialExpiringAsync(string tenantId, string? domainName, DateTime trialExpiresUtc, int daysLeft)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, "TenantTrialExpiring", OpsEventSeverity.Info,
$"Enterprise trial for tenant {tenantLabel} expires in {daysLeft} day(s) ({trialExpiresUtc:yyyy-MM-dd HH:mm}Z)",
tenantId, "System.TrialSweep", new { domainName, trialExpiresUtc, daysLeft });
}
/// <summary>
/// An Enterprise trial expired within the last sweep window — the tenant silently degraded
/// to Community at read time (retention cap, rate limits, MSP delegation, MCP plan).
/// Warning-tier: a conversion moment an operator likely wants a Telegram ping for.
/// </summary>
public Task RecordTenantTrialExpiredAsync(string tenantId, string? domainName, DateTime trialExpiredUtc)
{
var tenantLabel = string.IsNullOrWhiteSpace(domainName) ? tenantId : $"{domainName} ({tenantId})";
return WriteAsync(OpsEventCategory.Tenant, "TenantTrialExpired", OpsEventSeverity.Warning,
$"Enterprise trial for tenant {tenantLabel} expired ({trialExpiredUtc:yyyy-MM-dd HH:mm}Z) — tenant is now Community",
tenantId, "System.TrialSweep", new { domainName, trialExpiredUtc });
}
// ── Agent ──────────────────────────────────────────────────────────────
public Task RecordSessionTimeoutsAsync(string tenantId, int sessionCount, int timeoutHours)
=> WriteAsync(OpsEventCategory.Agent, "SessionTimeouts", OpsEventSeverity.Info,
$"{sessionCount} session(s) timed out after {timeoutHours}h",
tenantId, "System.Maintenance", new { sessionCount, timeoutHours });
public Task RecordExcessiveSessionEventsAsync(string tenantId, string sessionId, int eventCount, int threshold)
=> WriteAsync(OpsEventCategory.Agent, "ExcessiveSessionEvents", OpsEventSeverity.Warning,
$"Session {sessionId} has {eventCount} events (threshold {threshold}) — likely agent loop bug",
tenantId, "System.Maintenance", new { sessionId, eventCount, threshold });
/// <summary>
/// Fired when maintenance auto-blocks or auto-kills a device after its session crossed
/// <see cref="AutopilotMonitor.Shared.Models.AdminConfiguration.ExcessiveEventAutoActionThreshold"/>.
/// Critical-severity so operators can wire a Telegram rule independent of the warn-tier
/// <c>ExcessiveSessionEvents</c> rule. Details carry the resolved <c>serialNumber</c>
/// so the Ops Events detail modal's Block/Kill shortcuts deep-link correctly.
/// </summary>
public Task RecordExcessiveSessionEventsAutoActionedAsync(
string tenantId, string sessionId, string serialNumber, int eventCount, int threshold,
string action, int durationHours)
=> WriteAsync(OpsEventCategory.Security, "ExcessiveSessionEventsAutoActioned", OpsEventSeverity.Critical,
$"Auto-{action.ToLowerInvariant()} device {serialNumber} for session {sessionId} ({eventCount} events ≥ {threshold}, {durationHours}h)",
tenantId, "System.Maintenance",
new { sessionId, serialNumber, eventCount, threshold, action, durationHours });
public Task RecordNewImeVersionDetectedAsync(string version, string tenantId, string sessionId)
=> WriteAsync(OpsEventCategory.Agent, "NewImeVersionDetected", OpsEventSeverity.Warning,
$"New IME agent version detected: {version}",
tenantId, "System.Ingest", new { version, sessionId });
public Task RecordBlobStorageMissingAsync(string missingItem, int statusCode)
=> WriteAsync(OpsEventCategory.Agent, "BlobStorageMissing", OpsEventSeverity.Critical,
$"Agent blob storage check failed: {missingItem} is missing or unreachable (HTTP {statusCode})",
null, "System.Maintenance", new { missingItem, statusCode });
public Task RecordBlobStorageUnreachableAsync(string error)
=> WriteAsync(OpsEventCategory.Agent, "BlobStorageUnreachable", OpsEventSeverity.Critical,
$"Agent blob storage unreachable: {error}",
null, "System.Maintenance", new { error });
// ── SLA ────────────────────────────────────────────────────────────────
public Task RecordSlaBreachNotificationAsync(string tenantId, string breachType,
double currentRate, double targetRate, int totalSessions, int failedSessions)
=> WriteAsync(OpsEventCategory.Sla, "SlaBreachNotification", OpsEventSeverity.Warning,
$"SLA breach notification sent for tenant {tenantId}: {breachType} {currentRate:F1}% (target {targetRate:F1}%)",
tenantId, "System.SlaEvaluation",
new { breachType, currentRate, targetRate, totalSessions, failedSessions });
public Task RecordSlaConsecutiveFailuresAsync(string tenantId, int count, string? lastDevice, string? lastReason)
=> WriteAsync(OpsEventCategory.Sla, "SlaConsecutiveFailures", OpsEventSeverity.Error,
$"Consecutive failure alert for tenant {tenantId}: {count} failures in a row",
tenantId, "System.SlaEvaluation",
new { count, lastDevice, lastReason });
public Task RecordSlaEvaluationCompletedAsync(int tenantsEvaluated, int breachesDetected, int notificationsSent, int durationMs)
=> WriteAsync(OpsEventCategory.Sla, "SlaEvaluationCompleted", OpsEventSeverity.Info,
$"SLA evaluation: {tenantsEvaluated} tenants checked, {breachesDetected} breaches, {notificationsSent} notifications sent",
null, "System.SlaEvaluation",
new { tenantsEvaluated, breachesDetected, notificationsSent, durationMs });
// ── Core write method ──────────────────────────────────────────────────
private async Task WriteAsync(string category, string eventType, string severity,
string message, string? tenantId, string? userId, object? details)
{
try
{
var entry = new OpsEventEntry
{
Category = category,
EventType = eventType,
Severity = severity,
TenantId = tenantId,
UserId = userId,
Message = message,
Details = details != null ? JsonSerializer.Serialize(details) : null,
Timestamp = DateTime.UtcNow,
};
await _repository.SaveOpsEventAsync(entry);
// Fire-and-forget: dispatch alerts to enabled providers.
// TrySendAlerts has its own top-level try/catch so unobserved exceptions are safe.
_ = _alertDispatch.DispatchAsync(category, eventType, severity, message, tenantId);
}
catch (Exception ex)
{
// Never throw from ops event recording — it must not break the calling flow
_logger.LogWarning(ex, "Failed to record ops event {Category}/{EventType}", category, eventType);
}
}
}
}