using System; using System.Threading; using System.Threading.Tasks; using AutopilotMonitor.Functions.Services; using AutopilotMonitor.Functions.Services.Backup; using AutopilotMonitor.Shared; using AutopilotMonitor.Shared.Models.Backup; using Microsoft.Azure.Functions.Worker; using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Functions.Backup { /// /// Daily 04:00 UTC trigger of the critical-table backup feature. The TIMER is /// worker-equivalent topology-wise (Lease → Service → Release) but has no /// JobStatus + no queue pop: failure paths emit OpsEvents only, and the /// daily re-run is the natural recovery mechanism (no per-fire retries). /// public class CriticalTableBackupTimerFunction { private readonly BlobBackupStore _blobStore; private readonly ICriticalTableBackupService _backupService; private readonly OpsEventService _opsEvents; private readonly ILogger _logger; public CriticalTableBackupTimerFunction( BlobBackupStore blobStore, ICriticalTableBackupService backupService, OpsEventService opsEvents, ILogger logger) { _blobStore = blobStore; _backupService = backupService; _opsEvents = opsEvents; _logger = logger; } /// /// NCRONTAB "0 0 4 * * *" — every day at 04:00 UTC. After the daily /// VulnerabilityDataSync (03:00) and well outside the agent ingest peak. /// [Function("CriticalTableBackupTimer")] public async Task Run([TimerTrigger("0 0 4 * * *")] object timer, CancellationToken ct) { _logger.LogInformation("CriticalTableBackupTimer fired"); string triggeredBy = "Timer"; string? backupId = null; // ── Acquire maintenance lease ─────────────────────────────────────── // Codex-Hotfix #1: the lease must be renewed during the 50-min per-run budget; // otherwise the lease auto-expires after 60 s and another concurrent op could enter. await _blobStore.EnsureMaintenanceLockSentinelAsync(ct).ConfigureAwait(false); Azure.Storage.Blobs.Specialized.BlobLeaseClient? lease; try { lease = await _blobStore.AcquireMaintenanceLeaseAsync(BlobBackupStore.MaintenanceLeaseDuration, ct).ConfigureAwait(false); } catch (LeaseHeldException) { _logger.LogInformation("CriticalTableBackupTimer: maintenance lease held by another op — skipping today's run"); await TryRecordAsync(() => _opsEvents.RecordCriticalTableBackupSkippedLockedAsync( "another maintenance operation holds the lease at timer fire", triggeredBy)).ConfigureAwait(false); return; } using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(ct); var leaseHolder = new MaintenanceLeaseHolder(lease, handlerCts, _logger); try { var hct = handlerCts.Token; backupId = _backupService.GenerateBackupId(); BackupRunResult? result; try { result = await _backupService.RunBackupUnderLeaseAsync(backupId, triggeredBy, hct).ConfigureAwait(false); } catch (OperationCanceledException) when (handlerCts.IsCancellationRequested && leaseHolder.RenewalFailureReason is not null) { _logger.LogError("CriticalTableBackupTimer: maintenance lease lost mid-run (backupId={BackupId}) — {Reason}", backupId, leaseHolder.RenewalFailureReason); await TryRecordAsync(() => _opsEvents.RecordCriticalTableBackupFailedAsync( backupId, leaseHolder.RenewalFailureReason!, triggeredBy)).ConfigureAwait(false); return; } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogError(ex, "CriticalTableBackupTimer: backup run failed (backupId={BackupId})", backupId); await TryRecordAsync(() => _opsEvents.RecordCriticalTableBackupFailedAsync(backupId, ex.Message, triggeredBy)).ConfigureAwait(false); return; } var durationMs = (int)(result.Manifest.CompletedAtUtc - result.Manifest.StartedAtUtc).TotalMilliseconds; var failedOrSkipped = 0; foreach (var t in result.Manifest.Tables) { if (t.Status == TableBackupStatus.Failed || t.Status == TableBackupStatus.Skipped) failedOrSkipped++; } if (result.Outcome == BackupOutcome.Success) { await TryRecordAsync(() => _opsEvents.RecordCriticalTableBackupCompletedAsync( result.Manifest.BackupId, result.Manifest.Tables.Count, durationMs, Constants.BlobContainers.CriticalTableBackups, result.ManifestBlobName, triggeredBy)).ConfigureAwait(false); } else { await TryRecordAsync(() => _opsEvents.RecordCriticalTableBackupPartialAsync( result.Manifest.BackupId, result.Manifest.Tables.Count, failedOrSkipped, durationMs, Constants.BlobContainers.CriticalTableBackups, result.ManifestBlobName, triggeredBy)).ConfigureAwait(false); } } finally { await leaseHolder.DisposeAsync().ConfigureAwait(false); } } /// OpsEvent write must not bubble — backup itself already succeeded (or failed by other means). private async Task TryRecordAsync(Func writeAsync) { try { await writeAsync().ConfigureAwait(false); } catch (Exception ex) { _logger.LogWarning(ex, "CriticalTableBackupTimer: ops-event recording failed"); } } } }