mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
d4034842ee
Adds the full backup pipeline that snapshots 15 critical tables (Admin/Tenant config, rules, identities, audit, preview) into a logically-isolated blob container, daily at 04:00 UTC and on operator demand. Restore endpoints follow in PR2/PR3. Architecture: * Daily TimerTrigger + manual POST /api/global/backups/trigger (GA-only, fail-hard producer). HTTP returns 202 + jobId on enqueue, 500 if the queue write fails (and the BackupJobs row is rolled to Failed). * BackgroundService queue worker (not QueueTrigger — matches the Analyze/ IndexReconcile pattern that avoids the Functions-host connection-binding app-setting). Worker holds the maintenance lease for the whole run; service is intentionally lease-unaware so it stays unit-testable. * BlobBackupStore wraps the new 'critical-table-backups' container with a shared maintenance-lease sentinel at '_lock/maintenance.lock' that serialises every backup or restore op across worker/timer/single-row paths. * CriticalTableBackupService streams each table through CryptoStream(SHA256) into per-table NDJSON blobs, then writes manifest.json LAST as the durability anchor. Per-table failures degrade Outcome to Partial without aborting; per-run budget (50 min) → Skipped status. * BackupJobsRepository persists the six-state machine (Queued/Running/Completed/Failed/Skipped/BlockedTerminal) with ETag-CAS updates. Enum→string mapping rejects unknown values on read so refactors surface loudly instead of silently shifting state. * BackupJobWatchdog (30 min timer) heals stuck rows via a probe lease on the shared sentinel: Queued > 60 min → Failed, Running with stale heartbeat AND free lease → Failed (lease-busy means a live worker — leave alone). * OpsEvents: distinct CriticalTableBackupPartial (Warning) vs Failed (Error) so operators can wire softer Telegram rules for partial captures. * MaintenanceService.MonitoredPoisonQueues + OPS_EVENT_TYPES catalog registered (dual-register memory). Test coverage (15 new tests, 1952/1952 backend xUnit green): * BackupManifestJsonTests: enum-as-string (allowIntegerValues=false rejects numeric values), DictionaryKeyPolicy unset (Azure column names preserved case-faithfully), manifest field round-trip. * BackupJobsRepositoryMappingTests: full entity↔domain round-trip, unknown Kind/State/BackupOutcome rejected, legacy BackupOutcome="Failed" rejected. * CriticalTableBackupServiceTests: happy/partial/empty/budget-exceeded paths exercise the per-table loop with in-memory table + blob fakes. Setup: scripts/Backup/setup-backup-container.ps1 — idempotent az-CLI bootstrap for container + lifecycle rule (90 d delete on prefix). Soft-Delete and Versioning are NOT applied automatically (storage-account-wide blast radius — operator must validate per separate doc note). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
94 lines
4.0 KiB
C#
94 lines
4.0 KiB
C#
using System;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using AutopilotMonitor.Shared;
|
|
using Azure.Identity;
|
|
using Azure.Storage.Queues;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.Logging;
|
|
using Newtonsoft.Json;
|
|
|
|
namespace AutopilotMonitor.Functions.Services.Backup.Queue
|
|
{
|
|
/// <summary>
|
|
/// Azure-Storage-Queue producer for the critical-table backup feature.
|
|
/// Mirrors <c>AzureQueueAnalyzeOnEnrollmentEndProducer</c>'s connection setup
|
|
/// (Managed Identity preferred, connection string fallback) but flips the failure
|
|
/// semantics: SendMessageAsync errors propagate so the HTTP trigger can mark
|
|
/// the job Failed + return 5xx instead of leaving the operator with a 202 and
|
|
/// no actual work scheduled.
|
|
/// </summary>
|
|
public sealed class AzureQueueCriticalTableBackupProducer : ICriticalTableBackupProducer
|
|
{
|
|
private readonly QueueClient _queueClient;
|
|
private readonly ILogger<AzureQueueCriticalTableBackupProducer> _logger;
|
|
private int _queueEnsured;
|
|
|
|
public AzureQueueCriticalTableBackupProducer(
|
|
IConfiguration configuration,
|
|
ILogger<AzureQueueCriticalTableBackupProducer> logger)
|
|
{
|
|
_logger = logger;
|
|
|
|
var options = new QueueClientOptions
|
|
{
|
|
MessageEncoding = QueueMessageEncoding.Base64,
|
|
};
|
|
|
|
var storageAccountName = configuration["AzureStorageAccountName"];
|
|
var connectionString = configuration["AzureTableStorageConnectionString"];
|
|
|
|
if (!string.IsNullOrEmpty(storageAccountName))
|
|
{
|
|
var queueUri = new Uri(
|
|
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.CriticalTableBackup}");
|
|
_queueClient = new QueueClient(queueUri, new DefaultAzureCredential(), options);
|
|
_logger.LogInformation(
|
|
"CriticalTableBackup producer initialized with Managed Identity (account: {Account})",
|
|
storageAccountName);
|
|
}
|
|
else if (!string.IsNullOrEmpty(connectionString))
|
|
{
|
|
_queueClient = new QueueClient(
|
|
connectionString, Constants.QueueNames.CriticalTableBackup, options);
|
|
_logger.LogInformation("CriticalTableBackup producer initialized with connection string");
|
|
}
|
|
else
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Queue Storage not configured. Set either 'AzureStorageAccountName' or 'AzureTableStorageConnectionString'.");
|
|
}
|
|
}
|
|
|
|
public async Task EnqueueAsync(CriticalTableBackupEnvelope envelope, CancellationToken ct = default)
|
|
{
|
|
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
|
|
if (string.IsNullOrEmpty(envelope.JobId)) throw new ArgumentException("JobId required", nameof(envelope));
|
|
|
|
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
|
|
|
|
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and rolls the
|
|
// BackupJobStatus row to Failed + returns 500.
|
|
var body = JsonConvert.SerializeObject(envelope);
|
|
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
|
|
_logger.LogInformation("CriticalTableBackup enqueued (jobId={JobId})", envelope.JobId);
|
|
}
|
|
|
|
private async Task EnsureQueueExistsAsync(CancellationToken ct)
|
|
{
|
|
if (_queueEnsured == 1) return;
|
|
try
|
|
{
|
|
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
|
|
Interlocked.Exchange(ref _queueEnsured, 1);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
|
|
_logger.LogError(ex, "CriticalTableBackup queue CreateIfNotExists failed");
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
}
|