Files
AutopilotMonitor/src/Backend/AutopilotMonitor.Functions/Services/Backup/Queue/AzureQueueCriticalTableBackupProducer.cs
T
Oliver Kieselbach d4034842ee feat(backend/backup): PR1 — critical-table backup feature (daily timer + manual GA trigger)
Adds the full backup pipeline that snapshots 15 critical tables (Admin/Tenant
config, rules, identities, audit, preview) into a logically-isolated blob
container, daily at 04:00 UTC and on operator demand. Restore endpoints follow
in PR2/PR3.

Architecture:
* Daily TimerTrigger + manual POST /api/global/backups/trigger (GA-only,
  fail-hard producer). HTTP returns 202 + jobId on enqueue, 500 if the queue
  write fails (and the BackupJobs row is rolled to Failed).
* BackgroundService queue worker (not QueueTrigger — matches the Analyze/
  IndexReconcile pattern that avoids the Functions-host connection-binding
  app-setting). Worker holds the maintenance lease for the whole run; service
  is intentionally lease-unaware so it stays unit-testable.
* BlobBackupStore wraps the new 'critical-table-backups' container with a
  shared maintenance-lease sentinel at '_lock/maintenance.lock' that
  serialises every backup or restore op across worker/timer/single-row paths.
* CriticalTableBackupService streams each table through CryptoStream(SHA256)
  into per-table NDJSON blobs, then writes manifest.json LAST as the
  durability anchor. Per-table failures degrade Outcome to Partial without
  aborting; per-run budget (50 min) → Skipped status.
* BackupJobsRepository persists the six-state machine
  (Queued/Running/Completed/Failed/Skipped/BlockedTerminal) with ETag-CAS
  updates. Enum→string mapping rejects unknown values on read so refactors
  surface loudly instead of silently shifting state.
* BackupJobWatchdog (30 min timer) heals stuck rows via a probe lease on the
  shared sentinel: Queued > 60 min → Failed, Running with stale heartbeat
  AND free lease → Failed (lease-busy means a live worker — leave alone).
* OpsEvents: distinct CriticalTableBackupPartial (Warning) vs Failed (Error)
  so operators can wire softer Telegram rules for partial captures.
* MaintenanceService.MonitoredPoisonQueues + OPS_EVENT_TYPES catalog
  registered (dual-register memory).

Test coverage (15 new tests, 1952/1952 backend xUnit green):
* BackupManifestJsonTests: enum-as-string (allowIntegerValues=false rejects
  numeric values), DictionaryKeyPolicy unset (Azure column names preserved
  case-faithfully), manifest field round-trip.
* BackupJobsRepositoryMappingTests: full entity↔domain round-trip, unknown
  Kind/State/BackupOutcome rejected, legacy BackupOutcome="Failed" rejected.
* CriticalTableBackupServiceTests: happy/partial/empty/budget-exceeded paths
  exercise the per-table loop with in-memory table + blob fakes.

Setup: scripts/Backup/setup-backup-container.ps1 — idempotent az-CLI bootstrap
for container + lifecycle rule (90 d delete on prefix). Soft-Delete and
Versioning are NOT applied automatically (storage-account-wide blast radius
— operator must validate per separate doc note).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-26 15:19:07 +02:00

94 lines
4.0 KiB
C#

using System;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared;
using Azure.Identity;
using Azure.Storage.Queues;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Services.Backup.Queue
{
/// <summary>
/// Azure-Storage-Queue producer for the critical-table backup feature.
/// Mirrors <c>AzureQueueAnalyzeOnEnrollmentEndProducer</c>'s connection setup
/// (Managed Identity preferred, connection string fallback) but flips the failure
/// semantics: SendMessageAsync errors propagate so the HTTP trigger can mark
/// the job Failed + return 5xx instead of leaving the operator with a 202 and
/// no actual work scheduled.
/// </summary>
public sealed class AzureQueueCriticalTableBackupProducer : ICriticalTableBackupProducer
{
private readonly QueueClient _queueClient;
private readonly ILogger<AzureQueueCriticalTableBackupProducer> _logger;
private int _queueEnsured;
public AzureQueueCriticalTableBackupProducer(
IConfiguration configuration,
ILogger<AzureQueueCriticalTableBackupProducer> logger)
{
_logger = logger;
var options = new QueueClientOptions
{
MessageEncoding = QueueMessageEncoding.Base64,
};
var storageAccountName = configuration["AzureStorageAccountName"];
var connectionString = configuration["AzureTableStorageConnectionString"];
if (!string.IsNullOrEmpty(storageAccountName))
{
var queueUri = new Uri(
$"https://{storageAccountName}.queue.core.windows.net/{Constants.QueueNames.CriticalTableBackup}");
_queueClient = new QueueClient(queueUri, new DefaultAzureCredential(), options);
_logger.LogInformation(
"CriticalTableBackup producer initialized with Managed Identity (account: {Account})",
storageAccountName);
}
else if (!string.IsNullOrEmpty(connectionString))
{
_queueClient = new QueueClient(
connectionString, Constants.QueueNames.CriticalTableBackup, options);
_logger.LogInformation("CriticalTableBackup producer initialized with connection string");
}
else
{
throw new InvalidOperationException(
"Queue Storage not configured. Set either 'AzureStorageAccountName' or 'AzureTableStorageConnectionString'.");
}
}
public async Task EnqueueAsync(CriticalTableBackupEnvelope envelope, CancellationToken ct = default)
{
if (envelope == null) throw new ArgumentNullException(nameof(envelope));
if (string.IsNullOrEmpty(envelope.JobId)) throw new ArgumentException("JobId required", nameof(envelope));
await EnsureQueueExistsAsync(ct).ConfigureAwait(false);
// Fail-hard: do NOT swallow. Caller (HTTP trigger) catches and rolls the
// BackupJobStatus row to Failed + returns 500.
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(body, ct).ConfigureAwait(false);
_logger.LogInformation("CriticalTableBackup enqueued (jobId={JobId})", envelope.JobId);
}
private async Task EnsureQueueExistsAsync(CancellationToken ct)
{
if (_queueEnsured == 1) return;
try
{
await _queueClient.CreateIfNotExistsAsync(cancellationToken: ct).ConfigureAwait(false);
Interlocked.Exchange(ref _queueEnsured, 1);
}
catch (Exception ex)
{
// Ensure failure during enqueue is itself a fail-hard signal — surface to the caller.
_logger.LogError(ex, "CriticalTableBackup queue CreateIfNotExists failed");
throw;
}
}
}
}