mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
d4034842ee
Adds the full backup pipeline that snapshots 15 critical tables (Admin/Tenant config, rules, identities, audit, preview) into a logically-isolated blob container, daily at 04:00 UTC and on operator demand. Restore endpoints follow in PR2/PR3. Architecture: * Daily TimerTrigger + manual POST /api/global/backups/trigger (GA-only, fail-hard producer). HTTP returns 202 + jobId on enqueue, 500 if the queue write fails (and the BackupJobs row is rolled to Failed). * BackgroundService queue worker (not QueueTrigger — matches the Analyze/ IndexReconcile pattern that avoids the Functions-host connection-binding app-setting). Worker holds the maintenance lease for the whole run; service is intentionally lease-unaware so it stays unit-testable. * BlobBackupStore wraps the new 'critical-table-backups' container with a shared maintenance-lease sentinel at '_lock/maintenance.lock' that serialises every backup or restore op across worker/timer/single-row paths. * CriticalTableBackupService streams each table through CryptoStream(SHA256) into per-table NDJSON blobs, then writes manifest.json LAST as the durability anchor. Per-table failures degrade Outcome to Partial without aborting; per-run budget (50 min) → Skipped status. * BackupJobsRepository persists the six-state machine (Queued/Running/Completed/Failed/Skipped/BlockedTerminal) with ETag-CAS updates. Enum→string mapping rejects unknown values on read so refactors surface loudly instead of silently shifting state. * BackupJobWatchdog (30 min timer) heals stuck rows via a probe lease on the shared sentinel: Queued > 60 min → Failed, Running with stale heartbeat AND free lease → Failed (lease-busy means a live worker — leave alone). * OpsEvents: distinct CriticalTableBackupPartial (Warning) vs Failed (Error) so operators can wire softer Telegram rules for partial captures. * MaintenanceService.MonitoredPoisonQueues + OPS_EVENT_TYPES catalog registered (dual-register memory). Test coverage (15 new tests, 1952/1952 backend xUnit green): * BackupManifestJsonTests: enum-as-string (allowIntegerValues=false rejects numeric values), DictionaryKeyPolicy unset (Azure column names preserved case-faithfully), manifest field round-trip. * BackupJobsRepositoryMappingTests: full entity↔domain round-trip, unknown Kind/State/BackupOutcome rejected, legacy BackupOutcome="Failed" rejected. * CriticalTableBackupServiceTests: happy/partial/empty/budget-exceeded paths exercise the per-table loop with in-memory table + blob fakes. Setup: scripts/Backup/setup-backup-container.ps1 — idempotent az-CLI bootstrap for container + lifecycle rule (90 d delete on prefix). Soft-Delete and Versioning are NOT applied automatically (storage-account-wide blast radius — operator must validate per separate doc note). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
51 lines
2.2 KiB
C#
51 lines
2.2 KiB
C#
using System.Net;
|
|
using System.Text.Json;
|
|
using System.Threading.Tasks;
|
|
using AutopilotMonitor.Functions.Services.Backup;
|
|
using AutopilotMonitor.Shared.Models.Backup;
|
|
using Microsoft.Azure.Functions.Worker;
|
|
using Microsoft.Azure.Functions.Worker.Http;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace AutopilotMonitor.Functions.Functions.Backup
|
|
{
|
|
/// <summary>
|
|
/// <c>GET /api/global/backups/{backupId}</c> — fetches the manifest JSON for a
|
|
/// specific backup run. 404 if the manifest is missing (i.e. the prefix exists but
|
|
/// the durability anchor never got written — treated as <c>Incomplete</c> per plan).
|
|
/// GA-only.
|
|
/// </summary>
|
|
public class GetBackupManifestFunction
|
|
{
|
|
private readonly BlobBackupStore _store;
|
|
private readonly ILogger<GetBackupManifestFunction> _logger;
|
|
|
|
public GetBackupManifestFunction(BlobBackupStore store, ILogger<GetBackupManifestFunction> logger)
|
|
{
|
|
_store = store;
|
|
_logger = logger;
|
|
}
|
|
|
|
[Function("GetBackupManifest")]
|
|
public async Task<HttpResponseData> Run(
|
|
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "global/backups/{backupId}")] HttpRequestData req,
|
|
string backupId)
|
|
{
|
|
var (payload, _) = await _store.ReadManifestAsync(backupId, req.FunctionContext.CancellationToken);
|
|
if (payload is null)
|
|
{
|
|
var notFound = req.CreateResponse(HttpStatusCode.NotFound);
|
|
notFound.Headers.Add("Content-Type", "application/json; charset=utf-8");
|
|
await notFound.WriteStringAsync(JsonSerializer.Serialize(new { error = "ManifestNotFound", backupId }, BackupManifestJson.SerializerOptions));
|
|
return notFound;
|
|
}
|
|
|
|
// Stream the bytes verbatim — the manifest is already JSON, no need to deserialise + re-serialise.
|
|
var response = req.CreateResponse(HttpStatusCode.OK);
|
|
response.Headers.Add("Content-Type", "application/json; charset=utf-8");
|
|
await response.Body.WriteAsync(payload, req.FunctionContext.CancellationToken);
|
|
return response;
|
|
}
|
|
}
|
|
}
|