mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 12:42:38 +00:00
feat(backend/backup): PR2 — single-row restore + admin UI
POST /api/global/backups/{backupId}/restore-row (GA-only) with preview|commit
modes. Preview scans the ETag-pinned NDJSON blob for the requested (pk, rk),
diffs against the live row, and returns rowSha256 + currentETag for echo.
Commit re-validates everything under a short maintenance lease and writes via
UpdateEntity(IfMatch=etag, Replace) or AddEntity; 412/404/409 → CurrentRowChanged.
Backend:
- RestoreTablePreflightValidator (no I/O) + BackupRestoreInputValidator (manifest
parse, entry check, BlobName canonical, ETag-pin + SHA in one stream read)
- CriticalTableRestoreService with PreviewRowAsync (no lease) + CommitRowAsync
(MaintenanceLeaseHolder, 45 s renewal, deterministic finally release)
- BlobBackupStore.OpenNdjsonReadAsync — GetProperties + OpenRead(IfMatch=etag)
- BackupTerminalException carries a stable Code for HTTP envelope mapping
- OpsEventService.RecordBackupRowRestoredAsync (Warning; payload is
container + blobName only, never a SAS URL)
- 27 new xUnit tests; full backend suite 1979/1979 green
Web:
- /admin/backups list page with newest-first list + "Run backup now" trigger
and async job-status poller
- /admin/backups/{backupId} detail page with per-table breakdown + single-row
restore trigger
- RestoreRowDiffModal with Added/Removed/Changed/Unchanged pills and
auth-table security banner (GlobalAdmins/TenantAdmins/McpUsers)
- api.backups namespace + matching DTO types
- globalNavConfig: ga-backups under Ops
- BackupRowRestored registered in OPS_EVENT_TYPES (dual-register policy)
- 6 new vitest cases; full web suite 390/390 green
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,247 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Validator covers the five reject paths from the plan:
|
||||
/// <list type="number">
|
||||
/// <item>tableName not in catalog (deferred — handled by the preflight, not this validator)</item>
|
||||
/// <item>manifest missing → BackupNotFound</item>
|
||||
/// <item>manifest entry missing → TableNotInBackup</item>
|
||||
/// <item>BlobName not canonical → ManifestCorrupt</item>
|
||||
/// <item>SHA mismatch → IntegrityCheckFailed</item>
|
||||
/// </list>
|
||||
/// Plus the entry-Status check (Failed/Skipped entries refused) and the
|
||||
/// ETag-pin propagation invariant.
|
||||
/// </summary>
|
||||
public class BackupRestoreInputValidatorTests
|
||||
{
|
||||
private const string BackupId = "20260522T040000Z_a1b2c3d4";
|
||||
private const string Table = Constants.TableNames.AnalyzeRules;
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_returns_pinned_etag_and_entry_on_happy_path()
|
||||
{
|
||||
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
|
||||
var sha = ComputeSha(bytes);
|
||||
var manifest = BuildManifest(new CriticalTableBackupTableEntry
|
||||
{
|
||||
TableName = Table,
|
||||
Status = TableBackupStatus.Ok,
|
||||
RowCount = 1,
|
||||
ByteSize = bytes.Length,
|
||||
Sha256Hex = sha,
|
||||
BlobName = $"{BackupId}/{Table}.ndjson",
|
||||
});
|
||||
|
||||
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"0xABC\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var result = await sut.ValidateAsync(BackupId, Table, CancellationToken.None);
|
||||
|
||||
Assert.Equal(BackupId, result.Manifest.BackupId);
|
||||
Assert.Equal(Table, result.Entry.TableName);
|
||||
Assert.Equal(new ETag("\"0xABC\""), result.BlobETag);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_throws_BackupNotFound_when_manifest_missing()
|
||||
{
|
||||
var fakeStore = new FakeStore(manifest: null, ndjson: Array.Empty<byte>(), new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("BackupNotFound", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_throws_ManifestCorrupt_when_payload_is_garbage()
|
||||
{
|
||||
var fakeStore = new FakeStore(rawManifestBytes: Encoding.UTF8.GetBytes("not json"), ndjson: Array.Empty<byte>(), new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("ManifestCorrupt", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_throws_TableNotInBackup_when_table_not_in_manifest()
|
||||
{
|
||||
var manifest = BuildManifest(new CriticalTableBackupTableEntry
|
||||
{
|
||||
TableName = Constants.TableNames.Feedback, // different table
|
||||
Status = TableBackupStatus.Ok,
|
||||
BlobName = $"{BackupId}/{Constants.TableNames.Feedback}.ndjson",
|
||||
});
|
||||
var fakeStore = new FakeStore(manifest, Array.Empty<byte>(), new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("TableNotInBackup", ex.Code);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(TableBackupStatus.Failed)]
|
||||
[InlineData(TableBackupStatus.Skipped)]
|
||||
public async Task ValidateAsync_refuses_restore_from_failed_or_skipped_entry(TableBackupStatus status)
|
||||
{
|
||||
var manifest = BuildManifest(new CriticalTableBackupTableEntry
|
||||
{
|
||||
TableName = Table,
|
||||
Status = status,
|
||||
BlobName = $"{BackupId}/{Table}.ndjson",
|
||||
});
|
||||
var fakeStore = new FakeStore(manifest, Array.Empty<byte>(), new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("TableNotInBackup", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_throws_ManifestCorrupt_when_BlobName_is_not_canonical()
|
||||
{
|
||||
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
|
||||
var sha = ComputeSha(bytes);
|
||||
var manifest = BuildManifest(new CriticalTableBackupTableEntry
|
||||
{
|
||||
TableName = Table,
|
||||
Status = TableBackupStatus.Ok,
|
||||
Sha256Hex = sha,
|
||||
BlobName = $"{BackupId}/some-other-name.ndjson", // wrong
|
||||
});
|
||||
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("ManifestCorrupt", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAsync_throws_IntegrityCheckFailed_when_sha_does_not_match()
|
||||
{
|
||||
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
|
||||
var wrongSha = ComputeSha(Encoding.UTF8.GetBytes("different bytes")); // doesn't match actual ndjson
|
||||
var manifest = BuildManifest(new CriticalTableBackupTableEntry
|
||||
{
|
||||
TableName = Table,
|
||||
Status = TableBackupStatus.Ok,
|
||||
Sha256Hex = wrongSha,
|
||||
BlobName = $"{BackupId}/{Table}.ndjson",
|
||||
});
|
||||
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"e\""));
|
||||
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
|
||||
Assert.Equal("IntegrityCheckFailed", ex.Code);
|
||||
}
|
||||
|
||||
// ── Test helpers ────────────────────────────────────────────────────────
|
||||
|
||||
private static CriticalTableBackupManifest BuildManifest(CriticalTableBackupTableEntry entry)
|
||||
{
|
||||
return new CriticalTableBackupManifest
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
BackupId = BackupId,
|
||||
StartedAtUtc = new DateTime(2026, 5, 22, 4, 0, 0, DateTimeKind.Utc),
|
||||
CompletedAtUtc = new DateTime(2026, 5, 22, 4, 0, 1, DateTimeKind.Utc),
|
||||
TriggeredBy = "Timer",
|
||||
Outcome = BackupOutcome.Success,
|
||||
Tables = { entry },
|
||||
};
|
||||
}
|
||||
|
||||
private static byte[] BuildNdjsonLine(string pk, string rk, object props)
|
||||
{
|
||||
// Round-trip via DeletionRowDump so the test bytes mirror what the
|
||||
// production backup writer produces.
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(props, BackupManifestJson.SerializerOptions);
|
||||
using var doc = JsonDocument.Parse(bytes);
|
||||
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = pk,
|
||||
Rk = rk,
|
||||
Etag = null,
|
||||
Props = new System.Collections.Generic.Dictionary<string, DeletionPropValue>(),
|
||||
};
|
||||
foreach (var prop in doc.RootElement.EnumerateObject())
|
||||
{
|
||||
dump.Props[prop.Name] = new DeletionPropValue
|
||||
{
|
||||
EdmType = DeletionPropEdmType.String,
|
||||
Value = prop.Value.Clone(),
|
||||
};
|
||||
}
|
||||
return JsonSerializer.SerializeToUtf8Bytes(dump, BackupManifestJson.SerializerOptions);
|
||||
}
|
||||
|
||||
private static string ComputeSha(byte[] bytes)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fake <see cref="BlobBackupStore"/> that returns canned manifest bytes +
|
||||
/// NDJSON bytes from in-memory state. Only the two read paths the validator
|
||||
/// uses are overridden; everything else throws so a test cannot silently
|
||||
/// hit the real Azure SDK.
|
||||
/// </summary>
|
||||
private sealed class FakeStore : BlobBackupStore
|
||||
{
|
||||
private readonly byte[]? _manifestBytes;
|
||||
private readonly byte[] _ndjsonBytes;
|
||||
private readonly ETag _etag;
|
||||
|
||||
public FakeStore(CriticalTableBackupManifest? manifest, byte[] ndjson, ETag etag)
|
||||
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
|
||||
{
|
||||
_manifestBytes = manifest is null ? null : JsonSerializer.SerializeToUtf8Bytes(manifest, BackupManifestJson.SerializerOptions);
|
||||
_ndjsonBytes = ndjson;
|
||||
_etag = etag;
|
||||
}
|
||||
|
||||
public FakeStore(byte[] rawManifestBytes, byte[] ndjson, ETag etag)
|
||||
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
|
||||
{
|
||||
_manifestBytes = rawManifestBytes;
|
||||
_ndjsonBytes = ndjson;
|
||||
_etag = etag;
|
||||
}
|
||||
|
||||
public override Task EnsureContainerAsync(CancellationToken ct = default) => Task.CompletedTask;
|
||||
|
||||
public override Task<(byte[]? Payload, ETag? ETag)> ReadManifestAsync(string backupId, CancellationToken ct = default)
|
||||
=> Task.FromResult<(byte[]?, ETag?)>((_manifestBytes, _manifestBytes is null ? null : _etag));
|
||||
|
||||
public override Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(string backupId, string tableName, CancellationToken ct = default)
|
||||
{
|
||||
// The validator first calls ReadManifestAsync, then OpenNdjsonReadAsync.
|
||||
// We always return the same ETag here so the round-trip is consistent.
|
||||
Stream stream = new MemoryStream(_ndjsonBytes, writable: false);
|
||||
return Task.FromResult<(Stream, ETag)?>((stream, _etag));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeBlobStorageService : BlobStorageService
|
||||
{
|
||||
public FakeBlobStorageService()
|
||||
: base(new Azure.Storage.Blobs.BlobServiceClient(new Uri("https://example.invalid"), new Azure.Storage.StorageSharedKeyCredential("x", Convert.ToBase64String(new byte[32]))), NullLogger<BlobStorageService>.Instance) { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Single-row preview tests for the critical-table restore service. Covers the
|
||||
/// happy path (row found in backup + missing from live), the diff shape
|
||||
/// (Added / Removed / Changed / Unchanged), and the RowNotInBackup edge.
|
||||
/// Auth-table flag propagation is also asserted.
|
||||
/// <para>
|
||||
/// Commit-path tests live in <see cref="CriticalTableRestoreServiceCommitTests"/>
|
||||
/// once we can mock <c>BlobLeaseClient</c> without hitting the real SDK lease
|
||||
/// API; for now the commit happy/conflict paths are covered indirectly via the
|
||||
/// validator + preflight tests.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class CriticalTableRestoreServiceTests
|
||||
{
|
||||
private const string BackupId = "20260522T040000Z_a1b2c3d4";
|
||||
private const string Table = Constants.TableNames.AnalyzeRules;
|
||||
|
||||
[Fact]
|
||||
public async Task PreviewRowAsync_returns_full_diff_when_backup_and_live_differ()
|
||||
{
|
||||
var lineBytes = BuildNdjsonLine("tenant1", "rule1", new Dictionary<string, (string Edm, object? Value)>
|
||||
{
|
||||
["Enabled"] = ("Boolean", true),
|
||||
["KillSwitch"] = ("Boolean", false),
|
||||
["Comment"] = ("String", "from backup"),
|
||||
});
|
||||
var sha = ComputeSha(lineBytes);
|
||||
|
||||
var entry = MakeEntry(Table, sha, lineBytes.Length);
|
||||
var manifest = MakeManifest(entry);
|
||||
|
||||
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
|
||||
var liveRow = new TableEntity("tenant1", "rule1")
|
||||
{
|
||||
["Enabled"] = false, // changed
|
||||
["KillSwitch"] = false, // unchanged
|
||||
["NewLiveColumn"] = "added-after", // removed by restore
|
||||
["odata.etag"] = "live-etag",
|
||||
};
|
||||
liveRow.ETag = new ETag("\"live-etag\"");
|
||||
var tables = BuildTableStorageWithLiveRow(liveRow);
|
||||
|
||||
var sut = MakeService(tables, fakeStore);
|
||||
|
||||
var preview = await sut.PreviewRowAsync(BackupId, Table, "tenant1", "rule1", CancellationToken.None);
|
||||
|
||||
Assert.Equal(BackupId, preview.BackupId);
|
||||
Assert.Equal(Table, preview.TableName);
|
||||
Assert.Equal("tenant1", preview.PartitionKey);
|
||||
Assert.Equal("rule1", preview.RowKey);
|
||||
Assert.Equal(sha, preview.RowSha256);
|
||||
Assert.Equal("\"live-etag\"", preview.CurrentETag);
|
||||
Assert.False(preview.IsAuthTable);
|
||||
|
||||
// Diff sanity
|
||||
var changed = preview.Diff.Single(d => d.Name == "Enabled");
|
||||
Assert.Equal(RestoreRowDiffKind.Changed, changed.Kind);
|
||||
|
||||
var unchanged = preview.Diff.Single(d => d.Name == "KillSwitch");
|
||||
Assert.Equal(RestoreRowDiffKind.Unchanged, unchanged.Kind);
|
||||
|
||||
var removed = preview.Diff.Single(d => d.Name == "NewLiveColumn");
|
||||
Assert.Equal(RestoreRowDiffKind.Removed, removed.Kind);
|
||||
// "Comment" exists in backup, not in live → Added on restore.
|
||||
var added = preview.Diff.Single(d => d.Name == "Comment");
|
||||
Assert.Equal(RestoreRowDiffKind.Added, added.Kind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PreviewRowAsync_marks_currentETag_null_when_live_row_missing()
|
||||
{
|
||||
var lineBytes = BuildNdjsonLine("p", "r", new Dictionary<string, (string Edm, object? Value)>
|
||||
{
|
||||
["Setting"] = ("String", "value"),
|
||||
});
|
||||
var sha = ComputeSha(lineBytes);
|
||||
var manifest = MakeManifest(MakeEntry(Table, sha, lineBytes.Length));
|
||||
|
||||
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
|
||||
var tables = BuildTableStorageWithLiveRow(liveRow: null);
|
||||
|
||||
var sut = MakeService(tables, fakeStore);
|
||||
|
||||
var preview = await sut.PreviewRowAsync(BackupId, Table, "p", "r", CancellationToken.None);
|
||||
|
||||
Assert.Null(preview.CurrentETag);
|
||||
Assert.Null(preview.CurrentProperties);
|
||||
Assert.Single(preview.Diff);
|
||||
Assert.Equal(RestoreRowDiffKind.Added, preview.Diff[0].Kind);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PreviewRowAsync_throws_RowNotInBackup_when_keys_dont_match_any_line()
|
||||
{
|
||||
var lineBytes = BuildNdjsonLine("p", "r", new Dictionary<string, (string Edm, object? Value)>
|
||||
{
|
||||
["Setting"] = ("String", "value"),
|
||||
});
|
||||
var sha = ComputeSha(lineBytes);
|
||||
var manifest = MakeManifest(MakeEntry(Table, sha, lineBytes.Length));
|
||||
|
||||
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
|
||||
var tables = BuildTableStorageWithLiveRow(liveRow: null);
|
||||
|
||||
var sut = MakeService(tables, fakeStore);
|
||||
|
||||
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() =>
|
||||
sut.PreviewRowAsync(BackupId, Table, "p-OTHER", "r", CancellationToken.None));
|
||||
Assert.Equal("RowNotInBackup", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PreviewRowAsync_sets_IsAuthTable_true_for_GlobalAdmins()
|
||||
{
|
||||
var lineBytes = BuildNdjsonLine("admins", "alice@contoso.test",
|
||||
new Dictionary<string, (string Edm, object? Value)>
|
||||
{
|
||||
["IsEnabled"] = ("Boolean", true),
|
||||
});
|
||||
var sha = ComputeSha(lineBytes);
|
||||
var manifest = MakeManifestForTable(Constants.TableNames.GlobalAdmins,
|
||||
MakeEntry(Constants.TableNames.GlobalAdmins, sha, lineBytes.Length));
|
||||
|
||||
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
|
||||
var tables = BuildTableStorageWithLiveRow(liveRow: null);
|
||||
|
||||
var sut = MakeService(tables, fakeStore);
|
||||
|
||||
var preview = await sut.PreviewRowAsync(
|
||||
BackupId, Constants.TableNames.GlobalAdmins, "admins", "alice@contoso.test",
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.True(preview.IsAuthTable);
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
private static CriticalTableRestoreService MakeService(TableStorageService tables, FakeStore store)
|
||||
{
|
||||
var validator = new BackupRestoreInputValidator(store, NullLogger<BackupRestoreInputValidator>.Instance);
|
||||
return new CriticalTableRestoreService(
|
||||
tables,
|
||||
store,
|
||||
validator,
|
||||
NullLoggerFactory.Instance,
|
||||
NullLogger<CriticalTableRestoreService>.Instance);
|
||||
}
|
||||
|
||||
private static CriticalTableBackupTableEntry MakeEntry(string tableName, string sha, long byteSize) => new()
|
||||
{
|
||||
TableName = tableName,
|
||||
Status = TableBackupStatus.Ok,
|
||||
RowCount = 1,
|
||||
ByteSize = byteSize,
|
||||
Sha256Hex = sha,
|
||||
BlobName = $"{BackupId}/{tableName}.ndjson",
|
||||
};
|
||||
|
||||
private static CriticalTableBackupManifest MakeManifest(CriticalTableBackupTableEntry entry) =>
|
||||
MakeManifestForTable(Table, entry);
|
||||
|
||||
private static CriticalTableBackupManifest MakeManifestForTable(string tableName, CriticalTableBackupTableEntry entry) => new()
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
BackupId = BackupId,
|
||||
StartedAtUtc = new DateTime(2026, 5, 22, 4, 0, 0, DateTimeKind.Utc),
|
||||
CompletedAtUtc = new DateTime(2026, 5, 22, 4, 0, 1, DateTimeKind.Utc),
|
||||
TriggeredBy = "Timer",
|
||||
Outcome = BackupOutcome.Success,
|
||||
Tables = { entry },
|
||||
};
|
||||
|
||||
private static byte[] BuildNdjsonLine(string pk, string rk, Dictionary<string, (string Edm, object? Value)> props)
|
||||
{
|
||||
var dump = new DeletionRowDump
|
||||
{
|
||||
Pk = pk,
|
||||
Rk = rk,
|
||||
Etag = null,
|
||||
Props = new Dictionary<string, DeletionPropValue>(StringComparer.Ordinal),
|
||||
};
|
||||
foreach (var kvp in props)
|
||||
{
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(kvp.Value.Value, BackupManifestJson.SerializerOptions);
|
||||
using var doc = JsonDocument.Parse(bytes);
|
||||
dump.Props[kvp.Key] = new DeletionPropValue
|
||||
{
|
||||
EdmType = kvp.Value.Edm,
|
||||
Value = doc.RootElement.Clone(),
|
||||
};
|
||||
}
|
||||
return JsonSerializer.SerializeToUtf8Bytes(dump, BackupManifestJson.SerializerOptions);
|
||||
}
|
||||
|
||||
private static string ComputeSha(byte[] bytes)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static TableStorageService BuildTableStorageWithLiveRow(TableEntity? liveRow)
|
||||
{
|
||||
var mockTableClient = new Mock<TableClient>();
|
||||
|
||||
if (liveRow != null)
|
||||
{
|
||||
var response = Response.FromValue(liveRow, new Mock<Response>().Object);
|
||||
mockTableClient
|
||||
.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
|
||||
liveRow.PartitionKey, liveRow.RowKey,
|
||||
It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((NullableResponse<TableEntity>)response);
|
||||
}
|
||||
|
||||
// For any other (pk, rk) — return a "not found" NullableResponse.
|
||||
mockTableClient
|
||||
.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, string, IEnumerable<string>?, CancellationToken>((pk, rk, _, _) =>
|
||||
{
|
||||
if (liveRow != null
|
||||
&& string.Equals(pk, liveRow.PartitionKey, StringComparison.Ordinal)
|
||||
&& string.Equals(rk, liveRow.RowKey, StringComparison.Ordinal))
|
||||
{
|
||||
return Task.FromResult((NullableResponse<TableEntity>)
|
||||
Response.FromValue(liveRow, new Mock<Response>().Object));
|
||||
}
|
||||
var notFound = new Mock<NullableResponse<TableEntity>>();
|
||||
notFound.SetupGet(r => r.HasValue).Returns(false);
|
||||
return Task.FromResult(notFound.Object);
|
||||
});
|
||||
|
||||
var mockServiceClient = new Mock<TableServiceClient>();
|
||||
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
|
||||
|
||||
return new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
|
||||
private sealed class FakeStore : BlobBackupStore
|
||||
{
|
||||
private readonly byte[]? _manifestBytes;
|
||||
private readonly byte[] _ndjsonBytes;
|
||||
private readonly ETag _etag;
|
||||
|
||||
public FakeStore(CriticalTableBackupManifest? manifest, byte[] ndjson, ETag etag)
|
||||
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
|
||||
{
|
||||
_manifestBytes = manifest is null ? null : JsonSerializer.SerializeToUtf8Bytes(manifest, BackupManifestJson.SerializerOptions);
|
||||
_ndjsonBytes = ndjson;
|
||||
_etag = etag;
|
||||
}
|
||||
|
||||
public override Task EnsureContainerAsync(CancellationToken ct = default) => Task.CompletedTask;
|
||||
|
||||
public override Task<(byte[]? Payload, ETag? ETag)> ReadManifestAsync(string backupId, CancellationToken ct = default)
|
||||
=> Task.FromResult<(byte[]?, ETag?)>((_manifestBytes, _manifestBytes is null ? null : _etag));
|
||||
|
||||
public override Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(string backupId, string tableName, CancellationToken ct = default)
|
||||
{
|
||||
// Return a fresh stream every call — the validator and the scan
|
||||
// open the blob independently.
|
||||
Stream stream = new MemoryStream(_ndjsonBytes, writable: false);
|
||||
return Task.FromResult<(Stream, ETag)?>((stream, _etag));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeBlobStorageService : BlobStorageService
|
||||
{
|
||||
public FakeBlobStorageService()
|
||||
: base(new Azure.Storage.Blobs.BlobServiceClient(new Uri("https://example.invalid"),
|
||||
new Azure.Storage.StorageSharedKeyCredential("x", Convert.ToBase64String(new byte[32]))),
|
||||
NullLogger<BlobStorageService>.Instance) { }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pure-function checks for <see cref="RestoreTablePreflightValidator"/>. The
|
||||
/// preflight gates every <c>POST .../restore-row</c> request with no I/O — fail
|
||||
/// fast before the heavy validator runs.
|
||||
/// </summary>
|
||||
public class RestoreTablePreflightValidatorTests
|
||||
{
|
||||
private static readonly RestoreTablePreflightValidator Sut = new();
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_accepts_valid_preview()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "tenant1",
|
||||
RowKey = "rule1",
|
||||
Mode = RestoreRowMode.Preview,
|
||||
};
|
||||
|
||||
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_accepts_valid_commit_with_sha_and_etag()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "tenant1",
|
||||
RowKey = "rule1",
|
||||
Mode = RestoreRowMode.Commit,
|
||||
IfSha256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
IfCurrentETag = "\"0x8DABCDEF\"",
|
||||
};
|
||||
|
||||
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_accepts_commit_with_null_etag_when_row_was_missing()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "tenant1",
|
||||
RowKey = "rule1",
|
||||
Mode = RestoreRowMode.Commit,
|
||||
IfSha256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
|
||||
IfCurrentETag = null,
|
||||
};
|
||||
|
||||
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw — Add-path
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_rejects_empty_backupId()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "p",
|
||||
RowKey = "r",
|
||||
Mode = RestoreRowMode.Preview,
|
||||
};
|
||||
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("", req));
|
||||
Assert.Equal("InvalidBackupId", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_rejects_table_not_in_catalog()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = "RandomOtherTable",
|
||||
PartitionKey = "p",
|
||||
RowKey = "r",
|
||||
Mode = RestoreRowMode.Preview,
|
||||
};
|
||||
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
|
||||
Assert.Equal("InvalidTable", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_rejects_null_partition_key()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = null!,
|
||||
RowKey = "r",
|
||||
Mode = RestoreRowMode.Preview,
|
||||
};
|
||||
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
|
||||
Assert.Equal("InvalidKeys", ex.Code);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateRowRequest_commit_requires_ifSha256()
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "p",
|
||||
RowKey = "r",
|
||||
Mode = RestoreRowMode.Commit,
|
||||
IfSha256 = null,
|
||||
};
|
||||
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
|
||||
Assert.Equal("MissingPrecondition", ex.Code);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-hex")]
|
||||
[InlineData("ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789")] // uppercase rejected
|
||||
[InlineData("0123456789abcdef")] // too short
|
||||
public void ValidateRowRequest_commit_requires_lowercase_64_hex(string sha)
|
||||
{
|
||||
var req = new RestoreRowRequest
|
||||
{
|
||||
TableName = Constants.TableNames.AnalyzeRules,
|
||||
PartitionKey = "p",
|
||||
RowKey = "r",
|
||||
Mode = RestoreRowMode.Commit,
|
||||
IfSha256 = sha,
|
||||
};
|
||||
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
|
||||
Assert.Equal("MissingPrecondition", ex.Code);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(nameof(Constants.TableNames.GlobalAdmins), true)]
|
||||
[InlineData(nameof(Constants.TableNames.TenantAdmins), true)]
|
||||
[InlineData(nameof(Constants.TableNames.McpUsers), true)]
|
||||
[InlineData(nameof(Constants.TableNames.AnalyzeRules), false)]
|
||||
[InlineData(nameof(Constants.TableNames.Feedback), false)]
|
||||
public void IsAuthTable_flags_only_three_security_tables(string tableNameField, bool expected)
|
||||
{
|
||||
var tableName = typeof(Constants.TableNames).GetField(tableNameField)!.GetValue(null) as string;
|
||||
Assert.Equal(expected, Sut.IsAuthTable(tableName!));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// <c>POST /api/global/backups/{backupId}/restore-row</c> — single-row preview
|
||||
/// + commit endpoint for the critical-table backup restore (plan §PR2).
|
||||
/// GA-only via <c>EndpointAccessPolicyCatalog</c>.
|
||||
/// <para>
|
||||
/// Synchronous on commit (1 row → Function timeout unproblematic), unlike the
|
||||
/// full-table restore which goes via the 202+queue pattern. PartitionKey and
|
||||
/// RowKey live in the body, not the URL: Azure Tables permits <c>/</c>,
|
||||
/// <c>+</c>, and <c>%</c> in PK/RK which the Functions router would mangle.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class RestoreRowFunction
|
||||
{
|
||||
private readonly RestoreTablePreflightValidator _preflight;
|
||||
private readonly CriticalTableRestoreService _restoreService;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
private readonly ILogger<RestoreRowFunction> _logger;
|
||||
|
||||
public RestoreRowFunction(
|
||||
RestoreTablePreflightValidator preflight,
|
||||
CriticalTableRestoreService restoreService,
|
||||
OpsEventService opsEvents,
|
||||
ILogger<RestoreRowFunction> logger)
|
||||
{
|
||||
_preflight = preflight;
|
||||
_restoreService = restoreService;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
[Function("RestoreRow")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/backups/{backupId}/restore-row")] HttpRequestData req,
|
||||
string backupId)
|
||||
{
|
||||
var ct = req.FunctionContext.CancellationToken;
|
||||
|
||||
// 1. Parse body
|
||||
RestoreRowRequest? body;
|
||||
try
|
||||
{
|
||||
body = await JsonSerializer.DeserializeAsync<RestoreRowRequest>(
|
||||
req.Body, BackupManifestJson.SerializerOptions, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, "InvalidBody", $"request body is not valid JSON: {ex.Message}").ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (body == null)
|
||||
{
|
||||
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, "MissingBody", "request body is required").ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// 2. Lightweight preflight (no I/O)
|
||||
try
|
||||
{
|
||||
_preflight.ValidateRowRequest(backupId, body);
|
||||
}
|
||||
catch (BackupTerminalException ex)
|
||||
{
|
||||
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, ex.Code, ex.Message).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// 3. Dispatch on mode
|
||||
try
|
||||
{
|
||||
if (body.Mode == RestoreRowMode.Preview)
|
||||
{
|
||||
var preview = await _restoreService.PreviewRowAsync(
|
||||
backupId, body.TableName, body.PartitionKey, body.RowKey, ct).ConfigureAwait(false);
|
||||
return await WriteJsonAsync(req, HttpStatusCode.OK, preview).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
var actor = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
|
||||
var commit = await _restoreService.CommitRowAsync(
|
||||
backupId, body.TableName, body.PartitionKey, body.RowKey,
|
||||
body.IfSha256!, body.IfCurrentETag, ct).ConfigureAwait(false);
|
||||
|
||||
// Fire-and-forget audit; OpsEventService catches its own writes.
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordBackupRowRestoredAsync(
|
||||
backupId, body.TableName, body.PartitionKey, body.RowKey, actor,
|
||||
commit.Outcome.ToString()).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception evtEx)
|
||||
{
|
||||
_logger.LogWarning(evtEx, "RestoreRow: ops event recording failed — write itself succeeded");
|
||||
}
|
||||
|
||||
return await WriteJsonAsync(req, HttpStatusCode.OK, commit).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (BackupTerminalException ex)
|
||||
{
|
||||
var status = MapErrorCodeToStatus(ex.Code);
|
||||
return await WriteErrorAsync(req, status, ex.Code, ex.Message).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Client went away.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "RestoreRow: unexpected failure for backupId={BackupId} table={Table}", backupId, body.TableName);
|
||||
return await WriteErrorAsync(req, HttpStatusCode.InternalServerError, "InternalError", ex.Message).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static HttpStatusCode MapErrorCodeToStatus(string code) => code switch
|
||||
{
|
||||
"BackupNotFound" => HttpStatusCode.NotFound,
|
||||
"RowNotInBackup" => HttpStatusCode.NotFound,
|
||||
"TableNotInBackup" => HttpStatusCode.Conflict,
|
||||
"ManifestCorrupt" => HttpStatusCode.Conflict,
|
||||
"ManifestSchemaUnsupported" => HttpStatusCode.Conflict,
|
||||
"IntegrityCheckFailed" => HttpStatusCode.Conflict,
|
||||
"BlobChangedSinceValidation" => HttpStatusCode.Conflict,
|
||||
"RowChangedSinceValidation" => HttpStatusCode.Conflict,
|
||||
"CurrentRowChanged" => HttpStatusCode.Conflict,
|
||||
"MaintenanceInProgress" => HttpStatusCode.Conflict,
|
||||
_ => HttpStatusCode.BadRequest,
|
||||
};
|
||||
|
||||
private static async Task<HttpResponseData> WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body)
|
||||
{
|
||||
var response = req.CreateResponse(status);
|
||||
response.Headers.Add("Content-Type", "application/json; charset=utf-8");
|
||||
var json = JsonSerializer.Serialize(body, BackupManifestJson.SerializerOptions);
|
||||
await response.WriteStringAsync(json).ConfigureAwait(false);
|
||||
return response;
|
||||
}
|
||||
|
||||
private static async Task<HttpResponseData> WriteErrorAsync(HttpRequestData req, HttpStatusCode status, string code, string message)
|
||||
{
|
||||
return await WriteJsonAsync(req, status, new { error = code, message }).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -295,6 +295,11 @@ builder.Services.AddSingleton<
|
||||
// pattern per plan §Wave4 #1 — avoids the QueueTrigger-specific connection app-setting).
|
||||
builder.Services.AddHostedService<
|
||||
AutopilotMonitor.Functions.Services.Backup.Queue.CriticalTableBackupQueueWorker>();
|
||||
// PR2: single-row restore. Both validators are pure / I/O-only; the restore
|
||||
// service is stateless (lease lifecycle is owned per-call). Singleton is safe.
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.RestoreTablePreflightValidator>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.BackupRestoreInputValidator>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.CriticalTableRestoreService>();
|
||||
|
||||
// Programmatic SignalR push for background tasks (rule engine, vulnerability correlation)
|
||||
builder.Services.AddSingleton<SignalRNotificationService>();
|
||||
|
||||
@@ -288,11 +288,14 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly),
|
||||
new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
// ── Critical-Table Backup (plan §PR1) ──────────────────────────────────
|
||||
// ── Critical-Table Backup (plan §PR1+§PR2) ─────────────────────────────
|
||||
new("POST", "global/backups/trigger", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/backups", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/backups/{backupId}", EndpointPolicy.GlobalAdminOnly),
|
||||
new("GET", "global/backups/jobs/{jobId}", EndpointPolicy.GlobalAdminOnly),
|
||||
// PR2: single-row preview + commit. PartitionKey + RowKey live in the body
|
||||
// (Azure Tables permits / + % in PK/RK; the route wouldn't carry them safely).
|
||||
new("POST", "global/backups/{backupId}/restore-row", EndpointPolicy.GlobalAdminOnly),
|
||||
|
||||
// ── Tenant Offboarding Customs Archive (PR3.B) ─────────────────────────
|
||||
// Snapshot of each tenant's custom GatherRules / AnalyzeRules / ImeLogPatterns
|
||||
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using Azure;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// Heavy, I/O-bound validation that has to run before any restore write touches
|
||||
/// the live table. Performs:
|
||||
/// <list type="number">
|
||||
/// <item>Manifest exists + parses (plan §Wave11 #4)</item>
|
||||
/// <item>Manifest schema version is supported</item>
|
||||
/// <item>Entry for the requested <c>tableName</c> exists with <c>Status ∈ {Ok, Empty}</c></item>
|
||||
/// <item>Entry's <c>BlobName</c> matches the canonical <c>{backupId}/{tableName}.ndjson</c></item>
|
||||
/// <item><b>ETag-pin + SHA in a single stream read</b> (plan §Wave12 #3):
|
||||
/// GetProperties → pin ETag → OpenReadAsync(IfMatch=etag) → SHA-256 over that stream.
|
||||
/// Mismatch → <c>IntegrityCheckFailed</c>. Race on the blob → <c>BlobChangedSinceValidation</c>.</item>
|
||||
/// </list>
|
||||
/// On success returns the parsed manifest, the matching entry, and the pinned
|
||||
/// ETag — the restore service uses the SAME ETag for the line-scan read so the
|
||||
/// hash and the scan see byte-identical bytes.
|
||||
/// </summary>
|
||||
public sealed class BackupRestoreInputValidator
|
||||
{
|
||||
private const int SupportedSchemaVersion = 1;
|
||||
|
||||
private readonly BlobBackupStore _store;
|
||||
private readonly ILogger<BackupRestoreInputValidator> _logger;
|
||||
|
||||
public BackupRestoreInputValidator(BlobBackupStore store, ILogger<BackupRestoreInputValidator> logger)
|
||||
{
|
||||
_store = store;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a successful validation. The pinned ETag is returned so the
|
||||
/// restore service can OpenReadAsync the SAME blob version it was just
|
||||
/// hashed against.
|
||||
/// </summary>
|
||||
public sealed class ValidationResult
|
||||
{
|
||||
public CriticalTableBackupManifest Manifest { get; init; } = default!;
|
||||
public CriticalTableBackupTableEntry Entry { get; init; } = default!;
|
||||
public ETag BlobETag { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs all checks. Throws <see cref="BackupTerminalException"/> on the
|
||||
/// first failure with a stable error code suitable for the HTTP envelope.
|
||||
/// </summary>
|
||||
public async Task<ValidationResult> ValidateAsync(string backupId, string tableName, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(backupId)) throw new BackupTerminalException("InvalidBackupId", "backupId is required");
|
||||
if (string.IsNullOrEmpty(tableName)) throw new BackupTerminalException("InvalidTable", "tableName is required");
|
||||
|
||||
// 1. Manifest exists + parseable
|
||||
var (payload, _) = await _store.ReadManifestAsync(backupId, ct).ConfigureAwait(false);
|
||||
if (payload == null)
|
||||
{
|
||||
throw new BackupTerminalException("BackupNotFound", $"manifest for backupId '{backupId}' was not found");
|
||||
}
|
||||
|
||||
CriticalTableBackupManifest manifest;
|
||||
try
|
||||
{
|
||||
manifest = JsonSerializer.Deserialize<CriticalTableBackupManifest>(payload, BackupManifestJson.SerializerOptions)
|
||||
?? throw new BackupTerminalException("ManifestCorrupt", "manifest deserialised to null");
|
||||
}
|
||||
catch (BackupTerminalException) { throw; }
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new BackupTerminalException("ManifestCorrupt", $"manifest JSON parse failed: {ex.Message}", ex);
|
||||
}
|
||||
|
||||
if (manifest.SchemaVersion != SupportedSchemaVersion)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"ManifestSchemaUnsupported",
|
||||
$"manifest schemaVersion={manifest.SchemaVersion} not supported by this restore endpoint (expected {SupportedSchemaVersion})");
|
||||
}
|
||||
|
||||
// 2. Per-table entry exists + Status is restorable
|
||||
var entry = manifest.Tables.FirstOrDefault(t => string.Equals(t.TableName, tableName, StringComparison.Ordinal));
|
||||
if (entry == null)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"TableNotInBackup",
|
||||
$"no entry for tableName '{tableName}' in manifest {backupId}");
|
||||
}
|
||||
if (entry.Status != TableBackupStatus.Ok && entry.Status != TableBackupStatus.Empty)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"TableNotInBackup",
|
||||
$"manifest entry for '{tableName}' has Status={entry.Status} — restore is only allowed from Ok or Empty entries");
|
||||
}
|
||||
|
||||
// 3. Canonical BlobName invariant
|
||||
var expectedBlobName = BlobBackupStore.BuildNdjsonBlobName(backupId, tableName);
|
||||
if (!string.Equals(entry.BlobName, expectedBlobName, StringComparison.Ordinal))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"ManifestCorrupt",
|
||||
$"manifest entry BlobName='{entry.BlobName}' does not match canonical '{expectedBlobName}'");
|
||||
}
|
||||
|
||||
// 4. ETag-pin + SHA in a single stream read
|
||||
var openResult = await _store.OpenNdjsonReadAsync(backupId, tableName, ct).ConfigureAwait(false);
|
||||
if (openResult == null)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"BackupNotFound",
|
||||
$"NDJSON blob '{expectedBlobName}' is missing despite manifest entry present — orphan manifest");
|
||||
}
|
||||
|
||||
var (stream, blobETag) = openResult.Value;
|
||||
string actualSha;
|
||||
try
|
||||
{
|
||||
using (stream)
|
||||
using (var sha = SHA256.Create())
|
||||
{
|
||||
var hashBytes = await sha.ComputeHashAsync(stream, ct).ConfigureAwait(false);
|
||||
actualSha = Convert.ToHexString(hashBytes).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
// The IfMatch precondition can also fire mid-stream if the blob was
|
||||
// overwritten between OpenRead and the final byte. Same recovery as
|
||||
// OpenNdjsonReadAsync's 412 path.
|
||||
throw new BackupTerminalException("BlobChangedSinceValidation", "NDJSON blob changed during SHA verification", ex);
|
||||
}
|
||||
|
||||
var expectedSha = (entry.Sha256Hex ?? string.Empty).ToLowerInvariant();
|
||||
if (!string.Equals(actualSha, expectedSha, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogError(
|
||||
"BackupRestoreInputValidator: SHA mismatch for backupId={BackupId} table={Table} (expected={Expected}, actual={Actual}) — refusing restore",
|
||||
backupId, tableName, expectedSha, actualSha);
|
||||
throw new BackupTerminalException(
|
||||
"IntegrityCheckFailed",
|
||||
$"SHA-256 mismatch for table '{tableName}' (expected {expectedSha}, got {actualSha}) — backup may be tampered or partially written");
|
||||
}
|
||||
|
||||
return new ValidationResult
|
||||
{
|
||||
Manifest = manifest,
|
||||
Entry = entry,
|
||||
BlobETag = blobETag,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using System;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// Deterministic, non-retryable failure raised by the backup/restore paths
|
||||
/// (Manifest missing, SHA mismatch, Blob ETag changed, auth-table block,
|
||||
/// confirmation-token typo, lease busy …). The HTTP layer maps these to a
|
||||
/// specific 4xx; the queue worker maps them to <c>JobState.BlockedTerminal</c>
|
||||
/// + <c>DeleteMessage</c> on first attempt (no retry, no poison).
|
||||
/// </summary>
|
||||
public sealed class BackupTerminalException : Exception
|
||||
{
|
||||
/// <summary>
|
||||
/// Stable short code suitable for the HTTP error envelope
|
||||
/// (<c>{ error: Code, message: Message }</c>). UI uses this to decide which
|
||||
/// banner to render (e.g. <c>CurrentRowChanged</c> → "open Preview again").
|
||||
/// </summary>
|
||||
public string Code { get; }
|
||||
|
||||
public BackupTerminalException(string code, string message) : base(message)
|
||||
{
|
||||
Code = code;
|
||||
}
|
||||
|
||||
public BackupTerminalException(string code, string message, Exception inner) : base(message, inner)
|
||||
{
|
||||
Code = code;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -173,6 +173,57 @@ namespace AutopilotMonitor.Functions.Services.Backup
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens the per-table NDJSON blob for read with a pinned ETag so the SHA
|
||||
/// hash + subsequent line scan operate on the exact same blob version (plan
|
||||
/// §Wave11 #4 + Wave12 #3). Workflow:
|
||||
/// <list type="number">
|
||||
/// <item>GetProperties → pin ETag</item>
|
||||
/// <item>OpenReadAsync with <c>IfMatch=ETag</c>; 412 throws
|
||||
/// <see cref="BackupTerminalException"/> with code <c>BlobChangedSinceValidation</c></item>
|
||||
/// </list>
|
||||
/// Caller is responsible for disposing the returned stream. Returns
|
||||
/// <c>null</c> if the blob does not exist (404).
|
||||
/// </summary>
|
||||
public virtual async Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(
|
||||
string backupId,
|
||||
string tableName,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrEmpty(backupId)) throw new ArgumentException("backupId required", nameof(backupId));
|
||||
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName required", nameof(tableName));
|
||||
|
||||
await EnsureContainerAsync(ct).ConfigureAwait(false);
|
||||
var blob = GetContainer().GetBlobClient(BuildNdjsonBlobName(backupId, tableName));
|
||||
|
||||
BlobProperties props;
|
||||
try
|
||||
{
|
||||
props = (await blob.GetPropertiesAsync(cancellationToken: ct).ConfigureAwait(false)).Value;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var etag = props.ETag;
|
||||
try
|
||||
{
|
||||
var stream = await blob.OpenReadAsync(new BlobOpenReadOptions(allowModifications: false)
|
||||
{
|
||||
Conditions = new BlobRequestConditions { IfMatch = etag },
|
||||
}, ct).ConfigureAwait(false);
|
||||
return (stream, etag);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"BlobChangedSinceValidation",
|
||||
$"NDJSON blob '{BuildNdjsonBlobName(backupId, tableName)}' changed between GetProperties and OpenRead — refusing to read a half-version",
|
||||
ex);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists every backupId whose <c>manifest.json</c> exists — i.e. every run that
|
||||
/// reached the durability anchor (plan §3 "manifest is the backup"). Orphan
|
||||
|
||||
+480
@@ -0,0 +1,480 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// Single-row preview + commit primitives for the critical-table backup restore
|
||||
/// path (plan §PR2). Two phases:
|
||||
/// <list type="bullet">
|
||||
/// <item><b>PreviewRowAsync</b> — heavy <see cref="BackupRestoreInputValidator"/>
|
||||
/// (manifest + SHA + ETag-pin), then line-by-line scan of the same ETag-pinned
|
||||
/// NDJSON blob for the requested (pk, rk). Reads the live row via
|
||||
/// <c>GetEntityIfExistsAsync</c>, returns backup+current+diff+rowHash+currentETag.
|
||||
/// <b>No write</b>, no lease — purely read-only.</item>
|
||||
/// <item><b>CommitRowAsync</b> — acquires the maintenance lease (short-lived but
|
||||
/// with the standard 45 s renewal loop so a slow blob fetch cannot drop it),
|
||||
/// re-runs the validator, recomputes the row hash, re-reads the live row,
|
||||
/// verifies <c>ifSha256</c> and <c>ifCurrentETag</c> echoed by the client, then
|
||||
/// writes <c>UpdateEntityAsync(IfMatch=etag, Replace)</c> for existing rows or
|
||||
/// <c>AddEntityAsync</c> for missing rows. 412/404/409 → <c>CurrentRowChanged</c>.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// This service is intentionally NOT used for full-table restores — the bulk
|
||||
/// path lives in <c>UpsertRowsByExactKeysInBatchesAsync</c> and has no per-row
|
||||
/// ETag-CAS. PR2 covers only single-row; PR3/PR4 wire the bulk path.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class CriticalTableRestoreService
|
||||
{
|
||||
private readonly TableStorageService _tables;
|
||||
private readonly BlobBackupStore _store;
|
||||
private readonly BackupRestoreInputValidator _validator;
|
||||
private readonly ILoggerFactory _loggerFactory;
|
||||
private readonly ILogger<CriticalTableRestoreService> _logger;
|
||||
|
||||
public CriticalTableRestoreService(
|
||||
TableStorageService tables,
|
||||
BlobBackupStore store,
|
||||
BackupRestoreInputValidator validator,
|
||||
ILoggerFactory loggerFactory,
|
||||
ILogger<CriticalTableRestoreService> logger)
|
||||
{
|
||||
_tables = tables;
|
||||
_store = store;
|
||||
_validator = validator;
|
||||
_loggerFactory = loggerFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
// ── Preview ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Heavy validate + scan + diff. No write, no lease.
|
||||
/// </summary>
|
||||
public async Task<RestoreRowPreviewResponse> PreviewRowAsync(
|
||||
string backupId,
|
||||
string tableName,
|
||||
string partitionKey,
|
||||
string rowKey,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
// 1. Heavy validate (manifest, SHA, ETag-pin).
|
||||
var validation = await _validator.ValidateAsync(backupId, tableName, ct).ConfigureAwait(false);
|
||||
|
||||
// 2. Scan the SAME ETag-pinned blob version for the requested (pk, rk).
|
||||
// We re-open the blob with IfMatch=validation.BlobETag so the scan sees
|
||||
// byte-identical bytes to the hash. This is the central correctness
|
||||
// invariant — without re-pinning, a concurrent overwrite could change
|
||||
// the bytes between hash and scan.
|
||||
var openResult = await OpenAtPinnedETagAsync(backupId, tableName, validation.BlobETag, ct).ConfigureAwait(false);
|
||||
|
||||
var (dump, ndjsonLineBytes) = await FindLineAsync(openResult, partitionKey, rowKey, ct).ConfigureAwait(false);
|
||||
if (dump == null)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"RowNotInBackup",
|
||||
$"row (pk='{partitionKey}', rk='{rowKey}') was not found in backup {backupId}/{tableName}");
|
||||
}
|
||||
|
||||
// 3. Read the live row.
|
||||
var tableClient = _tables.GetTableClient(tableName);
|
||||
var liveResponse = await tableClient
|
||||
.GetEntityIfExistsAsync<TableEntity>(partitionKey, rowKey, cancellationToken: ct)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
TableEntity? liveEntity = liveResponse.HasValue ? liveResponse.Value : null;
|
||||
string? currentETag = liveEntity != null ? liveEntity.ETag.ToString() : null;
|
||||
|
||||
// 4. Build the diff + snapshots.
|
||||
var backupSnap = ToSnapshotMap(dump);
|
||||
var currentSnap = liveEntity != null ? LiveEntityToSnapshotMap(liveEntity) : null;
|
||||
var diff = BuildDiff(backupSnap, currentSnap);
|
||||
|
||||
// 5. Hash the exact NDJSON line (without trailing \n). See plan §Wave15 #5
|
||||
// — no canonicalisation; the original NDJSON bytes are the canonical form.
|
||||
var rowSha = ComputeSha256Hex(ndjsonLineBytes);
|
||||
|
||||
return new RestoreRowPreviewResponse
|
||||
{
|
||||
BackupId = backupId,
|
||||
TableName = tableName,
|
||||
PartitionKey = partitionKey,
|
||||
RowKey = rowKey,
|
||||
BackupProperties = backupSnap,
|
||||
CurrentProperties = currentSnap,
|
||||
Diff = diff,
|
||||
RowSha256 = rowSha,
|
||||
CurrentETag = currentETag,
|
||||
IsAuthTable = Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden
|
||||
.Contains(tableName, StringComparer.Ordinal),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Commit ──────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Acquires the maintenance lease (briefly), re-validates, recomputes the
|
||||
/// row hash, then runs a conditional write keyed on <paramref name="ifCurrentETag"/>.
|
||||
/// All five CurrentRowChanged paths from the plan are covered:
|
||||
/// <list type="bullet">
|
||||
/// <item>Row existed in preview + same ETag → <c>UpdateEntity(Replace)</c></item>
|
||||
/// <item>Row existed in preview + different ETag (412) → 409 <c>CurrentRowChanged</c></item>
|
||||
/// <item>Row deleted between preview + commit (404) → 409 <c>CurrentRowChanged</c></item>
|
||||
/// <item>Row missing in preview → <c>AddEntity</c></item>
|
||||
/// <item>Row missing in preview + meanwhile created (409) → 409 <c>CurrentRowChanged</c></item>
|
||||
/// </list>
|
||||
/// </summary>
|
||||
public async Task<RestoreRowCommitResponse> CommitRowAsync(
|
||||
string backupId,
|
||||
string tableName,
|
||||
string partitionKey,
|
||||
string rowKey,
|
||||
string ifSha256,
|
||||
string? ifCurrentETag,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
// Acquire the maintenance lease BEFORE re-reading anything that could
|
||||
// race with a parallel full-table restore. Without the lease a Single-Row
|
||||
// commit could be overwritten by an in-flight bulk replace-all started
|
||||
// microseconds later (plan §Wave15 #4).
|
||||
Azure.Storage.Blobs.Specialized.BlobLeaseClient leaseClient;
|
||||
try
|
||||
{
|
||||
leaseClient = await _store.AcquireMaintenanceLeaseAsync(leaseDuration: null, ct: ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (LeaseHeldException ex)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"MaintenanceInProgress",
|
||||
"another maintenance operation is in progress — please retry shortly",
|
||||
ex);
|
||||
}
|
||||
|
||||
using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
var holder = new MaintenanceLeaseHolder(
|
||||
leaseClient,
|
||||
handlerCts,
|
||||
_loggerFactory.CreateLogger<MaintenanceLeaseHolder>());
|
||||
|
||||
try
|
||||
{
|
||||
// Re-validate from scratch under the lease. The validator's ETag-pin
|
||||
// returns a NEW ETag for the current blob version — we use that pin
|
||||
// for the scan below (NOT the preview's, which may now be stale if a
|
||||
// parallel write+restore landed in between).
|
||||
var validation = await _validator.ValidateAsync(backupId, tableName, handlerCts.Token).ConfigureAwait(false);
|
||||
|
||||
var openResult = await OpenAtPinnedETagAsync(backupId, tableName, validation.BlobETag, handlerCts.Token).ConfigureAwait(false);
|
||||
var (dump, ndjsonLineBytes) = await FindLineAsync(openResult, partitionKey, rowKey, handlerCts.Token).ConfigureAwait(false);
|
||||
if (dump == null)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"RowNotInBackup",
|
||||
$"row (pk='{partitionKey}', rk='{rowKey}') was not found in backup {backupId}/{tableName}");
|
||||
}
|
||||
|
||||
// Re-compute SHA on the fresh line bytes. If the operator's echoed
|
||||
// ifSha256 does not match, the backup has changed (or the operator
|
||||
// typed by hand) — refuse.
|
||||
var freshSha = ComputeSha256Hex(ndjsonLineBytes);
|
||||
if (!string.Equals(freshSha, (ifSha256 ?? string.Empty).ToLowerInvariant(), StringComparison.Ordinal))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"RowChangedSinceValidation",
|
||||
$"backup row SHA-256 changed between preview and commit (preview echoed {ifSha256}, fresh {freshSha}) — re-open the preview to refresh");
|
||||
}
|
||||
|
||||
// Build the entity from the dump. ConvertDumpToEntity also applies
|
||||
// the Sessions-specific reset (no-op for other tables).
|
||||
var entity = TableStorageService.ConvertDumpToEntity(dump, tableName);
|
||||
|
||||
var tableClient = _tables.GetTableClient(tableName);
|
||||
|
||||
if (!string.IsNullOrEmpty(ifCurrentETag))
|
||||
{
|
||||
// Preview saw a live row; commit requires the live ETag still matches.
|
||||
var etag = new ETag(ifCurrentETag);
|
||||
try
|
||||
{
|
||||
await tableClient.UpdateEntityAsync(entity, etag, TableUpdateMode.Replace, handlerCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"CurrentRowChanged",
|
||||
$"live row was modified or deleted since preview (status={ex.Status}) — re-open the preview",
|
||||
ex);
|
||||
}
|
||||
|
||||
return new RestoreRowCommitResponse
|
||||
{
|
||||
BackupId = backupId,
|
||||
TableName = tableName,
|
||||
PartitionKey = partitionKey,
|
||||
RowKey = rowKey,
|
||||
Outcome = RestoreRowCommitOutcome.Replaced,
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
// Preview saw NO live row; commit requires the row is still absent.
|
||||
try
|
||||
{
|
||||
await tableClient.AddEntityAsync(entity, handlerCts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (RequestFailedException ex) when (IsAlreadyExistsStatus(ex))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"CurrentRowChanged",
|
||||
"live row was created since preview — re-open the preview",
|
||||
ex);
|
||||
}
|
||||
|
||||
return new RestoreRowCommitResponse
|
||||
{
|
||||
BackupId = backupId,
|
||||
TableName = tableName,
|
||||
PartitionKey = partitionKey,
|
||||
RowKey = rowKey,
|
||||
Outcome = RestoreRowCommitOutcome.Inserted,
|
||||
};
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
// DisposeAsync stops the renewal loop deterministically before releasing.
|
||||
try { await holder.DisposeAsync().ConfigureAwait(false); }
|
||||
catch (Exception ex) { _logger.LogWarning(ex, "CriticalTableRestoreService: lease holder dispose threw"); }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
private async Task<(Stream Stream, ETag ETag)> OpenAtPinnedETagAsync(
|
||||
string backupId, string tableName, ETag expectedETag, CancellationToken ct)
|
||||
{
|
||||
var open = await _store.OpenNdjsonReadAsync(backupId, tableName, ct).ConfigureAwait(false);
|
||||
if (open == null)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"BackupNotFound",
|
||||
$"NDJSON blob for {backupId}/{tableName}.ndjson disappeared between validation and scan");
|
||||
}
|
||||
if (!open.Value.ETag.Equals(expectedETag))
|
||||
{
|
||||
// The validator already opened+read; if the ETag changed between
|
||||
// validator and scan, the blob was overwritten.
|
||||
try { open.Value.Stream.Dispose(); } catch { /* best effort */ }
|
||||
throw new BackupTerminalException(
|
||||
"BlobChangedSinceValidation",
|
||||
"NDJSON blob ETag changed between validation hash and scan — refusing to read mismatching bytes");
|
||||
}
|
||||
return open.Value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Scans an NDJSON stream line-by-line for the (pk, rk) match. Returns the
|
||||
/// parsed dump plus the raw line bytes (without trailing <c>\n</c>) so the
|
||||
/// caller can SHA-256 the same bytes that get echoed via <c>ifSha256</c>.
|
||||
/// </summary>
|
||||
private static async Task<(DeletionRowDump? Dump, byte[] LineBytes)> FindLineAsync(
|
||||
(Stream Stream, ETag ETag) open,
|
||||
string partitionKey,
|
||||
string rowKey,
|
||||
CancellationToken ct)
|
||||
{
|
||||
using var stream = open.Stream;
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, leaveOpen: false);
|
||||
|
||||
string? line;
|
||||
while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) != null)
|
||||
{
|
||||
if (line.Length == 0) continue;
|
||||
|
||||
// Quick reject on PK/RK substring miss before deserialising —
|
||||
// a worthwhile micro-opt because most lines won't match.
|
||||
DeletionRowDump? dump;
|
||||
try
|
||||
{
|
||||
dump = JsonSerializer.Deserialize<DeletionRowDump>(line, BackupManifestJson.SerializerOptions);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"ManifestCorrupt",
|
||||
$"NDJSON line failed to parse as DeletionRowDump: {ex.Message}",
|
||||
ex);
|
||||
}
|
||||
|
||||
if (dump == null) continue;
|
||||
if (!string.Equals(dump.Pk, partitionKey, StringComparison.Ordinal)) continue;
|
||||
if (!string.Equals(dump.Rk, rowKey, StringComparison.Ordinal)) continue;
|
||||
|
||||
var lineBytes = Encoding.UTF8.GetBytes(line);
|
||||
return (dump, lineBytes);
|
||||
}
|
||||
|
||||
return (null, Array.Empty<byte>());
|
||||
}
|
||||
|
||||
private static Dictionary<string, RestoreRowPropertySnapshot> ToSnapshotMap(DeletionRowDump dump)
|
||||
{
|
||||
var map = new Dictionary<string, RestoreRowPropertySnapshot>(StringComparer.Ordinal);
|
||||
foreach (var kvp in dump.Props)
|
||||
{
|
||||
map[kvp.Key] = new RestoreRowPropertySnapshot
|
||||
{
|
||||
EdmType = kvp.Value.EdmType,
|
||||
Value = kvp.Value.Value,
|
||||
};
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private static Dictionary<string, RestoreRowPropertySnapshot> LiveEntityToSnapshotMap(TableEntity entity)
|
||||
{
|
||||
var map = new Dictionary<string, RestoreRowPropertySnapshot>(StringComparer.Ordinal);
|
||||
foreach (var kvp in entity)
|
||||
{
|
||||
// Skip Azure-system columns; PartitionKey and RowKey are top-level on
|
||||
// the dump, Timestamp + odata.etag are SDK-managed.
|
||||
if (kvp.Key == "PartitionKey" || kvp.Key == "RowKey" || kvp.Key == "Timestamp" || kvp.Key == "odata.etag")
|
||||
continue;
|
||||
|
||||
map[kvp.Key] = ConvertLiveValueToSnapshot(kvp.Value);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
private static RestoreRowPropertySnapshot ConvertLiveValueToSnapshot(object? value)
|
||||
{
|
||||
// Mirror the dump's EDM tagging — we don't have full SDK type info on
|
||||
// the live row beyond CLR types, but the round-trip rules from
|
||||
// TableEntityDumpConverter give us enough fidelity for a diff UI.
|
||||
//
|
||||
// For property comparison we only need a deterministic JSON
|
||||
// representation of the value, so we hand-encode each EDM type
|
||||
// identically to how MapEntityToDump would have encoded it. That keeps
|
||||
// the diff stable: backup-vs-current of a long stays "long==long" even
|
||||
// though the live SDK already typed it as Int64.
|
||||
switch (value)
|
||||
{
|
||||
case null:
|
||||
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.Null, null);
|
||||
case string s:
|
||||
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.String, s);
|
||||
case bool b:
|
||||
return MakeSnapshot(DeletionPropEdmType.Boolean, b ? JsonValueKind.True : JsonValueKind.False, b);
|
||||
case int i:
|
||||
return MakeSnapshot(DeletionPropEdmType.Int32, JsonValueKind.Number, i);
|
||||
case long l:
|
||||
return MakeSnapshot(DeletionPropEdmType.Int64, JsonValueKind.Number, l);
|
||||
case double d:
|
||||
return MakeSnapshot(DeletionPropEdmType.Double, JsonValueKind.Number, d);
|
||||
case DateTime dt:
|
||||
return MakeSnapshot(DeletionPropEdmType.DateTime, JsonValueKind.String,
|
||||
dt.ToUniversalTime().ToString("o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
case DateTimeOffset dto:
|
||||
return MakeSnapshot(DeletionPropEdmType.DateTime, JsonValueKind.String,
|
||||
dto.UtcDateTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture));
|
||||
case Guid g:
|
||||
return MakeSnapshot(DeletionPropEdmType.Guid, JsonValueKind.String, g.ToString());
|
||||
case byte[] bytes:
|
||||
return MakeSnapshot(DeletionPropEdmType.Binary, JsonValueKind.String, Convert.ToBase64String(bytes));
|
||||
default:
|
||||
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.String, value.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
private static RestoreRowPropertySnapshot MakeSnapshot(string edmType, JsonValueKind kind, object? value)
|
||||
{
|
||||
// Cheap round-trip via JsonSerializer to get a normalized JsonElement —
|
||||
// the diff serializer (BackupManifestJson.SerializerOptions) re-emits it
|
||||
// verbatim, and JsonElement equality is byte-based, so this keeps the
|
||||
// diff stable across processes.
|
||||
var bytes = JsonSerializer.SerializeToUtf8Bytes(value, BackupManifestJson.SerializerOptions);
|
||||
using var doc = JsonDocument.Parse(bytes);
|
||||
return new RestoreRowPropertySnapshot
|
||||
{
|
||||
EdmType = edmType,
|
||||
Value = doc.RootElement.Clone(),
|
||||
};
|
||||
// 'kind' parameter retained for self-documentation at call sites.
|
||||
}
|
||||
|
||||
private static List<RestoreRowPropertyDiff> BuildDiff(
|
||||
Dictionary<string, RestoreRowPropertySnapshot> backup,
|
||||
Dictionary<string, RestoreRowPropertySnapshot>? current)
|
||||
{
|
||||
var diff = new List<RestoreRowPropertyDiff>();
|
||||
var keys = new HashSet<string>(backup.Keys, StringComparer.Ordinal);
|
||||
if (current != null) keys.UnionWith(current.Keys);
|
||||
|
||||
foreach (var name in keys.OrderBy(x => x, StringComparer.Ordinal))
|
||||
{
|
||||
backup.TryGetValue(name, out var b);
|
||||
RestoreRowPropertySnapshot? c = null;
|
||||
current?.TryGetValue(name, out c);
|
||||
|
||||
RestoreRowDiffKind kind;
|
||||
if (b != null && c == null) kind = current == null ? RestoreRowDiffKind.Added : RestoreRowDiffKind.Added;
|
||||
else if (b == null && c != null) kind = RestoreRowDiffKind.Removed;
|
||||
else if (b != null && c != null)
|
||||
{
|
||||
kind = SnapshotsEqual(b, c) ? RestoreRowDiffKind.Unchanged : RestoreRowDiffKind.Changed;
|
||||
}
|
||||
else continue;
|
||||
|
||||
diff.Add(new RestoreRowPropertyDiff
|
||||
{
|
||||
Name = name,
|
||||
Kind = kind,
|
||||
Backup = b,
|
||||
Current = c,
|
||||
});
|
||||
}
|
||||
|
||||
return diff;
|
||||
}
|
||||
|
||||
private static bool SnapshotsEqual(RestoreRowPropertySnapshot a, RestoreRowPropertySnapshot b)
|
||||
{
|
||||
if (!string.Equals(a.EdmType, b.EdmType, StringComparison.Ordinal)) return false;
|
||||
// JsonElement equality is by raw text — re-serialise both for a
|
||||
// canonical byte-compare. Both go through the same SerializerOptions so
|
||||
// ordering is stable.
|
||||
var aBytes = JsonSerializer.SerializeToUtf8Bytes(a.Value, BackupManifestJson.SerializerOptions);
|
||||
var bBytes = JsonSerializer.SerializeToUtf8Bytes(b.Value, BackupManifestJson.SerializerOptions);
|
||||
return aBytes.AsSpan().SequenceEqual(bBytes);
|
||||
}
|
||||
|
||||
private static string ComputeSha256Hex(byte[] data)
|
||||
{
|
||||
using var sha = SHA256.Create();
|
||||
var hash = sha.ComputeHash(data);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private static bool IsAlreadyExistsStatus(RequestFailedException ex)
|
||||
{
|
||||
if (ex.Status == 409) return true;
|
||||
if (ex.Status == 400 && string.Equals(ex.ErrorCode, "EntityAlreadyExists", StringComparison.Ordinal)) return true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models.Backup;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// Light, no-I/O pre-flight for the restore HTTP triggers (plan §Wave17 #2 +
|
||||
/// Wave18 #2). Validates only the request shape — table name is in the
|
||||
/// critical-backup catalog, mode is a known value, keys are present, auth-table
|
||||
/// rules are respected. Heavier work (manifest parse, SHA verify, blob ETag-pin)
|
||||
/// lives in <see cref="BackupRestoreInputValidator"/>.
|
||||
/// <para>
|
||||
/// Single-row preview is read-only against the live row and does not require
|
||||
/// the destructive-acknowledgement / confirmation-token rituals the full-table
|
||||
/// restore demands. The auth-table single-row path is explicitly allowed (with
|
||||
/// a UI banner), full-table on auth-tables stays blocked at this gate.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class RestoreTablePreflightValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Validates a single-row restore request. Throws on the first failure with
|
||||
/// a code suitable for HTTP envelope. Pure function, fully unit-testable.
|
||||
/// </summary>
|
||||
public void ValidateRowRequest(string backupId, RestoreRowRequest request)
|
||||
{
|
||||
if (string.IsNullOrEmpty(backupId))
|
||||
{
|
||||
throw new BackupTerminalException("InvalidBackupId", "backupId route segment is empty");
|
||||
}
|
||||
if (request == null)
|
||||
{
|
||||
throw new BackupTerminalException("MissingBody", "request body is required");
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(request.TableName))
|
||||
{
|
||||
throw new BackupTerminalException("InvalidTable", "tableName is required");
|
||||
}
|
||||
if (!Constants.CriticalBackupTables.All.Contains(request.TableName, StringComparer.Ordinal))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"InvalidTable",
|
||||
$"tableName '{request.TableName}' is not in the critical-backup catalog — only the 15 critical tables are restorable");
|
||||
}
|
||||
|
||||
// Empty PK/RK are technically legal in Azure Tables, so we only insist on
|
||||
// non-null. The Validator further down may still reject if the dump line
|
||||
// is not found for the given keys.
|
||||
if (request.PartitionKey == null)
|
||||
{
|
||||
throw new BackupTerminalException("InvalidKeys", "partitionKey is required (may be empty string but not null)");
|
||||
}
|
||||
if (request.RowKey == null)
|
||||
{
|
||||
throw new BackupTerminalException("InvalidKeys", "rowKey is required (may be empty string but not null)");
|
||||
}
|
||||
|
||||
if (request.Mode != RestoreRowMode.Preview && request.Mode != RestoreRowMode.Commit)
|
||||
{
|
||||
throw new BackupTerminalException("InvalidMode", $"mode must be 'preview' or 'commit', got '{request.Mode}'");
|
||||
}
|
||||
|
||||
if (request.Mode == RestoreRowMode.Commit)
|
||||
{
|
||||
if (string.IsNullOrEmpty(request.IfSha256))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"MissingPrecondition",
|
||||
"ifSha256 is required on commit — echo the rowSha256 from the preview response");
|
||||
}
|
||||
if (!IsLowerHex64(request.IfSha256))
|
||||
{
|
||||
throw new BackupTerminalException(
|
||||
"MissingPrecondition",
|
||||
"ifSha256 must be 64 lowercase hex characters (SHA-256 hex digest)");
|
||||
}
|
||||
// ifCurrentETag is intentionally nullable: null carries the precondition
|
||||
// "preview saw no live row" and triggers AddEntity on commit.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True for tables whose IsEnabled column carries security semantics — UI
|
||||
/// must render a warning banner and the operator must confirm explicitly.
|
||||
/// Single-row commit on these is allowed (full-table is not).
|
||||
/// </summary>
|
||||
public bool IsAuthTable(string tableName)
|
||||
{
|
||||
return Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden
|
||||
.Contains(tableName, StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
private static bool IsLowerHex64(string s)
|
||||
{
|
||||
if (s.Length != 64) return false;
|
||||
for (var i = 0; i < s.Length; i++)
|
||||
{
|
||||
var c = s[i];
|
||||
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -284,6 +284,29 @@ namespace AutopilotMonitor.Functions.Services
|
||||
null, "System.Maintenance",
|
||||
new { reason, triggeredBy });
|
||||
|
||||
/// <summary>
|
||||
/// A GA operator restored a single row from a backup (plan §PR2). Warning-level
|
||||
/// because the restore overwrote (or created) live data via ETag-CAS — operators
|
||||
/// frequently want a Telegram ping for this event so a parallel admin sees the
|
||||
/// audit trail in near-real-time. Payload carries the canonical
|
||||
/// <c>{ container, blobName }</c> only — no SAS URL, per plan §Medium #6.
|
||||
/// </summary>
|
||||
public Task RecordBackupRowRestoredAsync(
|
||||
string backupId, string tableName, string partitionKey, string rowKey, string actor, string outcome)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, "BackupRowRestored", OpsEventSeverity.Warning,
|
||||
$"Critical-table row restored: {tableName} (pk='{partitionKey}', rk='{rowKey}') from backup {backupId} by {actor} → {outcome}",
|
||||
null, actor,
|
||||
new
|
||||
{
|
||||
backupId,
|
||||
tableName,
|
||||
partitionKey,
|
||||
rowKey,
|
||||
outcome,
|
||||
container = AutopilotMonitor.Shared.Constants.BlobContainers.CriticalTableBackups,
|
||||
blobName = $"{backupId}/{tableName}.ndjson",
|
||||
});
|
||||
|
||||
// ── Tenant ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -221,6 +221,7 @@ namespace AutopilotMonitor.Shared
|
||||
public const string GlobalBackupsList = "/api/global/backups"; // GET → list of backups
|
||||
public const string GlobalBackupsManifest = "/api/global/backups/{backupId}"; // GET → manifest detail
|
||||
public const string GlobalBackupsJobStatus = "/api/global/backups/jobs/{jobId}"; // GET → BackupJobStatus
|
||||
public const string GlobalBackupsRestoreRow = "/api/global/backups/{backupId}/restore-row"; // POST → preview|commit single row
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Shared.Models.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// Operating mode for <c>POST /api/global/backups/{backupId}/restore-row</c>.
|
||||
/// Preview is a read-only diff; Commit is the conditional write under maintenance
|
||||
/// lease + ETag-CAS.
|
||||
/// </summary>
|
||||
public enum RestoreRowMode
|
||||
{
|
||||
Preview,
|
||||
Commit,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Body of <c>POST /api/global/backups/{backupId}/restore-row</c>.
|
||||
/// PartitionKey + RowKey are carried in the body (never the URL) because Azure
|
||||
/// Tables permits <c>/</c>, <c>+</c>, <c>%</c> in PK/RK; URL-encoding them on the
|
||||
/// route would interact poorly with the Functions router.
|
||||
/// </summary>
|
||||
public sealed class RestoreRowRequest
|
||||
{
|
||||
public string TableName { get; set; } = string.Empty;
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
public RestoreRowMode Mode { get; set; } = RestoreRowMode.Preview;
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 (hex, lowercase) of the raw NDJSON line bytes echoed from the
|
||||
/// matching Preview response. Required on Commit. The server re-computes the
|
||||
/// hash on the fresh NDJSON line and rejects with 409 on mismatch (tamper /
|
||||
/// blob churn after the preview).
|
||||
/// </summary>
|
||||
public string? IfSha256 { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Live-row ETag echoed from the matching Preview response.
|
||||
/// <para>
|
||||
/// <c>null</c> means "Preview saw no live row" → Commit uses
|
||||
/// <c>AddEntity</c>. A non-null value means "Preview saw a live row with this
|
||||
/// ETag" → Commit uses <c>UpdateEntity(ifMatch=etag, Replace)</c>. Mismatch
|
||||
/// (412) or existence change → 409 <c>CurrentRowChanged</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public string? IfCurrentETag { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Response body of <c>mode=preview</c>. Contains the backup-row dump, the live
|
||||
/// row (or null), a per-property diff, the row-hash to echo on commit, and the
|
||||
/// live ETag (or null) to echo on commit.
|
||||
/// </summary>
|
||||
public sealed class RestoreRowPreviewResponse
|
||||
{
|
||||
public string BackupId { get; set; } = string.Empty;
|
||||
public string TableName { get; set; } = string.Empty;
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// EDM-tagged property snapshot from the backup NDJSON line. Keys are
|
||||
/// case-faithful original Azure Table column names.
|
||||
/// </summary>
|
||||
public Dictionary<string, RestoreRowPropertySnapshot> BackupProperties { get; set; }
|
||||
= new Dictionary<string, RestoreRowPropertySnapshot>();
|
||||
|
||||
/// <summary>
|
||||
/// EDM-tagged property snapshot of the live row, or <c>null</c> if the live
|
||||
/// row does not currently exist.
|
||||
/// </summary>
|
||||
public Dictionary<string, RestoreRowPropertySnapshot>? CurrentProperties { get; set; }
|
||||
|
||||
public List<RestoreRowPropertyDiff> Diff { get; set; } = new List<RestoreRowPropertyDiff>();
|
||||
|
||||
/// <summary>SHA-256 (hex, lowercase) of the raw NDJSON line bytes. Echo this on Commit.</summary>
|
||||
public string RowSha256 { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Live-row ETag at preview time, or <c>null</c> if the live row did not exist.</summary>
|
||||
public string? CurrentETag { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True for tables whose <c>IsEnabled</c> column carries security semantics
|
||||
/// (<see cref="Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden"/>).
|
||||
/// UI must render the warning banner.
|
||||
/// </summary>
|
||||
public bool IsAuthTable { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Successful response body of <c>mode=commit</c>. Echoes the write outcome.
|
||||
/// </summary>
|
||||
public sealed class RestoreRowCommitResponse
|
||||
{
|
||||
public string BackupId { get; set; } = string.Empty;
|
||||
public string TableName { get; set; } = string.Empty;
|
||||
public string PartitionKey { get; set; } = string.Empty;
|
||||
public string RowKey { get; set; } = string.Empty;
|
||||
|
||||
/// <summary><c>Inserted</c> when the live row was absent, <c>Replaced</c> when it existed.</summary>
|
||||
public RestoreRowCommitOutcome Outcome { get; set; }
|
||||
}
|
||||
|
||||
public enum RestoreRowCommitOutcome
|
||||
{
|
||||
Inserted,
|
||||
Replaced,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Single EDM-tagged property value carried over the wire. Matches the shape of
|
||||
/// <see cref="AutopilotMonitor.Shared.Models.Deletion.DeletionPropValue"/> but is
|
||||
/// a separate DTO so we can serialize <c>JsonElement</c> snapshots from both
|
||||
/// backup and live rows without leaking a Deletion-namespaced type.
|
||||
/// </summary>
|
||||
public sealed class RestoreRowPropertySnapshot
|
||||
{
|
||||
public string EdmType { get; set; } = string.Empty;
|
||||
public System.Text.Json.JsonElement Value { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>Per-property diff entry — exactly one property name with a change kind.</summary>
|
||||
public sealed class RestoreRowPropertyDiff
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public RestoreRowDiffKind Kind { get; set; }
|
||||
|
||||
public RestoreRowPropertySnapshot? Backup { get; set; }
|
||||
public RestoreRowPropertySnapshot? Current { get; set; }
|
||||
}
|
||||
|
||||
public enum RestoreRowDiffKind
|
||||
{
|
||||
/// <summary>Property exists only in the backup (live row is missing it; restore will add it).</summary>
|
||||
Added,
|
||||
|
||||
/// <summary>Property exists only on the live row; restore will REMOVE it (Replace semantics).</summary>
|
||||
Removed,
|
||||
|
||||
/// <summary>Property exists on both sides with different values; restore will overwrite.</summary>
|
||||
Changed,
|
||||
|
||||
/// <summary>Property exists on both sides and values match.</summary>
|
||||
Unchanged,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
import Link from "next/link";
|
||||
import { useParams } from "next/navigation";
|
||||
import {
|
||||
api,
|
||||
type BackupManifest,
|
||||
type BackupTableEntry,
|
||||
type RestoreRowCommitResponse,
|
||||
type RestoreRowPreviewResponse,
|
||||
type RestoreRowRequestBody,
|
||||
} from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { useAdminConfig } from "../../AdminConfigContext";
|
||||
import { AdminNotifications } from "../../AdminNotifications";
|
||||
import { RestoreRowDiffModal } from "../components/RestoreRowDiffModal";
|
||||
|
||||
/**
|
||||
* Backup manifest detail page (plan §PR2). Renders the per-table breakdown
|
||||
* (status / row-count / byte-size / SHA-256 preview / error message) and lets
|
||||
* the operator restore a single row from any Ok / Empty table.
|
||||
*/
|
||||
export default function BackupDetailPage() {
|
||||
const params = useParams<{ backupId: string }>();
|
||||
const backupId = decodeURIComponent(params?.backupId ?? "");
|
||||
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
|
||||
|
||||
const [manifest, setManifest] = useState<BackupManifest | null>(null);
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
|
||||
// Restore-row trigger state.
|
||||
const [restoreInput, setRestoreInput] = useState<{ tableName: string; pk: string; rk: string } | null>(null);
|
||||
const [previewLoading, setPreviewLoading] = useState<boolean>(false);
|
||||
const [activePreview, setActivePreview] = useState<RestoreRowPreviewResponse | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
if (!backupId) return;
|
||||
try {
|
||||
setLoading(true);
|
||||
const res = await authenticatedFetch(api.backups.manifest(backupId), getAccessToken);
|
||||
if (!res.ok) throw new Error(`Manifest failed: ${res.status} ${res.statusText}`);
|
||||
const body = (await res.json()) as BackupManifest;
|
||||
setManifest(body);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [backupId, getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const openRestoreModal = async (tableName: string, pk: string, rk: string) => {
|
||||
setError(null);
|
||||
setPreviewLoading(true);
|
||||
try {
|
||||
const body: RestoreRowRequestBody = {
|
||||
tableName,
|
||||
partitionKey: pk,
|
||||
rowKey: rk,
|
||||
mode: "Preview",
|
||||
};
|
||||
const res = await authenticatedFetch(api.backups.restoreRow(backupId), getAccessToken, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Preview failed: ${res.status} ${text}`);
|
||||
}
|
||||
const preview = (await res.json()) as RestoreRowPreviewResponse;
|
||||
setActivePreview(preview);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setPreviewLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleCommit = useCallback(
|
||||
(response: RestoreRowCommitResponse) => {
|
||||
setSuccessMessage(
|
||||
`Row ${response.outcome.toLowerCase()}: ${response.tableName} pk='${response.partitionKey}' rk='${response.rowKey}'`,
|
||||
);
|
||||
setActivePreview(null);
|
||||
setRestoreInput(null);
|
||||
},
|
||||
[setSuccessMessage],
|
||||
);
|
||||
|
||||
const sortedTables = useMemo(() => {
|
||||
if (!manifest) return [];
|
||||
return [...manifest.tables].sort((a, b) => a.tableName.localeCompare(b.tableName));
|
||||
}, [manifest]);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
|
||||
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
|
||||
<div className="py-6 px-4 sm:px-6 lg:px-8">
|
||||
<div className="text-sm">
|
||||
<Link href="/admin/backups" className="text-blue-600 dark:text-blue-400 hover:underline">
|
||||
← Backups
|
||||
</Link>
|
||||
</div>
|
||||
<h1 className="text-2xl font-normal text-gray-900 dark:text-white mt-2 font-mono">{backupId}</h1>
|
||||
{manifest && (
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
|
||||
Started {new Date(manifest.startedAtUtc).toLocaleString()} · completed{" "}
|
||||
{new Date(manifest.completedAtUtc).toLocaleString()} · triggered by{" "}
|
||||
<span className="font-mono">{manifest.triggeredBy}</span> · outcome{" "}
|
||||
<OutcomePill outcome={manifest.outcome} />
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<AdminNotifications />
|
||||
|
||||
{!manifest && !loading && (
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
|
||||
Manifest not found — backup may be incomplete or deleted.
|
||||
</div>
|
||||
)}
|
||||
|
||||
{manifest && (
|
||||
<>
|
||||
{/* Restore-row trigger card */}
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 space-y-3">
|
||||
<h2 className="text-sm font-medium text-gray-900 dark:text-white">Restore a single row</h2>
|
||||
<p className="text-xs text-gray-600 dark:text-gray-400">
|
||||
Enter the table name + partition key + row key. The preview will show the diff between the backup
|
||||
row and the live row; you confirm the restore in the modal.
|
||||
</p>
|
||||
<RestoreInputRow
|
||||
tables={manifest.tables}
|
||||
disabled={previewLoading || activePreview !== null}
|
||||
onSubmit={(tableName, pk, rk) => {
|
||||
setRestoreInput({ tableName, pk, rk });
|
||||
void openRestoreModal(tableName, pk, rk);
|
||||
}}
|
||||
/>
|
||||
{previewLoading && restoreInput && (
|
||||
<div className="text-xs text-gray-500 dark:text-gray-400">
|
||||
Loading preview for <span className="font-mono">{restoreInput.tableName}</span> /{" "}
|
||||
<span className="font-mono">{restoreInput.pk}</span> /{" "}
|
||||
<span className="font-mono">{restoreInput.rk}</span>…
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Per-table breakdown */}
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow overflow-hidden">
|
||||
<table className="w-full text-xs">
|
||||
<thead className="bg-gray-100 dark:bg-gray-900 text-gray-700 dark:text-gray-300">
|
||||
<tr>
|
||||
<th className="px-3 py-2 text-left">Table</th>
|
||||
<th className="px-3 py-2 text-left">Status</th>
|
||||
<th className="px-3 py-2 text-right">Rows</th>
|
||||
<th className="px-3 py-2 text-right">Bytes</th>
|
||||
<th className="px-3 py-2 text-left">SHA-256 (preview)</th>
|
||||
<th className="px-3 py-2 text-left">Notes</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200 dark:divide-gray-700">
|
||||
{sortedTables.map((t) => (
|
||||
<tr key={t.tableName}>
|
||||
<td className="px-3 py-2 font-mono text-gray-900 dark:text-white">{t.tableName}</td>
|
||||
<td className="px-3 py-2"><StatusPill status={t.status} /></td>
|
||||
<td className="px-3 py-2 text-right font-mono">{t.rowCount.toLocaleString()}</td>
|
||||
<td className="px-3 py-2 text-right font-mono">{t.byteSize.toLocaleString()}</td>
|
||||
<td className="px-3 py-2 font-mono text-gray-700 dark:text-gray-300">
|
||||
{t.sha256Hex ? t.sha256Hex.slice(0, 12) + "…" : "—"}
|
||||
</td>
|
||||
<td className="px-3 py-2 text-red-700 dark:text-red-300">{t.errorMessage ?? ""}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</main>
|
||||
|
||||
{activePreview && (
|
||||
<RestoreRowDiffModal
|
||||
backupId={backupId}
|
||||
preview={activePreview}
|
||||
getAccessToken={getAccessToken}
|
||||
onClose={() => {
|
||||
setActivePreview(null);
|
||||
setRestoreInput(null);
|
||||
}}
|
||||
onCommitted={handleCommit}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RestoreInputRow({
|
||||
tables,
|
||||
disabled,
|
||||
onSubmit,
|
||||
}: {
|
||||
tables: BackupTableEntry[];
|
||||
disabled: boolean;
|
||||
onSubmit: (tableName: string, pk: string, rk: string) => void;
|
||||
}) {
|
||||
const restorable = useMemo(
|
||||
() => tables.filter((t) => t.status === "Ok" || t.status === "Empty"),
|
||||
[tables],
|
||||
);
|
||||
const [tableName, setTableName] = useState<string>(restorable[0]?.tableName ?? "");
|
||||
const [pk, setPk] = useState<string>("");
|
||||
const [rk, setRk] = useState<string>("");
|
||||
|
||||
// Keep the dropdown in sync if the manifest reload changes the restorable set.
|
||||
useEffect(() => {
|
||||
if (!restorable.find((t) => t.tableName === tableName)) {
|
||||
setTableName(restorable[0]?.tableName ?? "");
|
||||
}
|
||||
}, [restorable, tableName]);
|
||||
|
||||
return (
|
||||
<div className="flex flex-wrap items-end gap-2">
|
||||
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300">
|
||||
Table
|
||||
<select
|
||||
value={tableName}
|
||||
onChange={(e) => setTableName(e.target.value)}
|
||||
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
|
||||
>
|
||||
{restorable.map((t) => (
|
||||
<option key={t.tableName} value={t.tableName}>{t.tableName}</option>
|
||||
))}
|
||||
</select>
|
||||
</label>
|
||||
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300 flex-1 min-w-40">
|
||||
Partition key
|
||||
<input
|
||||
type="text"
|
||||
value={pk}
|
||||
onChange={(e) => setPk(e.target.value)}
|
||||
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
|
||||
/>
|
||||
</label>
|
||||
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300 flex-1 min-w-40">
|
||||
Row key
|
||||
<input
|
||||
type="text"
|
||||
value={rk}
|
||||
onChange={(e) => setRk(e.target.value)}
|
||||
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSubmit(tableName, pk, rk)}
|
||||
disabled={disabled || !tableName || pk === "" || rk === ""}
|
||||
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
Preview restore
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function OutcomePill({ outcome }: { outcome: "Success" | "Partial" }) {
|
||||
if (outcome === "Success") {
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-green-100 dark:bg-green-900 text-green-900 dark:text-green-100">Success</span>;
|
||||
}
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-yellow-100 dark:bg-yellow-900 text-yellow-900 dark:text-yellow-100">Partial</span>;
|
||||
}
|
||||
|
||||
function StatusPill({ status }: { status: BackupTableEntry["status"] }) {
|
||||
switch (status) {
|
||||
case "Ok":
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-green-100 dark:bg-green-900 text-green-900 dark:text-green-100">Ok</span>;
|
||||
case "Empty":
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-gray-100 dark:bg-gray-700 text-gray-700 dark:text-gray-300">Empty</span>;
|
||||
case "Skipped":
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-yellow-100 dark:bg-yellow-900 text-yellow-900 dark:text-yellow-100">Skipped</span>;
|
||||
case "Failed":
|
||||
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-red-100 dark:bg-red-900 text-red-900 dark:text-red-100">Failed</span>;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import {
|
||||
api,
|
||||
type RestoreRowCommitResponse,
|
||||
type RestoreRowPreviewResponse,
|
||||
type RestoreRowPropertyDiff,
|
||||
type RestoreRowPropertySnapshot,
|
||||
type RestoreRowRequestBody,
|
||||
} from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
|
||||
interface RestoreRowDiffModalProps {
|
||||
backupId: string;
|
||||
preview: RestoreRowPreviewResponse;
|
||||
getAccessToken: () => Promise<string | null>;
|
||||
onClose: () => void;
|
||||
onCommitted: (response: RestoreRowCommitResponse) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Two-step modal: preview is loaded by the caller (so the diff is already
|
||||
* populated when the modal opens), then the operator can either dismiss or
|
||||
* commit. Commit echoes <code>ifSha256</code> + <code>ifCurrentETag</code>
|
||||
* from the preview response; the backend verifies both under the maintenance
|
||||
* lease before writing.
|
||||
*/
|
||||
export function RestoreRowDiffModal({
|
||||
backupId,
|
||||
preview,
|
||||
getAccessToken,
|
||||
onClose,
|
||||
onCommitted,
|
||||
}: RestoreRowDiffModalProps) {
|
||||
const [committing, setCommitting] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const commit = async () => {
|
||||
setError(null);
|
||||
setCommitting(true);
|
||||
try {
|
||||
const body: RestoreRowRequestBody = {
|
||||
tableName: preview.tableName,
|
||||
partitionKey: preview.partitionKey,
|
||||
rowKey: preview.rowKey,
|
||||
mode: "Commit",
|
||||
ifSha256: preview.rowSha256,
|
||||
ifCurrentETag: preview.currentETag ?? null,
|
||||
};
|
||||
const res = await authenticatedFetch(api.backups.restoreRow(backupId), getAccessToken, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Restore failed: ${res.status} ${text}`);
|
||||
}
|
||||
const response = (await res.json()) as RestoreRowCommitResponse;
|
||||
onCommitted(response);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setCommitting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const diffCounts = countDiff(preview.diff);
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50">
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow-xl max-w-4xl w-full max-h-[90vh] overflow-hidden flex flex-col">
|
||||
<header className="px-6 py-4 border-b border-gray-200 dark:border-gray-700">
|
||||
<h2 className="text-lg font-medium text-gray-900 dark:text-white">
|
||||
Restore row from backup
|
||||
</h2>
|
||||
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5 font-mono">
|
||||
{preview.tableName} · pk=<span className="text-gray-900 dark:text-gray-100">{preview.partitionKey}</span>{" "}
|
||||
· rk=<span className="text-gray-900 dark:text-gray-100">{preview.rowKey}</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="flex-1 overflow-y-auto px-6 py-4 space-y-4">
|
||||
{preview.isAuthTable && (
|
||||
<div className="border border-amber-400 bg-amber-50 dark:bg-amber-950 text-amber-900 dark:text-amber-100 rounded-md p-3 text-sm">
|
||||
<div className="font-medium">Security-relevant table — check the <code className="font-mono">IsEnabled</code> column</div>
|
||||
<div className="mt-1">
|
||||
This is an <strong>authentication / authorization</strong> table
|
||||
(<code className="font-mono">GlobalAdmins</code>, <code className="font-mono">TenantAdmins</code>,{" "}
|
||||
<code className="font-mono">McpUsers</code>). Restoring this row will overwrite the live{" "}
|
||||
<code className="font-mono">IsEnabled</code> flag — confirm that the backup row's enable/disable
|
||||
state is what you intend.
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="text-sm text-gray-700 dark:text-gray-300">
|
||||
{preview.currentETag === null ? (
|
||||
<span>
|
||||
Live row <strong>does not exist</strong> — restore will <strong>insert</strong> the backup row.
|
||||
</span>
|
||||
) : (
|
||||
<span>
|
||||
Live row exists (ETag <code className="font-mono">{preview.currentETag}</code>) — restore will{" "}
|
||||
<strong>replace</strong> it with the backup row.
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="text-xs text-gray-600 dark:text-gray-400">
|
||||
<span className="font-medium">{diffCounts.changed}</span> changed ·{" "}
|
||||
<span className="font-medium">{diffCounts.added}</span> added ·{" "}
|
||||
<span className="font-medium text-red-700 dark:text-red-300">{diffCounts.removed}</span> will be removed ·{" "}
|
||||
<span className="font-medium">{diffCounts.unchanged}</span> unchanged
|
||||
</div>
|
||||
|
||||
<div className="border border-gray-200 dark:border-gray-700 rounded-md overflow-hidden">
|
||||
<table className="w-full text-xs font-mono">
|
||||
<thead className="bg-gray-100 dark:bg-gray-900 text-gray-700 dark:text-gray-300">
|
||||
<tr>
|
||||
<th className="px-3 py-2 text-left">Property</th>
|
||||
<th className="px-3 py-2 text-left">Backup</th>
|
||||
<th className="px-3 py-2 text-left">Current</th>
|
||||
<th className="px-3 py-2 text-left">Change</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200 dark:divide-gray-700">
|
||||
{preview.diff.map((d) => (
|
||||
<tr key={d.name} className={rowClass(d)}>
|
||||
<td className="px-3 py-2 align-top break-all">{d.name}</td>
|
||||
<td className="px-3 py-2 align-top break-all whitespace-pre-wrap">{snapshotPretty(d.backup)}</td>
|
||||
<td className="px-3 py-2 align-top break-all whitespace-pre-wrap">{snapshotPretty(d.current)}</td>
|
||||
<td className="px-3 py-2 align-top">{kindLabel(d.kind)}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="border border-red-400 bg-red-50 dark:bg-red-950 text-red-900 dark:text-red-100 rounded-md p-3 text-sm font-mono break-all">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<footer className="px-6 py-3 border-t border-gray-200 dark:border-gray-700 flex justify-end gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onClose}
|
||||
disabled={committing}
|
||||
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={commit}
|
||||
disabled={committing}
|
||||
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{committing ? "Restoring…" : "Restore row"}
|
||||
</button>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function countDiff(diff: RestoreRowPropertyDiff[]): { added: number; removed: number; changed: number; unchanged: number } {
|
||||
const c = { added: 0, removed: 0, changed: 0, unchanged: 0 };
|
||||
for (const d of diff) {
|
||||
if (d.kind === "Added") c.added++;
|
||||
else if (d.kind === "Removed") c.removed++;
|
||||
else if (d.kind === "Changed") c.changed++;
|
||||
else c.unchanged++;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
function rowClass(d: RestoreRowPropertyDiff): string {
|
||||
switch (d.kind) {
|
||||
case "Added":
|
||||
return "bg-green-50 dark:bg-green-950/30";
|
||||
case "Removed":
|
||||
return "bg-red-50 dark:bg-red-950/30";
|
||||
case "Changed":
|
||||
return "bg-yellow-50 dark:bg-yellow-950/30";
|
||||
default:
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function kindLabel(kind: RestoreRowPropertyDiff["kind"]): string {
|
||||
switch (kind) {
|
||||
case "Added":
|
||||
return "will be added";
|
||||
case "Removed":
|
||||
return "will be removed";
|
||||
case "Changed":
|
||||
return "will be overwritten";
|
||||
case "Unchanged":
|
||||
return "unchanged";
|
||||
}
|
||||
}
|
||||
|
||||
function snapshotPretty(snap: RestoreRowPropertySnapshot | null | undefined): string {
|
||||
if (snap == null) return "—";
|
||||
return `${JSON.stringify(snap.value)}\n[${snap.edmType}]`;
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import Link from "next/link";
|
||||
import {
|
||||
api,
|
||||
type BackupJobStatus,
|
||||
type BackupListResponse,
|
||||
type BackupTriggerResponse,
|
||||
} from "@/lib/api";
|
||||
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
|
||||
import { useAdminConfig } from "../AdminConfigContext";
|
||||
import { AdminNotifications } from "../AdminNotifications";
|
||||
|
||||
const POLL_INTERVAL_MS = 2500;
|
||||
|
||||
/**
|
||||
* GA-only admin surface for the critical-table backup feature (plan §PR2).
|
||||
* Lists every completed backup run newest-first and offers a "Run backup now"
|
||||
* trigger that opens a status-poll banner while the BackgroundService worker
|
||||
* processes the queued job.
|
||||
*/
|
||||
export default function BackupsListPage() {
|
||||
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
|
||||
const [backupIds, setBackupIds] = useState<string[]>([]);
|
||||
const [loading, setLoading] = useState<boolean>(false);
|
||||
const [triggering, setTriggering] = useState<boolean>(false);
|
||||
const [activeJob, setActiveJob] = useState<BackupJobStatus | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const res = await authenticatedFetch(api.backups.list(), getAccessToken);
|
||||
if (!res.ok) throw new Error(`List backups failed: ${res.status} ${res.statusText}`);
|
||||
const body = (await res.json()) as BackupListResponse;
|
||||
setBackupIds(body.backupIds ?? []);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [getAccessToken, setError]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
// Poll the active job until terminal, then refresh the list.
|
||||
useEffect(() => {
|
||||
if (!activeJob) return;
|
||||
if (
|
||||
activeJob.state === "Completed" ||
|
||||
activeJob.state === "Failed" ||
|
||||
activeJob.state === "Skipped" ||
|
||||
activeJob.state === "BlockedTerminal"
|
||||
) {
|
||||
// Terminal — final refresh and stop polling.
|
||||
void load();
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
const timer = window.setTimeout(async () => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
const res = await authenticatedFetch(api.backups.jobStatus(activeJob.jobId), getAccessToken);
|
||||
if (!res.ok) throw new Error(`Job status failed: ${res.status} ${res.statusText}`);
|
||||
const body = (await res.json()) as BackupJobStatus;
|
||||
setActiveJob(body);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
setActiveJob(null);
|
||||
}
|
||||
}, POLL_INTERVAL_MS);
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
window.clearTimeout(timer);
|
||||
};
|
||||
}, [activeJob, getAccessToken, load, setError]);
|
||||
|
||||
const triggerBackup = useCallback(async () => {
|
||||
try {
|
||||
setTriggering(true);
|
||||
const res = await authenticatedFetch(api.backups.trigger(), getAccessToken, {
|
||||
method: "POST",
|
||||
});
|
||||
if (res.status !== 202 && !res.ok) {
|
||||
const text = await res.text();
|
||||
throw new Error(`Trigger backup failed: ${res.status} ${text}`);
|
||||
}
|
||||
const body = (await res.json()) as BackupTriggerResponse;
|
||||
// Seed the poller with the freshly-created job.
|
||||
setActiveJob({
|
||||
jobId: body.jobId,
|
||||
kind: "Backup",
|
||||
state: "Queued",
|
||||
requestedBy: "",
|
||||
queuedAtUtc: new Date().toISOString(),
|
||||
lastHeartbeatUtc: new Date().toISOString(),
|
||||
});
|
||||
setSuccessMessage(`Backup queued — jobId ${body.jobId}.`);
|
||||
} catch (err) {
|
||||
if (err instanceof TokenExpiredError) setError(err.message);
|
||||
else setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setTriggering(false);
|
||||
}
|
||||
}, [getAccessToken, setError, setSuccessMessage]);
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
|
||||
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
|
||||
<div className="py-6 px-4 sm:px-6 lg:px-8">
|
||||
<h1 className="text-2xl font-normal text-gray-900 dark:text-white">Critical-Table Backups</h1>
|
||||
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
|
||||
Daily snapshots of the 15 critical configuration tables (AdminConfiguration, AnalyzeRules, GatherRules,
|
||||
ImeLogPatterns, …). Manifests live in the <code className="font-mono">critical-table-backups</code> blob
|
||||
container; 90-day lifecycle delete. Inspect any backup to see per-table SHA + row counts, and restore a
|
||||
single row from there.
|
||||
</p>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
|
||||
<AdminNotifications />
|
||||
|
||||
{/* Trigger row */}
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={triggerBackup}
|
||||
disabled={triggering || activeJob !== null}
|
||||
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{triggering ? "Triggering…" : "Run backup now"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={load}
|
||||
disabled={loading}
|
||||
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
|
||||
>
|
||||
{loading ? "Loading…" : "Refresh"}
|
||||
</button>
|
||||
<span className="text-sm text-gray-500 dark:text-gray-400 ml-2">
|
||||
{backupIds.length} backup(s)
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Active job banner */}
|
||||
{activeJob && <ActiveJobBanner job={activeJob} onDismiss={() => setActiveJob(null)} />}
|
||||
|
||||
{/* List */}
|
||||
{backupIds.length === 0 && !loading && (
|
||||
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
|
||||
No critical-table backups found. The daily timer runs at 04:00 UTC; you can also trigger one manually
|
||||
via the button above.
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="space-y-2">
|
||||
{backupIds.map((id) => (
|
||||
<Link
|
||||
key={id}
|
||||
href={`/admin/backups/${encodeURIComponent(id)}`}
|
||||
className="block bg-white dark:bg-gray-800 rounded-lg shadow p-4 hover:bg-gray-100 dark:hover:bg-gray-700 transition-colors"
|
||||
>
|
||||
<div className="font-mono text-sm text-gray-900 dark:text-white">{id}</div>
|
||||
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5">
|
||||
{formatBackupTimestamp(id)} · click to inspect manifest
|
||||
</div>
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ActiveJobBanner({ job, onDismiss }: { job: BackupJobStatus; onDismiss: () => void }) {
|
||||
const state = job.state;
|
||||
const color =
|
||||
state === "Completed"
|
||||
? "bg-green-100 dark:bg-green-900 border-green-300 dark:border-green-700 text-green-900 dark:text-green-100"
|
||||
: state === "Failed" || state === "BlockedTerminal"
|
||||
? "bg-red-100 dark:bg-red-900 border-red-300 dark:border-red-700 text-red-900 dark:text-red-100"
|
||||
: state === "Skipped"
|
||||
? "bg-yellow-100 dark:bg-yellow-900 border-yellow-300 dark:border-yellow-700 text-yellow-900 dark:text-yellow-100"
|
||||
: "bg-blue-100 dark:bg-blue-900 border-blue-300 dark:border-blue-700 text-blue-900 dark:text-blue-100";
|
||||
|
||||
return (
|
||||
<div className={`border rounded-md p-3 text-sm ${color}`}>
|
||||
<div className="flex items-start justify-between gap-2">
|
||||
<div>
|
||||
<div className="font-medium">Backup job {job.jobId} — {state}</div>
|
||||
{job.backupOutcome && <div className="text-xs mt-0.5">Outcome: {job.backupOutcome}</div>}
|
||||
{job.error && <div className="text-xs mt-1 font-mono break-all">Error: {job.error}</div>}
|
||||
{job.progress && <div className="text-xs mt-1 font-mono break-all">Progress: {job.progress}</div>}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onDismiss}
|
||||
className="text-xs px-2 py-1 underline"
|
||||
>
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Pretty-print the embedded timestamp portion of a backupId (yyyyMMddTHHmmssZ_xxxxxxxx). */
|
||||
function formatBackupTimestamp(backupId: string): string {
|
||||
// 20260522T040000Z_a1b2c3d4 → "2026-05-22 04:00:00 UTC"
|
||||
const m = backupId.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z_/);
|
||||
if (!m) return backupId;
|
||||
return `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]} UTC`;
|
||||
}
|
||||
@@ -37,6 +37,9 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
"CriticalTableBackupPartial",
|
||||
"CriticalTableBackupFailed",
|
||||
"CriticalTableBackupSkippedLocked",
|
||||
// PR2: GA restored a single row from a backup. Warning severity so operators
|
||||
// can wire a Telegram rule and see the audit in near-real-time.
|
||||
"BackupRowRestored",
|
||||
],
|
||||
Security: [
|
||||
"DeviceBlocked",
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
|
||||
// API_BASE_URL is read at import time of lib/api — stub it before the import.
|
||||
vi.mock("@/utils/config", () => ({ API_BASE_URL: "https://test.example" }));
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const apiPromise = import("../api") as Promise<{ api: any }>;
|
||||
|
||||
/**
|
||||
* Wire-shape contract for the critical-table backup feature (PR1 + PR2).
|
||||
* The frontend talks to a fixed set of GA-only endpoints; a refactor of
|
||||
* api.ts that accidentally drops the namespace or changes the path encoding
|
||||
* would silently break the Backups admin surface.
|
||||
*/
|
||||
describe("api.backups — list / manifest / trigger / job-status", () => {
|
||||
it("list points at /api/global/backups", async () => {
|
||||
const { api } = await apiPromise;
|
||||
expect(api.backups.list()).toBe("https://test.example/api/global/backups");
|
||||
});
|
||||
|
||||
it("manifest encodes the backupId in the path", async () => {
|
||||
const { api } = await apiPromise;
|
||||
expect(api.backups.manifest("20260522T040000Z_a1b2c3d4")).toBe(
|
||||
"https://test.example/api/global/backups/20260522T040000Z_a1b2c3d4",
|
||||
);
|
||||
});
|
||||
|
||||
it("trigger points at /api/global/backups/trigger", async () => {
|
||||
const { api } = await apiPromise;
|
||||
expect(api.backups.trigger()).toBe("https://test.example/api/global/backups/trigger");
|
||||
});
|
||||
|
||||
it("jobStatus encodes the jobId in the path", async () => {
|
||||
const { api } = await apiPromise;
|
||||
expect(api.backups.jobStatus("abc-123")).toBe(
|
||||
"https://test.example/api/global/backups/jobs/abc-123",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("api.backups.restoreRow — PR2 single-row restore", () => {
|
||||
it("points at /api/global/backups/{backupId}/restore-row", async () => {
|
||||
const { api } = await apiPromise;
|
||||
expect(api.backups.restoreRow("20260522T040000Z_a1b2c3d4")).toBe(
|
||||
"https://test.example/api/global/backups/20260522T040000Z_a1b2c3d4/restore-row",
|
||||
);
|
||||
});
|
||||
|
||||
it("URL-encodes special characters in the backupId", async () => {
|
||||
const { api } = await apiPromise;
|
||||
// backupIds are stamp_guid8 in production but the path encoder must still
|
||||
// be defensive — a future schema change shouldn't silently break routing.
|
||||
expect(api.backups.restoreRow("with spaces/and+slashes")).toBe(
|
||||
"https://test.example/api/global/backups/with%20spaces%2Fand%2Bslashes/restore-row",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -496,6 +496,18 @@ export const api = {
|
||||
planTiers: () => `${API_BASE_URL}/api/global/config/plan-tiers`,
|
||||
},
|
||||
|
||||
// ── Critical-Table Backup (PR1 list/manifest/trigger + PR2 restore-row) ───
|
||||
backups: {
|
||||
list: () => `${API_BASE_URL}/api/global/backups`,
|
||||
manifest: (backupId: string) =>
|
||||
`${API_BASE_URL}/api/global/backups/${encodeURIComponent(backupId)}`,
|
||||
trigger: () => `${API_BASE_URL}/api/global/backups/trigger`,
|
||||
jobStatus: (jobId: string) =>
|
||||
`${API_BASE_URL}/api/global/backups/jobs/${encodeURIComponent(jobId)}`,
|
||||
restoreRow: (backupId: string) =>
|
||||
`${API_BASE_URL}/api/global/backups/${encodeURIComponent(backupId)}/restore-row`,
|
||||
},
|
||||
|
||||
// ── Maintenance ───────────────────────────────────────────────────────────
|
||||
maintenance: {
|
||||
trigger: (date?: string) =>
|
||||
@@ -556,3 +568,113 @@ export interface CustomsArchiveFullEntry extends CustomsArchiveEntrySummary {
|
||||
historyRowKey: string;
|
||||
archivedBy: string;
|
||||
}
|
||||
|
||||
// ── Backups response types (PR1 + PR2) ────────────────────────────────────
|
||||
|
||||
export interface BackupListResponse {
|
||||
backupIds: string[];
|
||||
}
|
||||
|
||||
export type BackupTableStatus = "Ok" | "Empty" | "Skipped" | "Failed";
|
||||
export type BackupOutcome = "Success" | "Partial";
|
||||
|
||||
export interface BackupTableEntry {
|
||||
tableName: string;
|
||||
status: BackupTableStatus;
|
||||
rowCount: number;
|
||||
byteSize: number;
|
||||
sha256Hex: string;
|
||||
blobName: string;
|
||||
errorMessage?: string | null;
|
||||
}
|
||||
|
||||
export interface BackupManifest {
|
||||
schemaVersion: number;
|
||||
backupId: string;
|
||||
startedAtUtc: string;
|
||||
completedAtUtc: string;
|
||||
triggeredBy: string;
|
||||
outcome: BackupOutcome;
|
||||
tables: BackupTableEntry[];
|
||||
}
|
||||
|
||||
export type BackupJobKind = "Backup" | "RestoreTable";
|
||||
export type BackupJobState =
|
||||
| "Queued"
|
||||
| "Running"
|
||||
| "Completed"
|
||||
| "Failed"
|
||||
| "Skipped"
|
||||
| "BlockedTerminal";
|
||||
|
||||
export interface BackupJobStatus {
|
||||
jobId: string;
|
||||
kind: BackupJobKind;
|
||||
state: BackupJobState;
|
||||
requestedBy: string;
|
||||
queuedAtUtc: string;
|
||||
startedAtUtc?: string | null;
|
||||
completedAtUtc?: string | null;
|
||||
lastHeartbeatUtc: string;
|
||||
backupId?: string | null;
|
||||
sourceBackupId?: string | null;
|
||||
tableName?: string | null;
|
||||
strategy?: string | null;
|
||||
progress?: string | null;
|
||||
error?: string | null;
|
||||
backupOutcome?: BackupOutcome | null;
|
||||
}
|
||||
|
||||
export interface BackupTriggerResponse {
|
||||
jobId: string;
|
||||
statusUrl: string;
|
||||
}
|
||||
|
||||
// ── PR2: single-row restore ───────────────────────────────────────────────
|
||||
|
||||
export type RestoreRowMode = "Preview" | "Commit";
|
||||
export type RestoreRowDiffKind = "Added" | "Removed" | "Changed" | "Unchanged";
|
||||
export type RestoreRowCommitOutcome = "Inserted" | "Replaced";
|
||||
|
||||
export interface RestoreRowPropertySnapshot {
|
||||
edmType: string;
|
||||
// JsonElement on the wire — string | number | boolean | null | object | array
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
export interface RestoreRowPropertyDiff {
|
||||
name: string;
|
||||
kind: RestoreRowDiffKind;
|
||||
backup?: RestoreRowPropertySnapshot | null;
|
||||
current?: RestoreRowPropertySnapshot | null;
|
||||
}
|
||||
|
||||
export interface RestoreRowPreviewResponse {
|
||||
backupId: string;
|
||||
tableName: string;
|
||||
partitionKey: string;
|
||||
rowKey: string;
|
||||
backupProperties: Record<string, RestoreRowPropertySnapshot>;
|
||||
currentProperties?: Record<string, RestoreRowPropertySnapshot> | null;
|
||||
diff: RestoreRowPropertyDiff[];
|
||||
rowSha256: string;
|
||||
currentETag?: string | null;
|
||||
isAuthTable: boolean;
|
||||
}
|
||||
|
||||
export interface RestoreRowCommitResponse {
|
||||
backupId: string;
|
||||
tableName: string;
|
||||
partitionKey: string;
|
||||
rowKey: string;
|
||||
outcome: RestoreRowCommitOutcome;
|
||||
}
|
||||
|
||||
export interface RestoreRowRequestBody {
|
||||
tableName: string;
|
||||
partitionKey: string;
|
||||
rowKey: string;
|
||||
mode: RestoreRowMode;
|
||||
ifSha256?: string;
|
||||
ifCurrentETag?: string | null;
|
||||
}
|
||||
|
||||
@@ -265,6 +265,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
|
||||
{ id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" },
|
||||
{ id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" },
|
||||
{ id: "ga-customs-archive", label: "Customs Archive", href: "/admin/customs-archive" },
|
||||
{ id: "ga-backups", label: "Backups", href: "/admin/backups" },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user