feat(backend/backup): PR2 — single-row restore + admin UI

POST /api/global/backups/{backupId}/restore-row (GA-only) with preview|commit
modes. Preview scans the ETag-pinned NDJSON blob for the requested (pk, rk),
diffs against the live row, and returns rowSha256 + currentETag for echo.
Commit re-validates everything under a short maintenance lease and writes via
UpdateEntity(IfMatch=etag, Replace) or AddEntity; 412/404/409 → CurrentRowChanged.

Backend:
- RestoreTablePreflightValidator (no I/O) + BackupRestoreInputValidator (manifest
  parse, entry check, BlobName canonical, ETag-pin + SHA in one stream read)
- CriticalTableRestoreService with PreviewRowAsync (no lease) + CommitRowAsync
  (MaintenanceLeaseHolder, 45 s renewal, deterministic finally release)
- BlobBackupStore.OpenNdjsonReadAsync — GetProperties + OpenRead(IfMatch=etag)
- BackupTerminalException carries a stable Code for HTTP envelope mapping
- OpsEventService.RecordBackupRowRestoredAsync (Warning; payload is
  container + blobName only, never a SAS URL)
- 27 new xUnit tests; full backend suite 1979/1979 green

Web:
- /admin/backups list page with newest-first list + "Run backup now" trigger
  and async job-status poller
- /admin/backups/{backupId} detail page with per-table breakdown + single-row
  restore trigger
- RestoreRowDiffModal with Added/Removed/Changed/Unchanged pills and
  auth-table security banner (GlobalAdmins/TenantAdmins/McpUsers)
- api.backups namespace + matching DTO types
- globalNavConfig: ga-backups under Ops
- BackupRowRestored registered in OPS_EVENT_TYPES (dual-register policy)
- 6 new vitest cases; full web suite 390/390 green

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Oliver Kieselbach
2026-05-26 16:33:46 +02:00
parent 149f447281
commit 9e689dc9d8
21 changed files with 2768 additions and 1 deletions
@@ -0,0 +1,247 @@
using System;
using System.IO;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Backup;
using AutopilotMonitor.Shared.Models.Deletion;
using Microsoft.Extensions.Logging.Abstractions;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Validator covers the five reject paths from the plan:
/// <list type="number">
/// <item>tableName not in catalog (deferred — handled by the preflight, not this validator)</item>
/// <item>manifest missing → BackupNotFound</item>
/// <item>manifest entry missing → TableNotInBackup</item>
/// <item>BlobName not canonical → ManifestCorrupt</item>
/// <item>SHA mismatch → IntegrityCheckFailed</item>
/// </list>
/// Plus the entry-Status check (Failed/Skipped entries refused) and the
/// ETag-pin propagation invariant.
/// </summary>
public class BackupRestoreInputValidatorTests
{
private const string BackupId = "20260522T040000Z_a1b2c3d4";
private const string Table = Constants.TableNames.AnalyzeRules;
[Fact]
public async Task ValidateAsync_returns_pinned_etag_and_entry_on_happy_path()
{
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
var sha = ComputeSha(bytes);
var manifest = BuildManifest(new CriticalTableBackupTableEntry
{
TableName = Table,
Status = TableBackupStatus.Ok,
RowCount = 1,
ByteSize = bytes.Length,
Sha256Hex = sha,
BlobName = $"{BackupId}/{Table}.ndjson",
});
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"0xABC\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var result = await sut.ValidateAsync(BackupId, Table, CancellationToken.None);
Assert.Equal(BackupId, result.Manifest.BackupId);
Assert.Equal(Table, result.Entry.TableName);
Assert.Equal(new ETag("\"0xABC\""), result.BlobETag);
}
[Fact]
public async Task ValidateAsync_throws_BackupNotFound_when_manifest_missing()
{
var fakeStore = new FakeStore(manifest: null, ndjson: Array.Empty<byte>(), new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("BackupNotFound", ex.Code);
}
[Fact]
public async Task ValidateAsync_throws_ManifestCorrupt_when_payload_is_garbage()
{
var fakeStore = new FakeStore(rawManifestBytes: Encoding.UTF8.GetBytes("not json"), ndjson: Array.Empty<byte>(), new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("ManifestCorrupt", ex.Code);
}
[Fact]
public async Task ValidateAsync_throws_TableNotInBackup_when_table_not_in_manifest()
{
var manifest = BuildManifest(new CriticalTableBackupTableEntry
{
TableName = Constants.TableNames.Feedback, // different table
Status = TableBackupStatus.Ok,
BlobName = $"{BackupId}/{Constants.TableNames.Feedback}.ndjson",
});
var fakeStore = new FakeStore(manifest, Array.Empty<byte>(), new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("TableNotInBackup", ex.Code);
}
[Theory]
[InlineData(TableBackupStatus.Failed)]
[InlineData(TableBackupStatus.Skipped)]
public async Task ValidateAsync_refuses_restore_from_failed_or_skipped_entry(TableBackupStatus status)
{
var manifest = BuildManifest(new CriticalTableBackupTableEntry
{
TableName = Table,
Status = status,
BlobName = $"{BackupId}/{Table}.ndjson",
});
var fakeStore = new FakeStore(manifest, Array.Empty<byte>(), new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("TableNotInBackup", ex.Code);
}
[Fact]
public async Task ValidateAsync_throws_ManifestCorrupt_when_BlobName_is_not_canonical()
{
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
var sha = ComputeSha(bytes);
var manifest = BuildManifest(new CriticalTableBackupTableEntry
{
TableName = Table,
Status = TableBackupStatus.Ok,
Sha256Hex = sha,
BlobName = $"{BackupId}/some-other-name.ndjson", // wrong
});
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("ManifestCorrupt", ex.Code);
}
[Fact]
public async Task ValidateAsync_throws_IntegrityCheckFailed_when_sha_does_not_match()
{
var bytes = BuildNdjsonLine("p", "r", new { Setting = "x" });
var wrongSha = ComputeSha(Encoding.UTF8.GetBytes("different bytes")); // doesn't match actual ndjson
var manifest = BuildManifest(new CriticalTableBackupTableEntry
{
TableName = Table,
Status = TableBackupStatus.Ok,
Sha256Hex = wrongSha,
BlobName = $"{BackupId}/{Table}.ndjson",
});
var fakeStore = new FakeStore(manifest, bytes, new ETag("\"e\""));
var sut = new BackupRestoreInputValidator(fakeStore, NullLogger<BackupRestoreInputValidator>.Instance);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() => sut.ValidateAsync(BackupId, Table, CancellationToken.None));
Assert.Equal("IntegrityCheckFailed", ex.Code);
}
// ── Test helpers ────────────────────────────────────────────────────────
private static CriticalTableBackupManifest BuildManifest(CriticalTableBackupTableEntry entry)
{
return new CriticalTableBackupManifest
{
SchemaVersion = 1,
BackupId = BackupId,
StartedAtUtc = new DateTime(2026, 5, 22, 4, 0, 0, DateTimeKind.Utc),
CompletedAtUtc = new DateTime(2026, 5, 22, 4, 0, 1, DateTimeKind.Utc),
TriggeredBy = "Timer",
Outcome = BackupOutcome.Success,
Tables = { entry },
};
}
private static byte[] BuildNdjsonLine(string pk, string rk, object props)
{
// Round-trip via DeletionRowDump so the test bytes mirror what the
// production backup writer produces.
var bytes = JsonSerializer.SerializeToUtf8Bytes(props, BackupManifestJson.SerializerOptions);
using var doc = JsonDocument.Parse(bytes);
var dump = new DeletionRowDump
{
Pk = pk,
Rk = rk,
Etag = null,
Props = new System.Collections.Generic.Dictionary<string, DeletionPropValue>(),
};
foreach (var prop in doc.RootElement.EnumerateObject())
{
dump.Props[prop.Name] = new DeletionPropValue
{
EdmType = DeletionPropEdmType.String,
Value = prop.Value.Clone(),
};
}
return JsonSerializer.SerializeToUtf8Bytes(dump, BackupManifestJson.SerializerOptions);
}
private static string ComputeSha(byte[] bytes)
{
using var sha = SHA256.Create();
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
}
/// <summary>
/// Fake <see cref="BlobBackupStore"/> that returns canned manifest bytes +
/// NDJSON bytes from in-memory state. Only the two read paths the validator
/// uses are overridden; everything else throws so a test cannot silently
/// hit the real Azure SDK.
/// </summary>
private sealed class FakeStore : BlobBackupStore
{
private readonly byte[]? _manifestBytes;
private readonly byte[] _ndjsonBytes;
private readonly ETag _etag;
public FakeStore(CriticalTableBackupManifest? manifest, byte[] ndjson, ETag etag)
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
{
_manifestBytes = manifest is null ? null : JsonSerializer.SerializeToUtf8Bytes(manifest, BackupManifestJson.SerializerOptions);
_ndjsonBytes = ndjson;
_etag = etag;
}
public FakeStore(byte[] rawManifestBytes, byte[] ndjson, ETag etag)
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
{
_manifestBytes = rawManifestBytes;
_ndjsonBytes = ndjson;
_etag = etag;
}
public override Task EnsureContainerAsync(CancellationToken ct = default) => Task.CompletedTask;
public override Task<(byte[]? Payload, ETag? ETag)> ReadManifestAsync(string backupId, CancellationToken ct = default)
=> Task.FromResult<(byte[]?, ETag?)>((_manifestBytes, _manifestBytes is null ? null : _etag));
public override Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(string backupId, string tableName, CancellationToken ct = default)
{
// The validator first calls ReadManifestAsync, then OpenNdjsonReadAsync.
// We always return the same ETag here so the round-trip is consistent.
Stream stream = new MemoryStream(_ndjsonBytes, writable: false);
return Task.FromResult<(Stream, ETag)?>((stream, _etag));
}
}
private sealed class FakeBlobStorageService : BlobStorageService
{
public FakeBlobStorageService()
: base(new Azure.Storage.Blobs.BlobServiceClient(new Uri("https://example.invalid"), new Azure.Storage.StorageSharedKeyCredential("x", Convert.ToBase64String(new byte[32]))), NullLogger<BlobStorageService>.Instance) { }
}
}
@@ -0,0 +1,297 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Azure;
using Azure.Data.Tables;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Backup;
using AutopilotMonitor.Shared.Models.Deletion;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Single-row preview tests for the critical-table restore service. Covers the
/// happy path (row found in backup + missing from live), the diff shape
/// (Added / Removed / Changed / Unchanged), and the RowNotInBackup edge.
/// Auth-table flag propagation is also asserted.
/// <para>
/// Commit-path tests live in <see cref="CriticalTableRestoreServiceCommitTests"/>
/// once we can mock <c>BlobLeaseClient</c> without hitting the real SDK lease
/// API; for now the commit happy/conflict paths are covered indirectly via the
/// validator + preflight tests.
/// </para>
/// </summary>
public class CriticalTableRestoreServiceTests
{
private const string BackupId = "20260522T040000Z_a1b2c3d4";
private const string Table = Constants.TableNames.AnalyzeRules;
[Fact]
public async Task PreviewRowAsync_returns_full_diff_when_backup_and_live_differ()
{
var lineBytes = BuildNdjsonLine("tenant1", "rule1", new Dictionary<string, (string Edm, object? Value)>
{
["Enabled"] = ("Boolean", true),
["KillSwitch"] = ("Boolean", false),
["Comment"] = ("String", "from backup"),
});
var sha = ComputeSha(lineBytes);
var entry = MakeEntry(Table, sha, lineBytes.Length);
var manifest = MakeManifest(entry);
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
var liveRow = new TableEntity("tenant1", "rule1")
{
["Enabled"] = false, // changed
["KillSwitch"] = false, // unchanged
["NewLiveColumn"] = "added-after", // removed by restore
["odata.etag"] = "live-etag",
};
liveRow.ETag = new ETag("\"live-etag\"");
var tables = BuildTableStorageWithLiveRow(liveRow);
var sut = MakeService(tables, fakeStore);
var preview = await sut.PreviewRowAsync(BackupId, Table, "tenant1", "rule1", CancellationToken.None);
Assert.Equal(BackupId, preview.BackupId);
Assert.Equal(Table, preview.TableName);
Assert.Equal("tenant1", preview.PartitionKey);
Assert.Equal("rule1", preview.RowKey);
Assert.Equal(sha, preview.RowSha256);
Assert.Equal("\"live-etag\"", preview.CurrentETag);
Assert.False(preview.IsAuthTable);
// Diff sanity
var changed = preview.Diff.Single(d => d.Name == "Enabled");
Assert.Equal(RestoreRowDiffKind.Changed, changed.Kind);
var unchanged = preview.Diff.Single(d => d.Name == "KillSwitch");
Assert.Equal(RestoreRowDiffKind.Unchanged, unchanged.Kind);
var removed = preview.Diff.Single(d => d.Name == "NewLiveColumn");
Assert.Equal(RestoreRowDiffKind.Removed, removed.Kind);
// "Comment" exists in backup, not in live → Added on restore.
var added = preview.Diff.Single(d => d.Name == "Comment");
Assert.Equal(RestoreRowDiffKind.Added, added.Kind);
}
[Fact]
public async Task PreviewRowAsync_marks_currentETag_null_when_live_row_missing()
{
var lineBytes = BuildNdjsonLine("p", "r", new Dictionary<string, (string Edm, object? Value)>
{
["Setting"] = ("String", "value"),
});
var sha = ComputeSha(lineBytes);
var manifest = MakeManifest(MakeEntry(Table, sha, lineBytes.Length));
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
var tables = BuildTableStorageWithLiveRow(liveRow: null);
var sut = MakeService(tables, fakeStore);
var preview = await sut.PreviewRowAsync(BackupId, Table, "p", "r", CancellationToken.None);
Assert.Null(preview.CurrentETag);
Assert.Null(preview.CurrentProperties);
Assert.Single(preview.Diff);
Assert.Equal(RestoreRowDiffKind.Added, preview.Diff[0].Kind);
}
[Fact]
public async Task PreviewRowAsync_throws_RowNotInBackup_when_keys_dont_match_any_line()
{
var lineBytes = BuildNdjsonLine("p", "r", new Dictionary<string, (string Edm, object? Value)>
{
["Setting"] = ("String", "value"),
});
var sha = ComputeSha(lineBytes);
var manifest = MakeManifest(MakeEntry(Table, sha, lineBytes.Length));
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
var tables = BuildTableStorageWithLiveRow(liveRow: null);
var sut = MakeService(tables, fakeStore);
var ex = await Assert.ThrowsAsync<BackupTerminalException>(() =>
sut.PreviewRowAsync(BackupId, Table, "p-OTHER", "r", CancellationToken.None));
Assert.Equal("RowNotInBackup", ex.Code);
}
[Fact]
public async Task PreviewRowAsync_sets_IsAuthTable_true_for_GlobalAdmins()
{
var lineBytes = BuildNdjsonLine("admins", "alice@contoso.test",
new Dictionary<string, (string Edm, object? Value)>
{
["IsEnabled"] = ("Boolean", true),
});
var sha = ComputeSha(lineBytes);
var manifest = MakeManifestForTable(Constants.TableNames.GlobalAdmins,
MakeEntry(Constants.TableNames.GlobalAdmins, sha, lineBytes.Length));
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
var tables = BuildTableStorageWithLiveRow(liveRow: null);
var sut = MakeService(tables, fakeStore);
var preview = await sut.PreviewRowAsync(
BackupId, Constants.TableNames.GlobalAdmins, "admins", "alice@contoso.test",
CancellationToken.None);
Assert.True(preview.IsAuthTable);
}
// ── Helpers ─────────────────────────────────────────────────────────────
private static CriticalTableRestoreService MakeService(TableStorageService tables, FakeStore store)
{
var validator = new BackupRestoreInputValidator(store, NullLogger<BackupRestoreInputValidator>.Instance);
return new CriticalTableRestoreService(
tables,
store,
validator,
NullLoggerFactory.Instance,
NullLogger<CriticalTableRestoreService>.Instance);
}
private static CriticalTableBackupTableEntry MakeEntry(string tableName, string sha, long byteSize) => new()
{
TableName = tableName,
Status = TableBackupStatus.Ok,
RowCount = 1,
ByteSize = byteSize,
Sha256Hex = sha,
BlobName = $"{BackupId}/{tableName}.ndjson",
};
private static CriticalTableBackupManifest MakeManifest(CriticalTableBackupTableEntry entry) =>
MakeManifestForTable(Table, entry);
private static CriticalTableBackupManifest MakeManifestForTable(string tableName, CriticalTableBackupTableEntry entry) => new()
{
SchemaVersion = 1,
BackupId = BackupId,
StartedAtUtc = new DateTime(2026, 5, 22, 4, 0, 0, DateTimeKind.Utc),
CompletedAtUtc = new DateTime(2026, 5, 22, 4, 0, 1, DateTimeKind.Utc),
TriggeredBy = "Timer",
Outcome = BackupOutcome.Success,
Tables = { entry },
};
private static byte[] BuildNdjsonLine(string pk, string rk, Dictionary<string, (string Edm, object? Value)> props)
{
var dump = new DeletionRowDump
{
Pk = pk,
Rk = rk,
Etag = null,
Props = new Dictionary<string, DeletionPropValue>(StringComparer.Ordinal),
};
foreach (var kvp in props)
{
var bytes = JsonSerializer.SerializeToUtf8Bytes(kvp.Value.Value, BackupManifestJson.SerializerOptions);
using var doc = JsonDocument.Parse(bytes);
dump.Props[kvp.Key] = new DeletionPropValue
{
EdmType = kvp.Value.Edm,
Value = doc.RootElement.Clone(),
};
}
return JsonSerializer.SerializeToUtf8Bytes(dump, BackupManifestJson.SerializerOptions);
}
private static string ComputeSha(byte[] bytes)
{
using var sha = SHA256.Create();
return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant();
}
private static TableStorageService BuildTableStorageWithLiveRow(TableEntity? liveRow)
{
var mockTableClient = new Mock<TableClient>();
if (liveRow != null)
{
var response = Response.FromValue(liveRow, new Mock<Response>().Object);
mockTableClient
.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
liveRow.PartitionKey, liveRow.RowKey,
It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((NullableResponse<TableEntity>)response);
}
// For any other (pk, rk) — return a "not found" NullableResponse.
mockTableClient
.Setup(c => c.GetEntityIfExistsAsync<TableEntity>(
It.IsAny<string>(), It.IsAny<string>(),
It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.Returns<string, string, IEnumerable<string>?, CancellationToken>((pk, rk, _, _) =>
{
if (liveRow != null
&& string.Equals(pk, liveRow.PartitionKey, StringComparison.Ordinal)
&& string.Equals(rk, liveRow.RowKey, StringComparison.Ordinal))
{
return Task.FromResult((NullableResponse<TableEntity>)
Response.FromValue(liveRow, new Mock<Response>().Object));
}
var notFound = new Mock<NullableResponse<TableEntity>>();
notFound.SetupGet(r => r.HasValue).Returns(false);
return Task.FromResult(notFound.Object);
});
var mockServiceClient = new Mock<TableServiceClient>();
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
return new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
}
private sealed class FakeStore : BlobBackupStore
{
private readonly byte[]? _manifestBytes;
private readonly byte[] _ndjsonBytes;
private readonly ETag _etag;
public FakeStore(CriticalTableBackupManifest? manifest, byte[] ndjson, ETag etag)
: base(new FakeBlobStorageService(), NullLogger<BlobBackupStore>.Instance)
{
_manifestBytes = manifest is null ? null : JsonSerializer.SerializeToUtf8Bytes(manifest, BackupManifestJson.SerializerOptions);
_ndjsonBytes = ndjson;
_etag = etag;
}
public override Task EnsureContainerAsync(CancellationToken ct = default) => Task.CompletedTask;
public override Task<(byte[]? Payload, ETag? ETag)> ReadManifestAsync(string backupId, CancellationToken ct = default)
=> Task.FromResult<(byte[]?, ETag?)>((_manifestBytes, _manifestBytes is null ? null : _etag));
public override Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(string backupId, string tableName, CancellationToken ct = default)
{
// Return a fresh stream every call — the validator and the scan
// open the blob independently.
Stream stream = new MemoryStream(_ndjsonBytes, writable: false);
return Task.FromResult<(Stream, ETag)?>((stream, _etag));
}
}
private sealed class FakeBlobStorageService : BlobStorageService
{
public FakeBlobStorageService()
: base(new Azure.Storage.Blobs.BlobServiceClient(new Uri("https://example.invalid"),
new Azure.Storage.StorageSharedKeyCredential("x", Convert.ToBase64String(new byte[32]))),
NullLogger<BlobStorageService>.Instance) { }
}
}
@@ -0,0 +1,149 @@
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Backup;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Pure-function checks for <see cref="RestoreTablePreflightValidator"/>. The
/// preflight gates every <c>POST .../restore-row</c> request with no I/O — fail
/// fast before the heavy validator runs.
/// </summary>
public class RestoreTablePreflightValidatorTests
{
private static readonly RestoreTablePreflightValidator Sut = new();
[Fact]
public void ValidateRowRequest_accepts_valid_preview()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "tenant1",
RowKey = "rule1",
Mode = RestoreRowMode.Preview,
};
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw
}
[Fact]
public void ValidateRowRequest_accepts_valid_commit_with_sha_and_etag()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "tenant1",
RowKey = "rule1",
Mode = RestoreRowMode.Commit,
IfSha256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
IfCurrentETag = "\"0x8DABCDEF\"",
};
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw
}
[Fact]
public void ValidateRowRequest_accepts_commit_with_null_etag_when_row_was_missing()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "tenant1",
RowKey = "rule1",
Mode = RestoreRowMode.Commit,
IfSha256 = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
IfCurrentETag = null,
};
Sut.ValidateRowRequest("20260522T040000Z_a1b2c3d4", req); // no throw — Add-path
}
[Fact]
public void ValidateRowRequest_rejects_empty_backupId()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "p",
RowKey = "r",
Mode = RestoreRowMode.Preview,
};
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("", req));
Assert.Equal("InvalidBackupId", ex.Code);
}
[Fact]
public void ValidateRowRequest_rejects_table_not_in_catalog()
{
var req = new RestoreRowRequest
{
TableName = "RandomOtherTable",
PartitionKey = "p",
RowKey = "r",
Mode = RestoreRowMode.Preview,
};
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
Assert.Equal("InvalidTable", ex.Code);
}
[Fact]
public void ValidateRowRequest_rejects_null_partition_key()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = null!,
RowKey = "r",
Mode = RestoreRowMode.Preview,
};
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
Assert.Equal("InvalidKeys", ex.Code);
}
[Fact]
public void ValidateRowRequest_commit_requires_ifSha256()
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "p",
RowKey = "r",
Mode = RestoreRowMode.Commit,
IfSha256 = null,
};
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
Assert.Equal("MissingPrecondition", ex.Code);
}
[Theory]
[InlineData("not-hex")]
[InlineData("ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789")] // uppercase rejected
[InlineData("0123456789abcdef")] // too short
public void ValidateRowRequest_commit_requires_lowercase_64_hex(string sha)
{
var req = new RestoreRowRequest
{
TableName = Constants.TableNames.AnalyzeRules,
PartitionKey = "p",
RowKey = "r",
Mode = RestoreRowMode.Commit,
IfSha256 = sha,
};
var ex = Assert.Throws<BackupTerminalException>(() => Sut.ValidateRowRequest("backup1", req));
Assert.Equal("MissingPrecondition", ex.Code);
}
[Theory]
[InlineData(nameof(Constants.TableNames.GlobalAdmins), true)]
[InlineData(nameof(Constants.TableNames.TenantAdmins), true)]
[InlineData(nameof(Constants.TableNames.McpUsers), true)]
[InlineData(nameof(Constants.TableNames.AnalyzeRules), false)]
[InlineData(nameof(Constants.TableNames.Feedback), false)]
public void IsAuthTable_flags_only_three_security_tables(string tableNameField, bool expected)
{
var tableName = typeof(Constants.TableNames).GetField(tableNameField)!.GetValue(null) as string;
Assert.Equal(expected, Sut.IsAuthTable(tableName!));
}
}
@@ -0,0 +1,156 @@
using System;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Backup;
using AutopilotMonitor.Shared.Models.Backup;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Backup
{
/// <summary>
/// <c>POST /api/global/backups/{backupId}/restore-row</c> — single-row preview
/// + commit endpoint for the critical-table backup restore (plan §PR2).
/// GA-only via <c>EndpointAccessPolicyCatalog</c>.
/// <para>
/// Synchronous on commit (1 row → Function timeout unproblematic), unlike the
/// full-table restore which goes via the 202+queue pattern. PartitionKey and
/// RowKey live in the body, not the URL: Azure Tables permits <c>/</c>,
/// <c>+</c>, and <c>%</c> in PK/RK which the Functions router would mangle.
/// </para>
/// </summary>
public class RestoreRowFunction
{
private readonly RestoreTablePreflightValidator _preflight;
private readonly CriticalTableRestoreService _restoreService;
private readonly OpsEventService _opsEvents;
private readonly ILogger<RestoreRowFunction> _logger;
public RestoreRowFunction(
RestoreTablePreflightValidator preflight,
CriticalTableRestoreService restoreService,
OpsEventService opsEvents,
ILogger<RestoreRowFunction> logger)
{
_preflight = preflight;
_restoreService = restoreService;
_opsEvents = opsEvents;
_logger = logger;
}
[Function("RestoreRow")]
public async Task<HttpResponseData> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "global/backups/{backupId}/restore-row")] HttpRequestData req,
string backupId)
{
var ct = req.FunctionContext.CancellationToken;
// 1. Parse body
RestoreRowRequest? body;
try
{
body = await JsonSerializer.DeserializeAsync<RestoreRowRequest>(
req.Body, BackupManifestJson.SerializerOptions, ct).ConfigureAwait(false);
}
catch (JsonException ex)
{
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, "InvalidBody", $"request body is not valid JSON: {ex.Message}").ConfigureAwait(false);
}
if (body == null)
{
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, "MissingBody", "request body is required").ConfigureAwait(false);
}
// 2. Lightweight preflight (no I/O)
try
{
_preflight.ValidateRowRequest(backupId, body);
}
catch (BackupTerminalException ex)
{
return await WriteErrorAsync(req, HttpStatusCode.BadRequest, ex.Code, ex.Message).ConfigureAwait(false);
}
// 3. Dispatch on mode
try
{
if (body.Mode == RestoreRowMode.Preview)
{
var preview = await _restoreService.PreviewRowAsync(
backupId, body.TableName, body.PartitionKey, body.RowKey, ct).ConfigureAwait(false);
return await WriteJsonAsync(req, HttpStatusCode.OK, preview).ConfigureAwait(false);
}
else
{
var actor = TenantHelper.GetUserIdentifier(req) ?? "GlobalAdmin";
var commit = await _restoreService.CommitRowAsync(
backupId, body.TableName, body.PartitionKey, body.RowKey,
body.IfSha256!, body.IfCurrentETag, ct).ConfigureAwait(false);
// Fire-and-forget audit; OpsEventService catches its own writes.
try
{
await _opsEvents.RecordBackupRowRestoredAsync(
backupId, body.TableName, body.PartitionKey, body.RowKey, actor,
commit.Outcome.ToString()).ConfigureAwait(false);
}
catch (Exception evtEx)
{
_logger.LogWarning(evtEx, "RestoreRow: ops event recording failed — write itself succeeded");
}
return await WriteJsonAsync(req, HttpStatusCode.OK, commit).ConfigureAwait(false);
}
}
catch (BackupTerminalException ex)
{
var status = MapErrorCodeToStatus(ex.Code);
return await WriteErrorAsync(req, status, ex.Code, ex.Message).ConfigureAwait(false);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested)
{
// Client went away.
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "RestoreRow: unexpected failure for backupId={BackupId} table={Table}", backupId, body.TableName);
return await WriteErrorAsync(req, HttpStatusCode.InternalServerError, "InternalError", ex.Message).ConfigureAwait(false);
}
}
private static HttpStatusCode MapErrorCodeToStatus(string code) => code switch
{
"BackupNotFound" => HttpStatusCode.NotFound,
"RowNotInBackup" => HttpStatusCode.NotFound,
"TableNotInBackup" => HttpStatusCode.Conflict,
"ManifestCorrupt" => HttpStatusCode.Conflict,
"ManifestSchemaUnsupported" => HttpStatusCode.Conflict,
"IntegrityCheckFailed" => HttpStatusCode.Conflict,
"BlobChangedSinceValidation" => HttpStatusCode.Conflict,
"RowChangedSinceValidation" => HttpStatusCode.Conflict,
"CurrentRowChanged" => HttpStatusCode.Conflict,
"MaintenanceInProgress" => HttpStatusCode.Conflict,
_ => HttpStatusCode.BadRequest,
};
private static async Task<HttpResponseData> WriteJsonAsync(HttpRequestData req, HttpStatusCode status, object body)
{
var response = req.CreateResponse(status);
response.Headers.Add("Content-Type", "application/json; charset=utf-8");
var json = JsonSerializer.Serialize(body, BackupManifestJson.SerializerOptions);
await response.WriteStringAsync(json).ConfigureAwait(false);
return response;
}
private static async Task<HttpResponseData> WriteErrorAsync(HttpRequestData req, HttpStatusCode status, string code, string message)
{
return await WriteJsonAsync(req, status, new { error = code, message }).ConfigureAwait(false);
}
}
}
@@ -295,6 +295,11 @@ builder.Services.AddSingleton<
// pattern per plan §Wave4 #1 — avoids the QueueTrigger-specific connection app-setting).
builder.Services.AddHostedService<
AutopilotMonitor.Functions.Services.Backup.Queue.CriticalTableBackupQueueWorker>();
// PR2: single-row restore. Both validators are pure / I/O-only; the restore
// service is stateless (lease lifecycle is owned per-call). Singleton is safe.
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.RestoreTablePreflightValidator>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.BackupRestoreInputValidator>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Backup.CriticalTableRestoreService>();
// Programmatic SignalR push for background tasks (rule engine, vulnerability correlation)
builder.Services.AddSingleton<SignalRNotificationService>();
@@ -288,11 +288,14 @@ public static class EndpointAccessPolicyCatalog
new("DELETE", "versions/block/{encodedPattern}", EndpointPolicy.GlobalAdminOnly),
new("POST", "maintenance/trigger", EndpointPolicy.GlobalAdminOnly),
// ── Critical-Table Backup (plan §PR1) ──────────────────────────────────
// ── Critical-Table Backup (plan §PR1+§PR2) ─────────────────────────────
new("POST", "global/backups/trigger", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/backups", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/backups/{backupId}", EndpointPolicy.GlobalAdminOnly),
new("GET", "global/backups/jobs/{jobId}", EndpointPolicy.GlobalAdminOnly),
// PR2: single-row preview + commit. PartitionKey + RowKey live in the body
// (Azure Tables permits / + % in PK/RK; the route wouldn't carry them safely).
new("POST", "global/backups/{backupId}/restore-row", EndpointPolicy.GlobalAdminOnly),
// ── Tenant Offboarding Customs Archive (PR3.B) ─────────────────────────
// Snapshot of each tenant's custom GatherRules / AnalyzeRules / ImeLogPatterns
@@ -0,0 +1,161 @@
using System;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.Models.Backup;
using Azure;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Backup
{
/// <summary>
/// Heavy, I/O-bound validation that has to run before any restore write touches
/// the live table. Performs:
/// <list type="number">
/// <item>Manifest exists + parses (plan §Wave11 #4)</item>
/// <item>Manifest schema version is supported</item>
/// <item>Entry for the requested <c>tableName</c> exists with <c>Status ∈ {Ok, Empty}</c></item>
/// <item>Entry's <c>BlobName</c> matches the canonical <c>{backupId}/{tableName}.ndjson</c></item>
/// <item><b>ETag-pin + SHA in a single stream read</b> (plan §Wave12 #3):
/// GetProperties → pin ETag → OpenReadAsync(IfMatch=etag) → SHA-256 over that stream.
/// Mismatch → <c>IntegrityCheckFailed</c>. Race on the blob → <c>BlobChangedSinceValidation</c>.</item>
/// </list>
/// On success returns the parsed manifest, the matching entry, and the pinned
/// ETag — the restore service uses the SAME ETag for the line-scan read so the
/// hash and the scan see byte-identical bytes.
/// </summary>
public sealed class BackupRestoreInputValidator
{
private const int SupportedSchemaVersion = 1;
private readonly BlobBackupStore _store;
private readonly ILogger<BackupRestoreInputValidator> _logger;
public BackupRestoreInputValidator(BlobBackupStore store, ILogger<BackupRestoreInputValidator> logger)
{
_store = store;
_logger = logger;
}
/// <summary>
/// Result of a successful validation. The pinned ETag is returned so the
/// restore service can OpenReadAsync the SAME blob version it was just
/// hashed against.
/// </summary>
public sealed class ValidationResult
{
public CriticalTableBackupManifest Manifest { get; init; } = default!;
public CriticalTableBackupTableEntry Entry { get; init; } = default!;
public ETag BlobETag { get; init; }
}
/// <summary>
/// Runs all checks. Throws <see cref="BackupTerminalException"/> on the
/// first failure with a stable error code suitable for the HTTP envelope.
/// </summary>
public async Task<ValidationResult> ValidateAsync(string backupId, string tableName, CancellationToken ct = default)
{
if (string.IsNullOrEmpty(backupId)) throw new BackupTerminalException("InvalidBackupId", "backupId is required");
if (string.IsNullOrEmpty(tableName)) throw new BackupTerminalException("InvalidTable", "tableName is required");
// 1. Manifest exists + parseable
var (payload, _) = await _store.ReadManifestAsync(backupId, ct).ConfigureAwait(false);
if (payload == null)
{
throw new BackupTerminalException("BackupNotFound", $"manifest for backupId '{backupId}' was not found");
}
CriticalTableBackupManifest manifest;
try
{
manifest = JsonSerializer.Deserialize<CriticalTableBackupManifest>(payload, BackupManifestJson.SerializerOptions)
?? throw new BackupTerminalException("ManifestCorrupt", "manifest deserialised to null");
}
catch (BackupTerminalException) { throw; }
catch (Exception ex)
{
throw new BackupTerminalException("ManifestCorrupt", $"manifest JSON parse failed: {ex.Message}", ex);
}
if (manifest.SchemaVersion != SupportedSchemaVersion)
{
throw new BackupTerminalException(
"ManifestSchemaUnsupported",
$"manifest schemaVersion={manifest.SchemaVersion} not supported by this restore endpoint (expected {SupportedSchemaVersion})");
}
// 2. Per-table entry exists + Status is restorable
var entry = manifest.Tables.FirstOrDefault(t => string.Equals(t.TableName, tableName, StringComparison.Ordinal));
if (entry == null)
{
throw new BackupTerminalException(
"TableNotInBackup",
$"no entry for tableName '{tableName}' in manifest {backupId}");
}
if (entry.Status != TableBackupStatus.Ok && entry.Status != TableBackupStatus.Empty)
{
throw new BackupTerminalException(
"TableNotInBackup",
$"manifest entry for '{tableName}' has Status={entry.Status} — restore is only allowed from Ok or Empty entries");
}
// 3. Canonical BlobName invariant
var expectedBlobName = BlobBackupStore.BuildNdjsonBlobName(backupId, tableName);
if (!string.Equals(entry.BlobName, expectedBlobName, StringComparison.Ordinal))
{
throw new BackupTerminalException(
"ManifestCorrupt",
$"manifest entry BlobName='{entry.BlobName}' does not match canonical '{expectedBlobName}'");
}
// 4. ETag-pin + SHA in a single stream read
var openResult = await _store.OpenNdjsonReadAsync(backupId, tableName, ct).ConfigureAwait(false);
if (openResult == null)
{
throw new BackupTerminalException(
"BackupNotFound",
$"NDJSON blob '{expectedBlobName}' is missing despite manifest entry present — orphan manifest");
}
var (stream, blobETag) = openResult.Value;
string actualSha;
try
{
using (stream)
using (var sha = SHA256.Create())
{
var hashBytes = await sha.ComputeHashAsync(stream, ct).ConfigureAwait(false);
actualSha = Convert.ToHexString(hashBytes).ToLowerInvariant();
}
}
catch (RequestFailedException ex) when (ex.Status == 412)
{
// The IfMatch precondition can also fire mid-stream if the blob was
// overwritten between OpenRead and the final byte. Same recovery as
// OpenNdjsonReadAsync's 412 path.
throw new BackupTerminalException("BlobChangedSinceValidation", "NDJSON blob changed during SHA verification", ex);
}
var expectedSha = (entry.Sha256Hex ?? string.Empty).ToLowerInvariant();
if (!string.Equals(actualSha, expectedSha, StringComparison.Ordinal))
{
_logger.LogError(
"BackupRestoreInputValidator: SHA mismatch for backupId={BackupId} table={Table} (expected={Expected}, actual={Actual}) — refusing restore",
backupId, tableName, expectedSha, actualSha);
throw new BackupTerminalException(
"IntegrityCheckFailed",
$"SHA-256 mismatch for table '{tableName}' (expected {expectedSha}, got {actualSha}) — backup may be tampered or partially written");
}
return new ValidationResult
{
Manifest = manifest,
Entry = entry,
BlobETag = blobETag,
};
}
}
}
@@ -0,0 +1,31 @@
using System;
namespace AutopilotMonitor.Functions.Services.Backup
{
/// <summary>
/// Deterministic, non-retryable failure raised by the backup/restore paths
/// (Manifest missing, SHA mismatch, Blob ETag changed, auth-table block,
/// confirmation-token typo, lease busy …). The HTTP layer maps these to a
/// specific 4xx; the queue worker maps them to <c>JobState.BlockedTerminal</c>
/// + <c>DeleteMessage</c> on first attempt (no retry, no poison).
/// </summary>
public sealed class BackupTerminalException : Exception
{
/// <summary>
/// Stable short code suitable for the HTTP error envelope
/// (<c>{ error: Code, message: Message }</c>). UI uses this to decide which
/// banner to render (e.g. <c>CurrentRowChanged</c> → "open Preview again").
/// </summary>
public string Code { get; }
public BackupTerminalException(string code, string message) : base(message)
{
Code = code;
}
public BackupTerminalException(string code, string message, Exception inner) : base(message, inner)
{
Code = code;
}
}
}
@@ -173,6 +173,57 @@ namespace AutopilotMonitor.Functions.Services.Backup
}
}
/// <summary>
/// Opens the per-table NDJSON blob for read with a pinned ETag so the SHA
/// hash + subsequent line scan operate on the exact same blob version (plan
/// §Wave11 #4 + Wave12 #3). Workflow:
/// <list type="number">
/// <item>GetProperties → pin ETag</item>
/// <item>OpenReadAsync with <c>IfMatch=ETag</c>; 412 throws
/// <see cref="BackupTerminalException"/> with code <c>BlobChangedSinceValidation</c></item>
/// </list>
/// Caller is responsible for disposing the returned stream. Returns
/// <c>null</c> if the blob does not exist (404).
/// </summary>
public virtual async Task<(Stream Stream, ETag ETag)?> OpenNdjsonReadAsync(
string backupId,
string tableName,
CancellationToken ct = default)
{
if (string.IsNullOrEmpty(backupId)) throw new ArgumentException("backupId required", nameof(backupId));
if (string.IsNullOrEmpty(tableName)) throw new ArgumentException("tableName required", nameof(tableName));
await EnsureContainerAsync(ct).ConfigureAwait(false);
var blob = GetContainer().GetBlobClient(BuildNdjsonBlobName(backupId, tableName));
BlobProperties props;
try
{
props = (await blob.GetPropertiesAsync(cancellationToken: ct).ConfigureAwait(false)).Value;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return null;
}
var etag = props.ETag;
try
{
var stream = await blob.OpenReadAsync(new BlobOpenReadOptions(allowModifications: false)
{
Conditions = new BlobRequestConditions { IfMatch = etag },
}, ct).ConfigureAwait(false);
return (stream, etag);
}
catch (RequestFailedException ex) when (ex.Status == 412)
{
throw new BackupTerminalException(
"BlobChangedSinceValidation",
$"NDJSON blob '{BuildNdjsonBlobName(backupId, tableName)}' changed between GetProperties and OpenRead — refusing to read a half-version",
ex);
}
}
/// <summary>
/// Lists every backupId whose <c>manifest.json</c> exists — i.e. every run that
/// reached the durability anchor (plan §3 "manifest is the backup"). Orphan
@@ -0,0 +1,480 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Backup;
using AutopilotMonitor.Shared.Models.Deletion;
using Azure;
using Azure.Data.Tables;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Backup
{
/// <summary>
/// Single-row preview + commit primitives for the critical-table backup restore
/// path (plan §PR2). Two phases:
/// <list type="bullet">
/// <item><b>PreviewRowAsync</b> — heavy <see cref="BackupRestoreInputValidator"/>
/// (manifest + SHA + ETag-pin), then line-by-line scan of the same ETag-pinned
/// NDJSON blob for the requested (pk, rk). Reads the live row via
/// <c>GetEntityIfExistsAsync</c>, returns backup+current+diff+rowHash+currentETag.
/// <b>No write</b>, no lease — purely read-only.</item>
/// <item><b>CommitRowAsync</b> — acquires the maintenance lease (short-lived but
/// with the standard 45 s renewal loop so a slow blob fetch cannot drop it),
/// re-runs the validator, recomputes the row hash, re-reads the live row,
/// verifies <c>ifSha256</c> and <c>ifCurrentETag</c> echoed by the client, then
/// writes <c>UpdateEntityAsync(IfMatch=etag, Replace)</c> for existing rows or
/// <c>AddEntityAsync</c> for missing rows. 412/404/409 → <c>CurrentRowChanged</c>.</item>
/// </list>
/// <para>
/// This service is intentionally NOT used for full-table restores — the bulk
/// path lives in <c>UpsertRowsByExactKeysInBatchesAsync</c> and has no per-row
/// ETag-CAS. PR2 covers only single-row; PR3/PR4 wire the bulk path.
/// </para>
/// </summary>
public sealed class CriticalTableRestoreService
{
private readonly TableStorageService _tables;
private readonly BlobBackupStore _store;
private readonly BackupRestoreInputValidator _validator;
private readonly ILoggerFactory _loggerFactory;
private readonly ILogger<CriticalTableRestoreService> _logger;
public CriticalTableRestoreService(
TableStorageService tables,
BlobBackupStore store,
BackupRestoreInputValidator validator,
ILoggerFactory loggerFactory,
ILogger<CriticalTableRestoreService> logger)
{
_tables = tables;
_store = store;
_validator = validator;
_loggerFactory = loggerFactory;
_logger = logger;
}
// ── Preview ─────────────────────────────────────────────────────────────
/// <summary>
/// Heavy validate + scan + diff. No write, no lease.
/// </summary>
public async Task<RestoreRowPreviewResponse> PreviewRowAsync(
string backupId,
string tableName,
string partitionKey,
string rowKey,
CancellationToken ct = default)
{
// 1. Heavy validate (manifest, SHA, ETag-pin).
var validation = await _validator.ValidateAsync(backupId, tableName, ct).ConfigureAwait(false);
// 2. Scan the SAME ETag-pinned blob version for the requested (pk, rk).
// We re-open the blob with IfMatch=validation.BlobETag so the scan sees
// byte-identical bytes to the hash. This is the central correctness
// invariant — without re-pinning, a concurrent overwrite could change
// the bytes between hash and scan.
var openResult = await OpenAtPinnedETagAsync(backupId, tableName, validation.BlobETag, ct).ConfigureAwait(false);
var (dump, ndjsonLineBytes) = await FindLineAsync(openResult, partitionKey, rowKey, ct).ConfigureAwait(false);
if (dump == null)
{
throw new BackupTerminalException(
"RowNotInBackup",
$"row (pk='{partitionKey}', rk='{rowKey}') was not found in backup {backupId}/{tableName}");
}
// 3. Read the live row.
var tableClient = _tables.GetTableClient(tableName);
var liveResponse = await tableClient
.GetEntityIfExistsAsync<TableEntity>(partitionKey, rowKey, cancellationToken: ct)
.ConfigureAwait(false);
TableEntity? liveEntity = liveResponse.HasValue ? liveResponse.Value : null;
string? currentETag = liveEntity != null ? liveEntity.ETag.ToString() : null;
// 4. Build the diff + snapshots.
var backupSnap = ToSnapshotMap(dump);
var currentSnap = liveEntity != null ? LiveEntityToSnapshotMap(liveEntity) : null;
var diff = BuildDiff(backupSnap, currentSnap);
// 5. Hash the exact NDJSON line (without trailing \n). See plan §Wave15 #5
// — no canonicalisation; the original NDJSON bytes are the canonical form.
var rowSha = ComputeSha256Hex(ndjsonLineBytes);
return new RestoreRowPreviewResponse
{
BackupId = backupId,
TableName = tableName,
PartitionKey = partitionKey,
RowKey = rowKey,
BackupProperties = backupSnap,
CurrentProperties = currentSnap,
Diff = diff,
RowSha256 = rowSha,
CurrentETag = currentETag,
IsAuthTable = Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden
.Contains(tableName, StringComparer.Ordinal),
};
}
// ── Commit ──────────────────────────────────────────────────────────────
/// <summary>
/// Acquires the maintenance lease (briefly), re-validates, recomputes the
/// row hash, then runs a conditional write keyed on <paramref name="ifCurrentETag"/>.
/// All five CurrentRowChanged paths from the plan are covered:
/// <list type="bullet">
/// <item>Row existed in preview + same ETag → <c>UpdateEntity(Replace)</c></item>
/// <item>Row existed in preview + different ETag (412) → 409 <c>CurrentRowChanged</c></item>
/// <item>Row deleted between preview + commit (404) → 409 <c>CurrentRowChanged</c></item>
/// <item>Row missing in preview → <c>AddEntity</c></item>
/// <item>Row missing in preview + meanwhile created (409) → 409 <c>CurrentRowChanged</c></item>
/// </list>
/// </summary>
public async Task<RestoreRowCommitResponse> CommitRowAsync(
string backupId,
string tableName,
string partitionKey,
string rowKey,
string ifSha256,
string? ifCurrentETag,
CancellationToken ct = default)
{
// Acquire the maintenance lease BEFORE re-reading anything that could
// race with a parallel full-table restore. Without the lease a Single-Row
// commit could be overwritten by an in-flight bulk replace-all started
// microseconds later (plan §Wave15 #4).
Azure.Storage.Blobs.Specialized.BlobLeaseClient leaseClient;
try
{
leaseClient = await _store.AcquireMaintenanceLeaseAsync(leaseDuration: null, ct: ct).ConfigureAwait(false);
}
catch (LeaseHeldException ex)
{
throw new BackupTerminalException(
"MaintenanceInProgress",
"another maintenance operation is in progress — please retry shortly",
ex);
}
using var handlerCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
var holder = new MaintenanceLeaseHolder(
leaseClient,
handlerCts,
_loggerFactory.CreateLogger<MaintenanceLeaseHolder>());
try
{
// Re-validate from scratch under the lease. The validator's ETag-pin
// returns a NEW ETag for the current blob version — we use that pin
// for the scan below (NOT the preview's, which may now be stale if a
// parallel write+restore landed in between).
var validation = await _validator.ValidateAsync(backupId, tableName, handlerCts.Token).ConfigureAwait(false);
var openResult = await OpenAtPinnedETagAsync(backupId, tableName, validation.BlobETag, handlerCts.Token).ConfigureAwait(false);
var (dump, ndjsonLineBytes) = await FindLineAsync(openResult, partitionKey, rowKey, handlerCts.Token).ConfigureAwait(false);
if (dump == null)
{
throw new BackupTerminalException(
"RowNotInBackup",
$"row (pk='{partitionKey}', rk='{rowKey}') was not found in backup {backupId}/{tableName}");
}
// Re-compute SHA on the fresh line bytes. If the operator's echoed
// ifSha256 does not match, the backup has changed (or the operator
// typed by hand) — refuse.
var freshSha = ComputeSha256Hex(ndjsonLineBytes);
if (!string.Equals(freshSha, (ifSha256 ?? string.Empty).ToLowerInvariant(), StringComparison.Ordinal))
{
throw new BackupTerminalException(
"RowChangedSinceValidation",
$"backup row SHA-256 changed between preview and commit (preview echoed {ifSha256}, fresh {freshSha}) — re-open the preview to refresh");
}
// Build the entity from the dump. ConvertDumpToEntity also applies
// the Sessions-specific reset (no-op for other tables).
var entity = TableStorageService.ConvertDumpToEntity(dump, tableName);
var tableClient = _tables.GetTableClient(tableName);
if (!string.IsNullOrEmpty(ifCurrentETag))
{
// Preview saw a live row; commit requires the live ETag still matches.
var etag = new ETag(ifCurrentETag);
try
{
await tableClient.UpdateEntityAsync(entity, etag, TableUpdateMode.Replace, handlerCts.Token).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (ex.Status == 412 || ex.Status == 404)
{
throw new BackupTerminalException(
"CurrentRowChanged",
$"live row was modified or deleted since preview (status={ex.Status}) — re-open the preview",
ex);
}
return new RestoreRowCommitResponse
{
BackupId = backupId,
TableName = tableName,
PartitionKey = partitionKey,
RowKey = rowKey,
Outcome = RestoreRowCommitOutcome.Replaced,
};
}
else
{
// Preview saw NO live row; commit requires the row is still absent.
try
{
await tableClient.AddEntityAsync(entity, handlerCts.Token).ConfigureAwait(false);
}
catch (RequestFailedException ex) when (IsAlreadyExistsStatus(ex))
{
throw new BackupTerminalException(
"CurrentRowChanged",
"live row was created since preview — re-open the preview",
ex);
}
return new RestoreRowCommitResponse
{
BackupId = backupId,
TableName = tableName,
PartitionKey = partitionKey,
RowKey = rowKey,
Outcome = RestoreRowCommitOutcome.Inserted,
};
}
}
finally
{
// DisposeAsync stops the renewal loop deterministically before releasing.
try { await holder.DisposeAsync().ConfigureAwait(false); }
catch (Exception ex) { _logger.LogWarning(ex, "CriticalTableRestoreService: lease holder dispose threw"); }
}
}
// ── Helpers ─────────────────────────────────────────────────────────────
private async Task<(Stream Stream, ETag ETag)> OpenAtPinnedETagAsync(
string backupId, string tableName, ETag expectedETag, CancellationToken ct)
{
var open = await _store.OpenNdjsonReadAsync(backupId, tableName, ct).ConfigureAwait(false);
if (open == null)
{
throw new BackupTerminalException(
"BackupNotFound",
$"NDJSON blob for {backupId}/{tableName}.ndjson disappeared between validation and scan");
}
if (!open.Value.ETag.Equals(expectedETag))
{
// The validator already opened+read; if the ETag changed between
// validator and scan, the blob was overwritten.
try { open.Value.Stream.Dispose(); } catch { /* best effort */ }
throw new BackupTerminalException(
"BlobChangedSinceValidation",
"NDJSON blob ETag changed between validation hash and scan — refusing to read mismatching bytes");
}
return open.Value;
}
/// <summary>
/// Scans an NDJSON stream line-by-line for the (pk, rk) match. Returns the
/// parsed dump plus the raw line bytes (without trailing <c>\n</c>) so the
/// caller can SHA-256 the same bytes that get echoed via <c>ifSha256</c>.
/// </summary>
private static async Task<(DeletionRowDump? Dump, byte[] LineBytes)> FindLineAsync(
(Stream Stream, ETag ETag) open,
string partitionKey,
string rowKey,
CancellationToken ct)
{
using var stream = open.Stream;
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, leaveOpen: false);
string? line;
while ((line = await reader.ReadLineAsync(ct).ConfigureAwait(false)) != null)
{
if (line.Length == 0) continue;
// Quick reject on PK/RK substring miss before deserialising —
// a worthwhile micro-opt because most lines won't match.
DeletionRowDump? dump;
try
{
dump = JsonSerializer.Deserialize<DeletionRowDump>(line, BackupManifestJson.SerializerOptions);
}
catch (JsonException ex)
{
throw new BackupTerminalException(
"ManifestCorrupt",
$"NDJSON line failed to parse as DeletionRowDump: {ex.Message}",
ex);
}
if (dump == null) continue;
if (!string.Equals(dump.Pk, partitionKey, StringComparison.Ordinal)) continue;
if (!string.Equals(dump.Rk, rowKey, StringComparison.Ordinal)) continue;
var lineBytes = Encoding.UTF8.GetBytes(line);
return (dump, lineBytes);
}
return (null, Array.Empty<byte>());
}
private static Dictionary<string, RestoreRowPropertySnapshot> ToSnapshotMap(DeletionRowDump dump)
{
var map = new Dictionary<string, RestoreRowPropertySnapshot>(StringComparer.Ordinal);
foreach (var kvp in dump.Props)
{
map[kvp.Key] = new RestoreRowPropertySnapshot
{
EdmType = kvp.Value.EdmType,
Value = kvp.Value.Value,
};
}
return map;
}
private static Dictionary<string, RestoreRowPropertySnapshot> LiveEntityToSnapshotMap(TableEntity entity)
{
var map = new Dictionary<string, RestoreRowPropertySnapshot>(StringComparer.Ordinal);
foreach (var kvp in entity)
{
// Skip Azure-system columns; PartitionKey and RowKey are top-level on
// the dump, Timestamp + odata.etag are SDK-managed.
if (kvp.Key == "PartitionKey" || kvp.Key == "RowKey" || kvp.Key == "Timestamp" || kvp.Key == "odata.etag")
continue;
map[kvp.Key] = ConvertLiveValueToSnapshot(kvp.Value);
}
return map;
}
private static RestoreRowPropertySnapshot ConvertLiveValueToSnapshot(object? value)
{
// Mirror the dump's EDM tagging — we don't have full SDK type info on
// the live row beyond CLR types, but the round-trip rules from
// TableEntityDumpConverter give us enough fidelity for a diff UI.
//
// For property comparison we only need a deterministic JSON
// representation of the value, so we hand-encode each EDM type
// identically to how MapEntityToDump would have encoded it. That keeps
// the diff stable: backup-vs-current of a long stays "long==long" even
// though the live SDK already typed it as Int64.
switch (value)
{
case null:
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.Null, null);
case string s:
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.String, s);
case bool b:
return MakeSnapshot(DeletionPropEdmType.Boolean, b ? JsonValueKind.True : JsonValueKind.False, b);
case int i:
return MakeSnapshot(DeletionPropEdmType.Int32, JsonValueKind.Number, i);
case long l:
return MakeSnapshot(DeletionPropEdmType.Int64, JsonValueKind.Number, l);
case double d:
return MakeSnapshot(DeletionPropEdmType.Double, JsonValueKind.Number, d);
case DateTime dt:
return MakeSnapshot(DeletionPropEdmType.DateTime, JsonValueKind.String,
dt.ToUniversalTime().ToString("o", System.Globalization.CultureInfo.InvariantCulture));
case DateTimeOffset dto:
return MakeSnapshot(DeletionPropEdmType.DateTime, JsonValueKind.String,
dto.UtcDateTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture));
case Guid g:
return MakeSnapshot(DeletionPropEdmType.Guid, JsonValueKind.String, g.ToString());
case byte[] bytes:
return MakeSnapshot(DeletionPropEdmType.Binary, JsonValueKind.String, Convert.ToBase64String(bytes));
default:
return MakeSnapshot(DeletionPropEdmType.String, JsonValueKind.String, value.ToString());
}
}
private static RestoreRowPropertySnapshot MakeSnapshot(string edmType, JsonValueKind kind, object? value)
{
// Cheap round-trip via JsonSerializer to get a normalized JsonElement —
// the diff serializer (BackupManifestJson.SerializerOptions) re-emits it
// verbatim, and JsonElement equality is byte-based, so this keeps the
// diff stable across processes.
var bytes = JsonSerializer.SerializeToUtf8Bytes(value, BackupManifestJson.SerializerOptions);
using var doc = JsonDocument.Parse(bytes);
return new RestoreRowPropertySnapshot
{
EdmType = edmType,
Value = doc.RootElement.Clone(),
};
// 'kind' parameter retained for self-documentation at call sites.
}
private static List<RestoreRowPropertyDiff> BuildDiff(
Dictionary<string, RestoreRowPropertySnapshot> backup,
Dictionary<string, RestoreRowPropertySnapshot>? current)
{
var diff = new List<RestoreRowPropertyDiff>();
var keys = new HashSet<string>(backup.Keys, StringComparer.Ordinal);
if (current != null) keys.UnionWith(current.Keys);
foreach (var name in keys.OrderBy(x => x, StringComparer.Ordinal))
{
backup.TryGetValue(name, out var b);
RestoreRowPropertySnapshot? c = null;
current?.TryGetValue(name, out c);
RestoreRowDiffKind kind;
if (b != null && c == null) kind = current == null ? RestoreRowDiffKind.Added : RestoreRowDiffKind.Added;
else if (b == null && c != null) kind = RestoreRowDiffKind.Removed;
else if (b != null && c != null)
{
kind = SnapshotsEqual(b, c) ? RestoreRowDiffKind.Unchanged : RestoreRowDiffKind.Changed;
}
else continue;
diff.Add(new RestoreRowPropertyDiff
{
Name = name,
Kind = kind,
Backup = b,
Current = c,
});
}
return diff;
}
private static bool SnapshotsEqual(RestoreRowPropertySnapshot a, RestoreRowPropertySnapshot b)
{
if (!string.Equals(a.EdmType, b.EdmType, StringComparison.Ordinal)) return false;
// JsonElement equality is by raw text — re-serialise both for a
// canonical byte-compare. Both go through the same SerializerOptions so
// ordering is stable.
var aBytes = JsonSerializer.SerializeToUtf8Bytes(a.Value, BackupManifestJson.SerializerOptions);
var bBytes = JsonSerializer.SerializeToUtf8Bytes(b.Value, BackupManifestJson.SerializerOptions);
return aBytes.AsSpan().SequenceEqual(bBytes);
}
private static string ComputeSha256Hex(byte[] data)
{
using var sha = SHA256.Create();
var hash = sha.ComputeHash(data);
return Convert.ToHexString(hash).ToLowerInvariant();
}
private static bool IsAlreadyExistsStatus(RequestFailedException ex)
{
if (ex.Status == 409) return true;
if (ex.Status == 400 && string.Equals(ex.ErrorCode, "EntityAlreadyExists", StringComparison.Ordinal)) return true;
return false;
}
}
}
@@ -0,0 +1,107 @@
using System;
using System.Linq;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models.Backup;
namespace AutopilotMonitor.Functions.Services.Backup
{
/// <summary>
/// Light, no-I/O pre-flight for the restore HTTP triggers (plan §Wave17 #2 +
/// Wave18 #2). Validates only the request shape — table name is in the
/// critical-backup catalog, mode is a known value, keys are present, auth-table
/// rules are respected. Heavier work (manifest parse, SHA verify, blob ETag-pin)
/// lives in <see cref="BackupRestoreInputValidator"/>.
/// <para>
/// Single-row preview is read-only against the live row and does not require
/// the destructive-acknowledgement / confirmation-token rituals the full-table
/// restore demands. The auth-table single-row path is explicitly allowed (with
/// a UI banner), full-table on auth-tables stays blocked at this gate.
/// </para>
/// </summary>
public sealed class RestoreTablePreflightValidator
{
/// <summary>
/// Validates a single-row restore request. Throws on the first failure with
/// a code suitable for HTTP envelope. Pure function, fully unit-testable.
/// </summary>
public void ValidateRowRequest(string backupId, RestoreRowRequest request)
{
if (string.IsNullOrEmpty(backupId))
{
throw new BackupTerminalException("InvalidBackupId", "backupId route segment is empty");
}
if (request == null)
{
throw new BackupTerminalException("MissingBody", "request body is required");
}
if (string.IsNullOrEmpty(request.TableName))
{
throw new BackupTerminalException("InvalidTable", "tableName is required");
}
if (!Constants.CriticalBackupTables.All.Contains(request.TableName, StringComparer.Ordinal))
{
throw new BackupTerminalException(
"InvalidTable",
$"tableName '{request.TableName}' is not in the critical-backup catalog — only the 15 critical tables are restorable");
}
// Empty PK/RK are technically legal in Azure Tables, so we only insist on
// non-null. The Validator further down may still reject if the dump line
// is not found for the given keys.
if (request.PartitionKey == null)
{
throw new BackupTerminalException("InvalidKeys", "partitionKey is required (may be empty string but not null)");
}
if (request.RowKey == null)
{
throw new BackupTerminalException("InvalidKeys", "rowKey is required (may be empty string but not null)");
}
if (request.Mode != RestoreRowMode.Preview && request.Mode != RestoreRowMode.Commit)
{
throw new BackupTerminalException("InvalidMode", $"mode must be 'preview' or 'commit', got '{request.Mode}'");
}
if (request.Mode == RestoreRowMode.Commit)
{
if (string.IsNullOrEmpty(request.IfSha256))
{
throw new BackupTerminalException(
"MissingPrecondition",
"ifSha256 is required on commit — echo the rowSha256 from the preview response");
}
if (!IsLowerHex64(request.IfSha256))
{
throw new BackupTerminalException(
"MissingPrecondition",
"ifSha256 must be 64 lowercase hex characters (SHA-256 hex digest)");
}
// ifCurrentETag is intentionally nullable: null carries the precondition
// "preview saw no live row" and triggers AddEntity on commit.
}
}
/// <summary>
/// True for tables whose IsEnabled column carries security semantics — UI
/// must render a warning banner and the operator must confirm explicitly.
/// Single-row commit on these is allowed (full-table is not).
/// </summary>
public bool IsAuthTable(string tableName)
{
return Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden
.Contains(tableName, StringComparer.Ordinal);
}
private static bool IsLowerHex64(string s)
{
if (s.Length != 64) return false;
for (var i = 0; i < s.Length; i++)
{
var c = s[i];
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'))) return false;
}
return true;
}
}
}
@@ -284,6 +284,29 @@ namespace AutopilotMonitor.Functions.Services
null, "System.Maintenance",
new { reason, triggeredBy });
/// <summary>
/// A GA operator restored a single row from a backup (plan §PR2). Warning-level
/// because the restore overwrote (or created) live data via ETag-CAS — operators
/// frequently want a Telegram ping for this event so a parallel admin sees the
/// audit trail in near-real-time. Payload carries the canonical
/// <c>{ container, blobName }</c> only — no SAS URL, per plan §Medium #6.
/// </summary>
public Task RecordBackupRowRestoredAsync(
string backupId, string tableName, string partitionKey, string rowKey, string actor, string outcome)
=> WriteAsync(OpsEventCategory.Maintenance, "BackupRowRestored", OpsEventSeverity.Warning,
$"Critical-table row restored: {tableName} (pk='{partitionKey}', rk='{rowKey}') from backup {backupId} by {actor} → {outcome}",
null, actor,
new
{
backupId,
tableName,
partitionKey,
rowKey,
outcome,
container = AutopilotMonitor.Shared.Constants.BlobContainers.CriticalTableBackups,
blobName = $"{backupId}/{tableName}.ndjson",
});
// ── Tenant ─────────────────────────────────────────────────────────────
/// <summary>
@@ -221,6 +221,7 @@ namespace AutopilotMonitor.Shared
public const string GlobalBackupsList = "/api/global/backups"; // GET → list of backups
public const string GlobalBackupsManifest = "/api/global/backups/{backupId}"; // GET → manifest detail
public const string GlobalBackupsJobStatus = "/api/global/backups/jobs/{jobId}"; // GET → BackupJobStatus
public const string GlobalBackupsRestoreRow = "/api/global/backups/{backupId}/restore-row"; // POST → preview|commit single row
}
// -----------------------------------------------------------------------
@@ -0,0 +1,147 @@
using System.Collections.Generic;
namespace AutopilotMonitor.Shared.Models.Backup
{
/// <summary>
/// Operating mode for <c>POST /api/global/backups/{backupId}/restore-row</c>.
/// Preview is a read-only diff; Commit is the conditional write under maintenance
/// lease + ETag-CAS.
/// </summary>
public enum RestoreRowMode
{
Preview,
Commit,
}
/// <summary>
/// Body of <c>POST /api/global/backups/{backupId}/restore-row</c>.
/// PartitionKey + RowKey are carried in the body (never the URL) because Azure
/// Tables permits <c>/</c>, <c>+</c>, <c>%</c> in PK/RK; URL-encoding them on the
/// route would interact poorly with the Functions router.
/// </summary>
public sealed class RestoreRowRequest
{
public string TableName { get; set; } = string.Empty;
public string PartitionKey { get; set; } = string.Empty;
public string RowKey { get; set; } = string.Empty;
public RestoreRowMode Mode { get; set; } = RestoreRowMode.Preview;
/// <summary>
/// SHA-256 (hex, lowercase) of the raw NDJSON line bytes echoed from the
/// matching Preview response. Required on Commit. The server re-computes the
/// hash on the fresh NDJSON line and rejects with 409 on mismatch (tamper /
/// blob churn after the preview).
/// </summary>
public string? IfSha256 { get; set; }
/// <summary>
/// Live-row ETag echoed from the matching Preview response.
/// <para>
/// <c>null</c> means "Preview saw no live row" → Commit uses
/// <c>AddEntity</c>. A non-null value means "Preview saw a live row with this
/// ETag" → Commit uses <c>UpdateEntity(ifMatch=etag, Replace)</c>. Mismatch
/// (412) or existence change → 409 <c>CurrentRowChanged</c>.
/// </para>
/// </summary>
public string? IfCurrentETag { get; set; }
}
/// <summary>
/// Response body of <c>mode=preview</c>. Contains the backup-row dump, the live
/// row (or null), a per-property diff, the row-hash to echo on commit, and the
/// live ETag (or null) to echo on commit.
/// </summary>
public sealed class RestoreRowPreviewResponse
{
public string BackupId { get; set; } = string.Empty;
public string TableName { get; set; } = string.Empty;
public string PartitionKey { get; set; } = string.Empty;
public string RowKey { get; set; } = string.Empty;
/// <summary>
/// EDM-tagged property snapshot from the backup NDJSON line. Keys are
/// case-faithful original Azure Table column names.
/// </summary>
public Dictionary<string, RestoreRowPropertySnapshot> BackupProperties { get; set; }
= new Dictionary<string, RestoreRowPropertySnapshot>();
/// <summary>
/// EDM-tagged property snapshot of the live row, or <c>null</c> if the live
/// row does not currently exist.
/// </summary>
public Dictionary<string, RestoreRowPropertySnapshot>? CurrentProperties { get; set; }
public List<RestoreRowPropertyDiff> Diff { get; set; } = new List<RestoreRowPropertyDiff>();
/// <summary>SHA-256 (hex, lowercase) of the raw NDJSON line bytes. Echo this on Commit.</summary>
public string RowSha256 { get; set; } = string.Empty;
/// <summary>Live-row ETag at preview time, or <c>null</c> if the live row did not exist.</summary>
public string? CurrentETag { get; set; }
/// <summary>
/// True for tables whose <c>IsEnabled</c> column carries security semantics
/// (<see cref="Constants.CriticalBackupTables.AuthTablesFullRestoreForbidden"/>).
/// UI must render the warning banner.
/// </summary>
public bool IsAuthTable { get; set; }
}
/// <summary>
/// Successful response body of <c>mode=commit</c>. Echoes the write outcome.
/// </summary>
public sealed class RestoreRowCommitResponse
{
public string BackupId { get; set; } = string.Empty;
public string TableName { get; set; } = string.Empty;
public string PartitionKey { get; set; } = string.Empty;
public string RowKey { get; set; } = string.Empty;
/// <summary><c>Inserted</c> when the live row was absent, <c>Replaced</c> when it existed.</summary>
public RestoreRowCommitOutcome Outcome { get; set; }
}
public enum RestoreRowCommitOutcome
{
Inserted,
Replaced,
}
/// <summary>
/// Single EDM-tagged property value carried over the wire. Matches the shape of
/// <see cref="AutopilotMonitor.Shared.Models.Deletion.DeletionPropValue"/> but is
/// a separate DTO so we can serialize <c>JsonElement</c> snapshots from both
/// backup and live rows without leaking a Deletion-namespaced type.
/// </summary>
public sealed class RestoreRowPropertySnapshot
{
public string EdmType { get; set; } = string.Empty;
public System.Text.Json.JsonElement Value { get; set; }
}
/// <summary>Per-property diff entry — exactly one property name with a change kind.</summary>
public sealed class RestoreRowPropertyDiff
{
public string Name { get; set; } = string.Empty;
public RestoreRowDiffKind Kind { get; set; }
public RestoreRowPropertySnapshot? Backup { get; set; }
public RestoreRowPropertySnapshot? Current { get; set; }
}
public enum RestoreRowDiffKind
{
/// <summary>Property exists only in the backup (live row is missing it; restore will add it).</summary>
Added,
/// <summary>Property exists only on the live row; restore will REMOVE it (Replace semantics).</summary>
Removed,
/// <summary>Property exists on both sides with different values; restore will overwrite.</summary>
Changed,
/// <summary>Property exists on both sides and values match.</summary>
Unchanged,
}
}
@@ -0,0 +1,293 @@
"use client";
import { useCallback, useEffect, useMemo, useState } from "react";
import Link from "next/link";
import { useParams } from "next/navigation";
import {
api,
type BackupManifest,
type BackupTableEntry,
type RestoreRowCommitResponse,
type RestoreRowPreviewResponse,
type RestoreRowRequestBody,
} from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
import { useAdminConfig } from "../../AdminConfigContext";
import { AdminNotifications } from "../../AdminNotifications";
import { RestoreRowDiffModal } from "../components/RestoreRowDiffModal";
/**
* Backup manifest detail page (plan §PR2). Renders the per-table breakdown
* (status / row-count / byte-size / SHA-256 preview / error message) and lets
* the operator restore a single row from any Ok / Empty table.
*/
export default function BackupDetailPage() {
const params = useParams<{ backupId: string }>();
const backupId = decodeURIComponent(params?.backupId ?? "");
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
const [manifest, setManifest] = useState<BackupManifest | null>(null);
const [loading, setLoading] = useState<boolean>(false);
// Restore-row trigger state.
const [restoreInput, setRestoreInput] = useState<{ tableName: string; pk: string; rk: string } | null>(null);
const [previewLoading, setPreviewLoading] = useState<boolean>(false);
const [activePreview, setActivePreview] = useState<RestoreRowPreviewResponse | null>(null);
const load = useCallback(async () => {
if (!backupId) return;
try {
setLoading(true);
const res = await authenticatedFetch(api.backups.manifest(backupId), getAccessToken);
if (!res.ok) throw new Error(`Manifest failed: ${res.status} ${res.statusText}`);
const body = (await res.json()) as BackupManifest;
setManifest(body);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
} finally {
setLoading(false);
}
}, [backupId, getAccessToken, setError]);
useEffect(() => {
void load();
}, [load]);
const openRestoreModal = async (tableName: string, pk: string, rk: string) => {
setError(null);
setPreviewLoading(true);
try {
const body: RestoreRowRequestBody = {
tableName,
partitionKey: pk,
rowKey: rk,
mode: "Preview",
};
const res = await authenticatedFetch(api.backups.restoreRow(backupId), getAccessToken, {
method: "POST",
body: JSON.stringify(body),
headers: { "Content-Type": "application/json" },
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Preview failed: ${res.status} ${text}`);
}
const preview = (await res.json()) as RestoreRowPreviewResponse;
setActivePreview(preview);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
} finally {
setPreviewLoading(false);
}
};
const handleCommit = useCallback(
(response: RestoreRowCommitResponse) => {
setSuccessMessage(
`Row ${response.outcome.toLowerCase()}: ${response.tableName} pk='${response.partitionKey}' rk='${response.rowKey}'`,
);
setActivePreview(null);
setRestoreInput(null);
},
[setSuccessMessage],
);
const sortedTables = useMemo(() => {
if (!manifest) return [];
return [...manifest.tables].sort((a, b) => a.tableName.localeCompare(b.tableName));
}, [manifest]);
return (
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
<div className="py-6 px-4 sm:px-6 lg:px-8">
<div className="text-sm">
<Link href="/admin/backups" className="text-blue-600 dark:text-blue-400 hover:underline">
← Backups
</Link>
</div>
<h1 className="text-2xl font-normal text-gray-900 dark:text-white mt-2 font-mono">{backupId}</h1>
{manifest && (
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
Started {new Date(manifest.startedAtUtc).toLocaleString()} · completed{" "}
{new Date(manifest.completedAtUtc).toLocaleString()} · triggered by{" "}
<span className="font-mono">{manifest.triggeredBy}</span> · outcome{" "}
<OutcomePill outcome={manifest.outcome} />
</p>
)}
</div>
</header>
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
<AdminNotifications />
{!manifest && !loading && (
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
Manifest not found — backup may be incomplete or deleted.
</div>
)}
{manifest && (
<>
{/* Restore-row trigger card */}
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-4 space-y-3">
<h2 className="text-sm font-medium text-gray-900 dark:text-white">Restore a single row</h2>
<p className="text-xs text-gray-600 dark:text-gray-400">
Enter the table name + partition key + row key. The preview will show the diff between the backup
row and the live row; you confirm the restore in the modal.
</p>
<RestoreInputRow
tables={manifest.tables}
disabled={previewLoading || activePreview !== null}
onSubmit={(tableName, pk, rk) => {
setRestoreInput({ tableName, pk, rk });
void openRestoreModal(tableName, pk, rk);
}}
/>
{previewLoading && restoreInput && (
<div className="text-xs text-gray-500 dark:text-gray-400">
Loading preview for <span className="font-mono">{restoreInput.tableName}</span> /{" "}
<span className="font-mono">{restoreInput.pk}</span> /{" "}
<span className="font-mono">{restoreInput.rk}</span>…
</div>
)}
</div>
{/* Per-table breakdown */}
<div className="bg-white dark:bg-gray-800 rounded-lg shadow overflow-hidden">
<table className="w-full text-xs">
<thead className="bg-gray-100 dark:bg-gray-900 text-gray-700 dark:text-gray-300">
<tr>
<th className="px-3 py-2 text-left">Table</th>
<th className="px-3 py-2 text-left">Status</th>
<th className="px-3 py-2 text-right">Rows</th>
<th className="px-3 py-2 text-right">Bytes</th>
<th className="px-3 py-2 text-left">SHA-256 (preview)</th>
<th className="px-3 py-2 text-left">Notes</th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200 dark:divide-gray-700">
{sortedTables.map((t) => (
<tr key={t.tableName}>
<td className="px-3 py-2 font-mono text-gray-900 dark:text-white">{t.tableName}</td>
<td className="px-3 py-2"><StatusPill status={t.status} /></td>
<td className="px-3 py-2 text-right font-mono">{t.rowCount.toLocaleString()}</td>
<td className="px-3 py-2 text-right font-mono">{t.byteSize.toLocaleString()}</td>
<td className="px-3 py-2 font-mono text-gray-700 dark:text-gray-300">
{t.sha256Hex ? t.sha256Hex.slice(0, 12) + "…" : "—"}
</td>
<td className="px-3 py-2 text-red-700 dark:text-red-300">{t.errorMessage ?? ""}</td>
</tr>
))}
</tbody>
</table>
</div>
</>
)}
</main>
{activePreview && (
<RestoreRowDiffModal
backupId={backupId}
preview={activePreview}
getAccessToken={getAccessToken}
onClose={() => {
setActivePreview(null);
setRestoreInput(null);
}}
onCommitted={handleCommit}
/>
)}
</div>
);
}
function RestoreInputRow({
tables,
disabled,
onSubmit,
}: {
tables: BackupTableEntry[];
disabled: boolean;
onSubmit: (tableName: string, pk: string, rk: string) => void;
}) {
const restorable = useMemo(
() => tables.filter((t) => t.status === "Ok" || t.status === "Empty"),
[tables],
);
const [tableName, setTableName] = useState<string>(restorable[0]?.tableName ?? "");
const [pk, setPk] = useState<string>("");
const [rk, setRk] = useState<string>("");
// Keep the dropdown in sync if the manifest reload changes the restorable set.
useEffect(() => {
if (!restorable.find((t) => t.tableName === tableName)) {
setTableName(restorable[0]?.tableName ?? "");
}
}, [restorable, tableName]);
return (
<div className="flex flex-wrap items-end gap-2">
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300">
Table
<select
value={tableName}
onChange={(e) => setTableName(e.target.value)}
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
>
{restorable.map((t) => (
<option key={t.tableName} value={t.tableName}>{t.tableName}</option>
))}
</select>
</label>
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300 flex-1 min-w-40">
Partition key
<input
type="text"
value={pk}
onChange={(e) => setPk(e.target.value)}
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
/>
</label>
<label className="flex flex-col gap-1 text-xs text-gray-700 dark:text-gray-300 flex-1 min-w-40">
Row key
<input
type="text"
value={rk}
onChange={(e) => setRk(e.target.value)}
className="px-2 py-1 border border-gray-300 dark:border-gray-600 rounded-md bg-white dark:bg-gray-800 text-sm font-mono"
/>
</label>
<button
type="button"
onClick={() => onSubmit(tableName, pk, rk)}
disabled={disabled || !tableName || pk === "" || rk === ""}
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
Preview restore
</button>
</div>
);
}
function OutcomePill({ outcome }: { outcome: "Success" | "Partial" }) {
if (outcome === "Success") {
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-green-100 dark:bg-green-900 text-green-900 dark:text-green-100">Success</span>;
}
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-yellow-100 dark:bg-yellow-900 text-yellow-900 dark:text-yellow-100">Partial</span>;
}
function StatusPill({ status }: { status: BackupTableEntry["status"] }) {
switch (status) {
case "Ok":
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-green-100 dark:bg-green-900 text-green-900 dark:text-green-100">Ok</span>;
case "Empty":
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-gray-100 dark:bg-gray-700 text-gray-700 dark:text-gray-300">Empty</span>;
case "Skipped":
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-yellow-100 dark:bg-yellow-900 text-yellow-900 dark:text-yellow-100">Skipped</span>;
case "Failed":
return <span className="inline-block px-2 py-0.5 rounded text-xs bg-red-100 dark:bg-red-900 text-red-900 dark:text-red-100">Failed</span>;
}
}
@@ -0,0 +1,212 @@
"use client";
import { useState } from "react";
import {
api,
type RestoreRowCommitResponse,
type RestoreRowPreviewResponse,
type RestoreRowPropertyDiff,
type RestoreRowPropertySnapshot,
type RestoreRowRequestBody,
} from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
interface RestoreRowDiffModalProps {
backupId: string;
preview: RestoreRowPreviewResponse;
getAccessToken: () => Promise<string | null>;
onClose: () => void;
onCommitted: (response: RestoreRowCommitResponse) => void;
}
/**
* Two-step modal: preview is loaded by the caller (so the diff is already
* populated when the modal opens), then the operator can either dismiss or
* commit. Commit echoes <code>ifSha256</code> + <code>ifCurrentETag</code>
* from the preview response; the backend verifies both under the maintenance
* lease before writing.
*/
export function RestoreRowDiffModal({
backupId,
preview,
getAccessToken,
onClose,
onCommitted,
}: RestoreRowDiffModalProps) {
const [committing, setCommitting] = useState(false);
const [error, setError] = useState<string | null>(null);
const commit = async () => {
setError(null);
setCommitting(true);
try {
const body: RestoreRowRequestBody = {
tableName: preview.tableName,
partitionKey: preview.partitionKey,
rowKey: preview.rowKey,
mode: "Commit",
ifSha256: preview.rowSha256,
ifCurrentETag: preview.currentETag ?? null,
};
const res = await authenticatedFetch(api.backups.restoreRow(backupId), getAccessToken, {
method: "POST",
body: JSON.stringify(body),
headers: { "Content-Type": "application/json" },
});
if (!res.ok) {
const text = await res.text();
throw new Error(`Restore failed: ${res.status} ${text}`);
}
const response = (await res.json()) as RestoreRowCommitResponse;
onCommitted(response);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
} finally {
setCommitting(false);
}
};
const diffCounts = countDiff(preview.diff);
return (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/50">
<div className="bg-white dark:bg-gray-800 rounded-lg shadow-xl max-w-4xl w-full max-h-[90vh] overflow-hidden flex flex-col">
<header className="px-6 py-4 border-b border-gray-200 dark:border-gray-700">
<h2 className="text-lg font-medium text-gray-900 dark:text-white">
Restore row from backup
</h2>
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5 font-mono">
{preview.tableName} · pk=<span className="text-gray-900 dark:text-gray-100">{preview.partitionKey}</span>{" "}
· rk=<span className="text-gray-900 dark:text-gray-100">{preview.rowKey}</span>
</div>
</header>
<div className="flex-1 overflow-y-auto px-6 py-4 space-y-4">
{preview.isAuthTable && (
<div className="border border-amber-400 bg-amber-50 dark:bg-amber-950 text-amber-900 dark:text-amber-100 rounded-md p-3 text-sm">
<div className="font-medium">Security-relevant table — check the <code className="font-mono">IsEnabled</code> column</div>
<div className="mt-1">
This is an <strong>authentication / authorization</strong> table
(<code className="font-mono">GlobalAdmins</code>, <code className="font-mono">TenantAdmins</code>,{" "}
<code className="font-mono">McpUsers</code>). Restoring this row will overwrite the live{" "}
<code className="font-mono">IsEnabled</code> flag — confirm that the backup row's enable/disable
state is what you intend.
</div>
</div>
)}
<div className="text-sm text-gray-700 dark:text-gray-300">
{preview.currentETag === null ? (
<span>
Live row <strong>does not exist</strong> — restore will <strong>insert</strong> the backup row.
</span>
) : (
<span>
Live row exists (ETag <code className="font-mono">{preview.currentETag}</code>) — restore will{" "}
<strong>replace</strong> it with the backup row.
</span>
)}
</div>
<div className="text-xs text-gray-600 dark:text-gray-400">
<span className="font-medium">{diffCounts.changed}</span> changed ·{" "}
<span className="font-medium">{diffCounts.added}</span> added ·{" "}
<span className="font-medium text-red-700 dark:text-red-300">{diffCounts.removed}</span> will be removed ·{" "}
<span className="font-medium">{diffCounts.unchanged}</span> unchanged
</div>
<div className="border border-gray-200 dark:border-gray-700 rounded-md overflow-hidden">
<table className="w-full text-xs font-mono">
<thead className="bg-gray-100 dark:bg-gray-900 text-gray-700 dark:text-gray-300">
<tr>
<th className="px-3 py-2 text-left">Property</th>
<th className="px-3 py-2 text-left">Backup</th>
<th className="px-3 py-2 text-left">Current</th>
<th className="px-3 py-2 text-left">Change</th>
</tr>
</thead>
<tbody className="divide-y divide-gray-200 dark:divide-gray-700">
{preview.diff.map((d) => (
<tr key={d.name} className={rowClass(d)}>
<td className="px-3 py-2 align-top break-all">{d.name}</td>
<td className="px-3 py-2 align-top break-all whitespace-pre-wrap">{snapshotPretty(d.backup)}</td>
<td className="px-3 py-2 align-top break-all whitespace-pre-wrap">{snapshotPretty(d.current)}</td>
<td className="px-3 py-2 align-top">{kindLabel(d.kind)}</td>
</tr>
))}
</tbody>
</table>
</div>
{error && (
<div className="border border-red-400 bg-red-50 dark:bg-red-950 text-red-900 dark:text-red-100 rounded-md p-3 text-sm font-mono break-all">
{error}
</div>
)}
</div>
<footer className="px-6 py-3 border-t border-gray-200 dark:border-gray-700 flex justify-end gap-2">
<button
type="button"
onClick={onClose}
disabled={committing}
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
>
Cancel
</button>
<button
type="button"
onClick={commit}
disabled={committing}
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{committing ? "Restoring…" : "Restore row"}
</button>
</footer>
</div>
</div>
);
}
function countDiff(diff: RestoreRowPropertyDiff[]): { added: number; removed: number; changed: number; unchanged: number } {
const c = { added: 0, removed: 0, changed: 0, unchanged: 0 };
for (const d of diff) {
if (d.kind === "Added") c.added++;
else if (d.kind === "Removed") c.removed++;
else if (d.kind === "Changed") c.changed++;
else c.unchanged++;
}
return c;
}
function rowClass(d: RestoreRowPropertyDiff): string {
switch (d.kind) {
case "Added":
return "bg-green-50 dark:bg-green-950/30";
case "Removed":
return "bg-red-50 dark:bg-red-950/30";
case "Changed":
return "bg-yellow-50 dark:bg-yellow-950/30";
default:
return "";
}
}
function kindLabel(kind: RestoreRowPropertyDiff["kind"]): string {
switch (kind) {
case "Added":
return "will be added";
case "Removed":
return "will be removed";
case "Changed":
return "will be overwritten";
case "Unchanged":
return "unchanged";
}
}
function snapshotPretty(snap: RestoreRowPropertySnapshot | null | undefined): string {
if (snap == null) return "—";
return `${JSON.stringify(snap.value)}\n[${snap.edmType}]`;
}
@@ -0,0 +1,221 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import Link from "next/link";
import {
api,
type BackupJobStatus,
type BackupListResponse,
type BackupTriggerResponse,
} from "@/lib/api";
import { authenticatedFetch, TokenExpiredError } from "@/lib/authenticatedFetch";
import { useAdminConfig } from "../AdminConfigContext";
import { AdminNotifications } from "../AdminNotifications";
const POLL_INTERVAL_MS = 2500;
/**
* GA-only admin surface for the critical-table backup feature (plan §PR2).
* Lists every completed backup run newest-first and offers a "Run backup now"
* trigger that opens a status-poll banner while the BackgroundService worker
* processes the queued job.
*/
export default function BackupsListPage() {
const { getAccessToken, setError, setSuccessMessage } = useAdminConfig();
const [backupIds, setBackupIds] = useState<string[]>([]);
const [loading, setLoading] = useState<boolean>(false);
const [triggering, setTriggering] = useState<boolean>(false);
const [activeJob, setActiveJob] = useState<BackupJobStatus | null>(null);
const load = useCallback(async () => {
try {
setLoading(true);
const res = await authenticatedFetch(api.backups.list(), getAccessToken);
if (!res.ok) throw new Error(`List backups failed: ${res.status} ${res.statusText}`);
const body = (await res.json()) as BackupListResponse;
setBackupIds(body.backupIds ?? []);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
} finally {
setLoading(false);
}
}, [getAccessToken, setError]);
useEffect(() => {
void load();
}, [load]);
// Poll the active job until terminal, then refresh the list.
useEffect(() => {
if (!activeJob) return;
if (
activeJob.state === "Completed" ||
activeJob.state === "Failed" ||
activeJob.state === "Skipped" ||
activeJob.state === "BlockedTerminal"
) {
// Terminal — final refresh and stop polling.
void load();
return;
}
let cancelled = false;
const timer = window.setTimeout(async () => {
if (cancelled) return;
try {
const res = await authenticatedFetch(api.backups.jobStatus(activeJob.jobId), getAccessToken);
if (!res.ok) throw new Error(`Job status failed: ${res.status} ${res.statusText}`);
const body = (await res.json()) as BackupJobStatus;
setActiveJob(body);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
setActiveJob(null);
}
}, POLL_INTERVAL_MS);
return () => {
cancelled = true;
window.clearTimeout(timer);
};
}, [activeJob, getAccessToken, load, setError]);
const triggerBackup = useCallback(async () => {
try {
setTriggering(true);
const res = await authenticatedFetch(api.backups.trigger(), getAccessToken, {
method: "POST",
});
if (res.status !== 202 && !res.ok) {
const text = await res.text();
throw new Error(`Trigger backup failed: ${res.status} ${text}`);
}
const body = (await res.json()) as BackupTriggerResponse;
// Seed the poller with the freshly-created job.
setActiveJob({
jobId: body.jobId,
kind: "Backup",
state: "Queued",
requestedBy: "",
queuedAtUtc: new Date().toISOString(),
lastHeartbeatUtc: new Date().toISOString(),
});
setSuccessMessage(`Backup queued — jobId ${body.jobId}.`);
} catch (err) {
if (err instanceof TokenExpiredError) setError(err.message);
else setError(err instanceof Error ? err.message : String(err));
} finally {
setTriggering(false);
}
}, [getAccessToken, setError, setSuccessMessage]);
return (
<div className="min-h-screen bg-gray-50 dark:bg-gray-900">
<header className="bg-white dark:bg-gray-800 shadow dark:shadow-gray-700">
<div className="py-6 px-4 sm:px-6 lg:px-8">
<h1 className="text-2xl font-normal text-gray-900 dark:text-white">Critical-Table Backups</h1>
<p className="text-sm text-gray-600 dark:text-gray-400 mt-1">
Daily snapshots of the 15 critical configuration tables (AdminConfiguration, AnalyzeRules, GatherRules,
ImeLogPatterns, …). Manifests live in the <code className="font-mono">critical-table-backups</code> blob
container; 90-day lifecycle delete. Inspect any backup to see per-table SHA + row counts, and restore a
single row from there.
</p>
</div>
</header>
<main className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 py-8 space-y-6">
<AdminNotifications />
{/* Trigger row */}
<div className="flex flex-wrap items-center gap-2">
<button
type="button"
onClick={triggerBackup}
disabled={triggering || activeJob !== null}
className="px-3 py-1.5 bg-blue-600 hover:bg-blue-700 disabled:opacity-50 text-white text-sm rounded-md transition-colors"
>
{triggering ? "Triggering…" : "Run backup now"}
</button>
<button
type="button"
onClick={load}
disabled={loading}
className="px-3 py-1.5 bg-gray-200 hover:bg-gray-300 dark:bg-gray-700 dark:hover:bg-gray-600 text-gray-900 dark:text-white text-sm rounded-md transition-colors"
>
{loading ? "Loading…" : "Refresh"}
</button>
<span className="text-sm text-gray-500 dark:text-gray-400 ml-2">
{backupIds.length} backup(s)
</span>
</div>
{/* Active job banner */}
{activeJob && <ActiveJobBanner job={activeJob} onDismiss={() => setActiveJob(null)} />}
{/* List */}
{backupIds.length === 0 && !loading && (
<div className="bg-white dark:bg-gray-800 rounded-lg shadow p-6 text-sm text-gray-600 dark:text-gray-400">
No critical-table backups found. The daily timer runs at 04:00 UTC; you can also trigger one manually
via the button above.
</div>
)}
<div className="space-y-2">
{backupIds.map((id) => (
<Link
key={id}
href={`/admin/backups/${encodeURIComponent(id)}`}
className="block bg-white dark:bg-gray-800 rounded-lg shadow p-4 hover:bg-gray-100 dark:hover:bg-gray-700 transition-colors"
>
<div className="font-mono text-sm text-gray-900 dark:text-white">{id}</div>
<div className="text-xs text-gray-500 dark:text-gray-400 mt-0.5">
{formatBackupTimestamp(id)} · click to inspect manifest
</div>
</Link>
))}
</div>
</main>
</div>
);
}
function ActiveJobBanner({ job, onDismiss }: { job: BackupJobStatus; onDismiss: () => void }) {
const state = job.state;
const color =
state === "Completed"
? "bg-green-100 dark:bg-green-900 border-green-300 dark:border-green-700 text-green-900 dark:text-green-100"
: state === "Failed" || state === "BlockedTerminal"
? "bg-red-100 dark:bg-red-900 border-red-300 dark:border-red-700 text-red-900 dark:text-red-100"
: state === "Skipped"
? "bg-yellow-100 dark:bg-yellow-900 border-yellow-300 dark:border-yellow-700 text-yellow-900 dark:text-yellow-100"
: "bg-blue-100 dark:bg-blue-900 border-blue-300 dark:border-blue-700 text-blue-900 dark:text-blue-100";
return (
<div className={`border rounded-md p-3 text-sm ${color}`}>
<div className="flex items-start justify-between gap-2">
<div>
<div className="font-medium">Backup job {job.jobId} — {state}</div>
{job.backupOutcome && <div className="text-xs mt-0.5">Outcome: {job.backupOutcome}</div>}
{job.error && <div className="text-xs mt-1 font-mono break-all">Error: {job.error}</div>}
{job.progress && <div className="text-xs mt-1 font-mono break-all">Progress: {job.progress}</div>}
</div>
<button
type="button"
onClick={onDismiss}
className="text-xs px-2 py-1 underline"
>
Dismiss
</button>
</div>
</div>
);
}
/** Pretty-print the embedded timestamp portion of a backupId (yyyyMMddTHHmmssZ_xxxxxxxx). */
function formatBackupTimestamp(backupId: string): string {
// 20260522T040000Z_a1b2c3d4 → "2026-05-22 04:00:00 UTC"
const m = backupId.match(/^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z_/);
if (!m) return backupId;
return `${m[1]}-${m[2]}-${m[3]} ${m[4]}:${m[5]}:${m[6]} UTC`;
}
@@ -37,6 +37,9 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
"CriticalTableBackupPartial",
"CriticalTableBackupFailed",
"CriticalTableBackupSkippedLocked",
// PR2: GA restored a single row from a backup. Warning severity so operators
// can wire a Telegram rule and see the audit in near-real-time.
"BackupRowRestored",
],
Security: [
"DeviceBlocked",
@@ -0,0 +1,57 @@
import { describe, it, expect, vi } from "vitest";
// API_BASE_URL is read at import time of lib/api — stub it before the import.
vi.mock("@/utils/config", () => ({ API_BASE_URL: "https://test.example" }));
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const apiPromise = import("../api") as Promise<{ api: any }>;
/**
* Wire-shape contract for the critical-table backup feature (PR1 + PR2).
* The frontend talks to a fixed set of GA-only endpoints; a refactor of
* api.ts that accidentally drops the namespace or changes the path encoding
* would silently break the Backups admin surface.
*/
describe("api.backups — list / manifest / trigger / job-status", () => {
it("list points at /api/global/backups", async () => {
const { api } = await apiPromise;
expect(api.backups.list()).toBe("https://test.example/api/global/backups");
});
it("manifest encodes the backupId in the path", async () => {
const { api } = await apiPromise;
expect(api.backups.manifest("20260522T040000Z_a1b2c3d4")).toBe(
"https://test.example/api/global/backups/20260522T040000Z_a1b2c3d4",
);
});
it("trigger points at /api/global/backups/trigger", async () => {
const { api } = await apiPromise;
expect(api.backups.trigger()).toBe("https://test.example/api/global/backups/trigger");
});
it("jobStatus encodes the jobId in the path", async () => {
const { api } = await apiPromise;
expect(api.backups.jobStatus("abc-123")).toBe(
"https://test.example/api/global/backups/jobs/abc-123",
);
});
});
describe("api.backups.restoreRow — PR2 single-row restore", () => {
it("points at /api/global/backups/{backupId}/restore-row", async () => {
const { api } = await apiPromise;
expect(api.backups.restoreRow("20260522T040000Z_a1b2c3d4")).toBe(
"https://test.example/api/global/backups/20260522T040000Z_a1b2c3d4/restore-row",
);
});
it("URL-encodes special characters in the backupId", async () => {
const { api } = await apiPromise;
// backupIds are stamp_guid8 in production but the path encoder must still
// be defensive — a future schema change shouldn't silently break routing.
expect(api.backups.restoreRow("with spaces/and+slashes")).toBe(
"https://test.example/api/global/backups/with%20spaces%2Fand%2Bslashes/restore-row",
);
});
});
+122
View File
@@ -496,6 +496,18 @@ export const api = {
planTiers: () => `${API_BASE_URL}/api/global/config/plan-tiers`,
},
// ── Critical-Table Backup (PR1 list/manifest/trigger + PR2 restore-row) ───
backups: {
list: () => `${API_BASE_URL}/api/global/backups`,
manifest: (backupId: string) =>
`${API_BASE_URL}/api/global/backups/${encodeURIComponent(backupId)}`,
trigger: () => `${API_BASE_URL}/api/global/backups/trigger`,
jobStatus: (jobId: string) =>
`${API_BASE_URL}/api/global/backups/jobs/${encodeURIComponent(jobId)}`,
restoreRow: (backupId: string) =>
`${API_BASE_URL}/api/global/backups/${encodeURIComponent(backupId)}/restore-row`,
},
// ── Maintenance ───────────────────────────────────────────────────────────
maintenance: {
trigger: (date?: string) =>
@@ -556,3 +568,113 @@ export interface CustomsArchiveFullEntry extends CustomsArchiveEntrySummary {
historyRowKey: string;
archivedBy: string;
}
// ── Backups response types (PR1 + PR2) ────────────────────────────────────
export interface BackupListResponse {
backupIds: string[];
}
export type BackupTableStatus = "Ok" | "Empty" | "Skipped" | "Failed";
export type BackupOutcome = "Success" | "Partial";
export interface BackupTableEntry {
tableName: string;
status: BackupTableStatus;
rowCount: number;
byteSize: number;
sha256Hex: string;
blobName: string;
errorMessage?: string | null;
}
export interface BackupManifest {
schemaVersion: number;
backupId: string;
startedAtUtc: string;
completedAtUtc: string;
triggeredBy: string;
outcome: BackupOutcome;
tables: BackupTableEntry[];
}
export type BackupJobKind = "Backup" | "RestoreTable";
export type BackupJobState =
| "Queued"
| "Running"
| "Completed"
| "Failed"
| "Skipped"
| "BlockedTerminal";
export interface BackupJobStatus {
jobId: string;
kind: BackupJobKind;
state: BackupJobState;
requestedBy: string;
queuedAtUtc: string;
startedAtUtc?: string | null;
completedAtUtc?: string | null;
lastHeartbeatUtc: string;
backupId?: string | null;
sourceBackupId?: string | null;
tableName?: string | null;
strategy?: string | null;
progress?: string | null;
error?: string | null;
backupOutcome?: BackupOutcome | null;
}
export interface BackupTriggerResponse {
jobId: string;
statusUrl: string;
}
// ── PR2: single-row restore ───────────────────────────────────────────────
export type RestoreRowMode = "Preview" | "Commit";
export type RestoreRowDiffKind = "Added" | "Removed" | "Changed" | "Unchanged";
export type RestoreRowCommitOutcome = "Inserted" | "Replaced";
export interface RestoreRowPropertySnapshot {
edmType: string;
// JsonElement on the wire — string | number | boolean | null | object | array
value: unknown;
}
export interface RestoreRowPropertyDiff {
name: string;
kind: RestoreRowDiffKind;
backup?: RestoreRowPropertySnapshot | null;
current?: RestoreRowPropertySnapshot | null;
}
export interface RestoreRowPreviewResponse {
backupId: string;
tableName: string;
partitionKey: string;
rowKey: string;
backupProperties: Record<string, RestoreRowPropertySnapshot>;
currentProperties?: Record<string, RestoreRowPropertySnapshot> | null;
diff: RestoreRowPropertyDiff[];
rowSha256: string;
currentETag?: string | null;
isAuthTable: boolean;
}
export interface RestoreRowCommitResponse {
backupId: string;
tableName: string;
partitionKey: string;
rowKey: string;
outcome: RestoreRowCommitOutcome;
}
export interface RestoreRowRequestBody {
tableName: string;
partitionKey: string;
rowKey: string;
mode: RestoreRowMode;
ifSha256?: string;
ifCurrentETag?: string | null;
}
@@ -265,6 +265,7 @@ export const EXPANDABLE_NAV_GROUPS: ExpandableNavGroup[] = [
{ id: "ga-maintenance", label: "Maintenance", href: "/admin/ops" },
{ id: "ga-session-cleanup", label: "Session Cleanup", href: "/admin/ops/session-cleanup" },
{ id: "ga-customs-archive", label: "Customs Archive", href: "/admin/customs-archive" },
{ id: "ga-backups", label: "Backups", href: "/admin/backups" },
],
},
{