diff --git a/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageChunkingTests.cs b/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageChunkingTests.cs index 3216afce..c0f3d0dc 100644 --- a/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageChunkingTests.cs +++ b/src/Backend/AutopilotMonitor.Functions.Tests/TableStorageChunkingTests.cs @@ -1,5 +1,6 @@ using Azure.Data.Tables; using AutopilotMonitor.Functions.DataAccess.TableStorage; +using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.Tests; @@ -69,6 +70,35 @@ public class TableStorageChunkingTests Assert.Equal("", result["Prop"]); } + [Fact] + public void ChunkProperty_SurrogatePairOnBoundary_DoesNotSplitPair() + { + // Place an emoji (UTF-16 surrogate pair) such that the high surrogate lands + // at index 29_999 and the low surrogate at index 30_000 — the naive split point. + var prefix = new string('a', 29_999); + var emoji = "\uD83D\uDE00"; // U+1F600 GRINNING FACE + var suffix = new string('b', 5); + var value = prefix + emoji + suffix; // length 30_006 + + var result = TableStorageChunking.ChunkProperty("Prop", value); + + Assert.Equal("2", result["Prop_ChunkCount"]); + var chunk0 = result["Prop_0"]; + var chunk1 = result["Prop_1"]; + + // Neither chunk may contain an orphan surrogate. + Assert.False(char.IsHighSurrogate(chunk0[^1]), "Chunk 0 ends with orphan high surrogate"); + Assert.False(char.IsLowSurrogate(chunk1[0]), "Chunk 1 starts with orphan low surrogate"); + + // Round-trip preserves data. + Assert.Equal(value, chunk0 + chunk1); + + // And the high surrogate was moved into chunk 1 (not chunk 0). + Assert.Equal(29_999, chunk0.Length); + Assert.Equal('\uD83D', chunk1[0]); + Assert.Equal('\uDE00', chunk1[1]); + } + // ===== ReassembleProperty (IDictionary) ===== [Fact] @@ -158,4 +188,115 @@ public class TableStorageChunkingTests var reassembled = TableStorageChunking.ReassembleProperty(entity, "Data"); Assert.Equal(original, reassembled); } + + // ===== Corruption detection ===== + + [Fact] + public void ReassembleDict_MissingMiddleChunk_ReturnsPartialAndLogsWarning() + { + var entity = new Dictionary + { + { "Prop_0", "aaa" }, + { "Prop_2", "ccc" }, + { "Prop_ChunkCount", "3" } + }; + var logger = new CapturingLogger(); + + var result = TableStorageChunking.ReassembleProperty(entity, "Prop", logger, "pk/rk"); + + Assert.Equal("aaaccc", result); + var warning = Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning); + Assert.Contains("expected 3", warning.Message); + Assert.Contains("got 2", warning.Message); + Assert.Contains("pk/rk", warning.Message); + } + + [Fact] + public void ReassembleDict_ChunkCountWithoutChunks_ReturnsNullAndLogsWarning() + { + var entity = new Dictionary + { + { "Prop_ChunkCount", "3" } + }; + var logger = new CapturingLogger(); + + var result = TableStorageChunking.ReassembleProperty(entity, "Prop", logger, "pk/rk"); + + Assert.Null(result); + var warning = Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning); + Assert.Contains("no chunks found", warning.Message); + } + + [Fact] + public void ReassembleDict_InvalidChunkCount_ReturnsNullAndLogsWarning() + { + var entity = new Dictionary + { + { "Prop_0", "aaa" }, + { "Prop_ChunkCount", "abc" } + }; + var logger = new CapturingLogger(); + + var result = TableStorageChunking.ReassembleProperty(entity, "Prop", logger, "pk/rk"); + + Assert.Null(result); + var warning = Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning); + Assert.Contains("invalid ChunkCount", warning.Message); + Assert.Contains("abc", warning.Message); + } + + [Fact] + public void ReassembleTableEntity_MissingMiddleChunk_ReturnsPartialAndLogsWarning() + { + var entity = new TableEntity("pk", "rk") + { + ["Prop_0"] = "aaa", + ["Prop_2"] = "ccc", + ["Prop_ChunkCount"] = "3" + }; + var logger = new CapturingLogger(); + + var result = TableStorageChunking.ReassembleProperty(entity, "Prop", logger, "pk/rk"); + + Assert.Equal("aaaccc", result); + var warning = Assert.Single(logger.Entries, e => e.Level == LogLevel.Warning); + Assert.Contains("expected 3", warning.Message); + Assert.Contains("got 2", warning.Message); + } + + [Fact] + public void ReassembleDict_HappyPath_DoesNotLog() + { + var entity = new Dictionary + { + { "Prop_0", "aaa" }, + { "Prop_1", "bbb" }, + { "Prop_ChunkCount", "2" } + }; + var logger = new CapturingLogger(); + + var result = TableStorageChunking.ReassembleProperty(entity, "Prop", logger, "pk/rk"); + + Assert.Equal("aaabbb", result); + Assert.Empty(logger.Entries); + } + + private sealed class CapturingLogger : ILogger + { + public List<(LogLevel Level, string Message)> Entries { get; } = new(); + + public IDisposable BeginScope(TState state) where TState : notnull => NullScope.Instance; + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + Entries.Add((logLevel, formatter(state, exception))); + } + + private sealed class NullScope : IDisposable + { + public static readonly NullScope Instance = new(); + public void Dispose() { } + } + } } diff --git a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableStorageChunking.cs b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableStorageChunking.cs index 78c41606..28d80852 100644 --- a/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableStorageChunking.cs +++ b/src/Backend/AutopilotMonitor.Functions/DataAccess/TableStorage/TableStorageChunking.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Text; using Azure.Data.Tables; +using Microsoft.Extensions.Logging; namespace AutopilotMonitor.Functions.DataAccess.TableStorage { @@ -26,11 +27,19 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage var result = new Dictionary(); int chunkIndex = 0; - for (int offset = 0; offset < value.Length; offset += maxChunkSize) + int offset = 0; + while (offset < value.Length) { int length = Math.Min(maxChunkSize, value.Length - offset); + // Avoid splitting a UTF-16 surrogate pair across chunks: if the last + // char of this slice is a high surrogate, push it into the next chunk. + // (No-op when length already covers the remainder.) + if (length < value.Length - offset && char.IsHighSurrogate(value[offset + length - 1])) + length--; + result[$"{propertyName}_{chunkIndex}"] = value.Substring(offset, length); chunkIndex++; + offset += length; } result[$"{propertyName}_ChunkCount"] = chunkIndex.ToString(); return result; @@ -39,8 +48,14 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage /// /// Reassembles a potentially chunked property from a dictionary entity. /// Handles both old (single property) and new (chunked) formats. + /// When is supplied, emits warnings on detected corruption + /// (unparseable ChunkCount, zero chunks present, or missing chunks in the middle). /// - public static string? ReassembleProperty(IDictionary entity, string propertyName) + public static string? ReassembleProperty( + IDictionary entity, + string propertyName, + ILogger? logger = null, + string? context = null) { // Backward compat: single non-chunked property if (entity.TryGetValue(propertyName, out var single) && single != null) @@ -54,23 +69,37 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage if (!entity.TryGetValue($"{propertyName}_ChunkCount", out var countObj)) return null; - if (!int.TryParse(countObj?.ToString(), out var count) || count <= 0) + var rawCount = countObj?.ToString(); + if (!int.TryParse(rawCount, out var count) || count <= 0) + { + LogInvalidChunkCount(logger, propertyName, rawCount, context); return null; + } var sb = new StringBuilder(); + int foundChunks = 0; for (int i = 0; i < count; i++) { if (entity.TryGetValue($"{propertyName}_{i}", out var chunk) && chunk != null) + { sb.Append(chunk.ToString()); + foundChunks++; + } } - return sb.Length > 0 ? sb.ToString() : null; + + return FinalizeReassembly(sb, foundChunks, count, propertyName, logger, context); } /// /// Reassembles a potentially chunked property from a TableEntity. /// Handles both old (single property) and new (chunked) formats. + /// When is supplied, emits warnings on detected corruption. /// - public static string? ReassembleProperty(TableEntity entity, string propertyName) + public static string? ReassembleProperty( + TableEntity entity, + string propertyName, + ILogger? logger = null, + string? context = null) { // Backward compat: single non-chunked property var single = entity.GetString(propertyName); @@ -78,18 +107,65 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage return single; // Chunked format - var countStr = entity.GetString($"{propertyName}_ChunkCount"); - if (!int.TryParse(countStr, out var count) || count <= 0) + if (!entity.ContainsKey($"{propertyName}_ChunkCount")) return null; + var countStr = entity.GetString($"{propertyName}_ChunkCount"); + if (!int.TryParse(countStr, out var count) || count <= 0) + { + LogInvalidChunkCount(logger, propertyName, countStr, context); + return null; + } + var sb = new StringBuilder(); + int foundChunks = 0; for (int i = 0; i < count; i++) { var chunk = entity.GetString($"{propertyName}_{i}"); if (chunk != null) + { sb.Append(chunk); + foundChunks++; + } } - return sb.Length > 0 ? sb.ToString() : null; + + return FinalizeReassembly(sb, foundChunks, count, propertyName, logger, context); } + + private static string? FinalizeReassembly( + StringBuilder sb, + int foundChunks, + int expectedCount, + string propertyName, + ILogger? logger, + string? context) + { + if (foundChunks == 0) + { + logger?.LogWarning( + "Chunked property {PropertyName} has ChunkCount={Expected} but no chunks found{Context}", + propertyName, expectedCount, FormatContext(context)); + return null; + } + + if (foundChunks < expectedCount) + { + logger?.LogWarning( + "Chunked property {PropertyName} incomplete: expected {Expected} chunks, got {Actual}{Context}. Returning partial value of {Length} chars.", + propertyName, expectedCount, foundChunks, FormatContext(context), sb.Length); + } + + return sb.ToString(); + } + + private static void LogInvalidChunkCount(ILogger? logger, string propertyName, string? rawValue, string? context) + { + logger?.LogWarning( + "Chunked property {PropertyName} has invalid ChunkCount value '{RawValue}'{Context}", + propertyName, rawValue ?? "", FormatContext(context)); + } + + private static string FormatContext(string? context) + => string.IsNullOrEmpty(context) ? string.Empty : $" [{context}]"; } } diff --git a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Rules.cs b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Rules.cs index 6307a5ef..15b8ab2e 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Rules.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/TableStorageService.Rules.cs @@ -651,7 +651,7 @@ namespace AutopilotMonitor.Functions.Services var entity = response?.Value; if (entity == null) return null; - var reportJson = TableStorageChunking.ReassembleProperty(entity, "ReportJson"); + var reportJson = TableStorageChunking.ReassembleProperty(entity, "ReportJson", _logger, $"{partitionKey}/report"); return string.IsNullOrEmpty(reportJson) ? null : reportJson; } catch (Azure.RequestFailedException ex) when (ex.Status == 404) diff --git a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs index cf249689..ce3b73ab 100644 --- a/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs +++ b/src/Backend/AutopilotMonitor.Functions/Services/Vulnerability/VulnerabilityCorrelationService.cs @@ -556,7 +556,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability var cachedAtStr = entity.ContainsKey("CachedAt") ? entity["CachedAt"]?.ToString() : null; if (string.IsNullOrWhiteSpace(cachedAtStr)) return null; - var cveDataJson = TableStorageChunking.ReassembleProperty(entity, "CveDataJson") ?? "[]"; + var cveDataJson = TableStorageChunking.ReassembleProperty(entity, "CveDataJson", _logger, $"{cacheKey}/cache_result") ?? "[]"; var cves = JsonSerializer.Deserialize>(cveDataJson, JsonOptions) ?? new List(); return new CachedCveResult