Merge pull request #216 from okieselbach/agents/vuln-scanner-cve-daten-und-zuordnung

Vulnerability scan: CVE data, software mapping and monitoring
This commit is contained in:
Oliver Kieselbach
2026-11-12 23:28:09 +01:00
committed by GitHub
160 changed files with 26886 additions and 3407 deletions
+20 -6
View File
@@ -2,10 +2,10 @@
"$schema": "../schema/analyze-rule.schema.json",
"ruleId": "ANALYZE-ID-003",
"title": "Critical Vulnerability Detected",
"description": "Fires when the vulnerability correlation finds critical-severity CVEs or CISA KEV entries in installed software. This indicates the device has software with known, actively exploited vulnerabilities.",
"description": "Fires when the vulnerability correlation finds critical-severity CVEs in installed software that has to be updated by hand. Software that updates itself, such as browsers, is listed in the vulnerability report but does not trigger this rule.",
"severity": "critical",
"category": "security",
"version": "1.2.0",
"version": "1.3.0",
"author": "Autopilot Monitor",
"enabled": true,
"evaluateOn": [
@@ -19,7 +19,7 @@
"signal": "critical_vulnerabilities",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.critical_cves",
"dataField": "scan_summary.actionable_critical_cves",
"operator": "gte",
"value": "1",
"required": true
@@ -28,7 +28,16 @@
"signal": "kev_matches",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.kev_matches",
"dataField": "scan_summary.actionable_kev_matches",
"operator": "gte",
"value": "1",
"required": false
},
{
"signal": "self_updating_findings",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.self_updating_cves",
"operator": "gte",
"value": "1",
"required": false
@@ -42,7 +51,7 @@
}
],
"confidenceThreshold": 80,
"explanation": "The device has installed software with critical vulnerabilities. Software with CVEs rated CRITICAL (CVSS >= 9.0) or listed in the CISA Known Exploited Vulnerabilities catalog was detected. These vulnerabilities are actively targeted by attackers and should be remediated immediately.",
"explanation": "The device has installed software with critical vulnerabilities that an update of the software has to close. CVEs rated CRITICAL (CVSS >= 9.0) were found in software that is updated through your deployment. Findings listed in the CISA Known Exploited Vulnerabilities catalog are known to be used in attacks and should be remediated first. Software that updates itself, such as browsers, is not counted here, because it is expected to bring itself up to date after enrollment. Its findings are still listed in the vulnerability report.",
"remediation": [
{
"title": "Review vulnerability findings",
@@ -74,7 +83,7 @@
],
"presentation": {
"role": "advisory",
"summary": "Installed software has {{critical_vulnerabilities.value}} critical vulnerabilities",
"summary": "Critical CVEs in software that needs a manual update: {{critical_vulnerabilities.value}}",
"subject": {
"kind": "device",
"key": "vulnerabilities",
@@ -90,6 +99,11 @@
"label": "Known exploited",
"value": "{{kev_matches.value}}",
"format": "number"
},
{
"label": "CVEs in self-updating software",
"value": "{{self_updating_findings.value}}",
"format": "number"
}
]
},
+20 -6
View File
@@ -4259,10 +4259,10 @@
{
"ruleId": "ANALYZE-ID-003",
"title": "Critical Vulnerability Detected",
"description": "Fires when the vulnerability correlation finds critical-severity CVEs or CISA KEV entries in installed software. This indicates the device has software with known, actively exploited vulnerabilities.",
"description": "Fires when the vulnerability correlation finds critical-severity CVEs in installed software that has to be updated by hand. Software that updates itself, such as browsers, is listed in the vulnerability report but does not trigger this rule.",
"severity": "critical",
"category": "security",
"version": "1.2.0",
"version": "1.3.0",
"author": "Autopilot Monitor",
"enabled": true,
"evaluateOn": [
@@ -4276,7 +4276,7 @@
"signal": "critical_vulnerabilities",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.critical_cves",
"dataField": "scan_summary.actionable_critical_cves",
"operator": "gte",
"value": "1",
"required": true
@@ -4285,7 +4285,16 @@
"signal": "kev_matches",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.kev_matches",
"dataField": "scan_summary.actionable_kev_matches",
"operator": "gte",
"value": "1",
"required": false
},
{
"signal": "self_updating_findings",
"source": "event_type",
"eventType": "vulnerability_report",
"dataField": "scan_summary.self_updating_cves",
"operator": "gte",
"value": "1",
"required": false
@@ -4299,7 +4308,7 @@
}
],
"confidenceThreshold": 80,
"explanation": "The device has installed software with critical vulnerabilities. Software with CVEs rated CRITICAL (CVSS >= 9.0) or listed in the CISA Known Exploited Vulnerabilities catalog was detected. These vulnerabilities are actively targeted by attackers and should be remediated immediately.",
"explanation": "The device has installed software with critical vulnerabilities that an update of the software has to close. CVEs rated CRITICAL (CVSS >= 9.0) were found in software that is updated through your deployment. Findings listed in the CISA Known Exploited Vulnerabilities catalog are known to be used in attacks and should be remediated first. Software that updates itself, such as browsers, is not counted here, because it is expected to bring itself up to date after enrollment. Its findings are still listed in the vulnerability report.",
"remediation": [
{
"title": "Review vulnerability findings",
@@ -4331,7 +4340,7 @@
],
"presentation": {
"role": "advisory",
"summary": "Installed software has {{critical_vulnerabilities.value}} critical vulnerabilities",
"summary": "Critical CVEs in software that needs a manual update: {{critical_vulnerabilities.value}}",
"subject": {
"kind": "device",
"key": "vulnerabilities",
@@ -4347,6 +4356,11 @@
"label": "Known exploited",
"value": "{{kev_matches.value}}",
"format": "number"
},
{
"label": "CVEs in self-updating software",
"value": "{{self_updating_findings.value}}",
"format": "number"
}
]
},
+6148
View File
File diff suppressed because it is too large Load Diff
@@ -810,6 +810,36 @@ public class AdminWireParityTests
});
}
[Fact]
public void ReseedFromGitHubResponse_carries_the_inventory_counts_and_the_problems_of_a_refused_list()
{
AssertParity(
new
{
success = false,
message = "Reseed from GitHub complete, except for the CPE mappings",
gather = new { deleted = 0, written = 0, orphanStatesGcd = 0, sunsetSkipped = 0 },
analyze = new { deleted = 0, written = 0, orphanStatesGcd = 0, sunsetSkipped = 0 },
ime = new { deleted = 0, written = 0 },
cpeCommunityMappings = new { deleted = 0, written = 0 },
cpeSeedMappings = new { deleted = 0, written = 0 },
cpeInventory = new { rows = 9864, set = 0, cleared = 0, failed = 0, productsNoted = 0 },
cpeProblems = new[] { "seed list: the list holds no mapping" }
},
new ReseedFromGitHubResponse
{
Success = false,
Message = "Reseed from GitHub complete, except for the CPE mappings",
Gather = new ReseedRuleCountsNode { Deleted = 0, Written = 0, OrphanStatesGcd = 0, SunsetSkipped = 0 },
Analyze = new ReseedRuleCountsNode { Deleted = 0, Written = 0, OrphanStatesGcd = 0, SunsetSkipped = 0 },
Ime = new ReseedTableCountsNode { Deleted = 0, Written = 0 },
CpeCommunityMappings = new ReseedTableCountsNode { Deleted = 0, Written = 0 },
CpeSeedMappings = new ReseedTableCountsNode { Deleted = 0, Written = 0 },
CpeInventory = new ReseedInventoryNode { Rows = 9864, Set = 0, Cleared = 0, Failed = 0, ProductsNoted = 0 },
CpeProblems = new List<string> { "seed list: the list holds no mapping" }
});
}
// ---- SubmitOffboardingFeedback -------------------------------------------------------
[Fact]
@@ -0,0 +1,362 @@
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// POST vulnerability/cpe-mapping/auto-resolve tries the given programs one after the other and
/// says per program what came of it. The operator waits for the answer, so once NVD leaves a
/// request unanswered NVD is not asked about the remaining programs, which still get every
/// answer that needs no request; programs that got a mapping have it on their inventory rows
/// when the call returns.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class AutoResolveCpeMappingFunctionTests : IDisposable
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private const string ToolCpe = "cpe:2.3:a:northwind:tool";
private const string ReaderCpe = "cpe:2.3:a:adatum:reader";
private const string ViewerCpe = "cpe:2.3:a:adatum:viewer";
private const string EditorCpe = "cpe:2.3:a:adatum:editor";
private static readonly AutoResolveCpeMappingItem Widget = new() { SoftwareName = "Contoso Widget 4.2 (x64)", NormalizedVendor = "contoso", Publisher = "Contoso Ltd" };
private static readonly AutoResolveCpeMappingItem Gadget = new() { SoftwareName = "Fabrikam Gadget 2.0", NormalizedVendor = "fabrikam", Publisher = "Fabrikam Inc" };
private static readonly AutoResolveCpeMappingItem Tool = new() { SoftwareName = "Northwind Tool 3.1", NormalizedVendor = "northwind", Publisher = "Northwind Traders" };
private static readonly AutoResolveCpeMappingItem Reader = new() { SoftwareName = "Adatum Reader 2.0", NormalizedVendor = "adatum", Publisher = "Adatum Corporation" };
private static readonly AutoResolveCpeMappingItem Viewer = new() { SoftwareName = "Adatum Viewer 1.4", NormalizedVendor = "adatum", Publisher = "Adatum Corporation" };
private static readonly AutoResolveCpeMappingItem Editor = new() { SoftwareName = "Adatum Editor 6.0", NormalizedVendor = "adatum", Publisher = "Adatum Corporation" };
private static readonly AutoResolveCpeMappingItem Launcher = new() { SoftwareName = "Tailspin Launcher", NormalizedVendor = "tailspin", Publisher = "Tailspin Toys" };
private static readonly AutoResolveCpeMappingItem Terminal = new() { SoftwareName = "Tailspin Terminal", NormalizedVendor = "tailspin", Publisher = "Tailspin Toys" };
public AutoResolveCpeMappingFunctionTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private sealed class Rig
{
private Rig(MappingRig maps, CpeDictionaryService dictionary)
{
Maps = maps;
Function = new AutoResolveCpeMappingFunction(
maps.Resolver(dictionary), maps.Store, maps.Reconciler, NullLogger<AutoResolveCpeMappingFunction>.Instance);
}
/// <summary>
/// The gadget is mapped by the seed; widget, gadget, tool, reader, viewer and editor are
/// products of the stored dictionary, and NVD lists one CVE for whatever it is asked about.
/// </summary>
public static async Task<Rig> CreateAsync(bool dictionaryStored = true)
{
var maps = await MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"));
maps.Scan.NvdAnswer = OneCve;
var dictionary = dictionaryStored
? CpeDictionaryFixture.Stored(
("contoso:widget", "Contoso Widget"), ("fabrikam:gadget", "Fabrikam Gadget"),
("northwind:tool", "Northwind Tool"), ("adatum:reader", "Adatum Reader"),
("adatum:viewer", "Adatum Viewer"), ("adatum:editor", "Adatum Editor"))
: CpeDictionaryFixture.Stored();
return new Rig(maps, dictionary);
}
public MappingRig Maps { get; }
public AutoResolveCpeMappingFunction Function { get; }
public IEnumerable<string> AskedFor => Maps.Scan.Nvd.Requests.Select(r => r.Cpe);
public async Task<JsonElement> ResolveAsync(params AutoResolveCpeMappingItem[] items)
{
var response = await Function.Run(OperatorCall.Request(new AutoResolveCpeMappingRequest { Items = items.ToList() }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return OperatorCall.BodyOf(response);
}
}
private static HttpResponseMessage OneCve(NvdRequest request) =>
NvdFixture.Page(NvdFixture.Cve("CVE-2026-10001", 8.8, NvdFixture.Below(request.Cpe, "9.0")));
private static List<(string SoftwareName, string Reason)> FailedOf(JsonElement body) => body.GetProperty("failed").EnumerateArray()
.Select(f => (f.GetProperty("softwareName").GetString()!, f.GetProperty("reason").GetString()!))
.ToList();
private static List<(string SoftwareName, string CpeUri)> ResolvedOf(JsonElement body) => body.GetProperty("resolved").EnumerateArray()
.Select(r => (r.GetProperty("softwareName").GetString()!, r.GetProperty("cpeUri").GetString()!))
.ToList();
[Fact]
public async Task Run_MappedPrograms_AreReportedWithTheirCpe_AndHaveItOnTheirInventoryRows()
{
var rig = await Rig.CreateAsync();
var installed = rig.Maps.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2");
var otherVersion = rig.Maps.Tables.AddInventory("contoso", "Contoso Widget 5.0", "5.0");
var body = await rig.ResolveAsync(Widget, Tool);
Assert.Equal(new[] { (Widget.SoftwareName, WidgetCpe), (Tool.SoftwareName, ToolCpe) }, ResolvedOf(body));
Assert.All(body.GetProperty("resolved").EnumerateArray(), r => Assert.Equal(1, r.GetProperty("confidence").GetDouble()));
Assert.Empty(FailedOf(body));
Assert.Equal(2, body.GetProperty("totalProcessed").GetInt32());
Assert.Equal(2, body.GetProperty("totalResolved").GetInt32());
Assert.Equal(0, body.GetProperty("totalFailed").GetInt32());
Assert.Equal(new[] { WidgetCpe, ToolCpe }, rig.AskedFor);
Assert.Equal(WidgetCpe, installed["CpeUri"]);
Assert.Equal(WidgetCpe, otherVersion["CpeUri"]);
}
[Fact]
public async Task Run_FirstRequestNvdLeavesUnanswered_EndsTheAsking_AndTheRestIsReportedUnavailable()
{
var rig = await Rig.CreateAsync();
rig.Maps.Scan.NvdAnswer = request =>
request.Cpe == WidgetCpe ? NvdFixture.Status(HttpStatusCode.InternalServerError) : OneCve(request);
var installed = rig.Maps.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0");
var body = await rig.ResolveAsync(Reader, Widget, Tool);
Assert.Equal(new[] { (Reader.SoftwareName, ReaderCpe) }, ResolvedOf(body));
Assert.Equal(
new[] { (Widget.SoftwareName, "nvd_unavailable"), (Tool.SoftwareName, "nvd_unavailable") }, FailedOf(body));
Assert.Equal(3, body.GetProperty("totalProcessed").GetInt32());
Assert.Equal(1, body.GetProperty("totalResolved").GetInt32());
Assert.Equal(2, body.GetProperty("totalFailed").GetInt32());
// The tool was never asked about: its request would have waited for the same timeout.
Assert.Equal(new[] { ReaderCpe, WidgetCpe }, rig.AskedFor);
Assert.Equal(new[] { "adatum:adatum reader" }, rig.Maps.Tables.RowKeys(CpeMappingStore.AutoPartition));
// What was mapped before NVD stopped answering is kept and reaches the inventory.
Assert.Equal(ReaderCpe, installed["CpeUri"]);
}
[Fact]
public async Task Run_RequestNvdTurnsDown_FailsThatProgramOnly()
{
var rig = await Rig.CreateAsync();
rig.Maps.Scan.NvdAnswer = request =>
request.Cpe == WidgetCpe ? NvdFixture.Status(HttpStatusCode.NotFound) : OneCve(request);
var body = await rig.ResolveAsync(Widget, Tool);
Assert.Equal(new[] { (Widget.SoftwareName, "nvd_unavailable") }, FailedOf(body));
Assert.Equal(new[] { (Tool.SoftwareName, ToolCpe) }, ResolvedOf(body));
Assert.Equal(new[] { WidgetCpe, ToolCpe }, rig.AskedFor);
}
[Fact]
public async Task Run_SaysWhyAProgramGotNoMapping()
{
var rig = await Rig.CreateAsync();
rig.Maps.Scan.NvdAnswer = _ => NvdFixture.Page();
var ignore = CpeIgnoreEntry.Build(Tool.NormalizedVendor, Tool.SoftwareName, "manual")!.Value;
rig.Maps.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var installed = rig.Maps.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2");
var body = await rig.ResolveAsync(
new AutoResolveCpeMappingItem { SoftwareName = " " }, Gadget, Tool, Launcher, Widget);
Assert.Equal(
new[]
{
(" ", "empty_name"),
(Gadget.SoftwareName, "already_mapped"),
(Tool.SoftwareName, "ignored"),
(Launcher.SoftwareName, "no_exact_product"),
(Widget.SoftwareName, "no_cves"),
},
FailedOf(body));
Assert.Empty(ResolvedOf(body));
Assert.Equal(new[] { WidgetCpe }, rig.AskedFor);
// Nothing was mapped, so no inventory row is touched.
Assert.Empty(rig.Maps.Tables.InventoryWrites);
Assert.Equal(string.Empty, installed["CpeUri"]);
}
[Fact]
public async Task Run_NoDictionaryStored_SaysSoForEveryProgram()
{
var rig = await Rig.CreateAsync(dictionaryStored: false);
var body = await rig.ResolveAsync(Widget, Tool);
Assert.Equal(new[] { (Widget.SoftwareName, "no_dictionary"), (Tool.SoftwareName, "no_dictionary") }, FailedOf(body));
Assert.Empty(rig.AskedFor);
}
public static TheoryData<string> BodiesWithoutUsableItems() => new()
{
"{}",
"""{ "items": [] }""",
"""{ "items": null }""",
JsonSerializer.Serialize(new { items = Enumerable.Range(0, 26).Select(n => new { softwareName = $"Contoso Tool {n}" }) }),
};
[Theory]
[MemberData(nameof(BodiesWithoutUsableItems))]
public async Task Run_NoItemsOrMoreThanARequestMayCarry_IsABadRequest(string json)
{
var rig = await Rig.CreateAsync();
var response = await rig.Function.Run(OperatorCall.RequestWithJson(json));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.BadRequest, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.AskedFor);
}
[Fact]
public async Task Run_AsManyItemsAsARequestMayCarry_AreAllTried()
{
var rig = await Rig.CreateAsync();
var items = Enumerable.Range(0, 25)
.Select(n => new AutoResolveCpeMappingItem { SoftwareName = $"Tailspin Launcher {(char)('a' + n)}", NormalizedVendor = "tailspin" })
.ToArray();
var body = await rig.ResolveAsync(items);
Assert.Equal(25, body.GetProperty("totalProcessed").GetInt32());
Assert.All(FailedOf(body), failed => Assert.Equal("no_exact_product", failed.Reason));
}
[Theory]
[InlineData(CpeAutoMapOutcome.AlreadyMapped, "already_mapped")]
[InlineData(CpeAutoMapOutcome.Ignored, "ignored")]
[InlineData(CpeAutoMapOutcome.NoExactProduct, "no_exact_product")]
[InlineData(CpeAutoMapOutcome.NoCves, "no_cves")]
[InlineData(CpeAutoMapOutcome.UnversionedCves, "unversioned_cves")]
[InlineData(CpeAutoMapOutcome.StorePackage, "store_package")]
[InlineData(CpeAutoMapOutcome.NvdUnavailable, "nvd_unavailable")]
[InlineData(CpeAutoMapOutcome.NoDictionary, "no_dictionary")]
public void ReasonOf_NamesTheOutcome(CpeAutoMapOutcome outcome, string expected) =>
Assert.Equal(expected, AutoResolveCpeMappingFunction.ReasonOf(outcome));
[Fact]
public void ReasonOf_HasAReasonOfItsOwnForEveryOutcomeThatIsNotAMapping()
{
var reasons = Enum.GetValues<CpeAutoMapOutcome>()
.Where(outcome => outcome != CpeAutoMapOutcome.Mapped)
.Select(AutoResolveCpeMappingFunction.ReasonOf)
.ToList();
// An outcome added later must not fall back to the catch-all.
Assert.DoesNotContain("not_mapped", reasons);
Assert.Equal(reasons.Count, reasons.Distinct().Count());
}
// -----------------------------------------------------------------------
// Answers that need no request, Store packages, unreadable mappings
// -----------------------------------------------------------------------
[Fact]
public async Task Run_AfterNvdWentQuiet_WhateverNeedsNoRequestStillGetsItsRealAnswer()
{
var rig = await Rig.CreateAsync();
rig.Maps.Scan.NvdAnswer = _ => NvdFixture.Status(HttpStatusCode.InternalServerError);
var cachedAt = DateTime.UtcNow.AddDays(-2);
rig.Maps.Scan.Cache(VulnerabilityCacheRows.Filled(
ReaderCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10002", 7.5, ReaderCpe, "9.0") }, cachedAt, nvdTotal: 1));
rig.Maps.Scan.Cache(VulnerabilityCacheRows.Filled(ViewerCpe, Array.Empty<CachedCve>(), cachedAt));
rig.Maps.Scan.Cache(VulnerabilityCacheRows.Filled(
EditorCpe, new[] { VulnerabilityCacheRows.CveOfEveryVersion("CVE-2026-10004", 9.8, EditorCpe) }, cachedAt, nvdTotal: 1));
var ignore = CpeIgnoreEntry.Build(Launcher.NormalizedVendor, Launcher.SoftwareName, "manual")!.Value;
rig.Maps.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var installed = rig.Maps.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0");
var body = await rig.ResolveAsync(Widget, Gadget, Launcher, Terminal, Reader, Viewer, Editor, Tool);
// The widget's request is the one NVD left unanswered. Nothing is sent after it.
Assert.Equal(new[] { WidgetCpe }, rig.AskedFor);
Assert.Equal(new[] { (Reader.SoftwareName, ReaderCpe) }, ResolvedOf(body));
Assert.Equal(
new[]
{
(Widget.SoftwareName, "nvd_unavailable"),
(Gadget.SoftwareName, "already_mapped"),
(Launcher.SoftwareName, "ignored"),
(Terminal.SoftwareName, "no_exact_product"),
(Viewer.SoftwareName, "no_cves"),
(Editor.SoftwareName, "unversioned_cves"),
// The only other program whose answer NVD would have had to give.
(Tool.SoftwareName, "nvd_unavailable"),
},
FailedOf(body));
Assert.Equal(new[] { "adatum:adatum reader" }, rig.Maps.Tables.RowKeys(CpeMappingStore.AutoPartition));
Assert.Equal(ReaderCpe, installed["CpeUri"]);
// The tool was not noted as "no CVEs" or as waiting: the next attempt asks NVD about it.
Assert.DoesNotContain(CveCacheFiller.CacheKey(ToolCpe), rig.Maps.Scan.CacheRows.Keys);
}
[Fact]
public async Task Run_ProductWithACveThatNamesNoVersion_IsNotMapped_AndSaysWhy()
{
var rig = await Rig.CreateAsync();
rig.Maps.Scan.NvdAnswer = request =>
NvdFixture.Page(NvdFixture.Cve("CVE-2026-10004", 9.8, NvdFixture.EveryVersion(request.Cpe)));
var installed = rig.Maps.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2");
var body = await rig.ResolveAsync(Widget);
Assert.Equal(new[] { (Widget.SoftwareName, "unversioned_cves") }, FailedOf(body));
Assert.Empty(ResolvedOf(body));
Assert.Empty(rig.Maps.Tables.Rows(CpeMappingStore.AutoPartition));
Assert.Equal(string.Empty, installed["CpeUri"]);
}
[Fact]
public async Task Run_ProgramThatOccursOnlyAsAStorePackage_IsNotMapped_AndCostsNoRequest()
{
var rig = await Rig.CreateAsync();
// A package carries the version of its packaging, which need not be the product's.
var package = rig.Maps.Tables.AddInventory("contoso", "Contoso Widget", "1.0.3", registrySource: "AppX");
// The tool is a package on some devices and installed the classic way on others.
rig.Maps.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX");
var installed = rig.Maps.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1");
var body = await rig.ResolveAsync(Widget, Tool);
Assert.Equal(new[] { (Widget.SoftwareName, "store_package") }, FailedOf(body));
Assert.Equal(new[] { (Tool.SoftwareName, ToolCpe) }, ResolvedOf(body));
Assert.Equal(new[] { ToolCpe }, rig.AskedFor);
Assert.Equal(new[] { "northwind:northwind tool" }, rig.Maps.Tables.RowKeys(CpeMappingStore.AutoPartition));
Assert.Equal(string.Empty, package["CpeUri"]);
Assert.Equal(ToolCpe, installed["CpeUri"]);
}
[Fact]
public async Task Run_StoreOnlyProgram_ThatAMappingCoversOrAnOperatorIgnores_IsToldThat()
{
var rig = await Rig.CreateAsync();
rig.Maps.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0.3", cpeUri: GadgetCpe, registrySource: "AppX");
rig.Maps.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0.3", registrySource: "AppX");
var ignore = CpeIgnoreEntry.Build(Launcher.NormalizedVendor, Launcher.SoftwareName, "manual")!.Value;
rig.Maps.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var body = await rig.ResolveAsync(Gadget, Launcher);
// "Store package" would send the operator off to map by hand what is mapped or ignored already.
Assert.Equal(
new[] { (Gadget.SoftwareName, "already_mapped"), (Launcher.SoftwareName, "ignored") }, FailedOf(body));
Assert.Empty(rig.AskedFor);
}
[Fact]
public async Task Run_MappingsCannotBeRead_IsServiceUnavailableWithARetryHint_AndNothingIsAskedOrWritten()
{
var rig = await Rig.CreateAsync();
rig.Maps.Tables.ReadFailure = new InvalidOperationException("table storage is not reachable");
var response = await rig.Function.Run(OperatorCall.Request(
new AutoResolveCpeMappingRequest { Items = new List<AutoResolveCpeMappingItem> { Widget, Tool } }));
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.ServiceUnavailable, EndpointHarness.ErrorCode(response));
Assert.Equal(new[] { "60" }, response.Headers.GetValues("Retry-After"));
Assert.Empty(rig.AskedFor);
Assert.Empty(rig.Maps.Tables.Writes);
Assert.Empty(rig.Maps.Tables.InventoryWrites);
}
}
@@ -0,0 +1,67 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Operator tool, not a check: reduces a downloaded NVD dictionary feed with the real reader and
/// writes down what came out and what it cost, so a change to the reader or to NVD's feed can be
/// measured before it ships.
/// <code>AM_CPE_FEED_FILE=&lt;path to nvdcpe-2.0.tar.gz&gt; dotnet test --filter CpeDictionaryFeedCheckTests</code>
/// writes <c>&lt;feed&gt;.reduced.json</c> (product keys and titles) and <c>&lt;feed&gt;.summary.json</c> next to
/// the feed. Without the variable the test does nothing.
/// </summary>
public class CpeDictionaryFeedCheckTests
{
private const string FileVariable = "AM_CPE_FEED_FILE";
[Fact]
public async Task Reduce_a_downloaded_feed()
{
var file = Environment.GetEnvironmentVariable(FileVariable);
if (string.IsNullOrWhiteSpace(file)) return;
var heapBefore = GC.GetTotalMemory(forceFullCollection: true);
long heapPeak = heapBefore;
using var sampling = new CancellationTokenSource();
var sampler = Task.Run(async () =>
{
while (!sampling.IsCancellationRequested)
{
heapPeak = Math.Max(heapPeak, GC.GetTotalMemory(forceFullCollection: false));
await Task.Delay(20);
}
});
var watch = Stopwatch.StartNew();
Dictionary<string, CpeDictionaryProduct> products;
await using (var feed = File.OpenRead(file))
products = await CpeDictionaryService.ReduceAsync(feed, CancellationToken.None);
watch.Stop();
sampling.Cancel();
await sampler;
var stored = JsonSerializer.SerializeToUtf8Bytes(products);
File.WriteAllText(file + ".reduced.json", JsonSerializer.Serialize(
new SortedDictionary<string, string>(products.ToDictionary(p => p.Key, p => p.Value.T), StringComparer.Ordinal)));
File.WriteAllText(file + ".summary.json", JsonSerializer.Serialize(new
{
products = products.Count,
seconds = Math.Round(watch.Elapsed.TotalSeconds, 1),
storedBytes = stored.Length,
heapBeforeMb = heapBefore / 1024 / 1024,
heapPeakMb = heapPeak / 1024 / 1024,
processPeakWorkingSetMb = Process.GetCurrentProcess().PeakWorkingSet64 / 1024 / 1024,
}));
Assert.True(products.Count >= CpeDictionaryService.MinProducts);
}
}
@@ -0,0 +1,827 @@
using System.Globalization;
using System.Net;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeDictionaryResolver"/> gives a program an automatic mapping only when its name is
/// exactly an NVD product name, a scan could report a CVE for that product, and none of those
/// CVEs would be reported for every install whatever its version. It asks NVD once per product
/// and keeps what it learned as the product's cache row, writes nothing for an answer NVD did
/// not give, and a nightly pass stops asking when NVD stops answering or the pass has used its
/// requests or its time. A program that occurs only as a Store package is left to the operator,
/// and a cached row of the older shape is asked about again instead of being taken as an answer.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class CpeDictionaryResolverTests : IDisposable
{
private const string Auto = CpeMappingStore.AutoPartition;
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private const string ToolCpe = "cpe:2.3:a:northwind:tool";
private const string ReaderCpe = "cpe:2.3:a:adatum:reader";
private static readonly (string ProductKey, string Title)[] Dictionary =
{
("contoso:widget", "Contoso Widget"),
("fabrikam:gadget", "Fabrikam Gadget"),
("northwind:tool", "Northwind Tool"),
("adatum:reader", "Adatum Reader"),
};
public CpeDictionaryResolverTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
/// <summary>A rig in which the gadget is mapped by the seed and nothing else is.</summary>
private static Task<MappingRig> RigAsync() => MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"));
private static CpeDictionaryResolver ResolverOf(MappingRig rig) => rig.Resolver(CpeDictionaryFixture.Stored(Dictionary));
/// <summary>NVD lists one CVE for whatever product it is asked about.</summary>
private static HttpResponseMessage OneCve(NvdRequest request) =>
NvdFixture.Page(NvdFixture.Cve("CVE-2026-10001", 8.8, NvdFixture.Below(request.Cpe, "9.0")));
private static IEnumerable<string> AskedFor(MappingRig rig) => rig.Scan.Nvd.Requests.Select(r => r.Cpe);
// -----------------------------------------------------------------------
// One name
// -----------------------------------------------------------------------
[Fact]
public async Task Resolve_NoDictionaryStored_IsNoDictionary()
{
var rig = await RigAsync();
var resolver = rig.Resolver(CpeDictionaryFixture.Stored());
var result = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.NoDictionary, result.Outcome);
Assert.False(result.AskedNvd);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Resolve_NameThatIsExactlyAProductWithCves_GetsAnAutomaticMapping_AndItsFirstCacheFill()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
var before = DateTime.UtcNow;
var result = await ResolverOf(rig).ResolveAsync("Contoso", "Contoso Widget 4.2 (x64)", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Equal(WidgetCpe, result.CpeUri);
Assert.Equal("Contoso Widget", result.ProductTitle);
Assert.True(result.AskedNvd);
Assert.False(result.NvdDown);
// Filed under the vendor and the name without its version, so one row covers every version.
var row = Assert.Single(rig.Tables.Rows(Auto));
Assert.Equal(CpeSeedFile.RowKeyOf("contoso", "contoso widget"), row["RowKey"]);
Assert.Equal("contoso", row["NormalizedVendor"]);
Assert.Equal("contoso widget", row["NormalizedProduct"]);
Assert.Equal("contoso", row["CpeVendor"]);
Assert.Equal("widget", row["CpeProduct"]);
Assert.Equal(WidgetCpe, row["CpeUri"]);
Assert.Equal("auto", row["Source"]);
Assert.Equal("Contoso Widget", row["ProductTitle"]);
Assert.Equal("[]", row["DisplayNamePatternsJson"]);
Assert.Equal("[]", row["PublisherPatternsJson"]);
Assert.Equal("[]", row["ExcludePatternsJson"]);
var createdAt = DateTime.Parse((string)row["CreatedAt"], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
Assert.InRange(createdAt, before, DateTime.UtcNow);
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
var cached = rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)];
Assert.Equal("filled", cached["State"]);
Assert.Equal(1, cached["NvdTotal"]);
// The row is a mapping the matching reads, ranked as automatic.
var match = (await rig.Store.ReloadAsync()).Matcher.Match("contoso", "contoso widget 5.0", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal(WidgetCpe, match!.Value.Mapping.CpeUri);
Assert.Equal(CpeMappingSource.Auto, match.Value.Mapping.Source);
}
[Fact]
public async Task Resolve_ProductNameWhoseVendorIsThePublisher_IsFiledUnderTheProgramsOwnVendor()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
var result = await ResolverOf(rig).ResolveAsync("northwind traders", "Tool 3.1", "Northwind Traders Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Equal(ToolCpe, result.CpeUri);
var row = Assert.Single(rig.Tables.Rows(Auto));
Assert.Equal("northwind traders:tool", row["RowKey"]);
Assert.Equal("northwind traders", row["NormalizedVendor"]);
Assert.Equal("tool", row["NormalizedProduct"]);
}
[Fact]
public async Task Resolve_ProductWhoseCveDataIsAlreadyCached_IsMappedWithoutAskingNvd()
{
var rig = await RigAsync();
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10002", 7.5, WidgetCpe, "9.0") }, DateTime.UtcNow.AddDays(-2), nvdTotal: 3));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.False(result.AskedNvd);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Single(rig.Tables.Rows(Auto));
}
[Fact]
public async Task Resolve_NvdListsNoCve_IsNoCves_AndTheEmptyAnswerIsNotAskedForAgain()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Page();
var resolver = ResolverOf(rig);
var first = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
var second = await resolver.ResolveAsync("contoso", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.NoCves, first.Outcome);
Assert.Equal(WidgetCpe, first.CpeUri);
Assert.True(first.AskedNvd);
Assert.False(first.NvdDown);
Assert.Equal(CpeAutoMapOutcome.NoCves, second.Outcome);
Assert.False(second.AskedNvd);
// A mapping would only make the program look checked.
Assert.Empty(rig.Tables.Rows(Auto));
Assert.Empty(rig.Tables.Writes);
// The note is the product's cache row with nothing in it.
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
var note = rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)];
Assert.Equal("filled", note["State"]);
Assert.Equal(0, note["NvdTotal"]);
}
[Theory]
[InlineData("the product is only the platform another product runs on")]
[InlineData("the product's CVEs are older than the cache keeps")]
public async Task Resolve_NvdCountsCvesButAScanCouldReportNone_IsNoCves_AndIsNotAskedForAgain(string why)
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = request => NvdFixture.Page(why switch
{
"the product is only the platform another product runs on" => NvdFixture.Cve(
"CVE-2026-10003", 9.8,
NvdFixture.Below(GadgetCpe, "3.0"),
new NvdCpeMatch { Vulnerable = false, Criteria = $"{request.Cpe}:*:*:*:*:*:*:*:*" }),
_ => NvdFixture.Cve("CVE-2026-10003", 9.8, DateTime.UtcNow.AddYears(-7), NvdFixture.Below(request.Cpe, "3.0")),
});
var resolver = ResolverOf(rig);
var first = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
var second = await resolver.ResolveAsync("contoso", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.NoCves, first.Outcome);
Assert.True(first.AskedNvd);
Assert.False(first.NvdDown);
Assert.Equal(CpeAutoMapOutcome.NoCves, second.Outcome);
Assert.False(second.AskedNvd);
Assert.Empty(rig.Tables.Writes);
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
// NVD's own count says one; what the product's row holds is what a scan can report.
var note = rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)];
Assert.Equal(1, note["NvdTotal"]);
Assert.Equal(0, note["TotalCves"]);
}
[Theory]
// The EPSS walk rewrites a row from before the total was recorded with a total of 0.
[InlineData(0)]
[InlineData(40)]
public async Task Resolve_CachedRowWithCves_IsMapped_WhateverTotalItCarries(int nvdTotal)
{
var rig = await RigAsync();
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10002", 7.5, WidgetCpe, "9.0") }, DateTime.UtcNow.AddDays(-2), nvdTotal));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Empty(rig.Scan.Nvd.Requests);
}
[Fact]
public async Task Resolve_CachedRowWithoutCves_IsNoCves_WhateverTotalItCarries()
{
var rig = await RigAsync();
rig.Scan.Cache(VulnerabilityCacheRows.Filled(WidgetCpe, Array.Empty<CachedCve>(), DateTime.UtcNow.AddDays(-2), nvdTotal: 40));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.NoCves, result.Outcome);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
// -----------------------------------------------------------------------
// CVEs that name no version
// -----------------------------------------------------------------------
/// <summary>NVD lists one CVE for whatever product it is asked about, and for every version of it.</summary>
private static HttpResponseMessage OneUnversionedCve(NvdRequest request) =>
NvdFixture.Page(NvdFixture.Cve("CVE-2026-10004", 9.8, NvdFixture.EveryVersion(request.Cpe)));
[Theory]
[InlineData("*")]
[InlineData("windows")]
public async Task Resolve_ProductWithACveThatNamesNoVersion_IsLeftToTheOperator_AndIsNotAskedForAgain(string targetSw)
{
var rig = await RigAsync();
// One record of that kind is enough, next to any number of ordinary ones.
rig.Scan.NvdAnswer = request => NvdFixture.Page(
NvdFixture.Cve("CVE-2026-10001", 8.8, NvdFixture.Below(request.Cpe, "9.0")),
NvdFixture.Cve("CVE-2026-10004", 9.8, NvdFixture.EveryVersion(request.Cpe, targetSw)));
var resolver = ResolverOf(rig);
var first = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
var second = await resolver.ResolveAsync("contoso", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.UnversionedCves, first.Outcome);
Assert.Equal(WidgetCpe, first.CpeUri);
Assert.Equal("Contoso Widget", first.ProductTitle);
Assert.True(first.AskedNvd);
Assert.False(first.NvdDown);
Assert.Equal(CpeAutoMapOutcome.UnversionedCves, second.Outcome);
Assert.False(second.AskedNvd);
// With a mapping, every install of the program would carry that CVE, whatever its version.
Assert.Empty(rig.Tables.Rows(Auto));
Assert.Empty(rig.Tables.Writes);
// The answer is kept as the product's cache row, so asking again costs no request.
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
var kept = rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)];
Assert.Equal("filled", kept["State"]);
Assert.Equal(2, kept["TotalCves"]);
}
[Theory]
[InlineData("a range with an end")]
[InlineData("a range with a start")]
[InlineData("one version")]
// NVD's "-" stands for a version it does not know; no install is reported for it.
[InlineData("the unknown version")]
// Never reported for an install on Windows, so it cannot become a false alarm there.
[InlineData("every version on another platform")]
public async Task Resolve_ProductWhoseCveIsNotReportedForEveryInstall_IsMapped(string range)
{
var match = range switch
{
"a range with an end" => NvdFixture.Below(WidgetCpe, "9.0"),
"a range with a start" => new NvdCpeMatch { Vulnerable = true, Criteria = $"{WidgetCpe}:*:*:*:*:*:*:*:*", VersionStartIncluding = "2.0" },
"one version" => new NvdCpeMatch { Vulnerable = true, Criteria = $"{WidgetCpe}:4.2:*:*:*:*:*:*:*" },
"the unknown version" => new NvdCpeMatch { Vulnerable = true, Criteria = $"{WidgetCpe}:-:*:*:*:*:*:*:*" },
_ => NvdFixture.EveryVersion(WidgetCpe, "android"),
};
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Page(NvdFixture.Cve("CVE-2026-10005", 9.8, match));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Single(rig.Tables.Rows(Auto));
}
[Fact]
public async Task Resolve_CachedCveThatNamesNoVersion_IsFoundWithoutAskingNvd()
{
var rig = await RigAsync();
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.CveOfEveryVersion("CVE-2026-10004", 9.8, WidgetCpe) }, DateTime.UtcNow.AddDays(-2), nvdTotal: 1));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.UnversionedCves, result.Outcome);
Assert.False(result.AskedNvd);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
// -----------------------------------------------------------------------
// Without asking NVD
// -----------------------------------------------------------------------
[Fact]
public async Task Resolve_NotAskingNvd_ProductWithoutCachedCves_IsUnanswered_AndNothingIsSentOrNoted()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd", askNvd: false);
Assert.Equal(CpeAutoMapOutcome.NvdUnavailable, result.Outcome);
Assert.Equal(WidgetCpe, result.CpeUri);
Assert.False(result.AskedNvd);
// Nothing was learned about NVD: what the caller knows about it stays what it was.
Assert.False(result.NvdDown);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Scan.CacheRows);
Assert.Empty(rig.Scan.PendingMarkers);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Resolve_NotAskingNvd_StillSaysWhateverNeedsNoRequest()
{
var rig = await RigAsync();
var ignore = CpeIgnoreEntry.Build("northwind", "Northwind Tool", "manual")!.Value;
rig.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var resolver = ResolverOf(rig);
var alreadyMapped = await resolver.ResolveAsync("fabrikam", "Fabrikam Gadget 2.0", "Fabrikam Inc", askNvd: false);
var ignored = await resolver.ResolveAsync("northwind", "Northwind Tool 3.1", "Northwind Traders", askNvd: false);
var noProduct = await resolver.ResolveAsync("tailspin", "Tailspin Launcher", "Tailspin Toys", askNvd: false);
Assert.Equal(CpeAutoMapOutcome.AlreadyMapped, alreadyMapped.Outcome);
Assert.Equal(CpeAutoMapOutcome.Ignored, ignored.Outcome);
Assert.Equal(CpeAutoMapOutcome.NoExactProduct, noProduct.Outcome);
Assert.Empty(rig.Scan.Nvd.Requests);
}
[Fact]
public async Task Resolve_NotAskingNvd_ProductWhoseCvesAreCached_GetsTheAnswerTheCacheGives()
{
var rig = await RigAsync();
var cachedAt = DateTime.UtcNow.AddDays(-2);
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10002", 7.5, WidgetCpe, "9.0") }, cachedAt, nvdTotal: 1));
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
ToolCpe, new[] { VulnerabilityCacheRows.CveOfEveryVersion("CVE-2026-10004", 9.8, ToolCpe) }, cachedAt, nvdTotal: 1));
rig.Scan.Cache(VulnerabilityCacheRows.Filled(ReaderCpe, Array.Empty<CachedCve>(), cachedAt));
var resolver = ResolverOf(rig);
var mapped = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd", askNvd: false);
var unversioned = await resolver.ResolveAsync("northwind", "Northwind Tool 3.1", "Northwind Traders", askNvd: false);
var noCves = await resolver.ResolveAsync("adatum", "Adatum Reader 2.0", "Adatum Corporation", askNvd: false);
Assert.Equal(CpeAutoMapOutcome.Mapped, mapped.Outcome);
Assert.Equal(CpeAutoMapOutcome.UnversionedCves, unversioned.Outcome);
Assert.Equal(CpeAutoMapOutcome.NoCves, noCves.Outcome);
Assert.Equal(new[] { "contoso:contoso widget" }, rig.Tables.RowKeys(Auto));
Assert.Empty(rig.Scan.Nvd.Requests);
}
[Theory]
[InlineData(HttpStatusCode.InternalServerError, true)]
[InlineData(HttpStatusCode.TooManyRequests, true)]
// The gateway in front of NVD refusing the client leaves the next request no better off than a throttle.
[InlineData(HttpStatusCode.Forbidden, true)]
// NVD answered and turned this one request down: that says nothing about the next product.
[InlineData(HttpStatusCode.NotFound, false)]
public async Task Resolve_NvdGivesNoUsableAnswer_IsUnavailable_AndWritesNothing(HttpStatusCode status, bool nvdDown)
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Status(status);
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.NvdUnavailable, result.Outcome);
Assert.Equal(nvdDown, result.NvdDown);
Assert.True(result.AskedNvd);
Assert.Empty(rig.Tables.Writes);
Assert.Empty(rig.Scan.CacheWrites);
Assert.Empty(rig.Scan.CacheRows);
}
[Fact]
public async Task Resolve_AfterAnAnswerNvdDidNotGive_TheNameIsTriedAgain()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Status(HttpStatusCode.InternalServerError);
var resolver = ResolverOf(rig);
await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
rig.Scan.NvdAnswer = OneCve;
var result = await resolver.ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Equal(new[] { WidgetCpe, WidgetCpe }, AskedFor(rig));
}
[Fact]
public async Task Resolve_NameAMappingAlreadyCovers_IsAlreadyMapped()
{
var rig = await RigAsync();
var result = await ResolverOf(rig).ResolveAsync("fabrikam", "Fabrikam Gadget 2.0", "Fabrikam Inc");
Assert.Equal(CpeAutoMapOutcome.AlreadyMapped, result.Outcome);
Assert.Equal(GadgetCpe, result.CpeUri);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Resolve_NameOnTheIgnoreList_IsIgnored()
{
var rig = await RigAsync();
var ignore = CpeIgnoreEntry.Build("contoso", "Contoso Widget 4.2 (x64)", "manual")!.Value;
rig.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Ignored, result.Outcome);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
[Theory]
[InlineData("contoso", "Contoso Widget Pro 4.2", "Contoso Ltd")] // one word more than the product
[InlineData("tailspin", "Widget 4.2", "Tailspin Toys")] // the product's name under another vendor
[InlineData("tailspin", "Tailspin Launcher", "Tailspin Toys")] // not in the dictionary at all
[InlineData("contoso", "4.2 (x64)", "Contoso Ltd")] // no name left
public async Task Resolve_NameThatIsNotExactlyAProduct_IsNoExactProduct(string vendor, string displayName, string publisher)
{
var rig = await RigAsync();
var result = await ResolverOf(rig).ResolveAsync(vendor, displayName, publisher);
Assert.Equal(CpeAutoMapOutcome.NoExactProduct, result.Outcome);
Assert.Null(result.CpeUri);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Tables.Writes);
}
// -----------------------------------------------------------------------
// The nightly pass over the inventory
// -----------------------------------------------------------------------
[Fact]
public async Task ResolveInventory_NoDictionaryStored_SaysSoAndAsksNothing()
{
var rig = await RigAsync();
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2");
var run = await rig.Resolver(CpeDictionaryFixture.Stored()).ResolveInventoryAsync();
Assert.True(run.NoDictionary);
Assert.Equal(0, run.UnmappedNames);
Assert.Empty(rig.Scan.Nvd.Requests);
}
[Fact]
public async Task ResolveInventory_TriesTheMostUsedNamesFirst_WithTheSessionsOfAllRowsOfAName()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", sessions: 5);
rig.Tables.AddInventory("contoso", "Contoso Widget 1.0", "1.0", sessions: 3);
rig.Tables.AddInventory("contoso", "Contoso Widget 2.0", "2.0", sessions: 4);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 6);
// Used most of all, but mapped already, or no product NVD knows: neither costs a request.
rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget 1.0", "1.0", sessions: 100);
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 50);
var resolver = ResolverOf(rig);
var heldBefore = await rig.Store.GetAsync();
var run = await resolver.ResolveInventoryAsync();
Assert.False(run.NoDictionary);
Assert.Equal(4, run.UnmappedNames);
Assert.Equal(3, run.Mapped);
Assert.Equal(0, run.NoCves);
Assert.Equal(0, run.NvdUnavailable);
Assert.Equal(0, run.Postponed);
// Widget: 3 + 4 sessions, ahead of the tool with 6 and the reader with 5.
Assert.Equal(new[] { WidgetCpe, ToolCpe, ReaderCpe }, AskedFor(rig));
Assert.Equal(
new[] { "adatum:adatum reader", "contoso:contoso widget", "northwind:northwind tool" }, rig.Tables.RowKeys(Auto));
// This instance's copy of the mappings is dropped, so its next reader sees the new ones.
var heldAfter = await rig.Store.GetAsync();
Assert.NotSame(heldBefore, heldAfter);
Assert.NotNull(heldAfter.Matcher.Match("northwind", "northwind tool 3.1", "Northwind Tool 3.1", "northwind"));
}
[Fact]
public async Task ResolveInventory_StopsAskingOnceNvdLeftTwoRequestsUnanswered_AndPostponesTheRest()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Status(HttpStatusCode.InternalServerError);
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", sessions: 9);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 7);
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", sessions: 5);
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 3);
var run = await ResolverOf(rig).ResolveInventoryAsync();
Assert.Equal(2, CpeDictionaryResolver.MaxUnansweredPerRun);
Assert.Equal(new[] { WidgetCpe, ToolCpe }, AskedFor(rig));
Assert.Equal(2, run.NvdUnavailable);
// Only the reader waits for the next pass: the launcher is no product NVD could be asked about.
Assert.Equal(1, run.Postponed);
Assert.Equal(0, run.Mapped);
Assert.Equal(4, run.UnmappedNames);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task ResolveInventory_RequestNvdTurnsDown_DoesNotEndThePass()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = request => request.Cpe == ReaderCpe ? OneCve(request) : NvdFixture.Status(HttpStatusCode.NotFound);
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", sessions: 9);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 7);
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", sessions: 5);
var run = await ResolverOf(rig).ResolveInventoryAsync();
Assert.Equal(new[] { WidgetCpe, ToolCpe, ReaderCpe }, AskedFor(rig));
Assert.Equal(2, run.NvdUnavailable);
Assert.Equal(1, run.Mapped);
Assert.Equal(0, run.Postponed);
}
[Fact]
public async Task ResolveInventory_SendsNoMoreThanTheRequestsOfOnePass_AndTakesUpTheRestInTheNext()
{
const int extra = 7;
var names = Enumerable.Range(0, CpeDictionaryResolver.MaxNvdRequestsPerRun + extra)
.Select(n => $"{(char)('a' + n / 26)}{(char)('a' + n % 26)}")
.ToList();
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Page();
var resolver = rig.Resolver(CpeDictionaryFixture.Stored(
names.Select(name => ($"contoso:tool_{name}", $"Contoso Tool {name}")).ToArray()));
// Each name is used a little less than the one before it.
for (var n = 0; n < names.Count; n++)
rig.Tables.AddInventory("contoso", $"Contoso Tool {names[n]} 1.0", "1.0", sessions: 1000 - n);
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 5000);
var expectedOrder = names.Select(name => $"cpe:2.3:a:contoso:tool_{name}").ToList();
var first = await resolver.ResolveInventoryAsync();
Assert.Equal(names.Count + 1, first.UnmappedNames);
Assert.Equal(CpeDictionaryResolver.MaxNvdRequestsPerRun, first.NoCves);
Assert.Equal(extra, first.Postponed);
Assert.Equal(expectedOrder.Take(CpeDictionaryResolver.MaxNvdRequestsPerRun), AskedFor(rig));
var second = await resolver.ResolveInventoryAsync();
// What the first pass learned costs no request now, so the pass reaches the rest.
Assert.Equal(names.Count, second.NoCves);
Assert.Equal(0, second.Postponed);
Assert.Equal(expectedOrder, AskedFor(rig));
}
[Theory]
[InlineData(HttpStatusCode.TooManyRequests)]
[InlineData(HttpStatusCode.Forbidden)]
public async Task ResolveInventory_NvdRefusesTheClient_SendsNothingMoreThisPass(HttpStatusCode status)
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = _ => NvdFixture.Status(status);
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", sessions: 9);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 7);
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", sessions: 5);
var run = await ResolverOf(rig).ResolveInventoryAsync();
// The second product is turned away by the client's own cooldown, without a request.
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
Assert.Equal(0, run.Mapped);
Assert.Equal(3, run.NvdUnavailable + run.Postponed);
Assert.Empty(rig.Tables.Writes);
Assert.Empty(rig.Scan.CacheRows);
}
[Fact]
public async Task ResolveInventory_PassWithoutTimeLeft_AsksNothing_AndPostponesEveryProductItWouldHaveAskedAbout()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", sessions: 9);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 7);
rig.Tables.AddInventory("adatum", "Adatum Reader", "1.0.3", sessions: 5, registrySource: "AppX");
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 3);
var run = await rig.Resolver(CpeDictionaryFixture.Stored(Dictionary), runBudget: TimeSpan.Zero).ResolveInventoryAsync();
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Equal(4, run.UnmappedNames);
// Only what waits for a request is postponed: the launcher is no product NVD could be
// asked about, and the reader is a Store package whatever time the pass has.
Assert.Equal(2, run.Postponed);
Assert.Equal(1, run.StorePackages);
Assert.Equal(0, run.Mapped);
Assert.Equal(0, run.NvdUnavailable);
Assert.Empty(run.Created);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task ResolveInventory_ListsEachMappingItCreated_AsProgramAndProduct_MostUsedFirst()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = request => request.Cpe == ReaderCpe ? NvdFixture.Page() : OneCve(request);
rig.Tables.AddInventory("northwind traders", "Tool 3.1", "3.1", sessions: 7, publisher: "Northwind Traders Ltd");
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2", sessions: 9);
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", sessions: 8);
var run = await ResolverOf(rig).ResolveInventoryAsync();
Assert.Equal(2, run.Mapped);
Assert.Equal(1, run.NoCves);
// The program by its name without version and architecture, the product as vendor:product.
Assert.Equal(new[] { "contoso widget -> contoso:widget", "tool -> northwind:tool" }, run.Created);
}
[Fact]
public async Task ResolveInventory_CountsProductsLeftForACveWithoutVersion_AndDoesNotAskAboutThemTheNextNight()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = request => request.Cpe == WidgetCpe ? OneUnversionedCve(request) : OneCve(request);
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", sessions: 9);
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 7);
var resolver = ResolverOf(rig);
var first = await resolver.ResolveInventoryAsync();
var second = await resolver.ResolveInventoryAsync();
Assert.Equal(1, first.Unversioned);
Assert.Equal(1, first.Mapped);
Assert.Equal(new[] { "northwind tool -> northwind:tool" }, first.Created);
Assert.Equal(new[] { "northwind:northwind tool" }, rig.Tables.RowKeys(Auto));
// The widget is still unmapped, and still for the same reason; the tool is mapped now.
Assert.Equal(1, second.UnmappedNames);
Assert.Equal(1, second.Unversioned);
Assert.Equal(0, second.Mapped);
Assert.Empty(second.Created);
Assert.Equal(new[] { WidgetCpe, ToolCpe }, AskedFor(rig));
}
// -----------------------------------------------------------------------
// Store packages
// -----------------------------------------------------------------------
[Fact]
public async Task ResolveInventory_ProgramThatOccursOnlyAsAStorePackage_IsLeftToTheOperator_WithoutARequest()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
// A package carries the version of its packaging, which need not be the product's.
rig.Tables.AddInventory("contoso", "Contoso Widget", "1.0.3", sessions: 9, registrySource: "AppX");
rig.Tables.AddInventory("contoso", "Contoso Widget", "1.0.4", sessions: 2, registrySource: "AppX");
// Also installed the classic way on some devices: those rows carry the product's own version.
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", sessions: 7, registrySource: "AppX");
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", sessions: 1);
// A package whose name is no NVD product is not counted: nothing was held back for it.
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 3, registrySource: "AppX");
var run = await ResolverOf(rig).ResolveInventoryAsync();
Assert.Equal(3, run.UnmappedNames);
Assert.Equal(1, run.StorePackages);
Assert.Equal(1, run.Mapped);
Assert.Equal(0, run.Postponed);
Assert.Equal(new[] { ToolCpe }, AskedFor(rig));
Assert.Equal(new[] { "northwind:northwind tool" }, rig.Tables.RowKeys(Auto));
Assert.DoesNotContain(CveCacheFiller.CacheKey(WidgetCpe), rig.Scan.CacheRows.Keys);
}
[Fact]
public async Task StoreOnlyKeys_AreReadFromTheInventory_ForACallerThatResolvesSingleNames()
{
var rig = await RigAsync();
rig.Tables.AddInventory("contoso", "Contoso Widget", "1.0.3", registrySource: "AppX");
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX");
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1");
var keys = await ResolverOf(rig).StoreOnlyKeysAsync();
Assert.Equal(new[] { CpeMappingMatcher.SoftwareKey("contoso", "Contoso Widget") }, keys);
}
// -----------------------------------------------------------------------
// A cached row of the older shape
// -----------------------------------------------------------------------
/// <summary>
/// A row as it was written before only the product's own ranges were kept: what it holds for
/// the widget is a CVE of the gadget, which merely names the widget next to it.
/// </summary>
private static Dictionary<string, object> OlderWidgetRow() => VulnerabilityCacheRows.OlderShape(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10006", 7.5, GadgetCpe, "9.0") }, DateTime.UtcNow.AddDays(-20));
[Fact]
public async Task Resolve_CachedRowOfTheOlderShape_IsNoAnswer_NvdIsAsked_AndTheRowRewritten()
{
var rig = await RigAsync();
rig.Scan.Cache(OlderWidgetRow());
rig.Scan.NvdAnswer = _ => NvdFixture.Page();
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
// Taken at its word, the row would have said "this product has CVEs".
Assert.Equal(CpeAutoMapOutcome.NoCves, result.Outcome);
Assert.True(result.AskedNvd);
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
Assert.Empty(rig.Tables.Writes);
var rewritten = rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)];
Assert.Equal(CveCacheFiller.SchemaVersion, rewritten["SchemaVersion"]);
Assert.Equal(0, rewritten["TotalCves"]);
}
[Fact]
public async Task Resolve_CachedRowOfTheOlderShape_ProductWithCvesOfItsOwn_IsMappedFromTheFreshAnswer()
{
var rig = await RigAsync();
rig.Scan.Cache(OlderWidgetRow());
rig.Scan.NvdAnswer = OneCve;
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd");
Assert.Equal(CpeAutoMapOutcome.Mapped, result.Outcome);
Assert.Equal(new[] { WidgetCpe }, AskedFor(rig));
Assert.Single(rig.Tables.Rows(Auto));
}
[Theory]
[InlineData(HttpStatusCode.InternalServerError)]
[InlineData(null)]
public async Task Resolve_CachedRowOfTheOlderShape_WithoutAnAnswerFromNvd_IsUnavailable_AndTheRowStays(HttpStatusCode? nvdStatus)
{
var rig = await RigAsync();
var older = OlderWidgetRow();
rig.Scan.Cache(older);
rig.Scan.NvdAnswer = _ => NvdFixture.Status(nvdStatus ?? HttpStatusCode.OK);
// Without a status NVD is not asked at all.
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget 4.2", "Contoso Ltd", askNvd: nvdStatus != null);
Assert.Equal(CpeAutoMapOutcome.NvdUnavailable, result.Outcome);
Assert.Equal(nvdStatus != null, result.AskedNvd);
Assert.Equal(nvdStatus == null ? 0 : 1, rig.Scan.Nvd.Requests.Count);
// A scan of a program a mapping names goes on reading the row until the walk rewrites it.
Assert.Same(older, rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)]);
Assert.Empty(rig.Tables.Writes);
}
// -----------------------------------------------------------------------
// A caller that says the program is a Store package
// -----------------------------------------------------------------------
[Fact]
public async Task Resolve_StorePackage_GetsNoMapping_AndCostsNoRequest()
{
var rig = await RigAsync();
rig.Scan.NvdAnswer = OneCve;
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget", "Contoso Ltd", storePackage: true);
Assert.Equal(CpeAutoMapOutcome.StorePackage, result.Outcome);
Assert.Equal(WidgetCpe, result.CpeUri);
Assert.Equal("Contoso Widget", result.ProductTitle);
Assert.False(result.AskedNvd);
Assert.False(result.NvdDown);
Assert.Empty(rig.Scan.Nvd.Requests);
Assert.Empty(rig.Scan.CacheReads);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Resolve_StorePackage_WhoseCvesAreCached_StillGetsNoMapping()
{
var rig = await RigAsync();
rig.Scan.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-10002", 7.5, WidgetCpe, "9.0") }, DateTime.UtcNow.AddDays(-2), nvdTotal: 1));
var result = await ResolverOf(rig).ResolveAsync("contoso", "Contoso Widget", "Contoso Ltd", storePackage: true);
Assert.Equal(CpeAutoMapOutcome.StorePackage, result.Outcome);
Assert.Empty(rig.Tables.Rows(Auto));
}
[Fact]
public async Task Resolve_StorePackage_IsSaidOnlyAfterWhatIsKnownAboutTheProgramAnyway()
{
var rig = await RigAsync();
var ignore = CpeIgnoreEntry.Build("northwind", "Northwind Tool", "manual")!.Value;
rig.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var resolver = ResolverOf(rig);
// A curated mapping covers a Store package knowingly, and an operator's "ignore" stands.
var alreadyMapped = await resolver.ResolveAsync("fabrikam", "Fabrikam Gadget", "Fabrikam Inc", storePackage: true);
var ignored = await resolver.ResolveAsync("northwind", "Northwind Tool", "Northwind Traders", storePackage: true);
var noProduct = await resolver.ResolveAsync("tailspin", "Tailspin Launcher", "Tailspin Toys", storePackage: true);
Assert.Equal(CpeAutoMapOutcome.AlreadyMapped, alreadyMapped.Outcome);
Assert.Equal(GadgetCpe, alreadyMapped.CpeUri);
Assert.Equal(CpeAutoMapOutcome.Ignored, ignored.Outcome);
Assert.Equal(CpeAutoMapOutcome.NoExactProduct, noProduct.Outcome);
Assert.Empty(rig.Scan.Nvd.Requests);
}
}
@@ -0,0 +1,825 @@
using System.Formats.Tar;
using System.Globalization;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeDictionaryService"/> reduces the NVD product dictionary feed to the application
/// products that can be Windows software and keeps that copy as a blob. The feed is read as a
/// stream through a small buffer, one product name at a time; what the buffer size is must not
/// show in the result. A refresh never throws and never replaces the stored copy with less than
/// a whole dictionary.
/// </summary>
public class CpeDictionaryServiceTests
{
private const string FeedVersion = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF";
// -----------------------------------------------------------------------
// Feed fixtures
// -----------------------------------------------------------------------
private static string Quote(string text) =>
"\"" + JsonEncodedText.Encode(text, JavaScriptEncoder.UnsafeRelaxedJsonEscaping) + "\"";
/// <summary>One entry of a chunk's products array, laid out like the feed's own pretty-printed files.</summary>
private static string Entry(string cpeName, bool deprecated, int refs, IEnumerable<(string Lang, string Title)> titles)
{
var json = new StringBuilder();
json.Append("{\r\n \"cpe\" :\t{\n");
json.Append(" \"deprecated\": ").Append(deprecated ? "true" : "false").Append(",\n");
json.Append(" \"cpeName\": ").Append(Quote(cpeName)).Append(",\n");
json.Append(" \"cpeNameId\": \"00000000-0000-4000-8000-000000000000\",\n");
json.Append(" \"lastModified\": \"2026-10-01T12:00:00.000\",\n");
json.Append(" \"titles\": [ ");
json.Append(string.Join(", ", titles.Select(t => $"{{ \"title\": {Quote(t.Title)}, \"lang\": {Quote(t.Lang)} }}")));
json.Append(" ],\n \"refs\": [");
json.Append(string.Join(",", Enumerable.Range(1, refs).Select(n =>
$"\n {{ \"ref\": \"https://www.contoso.example/advisories/{n}\", \"type\": \"Vendor\" }}")));
json.Append("\n ]\n }\n }");
return json.ToString();
}
private static string Name(string cpeName, params (string Lang, string Title)[] titles) => Entry(cpeName, false, 1, titles);
private static string Withdrawn(string cpeName, params (string Lang, string Title)[] titles) => Entry(cpeName, true, 1, titles);
/// <summary>A chunk file: the feed's root object with <paramref name="entries"/> as its products.</summary>
private static byte[] Chunk(params string[] entries) => Encoding.UTF8.GetBytes(
"{\n \"resultsPerPage\": " + entries.Length.ToString(CultureInfo.InvariantCulture)
+ ",\n \"startIndex\": 0,\n \"format\": \"NVD_CPE\",\n \"version\": \"2.0\",\n \"timestamp\": \"2026-11-12T03:00:00.000\",\n \"products\": [\n "
+ string.Join(",\n ", entries)
+ "\n ]\n}\n");
private static byte[] Json(string text) => Encoding.UTF8.GetBytes(text);
private static async Task<Dictionary<string, CpeDictionaryService.ProductState>> ReadAsync(Stream chunk, int? bufferBytes = null)
{
var reduced = new Dictionary<string, CpeDictionaryService.ProductState>(StringComparer.Ordinal);
if (bufferBytes is { } size)
await CpeDictionaryService.ReadChunkAsync(chunk, reduced, CancellationToken.None, size);
else
await CpeDictionaryService.ReadChunkAsync(chunk, reduced, CancellationToken.None);
return reduced;
}
private static Task<Dictionary<string, CpeDictionaryService.ProductState>> ReadAsync(byte[] chunk, int? bufferBytes = null) =>
ReadAsync(new MemoryStream(chunk), bufferBytes);
/// <summary>What a read left per product, in key order, so two reads compare value by value.</summary>
private static List<(string Product, string Title, bool WindowsCapable)> StateOf(
Dictionary<string, CpeDictionaryService.ProductState> reduced) => reduced
.OrderBy(pair => pair.Key, StringComparer.Ordinal)
.Select(pair => (pair.Key, pair.Value.Title, pair.Value.WindowsCapable))
.ToList();
/// <summary>A chunk's bytes as a stream that tells how it was read, and can hand out a few bytes per read like a network stream.</summary>
private sealed class ProbeStream : MemoryStream
{
private readonly int _mostPerRead;
public ProbeStream(byte[] content, int mostPerRead = int.MaxValue) : base(content, writable: false) => _mostPerRead = mostPerRead;
public int Reads { get; private set; }
/// <summary>Room the reader offered on its first read: its whole buffer, still empty.</summary>
public int FirstCapacity { get; private set; }
public int LargestCapacity { get; private set; }
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
if (Reads++ == 0) FirstCapacity = buffer.Length;
LargestCapacity = Math.Max(LargestCapacity, buffer.Length);
return base.ReadAsync(buffer.Length > _mostPerRead ? buffer[.._mostPerRead] : buffer, cancellationToken);
}
}
// -----------------------------------------------------------------------
// ReadChunkAsync: what is kept
// -----------------------------------------------------------------------
[Fact]
public async Task ReadChunk_KeepsApplicationProductsThatAreNotWithdrawn()
{
var chunk = Chunk(
Name("cpe:2.3:a:contoso:widget:4.2:*:*:*:*:*:*:*", ("en", "Contoso Widget 4.2")),
Name("cpe:2.3:o:contoso:widget_os:1.0:*:*:*:*:*:*:*", ("en", "Contoso Widget OS 1.0")),
Name("cpe:2.3:h:contoso:widget_box:-:*:*:*:*:*:*:*", ("en", "Contoso Widget Box")),
Withdrawn("cpe:2.3:a:contoso:retired:1.0:*:*:*:*:*:*:*", ("en", "Contoso Retired 1.0")),
Name("cpe:2.3:a:contoso:*:1.0:*:*:*:*:*:*:*", ("en", "Contoso Anything")),
Name(@"cpe:2.3:a:fabrikam:gadget\+\+:7.0:*:*:*:*:*:*:*", ("en", "Fabrikam Gadget++ 7.0")));
var reduced = await ReadAsync(chunk);
Assert.Equal(
new[] { ("contoso:widget", "Contoso Widget 4.2", true), (@"fabrikam:gadget\+\+", "Fabrikam Gadget++ 7.0", true) },
StateOf(reduced));
}
[Theory]
[InlineData("*", true)]
[InlineData("-", true)]
[InlineData("windows", true)]
// NVD names a release for some Windows software instead of the platform.
[InlineData("windows_10", true)]
[InlineData("windows_7", true)]
// The phone, embedded and ARM-only lines share the prefix and are other platforms.
[InlineData("windows_phone", false)]
[InlineData("windows_mobile", false)]
[InlineData("windows_ce", false)]
[InlineData("windows_rt", false)]
[InlineData("wordpress", false)]
[InlineData("android", false)]
[InlineData("node.js", false)]
[InlineData("iphone_os", false)]
[InlineData("macos", false)]
public async Task ReadChunk_ProductCanBeWindowsSoftware_WhenANameTargetsWindowsOrNoPlatform(string targetSw, bool expected)
{
var reduced = await ReadAsync(Chunk(Name($"cpe:2.3:a:contoso:widget:4.2:*:*:*:*:{targetSw}:*:*", ("en", "Contoso Widget 4.2"))));
Assert.Equal(expected, reduced["contoso:widget"].WindowsCapable);
}
[Fact]
public async Task ReadChunk_WindowsLinesThatAreOtherPlatforms_AreOtherPlatformsForAScanToo()
{
foreach (var targetSw in new[] { "windows_phone", "windows_mobile", "windows_ce", "windows_rt" })
{
var name = $"cpe:2.3:a:contoso:widget:4.2:*:*:*:*:{targetSw}:*:*";
var reduced = await ReadAsync(Chunk(Name(name, ("en", "Contoso Widget 4.2"))));
Assert.False(reduced["contoso:widget"].WindowsCapable);
// A product the dictionary keeps out for its platform must not have that platform's
// CVE ranges compared with a desktop install either.
Assert.True(CveVersionFilter.IsNonWindowsPlatform(name), $"A scan judges a range for {targetSw}.");
}
}
[Fact]
public async Task ReadChunk_OneNameThatCanBeWindowsSoftware_IsEnoughForTheProduct()
{
var chunk = Chunk(
Name("cpe:2.3:a:contoso:widget:1.0:*:*:*:*:android:*:*", ("en", "Contoso Widget 1.0 for Android")),
Name("cpe:2.3:a:contoso:widget:2.0:*:*:*:*:windows:*:*", ("en", "Contoso Widget 2.0 for Windows")),
Name("cpe:2.3:a:contoso:widget:3.0:*:*:*:*:iphone_os:*:*", ("en", "Contoso Widget 3.0 for iPhone")),
// A withdrawn name says nothing about the product any more.
Name("cpe:2.3:a:contoso:plugin:1.0:*:*:*:*:wordpress:*:*", ("en", "Contoso Plugin 1.0")),
Withdrawn("cpe:2.3:a:contoso:plugin:0.9:*:*:*:*:*:*:*", ("en", "Contoso Plugin 0.9")),
// A name that ends before the target field targets no platform.
Name("cpe:2.3:a:contoso:short_form", ("en", "Contoso Short Form")));
var reduced = await ReadAsync(chunk);
Assert.True(reduced["contoso:widget"].WindowsCapable);
Assert.False(reduced["contoso:plugin"].WindowsCapable);
Assert.True(reduced["contoso:short_form"].WindowsCapable);
}
[Fact]
public async Task ReadChunk_TitleIsTheShortestEnglishOneOfAllNamesOfTheProduct()
{
var chunk = Chunk(
Name("cpe:2.3:a:contoso:widget:1.0:*:*:*:*:*:*:*", ("de", "Widget"), ("en", "Contoso Widget Professional 1.0")),
Name("cpe:2.3:a:contoso:widget:2.0:*:*:*:*:*:*:*", ("en", ""), ("en", "Contoso Widget 2.0"), ("en", "Contoso Widget 2.0 (64-bit)")),
Name("cpe:2.3:a:contoso:widget:3.0:*:*:*:*:*:*:*"),
Withdrawn("cpe:2.3:a:contoso:widget:0.1:*:*:*:*:*:*:*", ("en", "CW")),
Name("cpe:2.3:a:contoso:untitled:1.0:*:*:*:*:*:*:*", ("ja", "コントソ")));
var reduced = await ReadAsync(chunk);
Assert.Equal("Contoso Widget 2.0", reduced["contoso:widget"].Title);
Assert.Equal(string.Empty, reduced["contoso:untitled"].Title);
}
[Fact]
public async Task ReadChunk_EntriesOfAnotherShape_AreIgnored()
{
var chunk = Json("""
{ "products": [
42, "text", null, true,
[ 1, [ 2, 3 ], { "cpe": { "cpeName": "cpe:2.3:a:decoy:in_an_array:1.0:*:*:*:*:*:*:*" } } ],
{ "note": "no cpe here" },
{ "cpe": "not an object" },
{ "cpe": { "cpeName": 7 } },
{ "cpe": { "cpeName": "not a cpe name" } },
{ "cpe": { "deprecated": "no", "cpeName": "cpe:2.3:a:contoso:widget:4.2:*:*:*:*:*:*:*", "titles": "none" } },
{ "cpe": { "cpeName": "cpe:2.3:a:contoso:gadget:1.0:*:*:*:*:*:*:*", "titles": [ 7, { "lang": "en" }, { "lang": 1, "title": "x" }, { "lang": "en", "title": "Contoso Gadget" } ] } }
] }
""");
var reduced = await ReadAsync(chunk);
Assert.Equal(
new[] { ("contoso:gadget", "Contoso Gadget", true), ("contoso:widget", string.Empty, true) },
StateOf(reduced));
}
[Theory]
[InlineData("""{ "format": "NVD_CPE", "products": { "cpe": { "cpeName": "cpe:2.3:a:decoy:object:1.0:*:*:*:*:*:*:*" } }, "version": "2.0" }""")]
[InlineData("""{ "products": "none", "version": "2.0" }""")]
[InlineData("""{ "products": null }""")]
[InlineData("""{ "products": 42, "after": [ { "cpe": { "cpeName": "cpe:2.3:a:decoy:after:1.0:*:*:*:*:*:*:*" } } ] }""")]
[InlineData("""{ "format": "NVD_CPE" }""")]
[InlineData("""{ "feed": { "products": [ { "cpe": { "cpeName": "cpe:2.3:a:decoy:nested:1.0:*:*:*:*:*:*:*" } } ] } }""")]
[InlineData("""[ { "products": [ { "cpe": { "cpeName": "cpe:2.3:a:decoy:in_a_root_array:1.0:*:*:*:*:*:*:*" } } ] } ]""")]
[InlineData("{}")]
public async Task ReadChunk_NoProductsArrayAtTheRoot_YieldsNothingAndDoesNotThrow(string json)
{
Assert.Empty(await ReadAsync(Json(json)));
Assert.Empty(await ReadAsync(Json(json), bufferBytes: 16));
}
// -----------------------------------------------------------------------
// ReadChunkAsync: the buffer
// -----------------------------------------------------------------------
private const int LargeChunkProducts = 480;
private const int LargeChunkLargeEntries = 3;
/// <summary>The names of one product of a stress chunk; which kind it is goes by its number.</summary>
private static IEnumerable<string> NamesOfProduct(int number)
{
var id = number.ToString("000", CultureInfo.InvariantCulture);
var product = $"cpe:2.3:a:contoso:product_{id}";
switch (number % 6)
{
case 0:
yield return Name($"{product}:1.0:*:*:*:*:*:*:*", ("en", $"Contoso Product {id}"));
break;
case 1:
yield return Name($"{product}:1.0:*:*:*:*:android:*:*", ("en", $"Contoso Product {id} for Android"));
yield return Name($"{product}:2.0:*:*:*:*:windows:*:*", ("en", $"Contoso Product {id}"));
break;
case 2:
yield return Name($"{product}:1.0:*:*:*:*:wordpress:*:*", ("en", $"Contoso Product {id} Plugin"));
break;
case 3:
yield return Withdrawn($"{product}:1.0:*:*:*:*:*:*:*", ("en", $"Contoso Product {id}"));
break;
case 4:
yield return Name($"{product}:-:*:*:*:*:-:*:*", ("ja", "コントソ製品"), ("en", $"Contoso Prödükt {id} ✓"));
break;
default:
yield return Name($"{product}:1.0:*:*:*:*:*:*:*", ("en", $"Contoso \"Product\" {id} [x64] {{beta}} \\ edition"));
break;
}
}
/// <summary>
/// A chunk that puts every part of the reader between two buffer fills at some buffer size:
/// <paramref name="products"/> products of a few hundred bytes per name, entries of several
/// kilobytes near the end, entries that are no product names, and root properties around
/// the products array that hold objects and arrays of their own; <paramref name="rootPadding"/>
/// sizes the largest of them, which at 600 is larger than any entry. What the chunk reduces
/// to is stated by <see cref="AssertStressChunk"/>.
/// </summary>
private static byte[] StressChunk(int products, int largeEntries, int rootPadding)
{
const string decoy = "{ \"cpe\": { \"deprecated\": false, \"cpeName\": \"cpe:2.3:a:decoy:{0}:1.0:*:*:*:*:*:*:*\", \"titles\": [ { \"title\": \"Decoy\", \"lang\": \"en\" } ] } }";
string Decoy(string name) => decoy.Replace("{0}", name, StringComparison.Ordinal);
var entries = new List<string>();
for (var number = 0; number < products; number++)
{
entries.AddRange(NamesOfProduct(number));
if (number % 97 == 0)
entries.AddRange(new[] { "42", "\"text\"", "null", $"[ 1, [ 2, 3 ], {Decoy("in_an_array")} ]", "{ \"note\": \"no cpe here\" }" });
}
for (var large = 0; large < largeEntries; large++)
entries.Add(Entry($"cpe:2.3:a:contoso:large_{large}:1.0:*:*:*:*:*:*:*", false, 60, new[] { ("en", $"Contoso Large {large}") }));
entries.Add(Name("cpe:2.3:a:contoso:last:1.0:*:*:*:*:*:*:*", ("en", "Contoso Last")));
var json = new StringBuilder();
json.Append("{\n \"resultsPerPage\": 500,\n \"startIndex\": 0,\n \"format\": \"NVD_CPE\",\n");
json.Append(" \"source\": { \"name\": \"synthetic\", \"products\": [ ").Append(Decoy("before")).Append(" ], \"depth\": { \"a\": [ 1, 2, { \"b\": null } ] } },\n");
json.Append(" \"ids\": [ 1, [ 2, 3 ], { \"products\": [] } ],\n");
json.Append(" \"products\": [\n ").Append(string.Join(",\n ", entries)).Append("\n ],\n");
json.Append(" \"archive\": { \"note\": ").Append(Quote(string.Concat(Enumerable.Repeat("a } ] \" \\ é ", rootPadding))));
json.Append(", \"products\": [ ").Append(Decoy("after")).Append(" ], \"list\": [ ");
json.Append(string.Join(", ", Enumerable.Range(0, rootPadding))).Append(" ] },\n");
json.Append(" \"remarks\": [ \"one\", [ \"two\", [ \"three\" ] ], { \"products\": [ ").Append(Decoy("in_remarks")).Append(" ] } ],\n");
json.Append(" \"timestamp\": \"2026-11-12T03:00:00.000\",\n \"complete\": true,\n \"next\": null\n}\n \n");
return Encoding.UTF8.GetBytes(json.ToString());
}
private static void AssertStressChunk(Dictionary<string, CpeDictionaryService.ProductState> reduced, int products, int largeEntries)
{
var expectedCount = 0;
for (var number = 0; number < products; number++)
{
var id = number.ToString("000", CultureInfo.InvariantCulture);
var found = reduced.TryGetValue($"contoso:product_{id}", out var state);
(string Title, bool WindowsCapable)? expected = (number % 6) switch
{
0 or 1 => ($"Contoso Product {id}", true),
2 => ($"Contoso Product {id} Plugin", false),
3 => null,
4 => ($"Contoso Prödükt {id} ✓", true),
_ => ($"Contoso \"Product\" {id} [x64] {{beta}} \\ edition", true),
};
Assert.Equal(expected != null, found);
if (expected == null) continue;
Assert.Equal(expected.Value, (state!.Title, state.WindowsCapable));
expectedCount++;
}
for (var large = 0; large < largeEntries; large++)
{
var state = reduced[$"contoso:large_{large}"];
Assert.Equal(($"Contoso Large {large}", true), (state.Title, state.WindowsCapable));
}
Assert.Equal(("Contoso Last", true), (reduced["contoso:last"].Title, reduced["contoso:last"].WindowsCapable));
// Nothing but the root's own products array was read.
Assert.Equal(expectedCount + largeEntries + 1, reduced.Count);
Assert.DoesNotContain(reduced.Keys, key => key.StartsWith("decoy:", StringComparison.Ordinal));
}
private static readonly Lazy<byte[]> LargeChunk = new(() => StressChunk(LargeChunkProducts, LargeChunkLargeEntries, rootPadding: 600));
[Fact]
public async Task ReadChunk_LargeChunk_IsReducedToItsProducts()
{
var reduced = await ReadAsync(LargeChunk.Value);
AssertStressChunk(reduced, LargeChunkProducts, LargeChunkLargeEntries);
Assert.Equal(LargeChunkProducts / 6 * 5 + LargeChunkLargeEntries + 1, reduced.Count);
}
[Theory]
[InlineData(16)]
[InlineData(17)]
[InlineData(64)]
[InlineData(100)]
[InlineData(1000)]
public async Task ReadChunk_BufferFarSmallerThanTheChunk_YieldsExactlyWhatTheDefaultBufferYields(int bufferBytes)
{
var withDefaultBuffer = await ReadAsync(LargeChunk.Value);
var probe = new ProbeStream(LargeChunk.Value);
var withSmallBuffer = await ReadAsync(probe, bufferBytes);
Assert.Equal(StateOf(withDefaultBuffer), StateOf(withSmallBuffer));
AssertStressChunk(withSmallBuffer, LargeChunkProducts, LargeChunkLargeEntries);
// The comparison only says something when the buffer was filled many times over and had
// to grow for an entry that did not fit; a pool that hands out one large array would
// make it pass for nothing.
Assert.InRange(probe.FirstCapacity, bufferBytes, 4096);
Assert.True(probe.Reads >= 50, $"only {probe.Reads} reads");
Assert.True(probe.LargestCapacity > probe.FirstCapacity, "the buffer never grew");
Assert.True(probe.LargestCapacity < LargeChunk.Value.Length / 8, "the buffer came to hold a large part of the chunk");
}
[Theory]
[InlineData(13)]
[InlineData(4099)]
public async Task ReadChunk_StreamThatHandsOutLessThanAskedFor_YieldsTheSame(int bytesPerRead)
{
var whole = await ReadAsync(LargeChunk.Value);
var inPieces = await ReadAsync(new ProbeStream(LargeChunk.Value, bytesPerRead));
Assert.Equal(StateOf(whole), StateOf(inPieces));
}
[Theory]
[InlineData(null)]
[InlineData(16)]
public async Task ReadChunk_StreamThatHandsOutOneByteAtATime_CutsEveryTokenEverywhere_AndYieldsTheSame(int? bufferBytes)
{
// Small enough to be read byte by byte, with every kind of entry and one large one in it.
const int products = 36, largeEntries = 1;
var chunk = StressChunk(products, largeEntries, rootPadding: 30);
var probe = new ProbeStream(chunk, mostPerRead: 1);
var byteByByte = await ReadAsync(probe, bufferBytes);
AssertStressChunk(byteByByte, products, largeEntries);
Assert.Equal(StateOf(await ReadAsync(chunk)), StateOf(byteByByte));
Assert.True(probe.Reads > chunk.Length, "the chunk was not read byte by byte");
}
[Theory]
[InlineData(0.0)]
[InlineData(0.001)]
[InlineData(0.3)]
[InlineData(0.7)]
[InlineData(0.98)]
public async Task ReadChunk_ChunkThatEndsBeforeItsEnd_IsAJsonError_AtEveryBufferSize(double share)
{
var whole = Chunk(Enumerable.Range(0, 30).SelectMany(NamesOfProduct).ToArray());
var cut = whole[..(int)(whole.Length * share)];
await Assert.ThrowsAnyAsync<JsonException>(() => ReadAsync(cut));
await Assert.ThrowsAnyAsync<JsonException>(() => ReadAsync(cut, bufferBytes: 16));
await Assert.ThrowsAnyAsync<JsonException>(() => ReadAsync(cut, bufferBytes: 1000));
}
[Fact]
public async Task ReadChunk_ChunkWithoutItsClosingBrace_IsAJsonError()
{
var whole = Encoding.UTF8.GetString(Chunk(Name("cpe:2.3:a:contoso:widget:4.2:*:*:*:*:*:*:*", ("en", "Contoso Widget 4.2"))));
var cut = Json(whole[..whole.LastIndexOf('}')]);
await Assert.ThrowsAnyAsync<JsonException>(() => ReadAsync(cut));
await Assert.ThrowsAnyAsync<JsonException>(() => ReadAsync(cut, bufferBytes: 16));
}
/// <summary>A chunk whose first entry never ends: its bytes are produced as they are asked for.</summary>
private sealed class EndlessEntryStream : Stream
{
private static readonly byte[] Start = Encoding.UTF8.GetBytes("{ \"products\": [ { \"cpe\": { \"cpeName\": \"");
public long BytesRead { get; private set; }
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position { get => BytesRead; set => throw new NotSupportedException(); }
public override int Read(Span<byte> buffer)
{
var fromStart = (int)Math.Min(Math.Max(Start.Length - BytesRead, 0), buffer.Length);
Start.AsSpan((int)Math.Min(BytesRead, Start.Length), fromStart).CopyTo(buffer);
buffer[fromStart..].Fill((byte)'a');
BytesRead += buffer.Length;
return buffer.Length;
}
public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default) => new(Read(buffer.Span));
public override void Flush() { }
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
}
[Fact]
public async Task ReadChunk_EntryThatNeverEnds_IsRefusedOnceTheBufferReachedItsLimit()
{
var endless = new EndlessEntryStream();
await Assert.ThrowsAsync<InvalidDataException>(() => ReadAsync(endless));
// The buffer grew to its limit and no further: the file is given up, not loaded.
Assert.InRange(endless.BytesRead, CpeDictionaryService.MaxEntryBytes, 2L * CpeDictionaryService.MaxEntryBytes);
}
// -----------------------------------------------------------------------
// ReduceAsync
// -----------------------------------------------------------------------
/// <summary>The feed archive: a gzipped tar of the given files; an entry without content is a directory.</summary>
private static byte[] Archive(params (string Name, byte[]? Content)[] entries)
{
using var archive = new MemoryStream();
using (var gzip = new GZipStream(archive, CompressionLevel.Fastest, leaveOpen: true))
using (var tar = new TarWriter(gzip, TarEntryFormat.Ustar, leaveOpen: true))
{
foreach (var (name, content) in entries)
{
if (content == null)
tar.WriteEntry(new UstarTarEntry(TarEntryType.Directory, name));
else
tar.WriteEntry(new UstarTarEntry(TarEntryType.RegularFile, name) { DataStream = new MemoryStream(content) });
}
}
return archive.ToArray();
}
private static readonly byte[] SmallArchive = Archive(
("nvdcpe-2.0-chunks/", null),
("nvdcpe-2.0-chunks/README.txt", Json("{ \"products\": [ { \"cpe\": { \"cpeName\": \"cpe:2.3:a:decoy:not_a_chunk:1.0:*:*:*:*:*:*:*\" } } ] }")),
("nvdcpe-2.0-chunks/nvdcpe-2.0-chunk-00001.json", Chunk(
Name("cpe:2.3:a:contoso:widget:1.0:*:*:*:*:android:*:*", ("en", "Contoso Widget")),
Name("cpe:2.3:a:contoso:plugin:1.0:*:*:*:*:wordpress:*:*", ("en", "Contoso Plugin 1.0")),
Name("cpe:2.3:a:fabrikam:gadget:7.0:*:*:*:*:*:*:*", ("en", "Fabrikam Gadget 7.0")))),
("nvdcpe-2.0-chunks/nvdcpe-2.0-chunk-00002.JSON", Chunk(
// The second file completes a product the first one began.
Name("cpe:2.3:a:contoso:widget:2.0:*:*:*:*:windows:*:*", ("en", "Contoso Widget 2.0 for Windows")),
Name("cpe:2.3:o:contoso:widget_os:1.0:*:*:*:*:*:*:*", ("en", "Contoso Widget OS")),
Name("cpe:2.3:a:northwind:tool:3.1:*:*:*:*:-:*:*", ("en", "Northwind Tool 3.1")))));
[Fact]
public async Task Reduce_IsTheUnionOfAllChunkFiles_WithOnlyTheProductsThatCanBeWindowsSoftware()
{
var products = await CpeDictionaryService.ReduceAsync(new MemoryStream(SmallArchive), CancellationToken.None);
Assert.Equal(
new[]
{
("contoso:widget", "Contoso Widget"),
("fabrikam:gadget", "Fabrikam Gadget 7.0"),
("northwind:tool", "Northwind Tool 3.1"),
},
products.OrderBy(p => p.Key, StringComparer.Ordinal).Select(p => (p.Key, p.Value.T)));
}
[Fact]
public async Task Reduce_ArchiveWithoutChunkFiles_IsEmpty()
{
var archive = Archive(("nvdcpe-2.0-chunks/", null), ("nvdcpe-2.0-chunks/README.txt", Json("nothing to read")));
Assert.Empty(await CpeDictionaryService.ReduceAsync(new MemoryStream(archive), CancellationToken.None));
}
// -----------------------------------------------------------------------
// ParseFeedVersion
// -----------------------------------------------------------------------
[Theory]
[InlineData("lastModifiedDate:2026-11-12T03:00:01-05:00\r\nsize:913338293\r\ngzSize:85733215\r\nsha256:0123abcdef\r\n", "0123ABCDEF")]
[InlineData("sha256:0123ABCDEF", "0123ABCDEF")]
[InlineData("size:1\n SHA256: 0123abcdef \nsize:2", "0123ABCDEF")]
[InlineData("sha256:first\nsha256:second", "FIRST")]
public void ParseFeedVersion_IsTheHashLineInUpperCase(string meta, string expected) =>
Assert.Equal(expected, CpeDictionaryService.ParseFeedVersion(meta));
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData("lastModifiedDate:2026-11-12T03:00:01-05:00\r\nsize:913338293\r\n")]
[InlineData("sha256:\r\nsize:1")]
[InlineData("sha256: ")]
[InlineData("<html>maintenance</html>")]
public void ParseFeedVersion_WithoutAHash_IsNull(string? meta) =>
Assert.Null(CpeDictionaryService.ParseFeedVersion(meta));
// -----------------------------------------------------------------------
// RefreshAsync and the stored copy
// -----------------------------------------------------------------------
/// <summary>Stands in for the NVD feed directory.</summary>
private sealed class FeedStub : HttpMessageHandler
{
public Func<HttpResponseMessage> Meta { get; set; } = () => Text($"lastModifiedDate:2026-11-12T03:00:01-05:00\r\nsha256:{FeedVersion.ToLowerInvariant()}\r\n");
public Func<HttpResponseMessage> Archive { get; set; } = () => new HttpResponseMessage(HttpStatusCode.NotFound);
public List<string> Requests { get; } = new();
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var url = request.RequestUri!.AbsoluteUri;
Requests.Add(url);
return Task.FromResult(url switch
{
CpeDictionaryService.MetaUrl => Meta(),
CpeDictionaryService.FeedUrl => Archive(),
_ => new HttpResponseMessage(HttpStatusCode.NotFound),
});
}
}
private static HttpResponseMessage Text(string text) => new(HttpStatusCode.OK) { Content = new StringContent(text) };
private static HttpResponseMessage Bytes(byte[] bytes) => new(HttpStatusCode.OK) { Content = new ByteArrayContent(bytes) };
private static IEnumerable<(string ProductKey, string Title)> Products(int count) => Enumerable.Range(0, count)
.Select(n => ($"contoso:product_{n:00000}", $"Contoso Product {n:00000}"));
private const int FullSize = CpeDictionaryService.MinProducts + 5;
/// <summary>An archive that reduces to a dictionary of full size, in two chunk files.</summary>
private static readonly Lazy<byte[]> FullSizeArchive = new(() =>
{
byte[] ChunkOf(int from, int to) => Chunk(Enumerable.Range(from, to - from)
.Select(n => Entry($"cpe:2.3:a:contoso:product_{n:00000}:1.0:*:*:*:*:*:*:*", false, 0, new[] { ("en", $"Contoso Product {n:00000}") }))
.ToArray());
return Archive(
("nvdcpe-2.0-chunks/nvdcpe-2.0-chunk-00001.json", ChunkOf(0, FullSize / 2)),
("nvdcpe-2.0-chunks/nvdcpe-2.0-chunk-00002.json", ChunkOf(FullSize / 2, FullSize)));
});
private static BlobBackedReferenceCache<CpeDictionaryProduct>.Snapshot SnapshotIn(StoredBlob blob) =>
JsonSerializer.Deserialize<BlobBackedReferenceCache<CpeDictionaryProduct>.Snapshot>(blob.Content!)!;
[Fact]
public async Task Refresh_NewFeed_StoresTheReducedCopy_ServesIt_AndFindsNothingNewTheNextTime()
{
var blob = new StoredBlob();
var feed = new FeedStub { Archive = () => Bytes(FullSizeArchive.Value) };
var service = CpeDictionaryFixture.Service(feed, blob);
var before = DateTime.UtcNow;
var first = await service.RefreshAsync();
Assert.Null(first.Error);
Assert.True(first.Changed);
Assert.Equal(FullSize, first.Products);
Assert.Equal(FeedVersion, first.FeedVersion);
Assert.Equal(new[] { CpeDictionaryService.MetaUrl, CpeDictionaryService.FeedUrl }, feed.Requests);
Assert.Equal(1, blob.Uploads);
var stored = SnapshotIn(blob);
Assert.Equal(new[] { FeedVersion }, stored.CoveredDocuments);
Assert.Equal(FullSize, stored.Entries.Count);
Assert.Equal("Contoso Product 00042", stored.Entries["contoso:product_00042"].T);
// The new copy is in service without being read back from the blob.
var dictionary = await service.GetAsync();
Assert.Equal(FullSize, dictionary!.Count);
Assert.Equal("cpe:2.3:a:contoso:product_00042", dictionary.Resolve("product 00042", "contoso", null)!.Value.CpeUri);
Assert.Equal(0, blob.Downloads);
Assert.InRange((await service.LastRefreshAsync())!.Value, before, DateTime.UtcNow);
var second = await service.RefreshAsync();
Assert.Null(second.Error);
Assert.False(second.Changed);
Assert.Equal(FullSize, second.Products);
Assert.Equal(FeedVersion, second.FeedVersion);
Assert.Equal(new[] { CpeDictionaryService.MetaUrl, CpeDictionaryService.FeedUrl, CpeDictionaryService.MetaUrl }, feed.Requests);
Assert.Equal(1, blob.Uploads);
}
[Fact]
public async Task Refresh_FeedUnchanged_AndAStoredCopyOfFullSize_DownloadsNoArchive()
{
var blob = new StoredBlob(CpeDictionaryFixture.SnapshotJson(Products(CpeDictionaryService.MinProducts), FeedVersion));
var feed = new FeedStub { Archive = () => Bytes(FullSizeArchive.Value) };
var result = await CpeDictionaryFixture.Service(feed, blob).RefreshAsync();
Assert.Null(result.Error);
Assert.False(result.Changed);
Assert.Equal(CpeDictionaryService.MinProducts, result.Products);
Assert.Equal(FeedVersion, result.FeedVersion);
Assert.Equal(new[] { CpeDictionaryService.MetaUrl }, feed.Requests);
Assert.Equal(0, blob.Uploads);
}
[Theory]
[InlineData("another feed version")]
[InlineData("a stored copy that is too small")]
public async Task Refresh_StoredCopyThatIsNotTheCurrentFeedInFull_IsReplaced(string reason)
{
var stored = reason == "another feed version"
? CpeDictionaryFixture.SnapshotJson(Products(CpeDictionaryService.MinProducts), "AN-EARLIER-FEED")
: CpeDictionaryFixture.SnapshotJson(Products(CpeDictionaryService.MinProducts - 1), FeedVersion);
var blob = new StoredBlob(stored);
var feed = new FeedStub { Archive = () => Bytes(FullSizeArchive.Value) };
var result = await CpeDictionaryFixture.Service(feed, blob).RefreshAsync();
Assert.Null(result.Error);
Assert.True(result.Changed);
Assert.Equal(FullSize, result.Products);
Assert.Equal(1, blob.Uploads);
Assert.Equal(new[] { FeedVersion }, SnapshotIn(blob).CoveredDocuments);
}
[Fact]
public async Task Refresh_ArchiveThatReducesToTooFewProducts_ReportsIt_AndKeepsTheStoredCopy()
{
var earlier = CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:earlier", "Contoso Earlier") }, "AN-EARLIER-FEED");
var blob = new StoredBlob(earlier);
var feed = new FeedStub { Archive = () => Bytes(SmallArchive) };
var service = CpeDictionaryFixture.Service(feed, blob);
var result = await service.RefreshAsync();
Assert.False(result.Changed);
Assert.Contains("only 3 products", result.Error);
Assert.Equal(FeedVersion, result.FeedVersion);
Assert.Equal(0, blob.Uploads);
Assert.Equal(earlier, blob.Content);
Assert.NotNull((await service.GetAsync())!.Resolve("earlier", "contoso", null));
}
[Theory]
[InlineData("the version file is not served")]
[InlineData("the connection breaks")]
[InlineData("the archive is not served")]
[InlineData("the archive is no archive")]
[InlineData("the archive is cut off")]
public async Task Refresh_FailureOnTheWay_IsReportedInTheResult_NeverThrown(string failure)
{
var blob = new StoredBlob();
var feed = new FeedStub();
switch (failure)
{
case "the version file is not served": feed.Meta = () => new HttpResponseMessage(HttpStatusCode.InternalServerError); break;
case "the connection breaks": feed.Meta = () => throw new HttpRequestException("connection reset"); break;
case "the archive is not served": feed.Archive = () => new HttpResponseMessage(HttpStatusCode.BadGateway); break;
case "the archive is no archive": feed.Archive = () => Text("<html>maintenance</html>"); break;
default: feed.Archive = () => Bytes(FullSizeArchive.Value[..(FullSizeArchive.Value.Length / 2)]); break;
}
var service = CpeDictionaryFixture.Service(feed, blob);
var result = await service.RefreshAsync();
Assert.False(string.IsNullOrEmpty(result.Error));
Assert.False(result.Changed);
Assert.Equal(0, blob.Uploads);
Assert.Null(await service.GetAsync());
}
[Fact]
public async Task Get_NothingStored_IsNull_AndAStoredDictionaryIsLoadedOnce()
{
var nothingStored = CpeDictionaryFixture.Stored();
var blob = new StoredBlob(CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:widget", "Contoso Widget") }));
var stored = CpeDictionaryFixture.Service(new FeedStub(), blob);
Assert.Null(await nothingStored.GetAsync());
Assert.Null(await nothingStored.LastRefreshAsync());
var first = await stored.GetAsync();
var second = await stored.GetAsync();
Assert.Same(first, second);
Assert.Equal(1, blob.Downloads);
Assert.Equal("Contoso Widget", first!.Resolve("widget", "contoso", null)!.Value.Title);
Assert.Equal(new DateTime(2026, 11, 11, 3, 0, 0, DateTimeKind.Utc), await stored.LastRefreshAsync());
}
[Fact]
public async Task Get_StoredSnapshotWithoutProducts_CountsAsNoDictionary()
{
var service = CpeDictionaryFixture.Service(
new FeedStub(), new StoredBlob(CpeDictionaryFixture.SnapshotJson(Array.Empty<(string, string)>(), FeedVersion)));
Assert.Null(await service.GetAsync());
}
[Fact]
public async Task Get_InstanceThatFoundNoDictionary_FindsTheOneStoredLater()
{
var blob = new StoredBlob();
var service = CpeDictionaryFixture.Service(new FeedStub(), blob);
Assert.Null(await service.GetAsync());
// Another instance's nightly run stored the first dictionary.
blob.Content = CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:widget", "Contoso Widget") });
Assert.NotNull((await service.GetAsync())!.Resolve("widget", "contoso", null));
}
/// <summary>
/// Makes the copy an instance holds look as if it had been loaded <paramref name="age"/> ago.
/// The hours cannot be waited for, and the service reads the clock itself.
/// </summary>
private static void BackdateLoadedCopy(CpeDictionaryService service, TimeSpan age)
{
var loadedAt = typeof(CpeDictionaryService).GetField("_loadedAtUtc", BindingFlags.Instance | BindingFlags.NonPublic);
Assert.NotNull(loadedAt);
loadedAt!.SetValue(service, DateTime.UtcNow - age);
}
[Fact]
public async Task Get_CopyLoadedLongAgo_LooksAtTheBlobAgain_AndServesWhatAnotherInstanceStoredSince()
{
var blob = new StoredBlob(CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:widget", "Contoso Widget") }));
var service = CpeDictionaryFixture.Service(new FeedStub(), blob);
var first = await service.GetAsync();
// Only the instance whose nightly run downloaded the feed holds the new dictionary by itself.
blob.Content = CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:widget", "Contoso Widget"), ("fabrikam:gadget", "Fabrikam Gadget") });
BackdateLoadedCopy(service, CpeDictionaryService.LoadedCopyMaxAge - TimeSpan.FromMinutes(5));
Assert.Same(first, await service.GetAsync());
Assert.Equal(1, blob.Downloads);
BackdateLoadedCopy(service, CpeDictionaryService.LoadedCopyMaxAge + TimeSpan.FromMinutes(5));
var second = await service.GetAsync();
Assert.Equal(2, blob.Downloads);
Assert.NotNull(second!.Resolve("gadget", "fabrikam", null));
// The copy just read is held again for its own time.
Assert.Same(second, await service.GetAsync());
Assert.Equal(2, blob.Downloads);
}
[Theory]
[InlineData("the blob is gone")]
[InlineData("the blob cannot be parsed")]
[InlineData("the blob holds no product")]
public async Task Get_CopyLoadedLongAgo_AndNoUsableBlob_KeepsServingTheCopyItHolds(string state)
{
var blob = new StoredBlob(CpeDictionaryFixture.SnapshotJson(new[] { ("contoso:widget", "Contoso Widget") }));
var service = CpeDictionaryFixture.Service(new FeedStub(), blob);
var held = await service.GetAsync();
blob.Content = state switch
{
"the blob is gone" => null,
"the blob cannot be parsed" => "{ \"entries\": ",
_ => CpeDictionaryFixture.SnapshotJson(Array.Empty<(string, string)>(), FeedVersion),
};
BackdateLoadedCopy(service, CpeDictionaryService.LoadedCopyMaxAge + TimeSpan.FromMinutes(5));
var served = await service.GetAsync();
Assert.Same(held, served);
}
}
@@ -0,0 +1,135 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeDictionary"/> answers one question: is this program name an NVD product name?
/// A name resolves only when its words are the product's words and the vendor agrees, or when
/// they are the vendor's words followed by the product's, and only when exactly one product fits.
/// There is no similarity search: a near miss stays unresolved.
/// </summary>
public class CpeDictionaryTests
{
private static CpeDictionary Dictionary(params string[] productKeys) => new(
productKeys.ToDictionary(key => key, key => new CpeDictionaryProduct { T = $"Title of {key}" }, StringComparer.Ordinal),
"2026-11-11T03:00:00.0000000Z");
private static string? Resolved(CpeDictionary dictionary, string name, string? vendor = null, string? publisher = null) =>
dictionary.Resolve(name, vendor, publisher)?.CpeUri;
[Theory]
[InlineData("contoso", null)]
[InlineData(null, "Contoso")]
[InlineData("contoso inc", "")]
[InlineData("", "Contoso Ltd.")]
[InlineData("fabrikam", "The Contoso Company")] // either of the program's two vendor forms may agree
[InlineData("contoso software", null)] // a trailing word after the vendor's name
public void Resolve_NameThatIsTheProductName_ResolvesWhenTheProgramsVendorAgrees(string? vendor, string? publisher)
{
var match = Dictionary("contoso:widget").Resolve("widget", vendor, publisher);
Assert.NotNull(match);
Assert.Equal("contoso", match!.Value.Vendor);
Assert.Equal("widget", match.Value.Product);
Assert.Equal("cpe:2.3:a:contoso:widget", match.Value.CpeUri);
Assert.Equal("Title of contoso:widget", match.Value.Title);
}
[Theory]
[InlineData("fabrikam", "Fabrikam Inc")]
[InlineData("contosoft", null)]
[InlineData(null, null)]
[InlineData("", "")]
[InlineData("inc", "Ltd")] // nothing is left of a vendor that is only a legal form
public void Resolve_NameThatIsTheProductName_DoesNotResolveWithoutAnAgreeingVendor(string? vendor, string? publisher) =>
Assert.Null(Dictionary("contoso:widget").Resolve("widget", vendor, publisher));
[Fact]
public void Resolve_SameProductNameUnderTwoVendors_TakesTheOneWhoseVendorAgrees()
{
var dictionary = Dictionary("contoso:reader", "fabrikam:reader");
Assert.Equal("cpe:2.3:a:fabrikam:reader", Resolved(dictionary, "reader", "fabrikam"));
Assert.Equal("cpe:2.3:a:contoso:reader", Resolved(dictionary, "reader", publisher: "Contoso Ltd"));
Assert.Null(Resolved(dictionary, "reader"));
// Vendor and publisher each name one of the two: the name is not decided.
Assert.Null(Resolved(dictionary, "reader", "contoso", "Fabrikam Inc"));
}
[Fact]
public void Resolve_VendorWordsFollowedByProductWords_ResolveWithoutAVendor()
{
var dictionary = Dictionary("contoso:widget", "northwind_traders:tool_box");
Assert.Equal("cpe:2.3:a:contoso:widget", Resolved(dictionary, "contoso widget"));
Assert.Equal("cpe:2.3:a:northwind_traders:tool_box", Resolved(dictionary, "northwind traders tool box"));
// The product's words first is another name.
Assert.Null(Resolved(dictionary, "widget contoso"));
}
[Fact]
public void Resolve_TwoProductsThatFit_ResolveToNothing()
{
var sameWordsSplitDifferently = Dictionary("contoso:widget_pro", "contoso_widget:pro");
var productNameAndVendorPlusProduct = Dictionary("adatum:contoso_widget", "contoso:widget");
Assert.Null(Resolved(sameWordsSplitDifferently, "contoso widget pro"));
// Adatum's "contoso widget" and Contoso's "widget" both read "contoso widget".
Assert.Null(Resolved(productNameAndVendorPlusProduct, "contoso widget", "adatum"));
Assert.Equal("cpe:2.3:a:contoso:widget", Resolved(productNameAndVendorPlusProduct, "contoso widget"));
}
[Theory]
[InlineData("widget", "contoso")] // one word short
[InlineData("widget pro max", "contoso")] // one word more
[InlineData("pro widget", "contoso")] // other order
[InlineData("contoso widget", null)]
[InlineData("contoso widget pro 2", null)]
[InlineData("contoso widgets pro", null)] // nearly the same word
public void Resolve_NameWithAWordMoreOrLess_DoesNotResolve(string name, string? vendor)
{
var dictionary = Dictionary("contoso:widget_pro");
Assert.Null(Resolved(dictionary, name, vendor));
Assert.Equal("cpe:2.3:a:contoso:widget_pro", Resolved(dictionary, "widget pro", "contoso"));
Assert.Equal("cpe:2.3:a:contoso:widget_pro", Resolved(dictionary, "contoso widget pro"));
}
[Fact]
public void Resolve_CpeEscapesAndUnderscores_AreReadAsTheWordsTheyStandFor()
{
var dictionary = Dictionary(@"fabrikam:notepad\+\+", "contoso:visual_studio_code", "contoso:node.js", "northwind_traders:x-ray");
var escaped = dictionary.Resolve("notepad++", "fabrikam", null);
Assert.Equal(@"notepad\+\+", escaped!.Value.Product);
Assert.Equal(@"cpe:2.3:a:fabrikam:notepad\+\+", escaped.Value.CpeUri);
Assert.Equal(@"cpe:2.3:a:fabrikam:notepad\+\+", Resolved(dictionary, "fabrikam notepad++"));
// The plus signs are part of the word: "notepad" alone is another name.
Assert.Null(Resolved(dictionary, "notepad", "fabrikam"));
Assert.Equal("cpe:2.3:a:contoso:visual_studio_code", Resolved(dictionary, "visual studio code", "contoso"));
Assert.Equal("cpe:2.3:a:contoso:visual_studio_code", Resolved(dictionary, "Contoso Visual Studio Code"));
Assert.Null(Resolved(dictionary, "visual studio", "contoso"));
Assert.Equal("cpe:2.3:a:contoso:node.js", Resolved(dictionary, "node.js", "contoso"));
Assert.Equal("cpe:2.3:a:northwind_traders:x-ray", Resolved(dictionary, "X-Ray", publisher: "Northwind Traders Ltd"));
}
[Theory]
[InlineData("")]
[InlineData(" ")]
[InlineData("--")]
public void Resolve_NameWithoutAWord_IsNothing(string name) =>
Assert.Null(Dictionary("contoso:widget").Resolve(name, "contoso", "Contoso Ltd"));
[Fact]
public void Entries_WithoutAVendorOrAProduct_AreNeverResolved_AndDoNotGetInTheWayOfTheOthers()
{
var dictionary = Dictionary("widget", ":widget", "contoso:", "contoso:-", "contoso:reader");
Assert.Equal("2026-11-11T03:00:00.0000000Z", dictionary.LastRefresh);
Assert.Null(Resolved(dictionary, "widget", "contoso"));
Assert.Null(Resolved(dictionary, "contoso"));
Assert.Equal("cpe:2.3:a:contoso:reader", Resolved(dictionary, "reader", "contoso"));
}
}
@@ -0,0 +1,201 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Operator tool, not a check: runs an exported software inventory through the production
/// matcher and, when NVD answers are at hand, through the production projection, budget and
/// version filter. A change to the mapping list can so be compared with what is in service
/// before it ships, and a product where nearly every install has a finding can be looked at by hand.
/// <code>AM_CPE_DIFF_DIR=&lt;dir&gt; dotnet test --filter CpeMappingExportMatchTests</code>
/// reads <c>inventory.json</c> (an array of SoftwareInventory rows) from the directory and works
/// once per mapping set: every <c>mappings*.json</c> is an array of mapping table rows, every
/// <c>seed*.json</c> a mapping list file as the rules repository holds it. The rows of an optional
/// <c>next-to-seed.json</c> (table rows, an operator's own mappings for example) join every list
/// file. Per set it writes <c>matches-&lt;file&gt;.json</c> and, when the directory has an
/// <c>nvd</c> folder, <c>scan-&lt;file&gt;.json</c>; both hold one entry per inventory row, in the
/// same order. A file in <c>nvd</c> is one NVD 2.0 answer for a product, named like its CPE in
/// lower case with every character outside <c>a-z 0-9 . _ + -</c> replaced by an underscore.
/// Without the variable the test does nothing. An export holds tenant data: the directory lies
/// outside the repository.
/// </summary>
public class CpeMappingExportMatchTests
{
private const string DirectoryVariable = "AM_CPE_DIFF_DIR";
private sealed record MappingSet(string Name, List<Dictionary<string, object>> Rows, HashSet<string> SelfUpdating);
[Fact]
public void Map_an_exported_inventory()
{
var directory = Environment.GetEnvironmentVariable(DirectoryVariable);
if (string.IsNullOrWhiteSpace(directory)) return;
var inventory = ReadRows(Path.Combine(directory, "inventory.json"));
var nextToSeedFile = Path.Combine(directory, "next-to-seed.json");
var nextToSeed = File.Exists(nextToSeedFile) ? ReadRows(nextToSeedFile) : new List<Dictionary<string, object>>();
var nvdDirectory = Path.Combine(directory, "nvd");
var sets = new List<MappingSet>();
foreach (var file in Directory.GetFiles(directory, "mappings*.json"))
sets.Add(new MappingSet(Path.GetFileNameWithoutExtension(file), ReadRows(file), new HashSet<string>(StringComparer.Ordinal)));
foreach (var file in Directory.GetFiles(directory, "seed*.json"))
sets.Add(SeedSet(file, nextToSeed));
Assert.NotEmpty(sets);
foreach (var set in sets)
{
var mappings = set.Rows
.Select(CpeMappingStore.MappingFromRow)
.Where(m => m != null)
.Select(m => m!)
.OrderBy(m => m.PartitionKey, StringComparer.Ordinal)
.ThenBy(m => m.RowKey, StringComparer.Ordinal)
.ToList();
var matcher = new CpeMappingMatcher(mappings);
var scan = Directory.Exists(nvdDirectory) ? new ExportScan(nvdDirectory, set.SelfUpdating) : null;
var matches = new List<Dictionary<string, object?>>(inventory.Count);
var findings = new List<Dictionary<string, object?>>(inventory.Count);
foreach (var row in inventory)
{
var vendor = Text(row, "NormalizedVendor");
var name = Text(row, "NormalizedName");
var display = Text(row, "DisplayName");
var shown = display.Length > 0 ? display : name;
// A stored key is canonicalized again on load, so a second pass must not move it.
var canonical = SoftwareNameCanonicalizer.Canonicalize(shown);
Assert.Equal(canonical, SoftwareNameCanonicalizer.Canonicalize(canonical));
var match = matcher.Match(vendor, name, display, Text(row, "Publisher"));
matches.Add(new Dictionary<string, object?>
{
["key"] = CpeMappingMatcher.SoftwareKey(vendor, shown),
["cpeUri"] = match?.Mapping.CpeUri,
["edition"] = match?.Mapping.Edition,
["confidence"] = match?.Confidence,
["source"] = match?.Mapping.Source.ToString(),
["mapping"] = match?.Mapping.RowKey,
});
if (scan != null)
findings.Add(scan.Evaluate(match?.Mapping, Text(row, "NormalizedVersion"), Text(row, "RegistrySource")));
}
File.WriteAllText(Path.Combine(directory, $"matches-{set.Name}.json"), JsonSerializer.Serialize(matches));
if (scan != null)
File.WriteAllText(Path.Combine(directory, $"scan-{set.Name}.json"), JsonSerializer.Serialize(findings));
}
}
/// <summary>What the scan would report for one program version, from NVD answers on disk.</summary>
private sealed class ExportScan
{
private readonly string _nvdDirectory;
private readonly HashSet<string> _selfUpdating;
private readonly Dictionary<string, List<CachedCve>?> _cves = new(StringComparer.Ordinal);
public ExportScan(string nvdDirectory, HashSet<string> selfUpdating)
{
_nvdDirectory = nvdDirectory;
_selfUpdating = selfUpdating;
}
public Dictionary<string, object?> Evaluate(CpeMapping? mapping, string version, string registrySource)
{
var entry = new Dictionary<string, object?> { ["product"] = mapping?.ProductKey };
if (mapping == null) return entry;
entry["selfUpdating"] = _selfUpdating.Contains(mapping.ProductKey);
var cves = CvesOf(mapping.CpeUri);
if (cves == null)
{
entry["state"] = "no_data";
return entry;
}
if (VulnerabilityCorrelationService.IsPlaceholderVersion(registrySource, version))
{
entry["state"] = "placeholder_version";
return entry;
}
if (!VulnerabilityCorrelationService.HasVersionNumber(version))
{
entry["state"] = "no_version";
return entry;
}
var filtered = CveVersionFilter.Filter(cves, mapping.CpeUri, version, mapping.Edition, mapping.ExcludeEditions);
var exploited = new HashSet<string>(cves.Where(c => c.KevAdded != null).Select(c => c.CveId), StringComparer.OrdinalIgnoreCase);
var found = filtered.Vulnerabilities;
entry["state"] = found.Count > 0 ? "findings" : filtered.IncomparableCves > 0 ? "version_scheme" : "clean";
entry["cached"] = cves.Count;
entry["findings"] = found.Count;
entry["critical"] = found.Count(v => string.Equals(v.GetValueOrDefault("cvssSeverity")?.ToString(), "CRITICAL", StringComparison.OrdinalIgnoreCase));
entry["kev"] = found.Count(v => exploited.Contains(v.GetValueOrDefault("cveId")?.ToString() ?? string.Empty));
entry["everyVersion"] = found.Count(v => v.ContainsKey("versionMatchConfidence"));
entry["incomparable"] = filtered.IncomparableCves;
return entry;
}
private List<CachedCve>? CvesOf(string cpeUri)
{
if (_cves.TryGetValue(cpeUri, out var held)) return held;
var file = Path.Combine(_nvdDirectory, Regex.Replace(cpeUri.ToLowerInvariant(), "[^a-z0-9._+-]", "_") + ".json");
List<CachedCve>? cves = null;
if (File.Exists(file))
{
var answer = JsonSerializer.Deserialize<NvdCveResponse>(File.ReadAllText(file), CveCacheFiller.JsonOptions);
cves = CveCacheBudget.Apply(CveCacheFiller.Project(answer, cpeUri), DateTime.UtcNow).Cves;
}
_cves[cpeUri] = cves;
return cves;
}
}
/// <summary>The table rows a list file becomes on import; a file the import would refuse fails here.</summary>
private static MappingSet SeedSet(string path, List<Dictionary<string, object>> nextToSeed)
{
var (document, problems) = CpeSeedFile.Read(File.ReadAllText(path), mappingsInService: 0);
Assert.True(problems.Count == 0, $"{Path.GetFileName(path)}: {string.Join("; ", problems.Take(20))}");
var rows = CpeSeedFile.ToRows(document!, "seed")
.Select(row => new Dictionary<string, object>(row.Properties, StringComparer.Ordinal)
{
["PartitionKey"] = CpeMappingStore.SeedPartition,
["RowKey"] = row.RowKey,
})
.Concat(nextToSeed)
.ToList();
var selfUpdating = new HashSet<string>(
(document!.SelfUpdatingProducts ?? new List<string>()).Select(p => p.Trim().ToLowerInvariant()), StringComparer.Ordinal);
return new MappingSet(Path.GetFileNameWithoutExtension(path), rows, selfUpdating);
}
private static List<Dictionary<string, object>> ReadRows(string path)
{
using var document = JsonDocument.Parse(File.ReadAllText(path));
var rows = new List<Dictionary<string, object>>();
foreach (var element in document.RootElement.EnumerateArray())
{
var row = new Dictionary<string, object>(StringComparer.Ordinal);
foreach (var property in element.EnumerateObject())
{
if (property.Value.ValueKind == JsonValueKind.String) row[property.Name] = property.Value.GetString()!;
else if (property.Value.ValueKind != JsonValueKind.Null) row[property.Name] = property.Value.ToString();
}
rows.Add(row);
}
return rows;
}
private static string Text(Dictionary<string, object> row, string column) =>
row.TryGetValue(column, out var value) ? value?.ToString() ?? string.Empty : string.Empty;
}
@@ -0,0 +1,261 @@
using Azure;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using static AutopilotMonitor.Functions.Tests.MappingListFixture;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeMappingImporter"/> replaces the mappings of one list: fetch, check, write, then
/// remove what the list no longer holds. Whatever goes wrong before the first write leaves the
/// mappings in service exactly as they were, so a list that cannot be fetched or is refused never
/// empties them. A refused list is recorded as an ops event, whoever asked for the import.
/// </summary>
public class CpeMappingImporterTests
{
private const string Seed = CpeMappingStore.SeedPartition;
private const string Community = CpeMappingStore.CommunityPartition;
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private sealed class Rig
{
public Rig()
{
Store = new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance);
Importer = new CpeMappingImporter(
Tables.Repo.Object,
Rules.Repository(),
new CpeProductFlagsService(Tables.Repo.Object, Store),
Store,
OpsEvents.Object,
NullLogger<CpeMappingImporter>.Instance);
}
/// <summary>Records nothing; a test verifies what the import reported.</summary>
public Mock<OpsEventService> OpsEvents { get; } = new(null!, null!, null!) { CallBase = false };
/// <summary>The lists reported as refused, each with its problems, one entry per report.</summary>
public List<(string List, IReadOnlyList<string> Problems)> RefusalsReported => OpsEvents.Invocations
.Where(invocation => invocation.Method.Name == nameof(OpsEventService.RecordCpeMappingListRefusedAsync))
.Select(invocation => ((string)invocation.Arguments[0]!, (IReadOnlyList<string>)invocation.Arguments[1]!))
.ToList();
public MappingTables Tables { get; } = new();
public RulesRepositoryStub Rules { get; } = new();
public CpeMappingStore Store { get; }
public CpeMappingImporter Importer { get; }
/// <summary>Puts <paramref name="count"/> seed mappings into service, as an earlier import left them.</summary>
public void SeedInService(int count)
{
foreach (var entry in Tools(count))
Tables.Put(VulnerabilityCacheRows.Mapping(entry.NormalizedVendor!, entry.NormalizedProduct!, entry.CpeUri!));
}
/// <summary>Makes <paramref name="product"/> self-updating by default, as an earlier seed import left it.</summary>
public void DefaultInService(string product) => Tables.Put(
CpeMappingStore.ProductFlagsPartition, product,
new Dictionary<string, object> { ["Product"] = product, ["SeedSelfUpdating"] = true });
}
/// <summary>A rig with widget, gadget and tool in service and a seed list that corrects the widget, keeps the gadget, drops the tool and adds a reader.</summary>
private static Rig RigWithAChangedSeedList()
{
var rig = new Rig();
rig.Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget_old", "contoso widget"));
rig.Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"));
rig.Tables.Put(VulnerabilityCacheRows.Mapping("northwind", "northwind tool", "cpe:2.3:a:northwind:tool", "northwind tool"));
rig.DefaultInService("northwind:tool");
rig.Rules.SeedList = Json(
new[]
{
Entry("contoso", "contoso widget", WidgetCpe),
Entry("fabrikam", "fabrikam gadget", GadgetCpe),
Entry("adatum", "adatum reader", "cpe:2.3:a:adatum:reader"),
},
"contoso:widget");
return rig;
}
[Fact]
public async Task ImportSeed_WritesEveryRow_ThenRemovesWhatTheListNoLongerHolds_AndAppliesTheDefaults()
{
var rig = RigWithAChangedSeedList();
var custom = VulnerabilityCacheRows.MappingIn(
CpeMappingStore.CustomPartition, "manual", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool_custom");
rig.Tables.Put(custom);
var before = await rig.Store.GetAsync();
var result = await rig.Importer.ImportSeedAsync();
Assert.False(result.Refused);
Assert.Empty(result.Problems);
Assert.Equal(3, result.Written);
Assert.Equal(1, result.Removed);
Assert.Equal(new[] { RulesRepositoryStub.SeedListPath }, rig.Rules.RequestedPaths);
Assert.Equal(
new[] { "adatum:adatum reader", "contoso:contoso widget", "fabrikam:fabrikam gadget" }, rig.Tables.RowKeys(Seed));
var widget = rig.Tables.Row(Seed, "contoso:contoso widget")!;
Assert.Equal(WidgetCpe, widget["CpeUri"]);
Assert.Equal("seed", widget["Source"]);
Assert.Equal("""["contoso widget"]""", widget["DisplayNamePatternsJson"]);
// Another list's rows are not this import's to remove.
Assert.Equal(custom["CpeUri"], rig.Tables.Row(CpeMappingStore.CustomPartition, "northwind:northwind tool")!["CpeUri"]);
// Every row is in place before the first one goes.
Assert.Equal(
new[]
{
$"upsert {Seed}/contoso:contoso widget",
$"upsert {Seed}/fabrikam:fabrikam gadget",
$"upsert {Seed}/adatum:adatum reader",
$"delete {Seed}/northwind:northwind tool",
},
rig.Tables.Writes.Where(write => write.Contains($" {Seed}/", StringComparison.Ordinal)));
// The list's defaults replace the ones in service.
var flags = Assert.Single(rig.Tables.Rows(CpeMappingStore.ProductFlagsPartition));
Assert.Equal("contoso:widget", flags["Product"]);
Assert.Equal(true, flags["SeedSelfUpdating"]);
// The next reader on this instance sees the list, not the copy read before.
var after = await rig.Store.GetAsync();
Assert.NotSame(before, after);
Assert.Equal(WidgetCpe, after.Matcher.Match("contoso", "contoso widget", "Contoso Widget 4.2", "Contoso Ltd")!.Value.Mapping.CpeUri);
Assert.NotNull(after.Matcher.Match("adatum", "adatum reader", "Adatum Reader", "Adatum"));
Assert.True(after.IsSelfUpdating("contoso:widget"));
Assert.False(after.IsSelfUpdating("northwind:tool"));
}
[Fact]
public async Task ImportSeed_WriteThatFailsHalfWay_RemovesNothing()
{
var rig = RigWithAChangedSeedList();
rig.Tables.WriteFailure = write =>
write == $"upsert {Seed}/fabrikam:fabrikam gadget" ? new RequestFailedException(503, "busy") : null;
await Assert.ThrowsAsync<RequestFailedException>(() => rig.Importer.ImportSeedAsync());
// A failure on the way leaves more mappings in service than the list holds, never fewer.
Assert.Equal(new[] { $"upsert {Seed}/contoso:contoso widget" }, rig.Tables.Writes);
Assert.Equal(
new[] { "contoso:contoso widget", "fabrikam:fabrikam gadget", "northwind:northwind tool" }, rig.Tables.RowKeys(Seed));
Assert.Equal("northwind:tool", Assert.Single(rig.Tables.Rows(CpeMappingStore.ProductFlagsPartition))["Product"]);
}
public static TheoryData<string, string?> RefusedSeedLists() => new()
{
{ "shrinks too much", Json(Tools(1)) },
{ "not json", "<html>maintenance</html>" },
{ "empty list", Json(Array.Empty<CpeSeedEntry>()) },
{ "empty body", "" },
{ "entry with a version in its CPE", Json(Tools(3).Append(Entry("contoso", "contoso widget", "cpe:2.3:a:contoso:widget:4.2"))) },
};
[Theory]
[MemberData(nameof(RefusedSeedLists))]
public async Task ImportSeed_RefusedList_WritesAndRemovesNothing_AndReturnsTheProblems(string reason, string? list)
{
var rig = new Rig();
rig.SeedInService(4);
rig.DefaultInService("contoso:tool_a");
var inService = rig.Tables.RowKeys(Seed).ToList();
rig.Rules.SeedList = list;
var result = await rig.Importer.ImportSeedAsync();
Assert.True(result.Refused, reason);
Assert.NotEmpty(result.Problems);
Assert.Equal(0, result.Written);
Assert.Equal(0, result.Removed);
Assert.Empty(rig.Tables.Writes);
Assert.Equal(inService, rig.Tables.RowKeys(Seed));
Assert.Single(rig.Tables.Rows(CpeMappingStore.ProductFlagsPartition));
// The refusal is an operator's business whichever button asked for the import.
var reported = Assert.Single(rig.RefusalsReported);
Assert.Equal("seed", reported.List);
Assert.Equal(result.Problems, reported.Problems);
}
[Fact]
public async Task ImportCommunity_EmptyListWithNothingInService_ImportsNothingWithoutBeingRefused()
{
var rig = new Rig();
rig.SeedInService(2);
rig.Rules.CommunityList = """{ "version": "1.0.0", "lastUpdated": "2025-03-20", "mappings": [] }""";
var result = await rig.Importer.ImportCommunityAsync();
Assert.False(result.Refused);
Assert.Equal(0, result.Written);
Assert.Equal(0, result.Removed);
Assert.Empty(rig.Tables.Writes);
Assert.Equal(new[] { RulesRepositoryStub.CommunityListPath }, rig.Rules.RequestedPaths);
}
[Fact]
public async Task ImportCommunity_EmptyListWhileCommunityMappingsAreInService_IsRefused()
{
var rig = new Rig();
var inService = VulnerabilityCacheRows.MappingIn(Community, "community", "contoso", "contoso widget", WidgetCpe);
rig.Tables.Put(inService);
rig.Rules.CommunityList = """{ "version": "1.0.0", "mappings": [] }""";
var result = await rig.Importer.ImportCommunityAsync();
Assert.True(result.Refused);
Assert.Empty(rig.Tables.Writes);
Assert.Equal(new[] { (string)inService["RowKey"] }, rig.Tables.RowKeys(Community));
Assert.Equal("community", Assert.Single(rig.RefusalsReported).List);
}
[Fact]
public async Task ImportCommunity_WritesIntoItsOwnPartition_AndLeavesTheSeedAndItsDefaultsAlone()
{
var rig = new Rig();
rig.SeedInService(2);
rig.DefaultInService("contoso:tool_a");
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(Community, "community", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool"));
var seedBefore = rig.Tables.RowKeys(Seed).ToList();
rig.Rules.CommunityList = Json(new[] { Entry("contoso", "contoso widget", WidgetCpe) });
var result = await rig.Importer.ImportCommunityAsync();
Assert.False(result.Refused);
Assert.Empty(rig.RefusalsReported);
Assert.Equal(1, result.Written);
Assert.Equal(1, result.Removed);
var row = Assert.Single(rig.Tables.Rows(Community));
Assert.Equal("contoso:contoso widget", row["RowKey"]);
Assert.Equal("community", row["Source"]);
Assert.Equal(seedBefore, rig.Tables.RowKeys(Seed));
// The default set of self-updating products belongs to the seed alone.
Assert.All(rig.Tables.Writes, write => Assert.Contains($" {Community}/", write));
Assert.Single(rig.Tables.Rows(CpeMappingStore.ProductFlagsPartition));
Assert.Equal(CpeMappingSource.Community, (await rig.Store.GetAsync()).Matcher
.Match("contoso", "contoso widget", "Contoso Widget", "Contoso Ltd")!.Value.Mapping.Source);
}
[Theory]
[InlineData("the file is not there")]
[InlineData("the connection breaks")]
public async Task Import_FetchThatFails_LeavesTheTableUntouched(string failure)
{
var rig = new Rig();
rig.SeedInService(3);
var inService = rig.Tables.RowKeys(Seed).ToList();
if (failure == "the connection breaks") rig.Rules.Failure = new HttpRequestException("connection reset");
await Assert.ThrowsAsync<HttpRequestException>(() => rig.Importer.ImportSeedAsync());
await Assert.ThrowsAsync<HttpRequestException>(() => rig.Importer.ImportCommunityAsync());
// A list that could not be fetched was not refused: the caller hears about the failure.
Assert.Empty(rig.RefusalsReported);
Assert.Empty(rig.Tables.Writes);
Assert.Equal(inService, rig.Tables.RowKeys(Seed));
}
}
@@ -0,0 +1,88 @@
using System.Text.Json;
using System.Text.RegularExpressions;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The committed mapping list against program names as devices report them. Each entry of
/// <c>TestData/cpe-mapping-names.json</c> is a display name with its publisher and the product
/// the list must map it to, or no product when the list must leave it alone: an add-in, a
/// language pack, a program that only shares a word with a mapped one. A change to the list or
/// to the matcher that moves one of them fails here by name.
/// </summary>
public class CpeMappingListNamesTests
{
private static readonly Lazy<CpeMappingMatcher> CommittedList = new(BuildMatcher);
private static readonly Regex FromTheVersionOn = new(@"\s+v?\d+(\.\d+)+.*$", RegexOptions.CultureInvariant);
private static readonly Regex TrailingMarker = new(@"\s*\((?:x64|x86|64-bit|32-bit|user)\)$", RegexOptions.CultureInvariant);
private static readonly Regex TrailingLegalForm = new(@"\s+(?:inc|llc|ltd|gmbh|corporation|corp)$", RegexOptions.CultureInvariant);
public static TheoryData<string, string, string?, string?> Names()
{
var data = new TheoryData<string, string, string?, string?>();
using var document = JsonDocument.Parse(File.ReadAllText(
Path.Combine(FindRepoRoot(), "src", "Backend", "AutopilotMonitor.Functions.Tests", "TestData", "cpe-mapping-names.json")));
foreach (var entry in document.RootElement.EnumerateArray())
{
data.Add(
entry.GetProperty("name").GetString()!,
entry.GetProperty("publisher").GetString()!,
entry.TryGetProperty("product", out var product) ? product.GetString() : null,
entry.TryGetProperty("edition", out var edition) ? edition.GetString() : null);
}
return data;
}
/// <param name="product">The <c>vendor:product</c> the name must map to; null when it must stay unmapped.</param>
/// <param name="edition">When given, the edition of the mapping it lands on; empty for a mapping without one.</param>
[Theory]
[MemberData(nameof(Names))]
public void Name_MapsToTheProductTheListMeansForIt(string name, string publisher, string? product, string? edition)
{
var match = CommittedList.Value.Match(NormalizedVendor(publisher), NormalizedName(name), name, publisher);
Assert.Equal(product, match?.Mapping.ProductKey);
if (edition != null)
Assert.Equal(edition, match?.Mapping.Edition ?? string.Empty);
}
// The file holds what a device shows, the display name and the publisher. An inventory row
// also carries both in normalized form, and the matcher reads all four; the expectations in
// the file were worked out with the two derived by exactly these rules.
private static string NormalizedVendor(string publisher) =>
TrailingLegalForm.Replace(publisher.ToLowerInvariant().Replace(",", string.Empty).Replace(".", string.Empty).Trim(), string.Empty).Trim();
private static string NormalizedName(string name) =>
TrailingMarker.Replace(FromTheVersionOn.Replace(name.ToLowerInvariant(), string.Empty).Trim(), string.Empty).Trim();
/// <summary>The matcher over the rows the committed list becomes on import, in the order the store reads them.</summary>
private static CpeMappingMatcher BuildMatcher()
{
var (document, problems) = CpeSeedFile.Read(
File.ReadAllText(Path.Combine(FindRepoRoot(), "rules", "dist", "cpe-mappings.json")), mappingsInService: 0);
Assert.True(problems.Count == 0, string.Join(Environment.NewLine, problems.Take(20)));
var mappings = CpeSeedFile.ToRows(document!, "seed")
.Select(row => new Dictionary<string, object>(row.Properties, StringComparer.Ordinal)
{
["PartitionKey"] = CpeMappingStore.SeedPartition,
["RowKey"] = row.RowKey,
})
.Select(CpeMappingStore.MappingFromRow)
.Where(mapping => mapping != null)
.Select(mapping => mapping!)
.OrderBy(mapping => mapping.PartitionKey, StringComparer.Ordinal)
.ThenBy(mapping => mapping.RowKey, StringComparer.Ordinal);
return new CpeMappingMatcher(mappings);
}
private static string FindRepoRoot()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln")))
dir = dir.Parent;
return dir?.FullName ?? throw new InvalidOperationException("Repository root (AutopilotMonitor.sln) not found.");
}
}
@@ -0,0 +1,245 @@
using System;
using System.Collections.Generic;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Rules of <see cref="CpeMappingMatcher"/>. The negative cases are names that the former
/// substring matching filed under a foreign product.
/// </summary>
public class CpeMappingMatcherTests
{
private static CpeMapping Map(
string vendor, string product, string cpe,
string[]? patterns = null, string[]? publishers = null, string[]? excludes = null,
CpeMappingSource source = CpeMappingSource.Seed)
=> new()
{
PartitionKey = "test",
RowKey = $"{vendor}:{product}",
Source = source,
Vendor = vendor,
Product = product,
CpeUri = cpe,
ProductKey = CpeName.ProductKeyOf(cpe),
DisplayNamePatterns = patterns ?? Array.Empty<string>(),
PublisherPatterns = publishers ?? Array.Empty<string>(),
ExcludePatterns = excludes ?? Array.Empty<string>(),
};
private static readonly CpeMapping Chrome = Map("google", "google chrome", "cpe:2.3:a:google:chrome",
new[] { "google chrome", "chrome" }, new[] { "google", "google llc" });
private static readonly CpeMapping Edge = Map("microsoft", "microsoft edge", "cpe:2.3:a:microsoft:edge_chromium",
new[] { "microsoft edge", "microsoft edge webview2", "webview2 runtime" }, new[] { "microsoft" }, new[] { "edge update" });
private static readonly CpeMapping EdgeUpdate = Map("microsoft", "microsoft edge update", "cpe:2.3:a:microsoft:edge_update",
new[] { "microsoft edge update" }, new[] { "microsoft" });
private static readonly CpeMapping Rust = Map("rust-lang", "rust", "cpe:2.3:a:rust-lang:rust",
new[] { "rust" }, new[] { "the rust project developers" });
private static readonly CpeMapping Redis = Map("redis", "redis", "cpe:2.3:a:redis:redis",
new[] { "redis" }, new[] { "redis", "redis labs" });
private static readonly CpeMapping RLanguage = Map("r-project", "r", "cpe:2.3:a:r-project:r",
new[] { "r for windows" }, new[] { "r core team" });
private static readonly CpeMapping Opera = Map("opera", "opera", "cpe:2.3:a:opera:opera_browser",
new[] { "opera stable", "opera" }, new[] { "opera software" });
private static readonly CpeMapping Visio = Map("microsoft", "microsoft visio", "cpe:2.3:a:microsoft:visio",
new[] { "microsoft visio", "visio" }, new[] { "microsoft" });
private static readonly CpeMapping OracleJdk = Map("oracle", "java se development kit", "cpe:2.3:a:oracle:jdk",
new[] { "java se development kit", "jdk" }, new[] { "oracle", "oracle corporation" });
private static readonly CpeMapping VisualStudio = Map("microsoft", "visual studio", "cpe:2.3:a:microsoft:visual_studio",
new[] { "visual studio" }, new[] { "microsoft" });
private static readonly CpeMapping VsCode = Map("microsoft", "visual studio code", "cpe:2.3:a:microsoft:visual_studio_code",
new[] { "microsoft visual studio code", "visual studio code" }, new[] { "microsoft" });
private static readonly CpeMapping GoLanguage = Map("golang", "go", "cpe:2.3:a:golang:go",
new[] { "go programming language" }, new[] { "google", "the go authors" });
private static readonly CpeMapping Access = Map("microsoft", "microsoft access", "cpe:2.3:a:microsoft:access",
new[] { "microsoft access" }, new[] { "microsoft" });
private static readonly CpeMapping Office = Map("microsoft", "microsoft office", "cpe:2.3:a:microsoft:office",
new[] { "microsoft office" }, new[] { "microsoft" });
private static readonly CpeMapping Teams = Map("microsoft", "microsoft teams", "cpe:2.3:a:microsoft:teams",
new[] { "microsoft teams", "msteams", "teams" }, new[] { "microsoft" }, new[] { "add-in", "addin" });
private static readonly CpeMapping VisualCpp = Map("microsoft", "visual c++", @"cpe:2.3:a:microsoft:visual_c\+\+",
new[] { "microsoft visual c++" }, new[] { "microsoft" });
private static readonly CpeMapping NotepadPp = Map("notepad++", "notepad++", @"cpe:2.3:a:notepad-plus-plus:notepad\+\+",
new[] { "notepad++" }, new[] { "notepad++ team" });
private static readonly CpeMapping DotNet = Map("microsoft", ".net", "cpe:2.3:a:microsoft:.net",
new[] { "microsoft .net runtime", ".net runtime" }, new[] { "microsoft" });
private static readonly CpeMapping NodeJs = Map("openjs foundation", "node.js", "cpe:2.3:a:nodejs:node.js",
new[] { "node.js" }, new[] { "node.js foundation", "openjs foundation" });
private static readonly CpeMappingMatcher Matcher = new(new[]
{
Chrome, Edge, EdgeUpdate, Rust, Redis, RLanguage, Opera, Visio, OracleJdk, VisualStudio, VsCode,
GoLanguage, Access, Office, Teams, VisualCpp, NotepadPp, DotNet, NodeJs,
});
private static string? CpeOf(string vendor, string name, string display, string publisher = "")
=> Matcher.Match(vendor, name, display, publisher)?.Mapping.CpeUri;
// ---------------------------------------------------------------- key match
[Fact]
public void KeyMatch_VendorAndCanonicalName_IsHighConfidence()
{
var match = Matcher.Match("google", "google chrome", "Google Chrome", "Google LLC");
Assert.Equal("cpe:2.3:a:google:chrome", match!.Value.Mapping.CpeUri);
Assert.Equal("high", match.Value.Confidence);
}
[Fact]
public void KeyMatch_UsesTheDisplayNameNotTheAgentKey()
{
// The agent's normalized name can be mangled; the shown name carries the product.
var match = Matcher.Match("notepad++", "notepad++x64)", "Notepad++ (64-bit x64)", "Notepad++ Team");
Assert.Equal(@"cpe:2.3:a:notepad-plus-plus:notepad\+\+", match!.Value.Mapping.CpeUri);
Assert.Equal("high", match.Value.Confidence);
}
[Fact]
public void KeyMatch_HigherSourceWinsForTheSameKey()
{
var seed = Map("contoso", "contoso widget", "cpe:2.3:a:contoso:widget_old");
var custom = Map("contoso", "Contoso Widget 4.2 (x64)", "cpe:2.3:a:contoso:widget", source: CpeMappingSource.Custom);
var auto = Map("contoso", "contoso widget", "cpe:2.3:a:contoso:widget_guess", source: CpeMappingSource.Auto);
foreach (var order in new[] { new[] { seed, custom, auto }, new[] { auto, custom, seed }, new[] { custom, auto, seed } })
{
var match = new CpeMappingMatcher(order).Match("contoso", "widget", "Contoso Widget 5.0", "Contoso Ltd");
Assert.Equal("cpe:2.3:a:contoso:widget", match!.Value.Mapping.CpeUri);
}
}
[Fact]
public void KeyMatch_UnknownVendorPlaceholderEqualsNoVendor()
{
var custom = Map("unknown", "Microsoft Edge WebView2 Runtime", "cpe:2.3:a:microsoft:edge_chromium", source: CpeMappingSource.Custom);
var match = new CpeMappingMatcher(new[] { custom }).Match("", "microsoft edge webview2 runtime", "Microsoft Edge WebView2 Runtime", "");
Assert.Equal("high", match!.Value.Confidence);
}
[Fact]
public void KeyMatch_AnotherVendorWithTheSameNameIsNotMatched()
{
var custom = Map("contoso", "Agent", "cpe:2.3:a:contoso:agent", source: CpeMappingSource.Custom);
Assert.Null(new CpeMappingMatcher(new[] { custom }).Match("fabrikam", "agent", "Agent 2.1", "Fabrikam Inc"));
}
// ---------------------------------------------------------------- patterns
[Fact]
public void Pattern_WholeWordsInOrder_IsMediumConfidence()
{
var match = Matcher.Match("", "microsoft edge webview2 runtime", "Microsoft Edge WebView2 Runtime", "");
Assert.Equal("cpe:2.3:a:microsoft:edge_chromium", match!.Value.Mapping.CpeUri);
Assert.Equal("medium", match.Value.Confidence);
}
[Theory]
[InlineData("beyondtrust", "BeyondTrust Privilege Management for Windows (x64) 24.5.220.0", "BeyondTrust Corporation")] // "rust"
[InlineData("microsoft", "Microsoft Visual C++ 2013 Redistributable (x86) - 12.0.40664", "Microsoft Corporation")] // "redis"
[InlineData("microsoft", "Microsoft Interoperability Forms Redistributable Package 2.0a", "Microsoft Corporation")] // "opera"
[InlineData("contoso", "Envision Viewer 3.1", "Contoso Ltd")] // "visio"
[InlineData("google", "Google Earth Pro", "Google LLC")] // "go"
[InlineData("microsoft", "Global Secure Access Client", "Microsoft Corporation")] // "access"
public void Pattern_NeverMatchesInsideALongerWordOrByVendorAlone(string vendor, string display, string publisher)
{
var cpe = CpeOf(vendor, display.ToLowerInvariant(), display, publisher);
Assert.True(cpe is null or @"cpe:2.3:a:microsoft:visual_c\+\+", $"'{display}' was filed under {cpe}");
}
[Fact]
public void Pattern_LanguageSuffixDoesNotTriggerAOneLetterProduct()
=> Assert.Null(CpeOf("microsoft", "microsoft onenote - fr-fr", "Microsoft OneNote - fr-fr", "Microsoft Corporation"));
[Fact]
public void Pattern_MostWordsWins()
{
Assert.Equal("cpe:2.3:a:microsoft:visual_studio_code",
CpeOf("microsoft", "microsoft visual studio code (user)", "Microsoft Visual Studio Code (User)", "Microsoft Corporation"));
Assert.Equal("cpe:2.3:a:microsoft:visual_studio",
CpeOf("microsoft", "visual studio professional 2022", "Visual Studio Professional 2022", "Microsoft Corporation"));
Assert.Equal("cpe:2.3:a:microsoft:edge_update",
CpeOf("", "microsoft edge update", "Microsoft Edge Update", ""));
}
[Fact]
public void Pattern_AfterForIsAnAddOnNotTheProduct()
{
Assert.Null(CpeOf("contoso", "remote desktop host for google chrome", "Remote Desktop Host for Google Chrome", "Contoso Ltd"));
Assert.Null(CpeOf("contoso", "druckertreiber für microsoft office", "Druckertreiber für Microsoft Office", "Contoso GmbH"));
}
[Fact]
public void Pattern_ExcludeWordsBlockTheMapping()
{
// The add-in is not Teams, and what follows "for" is not Office either.
Assert.Null(CpeOf("microsoft", "microsoft teams meeting add-in for microsoft office",
"Microsoft Teams Meeting Add-in for Microsoft Office", "Microsoft"));
Assert.Equal("cpe:2.3:a:microsoft:teams",
CpeOf("microsoft", "teams machine-wide installer", "Teams Machine-Wide Installer", "Microsoft Corporation"));
}
[Fact]
public void OneWordPattern_NeedsTheVendorOrTheWholeName()
{
// Vendor agrees.
Assert.Equal("cpe:2.3:a:google:chrome", CpeOf("google", "chrome beta", "Chrome Beta 131.0", "Google LLC"));
// Vendor does not: a helper that carries the browser's name.
Assert.Null(CpeOf("contoso", "contoso chrome helper", "Contoso Chrome Helper 3.0", "Contoso Ltd"));
// The word is the whole name: an AppX package has no publisher to compare.
Assert.Equal("cpe:2.3:a:microsoft:teams", CpeOf("msteams", "msteams", "MSTeams", "MSTeams"));
}
[Fact]
public void OneWordPattern_VendorAgreesUpToLegalFormAndTrailingWords()
{
Assert.Equal("cpe:2.3:a:oracle:jdk", CpeOf("oracle", "jdk 21", "JDK 21.0.2", "Oracle Corporation"));
Assert.Equal("cpe:2.3:a:google:chrome", CpeOf("google llc", "chrome", "Chrome Canary", "Google LLC"));
// Another vendor's JDK is not Oracle's.
Assert.Null(CpeOf("azul", "azul zulu jdk", "Azul Zulu JDK 17.0.9 (64-bit)", "Azul Systems, Inc."));
}
[Theory]
[InlineData("microsoft", "Microsoft Visual C++ 2015-2022 Redistributable (x64) - 14.38.33135", @"cpe:2.3:a:microsoft:visual_c\+\+")]
[InlineData("contoso", "Notepad++ Portable Launcher", null)] // one word, foreign vendor
[InlineData("notepad++ team", "Notepad++ Portable Launcher", @"cpe:2.3:a:notepad-plus-plus:notepad\+\+")]
[InlineData("microsoft", "Microsoft .NET Runtime - 8.0.11 (x64)", "cpe:2.3:a:microsoft:.net")]
[InlineData("node.js foundation", "Node.js 20.11.0", "cpe:2.3:a:nodejs:node.js")]
public void Tokens_KeepPlusSignsAndSplitOnDots(string vendor, string display, string? expectedCpe)
=> Assert.Equal(expectedCpe, CpeOf(vendor, display.ToLowerInvariant(), display, vendor));
[Fact]
public void Tokenize_SplitsOnPunctuationAndKeepsPlusAndHash()
{
Assert.Equal(new[] { "microsoft", "visual", "c++", "2015", "2022" }, CpeMappingMatcher.Tokenize("Microsoft Visual C++ 2015-2022"));
Assert.Equal(new[] { "f#", "tools" }, CpeMappingMatcher.Tokenize("F# Tools"));
Assert.Equal(new[] { "intel", "wireless" }, CpeMappingMatcher.Tokenize("Intel(R) Wireless™"));
Assert.Empty(CpeMappingMatcher.Tokenize(" "));
}
[Fact]
public void Match_NothingKnown_IsNull()
{
Assert.Null(Matcher.Match("contoso", "widget", "Contoso Widget 4.2", "Contoso Ltd"));
Assert.Null(Matcher.Match(null, null, null, null));
}
// ---------------------------------------------------------------- keys
[Theory]
[InlineData("Google", "Google Chrome 131.0.6778.86", "google|google chrome")]
[InlineData("google llc", "Google Chrome", "google llc|google chrome")]
[InlineData("unknown", "7-Zip 24.08 (x64)", "|7-zip")]
[InlineData(null, "7-Zip", "|7-zip")]
[InlineData("contoso", "", "")]
public void SoftwareKey_IsVendorAndCanonicalName(string? vendor, string name, string expected)
=> Assert.Equal(expected, CpeMappingMatcher.SoftwareKey(vendor, name));
}
@@ -0,0 +1,324 @@
using System.Text.Json;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The two events that say the software-to-CPE mappings are not what they should be: the table
/// could not be read (<see cref="OpsEventService.RecordCpeMappingUnavailableAsync"/>), or an
/// import fetched a list the check refused (<see cref="OpsEventService.RecordCpeMappingListRefusedAsync"/>).
/// Every instance runs into the same outage, so each of them reads today's Maintenance events
/// first and records one event per UTC day, cause (for a refusal: list) and severity.
/// </summary>
public class CpeMappingOpsEventTests
{
private const string UnreadableWithoutACopy = "CPE mappings could not be loaded; vulnerability scans are postponed";
private const string UnreadableWithACopy = "CPE mappings could not be reloaded; an earlier copy keeps serving";
private sealed class Rig
{
public Rig()
{
var repository = new Mock<IOpsEventRepository>();
repository.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
.Returns<OpsEventEntry>(entry =>
{
Recorded.Add(entry);
return Task.CompletedTask;
});
repository.Setup(r => r.GetOpsEventsAsync(
It.IsAny<string?>(), It.IsAny<DateTime?>(), It.IsAny<DateTime?>(), It.IsAny<OpsEventQueryFilters?>()))
.Returns<string?, DateTime?, DateTime?, OpsEventQueryFilters?>((category, from, to, _) =>
{
Reads.Add((category, from, to));
if (ReadFailure != null) return Task.FromException<List<OpsEventEntry>>(ReadFailure);
return Task.FromResult(AlreadyInTheTable.Concat(Recorded)
.Where(e => (category == null || e.Category == category)
&& (from == null || e.Timestamp >= from)
&& (to == null || e.Timestamp < to))
.OrderByDescending(e => e.Timestamp)
.ToList());
});
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions())) { CallBase = false };
Service = new OpsEventService(
repository.Object, NullLogger<OpsEventService>.Instance, TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
}
public OpsEventService Service { get; }
/// <summary>The events the service recorded, in order.</summary>
public List<OpsEventEntry> Recorded { get; } = new();
/// <summary>Events another instance, or an earlier day, left in the table.</summary>
public List<OpsEventEntry> AlreadyInTheTable { get; } = new();
/// <summary>Every read of the events: the category and the window that were asked for.</summary>
public List<(string? Category, DateTime? From, DateTime? To)> Reads { get; } = new();
/// <summary>When set, reading the events fails with it.</summary>
public Exception? ReadFailure { get; set; }
public IEnumerable<(string Severity, string Message)> RecordedAs => Recorded.Select(e => (e.Severity, e.Message));
}
private static OpsEventEntry Stored(string eventType, string severity, string message, DateTime timestampUtc) => new()
{
Category = OpsEventCategory.Maintenance,
EventType = eventType,
Severity = severity,
Message = message,
Timestamp = timestampUtc,
};
// -----------------------------------------------------------------------
// The mapping table could not be read
// -----------------------------------------------------------------------
[Fact]
public async Task Unreadable_WithoutAnEarlierCopy_IsAnError_BecauseScansWait()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingUnavailableAsync("RequestFailedException: storage down", servingEarlierCopy: false);
var recorded = Assert.Single(rig.Recorded);
Assert.Equal(OpsEventCategory.Maintenance, recorded.Category);
Assert.Equal(OpsEventTypes.CpeMappingUnavailable, recorded.EventType);
Assert.Equal(OpsEventSeverity.Error, recorded.Severity);
Assert.Equal(UnreadableWithoutACopy, recorded.Message);
Assert.Null(recorded.TenantId);
using var details = JsonDocument.Parse(recorded.Details!);
Assert.Equal("RequestFailedException: storage down", details.RootElement.GetProperty("error").GetString());
Assert.False(details.RootElement.GetProperty("servingEarlierCopy").GetBoolean());
}
[Fact]
public async Task Unreadable_WhileAnEarlierCopyServes_IsAWarning()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingUnavailableAsync("RequestFailedException: storage down", servingEarlierCopy: true);
var recorded = Assert.Single(rig.Recorded);
Assert.Equal(OpsEventTypes.CpeMappingUnavailable, recorded.EventType);
Assert.Equal(OpsEventSeverity.Warning, recorded.Severity);
Assert.Equal(UnreadableWithACopy, recorded.Message);
using var details = JsonDocument.Parse(recorded.Details!);
Assert.True(details.RootElement.GetProperty("servingEarlierCopy").GetBoolean());
}
[Fact]
public async Task Unreadable_ReadsTodaysMaintenanceEventsBeforeItRecords()
{
var rig = new Rig();
var dayAtTheStart = DateTime.UtcNow.Date;
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
var read = Assert.Single(rig.Reads);
Assert.Equal(OpsEventCategory.Maintenance, read.Category);
// The day as it was when the call ran, which is the next one only across midnight.
Assert.InRange(read.From!.Value, dayAtTheStart, DateTime.UtcNow.Date);
Assert.Equal(TimeSpan.Zero, read.From.Value.TimeOfDay);
Assert.Equal(read.From.Value.AddDays(1), read.To);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Unreadable_ASecondTimeOnTheSameDay_RecordsNothing(bool servingEarlierCopy)
{
var rig = new Rig();
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy);
await rig.Service.RecordCpeMappingUnavailableAsync("storage down again", servingEarlierCopy);
await rig.Service.RecordCpeMappingUnavailableAsync("and again", servingEarlierCopy);
Assert.Single(rig.Recorded);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Unreadable_WhenAnotherInstanceRecordedItToday_RecordsNothing(bool servingEarlierCopy)
{
var rig = new Rig();
rig.AlreadyInTheTable.Add(Stored(
OpsEventTypes.CpeMappingUnavailable,
servingEarlierCopy ? OpsEventSeverity.Warning : OpsEventSeverity.Error,
servingEarlierCopy ? UnreadableWithACopy : UnreadableWithoutACopy,
DateTime.UtcNow));
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy);
Assert.Empty(rig.Recorded);
}
[Fact]
public async Task Unreadable_AnErrorAfterAWarning_IsRecorded()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: true);
// An instance without a copy cannot scan at all: that is worse news than the warning was.
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: true);
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
Assert.Equal(
new[] { (OpsEventSeverity.Warning, UnreadableWithACopy), (OpsEventSeverity.Error, UnreadableWithoutACopy) },
rig.RecordedAs);
}
[Fact]
public async Task Unreadable_AWarningAfterAnError_IsNotRecorded()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: true);
Assert.Equal(new[] { (OpsEventSeverity.Error, UnreadableWithoutACopy) }, rig.RecordedAs);
}
[Fact]
public async Task Unreadable_OnTheDayAfterAnEvent_IsRecordedAgain()
{
var rig = new Rig();
rig.AlreadyInTheTable.Add(Stored(
OpsEventTypes.CpeMappingUnavailable, OpsEventSeverity.Error, UnreadableWithoutACopy, DateTime.UtcNow.Date.AddMinutes(-1)));
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
Assert.Single(rig.Recorded);
}
[Fact]
public async Task NeitherEvent_IsHeldBackByOtherMaintenanceEventsOfTheDay()
{
var rig = new Rig();
// The night's run already said that its sync fell short, for this very reason.
rig.AlreadyInTheTable.Add(Stored(
OpsEventTypes.VulnerabilityDataSyncDegraded, OpsEventSeverity.Warning,
"Vulnerability data sync degraded: automatic mappings or inventory reconcile failed", DateTime.UtcNow));
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
Assert.Equal(2, rig.Recorded.Count);
}
// -----------------------------------------------------------------------
// A list the check refused
// -----------------------------------------------------------------------
[Fact]
public async Task RefusedList_IsAnError_ThatNamesTheFirstProblem_AndCarriesUpToTwentyOfThem()
{
var rig = new Rig();
var problems = Enumerable.Range(1, 25).Select(n => $"mapping {n} (contoso:contoso tool {n}): normalizedProduct is missing").ToList();
await rig.Service.RecordCpeMappingListRefusedAsync("seed", problems);
var recorded = Assert.Single(rig.Recorded);
Assert.Equal(OpsEventCategory.Maintenance, recorded.Category);
Assert.Equal(OpsEventTypes.CpeMappingUnavailable, recorded.EventType);
Assert.Equal(OpsEventSeverity.Error, recorded.Severity);
Assert.Equal($"CPE mapping list refused (seed), the mappings in service were kept: {problems[0]}", recorded.Message);
using var details = JsonDocument.Parse(recorded.Details!);
Assert.Equal("seed", details.RootElement.GetProperty("list").GetString());
Assert.Equal(problems.Take(20), details.RootElement.GetProperty("problems").EnumerateArray().Select(p => p.GetString()));
}
[Fact]
public async Task RefusedList_ASecondTimeOnTheSameDay_RecordsNothing()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
// Another import of the same list, turned down for another reason: still the same news for the day.
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "not valid JSON: unexpected end of data" });
Assert.Contains("the list holds no mapping", Assert.Single(rig.Recorded).Message);
}
[Fact]
public async Task RefusedList_TheOtherListOnTheSameDay_IsItsOwnEvent()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingListRefusedAsync("community", new[] { "not valid JSON: unexpected end of data" });
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
// One reseed imports both lists; the second refusal must not disappear behind the first.
Assert.Equal(
new[]
{
"CPE mapping list refused (community), the mappings in service were kept: not valid JSON: unexpected end of data",
"CPE mapping list refused (seed), the mappings in service were kept: the list holds no mapping",
},
rig.Recorded.Select(e => e.Message));
}
[Fact]
public async Task RefusedList_WithoutAProblem_RecordsNothing()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingListRefusedAsync("seed", Array.Empty<string>());
Assert.Empty(rig.Recorded);
}
// -----------------------------------------------------------------------
// The two causes next to each other
// -----------------------------------------------------------------------
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task RefusedListAndUnreadableTable_DoNotHoldEachOtherBack(bool refusedFirst)
{
var rig = new Rig();
Task Refused() => rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
Task Unreadable() => rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await (refusedFirst ? Refused() : Unreadable());
await (refusedFirst ? Unreadable() : Refused());
Assert.Equal(2, rig.Recorded.Count);
Assert.Single(rig.Recorded, e => e.Message == UnreadableWithoutACopy);
Assert.Single(rig.Recorded, e => e.Message.StartsWith("CPE mapping list refused", StringComparison.Ordinal));
}
[Fact]
public async Task RefusedList_DoesNotHoldBackTheWarningOfAnUnreadableTable_ThoughItIsTheMoreSevereEvent()
{
var rig = new Rig();
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: true);
Assert.Equal(new[] { OpsEventSeverity.Error, OpsEventSeverity.Warning }, rig.Recorded.Select(e => e.Severity));
}
[Fact]
public async Task TodaysEventsThatCannotBeRead_AreNoReasonToStaySilent()
{
var rig = new Rig { ReadFailure = new InvalidOperationException("table storage is not reachable") };
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await rig.Service.RecordCpeMappingUnavailableAsync("storage down", servingEarlierCopy: false);
await rig.Service.RecordCpeMappingListRefusedAsync("seed", new[] { "the list holds no mapping" });
// One event too many is the smaller harm.
Assert.Equal(3, rig.Recorded.Count);
}
}
@@ -0,0 +1,142 @@
using System.Reflection;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeMappingStore.LoadFailed"/> carries a failed load of the mappings to the ops
/// events. A load that keeps failing is retried again and again, by every scan without a copy
/// and once a minute otherwise, so the report is raised once per
/// <see cref="CpeMappingStore.ReportPause"/> and not per attempt. A load that succeeds ends the
/// pause: the next failure is a new incident.
/// </summary>
public class CpeMappingStoreLoadReportTests
{
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget", "contoso widget"));
Store = new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance)
{
LoadFailed = (_, servingEarlierCopy) =>
{
Reports.Add(servingEarlierCopy);
return ReportFailure == null ? Task.CompletedTask : Task.FromException(ReportFailure);
},
};
}
public MappingTables Tables { get; } = new();
public CpeMappingStore Store { get; }
/// <summary>One entry per report raised: whether an earlier copy was being served.</summary>
public List<bool> Reports { get; } = new();
/// <summary>When set, raising the report fails with it.</summary>
public Exception? ReportFailure { get; set; }
public void StorageGoesDown() => Tables.ReadFailure = new InvalidOperationException("table storage is not reachable");
public void StorageComesBack() => Tables.ReadFailure = null;
public int LoadAttempts => Tables.Repo.Invocations.Count(i => i.Method.Name == nameof(IVulnerabilityRepository.GetAllCpeMappingsAsync));
}
[Fact]
public async Task LoadThatKeepsFailing_WithoutAnEarlierCopy_IsReportedOnce_ThoughEveryCallTriesAgain()
{
var rig = new Rig();
rig.StorageGoesDown();
for (var call = 0; call < 4; call++)
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
Assert.Equal(4, rig.LoadAttempts);
Assert.Equal(new[] { false }, rig.Reports);
}
[Fact]
public async Task ReloadThatKeepsFailing_WhileAnEarlierCopyIsServed_IsReportedOnce()
{
var rig = new Rig();
var good = await rig.Store.GetAsync();
rig.StorageGoesDown();
// Every writer of a mapping asks for a fresh read, whatever the pause between retries says.
for (var call = 0; call < 4; call++)
Assert.Same(good, await rig.Store.ReloadAsync());
Assert.Equal(5, rig.LoadAttempts);
Assert.Equal(new[] { true }, rig.Reports);
}
[Fact]
public async Task LoadThatSucceedsAgain_EndsThePause_SoTheNextFailureIsReported()
{
var rig = new Rig();
rig.StorageGoesDown();
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
rig.StorageComesBack();
var good = await rig.Store.GetAsync();
Assert.Equal(new[] { false }, rig.Reports);
rig.StorageGoesDown();
Assert.Same(good, await rig.Store.ReloadAsync());
Assert.Same(good, await rig.Store.ReloadAsync());
// The second outage is told apart from the first: a copy is being served this time.
Assert.Equal(new[] { false, true }, rig.Reports);
}
/// <summary>
/// Lets the pause between two reports run out. The hour cannot be waited for, and the store
/// reads the clock itself.
/// </summary>
private static void LetTheReportPauseRunOut(CpeMappingStore store)
{
var notBefore = typeof(CpeMappingStore).GetField("_reportNotBeforeTicks", BindingFlags.Instance | BindingFlags.NonPublic);
Assert.NotNull(notBefore);
var until = new DateTime((long)notBefore!.GetValue(store)!, DateTimeKind.Utc);
Assert.InRange(until, DateTime.UtcNow.Add(CpeMappingStore.ReportPause).AddMinutes(-5), DateTime.UtcNow.Add(CpeMappingStore.ReportPause));
notBefore.SetValue(store, DateTime.UtcNow.AddSeconds(-1).Ticks);
}
[Fact]
public async Task OutageThatOutlastsThePause_IsReportedAgain()
{
var rig = new Rig();
rig.StorageGoesDown();
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
Assert.Single(rig.Reports);
LetTheReportPauseRunOut(rig.Store);
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
// Once per pause for as long as the outage lasts, not once for good.
Assert.Equal(new[] { false, false }, rig.Reports);
}
[Fact]
public async Task ReportThatCannotBeRaised_ChangesNothingAboutWhatTheCallerGets()
{
var rig = new Rig { ReportFailure = new InvalidOperationException("the ops events cannot be written either") };
rig.StorageGoesDown();
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
rig.StorageComesBack();
var good = await rig.Store.GetAsync();
rig.StorageGoesDown();
Assert.Same(good, await rig.Store.ReloadAsync());
Assert.Equal(new[] { false, true }, rig.Reports);
}
}
@@ -0,0 +1,238 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeMappingStore"/> serves one consistent snapshot, keeps the last good one when a
/// reload fails, and refuses to serve an empty one: correlating without mappings would take the
/// CPE off every inventory row it touches.
/// </summary>
public class CpeMappingStoreTests
{
private static Dictionary<string, object> Row(
string partition, string rowKey, string vendor, string product, string cpeUri,
string? source = null, string[]? patterns = null)
{
var row = new Dictionary<string, object>
{
["PartitionKey"] = partition,
["RowKey"] = rowKey,
["NormalizedVendor"] = vendor,
["NormalizedProduct"] = product,
["CpeUri"] = cpeUri,
["DisplayNamePatternsJson"] = System.Text.Json.JsonSerializer.Serialize(patterns ?? Array.Empty<string>()),
};
if (source != null) row["Source"] = source;
return row;
}
private sealed class Rig
{
public Mock<IVulnerabilityRepository> Repo { get; } = new();
public List<Dictionary<string, object>> Mappings { get; set; } = new();
public List<Dictionary<string, object>> Ignored { get; set; } = new();
public List<Dictionary<string, object>> Flags { get; set; } = new();
public Exception? Failure { get; set; }
public int Loads { get; private set; }
public CpeMappingStore Store { get; }
public Rig()
{
Repo.Setup(r => r.GetAllCpeMappingsAsync()).Returns(() =>
{
Loads++;
return Failure != null ? Task.FromException<List<Dictionary<string, object>>>(Failure) : Task.FromResult(Mappings);
});
Repo.Setup(r => r.GetCpeMappingsByPartitionAsync(CpeMappingStore.IgnoredPartition)).Returns(() => Task.FromResult(Ignored));
Repo.Setup(r => r.GetCpeMappingsByPartitionAsync(CpeMappingStore.ProductFlagsPartition)).Returns(() => Task.FromResult(Flags));
Store = new CpeMappingStore(Repo.Object, NullLogger<CpeMappingStore>.Instance);
}
}
private static Dictionary<string, object> ChromeSeed() =>
Row(CpeMappingStore.SeedPartition, "google:google chrome", "google", "google chrome", "cpe:2.3:a:google:chrome", patterns: new[] { "google chrome" });
[Fact]
public async Task Get_LoadsOnceAndServesTheSameSnapshot()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
var first = await rig.Store.GetAsync();
var second = await rig.Store.GetAsync();
Assert.Same(first, second);
Assert.Equal(1, rig.Loads);
Assert.Equal("cpe:2.3:a:google:chrome",
first.Matcher.Match("google", "google chrome", "Google Chrome", "Google LLC")!.Value.Mapping.CpeUri);
}
[Fact]
public async Task Reload_ReadsAgain()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
await rig.Store.GetAsync();
rig.Mappings.Add(Row(CpeMappingStore.CustomPartition, "contoso:widget", "contoso", "Contoso Widget", "cpe:2.3:a:contoso:widget", "manual"));
var reloaded = await rig.Store.ReloadAsync();
Assert.Equal(2, rig.Loads);
Assert.NotNull(reloaded.Matcher.Match("contoso", "widget", "Contoso Widget 2.0", "Contoso Ltd"));
}
[Fact]
public async Task Get_FailedReload_KeepsTheLastGoodSnapshotAndReportsIt()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
var reports = new List<bool>();
rig.Store.LoadFailed = (_, servedEarlierCopy) => { reports.Add(servedEarlierCopy); return Task.CompletedTask; };
var good = await rig.Store.GetAsync();
rig.Failure = new InvalidOperationException("storage down");
var served = await rig.Store.ReloadAsync();
Assert.Same(good, served);
Assert.Equal(new[] { true }, reports);
// While the pause after a failure runs, the held copy is served without another attempt.
var loadsAfterFailure = rig.Loads;
Assert.Same(good, await rig.Store.GetAsync());
Assert.Equal(loadsAfterFailure, rig.Loads);
}
[Fact]
public async Task Get_FailedFirstLoad_ThrowsUnavailable()
{
var rig = new Rig { Failure = new InvalidOperationException("storage down") };
var reports = new List<bool>();
rig.Store.LoadFailed = (_, servedEarlierCopy) => { reports.Add(servedEarlierCopy); return Task.CompletedTask; };
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
Assert.Equal(new[] { false }, reports);
}
[Fact]
public async Task Get_NoMappingAtAll_IsUnavailableNotAnEmptySnapshot()
{
var rig = new Rig();
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Store.GetAsync());
}
[Fact]
public async Task Get_EmptyReload_KeepsTheLastGoodSnapshot()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
var good = await rig.Store.GetAsync();
rig.Mappings = new List<Dictionary<string, object>>();
Assert.Same(good, await rig.Store.ReloadAsync());
}
[Fact]
public void MappingFromRow_ReducesTheCpeToItsProductForm()
{
var typedInFull = CpeMappingStore.MappingFromRow(
Row(CpeMappingStore.CustomPartition, "k", "Contoso", "Widget", "cpe:2.3:a:Contoso:Widget:*:*:*:*:*:*:*:*", "manual"));
var urlEncoded = CpeMappingStore.MappingFromRow(
Row(CpeMappingStore.CustomPartition, "k", "microsoft", "visual c++", "cpe:2.3:a:microsoft:visual_c%2b%2b", "manual"));
Assert.Equal("cpe:2.3:a:contoso:widget", typedInFull!.CpeUri);
Assert.Equal("contoso:widget", typedInFull.ProductKey);
Assert.Equal(@"cpe:2.3:a:microsoft:visual_c\+\+", urlEncoded!.CpeUri);
}
[Fact]
public void MappingFromRow_WithoutAUsableCpe_IsSkipped()
{
Assert.Null(CpeMappingStore.MappingFromRow(Row(CpeMappingStore.CustomPartition, "k", "contoso", "widget", "")));
Assert.Null(CpeMappingStore.MappingFromRow(Row(CpeMappingStore.CustomPartition, "k", "contoso", "widget", "cpe:2.3:a:*:*")));
}
[Fact]
public void MappingFromRow_FallsBackToTheVendorAndProductColumns()
{
var row = Row(CpeMappingStore.SeedPartition, "k", "contoso", "widget", "");
row["CpeVendor"] = "contoso";
row["CpeProduct"] = "widget";
Assert.Equal("cpe:2.3:a:contoso:widget", CpeMappingStore.MappingFromRow(row)!.CpeUri);
}
[Theory]
[InlineData(CpeMappingStore.SeedPartition, null, "Seed")]
[InlineData(CpeMappingStore.CommunityPartition, null, "Community")]
[InlineData(CpeMappingStore.AutoPartition, null, "Auto")]
[InlineData(CpeMappingStore.CustomPartition, "manual", "Custom")]
[InlineData(CpeMappingStore.CustomPartition, null, "Custom")]
// Rows the retired name search wrote into the custom partition rank below the seed.
[InlineData(CpeMappingStore.CustomPartition, "auto", "Auto")]
public void SourceOf_ReadsPartitionAndSourceColumn(string partition, string? sourceColumn, string expected)
=> Assert.Equal(expected, CpeMappingStore.SourceOf(partition, sourceColumn ?? string.Empty).ToString());
[Fact]
public async Task Snapshot_IgnoreListMatchesByVendorAndCanonicalName()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
rig.Ignored.Add(new Dictionary<string, object>
{
["PartitionKey"] = CpeMappingStore.IgnoredPartition,
["RowKey"] = "contoso ltd:contoso widget",
["NormalizedVendor"] = "Contoso Ltd", // older entries hold the raw publisher
["NormalizedProduct"] = "Contoso Widget 4.2 (x64)",
});
var snapshot = await rig.Store.GetAsync();
Assert.Equal(1, snapshot.IgnoredCount);
Assert.True(snapshot.IsIgnored("contoso", "Contoso Ltd", "Contoso Widget 5.0"));
Assert.True(snapshot.IsIgnored("contoso ltd", "", "Contoso Widget"));
Assert.False(snapshot.IsIgnored("fabrikam", "Fabrikam Inc", "Contoso Widget 5.0"));
Assert.False(snapshot.IsIgnored("contoso", "Contoso Ltd", "Contoso Gadget"));
}
[Fact]
public async Task Snapshot_SelfUpdating_AdminValueOverridesTheSeedDefault()
{
var rig = new Rig { Mappings = { ChromeSeed() } };
rig.Flags.AddRange(new[]
{
Flag("google:chrome", seed: true, admin: null),
Flag("mozilla:firefox", seed: true, admin: false),
Flag("microsoft:visual_studio_code", seed: null, admin: true),
Flag("contoso:widget", seed: false, admin: null),
});
var snapshot = await rig.Store.GetAsync();
Assert.True(snapshot.IsSelfUpdating("google:chrome"));
Assert.False(snapshot.IsSelfUpdating("mozilla:firefox"));
Assert.True(snapshot.IsSelfUpdating("microsoft:visual_studio_code"));
Assert.False(snapshot.IsSelfUpdating("contoso:widget"));
Assert.False(snapshot.IsSelfUpdating("never:listed"));
Assert.False(snapshot.IsSelfUpdating(null));
Assert.Equal(new[] { "google:chrome", "microsoft:visual_studio_code" }, snapshot.SelfUpdatingProducts().OrderBy(p => p));
}
private static Dictionary<string, object> Flag(string product, bool? seed, bool? admin)
{
var row = new Dictionary<string, object>
{
["PartitionKey"] = CpeMappingStore.ProductFlagsPartition,
["RowKey"] = product.Replace(':', '_'),
["Product"] = product,
};
if (seed.HasValue) row["SeedSelfUpdating"] = seed.Value;
if (admin.HasValue) row["AdminSelfUpdating"] = admin.Value;
return row;
}
}
@@ -0,0 +1,79 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeName"/> is the one parser for CPE 2.3 strings: product identity for grouping,
/// the version-less form mappings are keyed by, and the edition and platform fields the version
/// filter reads.
/// </summary>
public class CpeNameTests
{
[Theory]
[InlineData("cpe:2.3:a:7-zip:7-zip:24.08", "7-zip:7-zip")]
[InlineData("cpe:2.3:a:7-zip:7-zip:", "7-zip:7-zip")] // finding without a version
[InlineData("cpe:2.3:a:Microsoft:Teams:1.6.00.4472", "microsoft:teams")]
[InlineData("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*:1.2.3", "contoso:widget")] // custom mapping typed in full
[InlineData(@"cpe:2.3:a:microsoft:visual_c\+\+:14.0", @"microsoft:visual_c\+\+")] // CPE escapes stay part of the key
[InlineData("cpe:2.3:a:*:*:1.0", "")]
[InlineData("cpe:2.3:a", "")]
[InlineData("not a cpe:at:all:really:no", "")]
[InlineData("", "")]
[InlineData(null, "")]
public void ProductKeyOf_IsLowerCaseVendorAndProduct(string? cpe, string expected)
=> Assert.Equal(expected, CpeName.ProductKeyOf(cpe));
[Theory]
[InlineData("cpe:2.3:a:google:chrome", "cpe:2.3:a:google:chrome")]
[InlineData("cpe:2.3:a:Google:Chrome:120.0.1:*:*:*:*:*:*:*", "cpe:2.3:a:google:chrome")]
[InlineData(" cpe:2.3:a:mozilla:firefox:*:*:*:*:esr:*:*:* ", "cpe:2.3:a:mozilla:firefox")]
[InlineData("typed by hand", "typed by hand")]
[InlineData(null, "")]
public void ToProductUri_DropsEverythingAfterTheProduct(string? cpe, string expected)
=> Assert.Equal(expected, CpeName.ToProductUri(cpe));
[Fact]
public void TryParse_ReadsEditionAndPlatformFields()
{
Assert.True(CpeName.TryParse("cpe:2.3:a:mozilla:firefox:115.2.1:*:*:*:esr:android:arm64:*", out var name));
Assert.Equal("a", name.Part);
Assert.Equal("mozilla", name.Vendor);
Assert.Equal("firefox", name.Product);
Assert.Equal("115.2.1", name.Version);
Assert.Equal("esr", name.SwEdition);
Assert.Equal("android", name.TargetSw);
Assert.Equal("arm64", name.TargetHw);
}
[Fact]
public void TryParse_FillsMissingFieldsWithTheWildcard()
{
Assert.True(CpeName.TryParse("cpe:2.3:a:google:chrome", out var name));
Assert.Equal("*", name.Version);
Assert.Equal("*", name.SwEdition);
Assert.Equal("*", name.TargetSw);
}
[Fact]
public void TryParse_KeepsAnEscapedColonInsideItsField()
{
Assert.True(CpeName.TryParse(@"cpe:2.3:a:contoso:suite\:pro:2.0:*:*:*:*:*:*:*", out var name));
Assert.Equal(@"suite\:pro", name.Product);
Assert.Equal("2.0", name.Version);
}
[Fact]
public void IsSameProduct_ComparesVendorAndProductNotThePrefix()
{
Assert.True(CpeName.TryParse("cpe:2.3:a:adobe:acrobat", out var acrobat));
Assert.True(CpeName.TryParse("cpe:2.3:a:adobe:acrobat_reader_dc:23.006.20320:*:*:*:continuous:*:*:*", out var reader));
Assert.True(CpeName.TryParse("cpe:2.3:a:Adobe:Acrobat:20.0:*:*:*:classic:*:*:*", out var acrobatClassic));
Assert.False(acrobat.IsSameProduct(reader));
Assert.True(acrobat.IsSameProduct(acrobatClassic));
}
}
@@ -0,0 +1,321 @@
using System.Globalization;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeProductFlagsService"/> keeps, per product, the default the seed list ships and an
/// operator's own value only while it differs from that default. The row is written in replace
/// mode, so a value is unset by leaving its column out, and a row that would hold neither value
/// is removed.
/// </summary>
public class CpeProductFlagsServiceTests
{
private const string Flags = CpeMappingStore.ProductFlagsPartition;
private const string Widget = "contoso:widget";
private const string Gadget = "fabrikam:gadget";
private const string Operator = "admin@contoso.com";
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget", "contoso widget"));
Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget", "fabrikam gadget"));
Store = new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance);
Service = new CpeProductFlagsService(Tables.Repo.Object, Store);
}
public MappingTables Tables { get; } = new();
public CpeMappingStore Store { get; }
public CpeProductFlagsService Service { get; }
public Dictionary<string, object>? RowOf(string product) => Tables.Row(Flags, product);
public async Task<string[]> SelfUpdatingAsync() =>
(await Store.GetAsync()).SelfUpdatingProducts().OrderBy(p => p, StringComparer.Ordinal).ToArray();
/// <summary>How often the mappings were read from the table.</summary>
public int MappingReads =>
Tables.Repo.Invocations.Count(i => i.Method.Name == nameof(IVulnerabilityRepository.GetAllCpeMappingsAsync));
}
/// <summary>The columns a flags row holds besides its keys.</summary>
private static string[] ColumnsOf(Dictionary<string, object> row) => row.Keys
.Where(column => column is not ("PartitionKey" or "RowKey"))
.OrderBy(column => column, StringComparer.Ordinal)
.ToArray();
// -----------------------------------------------------------------------
// Seed defaults
// -----------------------------------------------------------------------
[Fact]
public async Task ApplySeedDefaults_WritesOneRowPerListedProduct_AndNothingWhenCalledAgain()
{
var rig = new Rig();
var first = await rig.Service.ApplySeedDefaultsAsync(new[] { Widget, Gadget });
var writesAfterFirst = rig.Tables.Writes.ToList();
var second = await rig.Service.ApplySeedDefaultsAsync(new[] { Widget, Gadget });
Assert.Equal(2, first);
Assert.Equal(new[] { Widget, Gadget }, rig.Tables.RowKeys(Flags));
Assert.All(new[] { Widget, Gadget }, product =>
{
var row = rig.RowOf(product)!;
Assert.Equal(new[] { "Product", "SeedSelfUpdating" }, ColumnsOf(row));
Assert.Equal(product, row["Product"]);
Assert.Equal(true, row["SeedSelfUpdating"]);
});
Assert.Equal(0, second);
Assert.Equal(writesAfterFirst, rig.Tables.Writes);
}
[Fact]
public async Task ApplySeedDefaults_KeysAreTrimmedAndLowerCased_AndEmptyOnesSkipped()
{
var rig = new Rig();
var changed = await rig.Service.ApplySeedDefaultsAsync(new[] { " Contoso:Widget ", "contoso:widget", "", " " });
Assert.Equal(1, changed);
Assert.Equal(new[] { Widget }, rig.Tables.RowKeys(Flags));
Assert.Equal(Widget, rig.RowOf(Widget)!["Product"]);
}
[Fact]
public async Task ApplySeedDefaults_DefaultThatNowEqualsTheOperatorsValue_DropsTheOperatorsValue()
{
var rig = new Rig();
await rig.Service.SetSelfUpdatingAsync(Widget, true, Operator);
Assert.Equal(new[] { "AdminSelfUpdating", "Product", "UpdatedAt", "UpdatedBy" }, ColumnsOf(rig.RowOf(Widget)!));
var changed = await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
// The product follows the default again, so a later change of the default reaches it.
Assert.Equal(1, changed);
Assert.Equal(new[] { "Product", "SeedSelfUpdating" }, ColumnsOf(rig.RowOf(Widget)!));
Assert.Equal(new[] { Widget }, await rig.SelfUpdatingAsync());
}
[Fact]
public async Task ApplySeedDefaults_ProductLeftTheList_LosesItsRow_WhateverTheOperatorSaidAboutTheDefault()
{
var rig = new Rig();
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget, Gadget });
// The operator switched the default off for the widget; with the default gone, "off" says nothing.
await rig.Service.SetSelfUpdatingAsync(Widget, false, Operator);
var changed = await rig.Service.ApplySeedDefaultsAsync(Array.Empty<string>());
Assert.Equal(2, changed);
Assert.Empty(rig.Tables.Rows(Flags));
Assert.Empty(await rig.SelfUpdatingAsync());
}
[Fact]
public async Task ApplySeedDefaults_LeavesAnOperatorsOwnFlagAlone()
{
var rig = new Rig();
await rig.Service.SetSelfUpdatingAsync(Gadget, true, Operator);
var before = rig.RowOf(Gadget)!;
var writes = rig.Tables.Writes.Count;
var changed = await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
// Only the widget's default was written: the gadget is flagged by the operator, not by the list.
Assert.Equal(1, changed);
Assert.Equal(writes + 1, rig.Tables.Writes.Count);
Assert.Equal(before, rig.RowOf(Gadget));
Assert.Equal(new[] { Widget, Gadget }, await rig.SelfUpdatingAsync());
}
// -----------------------------------------------------------------------
// The operator's value
// -----------------------------------------------------------------------
[Fact]
public async Task SetSelfUpdating_AwayFromTheDefault_StoresTheOperatorsValueNextToIt()
{
var rig = new Rig();
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
var before = DateTime.UtcNow;
var flags = await rig.Service.SetSelfUpdatingAsync(Widget, false, Operator);
var row = rig.RowOf(Widget)!;
Assert.Equal(new[] { "AdminSelfUpdating", "Product", "SeedSelfUpdating", "UpdatedAt", "UpdatedBy" }, ColumnsOf(row));
Assert.Equal(true, row["SeedSelfUpdating"]);
Assert.Equal(false, row["AdminSelfUpdating"]);
Assert.Equal(Operator, row["UpdatedBy"]);
var updatedAt = DateTime.Parse((string)row["UpdatedAt"], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
Assert.InRange(updatedAt, before, DateTime.UtcNow);
Assert.Equal(Widget, flags.Product);
Assert.False(flags.SelfUpdating);
Assert.True(flags.SeedSelfUpdating);
Assert.False(flags.AdminSelfUpdating);
Assert.Equal(Operator, flags.UpdatedBy);
Assert.Equal(row["UpdatedAt"], flags.UpdatedAt);
}
[Fact]
public async Task SetSelfUpdating_BackToTheDefault_LeavesOnlyTheDefaultOnTheRow()
{
var rig = new Rig();
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
await rig.Service.SetSelfUpdatingAsync(Widget, false, Operator);
var flags = await rig.Service.SetSelfUpdatingAsync(Widget, true, Operator);
// A merge would have kept the operator's columns; the row is replaced without them.
var row = rig.RowOf(Widget)!;
Assert.Equal(new[] { "Product", "SeedSelfUpdating" }, ColumnsOf(row));
Assert.Equal(true, row["SeedSelfUpdating"]);
Assert.True(flags.SelfUpdating);
Assert.True(flags.SeedSelfUpdating);
Assert.Null(flags.AdminSelfUpdating);
Assert.Equal(string.Empty, flags.UpdatedBy);
Assert.Equal(string.Empty, flags.UpdatedAt);
}
[Fact]
public async Task SetSelfUpdating_ProductTheSeedDoesNotList_StoresOnlyTheOperatorsValue_AndUnflaggingRemovesTheRow()
{
var rig = new Rig();
var flagged = await rig.Service.SetSelfUpdatingAsync(Gadget, true, Operator);
var row = rig.RowOf(Gadget)!;
var unflagged = await rig.Service.SetSelfUpdatingAsync(Gadget, false, Operator);
Assert.Equal(new[] { "AdminSelfUpdating", "Product", "UpdatedAt", "UpdatedBy" }, ColumnsOf(row));
Assert.Equal(true, row["AdminSelfUpdating"]);
Assert.True(flagged.SelfUpdating);
Assert.Null(flagged.SeedSelfUpdating);
Assert.True(flagged.AdminSelfUpdating);
Assert.Empty(rig.Tables.Rows(Flags));
Assert.Equal($"delete {Flags}/{Gadget}", rig.Tables.Writes[^1]);
Assert.False(unflagged.SelfUpdating);
Assert.Null(unflagged.SeedSelfUpdating);
Assert.Null(unflagged.AdminSelfUpdating);
}
[Fact]
public async Task SetSelfUpdating_ToTheValueItAlreadyHas_KeepsTheRowAsItIs()
{
var rig = new Rig();
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
var flags = await rig.Service.SetSelfUpdatingAsync(Widget, true, Operator);
Assert.True(flags.SelfUpdating);
Assert.Equal(new[] { "Product", "SeedSelfUpdating" }, ColumnsOf(rig.RowOf(Widget)!));
}
[Theory]
[InlineData("contoso:gadget")] // vendor and product of two different mappings
[InlineData("widget")]
[InlineData("")]
[InlineData(" ")]
public async Task SetSelfUpdating_ProductNoMappingPointsTo_ThrowsAndWritesNothing(string product)
{
var rig = new Rig();
await Assert.ThrowsAsync<ArgumentException>(() => rig.Service.SetSelfUpdatingAsync(product, true, Operator));
Assert.Empty(rig.Tables.Writes);
Assert.Empty(rig.Tables.Rows(Flags));
}
[Fact]
public async Task SetSelfUpdating_ProductKeyIsTrimmedAndLowerCased()
{
var rig = new Rig();
var flags = await rig.Service.SetSelfUpdatingAsync(" Contoso:WIDGET ", true, Operator);
Assert.Equal(Widget, flags.Product);
Assert.Equal(new[] { Widget }, rig.Tables.RowKeys(Flags));
Assert.Equal(Widget, rig.RowOf(Widget)!["Product"]);
Assert.Equal(new[] { Widget }, await rig.SelfUpdatingAsync());
}
[Fact]
public async Task SetSelfUpdating_MappingsUnreadable_IsUnavailableAndWritesNothing()
{
var rig = new Rig();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Service.SetSelfUpdatingAsync(Widget, true, Operator));
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task SetSelfUpdating_ProductMappedAfterThisInstanceReadTheMappings_IsFoundRightAway()
{
var rig = new Rig();
await rig.Store.GetAsync();
// Saved on another instance a moment ago; this one's copy is good for minutes yet.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(
CpeMappingStore.CustomPartition, "manual", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool"));
var flags = await rig.Service.SetSelfUpdatingAsync("northwind:tool", true, Operator);
Assert.True(flags.SelfUpdating);
Assert.Equal(true, rig.RowOf("northwind:tool")!["AdminSelfUpdating"]);
Assert.Equal(new[] { "northwind:tool" }, await rig.SelfUpdatingAsync());
}
[Theory]
// A product this instance knows is taken from the copy it holds.
[InlineData(Widget, false)]
// Before a product is refused, the mappings are read once more, and only once.
[InlineData("tailspin:launcher", true)]
public async Task SetSelfUpdating_ReadsTheMappingsAgain_OnlyBeforeItRefusesAProduct(string product, bool refused)
{
var rig = new Rig();
await rig.Store.GetAsync();
var readsBefore = rig.MappingReads;
var error = await Record.ExceptionAsync(() => rig.Service.SetSelfUpdatingAsync(product, true, Operator));
Assert.Equal(refused, error is ArgumentException);
Assert.Equal(readsBefore + (refused ? 1 : 0), rig.MappingReads);
}
// -----------------------------------------------------------------------
// What the matching sees
// -----------------------------------------------------------------------
[Fact]
public async Task EffectiveValue_SeenThroughTheMappingStore_FollowsEveryChange()
{
var rig = new Rig();
Assert.Empty(await rig.SelfUpdatingAsync());
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
Assert.Equal(new[] { Widget }, await rig.SelfUpdatingAsync());
Assert.True((await rig.Store.GetAsync()).IsSelfUpdating(Widget));
Assert.False((await rig.Store.GetAsync()).IsSelfUpdating(Gadget));
await rig.Service.SetSelfUpdatingAsync(Widget, false, Operator);
Assert.Empty(await rig.SelfUpdatingAsync());
await rig.Service.SetSelfUpdatingAsync(Gadget, true, Operator);
Assert.Equal(new[] { Gadget }, await rig.SelfUpdatingAsync());
await rig.Service.SetSelfUpdatingAsync(Widget, true, Operator);
Assert.Equal(new[] { Widget, Gadget }, await rig.SelfUpdatingAsync());
// The widget loses its default, the gadget keeps what the operator set.
await rig.Service.ApplySeedDefaultsAsync(Array.Empty<string>());
Assert.Equal(new[] { Gadget }, await rig.SelfUpdatingAsync());
}
}
@@ -1,24 +0,0 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeMatchScorer.ProductKey"/> turns a finding's CPE ("base CPE:installed version")
/// into the product identity the exposure summary groups by.
/// </summary>
public class CpeProductKeyTests
{
[Theory]
[InlineData("cpe:2.3:a:7-zip:7-zip:24.08", "7-zip:7-zip")]
[InlineData("cpe:2.3:a:7-zip:7-zip:", "7-zip:7-zip")] // finding without a version
[InlineData("cpe:2.3:a:Microsoft:Teams:1.6.00.4472", "microsoft:teams")]
[InlineData("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*:1.2.3", "contoso:widget")] // custom mapping typed in full
[InlineData(@"cpe:2.3:a:microsoft:visual_c\+\+:14.0", @"microsoft:visual_c\+\+")] // CPE escapes stay part of the key
[InlineData("cpe:2.3:a:*:*:1.0", "")]
[InlineData("cpe:2.3:a", "")]
[InlineData("", "")]
[InlineData(null, "")]
public void ProductKey_IsLowerCaseVendorAndProduct(string? cpe, string expected)
=> Assert.Equal(expected, CpeMatchScorer.ProductKey(cpe));
}
@@ -0,0 +1,450 @@
using System.Globalization;
using System.Text.Json;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CpeSeedFile"/> decides whether a mapping list may replace the mappings in service.
/// Every reason to refuse one is a problem in the result, never an exception, and a list with a
/// problem is not imported at all. The same check runs here over the committed community list.
/// </summary>
public class CpeSeedFileTests
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private static readonly JsonSerializerOptions FileJson = new(JsonSerializerDefaults.Web);
private static CpeSeedEntry Entry(string? vendor, string? product, string? cpeUri) =>
new() { NormalizedVendor = vendor, NormalizedProduct = product, CpeUri = cpeUri };
private static CpeSeedEntry Widget() => Entry("contoso", "contoso widget", WidgetCpe);
/// <summary><paramref name="count"/> entries of different programs, each with a product of its own.</summary>
private static CpeSeedEntry[] Entries(int count) => Enumerable.Range(0, count)
.Select(n => Entry("contoso", $"contoso tool {(char)('a' + n)}", $"cpe:2.3:a:contoso:tool_{(char)('a' + n)}"))
.ToArray();
private static string List(IEnumerable<CpeSeedEntry?> mappings, params string[] selfUpdatingProducts) =>
JsonSerializer.Serialize(new CpeSeedDocument
{
Version = "2.0.0",
LastUpdated = "2026-11-12",
SelfUpdatingProducts = selfUpdatingProducts.Length == 0 ? null : selfUpdatingProducts.ToList(),
Mappings = mappings.ToList()!,
}, FileJson);
private static List<string> ProblemsOf(params CpeSeedEntry?[] mappings) =>
CpeSeedFile.Read(List(mappings), mappingsInService: 0).Problems;
private static void AssertOnlyProblem(string expectedPart, List<string> problems) =>
Assert.Contains(expectedPart, Assert.Single(problems));
// -----------------------------------------------------------------------
// The list as a whole
// -----------------------------------------------------------------------
[Fact]
public void Read_WellFormedList_HasNoProblemsAndReturnsTheDocument()
{
// Written like the file in the rules repository: camelCase, a comment, a trailing comma,
// and the two CPE columns the earlier format carried next to the CPE itself.
const string json = """
{
"version": "2.0.0",
"lastUpdated": "2026-11-12",
// products that update themselves unless an operator says otherwise
"selfUpdatingProducts": ["contoso:widget"],
"mappings": [
{
"normalizedVendor": "contoso",
"normalizedProduct": "contoso widget",
"displayNamePatterns": ["contoso widget", "widget"],
"publisherPatterns": ["contoso", "contoso ltd"],
"excludePatterns": ["widget viewer"],
"cpeVendor": "contoso",
"cpeProduct": "widget",
"cpeUri": "cpe:2.3:a:contoso:widget",
"category": "utility",
"edition": "esr",
"excludeEditions": ["beta", "preview_2"],
},
{
"normalizedVendor": "fabrikam",
"normalizedProduct": "fabrikam gadget++",
"displayNamePatterns": ["fabrikam gadget++"],
"cpeUri": "cpe:2.3:a:fabrikam:gadget\\+\\+",
},
],
}
""";
var (document, problems) = CpeSeedFile.Read(json, mappingsInService: 2);
Assert.Empty(problems);
Assert.Equal("2.0.0", document!.Version);
Assert.Equal(new[] { "contoso:widget" }, document.SelfUpdatingProducts);
Assert.Equal(2, document.Mappings!.Count);
var widget = document.Mappings[0];
Assert.Equal("contoso widget", widget.NormalizedProduct);
Assert.Equal(new[] { "contoso widget", "widget" }, widget.DisplayNamePatterns);
Assert.Equal(new[] { "widget viewer" }, widget.ExcludePatterns);
Assert.Equal("esr", widget.Edition);
Assert.Equal(new[] { "beta", "preview_2" }, widget.ExcludeEditions);
Assert.Equal(@"cpe:2.3:a:fabrikam:gadget\+\+", document.Mappings[1].CpeUri);
}
[Theory]
[InlineData("{ \"mappings\": [ { \"normalizedVendor\": ")]
[InlineData("<html>maintenance</html>")]
[InlineData("[]")]
[InlineData("{ \"mappings\": \"none\" }")]
[InlineData("{ \"mappings\": [ 4 ] }")]
public void Read_TextThatIsNotAList_IsRefusedAsInvalidJson(string json)
{
var (document, problems) = CpeSeedFile.Read(json, mappingsInService: 0, mayBeEmpty: true);
Assert.Null(document);
AssertOnlyProblem("not valid JSON", problems);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" \r\n ")]
[InlineData("null")]
public void Read_NoTextAtAll_IsRefusedEvenWhereTheListMayBeEmpty(string? json)
{
var (document, problems) = CpeSeedFile.Read(json, mappingsInService: 0, mayBeEmpty: true);
Assert.Null(document);
AssertOnlyProblem("holds no mapping", problems);
}
[Theory]
[InlineData("{ \"version\": \"2.0.0\" }")]
[InlineData("{ \"version\": \"2.0.0\", \"mappings\": null }")]
[InlineData("{ \"version\": \"2.0.0\", \"mappings\": [] }")]
public void Read_ListWithoutMappings_IsAcceptedOnlyWhereItMayBeEmptyAndNothingIsInService(string json)
{
var asSeed = CpeSeedFile.Read(json, mappingsInService: 0);
var asCommunityListWithNothingInService = CpeSeedFile.Read(json, mappingsInService: 0, mayBeEmpty: true);
var asCommunityListThatWouldRemoveMappings = CpeSeedFile.Read(json, mappingsInService: 1, mayBeEmpty: true);
AssertOnlyProblem("holds no mapping", asSeed.Problems);
Assert.Empty(asCommunityListWithNothingInService.Problems);
Assert.Equal("2.0.0", asCommunityListWithNothingInService.Document!.Version);
AssertOnlyProblem("holds no mapping", asCommunityListThatWouldRemoveMappings.Problems);
}
[Fact]
public void Read_ListThatLostMoreThanTheAllowedShare_IsRefused_TheShareItselfIsAccepted()
{
const int inService = 10;
var smallestAccepted = (int)Math.Ceiling(inService * (1 - CpeSeedFile.MaxShrink));
var atTheLimit = CpeSeedFile.Read(List(Entries(smallestAccepted)), inService);
var belowIt = CpeSeedFile.Read(List(Entries(smallestAccepted - 1)), inService);
var withNothingInService = CpeSeedFile.Read(List(Entries(1)), mappingsInService: 0);
Assert.Equal(5, smallestAccepted);
Assert.Empty(atTheLimit.Problems);
AssertOnlyProblem("shrinks", belowIt.Problems);
Assert.Contains("4 mappings", belowIt.Problems[0]);
Assert.Contains("10", belowIt.Problems[0]);
// The first import has nothing to compare with.
Assert.Empty(withNothingInService.Problems);
}
// -----------------------------------------------------------------------
// One entry
// -----------------------------------------------------------------------
[Theory]
[InlineData("cpe:2.3:a:contoso:widget:4.2")]
[InlineData("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*")]
[InlineData("cpe:2.3:a:Contoso:Widget")]
[InlineData(" cpe:2.3:a:contoso:widget")]
public void Read_CpeWithAVersionPartOrUpperCase_IsRefusedWithTheFormToWrite(string cpeUri)
{
var problems = ProblemsOf(Entry("contoso", "contoso widget", cpeUri));
AssertOnlyProblem($"write the CPE as '{WidgetCpe}'", problems);
Assert.StartsWith("mapping 1 (contoso:contoso widget)", problems[0]);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData("contoso widget")]
[InlineData("cpe:2.3:a:contoso")]
[InlineData("cpe:2.3:a:contoso:*")]
[InlineData("cpe:2.3:a:-:widget")]
public void Read_CpeThatNamesNoProduct_IsRefused(string? cpeUri) =>
AssertOnlyProblem("names no CPE product", ProblemsOf(Entry("contoso", "contoso widget", cpeUri)));
[Theory]
[InlineData("cpe:2.3:a:fabrikam:gadget++")]
[InlineData("cpe:2.3:a:fabrikam:gadget%2b%2b")]
[InlineData(@"cpe:2.3:a:fabrikam:gadget\+\+")]
public void Read_PlusSignInAnyOfItsSpellings_IsTheSameProduct(string cpeUri)
{
var (_, problems) = CpeSeedFile.Read(
List(new[] { Entry("fabrikam", "fabrikam gadget++", cpeUri) }, @"fabrikam:gadget\+\+"), mappingsInService: 0);
Assert.Empty(problems);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
[InlineData("4.2.1 (x64)")] // nothing is left of it once version and architecture are gone
public void Read_EntryWithoutAProgramName_IsRefused(string? product) =>
AssertOnlyProblem("normalizedProduct is missing", ProblemsOf(Entry("contoso", product, WidgetCpe)));
[Theory]
[InlineData("contoso", "Contoso Widget 4.2 (x64)")] // the same program with its version
[InlineData("Contoso", "CONTOSO WIDGET")]
[InlineData(" contoso ", "contoso widget")]
public void Read_TwoEntriesForTheSameProgram_AreRefused(string vendor, string product)
{
var problems = ProblemsOf(Widget(), Entry(vendor, product, "cpe:2.3:a:contoso:widget_pro"));
AssertOnlyProblem("occur twice", problems);
Assert.StartsWith("mapping 2 ", problems[0]);
}
[Fact]
public void Read_EntriesWithoutAVendor_ShareOneKeyWhetherItIsEmptyOrThePlaceholder()
{
var problems = ProblemsOf(Entry("unknown", "contoso widget", WidgetCpe), Entry(null, "contoso widget", WidgetCpe));
AssertOnlyProblem("occur twice", problems);
}
[Fact]
public void Read_TwoEntriesThatWouldShareATableRow_AreRefused()
{
// '#' and '?' are part of a program's name, but a row key cannot hold either: both
// become '_', and the second entry would overwrite the first.
const string first = "widget#pro", second = "widget?pro";
Assert.NotEqual(CpeMappingMatcher.SoftwareKey("contoso", first), CpeMappingMatcher.SoftwareKey("contoso", second));
Assert.Equal(CpeSeedFile.RowKeyOf("contoso", first), CpeSeedFile.RowKeyOf("contoso", second));
var problems = ProblemsOf(
Entry("contoso", first, "cpe:2.3:a:contoso:widget_pro"), Entry("contoso", second, "cpe:2.3:a:contoso:widget_pro_2"));
AssertOnlyProblem("row of another mapping", problems);
Assert.StartsWith("mapping 2 (contoso:widget?pro)", problems[0]);
}
[Theory]
[InlineData("--")]
[InlineData("")]
[InlineData("(®)")]
public void Read_PatternWithoutAWord_IsRefused_WhetherItSelectsOrExcludes(string pattern)
{
var selecting = Widget();
selecting.DisplayNamePatterns = new List<string> { "contoso widget", pattern };
var excluding = Widget();
excluding.ExcludePatterns = new List<string> { pattern };
AssertOnlyProblem($"the pattern '{pattern}' holds no word", ProblemsOf(selecting));
AssertOnlyProblem($"the pattern '{pattern}' holds no word", ProblemsOf(excluding));
}
[Theory]
[InlineData("ESR")]
[InlineData("long term")]
[InlineData("")]
public void Read_EditionThatIsNotLowerCaseWithoutSpaces_IsRefused(string edition)
{
var only = Widget();
only.Edition = edition;
var excluded = Widget();
excluded.ExcludeEditions = new List<string> { "beta", edition };
AssertOnlyProblem($"'{edition}' is not a CPE software edition", ProblemsOf(only));
AssertOnlyProblem($"'{edition}' is not a CPE software edition", ProblemsOf(excluded));
}
[Fact]
public void Read_EditionsInCpeForm_AreAccepted()
{
var entry = Widget();
entry.Edition = "esr";
entry.ExcludeEditions = new List<string> { "lts_2021", "x.y-z", "7" };
Assert.Empty(ProblemsOf(entry));
}
[Fact]
public void Read_SelfUpdatingProductNoMappingPointsTo_IsRefused()
{
var known = CpeSeedFile.Read(List(new[] { Widget() }, " Contoso:Widget "), mappingsInService: 0);
var unknown = CpeSeedFile.Read(List(new[] { Widget() }, "contoso:widget", "contoso:gadget"), mappingsInService: 0);
// Compared the way the flags are keyed: trimmed, lower case.
Assert.Empty(known.Problems);
AssertOnlyProblem("selfUpdatingProducts names 'contoso:gadget'", unknown.Problems);
}
[Fact]
public void Read_ReportsEveryProblemOfAList_AndAnEmptyEntry()
{
var noWordPattern = Entry("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget");
noWordPattern.DisplayNamePatterns = new List<string> { "..." };
var problems = ProblemsOf(Widget(), null, Entry("contoso", "", "cpe:2.3:a:contoso"), noWordPattern);
Assert.Equal(4, problems.Count);
Assert.Equal("mapping 2 is empty", problems[0]);
Assert.Contains("names no CPE product", problems[1]);
Assert.Contains("normalizedProduct is missing", problems[2]);
Assert.Contains("holds no word", problems[3]);
}
// -----------------------------------------------------------------------
// Rows
// -----------------------------------------------------------------------
[Fact]
public void ToRows_BuildsOneTableRowPerEntry()
{
var document = new CpeSeedDocument
{
Mappings = new List<CpeSeedEntry>
{
new()
{
NormalizedVendor = "Contoso",
NormalizedProduct = "Contoso Widget",
CpeUri = "cpe:2.3:a:Contoso:Widget:*:*:*:*:*:*:*:*",
Category = "utility",
DisplayNamePatterns = new List<string> { "contoso widget", "widget" },
PublisherPatterns = new List<string> { "contoso ltd" },
ExcludePatterns = new List<string> { "widget viewer" },
Edition = " ESR ",
ExcludeEditions = new List<string> { " Beta ", "preview" },
},
new() { NormalizedVendor = "fabrikam", NormalizedProduct = "gadget#pro", CpeUri = "cpe:2.3:a:fabrikam:gadget%2b%2b" },
},
};
var before = DateTime.UtcNow;
var rows = CpeSeedFile.ToRows(document, "community");
Assert.Equal(2, rows.Count);
var (widgetKey, widget) = rows[0];
Assert.Equal(CpeSeedFile.RowKeyOf("Contoso", "Contoso Widget"), widgetKey);
Assert.Equal("contoso:contoso widget", widgetKey);
Assert.Equal("Contoso", widget["NormalizedVendor"]);
Assert.Equal("Contoso Widget", widget["NormalizedProduct"]);
Assert.Equal(WidgetCpe, widget["CpeUri"]);
Assert.Equal("contoso", widget["CpeVendor"]);
Assert.Equal("widget", widget["CpeProduct"]);
Assert.Equal("utility", widget["Category"]);
Assert.Equal("""["contoso widget","widget"]""", widget["DisplayNamePatternsJson"]);
Assert.Equal("""["contoso ltd"]""", widget["PublisherPatternsJson"]);
Assert.Equal("""["widget viewer"]""", widget["ExcludePatternsJson"]);
Assert.Equal("community", widget["Source"]);
Assert.Equal("esr", widget["Edition"]);
Assert.Equal("""["beta","preview"]""", widget["ExcludeEditionsJson"]);
var importedAt = DateTime.Parse((string)widget["ImportedAt"], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
Assert.InRange(importedAt, before, DateTime.UtcNow);
var (gadgetKey, gadget) = rows[1];
Assert.Equal("fabrikam:gadget_pro", gadgetKey);
Assert.Equal(@"cpe:2.3:a:fabrikam:gadget\+\+", gadget["CpeUri"]);
Assert.Equal("fabrikam", gadget["CpeVendor"]);
Assert.Equal(@"gadget\+\+", gadget["CpeProduct"]);
Assert.Equal(string.Empty, gadget["Category"]);
Assert.Equal("[]", gadget["DisplayNamePatternsJson"]);
Assert.Equal("[]", gadget["PublisherPatternsJson"]);
Assert.Equal("[]", gadget["ExcludePatternsJson"]);
// An edition column is only written for an entry that restricts editions.
Assert.False(gadget.ContainsKey("Edition"));
Assert.False(gadget.ContainsKey("ExcludeEditionsJson"));
}
[Fact]
public void ToRows_RowsAreWhatTheMappingStoreReadsBack()
{
var entry = Widget();
entry.DisplayNamePatterns = new List<string> { "contoso widget" };
entry.PublisherPatterns = new List<string> { "contoso ltd" };
entry.ExcludePatterns = new List<string> { "widget viewer" };
entry.Edition = "esr";
entry.ExcludeEditions = new List<string> { "beta" };
var (rowKey, properties) = Assert.Single(CpeSeedFile.ToRows(new CpeSeedDocument { Mappings = new List<CpeSeedEntry> { entry } }, "seed"));
var stored = new Dictionary<string, object>(properties) { ["PartitionKey"] = CpeMappingStore.SeedPartition, ["RowKey"] = rowKey };
var mapping = CpeMappingStore.MappingFromRow(stored)!;
Assert.Equal(CpeMappingSource.Seed, mapping.Source);
Assert.Equal(WidgetCpe, mapping.CpeUri);
Assert.Equal("contoso:widget", mapping.ProductKey);
Assert.Equal(new[] { "contoso widget" }, mapping.DisplayNamePatterns);
Assert.Equal(new[] { "contoso ltd" }, mapping.PublisherPatterns);
Assert.Equal(new[] { "widget viewer" }, mapping.ExcludePatterns);
Assert.Equal("esr", mapping.Edition);
Assert.Equal(new[] { "beta" }, mapping.ExcludeEditions);
Assert.Equal(properties["ImportedAt"], mapping.CreatedAt);
}
[Theory]
[InlineData("Contoso", "Contoso Widget", "contoso:contoso widget")]
[InlineData(null, "Contoso Widget", "unknown:contoso widget")]
[InlineData("", "Contoso Widget", "unknown:contoso widget")]
[InlineData("contoso", null, "contoso:unknown")]
[InlineData("contoso/emea", @"c:\tools\widget#1?", "contoso_emea:c:_tools_widget_1_")]
public void RowKeyOf_IsVendorAndProductInLowerCase_WithoutWhatAKeyCannotHold(string? vendor, string? product, string expected) =>
Assert.Equal(expected, CpeSeedFile.RowKeyOf(vendor, product));
[Fact]
public void RowKeyOf_IsCutAtTheLengthAKeyMayHave()
{
var rowKey = CpeSeedFile.RowKeyOf("contoso", new string('w', 600));
Assert.Equal(512, rowKey.Length);
Assert.StartsWith("contoso:www", rowKey);
}
// -----------------------------------------------------------------------
// The committed list
// -----------------------------------------------------------------------
[Fact]
public void CommittedCommunityList_PassesTheCheckAnImportRuns()
{
var json = File.ReadAllText(Path.Combine(FindRepoRoot(), "rules", "dist", "cpe-community-mappings.json"));
var (document, problems) = CpeSeedFile.Read(json, mappingsInService: 0, mayBeEmpty: true);
Assert.True(problems.Count == 0, string.Join(Environment.NewLine, problems));
Assert.NotNull(document);
}
[Fact]
public void CommittedSeedList_PassesTheCheckAnImportRuns_AndHoldsTheWholeList()
{
var json = File.ReadAllText(Path.Combine(FindRepoRoot(), "rules", "dist", "cpe-mappings.json"));
var (document, problems) = CpeSeedFile.Read(json, mappingsInService: 0);
Assert.True(problems.Count == 0, string.Join(Environment.NewLine, problems));
// A list cut short by a merge accident would still pass the check above.
var mappings = document!.Mappings!.Count;
Assert.True(mappings >= 400, $"The committed list holds {mappings} mappings.");
}
private static string FindRepoRoot()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir != null && !File.Exists(Path.Combine(dir.FullName, "AutopilotMonitor.sln")))
dir = dir.Parent;
return dir?.FullName ?? throw new InvalidOperationException("Repository root (AutopilotMonitor.sln) not found.");
}
}
@@ -203,4 +203,72 @@ public class CveCacheBudgetTests
Assert.Equal(0, r.TotalIn);
Assert.Empty(r.Cves);
}
// -----------------------------------------------------------------------
// Known-exploited CVEs and the cap
// -----------------------------------------------------------------------
private static CachedCve Exploited(string id, double score, DateTime publishedUtc)
{
var cve = Cve(id, score, publishedUtc);
cve.KevAdded = "2026-03-01";
return cve;
}
[Fact]
public void Apply_KnownExploitedCves_SurviveTheCapAheadOfHigherScores()
{
// More critical CVEs in the window than a row holds: by score alone the three exploited
// ones would be the first to go.
var cves = Enumerable.Range(1, 105)
.Select(i => Cve($"CVE-Critical-{i:000}", 9.8, Now.AddMonths(-2)))
.Concat(new[]
{
Exploited("CVE-Kev-1", 5.0, Now.AddMonths(-3)),
Exploited("CVE-Kev-2", 5.0, Now.AddMonths(-10)),
Exploited("CVE-Kev-3", 5.0, Now.AddMonths(-20)),
})
.ToList();
var r = CveCacheBudget.Apply(cves, Now);
Assert.Equal(108, r.KeptAfterDateFilter);
Assert.Equal(CveCacheBudget.MaxCachedCves, r.Cves.Count);
Assert.Equal(new[] { "CVE-Kev-1", "CVE-Kev-2", "CVE-Kev-3" }, r.Cves.Take(3).Select(c => c.CveId));
Assert.All(r.Cves.Skip(3), c => Assert.Equal(9.8, c.CvssScore));
}
[Fact]
public void Apply_KnownExploitedCveOutsideTheDateWindow_IsStillDropped()
{
var cves = new[]
{
Exploited("CVE-Kev-OldMedium", 5.0, Now.AddMonths(-30)), // below HIGH: 24 months
Exploited("CVE-Kev-OldCritical", 9.8, Now.AddMonths(-61)), // HIGH and above: 60 months
Exploited("CVE-Kev-OldHigh", 7.5, Now.AddMonths(-40)),
};
var r = CveCacheBudget.Apply(cves, Now);
Assert.Equal("CVE-Kev-OldHigh", Assert.Single(r.Cves).CveId);
}
[Fact]
public void Apply_OrderAmongTheOtherCves_StaysScoreThenDate()
{
var cves = new[]
{
Cve("CVE-Medium", 5.0, Now.AddMonths(-1)),
Cve("CVE-High-Older", 8.0, Now.AddMonths(-12)),
Exploited("CVE-Kev-Low", 3.1, Now.AddMonths(-6)),
Cve("CVE-Critical", 9.8, Now.AddMonths(-20)),
Cve("CVE-High-Newer", 8.0, Now.AddMonths(-1)),
};
var r = CveCacheBudget.Apply(cves, Now);
Assert.Equal(
new[] { "CVE-Kev-Low", "CVE-Critical", "CVE-High-Newer", "CVE-High-Older", "CVE-Medium" },
r.Cves.Select(c => c.CveId));
}
}
@@ -0,0 +1,463 @@
using System.Globalization;
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CveCacheFiller"/> is the one path from an NVD answer to a product's cache row.
/// Pinned here: a row holds only the product's own version ranges, an answer NVD did not give is
/// never written, a marker for missing data never overwrites a row, and a rewrite hands back the
/// columns the replace-mode write would otherwise lose.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class CveCacheFillerTests : IDisposable
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private static readonly string WidgetKey = CveCacheFiller.CacheKey(WidgetCpe);
public CveCacheFillerTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private static NvdCpeMatch Match(string criteria, string? endExcluding = null, bool vulnerable = true) => new()
{
Vulnerable = vulnerable,
Criteria = criteria,
VersionEndExcluding = endExcluding,
};
private static NvdCveResponse Response(params NvdVulnerabilityItem[] items) =>
new() { TotalResults = items.Length, Vulnerabilities = items.ToList() };
private static CveCacheFiller Filler(
Mock<IVulnerabilityRepository> repo, Func<NvdRequest, HttpResponseMessage> nvdAnswer, EpssApiClient? epss = null) =>
new(repo.Object, NvdFixture.Client(new NvdStubHandler(nvdAnswer)), NullLogger<CveCacheFiller>.Instance, epss);
// -----------------------------------------------------------------------
// Project
// -----------------------------------------------------------------------
[Fact]
public void Project_KeepsOnlyTheRangesOfTheProductItself()
{
// A prefix compare on the string would take acrobat_reader_dc and acrobat_dc for acrobat.
var response = Response(
NvdFixture.Cve("CVE-2025-0001", 7.8,
Match("cpe:2.3:a:adobe:acrobat:*:*:*:*:*:*:*:*", "24.0"),
Match("cpe:2.3:a:adobe:acrobat_reader_dc:*:*:*:*:continuous:*:*:*", "25.0"),
Match("cpe:2.3:a:adobe:acrobat_dc:*:*:*:*:continuous:*:*:*", "25.0"),
Match("cpe:2.3:o:microsoft:windows:-:*:*:*:*:*:*:*")),
NvdFixture.Cve("CVE-2025-0002", 9.8,
Match("cpe:2.3:a:adobe:acrobat_reader_dc:*:*:*:*:continuous:*:*:*", "25.0")));
var cves = CveCacheFiller.Project(response, "cpe:2.3:a:adobe:acrobat");
var kept = Assert.Single(cves);
Assert.Equal("CVE-2025-0001", kept.CveId);
var range = Assert.Single(kept.AffectedVersions);
Assert.Equal("cpe:2.3:a:adobe:acrobat:*:*:*:*:*:*:*:*", range.Criteria);
Assert.Equal("24.0", range.VersionEndExcluding);
}
[Fact]
public void Project_DropsOwnRangesThatAreNotVulnerable_AndACveLeftWithoutOne()
{
var response = Response(
NvdFixture.Cve("CVE-2025-0003", 7.5,
Match("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "3.0"),
Match("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "9.0", vulnerable: false)),
NvdFixture.Cve("CVE-2025-0004", 7.5,
Match("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "9.0", vulnerable: false)));
var kept = Assert.Single(CveCacheFiller.Project(response, WidgetCpe));
Assert.Equal("CVE-2025-0003", kept.CveId);
Assert.Equal("3.0", Assert.Single(kept.AffectedVersions).VersionEndExcluding);
}
/// <summary>
/// Shaped like the public CVE-2024-3661: one CVE names two products and gives a version range
/// for the other one only. The first product's row must not inherit that range.
/// </summary>
[Fact]
public void Project_CveNamingTwoProducts_KeepsOnlyTheOwnRange()
{
const string otherCpe = "cpe:2.3:a:fabrikam:secure_client";
var response = Response(NvdFixture.Cve("CVE-2024-3661", 7.6,
Match("cpe:2.3:a:contoso:widget:-:*:*:*:*:*:*:*"),
Match("cpe:2.3:a:fabrikam:secure_client:*:*:*:*:*:*:*:*", "24.06.1")));
var own = Assert.Single(CveCacheFiller.Project(response, WidgetCpe));
var other = Assert.Single(CveCacheFiller.Project(response, otherCpe));
var ownRange = Assert.Single(own.AffectedVersions);
Assert.Equal("cpe:2.3:a:contoso:widget:-:*:*:*:*:*:*:*", ownRange.Criteria);
Assert.Null(ownRange.VersionEndExcluding);
Assert.Equal("24.06.1", Assert.Single(other.AffectedVersions).VersionEndExcluding);
// Read back through the filter, the widget is not judged by the other product's range.
Assert.Empty(CveVersionFilter.Filter(new[] { own }, WidgetCpe, "3.2.0").Vulnerabilities);
Assert.Single(CveVersionFilter.Filter(new[] { other }, otherCpe, "3.2.0").Vulnerabilities);
}
[Theory]
[InlineData("2023-09-13", "2023-09-13")]
[InlineData(null, null)]
[InlineData(" ", null)]
public void Project_TakesTheExploitedDateFromNvd(string? cisaExploitAdd, string? expected)
{
var item = NvdFixture.Cve("CVE-2025-0005", 8.8, NvdFixture.Below(WidgetCpe, "2.0"));
item.Cve!.CisaExploitAdd = cisaExploitAdd;
Assert.Equal(expected, Assert.Single(CveCacheFiller.Project(Response(item), WidgetCpe)).KevAdded);
}
[Fact]
public void Project_FallsBackToCvss30_AndReportsUnknownWithoutMetrics()
{
var onlyV30 = NvdFixture.Cve("CVE-2025-0006", 0, NvdFixture.Below(WidgetCpe, "2.0"));
onlyV30.Cve!.Metrics = new NvdMetrics
{
CvssMetricV30 = { new NvdCvssMetricV31 { CvssData = new NvdCvssData { BaseScore = 6.5, BaseSeverity = "MEDIUM", VectorString = "CVSS:3.0/AV:N" } } },
};
var noMetrics = NvdFixture.Cve("CVE-2025-0007", 0, NvdFixture.Below(WidgetCpe, "2.0"));
noMetrics.Cve!.Metrics = null;
var cves = CveCacheFiller.Project(Response(onlyV30, noMetrics), WidgetCpe);
Assert.Equal(6.5, cves[0].CvssScore);
Assert.Equal("MEDIUM", cves[0].CvssSeverity);
Assert.Equal("CVSS:3.0/AV:N", cves[0].CvssVector);
Assert.Equal(0, cves[1].CvssScore);
Assert.Equal("UNKNOWN", cves[1].CvssSeverity);
Assert.Null(cves[1].CvssVector);
}
[Fact]
public void Project_CutsTheDescriptionAt500Chars()
{
var tooLong = NvdFixture.Cve("CVE-2025-0008", 5.0, NvdFixture.Below(WidgetCpe, "2.0"));
tooLong.Cve!.Descriptions = new List<NvdDescription>
{
new() { Lang = "es", Value = "Otra descripcion." },
new() { Lang = "en", Value = new string('a', 501) },
};
var fits = NvdFixture.Cve("CVE-2025-0009", 5.0, NvdFixture.Below(WidgetCpe, "2.0"));
fits.Cve!.Descriptions = new List<NvdDescription> { new() { Lang = "en", Value = new string('b', 500) } };
var cves = CveCacheFiller.Project(Response(tooLong, fits), WidgetCpe);
Assert.Equal(new string('a', 497) + "...", cves[0].Description);
Assert.Equal(new string('b', 500), cves[1].Description);
}
// -----------------------------------------------------------------------
// FillAsync
// -----------------------------------------------------------------------
[Fact]
public async Task Fill_WritesOneRowInTheCurrentShape_AndReturnsItsEntry()
{
var repo = new Mock<IVulnerabilityRepository>();
Dictionary<string, object>? written = null;
repo.Setup(r => r.UpsertCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()))
.Callback<string, string, Dictionary<string, object>>((_, _, row) => written = row)
.Returns(Task.CompletedTask);
var listed = NvdFixture.Cve("CVE-2025-0010", 8.8, NvdFixture.Below(WidgetCpe, "2.0"));
listed.Cve!.CisaExploitAdd = "2025-03-01";
// NVD reports more than it sends here; the row keeps the figure NVD gave.
var filler = Filler(repo, _ => NvdFixture.Page(4711, new[]
{
listed,
NvdFixture.Cve("CVE-2025-0011", 5.3, NvdFixture.Below(WidgetCpe, "1.5")),
NvdFixture.Cve("CVE-2025-0012", 9.8, NvdFixture.Below("cpe:2.3:a:fabrikam:gadget", "7.0")),
}));
var started = DateTime.UtcNow;
var result = await filler.FillAsync(WidgetCpe, CancellationToken.None);
Assert.Equal(CveCacheFillOutcome.Filled, result.Outcome);
Assert.Equal(4711, result.NvdTotal);
Assert.Equal(2, result.InWindow);
Assert.Equal(2, result.Kept);
Assert.Equal(new[] { "CVE-2025-0010", "CVE-2025-0011" }, result.Entry!.Cves.Select(c => c.CveId));
Assert.InRange(result.Entry.CachedAt, started, DateTime.UtcNow);
repo.Verify(r => r.UpsertCveCacheEntryAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, object>>()), Times.Once);
Assert.NotNull(written);
Assert.Equal(CveCacheFiller.SchemaVersion, written!["SchemaVersion"]);
Assert.Equal(2, written["SchemaVersion"]);
Assert.Equal("filled", written["State"]);
Assert.Equal(WidgetCpe, written["CpeUri"]);
Assert.Equal(2, written["TotalCves"]);
Assert.Equal(4711, written["NvdTotal"]);
Assert.Equal(result.Entry.CachedAt.ToString("o"), written["CachedAt"]);
var stored = JsonSerializer.Deserialize<List<CachedCve>>((string)written["CveDataJson"], CveCacheFiller.JsonOptions)!;
Assert.Equal(new[] { "CVE-2025-0010", "CVE-2025-0011" }, stored.Select(c => c.CveId));
Assert.Equal("2025-03-01", stored[0].KevAdded);
Assert.Equal("2.0", Assert.Single(stored[0].AffectedVersions).VersionEndExcluding);
}
[Fact]
public async Task Fill_SplitsALargePayloadOverSeveralColumns_AndTheReaderPutsItBackTogether()
{
var repo = new Mock<IVulnerabilityRepository>();
Dictionary<string, object>? written = null;
repo.Setup(r => r.UpsertCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()))
.Callback<string, string, Dictionary<string, object>>((_, _, row) => written = row)
.Returns(Task.CompletedTask);
var items = Enumerable.Range(1, 80).Select(n =>
{
var item = NvdFixture.Cve($"CVE-2025-1{n:000}", 7.5, NvdFixture.Below(WidgetCpe, "2.0"));
item.Cve!.Descriptions = new List<NvdDescription> { new() { Lang = "en", Value = new string('d', 500) } };
return item;
}).ToArray();
var filler = Filler(repo, _ => NvdFixture.Page(items));
var result = await filler.FillAsync(WidgetCpe, CancellationToken.None);
Assert.NotNull(written);
Assert.False(written!.ContainsKey("CveDataJson"));
Assert.True(int.Parse((string)written["CveDataJson_ChunkCount"], CultureInfo.InvariantCulture) >= 2);
Assert.Equal(80, written["TotalCves"]);
var readBack = filler.EntryFromRow(written, WidgetKey);
Assert.Equal(result.Entry!.Cves.Select(c => c.CveId), readBack!.Cves.Select(c => c.CveId));
Assert.Equal(80, readBack.Cves.Count);
}
[Theory]
[InlineData(HttpStatusCode.TooManyRequests, "Throttled")]
[InlineData(HttpStatusCode.InternalServerError, "Unanswered")]
[InlineData(HttpStatusCode.NotFound, "Failed")]
public async Task Fill_AnswerNvdDidNotGive_WritesNothing(HttpStatusCode status, string expected)
{
var repo = new Mock<IVulnerabilityRepository>(MockBehavior.Strict);
var filler = Filler(repo, _ => NvdFixture.Status(status));
var result = await filler.FillAsync(WidgetCpe, CancellationToken.None);
// A server error says NVD is not serving; a client error says only that this request is wrong.
Assert.Equal(expected, result.Outcome.ToString());
Assert.Null(result.Entry);
repo.VerifyNoOtherCalls();
}
[Fact]
public async Task Fill_RowCannotBeWritten_StillReturnsTheData()
{
var repo = new Mock<IVulnerabilityRepository>();
repo.Setup(r => r.UpsertCveCacheEntryAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, object>>()))
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
var filler = Filler(repo, _ => NvdFixture.Page(NvdFixture.Cve("CVE-2025-0013", 8.8, NvdFixture.Below(WidgetCpe, "2.0"))));
var result = await filler.FillAsync(WidgetCpe, CancellationToken.None);
Assert.Equal(CveCacheFillOutcome.Filled, result.Outcome);
Assert.Equal("CVE-2025-0013", Assert.Single(result.Entry!.Cves).CveId);
}
[Fact]
public async Task Fill_AddsEpssScoresInOneBatch_AndCachesWithoutThemWhenFirstFails()
{
var repo = new Mock<IVulnerabilityRepository>();
var epssRequests = 0;
var epssUp = true;
var epss = new EpssApiClient(new HttpClient(new LambdaHandler(_ =>
{
epssRequests++;
return epssUp
? NvdFixture.Json("""{"status":"OK","data":[{"cve":"CVE-2025-0014","epss":"0.42","percentile":"0.97","date":"2026-11-11"}]}""")
: NvdFixture.Status(HttpStatusCode.BadGateway);
})), NullLogger<EpssApiClient>.Instance);
var filler = Filler(repo, _ => NvdFixture.Page(
NvdFixture.Cve("CVE-2025-0014", 8.8, NvdFixture.Below(WidgetCpe, "2.0")),
NvdFixture.Cve("CVE-2025-0015", 5.0, NvdFixture.Below(WidgetCpe, "2.0"))), epss);
var scored = await filler.FillAsync(WidgetCpe, CancellationToken.None);
epssUp = false;
var unscored = await filler.FillAsync(WidgetCpe, CancellationToken.None);
Assert.Equal(2, epssRequests);
Assert.Equal(0.42, scored.Entry!.Cves.Single(c => c.CveId == "CVE-2025-0014").EpssScore);
Assert.Null(scored.Entry.Cves.Single(c => c.CveId == "CVE-2025-0015").EpssScore);
Assert.Equal(CveCacheFillOutcome.Filled, unscored.Outcome);
Assert.All(unscored.Entry!.Cves, c => Assert.Null(c.EpssScore));
}
// -----------------------------------------------------------------------
// MarkPendingAsync
// -----------------------------------------------------------------------
[Fact]
public async Task MarkPending_OnlyEverAddsTheRow()
{
var repo = new Mock<IVulnerabilityRepository>(MockBehavior.Strict);
Dictionary<string, object>? marker = null;
repo.Setup(r => r.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()))
.Callback<string, string, Dictionary<string, object>>((_, _, row) => marker = row)
.ReturnsAsync(false); // a row is already there: it stays as it is
var filler = Filler(repo, _ => NvdFixture.Page());
var before = DateTime.UtcNow;
await filler.MarkPendingAsync(WidgetCpe);
Assert.NotNull(marker);
Assert.Equal("pending", marker!["State"]);
Assert.Equal(WidgetCpe, marker["CpeUri"]);
Assert.Equal(CveCacheFiller.SchemaVersion, marker["SchemaVersion"]);
var requestedAt = DateTime.Parse((string)marker["RequestedAt"], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
Assert.InRange(requestedAt, before, DateTime.UtcNow);
Assert.False(marker.ContainsKey("CachedAt"));
// Nothing else was called: an upsert would replace a filled row with the marker.
repo.Verify(r => r.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()), Times.Once);
repo.VerifyNoOtherCalls();
}
[Fact]
public async Task MarkPending_StorageFailure_IsSwallowed()
{
var repo = new Mock<IVulnerabilityRepository>();
repo.Setup(r => r.TryAddCveCacheEntryAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, object>>()))
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
await Filler(repo, _ => NvdFixture.Page()).MarkPendingAsync(WidgetCpe);
repo.Verify(r => r.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()), Times.Once);
}
// -----------------------------------------------------------------------
// ReadAsync / EntryFromRow
// -----------------------------------------------------------------------
private static readonly DateTime CachedAt = new(2026, 11, 3, 4, 5, 6, DateTimeKind.Utc);
private static async Task<CveCacheEntry?> ReadAsync(Dictionary<string, object>? row)
{
var repo = new Mock<IVulnerabilityRepository>();
repo.Setup(r => r.GetCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey)).ReturnsAsync(row);
return await Filler(repo, _ => NvdFixture.Page()).ReadAsync(WidgetCpe);
}
[Fact]
public async Task Read_FilledRow_RoundTripsItsCvesAndFreshnessStamp()
{
var cve = VulnerabilityCacheRows.Cve("CVE-2025-0016", 8.8, WidgetCpe, "2.0");
cve.KevAdded = "2025-03-01";
cve.EpssScore = 0.42;
var entry = await ReadAsync(VulnerabilityCacheRows.Filled(WidgetCpe, new[] { cve }, CachedAt, nvdTotal: 12));
Assert.NotNull(entry);
Assert.Equal(CachedAt, entry!.CachedAt);
Assert.Equal(DateTimeKind.Utc, entry.CachedAt.Kind);
var read = Assert.Single(entry.Cves);
Assert.Equal("CVE-2025-0016", read.CveId);
Assert.Equal("2025-03-01", read.KevAdded);
Assert.Equal(0.42, read.EpssScore);
Assert.Equal("2.0", Assert.Single(read.AffectedVersions).VersionEndExcluding);
}
[Fact]
public async Task Read_RowOfTheOlderShape_StillServes()
{
var row = VulnerabilityCacheRows.OlderShape(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2025-0017", 7.5, WidgetCpe, "2.0") }, CachedAt);
var entry = await ReadAsync(row);
Assert.Equal("CVE-2025-0017", Assert.Single(entry!.Cves).CveId);
}
[Fact]
public async Task Read_PayloadSplitOverSeveralColumns_IsReassembled()
{
var cves = Enumerable.Range(1, 9)
.Select(n => VulnerabilityCacheRows.Cve($"CVE-2025-2{n:000}", 7.5, WidgetCpe, "2.0"))
.ToList();
var json = JsonSerializer.Serialize(cves, CveCacheFiller.JsonOptions);
var third = json.Length / 3;
var row = new Dictionary<string, object>
{
["CachedAt"] = CachedAt.ToString("o"),
["CpeUri"] = WidgetCpe,
["State"] = "filled",
["CveDataJson_0"] = json[..third],
["CveDataJson_1"] = json[third..(2 * third)],
["CveDataJson_2"] = json[(2 * third)..],
["CveDataJson_ChunkCount"] = "3",
};
var entry = await ReadAsync(row);
Assert.Equal(cves.Select(c => c.CveId), entry!.Cves.Select(c => c.CveId));
}
[Fact]
public async Task Read_PendingMarker_MissingRow_OrUnreadableStamp_IsNoData()
{
var unreadable = VulnerabilityCacheRows.Filled(WidgetCpe, Array.Empty<CachedCve>(), CachedAt);
unreadable["CachedAt"] = "not a date";
Assert.Null(await ReadAsync(VulnerabilityCacheRows.Pending(WidgetCpe, CachedAt)));
Assert.Null(await ReadAsync(null));
Assert.Null(await ReadAsync(unreadable));
}
[Fact]
public async Task Read_StorageFailure_IsNoData()
{
var repo = new Mock<IVulnerabilityRepository>();
repo.Setup(r => r.GetCveCacheEntryAsync(It.IsAny<string>(), It.IsAny<string>()))
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
Assert.Null(await Filler(repo, _ => NvdFixture.Page()).ReadAsync(WidgetCpe));
}
// -----------------------------------------------------------------------
// BuildRow
// -----------------------------------------------------------------------
[Fact]
public void BuildRow_HandsBackFreshnessTotalAndShapeAsGiven()
{
var cves = new List<CachedCve> { VulnerabilityCacheRows.Cve("CVE-2025-0018", 7.5, WidgetCpe, "2.0") };
// What an EPSS-only rewrite of an older row passes: its own stamp, total and shape.
var rewritten = CveCacheFiller.BuildRow(WidgetCpe, cves, CachedAt, nvdTotal: 4711, schemaVersion: 1);
var fresh = CveCacheFiller.BuildRow(WidgetCpe, cves, CachedAt, nvdTotal: 1);
Assert.Equal(CachedAt.ToString("o"), rewritten["CachedAt"]);
Assert.Equal(4711, rewritten["NvdTotal"]);
Assert.Equal(1, rewritten["SchemaVersion"]);
Assert.Equal("filled", rewritten["State"]);
Assert.Equal(1, rewritten["TotalCves"]);
Assert.Equal(CveCacheFiller.SchemaVersion, fresh["SchemaVersion"]);
}
/// <summary>The keys existing rows are stored under; the refresh walk compares them with the mappings.</summary>
[Theory]
[InlineData("cpe:2.3:a:contoso:widget", "cve_cpe:2.3:a:contoso:widget")]
[InlineData(@"cpe:2.3:a:microsoft:visual_c\+\+", "cve_cpe:2.3:a:microsoft:visual_c_+_+")] // a backslash cannot be part of a table key
public void CacheKey_IsThePartitionKeyOfTheProductsRow(string cpeUri, string expected)
{
Assert.Equal(expected, CveCacheFiller.CacheKey(cpeUri));
Assert.Equal("cache_result", CveCacheFiller.RowKey);
}
private sealed class LambdaHandler : HttpMessageHandler
{
private readonly Func<HttpRequestMessage, HttpResponseMessage> _respond;
public LambdaHandler(Func<HttpRequestMessage, HttpResponseMessage> respond) => _respond = respond;
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromResult(_respond(request));
}
}
@@ -0,0 +1,292 @@
using System.Collections.Concurrent;
using Azure;
using Azure.Data.Tables;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// A session's CveIndex rows follow its report: <see cref="TableStorageService.ReplaceCveIndexEntriesAsync"/>
/// writes the rows of the new findings and removes those only the replaced report had. A row that
/// stays keeps its detection date, so re-evaluating an old session does not move it into the
/// fleet summary's window, which <see cref="TableStorageService.ScanCveIndexAsync"/> applies in
/// the query. The index takes the five most urgent CVEs of a software entry plus every
/// known-exploited one.
/// </summary>
public class CveIndexReplaceTests
{
private const string Tenant = "11111111-1111-1111-1111-111111111111";
private const string Session = "22222222-2222-2222-2222-222222222222";
private static Dictionary<string, object> Vuln(string cveId, bool kev = false) => new()
{
["cveId"] = cveId,
["cvssScore"] = 7.5,
["cvssSeverity"] = "HIGH",
["isKev"] = kev,
};
private static Dictionary<string, object> Finding(string software, params Dictionary<string, object>[] vulns) => new()
{
["softwareName"] = software,
["cpeUri"] = "cpe:2.3:a:contoso:widget:3.2.0",
["riskLevel"] = "high",
["vulnerabilities"] = vulns.ToList(),
};
private static List<Dictionary<string, object>> Findings(params string[] cveIds) =>
new() { Finding("Contoso Widget", cveIds.Select(id => Vuln(id)).ToArray()) };
private sealed record Write(string Call, string CveId, bool HasDetectedAt, ETag IfMatch, TableUpdateMode Mode);
private sealed class Harness
{
public Mock<TableClient> Table { get; } = new();
public Mock<TableServiceClient> Service { get; } = new();
public ConcurrentQueue<Write> Writes { get; } = new();
public ConcurrentQueue<(string PartitionKey, string RowKey)> Deletes { get; } = new();
public List<(string? Filter, List<string> Select)> Queries { get; } = new();
public List<TableEntity> Rows { get; } = new();
/// <summary>Status the table answers with, per call and CVE id; none means success.</summary>
public Func<string, string, int?> Fail { get; set; } = (_, _) => null;
public TableStorageService Sut { get; }
public Harness()
{
Table.Setup(t => t.UpdateEntityAsync(It.IsAny<TableEntity>(), It.IsAny<ETag>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
.Returns<TableEntity, ETag, TableUpdateMode, CancellationToken>((entity, ifMatch, mode, _) =>
Record(new Write("update", entity.GetString("CveId"), entity.ContainsKey("DetectedAt"), ifMatch, mode)));
Table.Setup(t => t.UpsertEntityAsync(It.IsAny<TableEntity>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
.Returns<TableEntity, TableUpdateMode, CancellationToken>((entity, mode, _) =>
Record(new Write("upsert", entity.GetString("CveId"), entity.ContainsKey("DetectedAt"), default, mode)));
Table.Setup(t => t.DeleteEntityAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<ETag>(), It.IsAny<CancellationToken>()))
.Returns<string, string, ETag, CancellationToken>((partitionKey, rowKey, _, _) =>
{
Deletes.Enqueue((partitionKey, rowKey));
return Answer(Fail("delete", partitionKey.Substring(Tenant.Length + 1)));
});
Table.Setup(t => t.QueryAsync<TableEntity>(It.IsAny<string>(), It.IsAny<int?>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.Returns((string? filter, int? _, IEnumerable<string> select, CancellationToken _) =>
{
Queries.Add((filter, select.ToList()));
return AsyncPageable<TableEntity>.FromPages(new[] { Page<TableEntity>.FromValues(Rows, null, Mock.Of<Response>()) });
});
Service.Setup(s => s.GetTableClient(Constants.TableNames.CveIndex)).Returns(Table.Object);
Sut = new TableStorageService(Service.Object, NullLogger<TableStorageService>.Instance);
}
private Task<Response> Record(Write write)
{
Writes.Enqueue(write);
return Answer(Fail(write.Call, write.CveId));
}
private static Task<Response> Answer(int? failure) => failure.HasValue
? Task.FromException<Response>(new RequestFailedException(failure.Value, "synthetic failure"))
: Task.FromResult(Mock.Of<Response>());
public List<Write> WritesInOrder() => Writes.OrderBy(w => w.CveId, StringComparer.Ordinal).ThenBy(w => w.Call, StringComparer.Ordinal).ToList();
}
// -----------------------------------------------------------------------
// Which CVEs of a report get a row
// -----------------------------------------------------------------------
[Fact]
public void BuildCveIndexEntities_TakesTheFirstFiveOfAnEntry_AndEveryKnownExploitedOneBeyond()
{
var findings = new List<Dictionary<string, object>>
{
Finding("Contoso Widget",
Vuln("CVE-01"), Vuln("CVE-02"), Vuln("CVE-03"), Vuln("CVE-04"), Vuln("CVE-05"),
Vuln("CVE-06"), Vuln("CVE-07", kev: true), Vuln("CVE-08"), Vuln("CVE-09", kev: true)),
// The count runs per entry: this one lists CVE-08 among its first five.
Finding("Fabrikam Gadget",
Vuln("CVE-11"), Vuln("CVE-08"), Vuln("CVE-13"), Vuln("CVE-14"), Vuln("CVE-15"), Vuln("CVE-16")),
};
var indexed = TableStorageService.BuildCveIndexEntities(Tenant, Session, findings).Select(e => e.GetString("CveId"));
var all = TableStorageService.BuildCveIndexEntities(Tenant, Session, findings, mostUrgentOnly: false).Select(e => e.GetString("CveId"));
Assert.Equal(
new[] { "CVE-01", "CVE-02", "CVE-03", "CVE-04", "CVE-05", "CVE-07", "CVE-08", "CVE-09", "CVE-11", "CVE-13", "CVE-14", "CVE-15" },
indexed.OrderBy(id => id, StringComparer.Ordinal));
Assert.Equal(14, all.Count());
}
// -----------------------------------------------------------------------
// ReplaceCveIndexEntriesAsync
// -----------------------------------------------------------------------
[Fact]
public async Task Replace_KeptRowIsMergedWithoutItsDate_NewRowIsWrittenInFull_DroppedRowIsDeleted()
{
var harness = new Harness();
await harness.Sut.ReplaceCveIndexEntriesAsync(
Tenant, Session, Findings("CVE-KEPT", "CVE-NEW"), previousFindings: Findings("CVE-KEPT", "CVE-GONE"));
Assert.Equal(new[]
{
// A merge that does not name DetectedAt leaves the stored date alone.
new Write("update", "CVE-KEPT", HasDetectedAt: false, ETag.All, TableUpdateMode.Merge),
new Write("upsert", "CVE-NEW", HasDetectedAt: true, default, TableUpdateMode.Replace),
}, harness.WritesInOrder());
Assert.Equal(new[] { ($"{Tenant}_CVE-GONE", Session) }, harness.Deletes);
}
[Fact]
public async Task Replace_KeptRowThatNeverLanded_IsWrittenInFullWithADate()
{
var harness = new Harness { Fail = (call, _) => call == "update" ? 404 : null };
await harness.Sut.ReplaceCveIndexEntriesAsync(Tenant, Session, Findings("CVE-KEPT"), previousFindings: Findings("CVE-KEPT"));
Assert.Equal(new[]
{
new Write("update", "CVE-KEPT", HasDetectedAt: false, ETag.All, TableUpdateMode.Merge),
new Write("upsert", "CVE-KEPT", HasDetectedAt: true, default, TableUpdateMode.Replace),
}, harness.WritesInOrder());
Assert.Empty(harness.Deletes);
}
[Fact]
public async Task Replace_NoPreviousReport_WritesEveryRowInFull()
{
var harness = new Harness();
await harness.Sut.ReplaceCveIndexEntriesAsync(Tenant, Session, Findings("CVE-A", "CVE-B"), previousFindings: null);
Assert.Equal(new[]
{
new Write("upsert", "CVE-A", HasDetectedAt: true, default, TableUpdateMode.Replace),
new Write("upsert", "CVE-B", HasDetectedAt: true, default, TableUpdateMode.Replace),
}, harness.WritesInOrder());
Assert.Empty(harness.Deletes);
}
[Fact]
public async Task Replace_ReportRemoved_DeletesEveryRowItHad()
{
var harness = new Harness();
await harness.Sut.ReplaceCveIndexEntriesAsync(Tenant, Session, findings: null, previousFindings: Findings("CVE-A", "CVE-B"));
Assert.Empty(harness.Writes);
Assert.Equal(
new[] { ($"{Tenant}_CVE-A", Session), ($"{Tenant}_CVE-B", Session) },
harness.Deletes.OrderBy(d => d.PartitionKey, StringComparer.Ordinal));
}
/// <summary>
/// A report stored before the index was limited has a row for every CVE it lists. All of its
/// CVEs count as existing rows, so the ones the new selection leaves out are removed.
/// </summary>
[Fact]
public async Task Replace_PreviousReportIsReadInFull_SoRowsBeyondTheSelectionAreRemoved()
{
var harness = new Harness();
var sevenCves = new[] { "CVE-1", "CVE-2", "CVE-3", "CVE-4", "CVE-5", "CVE-6", "CVE-7" };
await harness.Sut.ReplaceCveIndexEntriesAsync(Tenant, Session, Findings(sevenCves), previousFindings: Findings(sevenCves));
Assert.Equal(new[] { "CVE-1", "CVE-2", "CVE-3", "CVE-4", "CVE-5" }, harness.WritesInOrder().Select(w => w.CveId));
Assert.All(harness.Writes, w => Assert.Equal("update", w.Call));
Assert.Equal(
new[] { $"{Tenant}_CVE-6", $"{Tenant}_CVE-7" },
harness.Deletes.Select(d => d.PartitionKey).OrderBy(pk => pk, StringComparer.Ordinal));
}
[Fact]
public async Task Replace_PreviousFindingsAsReadBackFromStorage_StillNameTheRows()
{
var harness = new Harness();
var stored = ReportFixture.AsStored(new Dictionary<string, object> { ["findings"] = Findings("CVE-KEPT", "CVE-GONE") });
await harness.Sut.ReplaceCveIndexEntriesAsync(
Tenant, Session, Findings("CVE-KEPT"), VulnerabilityCorrelationService.FindingsOf(stored));
Assert.Equal("update", Assert.Single(harness.Writes).Call);
Assert.Equal(new[] { ($"{Tenant}_CVE-GONE", Session) }, harness.Deletes);
}
[Fact]
public async Task Replace_NothingToWriteOrRemove_DoesNotTouchTheTable()
{
var harness = new Harness();
await harness.Sut.ReplaceCveIndexEntriesAsync(Tenant, Session, null, null);
await harness.Sut.ReplaceCveIndexEntriesAsync(
Tenant, Session, new List<Dictionary<string, object>>(), new List<Dictionary<string, object>>());
harness.Service.Verify(s => s.GetTableClient(It.IsAny<string>()), Times.Never);
harness.Table.VerifyNoOtherCalls();
}
[Theory]
[InlineData("delete", 404)] // already gone
[InlineData("delete", 503)]
[InlineData("upsert", 503)]
[InlineData("update", 503)]
public async Task Replace_FailingWrites_NeverThrow_AndTheOtherRowsAreStillWritten(string failingCall, int status)
{
var harness = new Harness { Fail = (call, _) => call == failingCall ? status : null };
await harness.Sut.ReplaceCveIndexEntriesAsync(
Tenant, Session, Findings("CVE-KEPT", "CVE-NEW"), previousFindings: Findings("CVE-KEPT", "CVE-GONE"));
Assert.Contains(harness.Writes, w => w.Call == "update" && w.CveId == "CVE-KEPT");
Assert.Contains(harness.Writes, w => w.Call == "upsert" && w.CveId == "CVE-NEW");
Assert.Equal(new[] { ($"{Tenant}_CVE-GONE", Session) }, harness.Deletes);
// Only a 404 means "the row is not there"; any other failed update is not retried as an insert.
Assert.DoesNotContain(harness.Writes, w => w.Call == "upsert" && w.CveId == "CVE-KEPT");
}
[Fact]
public async Task Replace_TableUnreachable_DoesNotThrow()
{
var service = new Mock<TableServiceClient>();
service.Setup(s => s.GetTableClient(It.IsAny<string>())).Throws(new RequestFailedException(503, "busy"));
var sut = new TableStorageService(service.Object, NullLogger<TableStorageService>.Instance);
await sut.ReplaceCveIndexEntriesAsync(Tenant, Session, Findings("CVE-A"), previousFindings: null);
}
// -----------------------------------------------------------------------
// ScanCveIndexAsync
// -----------------------------------------------------------------------
[Fact]
public async Task Scan_WithAWindow_FiltersByDetectionDateInTheQuery()
{
var harness = new Harness();
var detectedAt = new DateTimeOffset(2026, 11, 10, 9, 0, 0, TimeSpan.Zero);
harness.Rows.Add(new TableEntity($"{Tenant}_CVE-2026-0001", Session)
{
["TenantId"] = Tenant, ["SessionId"] = Session, ["CveId"] = "CVE-2026-0001", ["DetectedAt"] = detectedAt,
});
var since = new DateTime(2026, 11, 5, 8, 30, 0, DateTimeKind.Utc);
var (rows, truncated) = await harness.Sut.ScanCveIndexAsync(Tenant, maxRows: 100, detectedSinceUtc: since);
await harness.Sut.ScanCveIndexAsync(null, maxRows: 100, detectedSinceUtc: since);
await harness.Sut.ScanCveIndexAsync(Tenant, maxRows: 100);
Assert.Equal(new[]
{
$"(PartitionKey ge '{Tenant}_' and PartitionKey lt '{Tenant}_~') and DetectedAt ge datetime'2026-11-05T08:30:00Z'",
"DetectedAt ge datetime'2026-11-05T08:30:00Z'",
$"PartitionKey ge '{Tenant}_' and PartitionKey lt '{Tenant}_~'",
}, harness.Queries.Select(q => q.Filter));
Assert.All(harness.Queries, q => Assert.Contains("DetectedAt", q.Select));
Assert.False(truncated);
Assert.Equal(detectedAt.UtcDateTime, Assert.Single(rows).DetectedAt);
}
}
@@ -0,0 +1,165 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="CveVersionFilter"/> judges an installed version only by ranges that belong to the
/// product itself, fit the software edition its mapping asks for and count versions the way the
/// installed one does. Each rule here removes a class of CVEs a product was reported for without
/// having them. The platform and plain range rules are in <c>VulnerabilityCorrelationServiceTests</c>.
/// </summary>
public class CveVersionFilterTests
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string FirefoxCpe = "cpe:2.3:a:mozilla:firefox";
private static CachedVersionRange Below(string criteria, string? fixedIn = null) =>
new() { Criteria = criteria, VersionEndExcluding = fixedIn };
private static CachedCve Cve(string id, params CachedVersionRange[] ranges) => new()
{
CveId = id,
CvssScore = 8.8,
CvssSeverity = "HIGH",
Description = "Synthetic description.",
PublishedDate = "2026-06-01T00:00:00.000",
AffectedVersions = ranges.ToList(),
};
// -----------------------------------------------------------------------
// Ranges of another product (rows written before the projection dropped them)
// -----------------------------------------------------------------------
[Fact]
public void RangeOfAnotherProductInTheSameCve_DoesNotJudgeTheInstalledVersion()
{
var cve = Cve("CVE-2024-3661",
Below("cpe:2.3:a:contoso:widget:-:*:*:*:*:*:*:*"),
Below("cpe:2.3:a:fabrikam:secure_client:*:*:*:*:*:*:*:*", "24.06.1"));
var result = CveVersionFilter.Filter(new[] { cve }, WidgetCpe, "3.2.0");
Assert.Empty(result.Vulnerabilities);
Assert.Equal(0, result.IncomparableCves);
}
[Theory]
[InlineData("cpe:2.3:a:adobe:acrobat_reader_dc:*:*:*:*:*:*:*:*")] // shares the prefix, is another product
[InlineData("cpe:2.3:a:fabrikam:acrobat:*:*:*:*:*:*:*:*")] // same product name, another vendor
[InlineData("cpe:2.3:o:microsoft:windows:*:*:*:*:*:*:*:*")]
public void ProductIsComparedByVendorAndProduct_NotByPrefix(string foreignCriteria)
{
var cve = Cve("CVE-2025-0101", Below(foreignCriteria, "99.0"));
Assert.Empty(CveVersionFilter.Filter(new[] { cve }, "cpe:2.3:a:adobe:acrobat", "24.0").Vulnerabilities);
}
// -----------------------------------------------------------------------
// Software editions
// -----------------------------------------------------------------------
/// <summary>Shaped like the public CVE-2023-4863 for Firefox: one fix for the release channel, one for ESR.</summary>
private static readonly CachedCve FirefoxAndItsEsr = Cve("CVE-2023-4863",
Below("cpe:2.3:a:mozilla:firefox:*:*:*:*:-:*:*:*", "117.0.1"),
Below("cpe:2.3:a:mozilla:firefox:*:*:*:*:esr:*:*:*", "115.2.1"));
[Theory]
// Firefox, mapped with "never the ESR ranges"
[InlineData("116.0", null, true, true)]
[InlineData("117.0.1", null, true, false)]
// Firefox ESR, mapped with "only the ESR ranges"
[InlineData("115.2.0", "esr", false, true)]
[InlineData("115.3.0", "esr", false, false)]
// Without an edition on the mapping the fixed ESR build is judged by the release-channel
// range: the false positive the edition setting removes.
[InlineData("115.3.0", null, false, true)]
public void Edition_DecidesWhichRangesCount(string installed, string? edition, bool excludeEsr, bool affected)
{
var result = CveVersionFilter.Filter(
new[] { FirefoxAndItsEsr }, FirefoxCpe, installed, edition,
excludeEsr ? new[] { "esr" } : null);
Assert.Equal(affected, result.Vulnerabilities.Count == 1);
}
[Fact]
public void WithoutEditionOnTheMapping_EveryEditionCounts()
{
const string readerCpe = "cpe:2.3:a:adobe:acrobat_reader_dc";
var cve = Cve("CVE-2025-0102", Below("cpe:2.3:a:adobe:acrobat_reader_dc:*:*:*:*:continuous:*:*:*", "24.002.20759"));
Assert.Single(CveVersionFilter.Filter(new[] { cve }, readerCpe, "23.008.20470").Vulnerabilities);
Assert.Empty(CveVersionFilter.Filter(new[] { cve }, readerCpe, "23.008.20470", edition: "classic").Vulnerabilities);
Assert.Empty(CveVersionFilter.Filter(new[] { cve }, readerCpe, "23.008.20470", excludeEditions: new[] { "continuous" }).Vulnerabilities);
}
// -----------------------------------------------------------------------
// Version scheme
// -----------------------------------------------------------------------
[Theory]
[InlineData("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "2022.30050")] // a range that counts in years
[InlineData("cpe:2.3:a:contoso:widget:2022.30050:*:*:*:*:*:*:*", null)] // one exact version in years
public void RangeInAnotherVersionScheme_IsNotJudgedButCounted(string criteria, string? fixedIn)
{
var cve = Cve("CVE-2025-0103", Below(criteria, fixedIn));
var result = CveVersionFilter.Filter(new[] { cve }, WidgetCpe, "26.198");
// 26.198 is "below" every year-based version, so comparing would report each such CVE.
Assert.Empty(result.Vulnerabilities);
Assert.Equal(1, result.IncomparableCves);
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void CveMatchingThroughAComparableRange_IsReportedAndNotCountedAsIncomparable(bool comparableRangeFirst)
{
var inYears = Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "2022.30050");
var inReleases = Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "27.0");
var cve = comparableRangeFirst ? Cve("CVE-2025-0104", inReleases, inYears) : Cve("CVE-2025-0104", inYears, inReleases);
var result = CveVersionFilter.Filter(new[] { cve }, WidgetCpe, "26.198");
Assert.Equal("CVE-2025-0104", Assert.Single(result.Vulnerabilities)["cveId"]);
Assert.Equal(0, result.IncomparableCves);
}
[Fact]
public void IncomparableCves_CountsCvesNotRanges()
{
var twoRanges = Cve("CVE-2025-0105",
Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "2022.30050"),
Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "2023.10000"));
var fixedLongAgo = Cve("CVE-2025-0106", Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "20.0"));
var result = CveVersionFilter.Filter(new[] { twoRanges, fixedLongAgo }, WidgetCpe, "26.198");
Assert.Empty(result.Vulnerabilities);
Assert.Equal(1, result.IncomparableCves);
}
// -----------------------------------------------------------------------
// Platform named in the description
// -----------------------------------------------------------------------
[Theory]
[InlineData("The flaw can be triggered in certain scenarios when a crafted file is opened.", false)] // "ios" inside "scenarios"
[InlineData("A local attacker can gain elevated permissions in the agent on Linux and macOS.", true)]
[InlineData("A local attacker can gain elevated permissions in the agent on Linux and Windows.", false)]
[InlineData("FortiClientLinux may allow an attacker to execute code via a crafted website.", false)] // a product name, not the platform word
public void IsDescriptionNonWindowsOnly_MatchesWholeWords(string description, bool nonWindowsOnly)
{
Assert.Equal(nonWindowsOnly, CveVersionFilter.IsDescriptionNonWindowsOnly(description));
}
[Fact]
public void DescriptionThatOnlyContainsAPlatformNameInsideAWord_KeepsTheCve()
{
var cve = Cve("CVE-2025-0107", Below("cpe:2.3:a:contoso:widget:*:*:*:*:*:*:*:*", "4.0"));
cve.Description = "Memory corruption in certain scenarios allows code execution.";
Assert.Single(CveVersionFilter.Filter(new[] { cve }, WidgetCpe, "3.2.0").Vulnerabilities);
}
}
@@ -0,0 +1,298 @@
using System.Net;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// DELETE vulnerability/cpe-mapping removes an operator's mapping or an automatic one and brings
/// the inventory rows in line in the same call. An automatic mapping that is removed also puts
/// its program on the ignore list, or the next nightly pass would map it again; the seed and the
/// community list are not deleted from here.
/// </summary>
public class DeleteCustomCpeMappingFunctionTests
{
private const string Seed = CpeMappingStore.SeedPartition;
private const string Custom = CpeMappingStore.CustomPartition;
private const string Auto = CpeMappingStore.AutoPartition;
private const string Ignored = CpeMappingStore.IgnoredPartition;
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private const string ToolCpe = "cpe:2.3:a:northwind:tool";
private sealed class Rig
{
private Rig(MappingRig maps)
{
Maps = maps;
Function = new DeleteCustomCpeMappingFunction(
maps.Scan.Repo.Object, maps.Reconciler, NullLogger<DeleteCustomCpeMappingFunction>.Instance);
}
/// <summary>The gadget is mapped by the seed, the widget by an operator, the tool automatically.</summary>
public static async Task<Rig> CreateAsync() => new(await MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"),
VulnerabilityCacheRows.MappingIn(Custom, "manual", "contoso", "contoso widget", WidgetCpe),
VulnerabilityCacheRows.MappingIn(Auto, "auto", "northwind", "northwind tool", ToolCpe)));
public MappingRig Maps { get; }
public MappingTables Tables => Maps.Tables;
public DeleteCustomCpeMappingFunction Function { get; }
public Task<HttpResponseData> DeleteAsync(DeleteCustomCpeMappingRequest request) => Function.Run(OperatorCall.Request(request));
}
[Fact]
public async Task Delete_AutomaticMapping_RemovesIt_PutsTheProgramOnTheIgnoreList_AndTakesTheCpeOffItsRows()
{
var rig = await Rig.CreateAsync();
var installed = rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", cpeUri: ToolCpe);
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "northwind:northwind tool", Source = "auto" });
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.True(body.GetProperty("success").GetBoolean());
Assert.Contains("ignore list", body.GetProperty("message").GetString());
Assert.Empty(rig.Tables.Rows(Auto));
// Filed like an entry the operator adds by hand, so the ignore list shows and removes it the same way.
var entry = Assert.Single(rig.Tables.Rows(Ignored));
Assert.Equal(CpeIgnoreEntry.Build("northwind", "Northwind Tool 3.1", "manual")!.Value.RowKey, entry["RowKey"]);
Assert.Equal("northwind", entry["NormalizedVendor"]);
Assert.Equal("northwind tool", entry["NormalizedProduct"]);
Assert.Equal("automatic mapping removed", entry["Reason"]);
// The ignore entry first: a failure between the two writes must leave the mapping in
// place, not a program the next night maps again.
Assert.Equal(new[] { $"upsert {Ignored}/northwind:northwind tool", $"delete {Auto}/northwind:northwind tool" }, rig.Tables.Writes);
Assert.Equal(string.Empty, installed["CpeUri"]);
var mappings = await rig.Maps.Store.GetAsync();
Assert.Null(mappings.Matcher.Match("northwind", "northwind tool 4.0", "Northwind Tool 4.0", "Northwind Traders"));
Assert.True(mappings.IsIgnored("northwind", "Northwind Traders", "Northwind Tool 4.0"));
}
[Fact]
public async Task Delete_AutomaticMapping_LeavesAnIgnoreEntryTheIgnoreListItselfCanRemove()
{
var rig = await Rig.CreateAsync();
var ignoreList = new IgnoreSoftwareFunction(
rig.Maps.Scan.Repo.Object, rig.Maps.Store, NullLogger<IgnoreSoftwareFunction>.Instance);
await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "northwind:northwind tool", Source = "auto" });
var listed = Assert.Single(OperatorCall.BodyOf(await ignoreList.GetIgnored(OperatorCall.Request())).GetProperty("items").EnumerateArray());
// The list's delete sends back what the list shows.
var removed = await ignoreList.DeleteIgnored(OperatorCall.Request(new IgnoreSoftwareDeleteRequest
{
SoftwareName = listed.GetProperty("softwareName").GetString()!,
Publisher = listed.GetProperty("publisher").GetString(),
}));
Assert.Equal(HttpStatusCode.OK, removed.StatusCode);
Assert.Empty(rig.Tables.Rows(Ignored));
Assert.False((await rig.Maps.Store.GetAsync()).IsIgnored("northwind", "Northwind Traders", "Northwind Tool 4.0"));
}
[Fact]
public async Task Delete_OperatorsMapping_ByVendorAndProgramName_RemovesItWithoutIgnoringTheProgram()
{
var rig = await Rig.CreateAsync();
var installed = rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", cpeUri: WidgetCpe);
// The name as the program shows it; the source defaults to the operator's own mappings.
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest
{
NormalizedVendor = "Contoso",
NormalizedProduct = "Contoso Widget 4.2 (x64)",
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal("CPE mapping deleted", OperatorCall.BodyOf(response).GetProperty("message").GetString());
Assert.Empty(rig.Tables.Rows(Custom));
Assert.Empty(rig.Tables.Rows(Ignored));
Assert.Equal(new[] { $"delete {Custom}/contoso:contoso widget" }, rig.Tables.Writes);
Assert.Equal(string.Empty, installed["CpeUri"]);
}
[Fact]
public async Task Delete_ByRowKey_ReachesARowWhateverItIsKeyedBy()
{
var rig = await Rig.CreateAsync();
// Not the key today's save would give this program.
const string rowKey = ":contoso reader 2.0 (x64)";
rig.Tables.Put(Custom, rowKey, new Dictionary<string, object>
{
["NormalizedVendor"] = string.Empty,
["NormalizedProduct"] = "Contoso Reader 2.0 (x64)",
["CpeUri"] = "cpe:2.3:a:contoso:reader",
["Source"] = "manual",
});
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = rowKey, Source = "Custom" });
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(new[] { "contoso:contoso widget" }, rig.Tables.RowKeys(Custom));
Assert.Empty(rig.Tables.Rows(Ignored));
}
[Fact]
public async Task Delete_AutomaticMappingKeptAmongTheOperatorsMappings_IsDeletedByWhatTheListShowsForIt()
{
var rig = await Rig.CreateAsync();
// The retired name search wrote its mappings next to the operator's own; only the Source
// column tells them apart, and the list shows them as automatic.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(Custom, "auto", "adatum", "adatum reader", "cpe:2.3:a:adatum:reader"));
var list = new GetCpeMappingsFunction(rig.Maps.Store, NullLogger<GetCpeMappingsFunction>.Instance);
var listed = OperatorCall.BodyOf(await list.Run(OperatorCall.Request())).GetProperty("mappings").EnumerateArray()
.Single(m => m.GetProperty("normalizedProduct").GetString() == "adatum reader");
Assert.Equal("auto", listed.GetProperty("source").GetString());
// The portal names a mapping by the row key and the source the list gave it.
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest
{
RowKey = listed.GetProperty("rowKey").GetString(),
Source = listed.GetProperty("source").GetString(),
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(new[] { "contoso:contoso widget" }, rig.Tables.RowKeys(Custom));
}
[Theory]
[InlineData("seed")]
[InlineData("community")]
[InlineData("manual")]
public async Task Delete_SourceOtherThanTheOperatorsOrAutomatic_IsABadRequest(string source)
{
var rig = await Rig.CreateAsync();
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "fabrikam:fabrikam gadget", Source = source });
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.BadRequest, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.Tables.Writes);
Assert.Single(rig.Tables.Rows(Seed));
}
[Theory]
[InlineData("contoso:contoso gadget", null)]
// The row exists, but among the operator's mappings, not the automatic ones.
[InlineData("contoso:contoso widget", "auto")]
[InlineData("northwind:northwind tool", "custom")]
public async Task Delete_RowThatIsNotThere_IsNotFound(string rowKey, string? source)
{
var rig = await Rig.CreateAsync();
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = rowKey, Source = source });
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.NotFound, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.Tables.Writes);
}
[Theory]
[InlineData("{}")]
[InlineData("""{ "normalizedVendor": "contoso" }""")]
[InlineData("""{ "rowKey": " ", "normalizedProduct": " ", "source": "custom" }""")]
public async Task Delete_WithoutRowKeyOrProgramName_IsABadRequest(string json)
{
var rig = await Rig.CreateAsync();
var response = await rig.Function.Run(OperatorCall.RequestWithJson(json));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Delete_AutomaticMappingKeptAmongTheOperatorsMappings_LeavesTheIgnoreListAlone()
{
var rig = await Rig.CreateAsync();
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(Custom, "auto", "adatum", "adatum reader", "cpe:2.3:a:adatum:reader"));
var installed = rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", cpeUri: "cpe:2.3:a:adatum:reader");
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "adatum:adatum reader", Source = "auto" });
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
// The name search that wrote this row is retired, so nothing would write it again.
Assert.Equal("CPE mapping deleted", OperatorCall.BodyOf(response).GetProperty("message").GetString());
Assert.Empty(rig.Tables.Rows(Ignored));
Assert.Equal(new[] { $"delete {Custom}/adatum:adatum reader" }, rig.Tables.Writes);
Assert.Equal(string.Empty, installed["CpeUri"]);
Assert.False((await rig.Maps.Store.GetAsync()).IsIgnored("adatum", "Adatum Corporation", "Adatum Reader 2.0"));
}
[Fact]
public async Task Delete_AutomaticMappingKeptAmongTheOperatorsMappings_IsNotReachedAsOneOfTheOperatorsOwn()
{
var rig = await Rig.CreateAsync();
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(Custom, "auto", "adatum", "adatum reader", "cpe:2.3:a:adatum:reader"));
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "adatum:adatum reader", Source = "custom" });
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
Assert.Empty(rig.Tables.Writes);
}
[Theory]
[InlineData(null)]
[InlineData("")]
[InlineData(" ")]
public async Task Delete_WithoutASource_MeansTheOperatorsOwnMappings(string? source)
{
var rig = await Rig.CreateAsync();
var own = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "contoso:contoso widget", Source = source });
var automatic = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "northwind:northwind tool", Source = source });
Assert.Equal(HttpStatusCode.OK, own.StatusCode);
Assert.Empty(rig.Tables.Rows(Custom));
// An automatic mapping is deleted only by a request that names it as one.
Assert.Equal(HttpStatusCode.NotFound, automatic.StatusCode);
Assert.Single(rig.Tables.Rows(Auto));
Assert.Empty(rig.Tables.Rows(Ignored));
}
[Fact]
public async Task Delete_AutomaticMapping_WhoseIgnoreEntryCannotBeWritten_StaysInPlace()
{
var rig = await Rig.CreateAsync();
var installed = rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", cpeUri: ToolCpe);
rig.Tables.WriteFailure = write =>
write.StartsWith($"upsert {Ignored}/", StringComparison.Ordinal) ? new InvalidOperationException("storage down") : null;
var response = await rig.DeleteAsync(new DeleteCustomCpeMappingRequest { RowKey = "northwind:northwind tool", Source = "auto" });
Assert.Equal(HttpStatusCode.InternalServerError, response.StatusCode);
// Deleted without the entry, the mapping would be back after the next nightly pass.
Assert.Single(rig.Tables.Rows(Auto));
Assert.Empty(rig.Tables.Writes);
Assert.Equal(ToolCpe, installed["CpeUri"]);
}
[Fact]
public async Task Delete_AutomaticMapping_ThatCannotBeDeletedAfterItsIgnoreEntry_CanBeDeletedAgain()
{
var rig = await Rig.CreateAsync();
rig.Tables.WriteFailure = write =>
write.StartsWith("delete ", StringComparison.Ordinal) ? new InvalidOperationException("storage down") : null;
var request = new DeleteCustomCpeMappingRequest { RowKey = "northwind:northwind tool", Source = "auto" };
var first = await rig.DeleteAsync(request);
Assert.Equal(HttpStatusCode.InternalServerError, first.StatusCode);
Assert.Single(rig.Tables.Rows(Auto));
Assert.Single(rig.Tables.Rows(Ignored));
rig.Tables.WriteFailure = null;
var second = await rig.DeleteAsync(request);
Assert.Equal(HttpStatusCode.OK, second.StatusCode);
Assert.Empty(rig.Tables.Rows(Auto));
Assert.Single(rig.Tables.Rows(Ignored));
}
}
@@ -67,6 +67,8 @@ public class DerivedWriterLivenessTests
Mock.Of<ISessionRepository>(),
inventoryLoader.Object,
Mock.Of<IAnalyzeOnEnrollmentEndProducer>(),
Mock.Of<IVulnerabilityCorrelateProducer>(),
reportWriter: null!,
signalRNotification: null!,
GuardFor(row),
NullLogger<VulnerabilityCorrelateHandler>.Instance);
@@ -154,6 +154,12 @@ public class EpssAndNvdThrottleTests
var failed = NvdFetchResult<NvdCveResponse>.Failed();
Assert.False(failed.Succeeded);
Assert.False(failed.Throttled);
Assert.False(failed.Unanswered);
var unanswered = NvdFetchResult<NvdCveResponse>.UnansweredResult();
Assert.False(unanswered.Succeeded);
Assert.False(unanswered.Throttled);
Assert.True(unanswered.Unanswered);
var ok = NvdFetchResult<NvdCveResponse>.Ok(new NvdCveResponse { TotalResults = 1 });
Assert.True(ok.Succeeded);
@@ -209,27 +215,22 @@ public class EpssAndNvdThrottleTests
}
};
var cves = VulnerabilityCorrelationService.ProjectNvdResponse(response);
var cves = CveCacheFiller.Project(response, "cpe:2.3:a:v:p");
Assert.Equal(2, cves.Count);
var first = cves[0];
// The second CVE names no range of the product and is not kept.
var first = Assert.Single(cves);
Assert.Equal(9.8, first.CvssScore);
Assert.Equal("CRITICAL", first.CvssSeverity);
Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", first.CvssVector);
Assert.Single(first.AffectedVersions);
Assert.Equal("2.0", first.AffectedVersions[0].VersionEndExcluding);
Assert.Null(first.EpssScore); // never defaulted before enrichment
var second = cves[1];
Assert.Equal(0, second.CvssScore);
Assert.Equal("UNKNOWN", second.CvssSeverity);
Assert.Null(second.CvssVector);
}
[Fact]
public void ProjectNvdResponse_Null_ReturnsEmpty()
{
Assert.Empty(VulnerabilityCorrelationService.ProjectNvdResponse(null));
Assert.Empty(CveCacheFiller.Project(null, "cpe:2.3:a:v:p"));
}
// -----------------------------------------------------------------------
@@ -252,7 +253,7 @@ public class EpssAndNvdThrottleTests
// CVE-2 absent: FIRST did not score it (or its batch failed) ⇒ stays null
};
var changed = VulnerabilityCorrelationService.ApplyEpssScores(cves, scores);
var changed = CveCacheFiller.ApplyEpssScores(cves, scores);
Assert.Equal(1, changed);
Assert.Equal(0.25, cves[0].EpssScore);
@@ -265,7 +266,7 @@ public class EpssAndNvdThrottleTests
public void ApplyEpssScores_NeverErasesYesterdaysScore_WhenTodayHasNone()
{
var cves = new List<CachedCve> { new() { CveId = "CVE-1", EpssScore = 0.3, EpssPercentile = 0.7, EpssDate = "2026-08-27" } };
var changed = VulnerabilityCorrelationService.ApplyEpssScores(cves, new Dictionary<string, EpssScore>());
var changed = CveCacheFiller.ApplyEpssScores(cves, new Dictionary<string, EpssScore>());
Assert.Equal(0, changed);
Assert.Equal(0.3, cves[0].EpssScore);
}
@@ -274,7 +275,7 @@ public class EpssAndNvdThrottleTests
public void BuildCveCacheRow_PassesCachedAtThrough_SoEpssRewritesDoNotLookLikeNvdFetches()
{
var cachedAt = new DateTime(2026, 8, 20, 3, 0, 0, DateTimeKind.Utc);
var row = VulnerabilityCorrelationService.BuildCveCacheRow("cpe:2.3:a:v:p", new List<CachedCve> { new() { CveId = "CVE-1" } }, cachedAt);
var row = CveCacheFiller.BuildRow("cpe:2.3:a:v:p", new List<CachedCve> { new() { CveId = "CVE-1" } }, cachedAt, nvdTotal: 7);
Assert.Equal(cachedAt.ToString("o"), row["CachedAt"]);
Assert.Equal("cpe:2.3:a:v:p", row["CpeUri"]);
@@ -285,13 +286,13 @@ public class EpssAndNvdThrottleTests
[Fact]
public void CachedCve_RoundTrips_WithoutWritingNulls_SoLegacyRowsStayReadable()
{
var json = System.Text.Json.JsonSerializer.Serialize(new List<CachedCve> { new() { CveId = "CVE-1", CvssScore = 7 } }, JsonOptions);
var json = System.Text.Json.JsonSerializer.Serialize(new List<CachedCve> { new() { CveId = "CVE-1", CvssScore = 7 } }, CveCacheFiller.JsonOptions);
Assert.DoesNotContain("EpssScore", json);
Assert.DoesNotContain("CvssVector", json);
// A row written before this change (no EPSS/vector properties at all) still deserialises.
var legacy = """[{"CveId":"CVE-9","CvssScore":4.3,"CvssSeverity":"MEDIUM","Description":"","PublishedDate":"","AffectedVersions":[]}]""";
var cves = System.Text.Json.JsonSerializer.Deserialize<List<CachedCve>>(legacy, JsonOptions)!;
var cves = System.Text.Json.JsonSerializer.Deserialize<List<CachedCve>>(legacy, CveCacheFiller.JsonOptions)!;
Assert.Null(cves[0].EpssScore);
Assert.Null(cves[0].CvssVector);
}
@@ -0,0 +1,163 @@
using System.Net;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// GET vulnerability/cpe-mappings lists every mapping in service with where it comes from, the
/// row key a delete names, and whether its product counts as self-updating. It reads fresh, so
/// the operator who just changed something sees it.
/// </summary>
public class GetCpeMappingsFunctionTests
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private static CpeMapping MappingOf(Dictionary<string, object> row) => CpeMappingStore.MappingFromRow(row)!;
private static CpeMappingSnapshot Snapshot(CpeMapping mapping, params CpeProductFlags[] flags) =>
new(new[] { mapping }, Array.Empty<string>(), flags, DateTime.UtcNow);
[Theory]
[InlineData(CpeMappingStore.SeedPartition, "seed", "seed")]
[InlineData(CpeMappingStore.CommunityPartition, "community", "community")]
[InlineData(CpeMappingStore.AutoPartition, "auto", "auto")]
[InlineData(CpeMappingStore.CustomPartition, "manual", "custom")]
// A row the retired name search left among the operator's mappings is shown as what it is.
[InlineData(CpeMappingStore.CustomPartition, "auto", "auto")]
public void ToItem_NamesWhereAMappingComesFrom(string partition, string sourceColumn, string expected)
{
var mapping = MappingOf(VulnerabilityCacheRows.MappingIn(partition, sourceColumn, "contoso", "contoso widget", WidgetCpe));
Assert.Equal(expected, GetCpeMappingsFunction.ToItem(mapping, Snapshot(mapping)).Source);
}
[Fact]
public void ToItem_CarriesTheMappingItsRowKeyAndTheProductOfItsCpe()
{
var row = VulnerabilityCacheRows.MappingIn(
CpeMappingStore.CustomPartition, "manual", "Contoso", "Contoso Widget", "cpe:2.3:a:Contoso:Widget:*:*:*:*:*:*:*:*", "contoso widget");
row["Category"] = "utility";
row["PublisherPatternsJson"] = """["contoso ltd"]""";
row["ExcludePatternsJson"] = """["widget viewer"]""";
row["CreatedAt"] = "2026-11-12T03:00:00.0000000Z";
var mapping = MappingOf(row);
var item = GetCpeMappingsFunction.ToItem(mapping, Snapshot(mapping));
Assert.Equal("contoso:contoso widget", item.RowKey);
Assert.Equal("Contoso", item.NormalizedVendor);
Assert.Equal("Contoso Widget", item.NormalizedProduct);
Assert.Equal(WidgetCpe, item.CpeUri);
Assert.Equal("contoso", item.CpeVendor);
Assert.Equal("widget", item.CpeProduct);
Assert.Equal("utility", item.Category);
Assert.Equal(new[] { "contoso widget" }, item.DisplayNamePatterns);
Assert.Equal(new[] { "contoso ltd" }, item.PublisherPatterns);
Assert.Equal(new[] { "widget viewer" }, item.ExcludePatterns);
Assert.Equal("2026-11-12T03:00:00.0000000Z", item.CreatedAt);
Assert.False(item.SelfUpdating);
Assert.False(item.SelfUpdatingDefault);
}
[Theory]
[InlineData(true, null, true, true)] // the default alone
[InlineData(true, false, false, true)] // an operator switched the default off
[InlineData(null, true, true, false)] // an operator flagged a product the seed does not list
public void ToItem_SelfUpdating_IsTheValueInEffect_NextToTheSeedsDefault(
bool? seed, bool? admin, bool expectedSelfUpdating, bool expectedDefault)
{
var mapping = MappingOf(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", WidgetCpe));
var flags = new CpeProductFlags { Product = "contoso:widget", SeedSelfUpdating = seed, AdminSelfUpdating = admin };
var ofAnotherProduct = new CpeProductFlags { Product = "contoso:gadget", SeedSelfUpdating = !expectedDefault };
var item = GetCpeMappingsFunction.ToItem(mapping, Snapshot(mapping, ofAnotherProduct, flags));
Assert.Equal(expectedSelfUpdating, item.SelfUpdating);
Assert.Equal(expectedDefault, item.SelfUpdatingDefault);
}
private sealed class Rig
{
public Rig()
{
Function = new GetCpeMappingsFunction(
new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance), NullLogger<GetCpeMappingsFunction>.Instance);
}
public MappingTables Tables { get; } = new();
public GetCpeMappingsFunction Function { get; }
public async Task<List<(string Source, string Product)>> ListAsync()
{
var response = await Function.Run(OperatorCall.Request());
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
var listed = body.GetProperty("mappings").EnumerateArray()
.Select(m => (m.GetProperty("source").GetString()!, m.GetProperty("normalizedProduct").GetString()!))
.ToList();
Assert.Equal(listed.Count, body.GetProperty("total").GetInt32());
return listed;
}
}
[Fact]
public async Task Get_ListsTheOperatorsMappingsFirst_ThenAutomaticOnes_ThenTheLists_EachByProgramName()
{
var rig = new Rig();
rig.Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget"));
rig.Tables.Put(VulnerabilityCacheRows.Mapping("adatum", "Adatum Reader", "cpe:2.3:a:adatum:reader"));
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.CommunityPartition, "community", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool"));
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.AutoPartition, "auto", "tailspin", "tailspin launcher", "cpe:2.3:a:tailspin:launcher"));
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.CustomPartition, "manual", "contoso", "Contoso Widget", WidgetCpe));
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.CustomPartition, "manual", "contoso", "contoso agent", "cpe:2.3:a:contoso:agent"));
rig.Tables.Put(VulnerabilityCacheRows.SelfUpdating("contoso:widget"));
var response = await rig.Function.Run(OperatorCall.Request());
// Within a source the names are in order whatever their case.
var mappings = OperatorCall.BodyOf(response).GetProperty("mappings").EnumerateArray().ToList();
Assert.Equal(
new[]
{
("custom", "contoso agent"), ("custom", "Contoso Widget"), ("auto", "tailspin launcher"),
("community", "northwind tool"), ("seed", "Adatum Reader"), ("seed", "fabrikam gadget"),
},
mappings.Select(m => (m.GetProperty("source").GetString()!, m.GetProperty("normalizedProduct").GetString()!)));
Assert.Equal(6, OperatorCall.BodyOf(response).GetProperty("total").GetInt32());
var widget = mappings[1];
Assert.Equal("contoso:contoso widget", widget.GetProperty("rowKey").GetString());
Assert.True(widget.GetProperty("selfUpdating").GetBoolean());
Assert.True(widget.GetProperty("selfUpdatingDefault").GetBoolean());
Assert.False(mappings[0].GetProperty("selfUpdating").GetBoolean());
}
[Fact]
public async Task Get_ReadsTheMappingsFresh_SoAChangeShowsAtOnce()
{
var rig = new Rig();
rig.Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget"));
Assert.Equal(new[] { ("seed", "fabrikam gadget") }, await rig.ListAsync());
// Stored by another instance: nothing told this one's copy to go.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.CustomPartition, "manual", "contoso", "contoso widget", WidgetCpe));
Assert.Equal(new[] { ("custom", "contoso widget"), ("seed", "fabrikam gadget") }, await rig.ListAsync());
}
[Fact]
public async Task Get_MappingsUnreadable_IsServiceUnavailable()
{
var rig = new Rig();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
var response = await rig.Function.Run(OperatorCall.Request());
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.ServiceUnavailable, EndpointHarness.ErrorCode(response));
Assert.Equal(new[] { "60" }, response.Headers.GetValues("Retry-After"));
}
}
@@ -0,0 +1,120 @@
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// GET vulnerability/unmatched-software pages through the programs without a mapping, most used
/// first, in an order that stays the same from page to page. Every call reads the mappings anew,
/// so a program mapped or ignored a moment ago is gone from the list whichever instance took the
/// change.
/// </summary>
public class GetUnmatchedSoftwareFunctionTests
{
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget", "fabrikam gadget"));
Function = new GetUnmatchedSoftwareFunction(
Tables.Repo.Object,
new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance),
NullLogger<GetUnmatchedSoftwareFunction>.Instance);
}
public MappingTables Tables { get; } = new();
public GetUnmatchedSoftwareFunction Function { get; }
public async Task<JsonElement> GetAsync(string query = "")
{
var response = await Function.Run(EndpointHarness.Request(VulnerabilityScanRig.TenantId, queryString: query).Req);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return OperatorCall.BodyOf(response);
}
}
private static IEnumerable<string> NamesOf(JsonElement body) =>
body.GetProperty("software").EnumerateArray().Select(s => s.GetProperty("softwareName").GetString()!);
[Fact]
public async Task Get_ListsUnmappedProgramsMostUsedFirst_ThenByName_AndPagesThroughThem()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", sessions: 4);
rig.Tables.AddInventory("contoso", "contoso widget", "4.2", sessions: 4);
rig.Tables.AddInventory("adatum", "Adatum Reader", "2.0", sessions: 9);
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 1);
rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0", sessions: 100);
var all = await rig.GetAsync();
var firstPage = await rig.GetAsync("?skip=0&take=3");
var secondPage = await rig.GetAsync("?skip=3&take=3");
// The mapped gadget is not in the list; names used equally often are in order whatever their case.
Assert.Equal(new[] { "Adatum Reader", "contoso widget", "Northwind Tool", "Tailspin Launcher" }, NamesOf(all));
Assert.Equal(4, all.GetProperty("total").GetInt32());
Assert.Equal(0, all.GetProperty("skip").GetInt32());
Assert.Equal(50, all.GetProperty("take").GetInt32());
Assert.Equal(new[] { "Adatum Reader", "contoso widget", "Northwind Tool" }, NamesOf(firstPage));
Assert.Equal(new[] { "Tailspin Launcher" }, NamesOf(secondPage));
Assert.Equal(4, secondPage.GetProperty("total").GetInt32());
Assert.Equal(3, secondPage.GetProperty("skip").GetInt32());
var reader = all.GetProperty("software")[0];
Assert.Equal(9, reader.GetProperty("frequency").GetInt32());
Assert.Equal("adatum", reader.GetProperty("normalizedVendor").GetString());
Assert.Equal("adatum reader", reader.GetProperty("canonicalName").GetString());
}
[Fact]
public async Task Get_MappingsUnreadable_IsServiceUnavailable()
{
var rig = new Rig();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
var response = await rig.Function.Run(EndpointHarness.Request(VulnerabilityScanRig.TenantId).Req);
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.ServiceUnavailable, EndpointHarness.ErrorCode(response));
}
[Fact]
public async Task Get_ProgramMappedOrIgnoredOnAnotherInstance_IsGoneFromTheListOnTheNextCall()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", sessions: 4);
rig.Tables.AddInventory("adatum", "Adatum Reader", "2.0", sessions: 9);
rig.Tables.AddInventory("tailspin", "Tailspin Launcher", "1.0", sessions: 1);
Assert.Equal(3, (await rig.GetAsync()).GetProperty("total").GetInt32());
// Neither write went through this instance, whose copy of the mappings is seconds old.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(
CpeMappingStore.CustomPartition, "manual", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool"));
var ignore = CpeIgnoreEntry.Build("adatum", "Adatum Reader", "manual")!.Value;
rig.Tables.Put(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
var body = await rig.GetAsync();
Assert.Equal(new[] { "Tailspin Launcher" }, NamesOf(body));
Assert.Equal(1, body.GetProperty("total").GetInt32());
}
[Fact]
public async Task Get_MappingsUnreadableOnALaterCall_AnswersFromTheCopyThisInstanceHolds()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1");
rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0");
await rig.GetAsync();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
var body = await rig.GetAsync();
Assert.Equal(new[] { "Northwind Tool" }, NamesOf(body));
}
}
@@ -0,0 +1,218 @@
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The ignore list (<c>vulnerability/ignored-software</c>): an entry is filed like a mapping,
/// under the vendor and the program's name without version, so one entry covers every version,
/// and what the list shows for an entry is what removes it again.
/// </summary>
public class IgnoreSoftwareFunctionTests
{
private const string Ignored = CpeMappingStore.IgnoredPartition;
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget", "fabrikam gadget"));
Store = new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance);
Function = new IgnoreSoftwareFunction(Tables.Repo.Object, Store, NullLogger<IgnoreSoftwareFunction>.Instance);
}
public MappingTables Tables { get; } = new();
public CpeMappingStore Store { get; }
public IgnoreSoftwareFunction Function { get; }
public async Task<JsonElement> AddAsync(params IgnoreSoftwareItem[] items)
{
var response = await Function.AddIgnored(OperatorCall.Request(new IgnoreSoftwareRequest { Items = items.ToList() }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return OperatorCall.BodyOf(response);
}
public async Task<List<JsonElement>> ListAsync()
{
var response = await Function.GetIgnored(OperatorCall.Request());
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return OperatorCall.BodyOf(response).GetProperty("items").EnumerateArray().ToList();
}
/// <summary>Removes an entry the way the portal does: with the name and the publisher the list shows for it.</summary>
public async Task RemoveAsListedAsync(JsonElement listed)
{
var response = await Function.DeleteIgnored(OperatorCall.Request(new IgnoreSoftwareDeleteRequest
{
SoftwareName = listed.GetProperty("softwareName").GetString()!,
Publisher = listed.GetProperty("publisher").GetString(),
}));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}
}
[Fact]
public async Task Add_FilesTheEntryUnderVendorAndNameWithoutVersion_SoItCoversEveryVersion()
{
var rig = new Rig();
Assert.False((await rig.Store.GetAsync()).IsIgnored("contoso", "Contoso Ltd", "Contoso Widget 5.0"));
var body = await rig.AddAsync(new IgnoreSoftwareItem
{
SoftwareName = "Contoso Widget 4.2 (x64)", Publisher = "Contoso Ltd", NormalizedVendor = "contoso",
});
Assert.True(body.GetProperty("success").GetBoolean());
Assert.Equal(1, body.GetProperty("added").GetInt32());
var entry = Assert.Single(rig.Tables.Rows(Ignored));
Assert.Equal("contoso:contoso widget", entry["RowKey"]);
Assert.Equal("contoso", entry["NormalizedVendor"]);
Assert.Equal("contoso widget", entry["NormalizedProduct"]);
Assert.Equal("manual", entry["Reason"]);
// The next reader on this instance already leaves the program out.
Assert.True((await rig.Store.GetAsync()).IsIgnored("contoso", "Contoso Ltd", "Contoso Widget 5.0"));
}
[Fact]
public async Task Add_WithoutAVendor_FilesTheEntryUnderThePublisher_AndSkipsANameThatIsOnlyAVersion()
{
var rig = new Rig();
var body = await rig.AddAsync(
new IgnoreSoftwareItem { SoftwareName = "Contoso Widget 4.2", Publisher = "Contoso Ltd", NormalizedVendor = " ", Reason = "not ours" },
new IgnoreSoftwareItem { SoftwareName = "4.2 (x64)", Publisher = "Contoso Ltd" });
Assert.Equal(1, body.GetProperty("added").GetInt32());
var entry = Assert.Single(rig.Tables.Rows(Ignored));
Assert.Equal("contoso ltd:contoso widget", entry["RowKey"]);
Assert.Equal("contoso ltd", entry["NormalizedVendor"]);
Assert.Equal("not ours", entry["Reason"]);
}
[Fact]
public async Task Delete_WithWhatTheListShowsForAnEntry_RemovesIt()
{
var rig = new Rig();
await rig.AddAsync(
new IgnoreSoftwareItem { SoftwareName = "Contoso Widget 4.2 (x64)", Publisher = "Contoso Ltd", NormalizedVendor = "contoso" },
new IgnoreSoftwareItem { SoftwareName = "Northwind Tool 3.1", Publisher = "Northwind Traders" });
var listed = await rig.ListAsync();
Assert.Equal(new[] { "contoso widget", "northwind tool" }, listed.Select(e => e.GetProperty("softwareName").GetString()));
await rig.RemoveAsListedAsync(listed[0]);
Assert.Equal(new[] { "northwind traders:northwind tool" }, rig.Tables.RowKeys(Ignored));
Assert.False((await rig.Store.GetAsync()).IsIgnored("contoso", "Contoso Ltd", "Contoso Widget 5.0"));
}
[Fact]
public async Task Delete_EntryStoredUnderAnEarlierKey_IsRemovedToo()
{
var rig = new Rig();
// Written before entries were filed under the name without version: the row key and the
// columns hold the publisher and the name as the program showed them.
rig.Tables.Put(Ignored, "contoso ltd:contoso widget 4.2 (x64)", new Dictionary<string, object>
{
["NormalizedVendor"] = "Contoso Ltd",
["NormalizedProduct"] = "Contoso Widget 4.2 (x64)",
["Reason"] = "manual",
["CreatedAt"] = "2026-08-01T08:00:00.0000000Z",
});
// The entry is in effect and listed.
Assert.True((await rig.Store.GetAsync()).IsIgnored("contoso", "Contoso Ltd", "Contoso Widget 5.0"));
var listed = Assert.Single(await rig.ListAsync());
await rig.RemoveAsListedAsync(listed);
// The call answered that the entry was removed.
Assert.Empty(rig.Tables.Rows(Ignored));
}
[Theory]
[InlineData("""{ "items": [] }""")]
[InlineData("{}")]
public async Task Add_WithoutItems_IsABadRequest(string json)
{
var rig = new Rig();
var response = await rig.Function.AddIgnored(OperatorCall.RequestWithJson(json));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Delete_WithoutAName_IsABadRequest()
{
var rig = new Rig();
var response = await rig.Function.DeleteIgnored(OperatorCall.RequestWithJson("""{ "publisher": "Contoso Ltd" }"""));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Empty(rig.Tables.Writes);
}
/// <summary>An entry as it was written while the list took the publisher and the name as the program showed them.</summary>
private static Dictionary<string, object> EarlierEntry(string publisher, string softwareName) => new()
{
["NormalizedVendor"] = publisher,
["NormalizedProduct"] = softwareName,
["Reason"] = "manual",
["CreatedAt"] = "2026-08-01T08:00:00.0000000Z",
};
[Fact]
public async Task Delete_RemovesEveryEntryThatStandsForTheProgram_AndNoOther()
{
var rig = new Rig();
// Two entries from before, for two versions of the program, and one written since.
rig.Tables.Put(Ignored, "contoso ltd:contoso widget 4.2 (x64)", EarlierEntry("Contoso Ltd", "Contoso Widget 4.2 (x64)"));
rig.Tables.Put(Ignored, "contoso ltd:contoso widget 5.0", EarlierEntry("Contoso Ltd", "Contoso Widget 5.0"));
await rig.AddAsync(
new IgnoreSoftwareItem { SoftwareName = "Contoso Widget 6.0", Publisher = "Contoso Ltd" },
// Another program of the publisher, and the same name under another publisher.
new IgnoreSoftwareItem { SoftwareName = "Contoso Widget Viewer", Publisher = "Contoso Ltd" },
new IgnoreSoftwareItem { SoftwareName = "Contoso Widget", Publisher = "Fabrikam Inc" });
Assert.Equal(5, rig.Tables.Rows(Ignored).Count);
var response = await rig.Function.DeleteIgnored(OperatorCall.Request(
new IgnoreSoftwareDeleteRequest { SoftwareName = "contoso widget", Publisher = "contoso ltd" }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(new[] { "contoso ltd:contoso widget viewer", "fabrikam inc:contoso widget" }, rig.Tables.RowKeys(Ignored));
Assert.False((await rig.Store.GetAsync()).IsIgnored("contoso ltd", "Contoso Ltd", "Contoso Widget 7.0"));
}
[Fact]
public async Task Delete_EntryThatIsNotOnTheList_ChangesNothing()
{
var rig = new Rig();
await rig.AddAsync(new IgnoreSoftwareItem { SoftwareName = "Contoso Widget 4.2", Publisher = "Contoso Ltd" });
var writesBefore = rig.Tables.Writes.ToList();
var response = await rig.Function.DeleteIgnored(OperatorCall.Request(
new IgnoreSoftwareDeleteRequest { SoftwareName = "Northwind Tool", Publisher = "Northwind Traders" }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(writesBefore, rig.Tables.Writes);
}
[Fact]
public async Task Delete_ListedEntryWhoseNameIsNoProgramName_IsRemovedToo()
{
var rig = new Rig();
// The list once took any name. This one is a patch number, which names no program: the
// entry ignores nothing any more, but the list still shows it.
rig.Tables.Put(Ignored, "contoso ltd:kb9900001", EarlierEntry("Contoso Ltd", "KB9900001"));
var listed = Assert.Single(await rig.ListAsync());
await rig.RemoveAsListedAsync(listed);
// The call answered that the entry was removed.
Assert.Empty(rig.Tables.Rows(Ignored));
}
}
@@ -0,0 +1,143 @@
using System.Net;
using System.Text.Json;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="KevDataService"/> answers lookups from the catalog it holds and never fetches on
/// that path: a scan asks once per CVE. The catalog changes only through a refresh, and a refresh
/// that fails or comes back empty says so and leaves the loaded catalog in service.
/// </summary>
public class KevDataServiceTests
{
/// <summary>Answers each request with what the test queued for it and counts the requests.</summary>
private sealed class CisaStub : HttpMessageHandler
{
public Func<HttpResponseMessage> Answer { get; set; } = () => new HttpResponseMessage(HttpStatusCode.InternalServerError);
public int Requests { get; private set; }
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
Requests++;
return Task.FromResult(Answer());
}
}
private static Func<HttpResponseMessage> Catalog(params string[] cveIds) =>
() => NvdFixture.Json(ReferenceDataFixture.KevCatalogJson(cveIds));
[Fact]
public void Lookup_BeforeAnythingIsLoaded_FindsNothingAndSendsNoRequest()
{
var cisa = new CisaStub { Answer = Catalog("CVE-2023-4863") };
var kev = ReferenceDataFixture.Kev(cisa);
Assert.Null(kev.Lookup("CVE-2023-4863"));
Assert.Null(kev.Lookup(null));
Assert.Null(kev.Lookup(" "));
Assert.Equal(0, cisa.Requests);
Assert.Equal(0, kev.CatalogSize);
}
[Fact]
public async Task Refresh_LoadsTheCatalog_AndLookupsReadItWithoutFetching()
{
var cisa = new CisaStub { Answer = Catalog("CVE-2023-4863", "CVE-2024-0001") };
var blob = new Mock<BlobClient>();
var kev = ReferenceDataFixture.Kev(cisa, blob);
Assert.True(await kev.RefreshAsync());
var entry = kev.Lookup("cve-2023-4863"); // ids compare without case
Assert.Equal("CVE-2023-4863", entry!.CveID);
Assert.Equal("2026-10-22", entry.DueDate);
Assert.Equal("Apply mitigations per vendor instructions.", entry.RequiredAction);
Assert.Equal("Unknown", entry.KnownRansomwareCampaignUse);
Assert.Null(kev.Lookup("CVE-2020-9999"));
Assert.Equal(2, kev.CatalogSize);
Assert.Equal(1, cisa.Requests);
// The snapshot is kept for the next cold start.
blob.Verify(b => b.UploadAsync(It.IsAny<Stream>(), true, It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task Refresh_ReplacesTheCatalog()
{
var cisa = new CisaStub { Answer = Catalog("CVE-2023-4863") };
var kev = ReferenceDataFixture.Kev(cisa);
await kev.RefreshAsync();
cisa.Answer = Catalog("CVE-2024-0001");
Assert.True(await kev.RefreshAsync());
Assert.Null(kev.Lookup("CVE-2023-4863"));
Assert.NotNull(kev.Lookup("CVE-2024-0001"));
Assert.Equal(1, kev.CatalogSize);
}
private static HttpResponseMessage NoCatalog(string kind) => kind switch
{
"server error" => new HttpResponseMessage(HttpStatusCode.BadGateway),
"empty list" => NvdFixture.Json("""{"title":"CISA Catalog","count":0,"vulnerabilities":[]}"""),
"no list" => NvdFixture.Json("""{"title":"CISA Catalog"}"""),
"not json" => NvdFixture.Json("<html>maintenance</html>"),
"connection lost" => throw new HttpRequestException("connection reset"),
_ => throw new ArgumentOutOfRangeException(nameof(kind)),
};
[Theory]
[InlineData("server error")]
[InlineData("empty list")]
[InlineData("no list")]
[InlineData("not json")]
[InlineData("connection lost")]
public async Task Refresh_FailedOrEmptyAnswer_ReportsItAndKeepsTheLoadedCatalog(string answer)
{
var cisa = new CisaStub { Answer = Catalog("CVE-2023-4863") };
var blob = new Mock<BlobClient>();
var kev = ReferenceDataFixture.Kev(cisa, blob);
Assert.True(await kev.RefreshAsync());
var loadedAt = kev.LastRefresh;
cisa.Answer = () => NoCatalog(answer);
var refreshed = await kev.RefreshAsync();
Assert.False(refreshed);
Assert.NotNull(kev.Lookup("CVE-2023-4863"));
Assert.Equal(1, kev.CatalogSize);
Assert.Equal(loadedAt, kev.LastRefresh);
// Nothing new to keep: only the successful refresh wrote the snapshot.
blob.Verify(b => b.UploadAsync(It.IsAny<Stream>(), true, It.IsAny<CancellationToken>()), Times.Once);
}
[Fact]
public async Task EnsureHydrated_LoadsTheSnapshotFromItsBlob_WithoutAskingCisa()
{
var snapshot = new BlobBackedReferenceCache<KevEntry>.Snapshot
{
LastRefresh = "2026-11-11T03:00:00.0000000Z",
Entries = { ["CVE-2023-4863"] = new KevEntry { CveID = "CVE-2023-4863", DueDate = "2023-10-04" } },
};
var blob = new Mock<BlobClient>();
blob.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>())).ReturnsAsync(Response.FromValue(true, Mock.Of<Response>()));
blob.Setup(b => b.DownloadContentAsync(It.IsAny<CancellationToken>())).ReturnsAsync(Response.FromValue(
BlobsModelFactory.BlobDownloadResult(content: BinaryData.FromString(JsonSerializer.Serialize(snapshot))),
Mock.Of<Response>()));
var cisa = new CisaStub();
var kev = ReferenceDataFixture.Kev(cisa, blob);
await kev.EnsureHydratedAsync();
await kev.EnsureHydratedAsync();
Assert.Equal("2023-10-04", kev.Lookup("CVE-2023-4863")!.DueDate);
Assert.Equal(new DateTime(2026, 11, 11, 3, 0, 0, DateTimeKind.Utc), kev.LastRefresh);
Assert.Equal(0, cisa.Requests);
blob.Verify(b => b.DownloadContentAsync(It.IsAny<CancellationToken>()), Times.Once);
}
}
@@ -0,0 +1,213 @@
using System.Net;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="NvdApiClient.SearchCvesByCpeAsync"/> reads a product's CVEs in full pages: one page
/// when NVD has no more, the first and the last page when it has, and never half an answer. A
/// request that runs into its own time limit fails that request; only the caller's cancellation
/// ends the call with an exception.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class NvdApiClientPagingTests : IDisposable
{
private const string Cpe = "cpe:2.3:a:google:chrome";
public NvdApiClientPagingTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
/// <summary>NVD pages by position: the page at <c>startIndex</c> holds the CVEs numbered from there.</summary>
private static HttpResponseMessage PageOf(int totalResults, NvdRequest request) => NvdFixture.Page(
totalResults,
Enumerable.Range(request.StartIndex + 1, Math.Min(request.ResultsPerPage, totalResults - request.StartIndex))
.Select(n => new NvdVulnerabilityItem { Cve = new NvdCve { Id = $"CVE-2020-{n:00000}" } }));
[Theory]
[InlineData(0)]
[InlineData(137)]
[InlineData(2000)]
public async Task UpToOnePage_IsOneRequestForAFullPageFromTheStart(int totalResults)
{
var nvd = new NvdStubHandler(request => PageOf(totalResults, request));
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
var request = Assert.Single(nvd.Requests);
Assert.Equal(Cpe, request.Cpe);
Assert.EndsWith("&resultsPerPage=2000&startIndex=0", request.Uri.Query);
Assert.True(result.Succeeded);
Assert.Equal(totalResults, result.Data.TotalResults);
Assert.Equal(totalResults, result.Data.Vulnerabilities.Count);
}
[Theory]
[InlineData(4500, 2500, 4000)] // the middle 500 lie years back and are left out
[InlineData(2500, 500, 2500)] // the two pages overlap by 1500
[InlineData(2001, 1, 2001)]
public async Task MoreThanOnePage_AlsoFetchesTheLastPageAndJoinsBoth(int totalResults, int lastPageStart, int expectedCves)
{
var nvd = new NvdStubHandler(request => PageOf(totalResults, request));
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
Assert.Equal(new[] { 0, lastPageStart }, nvd.Requests.Select(r => r.StartIndex));
Assert.All(nvd.Requests, r => Assert.Equal(2000, r.ResultsPerPage));
Assert.True(result.Succeeded);
Assert.Equal(totalResults, result.Data.TotalResults);
var ids = result.Data.Vulnerabilities.Select(v => v.Cve!.Id!).ToList();
Assert.Equal(expectedCves, ids.Count);
Assert.Equal(expectedCves, ids.Distinct(StringComparer.OrdinalIgnoreCase).Count());
Assert.Contains("CVE-2020-00001", ids);
Assert.Contains($"CVE-2020-{totalResults:00000}", ids);
}
[Theory]
[InlineData(HttpStatusCode.InternalServerError, false)]
[InlineData(HttpStatusCode.TooManyRequests, true)]
public async Task LastPageFails_TheWholeCallFails(HttpStatusCode lastPageStatus, bool throttled)
{
var nvd = new NvdStubHandler(request =>
request.StartIndex == 0 ? PageOf(4500, request) : NvdFixture.Status(lastPageStatus));
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
Assert.Equal(2, nvd.Requests.Count);
Assert.False(result.Succeeded);
Assert.Equal(throttled, result.Throttled);
// No half answer: the first page is not handed out without the newest CVEs.
Assert.Empty(result.Data.Vulnerabilities);
Assert.Equal(throttled, NvdApiClient.IsInCooldown);
}
[Fact]
public async Task FirstPageFails_NoSecondRequest()
{
var nvd = new NvdStubHandler(_ => NvdFixture.Status(HttpStatusCode.ServiceUnavailable));
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
Assert.Single(nvd.Requests);
Assert.False(result.Succeeded);
Assert.False(result.Throttled);
}
/// <summary>
/// The 45 s limit itself is not waited for here. What it does to a request is the same as
/// this: the send is cancelled although the caller's token is still live.
/// </summary>
[Fact]
public async Task RequestCancelledByItsOwnTimeLimit_IsAFailedResultNotAnException()
{
var nvd = new NvdStubHandler((_, _) => throw new TaskCanceledException("the request ran out of time"));
using var caller = new CancellationTokenSource();
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe, caller.Token);
Assert.False(result.Succeeded);
Assert.False(result.Throttled);
Assert.Empty(result.Data.Vulnerabilities);
}
/// <summary>The limit also covers the body, which is read after the headers have arrived.</summary>
[Fact]
public async Task BodyReadCancelledByItsOwnTimeLimit_IsAFailedResultNotAnException()
{
var nvd = new NvdStubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new CancelledBody() });
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
Assert.False(result.Succeeded);
Assert.False(result.Throttled);
}
[Fact]
public async Task CallerCancelsDuringTheRequest_Propagates()
{
var sent = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var nvd = new NvdStubHandler(async (_, ct) =>
{
sent.SetResult();
await Task.Delay(Timeout.Infinite, ct);
return NvdFixture.Page();
});
using var caller = new CancellationTokenSource();
var call = NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe, caller.Token);
await sent.Task.WaitAsync(TimeSpan.FromSeconds(30));
caller.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => call);
}
[Fact]
public async Task CallerCancelledBeforeTheCall_PropagatesWithoutARequest()
{
var nvd = new NvdStubHandler(_ => NvdFixture.Page());
using var caller = new CancellationTokenSource();
caller.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(
() => NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe, caller.Token));
Assert.Empty(nvd.Requests);
}
[Fact]
public async Task ReadsTheDateCisaListedACveAsExploited()
{
// Field names as NVD sends them (API 2.0), shaped like the public CVE-2023-4863.
const string body = """
{"resultsPerPage":2,"startIndex":0,"totalResults":2,"format":"NVD_CVE","version":"2.0",
"vulnerabilities":[
{"cve":{"id":"CVE-2023-4863","published":"2023-09-12T15:15:24.327","lastModified":"2025-07-30T01:00:01.000",
"vulnStatus":"Analyzed","cisaExploitAdd":"2023-09-13","cisaActionDue":"2023-10-04",
"descriptions":[{"lang":"en","value":"Heap buffer overflow in libwebp."}],
"metrics":{"cvssMetricV31":[{"type":"Primary","cvssData":{"version":"3.1","vectorString":"CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H","baseScore":8.8,"baseSeverity":"HIGH"}}]},
"configurations":[{"nodes":[{"operator":"OR","negate":false,"cpeMatch":[
{"vulnerable":true,"criteria":"cpe:2.3:a:google:chrome:*:*:*:*:*:*:*:*","versionEndExcluding":"116.0.5845.187"}]}]}]}},
{"cve":{"id":"CVE-2024-0001","published":"2024-01-02T10:00:00.000","descriptions":[],"configurations":[]}}
]}
""";
var nvd = new NvdStubHandler(_ => NvdFixture.Json(body));
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(Cpe);
Assert.True(result.Succeeded);
var listed = result.Data.Vulnerabilities.Single(v => v.Cve!.Id == "CVE-2023-4863").Cve!;
Assert.Equal("2023-09-13", listed.CisaExploitAdd);
Assert.Equal("116.0.5845.187", listed.Configurations.Single().Nodes.Single().CpeMatch.Single().VersionEndExcluding);
Assert.Null(result.Data.Vulnerabilities.Single(v => v.Cve!.Id == "CVE-2024-0001").Cve!.CisaExploitAdd);
}
/// <summary>A body whose read ends the way a cancelled read does.</summary>
private sealed class CancelledBody : HttpContent
{
protected override Task<Stream> CreateContentReadStreamAsync() => Task.FromResult<Stream>(new CancelledStream());
protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) =>
throw new OperationCanceledException();
protected override bool TryComputeLength(out long length)
{
length = 0;
return false;
}
}
private sealed class CancelledStream : Stream
{
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override long Length => throw new NotSupportedException();
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
public override void Flush() { }
public override int Read(byte[] buffer, int offset, int count) => throw new OperationCanceledException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
}
}
@@ -0,0 +1,129 @@
using System.Net;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// What <see cref="NvdApiClient"/> makes of an answer that is no result. NVD refusing the client,
/// with 429 from the API or 403 from the gateway in front of it, opens the process-wide cooldown
/// in which nothing more is sent. A server error, a timeout or a broken connection is no answer:
/// a caller that asks in a loop ends the loop on it. Any other refusal concerns the one request.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class NvdApiClientRefusalTests : IDisposable
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
public NvdApiClientRefusalTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
[Theory]
[InlineData(HttpStatusCode.TooManyRequests)]
[InlineData(HttpStatusCode.Forbidden)]
public async Task RefusalOfTheClient_IsThrottled_AndNothingIsSentWhileTheCooldownRuns(HttpStatusCode status)
{
var nvd = new NvdStubHandler(_ => NvdFixture.Status(status));
var client = NvdFixture.Client(nvd);
var refused = await client.SearchCvesByCpeAsync(WidgetCpe);
Assert.False(refused.Succeeded);
Assert.True(refused.Throttled);
Assert.False(refused.Unanswered);
Assert.Empty(refused.Data.Vulnerabilities);
Assert.True(NvdApiClient.IsInCooldown);
// The next product is not asked about, by this client or by another one of the process.
var otherNvd = new NvdStubHandler(_ => NvdFixture.Page());
var next = await client.SearchCvesByCpeAsync(GadgetCpe);
var fromAnotherClient = await NvdFixture.Client(otherNvd).SearchCvesByCpeAsync(GadgetCpe);
Assert.All(new[] { next, fromAnotherClient }, result =>
{
Assert.False(result.Succeeded);
Assert.True(result.Throttled);
});
Assert.Equal(new[] { WidgetCpe }, nvd.Requests.Select(r => r.Cpe));
Assert.Empty(otherNvd.Requests);
}
[Fact]
public async Task CooldownRemaining_IsHowLongARefusalStillBlocks_AndZeroWithoutOne()
{
Assert.Equal(TimeSpan.Zero, NvdApiClient.CooldownRemaining);
await NvdFixture.Client(new NvdStubHandler(_ => NvdFixture.Status(HttpStatusCode.TooManyRequests))).SearchCvesByCpeAsync(WidgetCpe);
Assert.InRange(
NvdApiClient.CooldownRemaining, NvdApiClient.ThrottleCooldown - TimeSpan.FromSeconds(5), NvdApiClient.ThrottleCooldown);
NvdApiClient.ResetThrottleState();
Assert.Equal(TimeSpan.Zero, NvdApiClient.CooldownRemaining);
}
[Theory]
[InlineData(HttpStatusCode.InternalServerError)]
[InlineData(HttpStatusCode.BadGateway)]
[InlineData(HttpStatusCode.ServiceUnavailable)]
[InlineData(HttpStatusCode.GatewayTimeout)]
public async Task ServerError_IsNoAnswer_AndOpensNoCooldown(HttpStatusCode status)
{
var nvd = new NvdStubHandler(_ => NvdFixture.Status(status));
var client = NvdFixture.Client(nvd);
var result = await client.SearchCvesByCpeAsync(WidgetCpe);
Assert.False(result.Succeeded);
Assert.False(result.Throttled);
Assert.True(result.Unanswered);
Assert.False(NvdApiClient.IsInCooldown);
await client.SearchCvesByCpeAsync(GadgetCpe);
Assert.Equal(new[] { WidgetCpe, GadgetCpe }, nvd.Requests.Select(r => r.Cpe));
}
[Theory]
[InlineData(HttpStatusCode.BadRequest)]
[InlineData(HttpStatusCode.NotFound)]
public async Task RequestNvdTurnsDown_FailsThatRequestOnly(HttpStatusCode status)
{
var nvd = new NvdStubHandler(request => request.Cpe == WidgetCpe ? NvdFixture.Status(status) : NvdFixture.Page());
var client = NvdFixture.Client(nvd);
var turnedDown = await client.SearchCvesByCpeAsync(WidgetCpe);
Assert.False(turnedDown.Succeeded);
Assert.False(turnedDown.Throttled);
// NVD answered: that says nothing about the next product.
Assert.False(turnedDown.Unanswered);
Assert.False(NvdApiClient.IsInCooldown);
var next = await client.SearchCvesByCpeAsync(GadgetCpe);
Assert.True(next.Succeeded);
Assert.Equal(new[] { WidgetCpe, GadgetCpe }, nvd.Requests.Select(r => r.Cpe));
}
[Theory]
[InlineData("the connection breaks")]
[InlineData("the request runs out of time")]
[InlineData("the body is not what NVD sends")]
public async Task RequestWithoutAUsableAnswer_IsUnanswered_AndOpensNoCooldown(string way)
{
var nvd = new NvdStubHandler((_, _) => way switch
{
"the connection breaks" => throw new HttpRequestException("connection reset"),
"the request runs out of time" => throw new TaskCanceledException("the request ran out of time"),
_ => Task.FromResult(NvdFixture.Json("<html>maintenance</html>")),
});
var result = await NvdFixture.Client(nvd).SearchCvesByCpeAsync(WidgetCpe);
Assert.False(result.Succeeded);
Assert.False(result.Throttled);
Assert.True(result.Unanswered);
Assert.Empty(result.Data.Vulnerabilities);
Assert.False(NvdApiClient.IsInCooldown);
}
}
@@ -0,0 +1,66 @@
using System.Net;
using AutopilotMonitor.Functions.Helpers;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The HTTP policy in front of the NVD client (<see cref="ResiliencePolicies.NvdApi"/>) retries
/// once for a server error or a broken connection and for nothing else. A 429 has to reach the
/// client, which opens its cooldown instead of asking again, and a page that timed out is not
/// fetched a second time inside a run with a time budget.
/// </summary>
public class NvdResiliencePolicyTests
{
/// <summary>Runs one request through a fresh policy; <paramref name="attempt"/> gets the attempt number, from 1.</summary>
private static async Task<(HttpStatusCode? Status, Exception? Error, int Attempts)> SendAsync(
Func<int, HttpResponseMessage> attempt)
{
var policy = new ResiliencePolicies(NullLoggerFactory.Instance).NvdApi;
var attempts = 0;
try
{
var response = await policy.ExecuteAsync(() => Task.FromResult(attempt(++attempts)));
return (response.StatusCode, null, attempts);
}
catch (Exception ex)
{
return (null, ex, attempts);
}
}
[Fact]
public async Task Refusal_IsNotRetried()
{
var (status, _, attempts) = await SendAsync(_ => new HttpResponseMessage(HttpStatusCode.TooManyRequests));
Assert.Equal(HttpStatusCode.TooManyRequests, status);
Assert.Equal(1, attempts);
}
[Fact]
public async Task Timeout_IsNotRetried()
{
var (_, error, attempts) = await SendAsync(_ => throw new TaskCanceledException("the request ran out of time"));
Assert.IsType<TaskCanceledException>(error);
Assert.Equal(1, attempts);
}
/// <summary>Both cases wait out the policy's pause before the retry, so they run side by side.</summary>
[Fact]
public async Task ServerErrorOrBrokenConnection_IsRetriedExactlyOnce()
{
var serverError = SendAsync(_ => new HttpResponseMessage(HttpStatusCode.BadGateway));
var brokenConnection = SendAsync(n =>
n == 1 ? throw new HttpRequestException("connection reset") : new HttpResponseMessage(HttpStatusCode.OK));
var afterServerError = await serverError;
var afterBrokenConnection = await brokenConnection;
Assert.Equal(HttpStatusCode.BadGateway, afterServerError.Status);
Assert.Equal(2, afterServerError.Attempts);
Assert.Equal(HttpStatusCode.OK, afterBrokenConnection.Status);
Assert.Equal(2, afterBrokenConnection.Attempts);
}
}
@@ -0,0 +1,240 @@
using System.Net;
using AutopilotMonitor.Functions.Functions.Admin;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using static AutopilotMonitor.Functions.Tests.MappingListFixture;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The CPE part of POST rules/reseed-from-github (<c>?type=cpe</c>) in
/// <see cref="ReseedFromGitHubFunction"/>: both mapping lists are imported, the inventory rows
/// follow in the same call, and a list that is refused is reported with its reasons, in the
/// answer and (by the importer) as an ops event, while the mappings in service stay.
/// </summary>
public class ReseedFromGitHubFunctionTests
{
private const string Seed = CpeMappingStore.SeedPartition;
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private const string ReaderCpe = "cpe:2.3:a:adatum:reader";
private const string EmptyCommunityList = """{ "version": "1.0.0", "lastUpdated": "2025-03-20", "mappings": [] }""";
private sealed class Rig
{
private Rig(MappingRig maps)
{
Maps = maps;
var rules = Rules.Repository();
// A reseed of the CPE lists alone reaches neither the rule services nor the rule repository.
Function = new ReseedFromGitHubFunction(
NullLogger<ReseedFromGitHubFunction>.Instance,
rules,
gatherRuleService: null!,
analyzeRuleService: null!,
new Mock<IRuleRepository>(MockBehavior.Strict).Object,
new CpeMappingImporter(
maps.Scan.Repo.Object, rules, maps.Flags, maps.Store, OpsEvents.Object, NullLogger<CpeMappingImporter>.Instance),
maps.Reconciler);
}
/// <summary>Records nothing; a test verifies what the import reported.</summary>
public Mock<OpsEventService> OpsEvents { get; } = new(null!, null!, null!) { CallBase = false };
public static async Task<Rig> CreateAsync(params Dictionary<string, object>[] mappingsInService) =>
new(await MappingRig.CreateAsync(mappingsInService));
public MappingRig Maps { get; }
public MappingTables Tables => Maps.Tables;
public RulesRepositoryStub Rules { get; } = new() { CommunityList = EmptyCommunityList };
public ReseedFromGitHubFunction Function { get; }
public Task<HttpResponseData> ReseedCpeAsync() =>
Function.Reseed(EndpointHarness.Request(VulnerabilityScanRig.TenantId, queryString: "?type=cpe").Req);
}
[Fact]
public async Task ReseedCpe_ImportsBothLists_AndBringsTheInventoryInLineInTheSameCall()
{
var rig = await Rig.CreateAsync(
VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget_old", "contoso widget"),
VulnerabilityCacheRows.Mapping("northwind", "northwind tool", "cpe:2.3:a:northwind:tool", "northwind tool"));
rig.Rules.SeedList = Json(
new[] { Entry("contoso", "contoso widget", WidgetCpe), Entry("fabrikam", "fabrikam gadget", GadgetCpe) }, "contoso:widget");
rig.Rules.CommunityList = Json(new[] { Entry("adatum", "adatum reader", ReaderCpe) });
var corrected = rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2", cpeUri: "cpe:2.3:a:contoso:widget_old");
var withdrawn = rig.Tables.AddInventory("northwind", "Northwind Tool 3.1", "3.1", cpeUri: "cpe:2.3:a:northwind:tool");
var added = rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0");
var response = await rig.ReseedCpeAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.True(body.GetProperty("success").GetBoolean());
Assert.Equal("Reseed from GitHub complete", body.GetProperty("message").GetString());
Assert.False(body.TryGetProperty("cpeProblems", out _));
Assert.Equal((1, 2), Counts(body.GetProperty("cpeSeedMappings")));
Assert.Equal((0, 1), Counts(body.GetProperty("cpeCommunityMappings")));
// The rule catalogs are no part of this reseed.
Assert.Equal((0, 0), Counts(body.GetProperty("gather")));
Assert.Equal((0, 0), Counts(body.GetProperty("analyze")));
Assert.Equal((0, 0), Counts(body.GetProperty("ime")));
var inventory = body.GetProperty("cpeInventory");
Assert.Equal(3, inventory.GetProperty("rows").GetInt32());
Assert.Equal(2, inventory.GetProperty("set").GetInt32());
Assert.Equal(1, inventory.GetProperty("cleared").GetInt32());
Assert.Equal(0, inventory.GetProperty("failed").GetInt32());
Assert.Equal(2, inventory.GetProperty("productsNoted").GetInt32());
Assert.Equal(WidgetCpe, corrected["CpeUri"]);
Assert.Equal(string.Empty, withdrawn["CpeUri"]);
Assert.Equal(ReaderCpe, added["CpeUri"]);
Assert.Equal(new[] { RulesRepositoryStub.CommunityListPath, RulesRepositoryStub.SeedListPath }, rig.Rules.RequestedPaths);
Assert.True((await rig.Maps.Store.GetAsync()).IsSelfUpdating("contoso:widget"));
Assert.Empty(RefusalsReported(rig));
}
[Fact]
public async Task ReseedCpe_ListThatIsRefused_IsReportedWithItsReasons_AndTheMappingsInServiceStay()
{
var rig = await Rig.CreateAsync(Tools(4)
.Select(entry => VulnerabilityCacheRows.Mapping(entry.NormalizedVendor!, entry.NormalizedProduct!, entry.CpeUri!))
.ToArray());
var inService = rig.Tables.RowKeys(Seed).ToList();
rig.Rules.SeedList = Json(Tools(1));
var response = await rig.ReseedCpeAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.False(body.GetProperty("success").GetBoolean());
// The message names the part that fell short; why it did is in the problems.
Assert.Contains("except for the CPE mappings", body.GetProperty("message").GetString());
var problem = Assert.Single(body.GetProperty("cpeProblems").EnumerateArray()).GetString();
Assert.StartsWith("seed list: ", problem);
Assert.Contains("shrinks", problem);
Assert.Contains("refused", problem);
Assert.Equal((0, 0), Counts(body.GetProperty("cpeSeedMappings")));
Assert.Equal(inService, rig.Tables.RowKeys(Seed));
Assert.Empty(rig.Tables.Writes);
// Nobody may be looking at the answer of a scheduled reseed: the refusal is an ops event too.
var reported = Assert.Single(RefusalsReported(rig));
Assert.Equal("seed", reported.List);
Assert.Equal(problem, "seed list: " + Assert.Single(reported.Problems));
}
[Fact]
public async Task ReseedCpe_BothListsRefused_AreBothInTheAnswer_AndEachIsReported()
{
var rig = await Rig.CreateAsync(Tools(4)
.Select(entry => VulnerabilityCacheRows.Mapping(entry.NormalizedVendor!, entry.NormalizedProduct!, entry.CpeUri!))
.ToArray());
rig.Rules.CommunityList = "{ \"mappings\": [";
rig.Rules.SeedList = Json(Tools(1));
var response = await rig.ReseedCpeAsync();
var problems = OperatorCall.BodyOf(response).GetProperty("cpeProblems").EnumerateArray().Select(p => p.GetString()!).ToList();
Assert.Equal(2, problems.Count);
Assert.StartsWith("community list: not valid JSON", problems[0]);
Assert.StartsWith("seed list: ", problems[1]);
Assert.Equal(problems, RefusalsReported(rig).Select(reported => $"{reported.List} list: {Assert.Single(reported.Problems)}"));
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task ReseedCpe_ImportedListsThatCannotBeReadBack_SayThatTheInventoryRowsWait()
{
var rig = await ImportedButNotReadBackAsync();
var installed = rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2");
var response = await rig.ReseedCpeAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.Equal((0, 1), Counts(body.GetProperty("cpeSeedMappings")));
Assert.Equal(new[] { "contoso:contoso widget" }, rig.Tables.RowKeys(Seed));
// The answer does not pass the import off as finished.
var problem = Assert.Single(body.GetProperty("cpeProblems").EnumerateArray()).GetString();
Assert.Contains("could not be read back", problem);
Assert.False(body.TryGetProperty("cpeInventory", out _));
Assert.Equal(string.Empty, installed["CpeUri"]);
}
[Fact]
public async Task ReseedCpe_ImportedListsThatCannotBeReadBack_AreNotReportedAsARefusedList()
{
var rig = await ImportedButNotReadBackAsync();
await rig.ReseedCpeAsync();
// No list was refused and the mappings in service were replaced: the event that says
// "list refused, the mappings in service were kept" is not this reseed's.
Assert.Empty(RefusalsReported(rig));
}
/// <summary>
/// A first import whose seed list is fine, on an instance that holds no copy of the mappings
/// and cannot read the table when the reconcile asks for them.
/// </summary>
private static async Task<Rig> ImportedButNotReadBackAsync()
{
var rig = await Rig.CreateAsync();
rig.Rules.SeedList = Json(new[] { Entry("contoso", "contoso widget", WidgetCpe) });
// The import reads and writes single partitions; only the reconcile loads the mappings as a whole.
rig.Maps.Scan.Repo.Setup(r => r.GetAllCpeMappingsAsync())
.ThrowsAsync(new InvalidOperationException("table storage is not reachable"));
return rig;
}
[Fact]
public async Task ReseedCpe_ListThatCannotBeFetched_IsABadGateway()
{
var rig = await Rig.CreateAsync(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", WidgetCpe, "contoso widget"));
rig.Rules.SeedList = null;
var response = await rig.ReseedCpeAsync();
Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.UpstreamError, EndpointHarness.ErrorCode(response));
Assert.Equal(new[] { "contoso:contoso widget" }, rig.Tables.RowKeys(Seed));
}
[Fact]
public async Task ReseedCpe_FirstImportThatIsRefused_StillSaysWhy()
{
// Nothing is in service yet, as on a new installation, and the first seed list has a mistake.
var rig = await Rig.CreateAsync();
rig.Rules.SeedList = Json(new[] { Entry("contoso", "contoso widget", "cpe:2.3:a:contoso:widget:4.2") });
var response = await rig.ReseedCpeAsync();
// Without a single mapping the inventory cannot be reconciled, but that is no reason to
// lose the one thing the operator needs: why the list was turned down.
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.False(body.GetProperty("success").GetBoolean());
Assert.Contains("write the CPE as", Assert.Single(body.GetProperty("cpeProblems").EnumerateArray()).GetString());
Assert.Contains("except for the CPE mappings", body.GetProperty("message").GetString());
// No inventory row was looked at, so the answer carries no counts for them.
Assert.False(body.TryGetProperty("cpeInventory", out _));
Assert.Empty(rig.Tables.Writes);
Assert.Empty(rig.Tables.InventoryWrites);
Assert.Single(RefusalsReported(rig));
}
/// <summary>The lists reported as refused, each with its problems, one entry per report.</summary>
private static List<(string List, IReadOnlyList<string> Problems)> RefusalsReported(Rig rig) => rig.OpsEvents.Invocations
.Where(invocation => invocation.Method.Name == nameof(OpsEventService.RecordCpeMappingListRefusedAsync))
.Select(invocation => ((string)invocation.Arguments[0]!, (IReadOnlyList<string>)invocation.Arguments[1]!))
.ToList();
private static (int Deleted, int Written) Counts(System.Text.Json.JsonElement node) =>
(node.GetProperty("deleted").GetInt32(), node.GetProperty("written").GetInt32());
}
@@ -0,0 +1,222 @@
using System.Net;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// POST vulnerability/cpe-mapping saves an operator's mapping for one program, filed under its
/// vendor and its name without version, architecture and language, and puts the CPE onto the
/// program's inventory rows in the same call.
/// </summary>
public class SaveCustomCpeMappingFunctionTests
{
private const string Custom = CpeMappingStore.CustomPartition;
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private sealed class Rig
{
private Rig(MappingRig maps)
{
Maps = maps;
Function = new SaveCustomCpeMappingFunction(
maps.Scan.Repo.Object, maps.Reconciler, NullLogger<SaveCustomCpeMappingFunction>.Instance);
}
public static async Task<Rig> CreateAsync() => new(await MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget", "fabrikam gadget")));
public MappingRig Maps { get; }
public MappingTables Tables => Maps.Tables;
public SaveCustomCpeMappingFunction Function { get; }
public Task<HttpResponseData> SaveAsync(SaveCustomCpeMappingRequest request) => Function.Run(OperatorCall.Request(request));
}
[Fact]
public async Task Save_FilesTheMappingUnderVendorAndNameWithoutVersion_AndPutsTheCpeOnEveryVersionInTheInventory()
{
var rig = await Rig.CreateAsync();
var named = rig.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2");
var otherVersion = rig.Tables.AddInventory("contoso", "Contoso Widget 5.0", "5.0");
var otherProgram = rig.Tables.AddInventory("contoso", "Contoso Widget Viewer", "1.0");
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest
{
NormalizedVendor = "Contoso",
NormalizedProduct = "Contoso Widget 4.2 (x64)",
CpeUri = "cpe:2.3:a:Contoso:Widget:4.2:*:*:*:*:*:*:*",
PublisherPatterns = new List<string> { "contoso ltd" },
// A pattern without a word could never match and would fail the next list check.
ExcludePatterns = new List<string> { "widget viewer", "--" },
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.True(body.GetProperty("success").GetBoolean());
Assert.Equal("contoso:contoso widget", body.GetProperty("rowKey").GetString());
Assert.Equal(2, body.GetProperty("inventoryRowsChanged").GetInt32());
var row = Assert.Single(rig.Tables.Rows(Custom));
Assert.Equal("contoso:contoso widget", row["RowKey"]);
Assert.Equal("contoso", row["NormalizedVendor"]);
Assert.Equal("contoso widget", row["NormalizedProduct"]);
Assert.Equal(WidgetCpe, row["CpeUri"]);
Assert.Equal("contoso", row["CpeVendor"]);
Assert.Equal("widget", row["CpeProduct"]);
Assert.Equal("custom", row["Category"]);
Assert.Equal("manual", row["Source"]);
Assert.Equal("[]", row["DisplayNamePatternsJson"]);
Assert.Equal("""["contoso ltd"]""", row["PublisherPatternsJson"]);
Assert.Equal("""["widget viewer"]""", row["ExcludePatternsJson"]);
Assert.Equal(OperatorCall.Upn, row["CreatedBy"]);
Assert.Equal(WidgetCpe, named["CpeUri"]);
Assert.Equal(WidgetCpe, otherVersion["CpeUri"]);
Assert.Equal(string.Empty, otherProgram["CpeUri"]);
}
[Fact]
public async Task Save_AgainForTheSameProgram_ReplacesItsMapping()
{
var rig = await Rig.CreateAsync();
var installed = rig.Tables.AddInventory("contoso", "Contoso Widget 4.2", "4.2");
await rig.SaveAsync(new SaveCustomCpeMappingRequest { NormalizedVendor = "contoso", NormalizedProduct = "Contoso Widget 4.2", CpeUri = "cpe:2.3:a:contoso:widget_wrong" });
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest { NormalizedVendor = "contoso", NormalizedProduct = "contoso widget", CpeUri = WidgetCpe });
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(WidgetCpe, Assert.Single(rig.Tables.Rows(Custom))["CpeUri"]);
Assert.Equal(WidgetCpe, installed["CpeUri"]);
}
[Fact]
public async Task Save_CorrectionOfAMappingStoredUnderAnEarlierKey_TakesEffect()
{
var rig = await Rig.CreateAsync();
// Stored before programs were filed under their name without version: the key holds the
// name as it was typed, and no vendor.
rig.Tables.Put(Custom, ":contoso reader 2.0 (x64)", new Dictionary<string, object>
{
["NormalizedVendor"] = string.Empty,
["NormalizedProduct"] = "Contoso Reader 2.0 (x64)",
["CpeUri"] = "cpe:2.3:a:contoso:reader_wrong",
["Source"] = "manual",
});
var installed = rig.Tables.AddInventory("", "Contoso Reader 2.0 (x64)", "2.0", cpeUri: "cpe:2.3:a:contoso:reader_wrong");
// The portal's edit sends the vendor and the name the list shows for the mapping.
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest
{
NormalizedVendor = string.Empty,
NormalizedProduct = "Contoso Reader 2.0 (x64)",
CpeUri = "cpe:2.3:a:contoso:reader",
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal("cpe:2.3:a:contoso:reader", installed["CpeUri"]);
}
[Theory]
[InlineData("", WidgetCpe)]
[InlineData("4.2 (x64)", WidgetCpe)] // no name is left once the version is gone
[InlineData("Contoso Widget", "")]
[InlineData("Contoso Widget", "cpe:2.3:a:contoso")]
[InlineData("Contoso Widget", "cpe:2.3:a:contoso:*")]
[InlineData("Contoso Widget", "contoso widget")]
public async Task Save_WithoutAProgramNameOrACpeThatNamesAProduct_IsABadRequest(string product, string cpeUri)
{
var rig = await Rig.CreateAsync();
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest { NormalizedVendor = "contoso", NormalizedProduct = product, CpeUri = cpeUri });
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.BadRequest, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Save_MappingsUnreadableAfterTheWrite_StillSavesAndLeavesTheRowCountOut()
{
var rig = await Rig.CreateAsync();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest { NormalizedVendor = "contoso", NormalizedProduct = "Contoso Widget", CpeUri = WidgetCpe });
// The mapping is stored; the nightly run brings the inventory rows in line.
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.False(OperatorCall.BodyOf(response).TryGetProperty("inventoryRowsChanged", out _));
Assert.Equal(new[] { $"upsert {Custom}/contoso:contoso widget" }, rig.Tables.Writes);
}
// -----------------------------------------------------------------------
// Earlier rows of the same program
// -----------------------------------------------------------------------
/// <summary>A mapping as it was stored before programs were filed under their name without version.</summary>
private static Dictionary<string, object> Earlier(string vendor, string product, string cpeUri) => new()
{
["NormalizedVendor"] = vendor,
["NormalizedProduct"] = product,
["CpeUri"] = cpeUri,
["Source"] = "manual",
};
[Fact]
public async Task Save_RemovesEarlierRowsOfTheSameProgram_AndNoOtherRow()
{
var rig = await Rig.CreateAsync();
rig.Tables.Put(Custom, "contoso:contoso reader 2.0 (x64)", Earlier("contoso", "Contoso Reader 2.0 (x64)", "cpe:2.3:a:contoso:reader_wrong"));
rig.Tables.Put(Custom, "contoso:contoso reader 1.0", Earlier("Contoso", "Contoso Reader 1.0", "cpe:2.3:a:contoso:reader_older"));
// Another program of the vendor, and the same name under another vendor.
rig.Tables.Put(Custom, "contoso:contoso reader pro", Earlier("contoso", "Contoso Reader Pro", "cpe:2.3:a:contoso:reader_pro"));
rig.Tables.Put(Custom, "fabrikam:contoso reader 2.0", Earlier("fabrikam", "Contoso Reader 2.0", "cpe:2.3:a:fabrikam:reader"));
// The automatic mapping of the same program is not an operator's row.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(CpeMappingStore.AutoPartition, "auto", "contoso", "contoso reader", "cpe:2.3:a:contoso:reader_auto"));
var response = await rig.SaveAsync(new SaveCustomCpeMappingRequest
{
NormalizedVendor = "contoso", NormalizedProduct = "Contoso Reader 3.0", CpeUri = "cpe:2.3:a:contoso:reader",
});
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.Equal(
new[] { "contoso:contoso reader", "contoso:contoso reader pro", "fabrikam:contoso reader 2.0" }, rig.Tables.RowKeys(Custom));
Assert.Single(rig.Tables.Rows(CpeMappingStore.AutoPartition));
// The mapping is stored before anything is taken away.
Assert.Equal($"upsert {Custom}/contoso:contoso reader", rig.Tables.Writes[0]);
Assert.Equal(
new[] { $"delete {Custom}/contoso:contoso reader 1.0", $"delete {Custom}/contoso:contoso reader 2.0 (x64)" },
rig.Tables.Writes.Skip(1).OrderBy(write => write, StringComparer.Ordinal));
}
[Fact]
public async Task Save_EarlierRowThatCannotBeRemoved_StillSavesTheMapping_AndTheNextSaveRemovesIt()
{
var rig = await Rig.CreateAsync();
rig.Tables.Put(Custom, "contoso:contoso reader 2.0 (x64)", Earlier("contoso", "Contoso Reader 2.0 (x64)", "cpe:2.3:a:contoso:reader_wrong"));
var request = new SaveCustomCpeMappingRequest
{
NormalizedVendor = "contoso", NormalizedProduct = "Contoso Reader 3.0", CpeUri = "cpe:2.3:a:contoso:reader",
};
rig.Tables.WriteFailure = write =>
write.StartsWith("delete ", StringComparison.Ordinal) ? new InvalidOperationException("storage down") : null;
var first = await rig.SaveAsync(request);
Assert.Equal(HttpStatusCode.OK, first.StatusCode);
Assert.True(OperatorCall.BodyOf(first).GetProperty("success").GetBoolean());
Assert.Equal(new[] { "contoso:contoso reader", "contoso:contoso reader 2.0 (x64)" }, rig.Tables.RowKeys(Custom));
Assert.Equal("cpe:2.3:a:contoso:reader", rig.Tables.Row(Custom, "contoso:contoso reader")!["CpeUri"]);
rig.Tables.WriteFailure = null;
var second = await rig.SaveAsync(request);
Assert.Equal(HttpStatusCode.OK, second.StatusCode);
Assert.Equal(new[] { "contoso:contoso reader" }, rig.Tables.RowKeys(Custom));
}
}
@@ -0,0 +1,124 @@
using System.Net;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// POST vulnerability/cpe-product-flags sets whether a product counts as self-updating and
/// answers with the value in effect and the default the seed ships. A product no mapping points
/// to is not found, and mappings that cannot be read change nothing.
/// </summary>
public class SetCpeProductFlagsFunctionTests
{
private const string Flags = CpeMappingStore.ProductFlagsPartition;
private const string Widget = "contoso:widget";
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget", "contoso widget"));
var store = new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance);
Service = new CpeProductFlagsService(Tables.Repo.Object, store);
Function = new SetCpeProductFlagsFunction(Service, NullLogger<SetCpeProductFlagsFunction>.Instance);
}
public MappingTables Tables { get; } = new();
public CpeProductFlagsService Service { get; }
public SetCpeProductFlagsFunction Function { get; }
}
[Fact]
public void Route_IsRegisteredForGlobalAdminsOnly()
{
var entry = EndpointAccessPolicyCatalog.FindPolicy("POST", "vulnerability/cpe-product-flags");
Assert.Equal(EndpointPolicy.GlobalAdminOnly, entry!.Policy);
}
[Fact]
public async Task Post_AwayFromTheDefault_StoresTheOperatorsValueUnderTheirName_AndAnswersWithBoth()
{
var rig = new Rig();
await rig.Service.ApplySeedDefaultsAsync(new[] { Widget });
var response = await rig.Function.Run(OperatorCall.Request(
new SetCpeProductFlagsRequest { Product = " Contoso:Widget ", SelfUpdating = false }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.True(body.GetProperty("success").GetBoolean());
Assert.Equal(Widget, body.GetProperty("product").GetString());
Assert.False(body.GetProperty("selfUpdating").GetBoolean());
Assert.True(body.GetProperty("selfUpdatingDefault").GetBoolean());
var row = rig.Tables.Row(Flags, Widget)!;
Assert.Equal(false, row["AdminSelfUpdating"]);
Assert.Equal(OperatorCall.Upn, row["UpdatedBy"]);
}
[Fact]
public async Task Post_ForAProductTheSeedDoesNotList_AnswersWithADefaultOfFalse()
{
var rig = new Rig();
var response = await rig.Function.Run(OperatorCall.Request(
new SetCpeProductFlagsRequest { Product = Widget, SelfUpdating = true }));
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = OperatorCall.BodyOf(response);
Assert.True(body.GetProperty("selfUpdating").GetBoolean());
Assert.False(body.GetProperty("selfUpdatingDefault").GetBoolean());
Assert.Equal(true, rig.Tables.Row(Flags, Widget)!["AdminSelfUpdating"]);
}
[Fact]
public async Task Post_ProductNoMappingPointsTo_IsNotFound()
{
var rig = new Rig();
var response = await rig.Function.Run(OperatorCall.Request(
new SetCpeProductFlagsRequest { Product = "contoso:gadget", SelfUpdating = true }));
Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.NotFound, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.Tables.Writes);
}
[Theory]
[InlineData("{}")]
[InlineData("""{ "selfUpdating": true }""")]
[InlineData("""{ "product": " ", "selfUpdating": true }""")]
[InlineData("""{ "product": "contoso:widget" }""")]
[InlineData("""{ "product": "contoso:widget", "selfUpdating": null }""")]
public async Task Post_WithoutProductOrValue_IsABadRequest(string json)
{
var rig = new Rig();
var response = await rig.Function.Run(OperatorCall.RequestWithJson(json));
Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.BadRequest, EndpointHarness.ErrorCode(response));
Assert.Empty(rig.Tables.Writes);
}
[Fact]
public async Task Post_MappingsUnreadable_IsServiceUnavailable_AndChangesNothing()
{
var rig = new Rig();
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
var response = await rig.Function.Run(OperatorCall.Request(
new SetCpeProductFlagsRequest { Product = Widget, SelfUpdating = true }));
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.ServiceUnavailable, EndpointHarness.ErrorCode(response));
Assert.Equal(new[] { "60" }, response.Headers.GetValues("Retry-After"));
Assert.Empty(rig.Tables.Writes);
}
}
@@ -0,0 +1,167 @@
using Azure;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="SoftwareInventoryCpeReconciler"/> brings the CPE on every inventory row in line with
/// the mappings in service: it writes only the rows that differ, goes on when one write fails,
/// notes products that have no CVE data yet, and touches nothing when the mappings cannot be read.
/// </summary>
public class SoftwareInventoryCpeReconcilerTests
{
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private const string GadgetCpe = "cpe:2.3:a:fabrikam:gadget";
private const string OtherTenant = "33333333-3333-3333-3333-333333333333";
private static Task<MappingRig> RigAsync() => MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("contoso", "contoso widget", WidgetCpe, "contoso widget"),
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"));
private static void HasCveData(MappingRig rig, params string[] cpeUris)
{
foreach (var cpeUri in cpeUris)
rig.Scan.Cache(VulnerabilityCacheRows.Filled(cpeUri, Array.Empty<CachedCve>(), DateTime.UtcNow.AddDays(-1)));
}
private static string RowKeyOf(Dictionary<string, object> row) => (string)row["RowKey"];
[Fact]
public async Task Reconcile_SetsTheCpeWhereAMappingCovers_TakesItAwayWhereNoneDoes_AndWritesNoRowThatIsRight()
{
var rig = await RigAsync();
HasCveData(rig, WidgetCpe, GadgetCpe);
var toSet = rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2");
var alreadySet = rig.Tables.AddInventory("contoso", "Contoso Widget", "5.0", cpeUri: WidgetCpe);
var toCorrect = rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0", cpeUri: "cpe:2.3:a:fabrikam:gadget_old");
var toClear = rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", cpeUri: "cpe:2.3:a:northwind:tool");
var neverMapped = rig.Tables.AddInventory("northwind", "Northwind Viewer", "1.0");
// The same program in a second tenant is a row of its own.
var inOtherTenant = rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2", tenantId: OtherTenant);
var result = await rig.Reconciler.ReconcileAsync("test");
Assert.Equal(6, result.Rows);
Assert.Equal(4, result.Mapped);
Assert.Equal(2, result.Unmapped);
Assert.Equal(3, result.Set);
Assert.Equal(1, result.Cleared);
Assert.Equal(0, result.Failed);
Assert.Equal(0, result.ProductsNoted);
Assert.Equal(WidgetCpe, toSet["CpeUri"]);
Assert.Equal(WidgetCpe, alreadySet["CpeUri"]);
Assert.Equal(GadgetCpe, toCorrect["CpeUri"]);
Assert.Equal(string.Empty, toClear["CpeUri"]);
Assert.Equal(string.Empty, neverMapped["CpeUri"]);
Assert.Equal(WidgetCpe, inOtherTenant["CpeUri"]);
// The writes run side by side, so their order is not fixed.
Assert.Equal(
new[] { RowKeyOf(toSet), RowKeyOf(inOtherTenant), RowKeyOf(toCorrect), RowKeyOf(toClear) }.OrderBy(k => k, StringComparer.Ordinal),
rig.Tables.InventoryWrites.OrderBy(k => k, StringComparer.Ordinal));
}
[Fact]
public async Task Reconcile_ReadsTheMappingsFresh_NotFromTheCopyThisInstanceHolds()
{
var rig = await RigAsync();
HasCveData(rig, "cpe:2.3:a:northwind:tool");
await rig.Store.GetAsync();
// Stored after this instance read the mappings, as the save of an operator's mapping is.
rig.Tables.Put(VulnerabilityCacheRows.MappingIn(
CpeMappingStore.CustomPartition, "manual", "northwind", "northwind tool", "cpe:2.3:a:northwind:tool"));
var row = rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1");
var result = await rig.Reconciler.ReconcileAsync("mapping_saved");
Assert.Equal(1, result.Set);
Assert.Equal("cpe:2.3:a:northwind:tool", row["CpeUri"]);
}
[Fact]
public async Task Reconcile_RowDeletedSinceItWasRead_IsNeitherSetNorFailed()
{
var rig = await RigAsync();
HasCveData(rig, WidgetCpe, GadgetCpe);
var deleted = rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2");
var staying = rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0");
rig.Tables.AfterInventoryRead = () => rig.Tables.RemoveInventory(deleted);
var result = await rig.Reconciler.ReconcileAsync("test");
Assert.Equal(2, result.Rows);
Assert.Equal(2, result.Mapped);
Assert.Equal(1, result.Set);
Assert.Equal(0, result.Cleared);
Assert.Equal(0, result.Failed);
Assert.Equal(new[] { RowKeyOf(staying) }, rig.Tables.InventoryWrites);
Assert.Equal(GadgetCpe, staying["CpeUri"]);
}
[Fact]
public async Task Reconcile_WriteThatThrows_CountsAsFailed_AndTheOtherRowsAreStillWritten()
{
var rig = await RigAsync();
HasCveData(rig, WidgetCpe, GadgetCpe);
var failing = rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2");
var set = new[] { "1.0", "1.1", "2.0" }.Select(version => rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", version)).ToList();
var cleared = rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", cpeUri: "cpe:2.3:a:northwind:tool");
rig.Tables.InventoryWriteFailure = rowKey => rowKey == RowKeyOf(failing) ? new RequestFailedException(503, "busy") : null;
var result = await rig.Reconciler.ReconcileAsync("test");
Assert.Equal(1, result.Failed);
Assert.Equal(3, result.Set);
Assert.Equal(1, result.Cleared);
Assert.Equal(string.Empty, failing["CpeUri"]);
Assert.All(set, row => Assert.Equal(GadgetCpe, row["CpeUri"]));
Assert.Equal(string.Empty, cleared["CpeUri"]);
}
[Fact]
public async Task Reconcile_ProductInUseWithoutCveData_IsNotedForTheRefresh_OnceAndOnlyWhileItHasNone()
{
var rig = await MappingRig.CreateAsync(
VulnerabilityCacheRows.Mapping("contoso", "contoso widget", WidgetCpe, "contoso widget"),
VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", GadgetCpe, "fabrikam gadget"),
VulnerabilityCacheRows.Mapping("adatum", "adatum reader", "cpe:2.3:a:adatum:reader", "adatum reader"));
HasCveData(rig, GadgetCpe);
rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2");
rig.Tables.AddInventory("contoso", "Contoso Widget", "5.0", cpeUri: WidgetCpe);
rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget", "1.0");
var first = await rig.Reconciler.ReconcileAsync("test");
var second = await rig.Reconciler.ReconcileAsync("test");
// The widget is in use and has no data; the gadget has data; nobody has the reader installed.
Assert.Equal(1, first.ProductsNoted);
var marker = Assert.Single(rig.Scan.PendingMarkers);
Assert.Equal(WidgetCpe, marker["CpeUri"]);
Assert.Equal(CveCacheFiller.StatePending, marker["State"]);
Assert.Equal(CveCacheFiller.StatePending, rig.Scan.CacheRows[CveCacheFiller.CacheKey(WidgetCpe)]["State"]);
Assert.False(rig.Scan.CacheRows.ContainsKey(CveCacheFiller.CacheKey("cpe:2.3:a:adatum:reader")));
// The marker is a cache row: the product is not noted again.
Assert.Equal(0, second.ProductsNoted);
Assert.Single(rig.Scan.PendingMarkers);
Assert.Empty(rig.Scan.Nvd.Requests);
}
[Fact]
public async Task Reconcile_MappingsUnreadable_IsUnavailable_AndNoRowIsWritten()
{
var rig = await RigAsync();
var mapped = rig.Tables.AddInventory("contoso", "Contoso Widget", "4.2");
var withCpe = rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", cpeUri: "cpe:2.3:a:northwind:tool");
rig.Tables.ReadFailure = new InvalidOperationException("storage down");
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Reconciler.ReconcileAsync("test"));
// Without mappings every row would read as unmapped and lose its CPE.
Assert.Empty(rig.Tables.InventoryWrites);
Assert.Equal(string.Empty, mapped["CpeUri"]);
Assert.Equal("cpe:2.3:a:northwind:tool", withCpe["CpeUri"]);
Assert.Empty(rig.Scan.PendingMarkers);
}
}
@@ -0,0 +1,94 @@
using System.Collections.Generic;
using System.Linq;
using AutopilotMonitor.Functions.Services.Vulnerability;
using Xunit;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The canonical name is the key for manual mappings, the ignore list and the admin's unmatched
/// list. Every version, architecture and language of one product must land on one key, and two
/// products must not.
/// </summary>
public class SoftwareNameCanonicalizerTests
{
public static IEnumerable<object[]> Names() => new[]
{
// versions, architectures, editions in parentheses
new object[] { "7-Zip 24.08 (x64)", "7-zip" },
new object[] { "7-Zip 26.03 (x64 edition)", "7-zip" },
new object[] { "7-Zip", "7-zip" },
new object[] { "Zoom Workplace (64-bit)", "zoom workplace" },
new object[] { "Python 3.12.1 (64-bit)", "python" },
new object[] { "WinRAR 7.01 (64-bit)", "winrar" },
new object[] { "Notepad++ (64-bit x64)", "notepad++" },
new object[] { "Foo Tool v2.5", "foo tool" },
new object[] { "Foo Tool version 2.5-beta", "foo tool" },
// language tags
new object[] { "Mozilla Firefox (x64 de)", "mozilla firefox" },
new object[] { "Mozilla Firefox (x64 en-US)", "mozilla firefox" },
new object[] { "Mozilla Firefox ESR (x64 en-US)", "mozilla firefox esr" },
new object[] { "Microsoft 365 Apps for enterprise - de-de", "microsoft 365 apps for enterprise" },
new object[] { "Microsoft OneNote - fr-fr", "microsoft onenote" },
// marks and separators
new object[] { "Intel(R) Wireless Bluetooth(R)", "intel wireless bluetooth" },
new object[] { "Java(TM) SE Development Kit 17.0.9 (64-bit)", "java se development kit" },
new object[] { "HP Wolf Security - Console", "hp wolf security console" },
new object[] { "Google Chrome", "google chrome" },
// counters that change with every patch or build
new object[] { "Java 8 Update 401 (64-bit)", "java 8" },
new object[] { "Update for Windows 10 for x64-based Systems (KB5001716)", "update for windows 10 for x64-based systems" },
new object[] { "Citrix Workspace 2402", "citrix workspace" },
new object[] { "Microsoft .NET Runtime - 8.0.11 (x64)", "microsoft .net runtime" },
new object[] { "Contoso Agent {5A3B9C10-1D2E-4F60-8A7B-0C9D8E7F6A5B}", "contoso agent" },
new object[] { "Windows Driver Package - Contoso (ctsdrv) System (06/12/2020 2028.15.0.1688)", "windows driver package contoso ctsdrv system" },
// a year is part of the name, a release range too
new object[] { "Microsoft Visual Studio Professional 2022", "microsoft visual studio professional 2022" },
new object[] { "Microsoft Office Professional Plus 2016", "microsoft office professional plus 2016" },
new object[] { "Microsoft Visual C++ 2015-2022 Redistributable (x64) - 14.38.33135", "microsoft visual c++ 2015-2022 redistributable" },
// a small number is part of the name
new object[] { "Office 16 Click-to-Run Licensing Component", "office 16 click-to-run licensing component" },
new object[] { "Node.js", "node.js" },
new object[] { "", "" },
new object[] { " ", "" },
};
[Theory]
[MemberData(nameof(Names))]
public void Canonicalize_KeepsOnlyWhatNamesTheProduct(string displayName, string expected)
=> Assert.Equal(expected, SoftwareNameCanonicalizer.Canonicalize(displayName));
[Fact]
public void Canonicalize_Null_IsEmpty()
=> Assert.Equal(string.Empty, SoftwareNameCanonicalizer.Canonicalize(null));
/// <summary>Stored keys are canonicalized again when loaded, so a second pass must not move them.</summary>
[Fact]
public void Canonicalize_IsStableOnItsOwnOutput()
{
foreach (var name in Names().Select(row => (string)row[0]))
{
var once = SoftwareNameCanonicalizer.Canonicalize(name);
Assert.Equal(once, SoftwareNameCanonicalizer.Canonicalize(once));
}
}
[Fact]
public void Canonicalize_KeepsDifferentProductsApart()
{
var keys = new[]
{
"Microsoft Visual Studio Professional 2019", "Microsoft Visual Studio Professional 2022",
"Microsoft Visual Studio Code", "Mozilla Firefox", "Mozilla Firefox ESR",
"Microsoft Edge", "Microsoft Edge Update", "Microsoft Edge WebView2 Runtime",
}.Select(SoftwareNameCanonicalizer.Canonicalize).ToList();
Assert.Equal(keys.Count, keys.Distinct().Count());
}
}
@@ -140,6 +140,48 @@ public class TableStorageInventoryCasTests
harness.SoftwareInventory.Verify(t => t.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()), Times.Never);
}
/// <summary>
/// The CPE on an existing row follows the matching: a computed value replaces it, the empty
/// "no mapping" clears it, and null (matching did not run) leaves it alone.
/// </summary>
[Theory]
[InlineData("cpe:2.3:a:contoso:widget_pro", "cpe:2.3:a:contoso:widget_pro")]
[InlineData("", "")]
[InlineData(null, "cpe:2.3:a:contoso:widget")]
public async Task Increment_writes_the_computed_cpe_onto_an_existing_row(string? computedCpe, string expectedOnRow)
{
var existing = new TableEntity(TenantId, "contoso:widget:1.0")
{
["NormalizedName"] = "widget",
["SessionCount"] = 3,
["CpeUri"] = "cpe:2.3:a:contoso:widget",
};
existing.ETag = new ETag("0xABC");
var harness = new Harness();
harness.SoftwareInventory.Setup(t =>
t.GetEntityAsync<TableEntity>(TenantId, It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(Response.FromValue(existing, new Mock<Response>().Object));
TableEntity? written = null;
harness.SoftwareInventory.Setup(t => t.UpdateEntityAsync(
It.IsAny<TableEntity>(), It.IsAny<ETag>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
.Returns<TableEntity, ETag, TableUpdateMode, CancellationToken>((e, _, _, _) =>
{
written = e;
return Task.FromResult(new Mock<Response>().Object);
});
var item = new SoftwareInventoryItem
{
NormalizedVendor = "contoso", NormalizedName = "widget", NormalizedVersion = "1.0", CpeUri = computedCpe,
};
await harness.Sut.IncrementSoftwareInventoryEntryAsync(TenantId, item, SessionId);
Assert.Equal(expectedOnRow, written!.GetString("CpeUri"));
Assert.Equal(4, written.GetInt32("SessionCount"));
}
[Fact]
public async Task Increment_retries_on_412_etag_conflict_and_succeeds_on_second_attempt()
{
@@ -0,0 +1,409 @@
[
{"name": "Microsoft Teams Meeting Add-in for Microsoft Office", "publisher": "Microsoft"},
{"name": "Chrome Remote Desktop Host", "publisher": "Google LLC"},
{"name": "Google Chrome", "publisher": "Google LLC", "product": "google:chrome"},
{"name": "Microsoft Edge Update", "publisher": "", "product": "microsoft:edge_update"},
{"name": "Microsoft Edge WebView2 Runtime", "publisher": "", "product": "microsoft:edge_chromium"},
{"name": "Mozilla Firefox ESR (x64 en-US)", "publisher": "Mozilla", "product": "mozilla:firefox", "edition": "esr"},
{"name": "Mozilla Firefox (x64 de)", "publisher": "Mozilla", "product": "mozilla:firefox", "edition": ""},
{"name": "Mozilla Maintenance Service", "publisher": "Mozilla"},
{"name": "Adobe Acrobat Reader DC - Deutsch", "publisher": "Adobe Systems Incorporated", "product": "adobe:acrobat_reader_dc"},
{"name": "Extended Asian Language font pack for Adobe Acrobat Reader DC", "publisher": "Adobe Systems Incorporated"},
{"name": "Adobe Acrobat (64-bit)", "publisher": "Adobe", "product": "adobe:acrobat_dc"},
{"name": "Microsoft 365 Copilot", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Dynamics 365 Business Central", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Office 365 ProPlus - en-us", "publisher": "Microsoft Corporation", "product": "microsoft:365_apps"},
{"name": "Microsoft 365 Apps for enterprise - en-us", "publisher": "Microsoft Corporation", "product": "microsoft:365_apps"},
{"name": "Kurumlar için Microsoft 365 Uygulamaları - tr-tr", "publisher": "", "product": "microsoft:365_apps"},
{"name": "Microsoft 365-apps voor ondernemingen - nl-nl", "publisher": "", "product": "microsoft:365_apps"},
{"name": "Microsoft Office Professional Plus 2019 - en-us", "publisher": "Microsoft Corporation", "product": "microsoft:office"},
{"name": "Microsoft Office LTSC Professional Plus 2024 - de-de", "publisher": "Microsoft Corporation", "product": "microsoft:office_2024"},
{"name": "Microsoft Office Hub", "publisher": "Microsoft"},
{"name": "Microsoft Visio Viewer 2016", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Project - en-us", "publisher": "Microsoft Corporation", "product": "microsoft:project"},
{"name": "Microsoft OneNote - fr-fr", "publisher": "", "product": "microsoft:onenote"},
{"name": "Teams Machine-Wide Installer", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Teams", "publisher": "MicrosoftTeams", "product": "microsoft:teams"},
{"name": "MSTeams", "publisher": "MSTeams", "product": "microsoft:teams"},
{"name": "Microsoft Teams Rooms", "publisher": "Microsoft Corporation"},
{"name": "Zoom Rooms", "publisher": "Zoom Video Communications, Inc."},
{"name": "Zoom", "publisher": "Zoom Video Communications, Inc.", "product": "zoom:zoom"},
{"name": "Zoom Workplace (64-bit)", "publisher": "Zoom", "product": "zoom:workplace_desktop"},
{"name": "Zoom Outlook Plugin", "publisher": "Zoom"},
{"name": "TeamViewer Patch Management", "publisher": "TeamViewer"},
{"name": "TeamViewer Host", "publisher": "TeamViewer", "product": "teamviewer:teamviewer"},
{"name": "Git Extensions 4.2.1", "publisher": "Git Extensions Team"},
{"name": "Git", "publisher": "The Git Development Community", "product": "git-scm:git"},
{"name": "Git LFS", "publisher": "GitHub, Inc.", "product": "git_large_file_storage_project:git_large_file_storage"},
{"name": "PowerShell 7-x64", "publisher": "Microsoft Corporation", "product": "microsoft:powershell"},
{"name": "Microsoft Azure PowerShell - April 2018", "publisher": "Microsoft Corporation"},
{"name": "Python 3.12.4 (64-bit)", "publisher": "Python Software Foundation", "product": "python:python"},
{"name": "Python Launcher", "publisher": "Python Software Foundation"},
{"name": "Anaconda3 2024.10-1 (Python 3.12.7 64-bit)", "publisher": "Anaconda, Inc.", "product": "anaconda:anaconda3"},
{"name": "Miniconda3 py312_24.5.0-0 (Python 3.12.4 64-bit)", "publisher": "Anaconda, Inc."},
{"name": "Microsoft SQL Server 2019 (64-bit)", "publisher": "Microsoft Corporation", "product": "microsoft:sql_server"},
{"name": "Microsoft SQL Server Management Studio - 20.2", "publisher": "Microsoft Corporation", "product": "microsoft:sql_server_management_studio"},
{"name": "Microsoft SQL Server 2012 Native Client", "publisher": "Microsoft Corporation"},
{"name": "Microsoft ODBC Driver 18 for SQL Server", "publisher": "Microsoft Corporation", "product": "microsoft:odbc_driver_for_sql_server"},
{"name": "Microsoft System CLR Types for SQL Server 2019", "publisher": "Microsoft Corporation"},
{"name": "Remote Desktop", "publisher": "Microsoft Corporation", "product": "microsoft:remote_desktop"},
{"name": "Remote Desktop Services Infrastructure Agent", "publisher": "Microsoft Corporation"},
{"name": "Remote Desktop Manager", "publisher": "Devolutions inc.", "product": "devolutions:remote_desktop_manager"},
{"name": "Remote Desktop Connection Manager", "publisher": "Microsoft Corporation"},
{"name": "Remote Help", "publisher": "Microsoft Corporation", "product": "microsoft:remote_help"},
{"name": "Rustup: the Rust toolchain installer", "publisher": "The Rust Project Developers"},
{"name": "Rust 1.80 (MSVC 64-bit)", "publisher": "The Rust Project Developers", "product": "rust-lang:rust"},
{"name": "Lenovo Vantage Service", "publisher": "Lenovo Group Ltd."},
{"name": "NVIDIA GeForce Experience 3.28.0.417", "publisher": "NVIDIA Corporation", "product": "nvidia:geforce_experience"},
{"name": "NVIDIA GeForce NOW", "publisher": "NVIDIA Corporation"},
{"name": "NVIDIA App 11.0.5.420", "publisher": "NVIDIA Corporation"},
{"name": "NVIDIA Grafiktreiber 573.44", "publisher": "NVIDIA Corporation", "product": "nvidia:gpu_display_driver"},
{"name": "Dell SupportAssist for Business PCs", "publisher": "Dell Inc.", "product": "dell:supportassist_for_business_pcs"},
{"name": "Dell SupportAssist", "publisher": "Dell Inc.", "product": "dell:supportassist_for_home_pcs"},
{"name": "Dell SupportAssist Remediation", "publisher": "Dell Inc."},
{"name": "Cisco Webex Meetings", "publisher": "Cisco Webex LLC", "product": "cisco:webex_meetings"},
{"name": "Webex", "publisher": "Cisco Systems, Inc", "product": "cisco:webex_teams"},
{"name": "OpenVPN Connect", "publisher": "OpenVPN Inc.", "product": "openvpn:connect"},
{"name": "OpenVPN 2.6.12-I001 amd64", "publisher": "OpenVPN, Inc.", "product": "openvpn:openvpn"},
{"name": "Windows Media Encoder 9 Series", "publisher": "Microsoft Corporation"},
{"name": "Intel(R) Management Engine Components", "publisher": "Intel Corporation"},
{"name": "Intel® Graphics Software", "publisher": "Intel Corporation"},
{"name": "Visual Studio Community 2022", "publisher": "Microsoft Corporation", "product": "microsoft:visual_studio_2022"},
{"name": "Microsoft Visual Studio Installer", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Visual Studio Code (User)", "publisher": "Microsoft Corporation", "product": "microsoft:visual_studio_code"},
{"name": "Microsoft Visual Studio 2010 Tools for Office Runtime (x64)", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Access database engine 2016 (English)", "publisher": "Microsoft Corporation", "product": "microsoft:access"},
{"name": "Microsoft Access Runtime 2016", "publisher": "Microsoft Corporation"},
{"name": "Microsoft Access 2016 - de-de", "publisher": "Microsoft Corporation", "product": "microsoft:access"},
{"name": "Java 8 Update 431 (64-bit)", "publisher": "Oracle Corporation", "product": "oracle:jre"},
{"name": "Java(TM) SE Development Kit 21.0.5 (64-bit)", "publisher": "Oracle Corporation", "product": "oracle:jdk"},
{"name": "Java Auto Updater", "publisher": "Oracle Corporation"},
{"name": "Eclipse Temurin JDK with Hotspot 17.0.13+11 (x64)", "publisher": "Eclipse Adoptium", "product": "oracle:openjdk"},
{"name": "Microsoft Build of OpenJDK 21.0.5+11 (x64)", "publisher": "Microsoft"},
{"name": "Azul Zulu JDK 17.54.21 (17.0.13), 64-bit", "publisher": "Azul Systems, Inc.", "product": "azul:zulu"},
{"name": "KeePassXC", "publisher": "KeePassXC Team", "product": "keepassxc:keepassxc"},
{"name": "KeePass Password Safe 2.57", "publisher": "Dominik Reichl", "product": "keepass:keepass"},
{"name": "FortiClient EMS", "publisher": "Fortinet Technologies Inc"},
{"name": "Citrix Workspace Environment Management Agent", "publisher": "Citrix Systems, Inc."},
{"name": "Splashtop Streamer", "publisher": "Splashtop Inc.", "product": "splashtop:streamer"},
{"name": "Splashtop Business", "publisher": "Splashtop Inc.", "product": "splashtop:splashtop"},
{"name": "Logi Options+", "publisher": "Logitech"},
{"name": "Logitech Options", "publisher": "Logitech", "product": "logitech:options"},
{"name": "Tableau Reader 2024.1", "publisher": "Tableau Software"},
{"name": "Tableau 2024.1 (20241.24.0312.0911)", "publisher": "Tableau Software", "product": "tableau:tableau_desktop"},
{"name": "AMD Chipset Software", "publisher": "Advanced Micro Devices, Inc."},
{"name": "Bonjour Print Services", "publisher": "Apple Inc."},
{"name": "Microsoft .NET Runtime - 8.0.11 (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:.net"},
{"name": "Microsoft Windows Desktop Runtime - 8.0.11 (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:.net"},
{"name": "Microsoft .NET SDK 8.0.404 (x64)", "publisher": "Microsoft Corporation"},
{"name": "Microsoft .NET Framework 4.8.1", "publisher": "Microsoft Corporation", "product": "microsoft:.net_framework"},
{"name": "SAP Crystal Reports runtime engine for .NET Framework (64-bit)", "publisher": "SAP"},
{"name": "VNC Viewer 7.12.0", "publisher": "RealVNC", "product": "realvnc:vnc_viewer"},
{"name": "VNC Server 7.12.0", "publisher": "RealVNC", "product": "realvnc:vnc_server"},
{"name": "Malwarebytes AdwCleaner", "publisher": "Malwarebytes"},
{"name": "MySQL Server 8.0", "publisher": "Oracle Corporation", "product": "oracle:mysql_server"},
{"name": "Zabbix Agent 2 (64-bit)", "publisher": "Zabbix SIA", "product": "zabbix:zabbix-agent2"},
{"name": "Zabbix Agent (64-bit)", "publisher": "Zabbix SIA", "product": "zabbix:zabbix-agent"},
{"name": "Nessus Agent (x64)", "publisher": "Tenable, Inc.", "product": "tenable:nessus_agent"},
{"name": "Tenable Nessus (x64)", "publisher": "Tenable, Inc.", "product": "tenable:nessus"},
{"name": "Autodesk AutoCAD LT 2025 - English", "publisher": "Autodesk, Inc."},
{"name": "AutoCAD Open in Desktop", "publisher": "Autodesk"},
{"name": "Citrix Secure Access Endpoint Analysis", "publisher": "Citrix Systems, Inc."},
{"name": "HP Security Update Service", "publisher": "HP Inc."},
{"name": "Microsoft Primary Interoperability Assemblies 2005", "publisher": "Microsoft Corporation"},
{"name": "Outlook For Windows", "publisher": "Microsoft"},
{"name": "Copilot", "publisher": "Microsoft Corporation"},
{"name": "Photos", "publisher": "Microsoft Corporation"},
{"name": "Microsoft PowerBI Desktop (x64)", "publisher": "Microsoft Corporation"},
{"name": "Mesh Agent", "publisher": ""},
{"name": "SketchUp 2026", "publisher": "Trimble, Inc."},
{"name": "Global Secure Access Client Installer (x64)", "publisher": "Microsoft Corporation"},
{"name": "Microsoft OLE DB Driver 19 for SQL Server", "publisher": "Microsoft Corporation", "product": "microsoft:ole_db_driver_for_sql_server"},
{"name": "Opera GX Stable 115.0.5322.124", "publisher": "Opera Software"},
{"name": "Opera Stable 115.0.5322.77", "publisher": "Opera Software", "product": "opera:opera_browser"},
{"name": "Microsoft OneDrive", "publisher": "Microsoft Corporation", "product": "microsoft:onedrive"},
{"name": "Dell Command | Update for Windows Universal", "publisher": "Dell Inc.", "product": "dell:command_update"},
{"name": "Dell Optimizer Service", "publisher": "Dell Inc."},
{"name": "Intel(R) PROSet/Wireless Software", "publisher": "Intel Corporation", "product": "intel:proset\\/wireless_wifi"},
{"name": "ADSelfService Plus Client Software", "publisher": "ZOHO Corp"},
{"name": "uniFLOW SmartClient", "publisher": "NT-ware Systemprogrammierung GmbH"},
{"name": "baramundi Management Agent", "publisher": "baramundi software GmbH"},
{"name": "YSoft SafeQ Client", "publisher": "Y Soft Corporation"},
{"name": "Brave", "publisher": "Brave Software Inc", "product": "brave:brave"},
{"name": "Microsoft Edge", "publisher": "Microsoft Corporation", "product": "microsoft:edge_chromium"},
{"name": "Visual Studio Code", "publisher": "Microsoft"},
{"name": "Microsoft Visual Studio Code", "publisher": "Microsoft Corporation", "product": "microsoft:visual_studio_code"},
{"name": "Citrix Workspace 2507", "publisher": "Citrix Systems, Inc.", "product": "citrix:workspace"},
{"name": "LOGO!Soft Comfort V8.4.0", "publisher": "Siemens AG"},
{"name": "7-Zip 24.08 (x64)", "publisher": "Igor Pavlov", "product": "7-zip:7-zip"},
{"name": "7-Zip 26.03 (x64 edition)", "publisher": "Igor Pavlov", "product": "7-zip:7-zip"},
{"name": "Notepad++ (64-bit x64)", "publisher": "Notepad++ Team", "product": "notepad-plus-plus:notepad\\+\\+"},
{"name": "PuTTY release 0.81 (64-bit)", "publisher": "Simon Tatham", "product": "putty:putty"},
{"name": "WinSCP 6.3.5", "publisher": "Martin Prikryl", "product": "winscp:winscp"},
{"name": "VLC media player", "publisher": "VideoLAN", "product": "videolan:vlc_media_player"},
{"name": "Node.js", "publisher": "Node.js Foundation", "product": "nodejs:node.js"},
{"name": "Wireshark 4.2.6 x64", "publisher": "The Wireshark developer community, https://www.wireshark.org", "product": "wireshark:wireshark"},
{"name": "FileZilla 3.67.0", "publisher": "Tim Kosse", "product": "filezilla-project:filezilla_client"},
{"name": "TeamViewer", "publisher": "TeamViewer", "product": "teamviewer:teamviewer"},
{"name": "AnyDesk", "publisher": "AnyDesk Software GmbH", "product": "anydesk:anydesk"},
{"name": "Foxit PDF Reader", "publisher": "Foxit Software Inc.", "product": "foxit:pdf_reader"},
{"name": "Docker Desktop", "publisher": "Docker Inc.", "product": "docker:desktop"},
{"name": "Java 8 Update 421 (64-bit)", "publisher": "Oracle Corporation", "product": "oracle:jre"},
{"name": "WireGuard", "publisher": "WireGuard LLC", "product": "wireguard:wireguard"},
{"name": "Microsoft Teams", "publisher": "Microsoft", "product": "microsoft:teams"},
{"name": "Opera Stable 112.0.5197.53", "publisher": "Opera Software", "product": "opera:opera_browser"},
{"name": "Mozilla Thunderbird (x64 en-US)", "publisher": "Mozilla", "product": "mozilla:thunderbird"},
{"name": "LibreOffice 24.2.5.2", "publisher": "The Document Foundation", "product": "libreoffice:libreoffice"},
{"name": "GIMP 2.10.38", "publisher": "The GIMP Team", "product": "gimp:gimp"},
{"name": "Inkscape", "publisher": "Inkscape", "product": "inkscape:inkscape"},
{"name": "Audacity 3.6.1", "publisher": "Audacity Team", "product": "audacityteam:audacity"},
{"name": "WinRAR 7.01 (64-bit)", "publisher": "win.rar GmbH", "product": "rarlab:winrar"},
{"name": "PDF24 Creator 11.18.0", "publisher": "geek software GmbH", "product": "pdf24:pdf24_creator"},
{"name": "Greenshot 1.2.10.6", "publisher": "Greenshot", "product": "getgreenshot:greenshot"},
{"name": "Paint.NET", "publisher": "dotPDN LLC", "product": "dotpdn:paint.net"},
{"name": "IrfanView 4.67 (64-bit)", "publisher": "Irfan Skiljan", "product": "irfanview:irfanview"},
{"name": "Oracle VM VirtualBox 7.0.20", "publisher": "Oracle and/or its affiliates", "product": "oracle:vm_virtualbox"},
{"name": "VMware Workstation", "publisher": "VMware, Inc.", "product": "vmware:workstation"},
{"name": "VMware Tools", "publisher": "VMware, Inc.", "product": "vmware:tools"},
{"name": "Cisco Secure Client - AnyConnect VPN", "publisher": "Cisco Systems, Inc.", "product": "cisco:secure_client"},
{"name": "FortiClient VPN", "publisher": "Fortinet Technologies Inc", "product": "fortinet:forticlient"},
{"name": "GlobalProtect", "publisher": "Palo Alto Networks", "product": "paloaltonetworks:globalprotect"},
{"name": "Zscaler", "publisher": "Zscaler Inc.", "product": "zscaler:client_connector"},
{"name": "Sophos Endpoint Agent", "publisher": "Sophos Limited", "product": "sophos:endpoint_protection"},
{"name": "CrowdStrike Windows Sensor", "publisher": "CrowdStrike, Inc.", "product": "crowdstrike:falcon"},
{"name": "1Password", "publisher": "AgileBits Inc.", "product": "1password:1password"},
{"name": "Bitwarden", "publisher": "Bitwarden Inc.", "product": "bitwarden:bitwarden"},
{"name": "Dropbox", "publisher": "Dropbox, Inc.", "product": "dropbox:dropbox"},
{"name": "Microsoft SQL Server Management Studio - 19.3", "publisher": "Microsoft Corporation", "product": "microsoft:sql_server_management_studio"},
{"name": "Microsoft .NET Runtime - 8.0.8 (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:.net"},
{"name": "Microsoft Windows Desktop Runtime - 8.0.8 (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:.net"},
{"name": "Microsoft ASP.NET Core 8.0.8 - Shared Framework (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:asp.net_core"},
{"name": "Microsoft Visual C++ 2015-2022 Redistributable (x64) - 14.40.33810", "publisher": "Microsoft Corporation", "product": "microsoft:visual_c\\+\\+"},
{"name": "Windows Terminal", "publisher": "Microsoft Corporation", "product": "microsoft:terminal"},
{"name": "Zoom Workplace (64-bit)", "publisher": "Zoom Communications, Inc.", "product": "zoom:workplace_desktop"},
{"name": "Webex", "publisher": "Cisco Webex LLC", "product": "cisco:webex_teams"},
{"name": "Slack", "publisher": "Slack Technologies Inc."},
{"name": "Postman x86_64 11.6.2", "publisher": "Postman", "product": "postman:postman"},
{"name": "Tableau Desktop 2024.2", "publisher": "Tableau Software", "product": "tableau:tableau_desktop"},
{"name": "Snagit 2024", "publisher": "TechSmith Corporation", "product": "techsmith:snagit"},
{"name": "Adobe Creative Cloud", "publisher": "Adobe Inc.", "product": "adobe:creative_cloud"},
{"name": "Adobe Photoshop 2024", "publisher": "Adobe Inc.", "product": "adobe:photoshop"},
{"name": "Autodesk AutoCAD 2025 - English", "publisher": "Autodesk", "product": "autodesk:autocad"},
{"name": "Dell Command | Update", "publisher": "Dell Inc.", "product": "dell:command_update"},
{"name": "HP Support Assistant", "publisher": "HP Inc.", "product": "hp:support_assistant"},
{"name": "Lenovo System Update", "publisher": "Lenovo", "product": "lenovo:system_update"},
{"name": "Intel(R) Graphics Driver", "publisher": "Intel Corporation", "product": "intel:graphics_driver"},
{"name": "NVIDIA Graphics Driver 552.44", "publisher": "NVIDIA Corporation", "product": "nvidia:gpu_display_driver"},
{"name": "TortoiseSVN 1.14.7.29687 (64 bit)", "publisher": "TortoiseSVN", "product": "tortoisesvn:tortoisesvn"},
{"name": "PowerToys (Preview) x64", "publisher": "Microsoft Corporation", "product": "microsoft:powertoys"},
{"name": "Sysinternals Suite", "publisher": "Microsoft Corporation"},
{"name": "BeyondTrust Privilege Management", "publisher": "BeyondTrust", "product": "beyondtrust:privilege_management_for_windows"},
{"name": "R for Windows 4.4.1", "publisher": "R Core Team"},
{"name": "Go Programming Language amd64 go1.22.5", "publisher": "https://go.dev", "product": "golang:go"},
{"name": "Strawberry Perl (64-bit)", "publisher": "strawberryperl.com project", "product": "perl:perl"},
{"name": "Microsoft Power BI Desktop (x64)", "publisher": "Microsoft Corporation"},
{"name": "Mozilla Thunderbird (x86 de)", "publisher": "Mozilla", "product": "mozilla:thunderbird"},
{"name": "Mozilla Thunderbird 102.15.1 (x64 en-GB)", "publisher": "Mozilla", "product": "mozilla:thunderbird"},
{"name": "Mozilla Firefox (x64 en-US)", "publisher": "Mozilla", "product": "mozilla:firefox", "edition": ""},
{"name": "eM Client", "publisher": "eM Client Inc.", "product": "emclient:em_client"},
{"name": "eM Client", "publisher": "eM Client s.r.o.", "product": "emclient:em_client"},
{"name": "WPS Office (12.2.0.13431)", "publisher": "Kingsoft Corp.", "product": "kingsoft:wps_office"},
{"name": "WPS Office (11.2.0.11537)", "publisher": "Kingsoft Corp.", "product": "kingsoft:wps_office"},
{"name": "SumatraPDF", "publisher": "Krzysztof Kowalczyk", "product": "sumatrapdfreader:sumatrapdf"},
{"name": "SumatraPDF 3.4.6", "publisher": "Krzysztof Kowalczyk", "product": "sumatrapdfreader:sumatrapdf"},
{"name": "GPL Ghostscript", "publisher": "Artifex Software Inc.", "product": "artifex:ghostscript"},
{"name": "GPL Ghostscript 9.56.1", "publisher": "Artifex Software Inc.", "product": "artifex:ghostscript"},
{"name": "GPL Ghostscript Fonts", "publisher": "Artifex Software Inc."},
{"name": "calibre 64bit", "publisher": "Kovid Goyal", "product": "calibre-ebook:calibre"},
{"name": "calibre", "publisher": "Kovid Goyal", "product": "calibre-ebook:calibre"},
{"name": "Bandizip", "publisher": "Bandisoft International Inc.", "product": "bandisoft:bandizip"},
{"name": "Bandizip 7.37", "publisher": "Bandisoft.com", "product": "bandisoft:bandizip"},
{"name": "PeaZip 9.9.0 (WIN64)", "publisher": "Giorgio Tani", "product": "peazip:peazip"},
{"name": "PeaZip 10.4.0", "publisher": "Giorgio Tani", "product": "peazip:peazip"},
{"name": "XnView 2.51.6", "publisher": "Gougelet Pierre-e", "product": "xnview:xnview"},
{"name": "XnView 2.49.1", "publisher": "Gougelet Pierre-e", "product": "xnview:xnview"},
{"name": "XnViewMP 1.7.2", "publisher": "Gougelet Pierre-e", "product": "xnview:xnview_mp"},
{"name": "XnView MP 1.4.5", "publisher": "XnSoft", "product": "xnview:xnview_mp"},
{"name": "XnView Shell Extension 4.1.11", "publisher": "Gougelet Pierre-e"},
{"name": "ImageMagick 7.1.1-38 Q16-HDRI (64-bit) (2024-09-01)", "publisher": "ImageMagick Studio LLC", "product": "imagemagick:imagemagick"},
{"name": "ImageMagick 6.9.13-12 Q16 (32-bit) (2024-06-15)", "publisher": "ImageMagick Studio LLC", "product": "imagemagick:imagemagick"},
{"name": "MPC-HC 2.3.3 (64-bit)", "publisher": "MPC-HC Team", "product": "mpc-hc:mpc-hc"},
{"name": "MPC-HC 1.7.13 (64-bit)", "publisher": "MPC-HC Team", "product": "mpc-hc:mpc-hc"},
{"name": "MPC-BE x64 1.6.11", "publisher": "MPC-BE Team"},
{"name": "K-Lite Codec Pack 18.4.5 Full", "publisher": "KLCP", "product": "codecguide:k-lite_codec_pack"},
{"name": "Kodi", "publisher": "XBMC Foundation", "product": "kodi:kodi"},
{"name": "Kodi 20.2", "publisher": "XBMC Foundation", "product": "kodi:kodi"},
{"name": "Adobe Lightroom Classic", "publisher": "Adobe Inc.", "product": "adobe:lightroom", "edition": "classic"},
{"name": "Adobe Lightroom Classic CC", "publisher": "Adobe Systems Incorporated", "product": "adobe:lightroom", "edition": "classic"},
{"name": "Adobe Photoshop Lightroom Classic CC", "publisher": "Adobe Systems Incorporated", "product": "adobe:lightroom", "edition": "classic"},
{"name": "Adobe Lightroom", "publisher": "Adobe Inc.", "product": "adobe:lightroom", "edition": ""},
{"name": "Adobe Lightroom CC", "publisher": "Adobe Systems Incorporated", "product": "adobe:lightroom", "edition": ""},
{"name": "Adobe Dimension", "publisher": "Adobe Inc.", "product": "adobe:dimension"},
{"name": "Adobe Dimension CC", "publisher": "Adobe Systems Incorporated", "product": "adobe:dimension"},
{"name": "Adobe Audition 2025", "publisher": "Adobe Inc.", "product": "adobe:audition"},
{"name": "Adobe Audition CC 2019", "publisher": "Adobe Systems Incorporated", "product": "adobe:audition"},
{"name": "Adobe InCopy 2025", "publisher": "Adobe Inc.", "product": "adobe:incopy"},
{"name": "Adobe InCopy CC 2019", "publisher": "Adobe Systems Incorporated", "product": "adobe:incopy"},
{"name": "Adobe InDesign 2025", "publisher": "Adobe Inc.", "product": "adobe:indesign"},
{"name": "Microsoft ASP.NET Core 6.0.33 - Shared Framework (x86)", "publisher": "Microsoft Corporation", "product": "microsoft:asp.net_core"},
{"name": "Microsoft ASP.NET Core 3.1.32 - Shared Framework (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:asp.net_core"},
{"name": "Microsoft ASP.NET Core 8.0.8 Targeting Pack (x64)", "publisher": "Microsoft Corporation"},
{"name": "Microsoft ASP.NET Core Module V2", "publisher": "Microsoft Corporation"},
{"name": "Microsoft ASP.NET Core Module", "publisher": "Microsoft Corporation"},
{"name": "Microsoft ASP.NET MVC 4 Runtime", "publisher": "Microsoft Corporation"},
{"name": "Vim 9.1 (x64)", "publisher": "Bram Moolenaar et al.", "product": "vim:vim"},
{"name": "Vim 8.2 (x86)", "publisher": "Bram Moolenaar et al.", "product": "vim:vim"},
{"name": "Neovim", "publisher": "neovim.io"},
{"name": "AWS Command Line Interface v2", "publisher": "Amazon Web Services", "product": "amazon:aws_command_line_interface"},
{"name": "AWS Command Line Interface", "publisher": "Amazon Web Services", "product": "amazon:aws_command_line_interface"},
{"name": "AWS SAM Command Line Interface", "publisher": "Amazon Web Services"},
{"name": "AWS Tools for Windows", "publisher": "Amazon Web Services Developer Relations"},
{"name": "Session Manager Plugin", "publisher": "Amazon Web Services"},
{"name": "Microsoft Azure CLI (64-bit)", "publisher": "Microsoft Corporation", "product": "microsoft:azure_command-line_interface"},
{"name": "Vagrant", "publisher": "HashiCorp, Inc.", "product": "hashicorp:vagrant"},
{"name": "Vagrant 2.4.1", "publisher": "HashiCorp", "product": "hashicorp:vagrant"},
{"name": "Vagrant VMware Utility", "publisher": "HashiCorp, Inc."},
{"name": "Terraform", "publisher": "HashiCorp", "product": "hashicorp:terraform"},
{"name": "SoapUI 5.7.2", "publisher": "SmartBear Software", "product": "smartbear:soapui"},
{"name": "SoapUI 5.5.0", "publisher": "SmartBear Software", "product": "smartbear:soapui"},
{"name": "ReadyAPI 3.52.0", "publisher": "SmartBear Software"},
{"name": "SoapUI Pro 1.4.0", "publisher": "SmartBear Software"},
{"name": "GitKraken", "publisher": "Axosoft, LLC", "product": "axosoft:gitkraken_desktop"},
{"name": "GitKraken", "publisher": "GitKraken", "product": "axosoft:gitkraken_desktop"},
{"name": "MySQL Connector/ODBC 8.0", "publisher": "Oracle Corporation", "product": "oracle:mysql_connector\\/odbc"},
{"name": "MySQL Connector/ODBC 9.0", "publisher": "Oracle Corporation", "product": "oracle:mysql_connector\\/odbc"},
{"name": "MySQL Connector/NET 9.5.0", "publisher": "Oracle Corporation", "product": "oracle:mysql_connector\\/net"},
{"name": "MySQL Workbench 8.0 CE", "publisher": "Oracle Corporation", "product": "oracle:mysql_workbench"},
{"name": "VMware Player", "publisher": "VMware, Inc.", "product": "vmware:workstation_player"},
{"name": "VMware Player 17.5.2", "publisher": "VMware, Inc.", "product": "vmware:workstation_player"},
{"name": "VMware Workstation 17 Player", "publisher": "VMware, Inc.", "product": "vmware:workstation"},
{"name": "TightVNC", "publisher": "GlavSoft LLC.", "product": "tightvnc:tightvnc"},
{"name": "TightVNC 2.8.81", "publisher": "GlavSoft LLC.", "product": "tightvnc:tightvnc"},
{"name": "UltraVnc", "publisher": "uvnc bvba", "product": "uvnc:ultravnc"},
{"name": "UltraVNC 1.4.3.6", "publisher": "uvnc bvba", "product": "uvnc:ultravnc"},
{"name": "DameWare Mini Remote Control", "publisher": "SolarWinds", "product": "solarwinds:dameware_mini_remote_control"},
{"name": "SolarWinds Dameware Mini Remote Control x64", "publisher": "SolarWinds Worldwide, LLC", "product": "solarwinds:dameware_mini_remote_control"},
{"name": "Dameware Remote Everywhere", "publisher": "SolarWinds"},
{"name": "MobaXterm", "publisher": "Mobatek", "product": "mobatek:mobaxterm"},
{"name": "MobaXterm 24.2", "publisher": "Mobatek", "product": "mobatek:mobaxterm"},
{"name": "NordVPN", "publisher": "Nord Security", "product": "nordvpn:nordvpn"},
{"name": "NordVPN 7.31.2.0", "publisher": "Nord Security", "product": "nordvpn:nordvpn"},
{"name": "NordVPN network TAP", "publisher": "NordVPN"},
{"name": "NordVPN network TUN", "publisher": "NordVPN"},
{"name": "NordPass", "publisher": "Nord Security"},
{"name": "Proton VPN", "publisher": "Proton AG", "product": "proton:protonvpn"},
{"name": "Citrix Secure Access", "publisher": "Citrix Systems, Inc.", "product": "citrix:secure_access_client"},
{"name": "Citrix Secure Access 24.8.1.19", "publisher": "Citrix Systems, Inc.", "product": "citrix:secure_access_client"},
{"name": "Citrix Gateway Plug-in", "publisher": "Citrix Systems, Inc.", "product": "citrix:gateway_plug-in"},
{"name": "Citrix Gateway Plug-in 13.0.84.11", "publisher": "Citrix Systems, Inc.", "product": "citrix:gateway_plug-in"},
{"name": "Citrix Workspace 2409", "publisher": "Citrix Systems, Inc.", "product": "citrix:workspace"},
{"name": "Global VPN Client", "publisher": "SonicWall", "product": "sonicwall:global_vpn_client"},
{"name": "SonicWall Global VPN Client", "publisher": "SonicWall Inc.", "product": "sonicwall:global_vpn_client"},
{"name": "SonicWall NetExtender", "publisher": "SonicWall Inc.", "product": "sonicwall:netextender"},
{"name": "WatchGuard Mobile VPN with SSL client 12.10", "publisher": "WatchGuard", "product": "watchguard:mobile_vpn_with_ssl"},
{"name": "WatchGuard Mobile VPN with SSL client 12.7.2", "publisher": "WatchGuard", "product": "watchguard:mobile_vpn_with_ssl"},
{"name": "WatchGuard Authentication Client 12.7", "publisher": "WatchGuard"},
{"name": "WatchGuard System Manager 12.10", "publisher": "WatchGuard"},
{"name": "Avast Free Antivirus", "publisher": "Avast Software", "product": "avast:free_antivirus"},
{"name": "Avast Free Antivirus", "publisher": "AVAST Software", "product": "avast:free_antivirus"},
{"name": "Avast Business Security", "publisher": "Avast Software", "product": "avast:antivirus"},
{"name": "Avast Premium Security", "publisher": "Avast Software"},
{"name": "Avast Secure Browser", "publisher": "Avast Software"},
{"name": "AVG AntiVirus FREE", "publisher": "AVG Technologies"},
{"name": "Webroot SecureAnywhere", "publisher": "Webroot", "product": "webroot:secureanywhere"},
{"name": "Webroot SecureAnywhere Endpoint Protection", "publisher": "Webroot", "product": "webroot:secureanywhere"},
{"name": "McAfee DLP Endpoint", "publisher": "McAfee, LLC.", "product": "mcafee:data_loss_prevention_endpoint"},
{"name": "McAfee Data Loss Prevention - Endpoint", "publisher": "McAfee, Inc.", "product": "mcafee:data_loss_prevention_endpoint"},
{"name": "McAfee Agent", "publisher": "McAfee, LLC.", "product": "mcafee:agent"},
{"name": "McAfee Endpoint Security Platform", "publisher": "McAfee, LLC.", "product": "mcafee:endpoint_security"},
{"name": "McAfee", "publisher": "McAfee, LLC"},
{"name": "VeraCrypt", "publisher": "IDRIX", "product": "idrix:veracrypt"},
{"name": "VeraCrypt 1.26.14", "publisher": "IDRIX", "product": "idrix:veracrypt"},
{"name": "GNU Privacy Guard", "publisher": "The GnuPG Project", "product": "gnupg:gnupg"},
{"name": "GnuPG 2.4.5", "publisher": "The GnuPG Project", "product": "gnupg:gnupg"},
{"name": "GnuPG VS-Desktop", "publisher": "g10 Code GmbH"},
{"name": "Gpg4win (4.3.1)", "publisher": "The Gpg4win Project", "product": "gpg4win:gpg4win"},
{"name": "OpenSC smartcard framework (64bit)", "publisher": "OpenSC Project", "product": "opensc_project:opensc"},
{"name": "OpenSC smartcard framework (32bit)", "publisher": "OpenSC Project", "product": "opensc_project:opensc"},
{"name": "Entrust Entelligence Security Provider", "publisher": "Entrust", "product": "entrustdatacard:entelligence_security_provider"},
{"name": "Entrust Entelligence Security Provider 10.0", "publisher": "Entrust Datacard", "product": "entrustdatacard:entelligence_security_provider"},
{"name": "Dell Encryption 64-bit", "publisher": "Dell Inc.", "product": "dell:encryption"},
{"name": "Dell Encryption Management Agent", "publisher": "Dell Inc.", "product": "dell:encryption"},
{"name": "Dell Data Protection | Encryption", "publisher": "Dell Inc."},
{"name": "Telegram Desktop", "publisher": "Telegram FZ-LLC", "product": "telegram:telegram_desktop"},
{"name": "Telegram Desktop version 5.5.5", "publisher": "Telegram FZ-LLC", "product": "telegram:telegram_desktop"},
{"name": "WhatsApp", "publisher": "WhatsApp"},
{"name": "WhatsApp 2.2245.9", "publisher": "WhatsApp"},
{"name": "Nextcloud", "publisher": "Nextcloud GmbH", "product": "nextcloud:desktop"},
{"name": "Nextcloud 3.13.2", "publisher": "Nextcloud GmbH", "product": "nextcloud:desktop"},
{"name": "Nextcloud Talk", "publisher": "Nextcloud GmbH"},
{"name": "ownCloud", "publisher": "ownCloud GmbH", "product": "owncloud:owncloud_desktop_client"},
{"name": "ownCloud 5.3.1", "publisher": "ownCloud GmbH", "product": "owncloud:owncloud_desktop_client"},
{"name": "Synology Drive Client", "publisher": "Synology, Inc.", "product": "synology:drive_client"},
{"name": "Synology Drive Client 3.5.1", "publisher": "Synology Inc.", "product": "synology:drive_client"},
{"name": "Synology Assistant", "publisher": "Synology"},
{"name": "Synology Active Backup for Business Agent", "publisher": "Synology Inc."},
{"name": "IBM Aspera Connect", "publisher": "IBM", "product": "ibm:aspera_connect"},
{"name": "Aspera Connect", "publisher": "Aspera, Inc.", "product": "ibm:aspera_connect"},
{"name": "Macrium Reflect Workstation", "publisher": "Paramount Software (UK) Ltd.", "product": "macrium:reflect"},
{"name": "Macrium Reflect Free Edition", "publisher": "Paramount Software (UK) Ltd.", "product": "macrium:reflect"},
{"name": "Druva inSync 7.5.0", "publisher": "Druva Technologies Pte. Ltd.", "product": "druva:insync_client"},
{"name": "Druva inSync", "publisher": "Druva Inc.", "product": "druva:insync_client"},
{"name": "AutoHotkey 1.1.37.02", "publisher": "Lexikos", "product": "autohotkey:autohotkey"},
{"name": "AutoHotkey", "publisher": "AutoHotkey Foundation LLC", "product": "autohotkey:autohotkey"},
{"name": "Rufus", "publisher": "Akeo Consulting", "product": "akeo:rufus"},
{"name": "Rufus 4.5", "publisher": "Akeo Consulting", "product": "akeo:rufus"},
{"name": "CPUID CPU-Z 2.10", "publisher": "CPUID, Inc.", "product": "cpuid:cpu-z"},
{"name": "CPUID CPU-Z 1.81", "publisher": "CPUID, Inc.", "product": "cpuid:cpu-z"},
{"name": "CPUID HWMonitor 1.53", "publisher": "CPUID, Inc."},
{"name": "TeraCopy", "publisher": "Code Sector", "product": "codesector:teracopy"},
{"name": "TeraCopy 3.9.7", "publisher": "Code Sector", "product": "codesector:teracopy"},
{"name": "Samsung Magician", "publisher": "Samsung Electronics", "product": "samsung:magician"},
{"name": "Samsung Magician 8.1.0", "publisher": "Samsung Electronics Co., Ltd.", "product": "samsung:magician"},
{"name": "Samsung Data Migration", "publisher": "Samsung"},
{"name": "Western Digital SSD Dashboard", "publisher": "Western Digital Corporation", "product": "westerndigital:ssd_dashboard"},
{"name": "WD SSD Dashboard", "publisher": "Western Digital Corporation", "product": "westerndigital:ssd_dashboard"},
{"name": "Western Digital Dashboard", "publisher": "Western Digital Technologies, Inc.", "product": "westerndigital:dashboard"},
{"name": "Dashboard", "publisher": "Western Digital Technologies, Inc.", "product": "westerndigital:dashboard"},
{"name": "Dashboard", "publisher": ""},
{"name": "Dell Command | Monitor", "publisher": "Dell Inc.", "product": "dell:command_\\|_monitor"},
{"name": "Dell Command | Monitor 10.11", "publisher": "Dell Inc.", "product": "dell:command_\\|_monitor"},
{"name": "Dell Command | Configure", "publisher": "Dell Inc.", "product": "dell:command\\|configure"},
{"name": "Dell Update", "publisher": "Dell Inc.", "product": "dell:update"},
{"name": "Dell Update for Windows Universal", "publisher": "Dell Inc.", "product": "dell:update"},
{"name": "Dell Update - SupportAssist Update Plugin", "publisher": "Dell Inc."},
{"name": "HP System Event Utility", "publisher": "HP Inc.", "product": "hp:system_event_utility"},
{"name": "HP System Event Utility", "publisher": "HP", "product": "hp:system_event_utility"},
{"name": "HP PC Hardware Diagnostics Windows", "publisher": "HP Inc.", "product": "hp:pc_hardware_diagnostics"},
{"name": "HP PC Hardware Diagnostics Windows", "publisher": "HP", "product": "hp:pc_hardware_diagnostics"},
{"name": "HP PC Hardware Diagnostics UEFI", "publisher": "HP Inc."},
{"name": "Lenovo Commercial Vantage", "publisher": "Lenovo Group Ltd.", "product": "lenovo:commercial_vantage"},
{"name": "Lenovo Commercial Vantage", "publisher": "Lenovo", "product": "lenovo:commercial_vantage"},
{"name": "Lenovo Vantage", "publisher": "Lenovo Group Ltd.", "product": "lenovo:vantage"},
{"name": "Lenovo Service Bridge", "publisher": "Lenovo", "product": "lenovo:service_bridge"},
{"name": "Lenovo Service Bridge", "publisher": "Lenovo Group Ltd.", "product": "lenovo:service_bridge"},
{"name": "ASUS Live Update", "publisher": "ASUSTeK COMPUTER INC.", "product": "asus:live_update"},
{"name": "ASUS Live Update", "publisher": "ASUS", "product": "asus:live_update"},
{"name": "Power Automate for desktop", "publisher": "Microsoft Corporation", "product": "microsoft:power_automate_for_desktop"},
{"name": "Power Automate for desktop (x64)", "publisher": "Microsoft Corporation", "product": "microsoft:power_automate_for_desktop"},
{"name": "Power Automate Desktop", "publisher": "Microsoft Corporation", "product": "microsoft:power_automate_for_desktop"},
{"name": "Power Automate machine runtime", "publisher": "Microsoft Corporation"},
{"name": "Ivanti Workspace Control 2023.2", "publisher": "Ivanti", "product": "ivanti:workspace_control"},
{"name": "Ivanti Workspace Control Agent", "publisher": "Ivanti", "product": "ivanti:workspace_control"},
{"name": "Ivanti Secure Access Client", "publisher": "Ivanti, Inc.", "product": "ivanti:secure_access_client"},
{"name": "MicroStation CONNECT Edition", "publisher": "Bentley Systems, Incorporated", "product": "bentley:microstation"},
{"name": "MicroStation 2023", "publisher": "Bentley Systems, Incorporated", "product": "bentley:microstation"},
{"name": "MicroStation PowerDraft CONNECT Edition", "publisher": "Bentley Systems, Incorporated"},
{"name": "ProjectWise Integration for MicroStation", "publisher": "Bentley Systems, Incorporated"},
{"name": "IBM SPSS Statistics", "publisher": "IBM Corp.", "product": "ibm:spss_statistics"},
{"name": "IBM SPSS Statistics 27", "publisher": "IBM Corp.", "product": "ibm:spss_statistics"},
{"name": "IBM SPSS Statistics - Essentials for Python", "publisher": "IBM Corp."},
{"name": "IBM SPSS Modeler 18.4", "publisher": "IBM Corp."},
{"name": "QlikView x64", "publisher": "QlikTech International AB", "product": "qlik:qlikview"},
{"name": "QlikView", "publisher": "QlikTech International AB", "product": "qlik:qlikview"},
{"name": "QlikView Server x64", "publisher": "QlikTech International AB"},
{"name": "QlikView IE Plugin", "publisher": "QlikTech International AB"},
{"name": "Blue Prism", "publisher": "Blue Prism Limited", "product": "blueprism:robotic_process_automation"},
{"name": "Blue Prism 6.10.1", "publisher": "Blue Prism Limited", "product": "blueprism:robotic_process_automation"},
{"name": "Blue Prism Capture", "publisher": "Blue Prism Limited"}
]
@@ -50,7 +50,6 @@ public class TypedRequestGuardTests
// Not request bodies at all: telemetry item payloads (agent wire, D-200) and stored table columns.
["Functions/Ingest/TelemetryPayloadParser.cs"] = "telemetry item PayloadJson (agent wire, D-200)",
["Functions/Config/ListTenantConfigBackupsFunction.cs"] = "stored backup diff column",
["Functions/Vulnerability/GetCpeMappingsFunction.cs"] = "stored pattern-list columns",
};
/// <summary>
@@ -0,0 +1,252 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Logging.Abstractions;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="UnmappedSoftware"/> is the one list of programs without a mapping that the
/// operator's view and the nightly automatic mapper share: one entry per vendor and canonical
/// name, decided by the matcher and the ignore list, read from the columns it names. It also
/// says which programs occur only as Store packages, which the automatic mapper leaves alone.
/// </summary>
public class UnmappedSoftwareTests
{
private const string OtherTenant = "33333333-3333-3333-3333-333333333333";
private const string OlderSession = "44444444-4444-4444-4444-444444444444";
private const string NewestSession = "55555555-5555-5555-5555-555555555555";
private static readonly DateTime Now = new(2026, 11, 12, 8, 0, 0, DateTimeKind.Utc);
private sealed class Rig
{
public Rig() =>
Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso widget", "cpe:2.3:a:contoso:widget", "contoso widget"));
public MappingTables Tables { get; } = new();
public void Ignore(string vendor, string product) => Tables.Put(
CpeMappingStore.IgnoredPartition, CpeSeedFile.RowKeyOf(vendor, product),
new Dictionary<string, object> { ["NormalizedVendor"] = vendor, ["NormalizedProduct"] = product });
/// <summary>Aggregates what a reader of the listed columns gets, over the mappings in service.</summary>
public async Task<List<UnmatchedSoftwareItem>> AggregateAsync()
{
var mappings = await new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance).GetAsync();
var rows = await Tables.Repo.Object.GetAllSoftwareInventoryAsync(UnmappedSoftware.Columns);
return UnmappedSoftware.Aggregate(rows, mappings);
}
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public async Task Aggregate_VersionsAndArchitecturesOfOneProgram_AreOneEntry_ShownAsItsMostRecentlySeenRow(bool newestRowFirst)
{
var rig = new Rig();
void Older() => rig.Tables.AddInventory(
"northwind", "Northwind Tool 1.0 (x64)", "1.0", sessions: 3, publisher: "Northwind Traders Ltd",
lastSeenUtc: Now.AddDays(-3), lastSessionId: OlderSession);
void Newest() => rig.Tables.AddInventory(
"northwind", "Northwind Tool 2.0 (x86)", "2.0", sessions: 4, publisher: "Northwind Traders",
lastSeenUtc: Now.AddDays(-1), lastSessionId: NewestSession);
if (newestRowFirst) { Newest(); Older(); } else { Older(); Newest(); }
rig.Tables.AddInventory(
"northwind", "Northwind Tool 1.0 (x64)", "1.0", sessions: 2, publisher: "Northwind Traders Ltd",
lastSeenUtc: Now.AddDays(-2), lastSessionId: OlderSession, tenantId: OtherTenant);
var entry = Assert.Single(await rig.AggregateAsync());
Assert.Equal(9, entry.Frequency);
Assert.Equal("Northwind Tool 2.0 (x86)", entry.SoftwareName);
Assert.Equal("2.0", entry.NormalizedVersion);
Assert.Equal(NewestSession, entry.ExampleSessionId);
Assert.Equal(Now.AddDays(-1).ToString("o"), entry.LastSeenAt);
Assert.Equal("Northwind Traders", entry.Publisher);
Assert.Equal("northwind", entry.NormalizedVendor);
Assert.Equal("northwind tool", entry.CanonicalName);
}
[Fact]
public async Task Aggregate_GoesByTheMappingsInService_NotByTheCpeColumnOfTheRow()
{
var rig = new Rig();
// The column trails a mapping change until the rows are reconciled.
rig.Tables.AddInventory("contoso", "Contoso Widget 4.2 (x64)", "4.2", cpeUri: "");
rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", cpeUri: "cpe:2.3:a:northwind:tool");
var entry = Assert.Single(await rig.AggregateAsync());
Assert.Equal("Northwind Tool", entry.SoftwareName);
}
[Fact]
public async Task Aggregate_IgnoredProgram_IsLeftOutInEveryVersion()
{
var rig = new Rig();
rig.Ignore("northwind", "northwind tool");
rig.Tables.AddInventory("northwind", "Northwind Tool 1.0 (x64)", "1.0");
rig.Tables.AddInventory("northwind", "Northwind Tool 2.0", "2.0");
rig.Tables.AddInventory("northwind", "Northwind Viewer", "1.0");
var entry = Assert.Single(await rig.AggregateAsync());
Assert.Equal("Northwind Viewer", entry.SoftwareName);
}
[Fact]
public async Task Aggregate_IgnoreEntryFiledUnderTheRawPublisher_StillHidesTheProgram()
{
var rig = new Rig();
// Entries written before the vendor was normalized hold the publisher and the full name.
rig.Ignore("Northwind Traders Ltd", "Northwind Tool 1.0 (x64)");
rig.Tables.AddInventory("northwind", "Northwind Tool 2.0", "2.0", publisher: "Northwind Traders Ltd");
rig.Tables.AddInventory("fabrikam", "Northwind Tool 2.0", "2.0", publisher: "Fabrikam Inc");
var entry = Assert.Single(await rig.AggregateAsync());
// Another vendor's program of the same name was never ignored.
Assert.Equal("fabrikam", entry.NormalizedVendor);
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public async Task Aggregate_RowWithoutADisplayName_FallsBackToTheNormalizedName(bool emptyInsteadOfMissing)
{
var rig = new Rig();
var row = rig.Tables.AddInventory("northwind", "Northwind Tool", "3.1", sessions: 2, lastSeenUtc: Now);
if (emptyInsteadOfMissing) row["DisplayName"] = string.Empty;
else row.Remove("DisplayName");
rig.Tables.AddInventory("northwind", "Northwind Tool 4.0", "4.0", sessions: 1, lastSeenUtc: Now.AddDays(-1));
var entry = Assert.Single(await rig.AggregateAsync());
Assert.Equal("northwind tool", entry.SoftwareName);
Assert.Equal("northwind tool", entry.CanonicalName);
Assert.Equal(3, entry.Frequency);
}
[Theory]
[InlineData("1.2.3")]
[InlineData("(x64)")]
[InlineData("{11111111-2222-3333-4444-555555555555}")]
[InlineData("")]
public async Task Aggregate_RowWhoseNameCanonicalizesToNothing_IsSkipped(string displayName)
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", displayName, "1.0");
Assert.Empty(await rig.AggregateAsync());
}
[Fact]
public async Task Aggregate_SameNameUnderTwoVendors_IsTwoEntries_NoVendorAndThePlaceholderAreOne()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Agent 2.1", "2.1", sessions: 5);
rig.Tables.AddInventory("fabrikam", "Agent 7.0", "7.0", sessions: 3);
rig.Tables.AddInventory("", "Agent 1.0", "1.0", sessions: 1);
rig.Tables.AddInventory("unknown", "Agent 1.1", "1.1", sessions: 1);
var entries = await rig.AggregateAsync();
Assert.Equal(
new[] { ("", 2), ("fabrikam", 3), ("northwind", 5) },
entries.Select(e => (e.NormalizedVendor!, e.Frequency)).OrderBy(e => e.Item1, StringComparer.Ordinal));
Assert.All(entries, e => Assert.Equal("agent", e.CanonicalName));
}
[Fact]
public async Task Aggregate_RowWithoutASessionCount_CountsOnce()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool 1.0", "1.0").Remove("SessionCount");
rig.Tables.AddInventory("northwind", "Northwind Tool 2.0", "2.0", sessions: 4);
Assert.Equal(5, Assert.Single(await rig.AggregateAsync()).Frequency);
}
// -----------------------------------------------------------------------
// Programs that occur only as Store packages
// -----------------------------------------------------------------------
private static async Task<HashSet<string>> StoreOnlyKeysAsync(Rig rig) =>
UnmappedSoftware.StoreOnlyKeys(await rig.Tables.Repo.Object.GetAllSoftwareInventoryAsync(UnmappedSoftware.Columns));
[Fact]
public async Task StoreOnlyKeys_ProgramEveryRowReportsAsAStorePackage_IsNamedByItsSoftwareKey()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX");
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.4", registrySource: "appx", tenantId: OtherTenant);
rig.Tables.AddInventory("adatum", "Adatum Reader 2.0", "2.0", registrySource: "HKLM_64");
rig.Tables.AddInventory("fabrikam", "Fabrikam Gadget (x86)", "1.0", registrySource: "HKLM_32");
var keys = await StoreOnlyKeysAsync(rig);
Assert.Equal(new[] { CpeMappingMatcher.SoftwareKey("northwind", "Northwind Tool") }, keys);
}
[Theory]
[InlineData("HKLM_64")]
[InlineData("HKU_1001")]
// A row that does not say where the program was read from is no Store package.
[InlineData("")]
public async Task StoreOnlyKeys_ProgramAlsoInstalledAnotherWay_IsNotStoreOnly_WhereverThatRowComesFrom(string otherSource)
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX");
// The same program by vendor and name without version, on a device of another tenant.
rig.Tables.AddInventory("northwind", "Northwind Tool 3.1 (x64)", "3.1", registrySource: otherSource, tenantId: OtherTenant);
Assert.Empty(await StoreOnlyKeysAsync(rig));
}
[Fact]
public async Task StoreOnlyKeys_RowWithoutTheSourceColumn_IsNoStorePackage()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX").Remove("RegistrySource");
Assert.Empty(await StoreOnlyKeysAsync(rig));
}
[Fact]
public async Task StoreOnlyKeys_AreDecidedPerVendorAndName()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Agent", "1.0.3", registrySource: "AppX");
// Another vendor's program of the same name says nothing about this one.
rig.Tables.AddInventory("fabrikam", "Agent 7.0", "7.0");
rig.Tables.AddInventory("northwind", "Agent Console", "2.0");
var keys = await StoreOnlyKeysAsync(rig);
Assert.Equal(new[] { CpeMappingMatcher.SoftwareKey("northwind", "Agent") }, keys);
}
[Fact]
public async Task StoreOnlyKeys_RowWithoutADisplayName_GoesByItsNormalizedName_AndANameThatIsNoneIsSkipped()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool", "1.0.3", registrySource: "AppX")["DisplayName"] = string.Empty;
rig.Tables.AddInventory("northwind", "1.0.4 (x64)", "1.0.4", registrySource: "AppX");
var keys = await StoreOnlyKeysAsync(rig);
Assert.Equal(new[] { CpeMappingMatcher.SoftwareKey("northwind", "northwind tool") }, keys);
}
[Fact]
public async Task StoreOnlyKeys_UseTheKeyTheUnmappedListFilesAProgramUnder()
{
var rig = new Rig();
rig.Tables.AddInventory("northwind", "Northwind Tool 1.0 (x64)", "1.0.3", registrySource: "AppX", publisher: "Northwind Traders Ltd");
var keys = await StoreOnlyKeysAsync(rig);
var listed = Assert.Single(await rig.AggregateAsync());
// What the operator's list shows for the program is what a caller looks it up with.
Assert.Contains(CpeMappingMatcher.SoftwareKey(listed.NormalizedVendor, listed.SoftwareName), keys);
}
}
@@ -110,4 +110,22 @@ public class VersionComparerTests
// No constraints = all versions affected
Assert.True(VersionComparer.IsVersionAffected("99.99.99", null, null, null, null));
}
// -----------------------------------------------------------------------
// AreComparable
// -----------------------------------------------------------------------
[Theory]
[InlineData("26.198", "2022.30050", false)] // releases against years: the order says nothing
[InlineData("2022.30050", "26.198", false)]
[InlineData("2024.1", "2023.3", true)]
[InlineData("24.08", "25.0", true)]
[InlineData("", "2022.30050", true)]
[InlineData("26.198", "", true)]
[InlineData(null, "2022.30050", true)]
[InlineData("26.198", null, true)]
public void AreComparable_IsFalseOnlyForAYearSchemeAgainstAReleaseScheme(string? installed, string? rangeBoundary, bool expected)
{
Assert.Equal(expected, VersionComparer.AreComparable(installed, rangeBoundary));
}
}
@@ -0,0 +1,547 @@
using System.Net;
using System.Net.Http.Headers;
using System.Text.Json;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The nightly walks of <see cref="VulnerabilityCacheRefreshService"/> over the per-product CVE
/// cache. The NVD walk fills what scans are waiting for first, then rows of the older shape,
/// then rows older than a week; it removes rows no mapping names, stops at a refusal and never
/// replaces a row with an answer NVD did not give. A cooldown that is already running when the
/// walk starts is waited out once. The EPSS walk rewrites a row with the row's own freshness
/// stamp, total and shape.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class VulnerabilityCacheRefreshServiceTests : IDisposable
{
private static readonly DateTime Now = DateTime.UtcNow;
public VulnerabilityCacheRefreshServiceTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private static string Product(string name) => $"cpe:2.3:a:contoso:{name}";
private static string Key(string name) => CveCacheFiller.CacheKey(Product(name));
private sealed class Rig
{
private Rig(VulnerabilityScanRig scan, HttpMessageHandler epss)
{
Scan = scan;
AdminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions()));
Service = new VulnerabilityCacheRefreshService(
scan.Repo.Object,
scan.Filler,
new EpssApiClient(new HttpClient(epss), NullLogger<EpssApiClient>.Instance),
scan.MappingStore,
AdminConfig.Object,
NullLogger<VulnerabilityCacheRefreshService>.Instance);
}
/// <summary>A rig whose mappings name exactly <paramref name="mappedProducts"/>; NVD answers every product with no CVEs.</summary>
public static async Task<Rig> CreateAsync(IEnumerable<string> mappedProducts, HttpMessageHandler? epss = null)
{
var scan = await VulnerabilityScanRig.CreateAsync();
scan.Mappings.Clear();
scan.Mappings.AddRange(mappedProducts.Select(name =>
VulnerabilityCacheRows.Mapping("contoso", $"contoso {name}", Product(name), $"contoso {name}")));
scan.NvdAnswer = _ => NvdFixture.Page();
return new Rig(scan, epss ?? new ReferenceDataFixture.FailingHandler());
}
public VulnerabilityScanRig Scan { get; }
public Mock<AdminConfigurationService> AdminConfig { get; }
public VulnerabilityCacheRefreshService Service { get; }
/// <summary>A service over the same cache and mappings whose NVD walk may take <paramref name="walkBudget"/>.</summary>
public VulnerabilityCacheRefreshService WithWalkBudget(TimeSpan walkBudget) => new(
Scan.Repo.Object,
Scan.Filler,
new EpssApiClient(new HttpClient(new ReferenceDataFixture.FailingHandler()), NullLogger<EpssApiClient>.Instance),
Scan.MappingStore,
AdminConfig.Object,
NullLogger<VulnerabilityCacheRefreshService>.Instance)
{
WalkBudget = walkBudget,
};
/// <summary>A service over the same cache and mappings that waits through <paramref name="pause"/> instead of letting time pass.</summary>
public VulnerabilityCacheRefreshService WithPause(Func<TimeSpan, Task> pause) => new(
Scan.Repo.Object,
Scan.Filler,
new EpssApiClient(new HttpClient(new ReferenceDataFixture.FailingHandler()), NullLogger<EpssApiClient>.Instance),
Scan.MappingStore,
AdminConfig.Object,
NullLogger<VulnerabilityCacheRefreshService>.Instance)
{
Pause = pause,
};
/// <summary>The products NVD was asked for, in order.</summary>
public IEnumerable<string> Asked => Scan.Nvd.Requests.Select(r => r.Cpe.Substring(Product(string.Empty).Length));
public void VerifyStamped(Times times) => AdminConfig.Verify(
a => a.UpdateAsync(It.IsAny<Func<AdminConfiguration, string?>>(), It.IsAny<string>(), It.IsAny<string?>()), times);
}
private static Dictionary<string, object> Filled(string name, DateTime cachedAt, params CachedCve[] cves) =>
VulnerabilityCacheRows.Filled(Product(name), cves, cachedAt, nvdTotal: 7);
private static Dictionary<string, object> Pending(string name, DateTime requestedAt) =>
VulnerabilityCacheRows.Pending(Product(name), requestedAt);
/// <summary>NVD's answer that it lists no CVE for the product.</summary>
private static Dictionary<string, object> EmptyAnswer(string name, DateTime cachedAt) =>
VulnerabilityCacheRows.Filled(Product(name), Array.Empty<CachedCve>(), cachedAt, nvdTotal: 0);
// -----------------------------------------------------------------------
// NVD walk
// -----------------------------------------------------------------------
[Fact]
public async Task Refresh_FillsWaitingProductsFirst_ThenOlderShapes_ThenStaleRows_AndLeavesFreshRowsAlone()
{
var rig = await Rig.CreateAsync(new[]
{
"fresh", "stale", "staler", "unversioned", "version1", "waiting", "waiting_longer",
});
var version1 = Filled("version1", Now.AddDays(-2));
version1["SchemaVersion"] = 1;
rig.Scan.Cache(Filled("fresh", Now.AddDays(-6)));
rig.Scan.Cache(Filled("stale", Now.AddDays(-8)));
rig.Scan.Cache(Filled("staler", Now.AddDays(-30)));
rig.Scan.Cache(VulnerabilityCacheRows.OlderShape(Product("unversioned"), Array.Empty<CachedCve>(), Now.AddDays(-1)));
rig.Scan.Cache(version1);
rig.Scan.Cache(Pending("waiting", Now.AddHours(-2)));
rig.Scan.Cache(Pending("waiting_longer", Now.AddDays(-3)));
var freshBefore = rig.Scan.CacheRows[Key("fresh")];
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(new[] { "waiting_longer", "waiting", "version1", "unversioned", "staler", "stale" }, rig.Asked);
Assert.Equal(6, result!.Stale);
Assert.Equal(6, result.Refreshed);
Assert.Equal(0, result.Failed);
Assert.Equal(0, result.SkippedThrottled);
Assert.Equal(0, result.Removed);
Assert.Equal(0, result.PendingOverdue);
Assert.True(result.Completed);
rig.VerifyStamped(Times.Once());
Assert.Same(freshBefore, rig.Scan.CacheRows[Key("fresh")]);
Assert.All(rig.Asked, name =>
{
var row = rig.Scan.CacheRows[Key(name)];
Assert.Equal(CveCacheFiller.SchemaVersion, row["SchemaVersion"]);
Assert.Equal("filled", row["State"]);
});
}
[Fact]
public async Task Refresh_RemovesRowsNoMappingNames_FilledOrWaiting()
{
var rig = await Rig.CreateAsync(new[] { "mapped" });
rig.Scan.Cache(Filled("mapped", Now.AddDays(-8)));
rig.Scan.Cache(Filled("withdrawn", Now.AddDays(-8)));
rig.Scan.Cache(Pending("withdrawn_waiting", Now.AddDays(-3)));
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(new[] { Key("withdrawn"), Key("withdrawn_waiting") }, rig.Scan.CacheDeletes.OrderBy(k => k, StringComparer.Ordinal));
Assert.Equal(2, result!.Removed);
Assert.Equal(new[] { "mapped" }, rig.Asked);
Assert.Equal(1, result.Stale);
Assert.Equal(0, result.PendingOverdue);
}
[Fact]
public async Task Refresh_MappingsUnreadable_RemovesNothingAndStillRefreshes()
{
var rig = await Rig.CreateAsync(new[] { "mapped" });
rig.Scan.MappingFailure = new InvalidOperationException("storage down");
rig.Scan.Cache(Filled("mapped", Now.AddDays(-8)));
rig.Scan.Cache(Filled("unknown_whether_mapped", Now.AddDays(-9)));
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Empty(rig.Scan.CacheDeletes);
Assert.Equal(0, result!.Removed);
Assert.Equal(new[] { "unknown_whether_mapped", "mapped" }, rig.Asked);
Assert.Equal(2, result.Refreshed);
}
[Fact]
public async Task Refresh_NvdRefuses_StopsTheWalkAndLeavesTheRestForNextTime()
{
var rig = await Rig.CreateAsync(new[] { "first", "second", "third" });
rig.Scan.Cache(Pending("first", Now.AddDays(-3)));
rig.Scan.Cache(Pending("second", Now.AddDays(-2)));
rig.Scan.Cache(Pending("third", Now.AddHours(-2)));
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("second") ? NvdFixture.Status(HttpStatusCode.TooManyRequests) : NvdFixture.Page();
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(new[] { "first", "second" }, rig.Asked);
Assert.Equal(3, result!.Stale);
Assert.Equal(1, result.Refreshed);
Assert.Equal(2, result.SkippedThrottled);
Assert.Equal(0, result.Failed);
// "second" has waited more than a day and still has no data; "third" is within the day.
Assert.Equal(1, result.PendingOverdue);
Assert.False(result.Completed);
rig.VerifyStamped(Times.Never());
Assert.Equal("pending", rig.Scan.CacheRows[Key("second")]["State"]);
Assert.Equal("pending", rig.Scan.CacheRows[Key("third")]["State"]);
}
[Fact]
public async Task Refresh_NvdsGatewayRefusesTheClient_StopsTheWalkLikeAThrottle()
{
var rig = await Rig.CreateAsync(new[] { "first", "second", "third" });
rig.Scan.Cache(Pending("first", Now.AddDays(-3)));
rig.Scan.Cache(Pending("second", Now.AddDays(-2)));
rig.Scan.Cache(Pending("third", Now.AddHours(-2)));
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("second") ? NvdFixture.Status(HttpStatusCode.Forbidden) : NvdFixture.Page();
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
// Counted as one failed product, the walk would go on and be refused for every row left.
Assert.Equal(new[] { "first", "second" }, rig.Asked);
Assert.Equal(1, result!.Refreshed);
Assert.Equal(2, result.SkippedThrottled);
Assert.Equal(0, result.Failed);
Assert.False(result.Completed);
rig.VerifyStamped(Times.Never());
Assert.Equal("pending", rig.Scan.CacheRows[Key("second")]["State"]);
}
[Fact]
public async Task Refresh_NvdFailsForOneProduct_CountsItAndGoesOn_WithoutTouchingItsRow()
{
var rig = await Rig.CreateAsync(new[] { "waiting", "served", "stale" });
rig.Scan.Cache(Pending("waiting", Now.AddDays(-2)));
rig.Scan.Cache(Filled("served", Now.AddDays(-9), VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, Product("served"), "2.0")));
rig.Scan.Cache(Filled("stale", Now.AddDays(-8)));
var servedBefore = rig.Scan.CacheRows[Key("served")];
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("stale") ? NvdFixture.Page() : NvdFixture.Status(HttpStatusCode.InternalServerError);
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(new[] { "waiting", "served", "stale" }, rig.Asked);
Assert.Equal(2, result!.Failed);
Assert.Equal(1, result.Refreshed);
Assert.Equal(0, result.SkippedThrottled);
Assert.Equal(1, result.PendingOverdue);
// Real data is never replaced by a refusal: the row keeps serving what it has.
Assert.Same(servedBefore, rig.Scan.CacheRows[Key("served")]);
Assert.Equal(new[] { Key("stale") }, rig.Scan.CacheWrites);
// Every due row got its attempt, so the walk counts as done.
Assert.True(result.Completed);
rig.VerifyStamped(Times.Once());
}
[Fact]
public async Task Refresh_ThreeRequestsInARowWithoutAnAnswer_EndTheWalk_AndLeaveTheRestForNextTime()
{
var rig = await Rig.CreateAsync(new[] { "first", "second", "third", "fourth", "fifth" });
rig.Scan.Cache(Pending("first", Now.AddDays(-5)));
rig.Scan.Cache(Pending("second", Now.AddDays(-4)));
rig.Scan.Cache(Pending("third", Now.AddDays(-3)));
rig.Scan.Cache(Filled("fourth", Now.AddDays(-9)));
rig.Scan.Cache(Filled("fifth", Now.AddDays(-8)));
rig.Scan.NvdAnswer = _ => NvdFixture.Status(HttpStatusCode.InternalServerError);
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(3, VulnerabilityCacheRefreshService.MaxUnansweredInARow);
Assert.Equal(new[] { "first", "second", "third" }, rig.Asked);
Assert.Equal(5, result!.Stale);
Assert.Equal(3, result.Failed);
Assert.Equal(2, result.SkippedUnavailable);
Assert.Equal(0, result.SkippedThrottled);
Assert.Equal(0, result.Refreshed);
Assert.Equal(3, result.PendingOverdue);
Assert.Null(result.Error);
// The rows that were not asked about are due again; the walk does not count as done.
Assert.False(result.Completed);
rig.VerifyStamped(Times.Never());
Assert.Empty(rig.Scan.CacheWrites);
}
[Fact]
public async Task Refresh_RequestNvdTurnsDown_BreaksARunOfUnansweredOnes_AndTheWalkGoesOn()
{
var names = new[] { "first", "second", "third", "fourth", "fifth", "sixth" };
var rig = await Rig.CreateAsync(names);
for (var n = 0; n < names.Length; n++)
rig.Scan.Cache(Pending(names[n], Now.AddDays(-10 + n)));
// Two without an answer, one NVD turns down, two more without an answer: never three in a row.
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("third") ? NvdFixture.Status(HttpStatusCode.NotFound)
: request.Cpe == Product("sixth") ? NvdFixture.Page()
: NvdFixture.Status(HttpStatusCode.InternalServerError);
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(names, rig.Asked);
Assert.Equal(5, result!.Failed);
Assert.Equal(1, result.Refreshed);
Assert.Equal(0, result.SkippedUnavailable);
Assert.Equal(new[] { Key("sixth") }, rig.Scan.CacheWrites);
Assert.True(result.Completed);
rig.VerifyStamped(Times.Once());
}
[Fact]
public async Task Refresh_RequestsNvdTurnsDown_NeverEndTheWalk_HoweverManyInARow()
{
var names = new[] { "first", "second", "third", "fourth", "fifth" };
var rig = await Rig.CreateAsync(names);
for (var n = 0; n < names.Length; n++)
rig.Scan.Cache(Filled(names[n], Now.AddDays(-20 + n)));
// A request that is wrong for one product says nothing about the next one.
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("fifth") ? NvdFixture.Page() : NvdFixture.Status(HttpStatusCode.NotFound);
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(names, rig.Asked);
Assert.Equal(4, result!.Failed);
Assert.Equal(1, result.Refreshed);
Assert.Equal(0, result.SkippedUnavailable);
Assert.True(result.Completed);
rig.VerifyStamped(Times.Once());
}
[Fact]
public async Task Refresh_WalkWithoutTimeLeft_AsksNothing_AndLeavesEveryDueRowForNextTime()
{
var rig = await Rig.CreateAsync(new[] { "waiting", "stale", "fresh" });
rig.Scan.Cache(Pending("waiting", Now.AddDays(-3)));
rig.Scan.Cache(Filled("stale", Now.AddDays(-8)));
rig.Scan.Cache(Filled("fresh", Now.AddDays(-1)));
var result = await rig.WithWalkBudget(TimeSpan.Zero).RefreshStaleCveCacheAsync("test");
Assert.Empty(rig.Asked);
Assert.Equal(2, result!.Stale);
Assert.Equal(2, result.SkippedUnavailable);
Assert.Equal(0, result.SkippedThrottled);
Assert.Equal(0, result.Failed);
Assert.Equal(0, result.Refreshed);
Assert.Equal(1, result.PendingOverdue);
Assert.False(result.Completed);
rig.VerifyStamped(Times.Never());
Assert.Empty(rig.Scan.CacheWrites);
}
// -----------------------------------------------------------------------
// A cooldown that is running when the walk starts
// -----------------------------------------------------------------------
/// <summary>
/// NVD refuses a request that is not the walk's, as it can refuse the automatic mapper that
/// asks right before it: the client's cooldown is running when the walk starts.
/// </summary>
private static async Task NvdRefusesTheRequestBeforeTheWalkAsync(TimeSpan? retryAfter = null)
{
var refusing = new NvdStubHandler(_ =>
{
var refusal = NvdFixture.Status(HttpStatusCode.TooManyRequests);
if (retryAfter is { } wait) refusal.Headers.RetryAfter = new RetryConditionHeaderValue(wait);
return refusal;
});
await NvdFixture.Client(refusing).SearchCvesByCpeAsync(Product("asked_before_the_walk"));
Assert.True(NvdApiClient.IsInCooldown);
}
/// <summary>A way to wait that takes no time and keeps what it was asked to wait for.</summary>
private static Func<TimeSpan, Task> Recording(List<TimeSpan> waits, Action? whenOver = null) => wait =>
{
waits.Add(wait);
whenOver?.Invoke();
return Task.CompletedTask;
};
[Theory]
[InlineData(null, 31)]
// NVD said how long to stay away: the wait follows what is left of that, not a fixed time.
[InlineData(120, 121)]
public async Task Refresh_CooldownRunningAsTheWalkStarts_IsWaitedOutOnce_ThenTheRowsAreAsked(int? retryAfterSeconds, int waitSeconds)
{
var rig = await Rig.CreateAsync(new[] { "first", "second" });
rig.Scan.Cache(Pending("first", Now.AddDays(-3)));
rig.Scan.Cache(Pending("second", Now.AddDays(-2)));
await NvdRefusesTheRequestBeforeTheWalkAsync(retryAfterSeconds is { } seconds ? TimeSpan.FromSeconds(seconds) : null);
var waits = new List<TimeSpan>();
// By the end of the wait, time has ended the cooldown.
var service = rig.WithPause(Recording(waits, whenOver: NvdApiClient.ResetThrottleState));
var result = await service.RefreshStaleCveCacheAsync("test");
// What is left of the cooldown and a second on top; the test has used a moment of it.
Assert.InRange(Assert.Single(waits), TimeSpan.FromSeconds(waitSeconds - 5), TimeSpan.FromSeconds(waitSeconds));
Assert.Equal(new[] { "first", "second" }, rig.Asked);
Assert.Equal(2, result!.Refreshed);
Assert.Equal(0, result.SkippedThrottled);
Assert.True(result.Completed);
rig.VerifyStamped(Times.Once());
}
[Fact]
public async Task Refresh_NvdStillRefusingAfterTheWait_EndsTheWalk_WithoutWaitingAgain()
{
var rig = await Rig.CreateAsync(new[] { "first", "second" });
rig.Scan.Cache(Pending("first", Now.AddDays(-3)));
rig.Scan.Cache(Pending("second", Now.AddDays(-2)));
await NvdRefusesTheRequestBeforeTheWalkAsync();
var waits = new List<TimeSpan>();
var result = await rig.WithPause(Recording(waits)).RefreshStaleCveCacheAsync("test");
Assert.Single(waits);
Assert.Empty(rig.Asked);
Assert.Equal(2, result!.SkippedThrottled);
Assert.Equal(0, result.Refreshed);
Assert.False(result.Completed);
}
[Fact]
public async Task Refresh_CooldownRunning_ButNoRowDue_DoesNotWait()
{
var rig = await Rig.CreateAsync(new[] { "fresh" });
rig.Scan.Cache(Filled("fresh", Now.AddDays(-1)));
await NvdRefusesTheRequestBeforeTheWalkAsync();
var waits = new List<TimeSpan>();
var result = await rig.WithPause(Recording(waits)).RefreshStaleCveCacheAsync("test");
Assert.Empty(waits);
Assert.Equal(0, result!.Stale);
Assert.Empty(rig.Asked);
Assert.True(result.Completed);
}
[Fact]
public async Task Refresh_NoCooldownAsTheWalkStarts_DoesNotWait_AlsoNotWhenNvdRefusesOnTheWay()
{
var rig = await Rig.CreateAsync(new[] { "first", "second", "third" });
rig.Scan.Cache(Pending("first", Now.AddDays(-3)));
rig.Scan.Cache(Pending("second", Now.AddDays(-2)));
rig.Scan.Cache(Pending("third", Now.AddHours(-2)));
rig.Scan.NvdAnswer = request =>
request.Cpe == Product("second") ? NvdFixture.Status(HttpStatusCode.TooManyRequests) : NvdFixture.Page();
var waits = new List<TimeSpan>();
var result = await rig.WithPause(Recording(waits)).RefreshStaleCveCacheAsync("test");
// The one wait is for a cooldown the walk did not cause. A refusal of its own ends it.
Assert.Empty(waits);
Assert.Equal(new[] { "first", "second" }, rig.Asked);
Assert.Equal(1, result!.Refreshed);
Assert.Equal(2, result.SkippedThrottled);
}
[Fact]
public async Task Refresh_AnswerNoMappingNames_StaysForAWeek_OlderAnswersAndMarkersAreRemoved()
{
var rig = await Rig.CreateAsync(new[] { "mapped" });
rig.Scan.Cache(Filled("mapped", Now.AddDays(-1)));
// What the automatic mapper learned about a product it did not map (no CVEs, or CVEs it
// must not map on): the row keeps it from asking about the same product every night.
rig.Scan.Cache(EmptyAnswer("noted_this_week", Now.AddDays(-6)));
rig.Scan.Cache(Filled("looked_at_this_week", Now.AddDays(-1)));
rig.Scan.Cache(EmptyAnswer("noted_before_that", Now.AddDays(-8)));
rig.Scan.Cache(Filled("looked_at_before_that", Now.AddDays(-9)));
rig.Scan.Cache(Pending("waiting", Now.AddHours(-1)));
var result = await rig.Service.RefreshStaleCveCacheAsync("test");
Assert.Equal(
new[] { Key("looked_at_before_that"), Key("noted_before_that"), Key("waiting") },
rig.Scan.CacheDeletes.OrderBy(k => k, StringComparer.Ordinal));
Assert.Equal(3, result!.Removed);
Assert.True(rig.Scan.CacheRows.ContainsKey(Key("noted_this_week")));
Assert.True(rig.Scan.CacheRows.ContainsKey(Key("looked_at_this_week")));
// The rows are kept, not kept current: NVD is not asked about a product no mapping names.
Assert.Equal(0, result.Stale);
Assert.Empty(rig.Asked);
}
// -----------------------------------------------------------------------
// EPSS walk
// -----------------------------------------------------------------------
[Fact]
public async Task EpssRefresh_RewritesChangedRowsWithTheirOwnStampTotalAndShape_AndSkipsTheRest()
{
var current = VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, Product("current"), "2.0");
var alreadyScored = VulnerabilityCacheRows.Cve("CVE-2026-0002", 5.0, Product("current"), "2.0");
alreadyScored.EpssScore = 0.5;
alreadyScored.EpssPercentile = 0.9;
alreadyScored.EpssDate = "2026-11-11";
var unchanged = VulnerabilityCacheRows.Cve("CVE-2026-0004", 5.0, Product("unchanged"), "2.0");
unchanged.EpssScore = 0.25;
unchanged.EpssPercentile = 0.7;
unchanged.EpssDate = "2026-11-11";
var epss = new ReferenceDataFixture.FixedHandler("""
{"status":"OK","data":[
{"cve":"CVE-2026-0001","epss":"0.42","percentile":"0.97","date":"2026-11-11"},
{"cve":"CVE-2026-0002","epss":"0.5","percentile":"0.9","date":"2026-11-11"},
{"cve":"CVE-2026-0003","epss":"0.1","percentile":"0.6","date":"2026-11-11"},
{"cve":"CVE-2026-0004","epss":"0.25","percentile":"0.7","date":"2026-11-11"}
]}
""");
var rig = await Rig.CreateAsync(new[] { "current", "older_shape", "unchanged", "waiting", "empty" }, epss);
var currentCachedAt = Now.AddDays(-3);
var olderCachedAt = Now.AddDays(-5);
rig.Scan.Cache(Filled("current", currentCachedAt, current, alreadyScored));
rig.Scan.Cache(VulnerabilityCacheRows.OlderShape(
Product("older_shape"), new[] { VulnerabilityCacheRows.Cve("CVE-2026-0003", 7.5, Product("older_shape"), "2.0") }, olderCachedAt));
rig.Scan.Cache(Filled("unchanged", Now.AddDays(-1), unchanged));
rig.Scan.Cache(Pending("waiting", Now.AddDays(-2)));
rig.Scan.Cache(Filled("empty", Now.AddDays(-1)));
var result = await rig.Service.RefreshEpssScoresAsync("test");
Assert.Null(result!.Error);
Assert.Equal(3, result.Rows);
Assert.Equal(4, result.Cves);
Assert.Equal(4, result.Scored);
Assert.Equal(2, result.RowsRewritten);
Assert.Equal(new[] { Key("current"), Key("older_shape") }, rig.Scan.CacheWrites.OrderBy(k => k, StringComparer.Ordinal));
Assert.Empty(rig.Scan.Nvd.Requests);
rig.VerifyStamped(Times.Once());
// Scoring is not a fresh NVD fetch: stamp, total and shape are the row's own.
var rewritten = rig.Scan.CacheRows[Key("current")];
Assert.Equal(currentCachedAt.ToString("o"), rewritten["CachedAt"]);
Assert.Equal(7, rewritten["NvdTotal"]);
Assert.Equal(CveCacheFiller.SchemaVersion, rewritten["SchemaVersion"]);
var cves = JsonSerializer.Deserialize<List<CachedCve>>((string)rewritten["CveDataJson"], CveCacheFiller.JsonOptions)!;
Assert.Equal(0.42, cves.Single(c => c.CveId == "CVE-2026-0001").EpssScore);
Assert.Equal("2.0", cves.Single(c => c.CveId == "CVE-2026-0001").AffectedVersions.Single().VersionEndExcluding);
// A row of the older shape stays one, so the NVD walk still knows it is due.
var older = rig.Scan.CacheRows[Key("older_shape")];
Assert.Equal(olderCachedAt.ToString("o"), older["CachedAt"]);
Assert.Equal(1, older["SchemaVersion"]);
Assert.Equal(0, older["NvdTotal"]);
Assert.Equal("pending", rig.Scan.CacheRows[Key("waiting")]["State"]);
}
}
@@ -0,0 +1,333 @@
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Analyze;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Azure.Data.Tables;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using static AutopilotMonitor.Functions.Tests.ReportFixture;
using static AutopilotMonitor.Functions.Tests.VulnerabilityScanRig;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="VulnerabilityCorrelateHandler"/> driven through a real correlation over mocked
/// storage. Pinned: a session waits in the queue when the mappings cannot be read or CVE data is
/// missing on its first run, instead of storing a report that calls unjudged products clean; a
/// result that changes or disappears takes the report row and the CveIndex rows with it; and
/// nothing is removed that belongs to the other WhiteGlove phase.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class VulnerabilityCorrelateHandlerFlowTests : IDisposable
{
public VulnerabilityCorrelateHandlerFlowTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private sealed class Rig
{
private Rig(VulnerabilityScanRig scan)
{
Scan = scan;
Result = new SessionResultRecorder(scan.Repo);
Result.Sessions.Setup(s => s.GetSessionEventsAsync(TenantId, SessionId, It.IsAny<int>())).ReturnsAsync(new List<EnrollmentEvent>());
Result.Sessions.Setup(s => s.StoreEventsBatchAsync(It.IsAny<List<EnrollmentEvent>>()))
.Returns<List<EnrollmentEvent>>(events =>
{
Result.Steps.Add("store event");
Events.AddRange(events);
return Task.FromResult(events);
});
var requeue = new Mock<IVulnerabilityCorrelateProducer>();
requeue.Setup(p => p.EnqueueDelayedAsync(It.IsAny<VulnerabilityCorrelateEnvelope>(), It.IsAny<TimeSpan>(), It.IsAny<CancellationToken>()))
.Returns<VulnerabilityCorrelateEnvelope, TimeSpan, CancellationToken>((envelope, delay, _) =>
{
Requeued.Add((envelope, delay, envelope.Deferrals, envelope.PendingRuns));
return Task.CompletedTask;
});
var analyze = new Mock<IAnalyzeOnEnrollmentEndProducer>();
analyze.Setup(p => p.EnqueueAsync(It.IsAny<AnalyzeOnEnrollmentEndEnvelope>(), It.IsAny<CancellationToken>()))
.Returns<AnalyzeOnEnrollmentEndEnvelope, CancellationToken>((envelope, _) =>
{
AnalyzeRequests.Add(envelope);
return Task.CompletedTask;
});
var inventoryLoader = new Mock<IVulnerabilityInventoryLoader>();
inventoryLoader.Setup(l => l.LoadShutdownInventoryAsync(TenantId, SessionId, It.IsAny<int?>(), false, It.IsAny<CancellationToken>()))
.Returns(() => Task.FromResult<IReadOnlyList<Dictionary<string, object>>>(Inventory.ToList()));
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions()));
adminConfig.Setup(a => a.GetConfigurationAsync())
.ReturnsAsync(new AdminConfiguration { VulnerabilityCorrelationEnabled = true });
var sessionRows = new Mock<ISessionDeletionInventoryReader>();
sessionRows.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new TableEntity(TenantId, SessionId));
Handler = new VulnerabilityCorrelateHandler(
adminConfig.Object,
scan.Service,
scan.Repo.Object,
Result.Sessions.Object,
inventoryLoader.Object,
analyze.Object,
requeue.Object,
Result.Writer,
// The SignalR push sits in its own catch: without a hub the handler logs and goes on.
signalRNotification: null!,
new SessionDeletionGuard(sessionRows.Object, NullLogger<SessionDeletionGuard>.Instance),
NullLogger<VulnerabilityCorrelateHandler>.Instance);
}
public static async Task<Rig> CreateAsync() => new(await VulnerabilityScanRig.CreateAsync());
public VulnerabilityScanRig Scan { get; }
public SessionResultRecorder Result { get; }
public VulnerabilityCorrelateHandler Handler { get; }
public List<Dictionary<string, object>> Inventory { get; } = new();
public List<EnrollmentEvent> Events { get; } = new();
public List<AnalyzeOnEnrollmentEndEnvelope> AnalyzeRequests { get; } = new();
public List<(VulnerabilityCorrelateEnvelope Envelope, TimeSpan Delay, int Deferrals, int PendingRuns)> Requeued { get; } = new();
}
private static VulnerabilityCorrelateEnvelope Envelope(int? whiteGlovePart = null) => new()
{
TenantId = TenantId,
SessionId = SessionId,
WhiteGlovePart = whiteGlovePart,
Reason = VulnerabilityCorrelateHandler.ReasonShutdownInventory,
EnqueuedAt = DateTime.UtcNow,
};
private static Dictionary<string, object> Chrome(string version) => Installed("google", "Google Chrome", version);
private static Dictionary<string, object> Widget() => Installed("contoso", "Contoso Widget", "3.2.0");
private static List<Dictionary<string, object>> Entries(Dictionary<string, object> report, string key) =>
(List<Dictionary<string, object>>)report[key];
// -----------------------------------------------------------------------
// Mappings cannot be read
// -----------------------------------------------------------------------
[Fact]
public async Task MappingsUnreadable_PutsTheSessionBackAndWritesNothing()
{
var rig = await Rig.CreateAsync();
rig.Scan.MappingFailure = new InvalidOperationException("storage down");
rig.Inventory.Add(Chrome("120.0.6099.130"));
var envelope = Envelope();
envelope.Deferrals = 3;
await rig.Handler.HandleAsync(envelope);
var requeued = Assert.Single(rig.Requeued);
Assert.Same(envelope, requeued.Envelope);
Assert.Equal(VulnerabilityCorrelateHandler.MappingPause, requeued.Delay);
Assert.Equal(4, requeued.Deferrals);
Assert.Equal(0, requeued.PendingRuns);
Assert.Empty(rig.Result.Steps);
Assert.Empty(rig.Scan.InventoryWrites);
Assert.Empty(rig.AnalyzeRequests);
}
[Fact]
public async Task MappingsUnreadable_AfterTheLastPostponement_FailsTheMessage()
{
var rig = await Rig.CreateAsync();
rig.Scan.MappingFailure = new InvalidOperationException("storage down");
rig.Inventory.Add(Chrome("120.0.6099.130"));
var envelope = Envelope();
envelope.Deferrals = VulnerabilityCorrelateHandler.MaxDeferrals;
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.Handler.HandleAsync(envelope));
Assert.Empty(rig.Requeued);
Assert.Empty(rig.Result.Steps);
}
// -----------------------------------------------------------------------
// CVE data missing
// -----------------------------------------------------------------------
[Fact]
public async Task ProductWithoutCveData_OnTheFirstRun_PutsTheSessionBackAndStoresNothing()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001");
rig.Inventory.AddRange(new[] { Chrome("120.0.6099.130"), Widget() });
await rig.Handler.HandleAsync(Envelope());
var requeued = Assert.Single(rig.Requeued);
Assert.Equal(VulnerabilityCorrelateHandler.PendingDelay, requeued.Delay);
Assert.Equal(1, requeued.PendingRuns);
Assert.Equal(0, requeued.Deferrals);
// Not even the Chrome finding: one report and one timeline entry per session, not two.
Assert.Empty(rig.Result.Steps);
Assert.Empty(rig.AnalyzeRequests);
rig.Scan.Repo.Verify(r => r.GetVulnerabilityReportAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
}
[Fact]
public async Task ProductWithoutCveData_OnTheLaterRun_StoresTheReportThatNamesIt()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001");
rig.Inventory.AddRange(new[] { Chrome("120.0.6099.130"), Widget() });
var envelope = Envelope();
envelope.PendingRuns = VulnerabilityCorrelateHandler.MaxPendingRuns;
await rig.Handler.HandleAsync(envelope);
Assert.Empty(rig.Requeued);
Assert.Equal(new[] { "store report", "replace index", "store event" }, rig.Result.Steps);
var report = Assert.Single(rig.Result.Stored);
var notEvaluated = Assert.Single(Entries(report, "not_evaluated"));
Assert.Equal("Contoso Widget", notEvaluated["softwareName"]);
Assert.Equal("cve_data_pending", notEvaluated["reason"]);
Assert.Equal(new[] { "CVE-2026-0001" }, CveIds(Entries(report, "findings")));
}
[Fact]
public async Task OnlyProductsWithoutCveData_OnTheLaterRun_StillStoreAReport()
{
var rig = await Rig.CreateAsync();
rig.Inventory.Add(Widget());
var envelope = Envelope();
envelope.PendingRuns = 1;
await rig.Handler.HandleAsync(envelope);
var report = Assert.Single(rig.Result.Stored);
Assert.Empty(Entries(report, "findings"));
Assert.Single(Entries(report, "not_evaluated"));
// No findings, so the index has nothing to write and, without an earlier report, nothing to remove.
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Empty(index.Findings!);
Assert.Null(index.Previous);
}
// -----------------------------------------------------------------------
// Nothing found
// -----------------------------------------------------------------------
[Theory]
[InlineData(null, null, true)]
[InlineData(1, "device_setup", true)]
[InlineData(1, "user_enrollment", false)] // the stored report holds the other phase
[InlineData(1, "both", false)]
[InlineData(2, "device_setup", false)]
[InlineData(2, "user_enrollment", false)]
public async Task NothingFound_RemovesAnEarlierReportOfTheSameScope_AndLeavesTheOtherPhaseAlone(
int? whiteGlovePart, string? storedPhase, bool removed)
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001", "CVE-2026-0002");
rig.Inventory.Add(Chrome("121.0.6167.85")); // updated since: nothing applies any more
rig.Result.StoredReport = StoredChromeReport(storedPhase, "CVE-2026-0001", "CVE-2026-0002");
await rig.Handler.HandleAsync(Envelope(whiteGlovePart));
Assert.Empty(rig.Result.Stored);
Assert.Empty(rig.Events);
Assert.Empty(rig.AnalyzeRequests);
Assert.Empty(rig.Requeued);
if (!removed)
{
Assert.Empty(rig.Result.Steps);
return;
}
Assert.Equal(new[] { "delete report", "replace index" }, rig.Result.Steps);
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Null(index.Findings);
// The removed report's findings tell which index rows exist.
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
}
[Fact]
public async Task NothingFound_AndNoStoredReport_WritesNothing()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001");
rig.Inventory.Add(Chrome("121.0.6167.85"));
await rig.Handler.HandleAsync(Envelope());
Assert.Empty(rig.Result.Steps);
Assert.Single(rig.Scan.InventoryWrites);
}
// -----------------------------------------------------------------------
// Findings
// -----------------------------------------------------------------------
[Fact]
public async Task Findings_StoreTheReport_ThenBringTheIndexInLineWithThePreviousOne()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001", "CVE-2026-0003");
rig.Inventory.Add(Chrome("120.0.6099.130"));
rig.Result.StoredReport = StoredChromeReport(phase: null, "CVE-2026-0001", "CVE-2026-0002");
await rig.Handler.HandleAsync(Envelope());
Assert.Equal(new[] { "store report", "replace index", "store event" }, rig.Result.Steps);
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0003" }, CveIds(index.Findings!));
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
var finding = Assert.Single(Entries(Assert.Single(rig.Result.Stored), "findings"));
Assert.Equal("120.0.6099.130", finding["installedVersion"]);
Assert.False(finding.ContainsKey("phase"));
Assert.Equal(Constants.EventTypes.VulnerabilityReport, Assert.Single(rig.Events).EventType);
Assert.Equal(AnalyzeOnEnrollmentEndHandler.ReasonVulnerabilityCorrelated, Assert.Single(rig.AnalyzeRequests).Reason);
}
[Fact]
public async Task Findings_WithoutAnEarlierReport_HaveNoRowsToCompareWith()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001");
rig.Inventory.Add(Chrome("120.0.6099.130"));
await rig.Handler.HandleAsync(Envelope());
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Equal(new[] { "CVE-2026-0001" }, CveIds(index.Findings!));
Assert.Null(index.Previous);
}
[Fact]
public async Task WhiteGlovePart2_MergesIntoTheStoredPart1Report_AndTheIndexFollowsTheMergedReport()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001", "CVE-2026-0002");
rig.Inventory.Add(Chrome("120.0.6099.130"));
rig.Result.StoredReport = StoredChromeReport("device_setup", "CVE-2026-0001", "CVE-2026-0002"); // Chrome 119 at pre-provisioning
await rig.Handler.HandleAsync(Envelope(whiteGlovePart: 2));
var report = Assert.Single(rig.Result.Stored);
Assert.Equal(
new[] { ("119.0.6045.200", "device_setup"), ("120.0.6099.130", "user_enrollment") },
Entries(report, "findings").Select(f => ((string)f["installedVersion"], (string)f["phase"])).OrderBy(f => f.Item1));
Assert.Equal(4, ((Dictionary<string, object>)report["scan_summary"])["total_cves_found"]);
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Equal(2, index.Findings!.Count);
Assert.Single(index.Previous!);
}
}
@@ -0,0 +1,404 @@
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Services.Vulnerability;
using static AutopilotMonitor.Functions.Tests.VulnerabilityScanRig;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// One run of <see cref="VulnerabilityCorrelationService.CorrelateAsync"/> over a real mapping
/// store and cache filler. Pinned: every program is filed under its product before the first
/// lookup, CVE data is read once per product, a product without data is reported as not
/// evaluated (never as clean) and noted for a later fill, and software that updates itself is
/// listed without counting as urgent.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class VulnerabilityCorrelationFlowTests : IDisposable
{
private static readonly DateTime CachedAt = DateTime.UtcNow.AddDays(-1);
public VulnerabilityCorrelationFlowTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private static Dictionary<string, object> Chrome(string version = "120.0.6099.130") => Installed("google", "Google Chrome", version);
private static Dictionary<string, object> SevenZip(string version = "24.08") => Installed("igor pavlov", $"7-Zip {version} (x64)", version);
private static Dictionary<string, object> Widget(string version = "3.2.0", string registrySource = "HKLM_64") =>
Installed("contoso", "Contoso Widget", version, registrySource);
private static List<Dictionary<string, object>> Entries(Dictionary<string, object> report, string key) =>
(List<Dictionary<string, object>>)report[key];
private static Dictionary<string, object> Summary(Dictionary<string, object> report) =>
(Dictionary<string, object>)report["scan_summary"];
private static IEnumerable<string> Names(Dictionary<string, object> report, string key) =>
Entries(report, key).Select(e => (string)e["softwareName"]);
// -----------------------------------------------------------------------
// Mapping
// -----------------------------------------------------------------------
[Fact]
public async Task EveryProgramIsFiledUnderItsProduct_EvenWhenTheRunIsAlreadyOutOfTime()
{
var rig = await CreateAsync();
using var outOfTime = new CancellationTokenSource();
outOfTime.Cancel();
var outcome = await rig.CorrelateAsync(
new[] { Chrome(), Installed("fabrikam", "Fabrikam Tool", "1.0"), SevenZip() }, outOfTime.Token);
var report = outcome.Report!;
var unmatched = Assert.Single(Entries(report, "unmatched_software"));
Assert.Equal("Fabrikam Tool", unmatched["softwareName"]);
Assert.Equal("no_cpe_match", unmatched["reason"]);
Assert.Equal(3, Summary(report)["total_software_scanned"]);
Assert.Equal(2, Summary(report)["matched_to_cpe"]);
Assert.Equal(1, Summary(report)["unmatched"]);
// The inventory rows take the computed product, set or cleared, on this run as on any other.
var written = Assert.Single(rig.InventoryWrites);
Assert.Equal(
new[] { ("Google Chrome", ChromeCpe), ("Fabrikam Tool", ""), ("7-Zip 24.08 (x64)", SevenZipCpe) },
written.Select(i => (i.DisplayName, i.CpeUri ?? "<null>")));
}
[Fact]
public async Task MappingsUnreadable_ThrowsBeforeAnythingIsWritten()
{
var rig = await CreateAsync();
rig.MappingFailure = new InvalidOperationException("storage down");
await Assert.ThrowsAsync<CpeMappingUnavailableException>(() => rig.CorrelateAsync(new[] { Chrome() }));
Assert.Empty(rig.InventoryWrites);
Assert.Empty(rig.CacheReads);
}
// -----------------------------------------------------------------------
// CVE data per product
// -----------------------------------------------------------------------
[Fact]
public async Task SeveralProgramsOfOneProduct_ReadItsCveDataOnce()
{
var rig = await CreateAsync();
rig.Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, ChromeCpe, "121.0") }, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Chrome("120.0.6099.130"), Chrome("119.0.6045.200"), Chrome("121.0.6167.85") });
Assert.Equal(new[] { CveCacheFiller.CacheKey(ChromeCpe) }, rig.CacheReads);
Assert.Empty(rig.Nvd.Requests);
Assert.Equal(2, Entries(outcome.Report!, "findings").Count);
Assert.Equal("121.0.6167.85", Assert.Single(Entries(outcome.Report!, "matched_clean_software"))["installedVersion"]);
Assert.Empty(outcome.PendingProducts);
}
[Theory]
[InlineData(HttpStatusCode.InternalServerError)]
[InlineData(HttpStatusCode.TooManyRequests)]
public async Task ProductWithoutData_AndNvdNotAnswering_IsNotedAndReportedAsNotEvaluated(HttpStatusCode nvdStatus)
{
var rig = await CreateAsync();
rig.NvdAnswer = _ => NvdFixture.Status(nvdStatus);
var outcome = await rig.CorrelateAsync(new[] { Widget("3.2.0"), Widget("3.1.0") });
Assert.Equal(new[] { WidgetCpe }, outcome.PendingProducts);
Assert.Single(rig.CacheReads);
Assert.Single(rig.Nvd.Requests);
Assert.Empty(rig.CacheWrites);
var marker = Assert.Single(rig.PendingMarkers);
Assert.Equal(CveCacheFiller.CacheKey(WidgetCpe), marker["PartitionKey"]);
Assert.Equal(CveCacheFiller.RowKey, marker["RowKey"]);
Assert.Equal("pending", marker["State"]);
Assert.Equal(WidgetCpe, marker["CpeUri"]);
// A report although nothing was found: "clean" would be a claim nobody checked.
var report = outcome.Report!;
Assert.Empty(Entries(report, "findings"));
Assert.Empty(Entries(report, "matched_clean_software"));
var notEvaluated = Entries(report, "not_evaluated");
Assert.Equal(new[] { "3.2.0", "3.1.0" }, notEvaluated.Select(n => (string)n["installedVersion"]));
Assert.All(notEvaluated, n =>
{
Assert.Equal("cve_data_pending", n["reason"]);
Assert.Equal(WidgetCpe, n["cpeUri"]);
});
Assert.Equal(2, Summary(report)["not_evaluated"]);
}
[Fact]
public async Task NvdRefusing_IsNotAskedAgainForTheNextProductOfTheRun()
{
var rig = await CreateAsync();
rig.NvdAnswer = _ => NvdFixture.Status(HttpStatusCode.TooManyRequests);
var outcome = await rig.CorrelateAsync(new[] { Widget(), Chrome() });
Assert.Equal(WidgetCpe, Assert.Single(rig.Nvd.Requests).Cpe);
Assert.Equal(new[] { WidgetCpe, ChromeCpe }, outcome.PendingProducts);
Assert.Equal(new[] { WidgetCpe, ChromeCpe }, rig.PendingMarkers.Select(m => (string)m["CpeUri"]));
}
[Fact]
public async Task RunOutOfTime_StillJudgesAProductWhoseDataIsCached()
{
var rig = await CreateAsync();
rig.ChromeHas("CVE-2026-1001");
using var outOfTime = new CancellationTokenSource();
outOfTime.Cancel();
var outcome = await rig.CorrelateAsync(new[] { Chrome() }, outOfTime.Token);
// Only a fetch needs time. Cached data is read, so the product is judged, not reported as waiting.
Assert.Empty(outcome.PendingProducts);
Assert.Empty(rig.PendingMarkers);
Assert.Empty(rig.Nvd.Requests);
Assert.Equal("Google Chrome", Assert.Single(Entries(outcome.Report!, "findings"))["softwareName"]);
Assert.Empty(Entries(outcome.Report!, "not_evaluated"));
}
[Fact]
public async Task RunOutOfTime_NotesAProductWithoutDataInsteadOfFetchingIt()
{
var rig = await CreateAsync();
using var outOfTime = new CancellationTokenSource();
outOfTime.Cancel();
var outcome = await rig.CorrelateAsync(new[] { Widget() }, outOfTime.Token);
Assert.Empty(rig.Nvd.Requests);
Assert.Equal(new[] { WidgetCpe }, outcome.PendingProducts);
Assert.Equal(WidgetCpe, Assert.Single(rig.PendingMarkers)["CpeUri"]);
Assert.Equal("cve_data_pending", Assert.Single(Entries(outcome.Report!, "not_evaluated"))["reason"]);
}
[Fact]
public async Task RunOutOfTimeDuringAFetch_StillReturnsItsReport_AndNotesTheProduct()
{
var rig = await CreateAsync();
using var runTime = new CancellationTokenSource();
rig.NvdAnswer = _ =>
{
runTime.Cancel();
throw new TaskCanceledException("the run's time is up");
};
var outcome = await rig.CorrelateAsync(new[] { Widget() }, runTime.Token);
Assert.Equal(new[] { WidgetCpe }, outcome.PendingProducts);
Assert.Equal(WidgetCpe, Assert.Single(rig.PendingMarkers)["CpeUri"]);
Assert.Equal("cve_data_pending", Assert.Single(Entries(outcome.Report!, "not_evaluated"))["reason"]);
Assert.Single(rig.InventoryWrites);
}
[Fact]
public async Task ProductWithoutData_IsFetchedDuringTheRun_AndServedFromTheCacheAfterwards()
{
var rig = await CreateAsync();
rig.NvdAnswer = _ => NvdFixture.Page(NvdFixture.Cve("CVE-2026-0002", 7.8, NvdFixture.Below(WidgetCpe, "4.0")));
var first = await rig.CorrelateAsync(new[] { Widget() });
var second = await rig.CorrelateAsync(new[] { Widget() });
Assert.Empty(first.PendingProducts);
Assert.Empty(rig.PendingMarkers);
Assert.Equal(new[] { CveCacheFiller.CacheKey(WidgetCpe) }, rig.CacheWrites);
Assert.Single(rig.Nvd.Requests);
foreach (var outcome in new[] { first, second })
{
var finding = Assert.Single(Entries(outcome.Report!, "findings"));
Assert.Equal("Contoso Widget", finding["softwareName"]);
Assert.Equal(1, finding["totalCves"]);
}
}
// -----------------------------------------------------------------------
// Programs that are not judged
// -----------------------------------------------------------------------
[Theory]
[InlineData("1.0.0.0")]
[InlineData("1.0.0.1")]
public async Task StoreAppStubWithAPlaceholderVersion_IsNotEvaluated_AndCostsNoLookup(string placeholder)
{
var rig = await CreateAsync();
// The placeholder lies below every "fixed in" version: judged, it would report this CVE.
rig.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0003", 9.8, WidgetCpe, "5.0") }, CachedAt));
rig.Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, ChromeCpe, "121.0") }, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Widget(placeholder, registrySource: "AppX"), Chrome() });
var report = outcome.Report!;
Assert.Equal(new[] { "Google Chrome" }, Names(report, "findings"));
var skipped = Assert.Single(Entries(report, "not_evaluated"));
Assert.Equal("Contoso Widget", skipped["softwareName"]);
Assert.Equal("placeholder_version", skipped["reason"]);
Assert.Equal(WidgetCpe, skipped["cpeUri"]);
Assert.Equal(new[] { CveCacheFiller.CacheKey(ChromeCpe) }, rig.CacheReads);
Assert.Empty(Entries(report, "unmatched_software"));
Assert.Equal(2, Summary(report)["matched_to_cpe"]);
}
[Fact]
public async Task NothingFoundAndNothingLeftOpen_GivesNoReport()
{
var rig = await CreateAsync();
rig.Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, ChromeCpe, "100.0") }, CachedAt));
// A stub that is skipped for its version alone does not make a report either.
var outcome = await rig.CorrelateAsync(new[]
{
Widget("1.0.0.0", registrySource: "AppX"), Chrome(), Installed("fabrikam", "Fabrikam Tool", "1.0"),
});
Assert.Null(outcome.Report);
Assert.Empty(outcome.PendingProducts);
Assert.Equal(new[] { CveCacheFiller.CacheKey(ChromeCpe) }, rig.CacheReads);
Assert.Single(rig.InventoryWrites);
}
[Fact]
public async Task ProgramCountingVersionsInAnotherScheme_IsNotEvaluated_NotClean()
{
var rig = await CreateAsync();
rig.Cache(VulnerabilityCacheRows.Filled(
WidgetCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0004", 7.8, WidgetCpe, "2022.30050") }, CachedAt));
rig.Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, ChromeCpe, "121.0") }, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Widget("26.198"), Chrome() });
var report = outcome.Report!;
Assert.Equal("version_scheme", Assert.Single(Entries(report, "not_evaluated"))["reason"]);
Assert.Empty(Entries(report, "matched_clean_software"));
Assert.Equal(new[] { "Google Chrome" }, Names(report, "findings"));
Assert.Empty(outcome.PendingProducts);
}
[Fact]
public async Task EditionOnTheMapping_DecidesWhichRangesJudgeTheProgram()
{
const string firefoxCpe = "cpe:2.3:a:mozilla:firefox";
var rig = await CreateAsync();
var firefox = VulnerabilityCacheRows.Mapping("mozilla", "mozilla firefox", firefoxCpe, "mozilla firefox");
firefox["ExcludeEditionsJson"] = JsonSerializer.Serialize(new[] { "esr" });
var firefoxEsr = VulnerabilityCacheRows.Mapping("mozilla", "mozilla firefox esr", firefoxCpe, "mozilla firefox esr");
firefoxEsr["Edition"] = "ESR";
rig.Mappings.Add(firefox);
rig.Mappings.Add(firefoxEsr);
// Shaped like the public CVE-2023-4863: fixed in 117.0.1, and in 115.2.1 on the ESR channel.
rig.Cache(VulnerabilityCacheRows.Filled(firefoxCpe, new[]
{
new CachedCve
{
CveId = "CVE-2023-4863",
CvssScore = 8.8,
CvssSeverity = "HIGH",
Description = "Heap buffer overflow in libwebp.",
PublishedDate = "2023-09-12T15:15:24.327",
AffectedVersions =
{
new CachedVersionRange { Criteria = "cpe:2.3:a:mozilla:firefox:*:*:*:*:-:*:*:*", VersionEndExcluding = "117.0.1" },
new CachedVersionRange { Criteria = "cpe:2.3:a:mozilla:firefox:*:*:*:*:esr:*:*:*", VersionEndExcluding = "115.2.1" },
},
},
}, CachedAt));
var outcome = await rig.CorrelateAsync(new[]
{
Installed("mozilla", "Mozilla Firefox (x64 en-US)", "116.0"),
Installed("mozilla", "Mozilla Firefox ESR (x64 en-US)", "115.3.0"),
});
var report = outcome.Report!;
Assert.Equal(new[] { "Mozilla Firefox (x64 en-US)" }, Names(report, "findings"));
Assert.Equal(new[] { "Mozilla Firefox ESR (x64 en-US)" }, Names(report, "matched_clean_software"));
// Both mappings name the same product, so its data is read once.
Assert.Single(rig.CacheReads);
}
// -----------------------------------------------------------------------
// Self-updating software and the summary
// -----------------------------------------------------------------------
[Fact]
public async Task SelfUpdatingSoftware_IsListedLast_AndLeftOutOfTheActionableCounts()
{
var rig = await CreateAsync("CVE-2026-0001", "CVE-2026-0005");
rig.ProductFlags.Add(VulnerabilityCacheRows.SelfUpdating("google:chrome"));
rig.Cache(VulnerabilityCacheRows.Filled(ChromeCpe, new[]
{
VulnerabilityCacheRows.Cve("CVE-2026-0001", 9.6, ChromeCpe, "121.0"), // critical, known exploited
VulnerabilityCacheRows.Cve("CVE-2026-0002", 8.8, ChromeCpe, "121.0"),
}, CachedAt));
rig.Cache(VulnerabilityCacheRows.Filled(SevenZipCpe, new[]
{
VulnerabilityCacheRows.Cve("CVE-2026-0003", 9.8, SevenZipCpe, "25.0"),
VulnerabilityCacheRows.Cve("CVE-2026-0004", 5.5, SevenZipCpe, "25.0"),
}, CachedAt));
rig.Cache(VulnerabilityCacheRows.Filled(WidgetCpe, new[]
{
VulnerabilityCacheRows.Cve("CVE-2026-0005", 7.8, WidgetCpe, "4.0"), // high, known exploited
}, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Chrome(), SevenZip(), Widget() });
var report = outcome.Report!;
var findings = Entries(report, "findings");
// By hand first (the exploited one ahead on a tie of risk), self-updating last whatever its risk.
Assert.Equal(new[] { "Contoso Widget", "7-Zip 24.08 (x64)", "Google Chrome" }, findings.Select(f => (string)f["softwareName"]));
Assert.True(findings[2]["selfUpdating"] is true);
Assert.False(findings[0].ContainsKey("selfUpdating"));
Assert.False(findings[1].ContainsKey("selfUpdating"));
var summary = Summary(report);
Assert.Equal(5, summary["total_cves_found"]);
Assert.Equal(2, summary["critical_cves"]);
Assert.Equal(2, summary["high_cves"]);
Assert.Equal(1, summary["medium_cves"]);
Assert.Equal(2, summary["kev_matches"]);
Assert.Equal(1, summary["actionable_critical_cves"]);
Assert.Equal(1, summary["actionable_kev_matches"]);
// The header of the report counts by hand only, so it needs the whole breakdown of that part.
Assert.Equal(3, summary["actionable_cves"]);
Assert.Equal(2, summary["actionable_packages"]);
Assert.Equal(1, summary["actionable_high_cves"]);
Assert.Equal(1, summary["actionable_medium_cves"]);
Assert.Equal(0, summary["actionable_low_cves"]);
Assert.Equal(2, summary["self_updating_cves"]);
Assert.Equal(1, summary["self_updating_packages"]);
Assert.Equal("critical", summary["overall_risk"]);
var exploited = ((List<Dictionary<string, object>>)findings[2]["vulnerabilities"])[0];
Assert.Equal("CVE-2026-0001", exploited["cveId"]);
Assert.True(exploited["isKev"] is true);
Assert.Equal(CvePriority.Act, exploited["priority"]);
Assert.Equal("2026-10-22", exploited["kevDueDate"]);
}
[Fact]
public async Task MoreEntriesThanAReportLists_AreCountedInTheSummary()
{
var rig = await CreateAsync();
rig.Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-0001", 8.8, ChromeCpe, "200.0") }, CachedAt));
var outcome = await rig.CorrelateAsync(Enumerable.Range(100, 55).Select(major => Chrome($"{major}.0.1.2")));
var report = outcome.Report!;
Assert.Equal(VulnerabilityCorrelationService.MaxFindings, Entries(report, "findings").Count);
Assert.Equal(5, Summary(report)["additional_findings_count"]);
Assert.Equal(55, Summary(report)["total_cves_found"]);
Assert.Single(rig.CacheReads);
}
}
@@ -0,0 +1,206 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using static AutopilotMonitor.Functions.Tests.VulnerabilityScanRig;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// Which mapped programs a run of <see cref="VulnerabilityCorrelationService.CorrelateAsync"/>
/// judges at all, and what an entry says about its worst CVE. A program without a version number
/// is not compared with any range, and a Store package is not judged by a mapping nobody chose
/// for it; both are listed as not evaluated, never as clean. An entry lists only its most urgent
/// CVEs, so it carries the highest score of all of them itself.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class VulnerabilityCorrelationJudgingTests : IDisposable
{
private const string ToolCpe = "cpe:2.3:a:northwind:tool";
private static readonly DateTime CachedAt = DateTime.UtcNow.AddDays(-1);
public VulnerabilityCorrelationJudgingTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
/// <summary>A rig in which Chrome 120 has a finding, so a run has a report whatever becomes of the program a test looks at.</summary>
private static async Task<VulnerabilityScanRig> RigAsync()
{
var rig = await CreateAsync();
rig.ChromeHas("CVE-2026-11001");
return rig;
}
private static Dictionary<string, object> Chrome() => Installed("google", "Google Chrome", "120.0.6099.130");
private static Dictionary<string, object> Tool(string version, string registrySource = "HKLM_64") =>
Installed("northwind", "Northwind Tool", version, registrySource);
/// <summary>Maps the tool by its vendor and name, in the partition and with the source a test names.</summary>
private static Dictionary<string, object> ToolMapping(string partition, string source) =>
VulnerabilityCacheRows.MappingIn(partition, source, "northwind", "northwind tool", ToolCpe);
/// <summary>The tool has one CVE that every version below 50 carries: a package version of 1.0.3 lies far below it.</summary>
private static void ToolHasACve(VulnerabilityScanRig rig) => rig.Cache(VulnerabilityCacheRows.Filled(
ToolCpe, new[] { VulnerabilityCacheRows.Cve("CVE-2026-11006", 9.8, ToolCpe, "50.0") }, CachedAt));
private static List<Dictionary<string, object>> Entries(Dictionary<string, object> report, string key) =>
(List<Dictionary<string, object>>)report[key];
private static Dictionary<string, object> Summary(Dictionary<string, object> report) =>
(Dictionary<string, object>)report["scan_summary"];
private static IEnumerable<string> Names(Dictionary<string, object> report, string key) =>
Entries(report, key).Select(e => (string)e["softwareName"]);
// -----------------------------------------------------------------------
// A program without a version number
// -----------------------------------------------------------------------
[Theory]
[InlineData("")]
[InlineData(" ")]
[InlineData("Base Version")]
public async Task MappedProgramWithoutAVersionNumber_IsNotEvaluated_WhateverCvesItsProductHas(string version)
{
var rig = await RigAsync();
// Read as a number such a version is zero, below every "fixed in" version there is.
rig.Cache(VulnerabilityCacheRows.Filled(WidgetCpe, new[]
{
VulnerabilityCacheRows.Cve("CVE-2026-11003", 9.8, WidgetCpe, "5.0"),
VulnerabilityCacheRows.CveOfEveryVersion("CVE-2026-11004", 7.5, WidgetCpe),
}, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Installed("contoso", "Contoso Widget", version), Chrome() });
var report = outcome.Report!;
Assert.Equal(new[] { "Google Chrome" }, Names(report, "findings"));
var skipped = Assert.Single(Entries(report, "not_evaluated"));
Assert.Equal("Contoso Widget", skipped["softwareName"]);
Assert.Equal("no_version", skipped["reason"]);
Assert.Equal(WidgetCpe, skipped["cpeUri"]);
Assert.Equal(version, skipped["installedVersion"]);
// Mapped, and neither clean nor waiting for data.
Assert.Empty(Entries(report, "matched_clean_software"));
Assert.Empty(Entries(report, "unmatched_software"));
Assert.Empty(outcome.PendingProducts);
Assert.Equal(2, Summary(report)["matched_to_cpe"]);
Assert.Equal(1, Summary(report)["not_evaluated"]);
Assert.Equal(1, Summary(report)["total_cves_found"]);
// The product's CVE data is not even looked up for it.
Assert.Equal(new[] { CveCacheFiller.CacheKey(ChromeCpe) }, rig.CacheReads);
Assert.Empty(rig.Nvd.Requests);
}
[Fact]
public async Task ProgramWithoutAVersionNumber_OnItsOwn_MakesNoReport()
{
var rig = await CreateAsync();
var outcome = await rig.CorrelateAsync(new[] { Installed("contoso", "Contoso Widget", "Base Version") });
Assert.Null(outcome.Report);
Assert.Empty(outcome.PendingProducts);
Assert.Empty(rig.CacheReads);
Assert.Empty(rig.PendingMarkers);
// Its inventory row still takes the product it was filed under.
Assert.Equal(WidgetCpe, Assert.Single(Assert.Single(rig.InventoryWrites)).CpeUri);
}
// -----------------------------------------------------------------------
// A Store package under an automatic mapping
// -----------------------------------------------------------------------
[Theory]
[InlineData(CpeMappingStore.AutoPartition, "auto")]
// The retired name search filed its mappings among the operator's own.
[InlineData(CpeMappingStore.CustomPartition, "auto")]
public async Task StorePackageUnderAnAutomaticMapping_IsNotJudged(string partition, string source)
{
var rig = await RigAsync();
rig.Mappings.Add(ToolMapping(partition, source));
ToolHasACve(rig);
var outcome = await rig.CorrelateAsync(new[] { Tool("1.0.3", registrySource: "AppX"), Chrome() });
var report = outcome.Report!;
Assert.Equal(new[] { "Google Chrome" }, Names(report, "findings"));
var skipped = Assert.Single(Entries(report, "not_evaluated"));
Assert.Equal("Northwind Tool", skipped["softwareName"]);
Assert.Equal("version_scheme", skipped["reason"]);
Assert.Equal(ToolCpe, skipped["cpeUri"]);
Assert.Equal("1.0.3", skipped["installedVersion"]);
Assert.Empty(Entries(report, "matched_clean_software"));
Assert.Empty(outcome.PendingProducts);
Assert.DoesNotContain(CveCacheFiller.CacheKey(ToolCpe), rig.CacheReads);
}
[Theory]
[InlineData(CpeMappingStore.SeedPartition, "seed")]
[InlineData(CpeMappingStore.CommunityPartition, "community")]
[InlineData(CpeMappingStore.CustomPartition, "manual")]
public async Task StorePackageUnderAMappingSomeoneChoseForIt_IsJudgedByItsPackageVersion(string partition, string source)
{
var rig = await RigAsync();
rig.Mappings.Add(ToolMapping(partition, source));
ToolHasACve(rig);
var outcome = await rig.CorrelateAsync(new[] { Tool("1.0.3", registrySource: "AppX"), Chrome() });
var report = outcome.Report!;
Assert.Equal(new[] { "Google Chrome", "Northwind Tool" }, Names(report, "findings").OrderBy(n => n, StringComparer.Ordinal));
Assert.Empty(Entries(report, "not_evaluated"));
}
[Fact]
public async Task ClassicInstallUnderAnAutomaticMapping_IsJudged_NextToThePackageThatIsNot()
{
var rig = await CreateAsync();
rig.Mappings.Add(ToolMapping(CpeMappingStore.AutoPartition, "auto"));
ToolHasACve(rig);
var outcome = await rig.CorrelateAsync(new[] { Tool("1.0.3", registrySource: "AppX"), Tool("3.1") });
var report = outcome.Report!;
var finding = Assert.Single(Entries(report, "findings"));
Assert.Equal("3.1", finding["installedVersion"]);
var skipped = Assert.Single(Entries(report, "not_evaluated"));
Assert.Equal("1.0.3", skipped["installedVersion"]);
Assert.Equal("version_scheme", skipped["reason"]);
Assert.Equal(new[] { CveCacheFiller.CacheKey(ToolCpe) }, rig.CacheReads);
}
// -----------------------------------------------------------------------
// The highest score of an entry
// -----------------------------------------------------------------------
[Fact]
public async Task Finding_CarriesTheHighestScoreOfAllItsCves_AlsoOfOneItsListLeavesOut()
{
var rig = await RigAsync();
// Ten CVEs with a known likelihood of exploitation fill the list. The one with the highest
// score has none, ranks last among the urgent ones and is the one the list leaves out.
var cves = Enumerable.Range(1, VulnerabilityCorrelationService.MaxCvesPerFinding)
.Select(n =>
{
var cve = VulnerabilityCacheRows.Cve($"CVE-2026-120{n:00}", 5.0 + n / 10.0, WidgetCpe, "5.0");
cve.EpssScore = 0.2;
return cve;
})
.Append(VulnerabilityCacheRows.Cve("CVE-2026-12099", 9.9, WidgetCpe, "5.0"))
.ToList();
rig.Cache(VulnerabilityCacheRows.Filled(WidgetCpe, cves, CachedAt));
var outcome = await rig.CorrelateAsync(new[] { Installed("contoso", "Contoso Widget", "3.2.0"), Chrome() });
var findings = Entries(outcome.Report!, "findings");
var widget = findings.Single(f => (string)f["softwareName"] == "Contoso Widget");
var listed = ((List<Dictionary<string, object>>)widget["vulnerabilities"]).Select(v => (string)v["cveId"]).ToList();
Assert.Equal(11, widget["totalCves"]);
Assert.Equal(VulnerabilityCorrelationService.MaxCvesPerFinding, listed.Count);
Assert.DoesNotContain("CVE-2026-12099", listed);
Assert.Equal(9.9, widget["maxCvssScore"]);
Assert.Equal("critical", widget["riskLevel"]);
// An entry that lists all it found carries the same number its list shows.
var chrome = findings.Single(f => (string)f["softwareName"] == "Google Chrome");
Assert.Equal(8.8, chrome["maxCvssScore"]);
}
}
@@ -42,10 +42,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
["normalizationConfidence"] = "high",
},
};
var cpe = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
{
["edge"] = "cpe:2.3:a:microsoft:edge:*:*:*:*:*:*:*:*",
};
var cpe = new string?[] { "cpe:2.3:a:microsoft:edge:*:*:*:*:*:*:*:*" };
var items = VulnerabilityCorrelationService.ParseSoftwareInventoryItems(raw, cpe);
@@ -66,7 +63,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
new Dictionary<string, object> { ["displayName"] = "Empty triple — should be skipped" },
};
var items = VulnerabilityCorrelationService.ParseSoftwareInventoryItems(raw, cpeMappings: null);
var items = VulnerabilityCorrelationService.ParseSoftwareInventoryItems(raw, cpeUris: null);
Assert.Single(items);
Assert.Equal("good", items[0].NormalizedName);
@@ -97,7 +94,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
new Dictionary<string, object> { ["normalizedPublisher"] = "v2", ["normalizedName"] = "n2", ["normalizedVersion"] = "2.0" },
};
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeUris: null);
vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
TenantId,
@@ -137,7 +134,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
new Dictionary<string, object> { ["normalizedPublisher"] = "v1", ["normalizedName"] = "n1", ["normalizedVersion"] = "1.0" },
};
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeUris: null);
vulnRepo.Verify(r => r.DecrementSoftwareInventoryEntryAsync(
TenantId, "vOld", "nOld", "0.9", It.IsAny<CancellationToken>()), Times.Once);
@@ -180,7 +177,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
};
// Should not throw.
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeUris: null);
vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
It.IsAny<string>(), It.IsAny<SoftwareInventoryItem>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
@@ -237,7 +234,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
};
// Should NOT throw.
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeUris: null);
// Every counter write attempted.
vulnRepo.Verify(r => r.IncrementSoftwareInventoryEntryAsync(
@@ -287,7 +284,7 @@ public class VulnerabilityCorrelationServiceContributionsTests
new Dictionary<string, object> { ["normalizedPublisher"] = "v1", ["normalizedName"] = "n1", ["normalizedVersion"] = "1.0" },
};
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeMappings: null);
await service.ApplyInventoryContributionsAsync(TenantId, SessionId, raw, cpeUris: null);
Assert.Empty(saved);
}
@@ -320,11 +317,12 @@ public class VulnerabilityCorrelationServiceContributionsTests
{
return new VulnerabilityCorrelationService(
vulnRepo.Object,
nvdApi: null!,
cveCache: null!,
kevService: null!,
msrcCorrelation: null!,
telemetryClient: null,
logger: NullLogger<VulnerabilityCorrelationService>.Instance,
mappingStore: null!,
opsEvents: opsEvents);
}
}
@@ -18,7 +18,7 @@ public class VulnerabilityCorrelationServiceTests
[InlineData("cpe:2.3:a:cisco:anyconnect:*:*:*:*:*:ios:*:*", true)]
public void IsNonWindowsPlatform_NonWindowsTargetSw_ReturnsTrue(string criteria, bool expected)
{
Assert.Equal(expected, VulnerabilityCorrelationService.IsNonWindowsPlatform(criteria));
Assert.Equal(expected, CveVersionFilter.IsNonWindowsPlatform(criteria));
}
[Theory]
@@ -27,33 +27,33 @@ public class VulnerabilityCorrelationServiceTests
[InlineData("cpe:2.3:a:cisco:anyconnect:*:*:*:*:*:-:*:*")]
public void IsNonWindowsPlatform_WindowsOrWildcard_ReturnsFalse(string criteria)
{
Assert.False(VulnerabilityCorrelationService.IsNonWindowsPlatform(criteria));
Assert.False(CveVersionFilter.IsNonWindowsPlatform(criteria));
}
[Fact]
public void IsNonWindowsPlatform_ShortCpe_ReturnsFalse()
{
// CPE with fewer than 11 fields — no target_sw available
Assert.False(VulnerabilityCorrelationService.IsNonWindowsPlatform("cpe:2.3:a:cisco:anyconnect:1.0"));
Assert.False(CveVersionFilter.IsNonWindowsPlatform("cpe:2.3:a:cisco:anyconnect:1.0"));
}
[Fact]
public void IsNonWindowsPlatform_Null_ReturnsFalse()
{
Assert.False(VulnerabilityCorrelationService.IsNonWindowsPlatform(null));
Assert.False(CveVersionFilter.IsNonWindowsPlatform(null));
}
[Fact]
public void IsNonWindowsPlatform_Empty_ReturnsFalse()
{
Assert.False(VulnerabilityCorrelationService.IsNonWindowsPlatform(""));
Assert.False(CveVersionFilter.IsNonWindowsPlatform(""));
}
[Fact]
public void IsNonWindowsPlatform_TargetHw_NonWindows_ReturnsTrue()
{
// target_hw at index 11 = android (target_sw at index 10 = wildcard)
Assert.True(VulnerabilityCorrelationService.IsNonWindowsPlatform(
Assert.True(CveVersionFilter.IsNonWindowsPlatform(
"cpe:2.3:a:cisco:anyconnect:*:*:*:*:*:*:android:*"));
}
@@ -68,7 +68,7 @@ public class VulnerabilityCorrelationServiceTests
[InlineData("Vulnerability in agent for iOS devices", true)]
public void IsDescriptionNonWindowsOnly_OnlyNonWindowsPlatforms_ReturnsTrue(string description, bool expected)
{
Assert.Equal(expected, VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(description));
Assert.Equal(expected, CveVersionFilter.IsDescriptionNonWindowsOnly(description));
}
[Theory]
@@ -77,7 +77,7 @@ public class VulnerabilityCorrelationServiceTests
[InlineData("Cross-platform issue affecting Windows and Linux")]
public void IsDescriptionNonWindowsOnly_MentionsWindows_ReturnsFalse(string description)
{
Assert.False(VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(description));
Assert.False(CveVersionFilter.IsDescriptionNonWindowsOnly(description));
}
[Theory]
@@ -85,31 +85,24 @@ public class VulnerabilityCorrelationServiceTests
[InlineData("Remote code execution via crafted input")]
public void IsDescriptionNonWindowsOnly_NoPlatformMentioned_ReturnsFalse(string description)
{
Assert.False(VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(description));
Assert.False(CveVersionFilter.IsDescriptionNonWindowsOnly(description));
}
[Fact]
public void IsDescriptionNonWindowsOnly_NullOrEmpty_ReturnsFalse()
{
Assert.False(VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(null));
Assert.False(VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(""));
Assert.False(VulnerabilityCorrelationService.IsDescriptionNonWindowsOnly(" "));
Assert.False(CveVersionFilter.IsDescriptionNonWindowsOnly(null));
Assert.False(CveVersionFilter.IsDescriptionNonWindowsOnly(""));
Assert.False(CveVersionFilter.IsDescriptionNonWindowsOnly(" "));
}
// -----------------------------------------------------------------------
// FilterByVersion
// -----------------------------------------------------------------------
private VulnerabilityCorrelationService CreateServiceForFilterTests()
{
// FilterByVersion doesn't use any dependencies — we can pass nulls
return new VulnerabilityCorrelationService(null!, null!, null!, null!, null, null!);
}
[Fact]
public void FilterByVersion_VersionInRange_ReturnsMatch()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -131,7 +124,7 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161").Vulnerabilities;
Assert.Single(results);
Assert.Equal("CVE-2024-0213", results[0]["cveId"]);
}
@@ -139,7 +132,6 @@ public class VulnerabilityCorrelationServiceTests
[Fact]
public void FilterByVersion_VersionOutOfRange_ReturnsEmpty()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -160,14 +152,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:cisco:anyconnect_secure_mobility_client", "5.1.5.65");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:cisco:anyconnect_secure_mobility_client", "5.1.5.65").Vulnerabilities;
Assert.Empty(results);
}
[Fact]
public void FilterByVersion_WildcardVersion_FlagsLowConfidence()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -188,7 +179,7 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:vendor:product", "1.0.0");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:vendor:product", "1.0.0").Vulnerabilities;
Assert.Single(results);
Assert.True(results[0].ContainsKey("versionMatchConfidence"));
Assert.Equal("low", results[0]["versionMatchConfidence"]);
@@ -197,7 +188,6 @@ public class VulnerabilityCorrelationServiceTests
[Fact]
public void FilterByVersion_DashVersion_SkipsMatch()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -217,14 +207,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:vendor:product", "2.0");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:vendor:product", "2.0").Vulnerabilities;
Assert.Empty(results);
}
[Fact]
public void FilterByVersion_ExactVersionMatch_ReturnsMatch()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -244,7 +233,7 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:vendor:product", "2.5.0");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:vendor:product", "2.5.0").Vulnerabilities;
Assert.Single(results);
Assert.False(results[0].ContainsKey("versionMatchConfidence")); // Not a wildcard match
}
@@ -252,7 +241,6 @@ public class VulnerabilityCorrelationServiceTests
[Fact]
public void FilterByVersion_ExactVersionMismatch_ReturnsEmpty()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -272,14 +260,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:vendor:product", "2.6.0");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:vendor:product", "2.6.0").Vulnerabilities;
Assert.Empty(results);
}
[Fact]
public void FilterByVersion_NonWindowsPlatform_SkipsEntireCve()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -303,14 +290,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:cisco:anyconnect", "5.1.5.65");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:cisco:anyconnect", "5.1.5.65").Vulnerabilities;
Assert.Empty(results); // All ranges are non-Windows → entire CVE skipped
}
[Fact]
public void FilterByVersion_MixedPlatforms_KeepsWindowsRange()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -340,7 +326,7 @@ public class VulnerabilityCorrelationServiceTests
};
// Version 4.5 is within the Windows range
var results = svc.FilterByVersion(cves, "cpe:2.3:a:cisco:anyconnect", "4.5");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:cisco:anyconnect", "4.5").Vulnerabilities;
Assert.Single(results);
Assert.Equal("CVE-2021-1258", results[0]["cveId"]);
}
@@ -348,7 +334,6 @@ public class VulnerabilityCorrelationServiceTests
[Fact]
public void FilterByVersion_DescriptionNonWindowsOnly_SkipsCve()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -371,14 +356,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161").Vulnerabilities;
Assert.Empty(results); // Description says Linux/macOS only → filtered out
}
[Fact]
public void FilterByVersion_DescriptionMentionsWindows_KeepsCve()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -400,14 +384,13 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:trellix:agent", "5.8.0.161").Vulnerabilities;
Assert.Single(results); // Mentions Windows → keep it
}
[Fact]
public void FilterByVersion_EmptyAffectedVersions_ReturnsEmpty()
{
var svc = CreateServiceForFilterTests();
var cves = new List<CachedCve>
{
new CachedCve
@@ -421,174 +404,10 @@ public class VulnerabilityCorrelationServiceTests
}
};
var results = svc.FilterByVersion(cves, "cpe:2.3:a:vendor:product", "1.0");
var results = CveVersionFilter.Filter(cves, "cpe:2.3:a:vendor:product", "1.0").Vulnerabilities;
Assert.Empty(results);
}
// -----------------------------------------------------------------------
// FindCpeMapping
// -----------------------------------------------------------------------
private VulnerabilityCorrelationService CreateServiceWithMappings(List<CpeMappingEntry> mappings)
{
var svc = new VulnerabilityCorrelationService(null!, null!, null!, null!, null, null!);
svc._cpeMappings = mappings;
return svc;
}
[Fact]
public void FindCpeMapping_ExactMatch_ReturnsHighConfidence()
{
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "trellix",
NormalizedProduct = "trellix agent",
CpeUri = "cpe:2.3:a:trellix:agent",
CpeProduct = "agent"
}
});
var result = svc.FindCpeMapping("trellix", "trellix agent", "Trellix Agent 5.8.0");
Assert.NotNull(result);
Assert.Equal("high", result!.MatchConfidence);
Assert.Equal("cpe:2.3:a:trellix:agent", result.CpeUri);
}
[Fact]
public void FindCpeMapping_DisplayNamePattern_ReturnsMediumConfidence()
{
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "cisco",
NormalizedProduct = "cisco webex",
DisplayNamePatterns = new List<string> { "webex meetings", "cisco webex" },
CpeUri = "cpe:2.3:a:cisco:webex_meetings",
CpeProduct = "webex_meetings"
}
});
var result = svc.FindCpeMapping("unknown_vendor", "some product", "Cisco Webex Meetings Desktop App");
Assert.NotNull(result);
Assert.Equal("medium", result!.MatchConfidence);
}
[Fact]
public void FindCpeMapping_PublisherPartialProduct_ReturnsMediumConfidence()
{
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "adobe",
NormalizedProduct = "adobe acrobat dc",
PublisherPatterns = new List<string> { "adobe", "adobe systems" },
CpeUri = "cpe:2.3:a:adobe:acrobat_dc",
CpeProduct = "acrobat_dc"
}
});
var result = svc.FindCpeMapping("adobe", "adobe acrobat dc pro", "Adobe Acrobat DC Pro");
Assert.NotNull(result);
Assert.Equal("medium", result!.MatchConfidence);
}
[Fact]
public void FindCpeMapping_NoMatch_ReturnsNull()
{
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "cisco",
NormalizedProduct = "cisco webex",
DisplayNamePatterns = new List<string> { "webex" },
CpeUri = "cpe:2.3:a:cisco:webex_meetings",
CpeProduct = "webex_meetings"
}
});
var result = svc.FindCpeMapping("unknown", "unknown product", "Totally Unknown Software");
Assert.Null(result);
}
[Fact]
public void FindCpeMapping_CiscoSecureClient_MatchesSecureClientNotAnyConnect()
{
// Regression test for Fix 1: "Cisco Secure Client" must match the dedicated
// cisco:secure_client mapping, NOT the legacy anyconnect mapping.
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "cisco",
NormalizedProduct = "cisco anyconnect",
DisplayNamePatterns = new List<string> { "cisco anyconnect", "cisco anyconnect secure mobility" },
CpeUri = "cpe:2.3:a:cisco:anyconnect_secure_mobility_client",
CpeProduct = "anyconnect_secure_mobility_client"
},
new CpeMappingEntry
{
NormalizedVendor = "cisco",
NormalizedProduct = "cisco secure client",
DisplayNamePatterns = new List<string> { "cisco secure client" },
CpeUri = "cpe:2.3:a:cisco:secure_client",
CpeProduct = "secure_client"
}
});
var result = svc.FindCpeMapping("cisco", "cisco secure client", "Cisco Secure Client - AnyConnect VPN");
Assert.NotNull(result);
Assert.Equal("cpe:2.3:a:cisco:secure_client", result!.CpeUri);
}
[Fact]
public void FindCpeMapping_ExcludePattern_PreventsMatch()
{
// "Microsoft Edge Update" should NOT match edge_chromium because of excludePatterns
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "microsoft",
NormalizedProduct = "microsoft edge",
DisplayNamePatterns = new List<string> { "microsoft edge" },
ExcludePatterns = new List<string> { "edge update" },
CpeUri = "cpe:2.3:a:microsoft:edge_chromium",
CpeProduct = "edge_chromium"
}
});
// "Microsoft Edge Update" contains "microsoft edge" but also "edge update" → excluded
var result = svc.FindCpeMapping("microsoft", "microsoft edge update", "Microsoft Edge Update");
Assert.Null(result);
}
[Fact]
public void FindCpeMapping_ExcludePattern_DoesNotAffectLegitMatch()
{
var svc = CreateServiceWithMappings(new List<CpeMappingEntry>
{
new CpeMappingEntry
{
NormalizedVendor = "microsoft",
NormalizedProduct = "microsoft edge",
DisplayNamePatterns = new List<string> { "microsoft edge" },
ExcludePatterns = new List<string> { "edge update" },
CpeUri = "cpe:2.3:a:microsoft:edge_chromium",
CpeProduct = "edge_chromium"
}
});
// Real Edge browser → should still match (no exclude pattern hit)
var result = svc.FindCpeMapping("microsoft", "microsoft edge", "Microsoft Edge");
Assert.NotNull(result);
Assert.Equal("high", result!.MatchConfidence); // Exact match on normalizedProduct
}
// -----------------------------------------------------------------------
// MergeReports
// -----------------------------------------------------------------------
@@ -0,0 +1,214 @@
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="VulnerabilityDataSyncFunction.PartsThatFellShort"/> names, in plain words, the
/// parts of a nightly run an operator has to look at; a run that names none raises no ops event.
/// A walk NVD throttled part of the way is not among them, because its rows are due again the
/// next night. A walk that refreshed nothing is, and so are products that have waited for their
/// first data for more than a day. <see cref="VulnerabilityDataSyncFunction.StaleParts"/> adds
/// what a single run cannot see: a walk that has not reached its end for days, each night for a
/// reason that is harmless on its own, and a dictionary that has not been replaced for a week.
/// </summary>
public class VulnerabilityDataSyncFunctionTests
{
private static List<string> FellShort(
string? kevError = null, string? msrcError = null, string? dictionaryError = null, string? mappingError = null,
InventoryReconcileResult? reconciled = null, NvdCacheRefreshResult? cveStats = null, EpssRefreshResult? epssStats = null) =>
VulnerabilityDataSyncFunction.PartsThatFellShort(
kevError, msrcError, new CpeDictionaryRefreshResult { Error = dictionaryError }, mappingError, reconciled, cveStats, epssStats);
[Fact]
public void RunInWhichEveryPartDidItsWork_NamesNothing()
{
var parts = FellShort(
reconciled: new InventoryReconcileResult { Rows = 40, Set = 3, Cleared = 1 },
cveStats: new NvdCacheRefreshResult { Stale = 12, Refreshed = 12, Removed = 2 },
epssStats: new EpssRefreshResult { Rows = 30, Cves = 400, Scored = 380, RowsRewritten = 12 });
Assert.Empty(parts);
}
[Fact]
public void WalksThatDidNotRunBecauseAManualOneWasInFlight_NameNothing()
{
var parts = FellShort(reconciled: new InventoryReconcileResult { Rows = 40 }, cveStats: null, epssStats: null);
Assert.Empty(parts);
}
[Fact]
public void WalkWithNothingDue_NamesNothing()
{
Assert.Empty(FellShort(cveStats: new NvdCacheRefreshResult { Stale = 0, Refreshed = 0, Removed = 3 }));
}
[Theory]
[InlineData("HTTP 503", null, null, null, "KEV catalog not refreshed")]
[InlineData(null, "2 of 6 documents failed", null, null, "MSRC index not refreshed")]
[InlineData(null, null, "The reduced dictionary holds only 12 products; the stored copy is kept.", null, "NVD product dictionary not refreshed")]
[InlineData(null, null, null, "CPE mappings could not be loaded and no earlier copy is held.", "automatic mappings or inventory reconcile failed")]
public void SourceOrMappingStepThatFailed_IsNamed(
string? kevError, string? msrcError, string? dictionaryError, string? mappingError, string expected)
{
var parts = FellShort(kevError, msrcError, dictionaryError, mappingError);
Assert.Equal(new[] { expected }, parts);
}
[Fact]
public void InventoryRowsThatDidNotTakeTheirCpe_AreNamedWithTheirNumber()
{
var parts = FellShort(reconciled: new InventoryReconcileResult { Rows = 40, Set = 5, Failed = 3 });
Assert.Equal(new[] { "3 inventory rows did not take their CPE" }, parts);
}
[Fact]
public void WalkThatFailedAsAWhole_IsNamed()
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Error = "The table could not be read." });
Assert.Equal(new[] { "CVE cache walk failed" }, parts);
}
[Fact]
public void WalkNvdThrottledPartOfTheWay_NamesNothing()
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Stale = 20, Refreshed = 6, SkippedThrottled = 14 });
Assert.Empty(parts);
}
[Fact]
public void WalkCutOffBecauseNvdStoppedAnsweringOrTimeRanOut_IsNamedWithTheRowsLeft()
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Stale = 20, Refreshed = 10, Failed = 3, SkippedUnavailable = 7 });
Assert.Equal(new[] { "CVE cache walk cut off with 7 rows left (NVD not answering or out of time)" }, parts);
}
[Theory]
// NVD answered and turned every request down.
[InlineData(9, 0)]
// NVD refused the client at the first row: nothing was refreshed, however harmless a throttle is otherwise.
[InlineData(0, 9)]
public void WalkThatRefreshedNoneOfItsDueRows_IsNamed(int failed, int skippedThrottled)
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Stale = 9, Refreshed = 0, Failed = failed, SkippedThrottled = skippedThrottled });
Assert.Equal(new[] { "CVE cache walk refreshed none of 9 due rows (NVD refused or failed every request)" }, parts);
}
[Fact]
public void WalkCutOffBeforeItRefreshedAnything_IsNamedAsCutOff_NotAsRefreshedNone()
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Stale = 9, Refreshed = 0, Failed = 3, SkippedUnavailable = 6 });
Assert.Equal(new[] { "CVE cache walk cut off with 6 rows left (NVD not answering or out of time)" }, parts);
}
[Fact]
public void ProductsWithoutDataForMoreThanADay_AreNamed_EvenWhenTheWalkItselfIsNot()
{
var parts = FellShort(cveStats: new NvdCacheRefreshResult { Stale = 20, Refreshed = 6, SkippedThrottled = 14, PendingOverdue = 4 });
Assert.Equal(new[] { "4 products without CVE data for more than a day" }, parts);
}
[Fact]
public void EpssScoresThatWereNotRefreshed_AreNamed()
{
var parts = FellShort(epssStats: new EpssRefreshResult { Rows = 30, Error = "HTTP 500" });
Assert.Equal(new[] { "EPSS scores not refreshed" }, parts);
}
[Fact]
public void EverythingThatFellShort_IsNamedInTheOrderOfTheRun()
{
var parts = FellShort(
kevError: "HTTP 503",
msrcError: "HTTP 503",
dictionaryError: "HTTP 503",
mappingError: "storage down",
reconciled: new InventoryReconcileResult { Rows = 40, Failed = 2 },
cveStats: new NvdCacheRefreshResult { Stale = 9, Refreshed = 0, SkippedUnavailable = 9, PendingOverdue = 5 },
epssStats: new EpssRefreshResult { Error = "HTTP 503" });
Assert.Equal(
new[]
{
"KEV catalog not refreshed",
"MSRC index not refreshed",
"NVD product dictionary not refreshed",
"automatic mappings or inventory reconcile failed",
"2 inventory rows did not take their CPE",
"CVE cache walk cut off with 9 rows left (NVD not answering or out of time)",
"5 products without CVE data for more than a day",
"EPSS scores not refreshed",
},
parts);
}
// -----------------------------------------------------------------------
// What one run cannot see about itself
// -----------------------------------------------------------------------
private static readonly DateTime Now = new(2026, 11, 12, 3, 20, 0, DateTimeKind.Utc);
[Fact]
public void StaleParts_NothingStoredYet_NamesNothing()
{
// A new installation has no walk that reached its end and no dictionary: nothing is overdue.
Assert.Empty(VulnerabilityDataSyncFunction.StaleParts(null, null, Now));
}
[Fact]
public void StaleParts_WalkAndDictionaryWithinTheirTime_NameNothing()
{
var lastNight = VulnerabilityDataSyncFunction.StaleParts(Now.AddDays(-1), Now.AddDays(-1), Now);
var atTheLimit = VulnerabilityDataSyncFunction.StaleParts(
Now - VulnerabilityDataSyncFunction.WalkOverdueAfter, Now - VulnerabilityDataSyncFunction.DictionaryOverdueAfter, Now);
Assert.Empty(lastNight);
Assert.Empty(atTheLimit);
}
[Fact]
public void StaleParts_WalkThatHasNotReachedItsEndForMoreThanTwoDays_IsNamedWithTheDayItLastDid()
{
var justOver = Now - VulnerabilityDataSyncFunction.WalkOverdueAfter - TimeSpan.FromSeconds(1);
var parts = VulnerabilityDataSyncFunction.StaleParts(justOver, Now.AddDays(-1), Now);
var longAgo = VulnerabilityDataSyncFunction.StaleParts(new DateTime(2026, 10, 3, 3, 12, 0, DateTimeKind.Utc), null, Now);
Assert.Equal(new[] { "CVE cache walk has not reached its end since 2026-11-10" }, parts);
Assert.Equal(new[] { "CVE cache walk has not reached its end since 2026-10-03" }, longAgo);
}
[Fact]
public void StaleParts_DictionaryOlderThanAWeek_IsNamedWithTheDayItWasReplaced()
{
var justOver = Now - VulnerabilityDataSyncFunction.DictionaryOverdueAfter - TimeSpan.FromSeconds(1);
var parts = VulnerabilityDataSyncFunction.StaleParts(Now.AddDays(-1), justOver, Now);
var longAgo = VulnerabilityDataSyncFunction.StaleParts(null, new DateTime(2026, 9, 28, 3, 4, 0, DateTimeKind.Utc), Now);
Assert.Equal(new[] { "NVD product dictionary not replaced since 2026-11-05" }, parts);
Assert.Equal(new[] { "NVD product dictionary not replaced since 2026-09-28" }, longAgo);
}
[Fact]
public void StaleParts_WalkAndDictionaryBothOverdue_AreNamedWalkFirst()
{
var parts = VulnerabilityDataSyncFunction.StaleParts(
new DateTime(2026, 11, 8, 3, 31, 0, DateTimeKind.Utc), new DateTime(2026, 11, 1, 3, 5, 0, DateTimeKind.Utc), Now);
Assert.Equal(
new[] { "CVE cache walk has not reached its end since 2026-11-08", "NVD product dictionary not replaced since 2026-11-01" },
parts);
}
}
@@ -0,0 +1,350 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using Newtonsoft.Json;
using static AutopilotMonitor.Functions.Tests.ReportFixture;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="VulnerabilityCorrelationService.MergeReports"/> joins the WhiteGlove
/// pre-provisioning report with the user-enrollment one. A software entry lists only its most
/// urgent CVEs, so the merge adds up the entries' counts instead of their lists, keeps one entry
/// per program and version, and carries the programs that were not judged. The earlier report
/// arrives as it is read back from its table row. Phase tagging of distinct entries and the
/// summary fields carried over are in <c>VulnerabilityCorrelationServiceTests</c>.
/// </summary>
public class VulnerabilityReportMergeTests
{
private static Dictionary<string, object> Vuln(string cveId, double cvss, bool kev = false, int descriptionLength = 20) => new()
{
["cveId"] = cveId,
["cvssScore"] = cvss,
["cvssSeverity"] = NvdFixture.Severity(cvss),
["description"] = new string('d', descriptionLength),
["isKev"] = kev,
["priority"] = CvePriority.Compute(kev, null, cvss),
};
/// <summary>An entry as a scan writes it: counts for every CVE, a list of the most urgent.</summary>
private static Dictionary<string, object> Counted(
string software, string version, int total,
int critical = 0, int high = 0, int medium = 0, int low = 0, int kev = 0, int attend = 0,
int listed = 1, int descriptionLength = 20) => new()
{
["softwareName"] = software,
["installedVersion"] = version,
["riskLevel"] = kev > 0 || critical > 0 ? "critical" : high > 0 ? "high" : medium > 0 ? "medium" : "low",
["totalCves"] = total,
["severityCounts"] = new Dictionary<string, object>
{
["critical"] = critical, ["high"] = high, ["medium"] = medium, ["low"] = low,
},
["kevCount"] = kev,
["attendCount"] = attend,
["vulnerabilities"] = Enumerable.Range(1, listed)
.Select(n => Vuln($"CVE-2026-{n:0000}", 7.5, descriptionLength: descriptionLength)).ToList(),
};
/// <summary>An entry written before the lists were shortened: the complete list and no counts.</summary>
private static Dictionary<string, object> Uncounted(string software, string version, params Dictionary<string, object>[] vulns) => new()
{
["softwareName"] = software,
["installedVersion"] = version,
["riskLevel"] = "high",
["vulnerabilities"] = vulns.ToList(),
};
private static Dictionary<string, object> Listed(string software, string version, string? reason = null)
{
var entry = new Dictionary<string, object> { ["softwareName"] = software, ["installedVersion"] = version };
if (reason != null) entry["reason"] = reason;
return entry;
}
private static Dictionary<string, object> Report(
IEnumerable<Dictionary<string, object>> findings,
IEnumerable<Dictionary<string, object>>? notEvaluated = null,
IEnumerable<Dictionary<string, object>>? clean = null,
int additionalFindings = 0) => new()
{
["scan_summary"] = new Dictionary<string, object>
{
["total_software_scanned"] = 10,
["matched_to_cpe"] = 5,
["additional_findings_count"] = additionalFindings,
},
["findings"] = findings.ToList(),
["not_evaluated"] = (notEvaluated ?? Array.Empty<Dictionary<string, object>>()).ToList(),
["unmatched_software"] = new List<Dictionary<string, object>>(),
["matched_clean_software"] = (clean ?? Array.Empty<Dictionary<string, object>>()).ToList(),
};
private static List<Dictionary<string, object>> Findings(Dictionary<string, object> report) =>
(List<Dictionary<string, object>>)report["findings"];
private static Dictionary<string, object> Summary(Dictionary<string, object> report) =>
(Dictionary<string, object>)report["scan_summary"];
private static IEnumerable<string> Names(object list) =>
((IEnumerable<Dictionary<string, object>>)list).Select(e => (string)e["softwareName"]);
// -----------------------------------------------------------------------
// Sums
// -----------------------------------------------------------------------
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Sums_ComeFromAnEntrysCounts_AndFromItsListWhereItHasNone(bool earlierReportFromStorage)
{
var earlier = Report(new[]
{
Uncounted("Fabrikam Gadget", "1.0",
Vuln("CVE-2025-0001", 9.8, kev: true), Vuln("CVE-2025-0002", 5.0), Vuln("CVE-2025-0003", 5.0)),
});
var newer = Report(new[]
{
Counted("Contoso Widget", "3.2.0", total: 42, critical: 2, high: 30, medium: 8, low: 2, kev: 1, attend: 5, listed: 10),
});
var merged = VulnerabilityCorrelationService.MergeReports(earlierReportFromStorage ? AsStored(earlier) : earlier, newer);
var summary = Summary(merged);
Assert.Equal(45, summary["total_cves_found"]);
Assert.Equal(3, summary["critical_cves"]);
Assert.Equal(30, summary["high_cves"]);
Assert.Equal(10, summary["medium_cves"]);
Assert.Equal(2, summary["low_cves"]);
Assert.Equal(2, summary["kev_matches"]);
Assert.Equal(2, summary["act_cves"]);
Assert.Equal(5, summary["attend_cves"]);
Assert.Equal("critical", summary["overall_risk"]);
var widget = Findings(merged).Single(f => (string)f["softwareName"] == "Contoso Widget");
Assert.Equal(42, Convert.ToInt32(widget["totalCves"]));
Assert.Equal(10, ((IEnumerable<object>)widget["vulnerabilities"]).Count());
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void Sums_OfWhatNeedsAction_LeaveOutSoftwareThatUpdatesItself(bool earlierReportFromStorage)
{
var browser = Counted("Contoso Browser", "131.0", total: 12, critical: 1, high: 6, medium: 4, low: 1, kev: 1);
browser["selfUpdating"] = true;
var earlier = Report(new[]
{
browser,
Uncounted("Fabrikam Gadget", "1.0", Vuln("CVE-2025-0001", 9.8, kev: true), Vuln("CVE-2025-0002", 5.0)),
});
var newer = Report(new[]
{
Counted("Contoso Widget", "3.2.0", total: 7, high: 4, medium: 2, low: 1),
});
var merged = VulnerabilityCorrelationService.MergeReports(earlierReportFromStorage ? AsStored(earlier) : earlier, newer);
var summary = Summary(merged);
Assert.Equal(21, summary["total_cves_found"]);
Assert.Equal(2, summary["kev_matches"]);
// The gadget and the widget have to be patched by hand; the browser patches itself.
Assert.Equal(9, summary["actionable_cves"]);
Assert.Equal(2, summary["actionable_packages"]);
Assert.Equal(1, summary["actionable_critical_cves"]);
Assert.Equal(4, summary["actionable_high_cves"]);
Assert.Equal(3, summary["actionable_medium_cves"]);
Assert.Equal(1, summary["actionable_low_cves"]);
Assert.Equal(1, summary["actionable_kev_matches"]);
Assert.Equal(12, summary["self_updating_cves"]);
Assert.Equal(1, summary["self_updating_packages"]);
// The entry keeps its mark and comes after everything that needs action, whatever its risk.
var findings = Findings(merged);
Assert.Equal(new[] { "Fabrikam Gadget", "Contoso Widget", "Contoso Browser" }, Names(findings));
Assert.True(findings[^1]["selfUpdating"] is true);
Assert.All(findings.Take(2), finding => Assert.False(finding.ContainsKey("selfUpdating")));
}
// -----------------------------------------------------------------------
// One entry per program and version
// -----------------------------------------------------------------------
[Theory]
[InlineData(false)]
[InlineData(true)]
public void SameProgramAndVersionInBothReports_AppearsOnceFromTheNewerReport(bool earlierReportFromStorage)
{
var earlier = Report(new[]
{
Counted("Contoso Widget", "3.2.0", total: 5, high: 5),
Counted("Contoso Widget", "3.1.0", total: 9, high: 9),
});
var newer = Report(new[] { Counted("Contoso Widget", "3.2.0", total: 7, high: 6, medium: 1) });
var merged = VulnerabilityCorrelationService.MergeReports(earlierReportFromStorage ? AsStored(earlier) : earlier, newer);
var findings = Findings(merged);
Assert.Equal(2, findings.Count);
var both = findings.Single(f => (string)f["installedVersion"] == "3.2.0");
Assert.Equal("both", both["phase"]);
Assert.Equal(7, Convert.ToInt32(both["totalCves"]));
Assert.Equal("device_setup", findings.Single(f => (string)f["installedVersion"] == "3.1.0")["phase"]);
// 7 + 9: the same version carries the same CVEs, so it is not counted once per phase.
Assert.Equal(16, Summary(merged)["total_cves_found"]);
Assert.Equal(15, Summary(merged)["high_cves"]);
}
[Theory]
[InlineData("user_enrollment", "user_enrollment")] // the newer phase ran again and meets its own entry
[InlineData("device_setup", "both")]
[InlineData("both", "both")]
[InlineData(null, "both")] // an untagged earlier entry belongs to the earlier phase
public void PhaseOfAnEntryFoundAgain_FollowsWhatTheStoredEntryStoodFor(string? storedPhase, string expectedPhase)
{
var stored = Counted("Contoso Widget", "3.2.0", total: 5, high: 5);
if (storedPhase != null) stored["phase"] = storedPhase;
var earlier = AsStored(Report(new[] { stored }));
var newer = Report(new[] { Counted("Contoso Widget", "3.2.0", total: 5, high: 5) });
var merged = VulnerabilityCorrelationService.MergeReports(earlier, newer);
Assert.Equal(expectedPhase, Assert.Single(Findings(merged))["phase"]);
}
[Fact]
public void PhaseContributions_CountTheEntriesEachPhaseHasAlone()
{
var earlier = AsStored(Report(new[]
{
Counted("Only Before A", "1.0", total: 1, high: 1),
Counted("Only Before B", "1.0", total: 1, high: 1),
Counted("In Both", "1.0", total: 1, high: 1),
}));
var newer = Report(new[]
{
Counted("Only After", "1.0", total: 1, high: 1),
Counted("In Both", "1.0", total: 1, high: 1),
});
var merged = VulnerabilityCorrelationService.MergeReports(earlier, newer);
var contributions = (Dictionary<string, object>)Summary(merged)["phase_contributions"];
Assert.Equal(2, contributions["device_setup"]);
Assert.Equal(1, contributions["user_enrollment"]);
Assert.Equal(4, Findings(merged).Count);
}
// -----------------------------------------------------------------------
// Programs that were not judged, and clean ones
// -----------------------------------------------------------------------
[Fact]
public void NotEvaluatedAndCleanPrograms_AreMergedOnceEach_WithoutThoseThatHaveAFinding()
{
var earlier = AsStored(Report(
findings: new[] { Counted("Fabrikam Gadget", "1.0", total: 1, high: 1) },
notEvaluated: new[]
{
Listed("Contoso Widget", "3.2.0", "cve_data_pending"),
Listed("Contoso Viewer", "2.0", "cve_data_pending"),
},
clean: new[] { Listed("Contoso Editor", "5.0"), Listed("Contoso Viewer", "1.0") }));
var newer = Report(
// The viewer got its CVE data in the meantime and has a finding now.
findings: new[] { Counted("Contoso Viewer", "2.0", total: 3, high: 3) },
notEvaluated: new[]
{
Listed("Contoso Widget", "3.2.0", "version_scheme"),
Listed("Contoso Reader", "1.0.0.0", "placeholder_version"),
},
clean: new[] { Listed("Contoso Editor", "5.0"), Listed("Fabrikam Gadget", "1.0"), Listed("Contoso Player", "8.1") });
var merged = VulnerabilityCorrelationService.MergeReports(earlier, newer);
var notEvaluated = (List<Dictionary<string, object>>)merged["not_evaluated"];
Assert.Equal(new[] { "Contoso Widget", "Contoso Reader" }, Names(notEvaluated));
Assert.Equal("version_scheme", notEvaluated[0]["reason"]); // the newer run's verdict
Assert.Equal(2, Summary(merged)["not_evaluated"]);
// A program with a finding is neither "not judged" nor "clean", whichever report said so.
Assert.Equal(
new[] { "Contoso Editor", "Contoso Viewer", "Contoso Player" },
Names(merged["matched_clean_software"]));
Assert.Equal("1.0", ((List<Dictionary<string, object>>)merged["matched_clean_software"])[1]["installedVersion"]);
}
// -----------------------------------------------------------------------
// Size
// -----------------------------------------------------------------------
[Fact]
public void EntriesBeyondTheLimit_AreCountedOnTopOfWhatEitherReportHadAlreadyCut()
{
var earlier = AsStored(Report(
Enumerable.Range(1, 30).Select(n => Counted($"Before {n:00}", "1.0", total: 2, high: 2)), additionalFindings: 3));
var newer = Report(
Enumerable.Range(1, 30).Select(n => Counted($"After {n:00}", "1.0", total: 2, high: 2)), additionalFindings: 2);
var merged = VulnerabilityCorrelationService.MergeReports(earlier, newer);
Assert.Equal(VulnerabilityCorrelationService.MaxFindings, Findings(merged).Count);
Assert.Equal(10 + 3 + 2, Summary(merged)["additional_findings_count"]);
// The sums cover the entries that are no longer listed.
Assert.Equal(120, Summary(merged)["total_cves_found"]);
}
[Fact]
public void OlderEntry_GetsItsCountsAndTheMostUrgentListWhenMerged()
{
var older = Uncounted("Contoso Widget", "3.2.0",
Enumerable.Range(1, 14).Select(n => Vuln($"CVE-2025-{n:0000}", 5.0))
.Append(Vuln("CVE-2025-9001", 9.8))
.Append(Vuln("CVE-2025-9002", 6.0, kev: true)).ToArray());
var merged = VulnerabilityCorrelationService.MergeReports(
AsStored(Report(new[] { older })), Report(new[] { Counted("Fabrikam Tool", "1.0", total: 1, high: 1) }));
var entry = Findings(merged).Single(f => (string)f["softwareName"] == "Contoso Widget");
Assert.Equal(16, Convert.ToInt32(entry["totalCves"]));
Assert.Equal(1, Convert.ToInt32(entry["kevCount"]));
var listed = ((IEnumerable<object>)entry["vulnerabilities"]).Cast<Dictionary<string, object>>().ToList();
Assert.Equal(VulnerabilityCorrelationService.MaxCvesPerFinding, listed.Count);
Assert.Equal(new[] { "CVE-2025-9002", "CVE-2025-9001" }, listed.Take(2).Select(v => (string)v["cveId"]));
}
[Fact]
public void OlderEntriesThatAreTrimmed_StillCountInFullOnALaterMerge()
{
// Far more than a table row holds as long as every entry lists all of its CVEs.
var older = Enumerable.Range(1, 30).Select(i => Uncounted($"Contoso Suite {i}", "1.0",
Enumerable.Range(1, 100).Select(n => Vuln($"CVE-2025-{i:00}{n:000}", 7.5, descriptionLength: 500)).ToArray()));
var newer = Report(new[] { Counted("Fabrikam Tool", "1.0", total: 1, high: 1) });
var first = VulnerabilityCorrelationService.MergeReports(AsStored(Report(older)), newer);
var second = VulnerabilityCorrelationService.MergeReports(AsStored(first), newer);
Assert.All(Findings(first), f => Assert.True(((IEnumerable<object>)f["vulnerabilities"]).Count() <= VulnerabilityCorrelationService.MaxCvesPerFinding));
Assert.Equal(3001, Convert.ToInt32(Summary(first)["total_cves_found"]));
Assert.Equal(3001, Convert.ToInt32(Summary(second)["total_cves_found"]));
}
[Fact]
public void MergedReportTooLargeForItsRow_IsTrimmedAndKeepsItsCounts()
{
var earlier = AsStored(Report(Enumerable.Range(1, 25).Select(n =>
Counted($"Before {n:00}", "1.0", total: 100, high: 100, listed: 100, descriptionLength: 500))));
var newer = Report(Enumerable.Range(1, 25).Select(n =>
Counted($"After {n:00}", "1.0", total: 100, high: 100, listed: 100, descriptionLength: 500)));
var merged = VulnerabilityCorrelationService.MergeReports(earlier, newer);
Assert.True(JsonConvert.SerializeObject(merged).Length <= VulnerabilityCorrelationService.MaxReportChars);
Assert.True(Summary(merged)["report_trimmed"] is true);
Assert.Equal(5000, Summary(merged)["total_cves_found"]);
Assert.Equal(50, Findings(merged).Count);
Assert.All(Findings(merged), finding =>
{
Assert.Equal(100, Convert.ToInt32(finding["totalCves"]));
Assert.InRange(((IEnumerable<object>)finding["vulnerabilities"]).Count(), 1, 99);
});
}
}
@@ -0,0 +1,264 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using Newtonsoft.Json;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The shape of a report: a software entry lists every known-exploited CVE and the most urgent
/// others but counts all of them, software that has to be patched by hand comes before software
/// that updates itself, and a report too large for its table row is cut without losing its counts.
/// </summary>
public class VulnerabilityReportShapeTests
{
private static Dictionary<string, object> Vuln(string cveId, double cvss, bool kev = false, double? epss = null, int descriptionLength = 40)
{
var vuln = new Dictionary<string, object>
{
["cveId"] = cveId,
["cvssScore"] = cvss,
["cvssSeverity"] = NvdFixture.Severity(cvss),
["description"] = new string('d', descriptionLength),
["publishedDate"] = "2026-06-01T00:00:00.000",
["source"] = "NVD",
["isKev"] = kev,
["priority"] = CvePriority.Compute(kev, epss, cvss),
};
if (epss.HasValue) vuln["epssScore"] = epss.Value;
return vuln;
}
private static List<Dictionary<string, object>> Vulns(int first, int count, double cvss, bool kev = false) =>
Enumerable.Range(first, count).Select(n => Vuln($"CVE-2026-{n:0000}", cvss, kev)).ToList();
private static List<string> ListedIds(Dictionary<string, object> finding) =>
((IEnumerable<object>)finding["vulnerabilities"])
.Cast<Dictionary<string, object>>().Select(v => (string)v["cveId"]).ToList();
private static Dictionary<string, object> Finding(List<Dictionary<string, object>> vulns, bool selfUpdating = false, string software = "Contoso Widget") =>
VulnerabilityCorrelationService.BuildFinding(software, "3.2.0", "cpe:2.3:a:contoso:widget", "high", selfUpdating, vulns);
// -----------------------------------------------------------------------
// BuildFinding
// -----------------------------------------------------------------------
[Fact]
public void BuildFinding_CountsEveryCve_AndListsTheMostUrgentTen()
{
var vulns = new List<Dictionary<string, object>>
{
Vuln("CVE-2026-0001", 5.0, kev: true),
Vuln("CVE-2026-0002", 9.1, kev: true),
Vuln("CVE-2026-0201", 8.0, epss: 0.5),
Vuln("CVE-2026-0202", 8.0, epss: 0.9),
};
vulns.AddRange(Vulns(101, 3, 9.8));
vulns.AddRange(Vulns(203, 4, 7.5));
vulns.AddRange(Vulns(301, 8, 5.0));
vulns.AddRange(Vulns(401, 6, 2.0));
var finding = Finding(vulns);
Assert.Equal(25, finding["totalCves"]);
var severity = (Dictionary<string, object>)finding["severityCounts"];
Assert.Equal(4, severity["critical"]);
Assert.Equal(6, severity["high"]);
Assert.Equal(9, severity["medium"]);
Assert.Equal(6, severity["low"]);
Assert.Equal(2, finding["kevCount"]);
// "attend" is likely or critical without being listed as exploited: 3 critical, 2 by EPSS.
Assert.Equal(5, finding["attendCount"]);
// Known exploited first, then by exploitation likelihood, then by severity.
Assert.Equal(new[]
{
"CVE-2026-0002", "CVE-2026-0001",
"CVE-2026-0202", "CVE-2026-0201",
"CVE-2026-0101", "CVE-2026-0102", "CVE-2026-0103",
"CVE-2026-0203", "CVE-2026-0204", "CVE-2026-0205",
}, ListedIds(finding));
Assert.Equal("critical", finding["riskLevel"]);
Assert.Equal(CvePriority.Act, finding["priority"]);
Assert.Equal(0.9, finding["maxEpssScore"]);
Assert.Equal("cpe:2.3:a:contoso:widget:3.2.0", finding["cpeUri"]);
Assert.Equal("high", finding["matchConfidence"]);
}
[Fact]
public void BuildFinding_ListsEveryKnownExploitedCve_EvenBeyondTen()
{
var vulns = Vulns(1, 12, 6.5, kev: true);
vulns.AddRange(Vulns(101, 5, 9.8));
var finding = Finding(vulns);
Assert.Equal(Enumerable.Range(1, 12).Select(n => $"CVE-2026-{n:0000}"), ListedIds(finding));
Assert.Equal(17, finding["totalCves"]);
Assert.Equal(12, finding["kevCount"]);
Assert.Equal(5, ((Dictionary<string, object>)finding["severityCounts"])["critical"]);
}
[Fact]
public void BuildFinding_MarksSelfUpdatingSoftware_AndLeavesTheKeyOutOtherwise()
{
Assert.True(Finding(Vulns(1, 1, 7.5), selfUpdating: true)["selfUpdating"] is true);
Assert.False(Finding(Vulns(1, 1, 7.5)).ContainsKey("selfUpdating"));
}
// -----------------------------------------------------------------------
// RankFindings
// -----------------------------------------------------------------------
[Fact]
public void RankFindings_SoftwareThatNeedsAHandComesFirst_EachGroupByRisk()
{
var findings = new List<Dictionary<string, object>>
{
Finding(Vulns(1, 3, 9.8, kev: true), selfUpdating: true, software: "self-updating, exploited"),
Finding(Vulns(1, 1, 2.0), software: "manual, low"),
Finding(Vulns(1, 40, 9.8), software: "manual, critical, 40 CVEs"),
Finding(Vulns(1, 1, 7.5), selfUpdating: true, software: "self-updating, high"),
Finding(Vulns(1, 2, 7.5), software: "manual, high"),
Finding(Vulns(1, 2, 9.8), software: "manual, critical, 2 CVEs"),
Finding(Vulns(1, 1, 5.0, kev: true), software: "manual, exploited"),
};
var ranked = VulnerabilityCorrelationService.RankFindings(findings).Select(f => (string)f["softwareName"]);
Assert.Equal(new[]
{
"manual, exploited", // critical by being exploited; more exploited CVEs win a tie on risk
"manual, critical, 40 CVEs", // then more CVEs
"manual, critical, 2 CVEs",
"manual, high",
"manual, low",
"self-updating, exploited",
"self-updating, high",
}, ranked);
}
// -----------------------------------------------------------------------
// FitToRow
// -----------------------------------------------------------------------
private static int Serialized(Dictionary<string, object> report) => JsonConvert.SerializeObject(report).Length;
/// <summary><paramref name="entries"/> software entries, each listing <paramref name="cvesPerEntry"/> CVEs of a stated total of 100.</summary>
private static Dictionary<string, object> Report(int entries, int cvesPerEntry, int descriptionLength, bool untypedLists = false)
{
var findings = Enumerable.Range(1, entries).Select(n =>
{
var listed = Enumerable.Range(1, cvesPerEntry)
.Select(c => Vuln($"CVE-2026-{c:0000}", 8.8, descriptionLength: descriptionLength)).ToList();
var finding = new Dictionary<string, object>
{
["softwareName"] = $"Contoso Widget {n:00}",
["installedVersion"] = "3.2.0",
["riskLevel"] = "high",
["totalCves"] = 100,
["severityCounts"] = new Dictionary<string, object> { ["critical"] = 0, ["high"] = 100, ["medium"] = 0, ["low"] = 0 },
["kevCount"] = 0,
["attendCount"] = 0,
};
// A merged report carries its lists as List<object>.
finding["vulnerabilities"] = untypedLists ? listed.Cast<object>().ToList() : listed;
return finding;
}).ToList();
return new Dictionary<string, object>
{
["scan_summary"] = new Dictionary<string, object>
{
["total_cves_found"] = entries * 100,
["additional_findings_count"] = 7,
},
["findings"] = findings,
["not_evaluated"] = new List<Dictionary<string, object>>(),
["unmatched_software"] = new List<Dictionary<string, object>>(),
["matched_clean_software"] = new List<Dictionary<string, object>>
{
new() { ["softwareName"] = "Fabrikam Gadget", ["installedVersion"] = "1.0" },
},
};
}
private static List<Dictionary<string, object>> Findings(Dictionary<string, object> report) =>
(List<Dictionary<string, object>>)report["findings"];
private static Dictionary<string, object> Summary(Dictionary<string, object> report) =>
(Dictionary<string, object>)report["scan_summary"];
[Fact]
public void FitToRow_ReportThatFits_IsLeftAsItIs()
{
var report = Report(entries: 5, cvesPerEntry: 10, descriptionLength: 500);
var before = JsonConvert.SerializeObject(report);
VulnerabilityCorrelationService.FitToRow(report);
Assert.Equal(before, JsonConvert.SerializeObject(report));
Assert.False(Summary(report).ContainsKey("report_trimmed"));
}
[Theory]
[InlineData(false)]
[InlineData(true)]
public void FitToRow_ReportTooLarge_ShortensTheListsAndKeepsTheCounts(bool untypedLists)
{
var report = Report(entries: 50, cvesPerEntry: 100, descriptionLength: 500, untypedLists);
Assert.True(Serialized(report) > VulnerabilityCorrelationService.MaxReportChars);
VulnerabilityCorrelationService.FitToRow(report);
Assert.True(Serialized(report) <= VulnerabilityCorrelationService.MaxReportChars);
Assert.True(Summary(report)["report_trimmed"] is true);
// Shorter lists were enough: no entry is dropped and the summary still states the totals.
Assert.Equal(50, Findings(report).Count);
Assert.Equal(7, Summary(report)["additional_findings_count"]);
Assert.Equal(5000, Summary(report)["total_cves_found"]);
Assert.All(Findings(report), finding =>
{
Assert.Equal(100, finding["totalCves"]);
var listed = ListedIds(finding);
Assert.InRange(listed.Count, 1, 99);
Assert.Equal("CVE-2026-0001", listed[0]); // the lists are cut from their less urgent end
});
}
[Fact]
public void FitToRow_ShorterListsNotEnough_DropsEntriesFromTheEndAndCountsThem()
{
var report = Report(entries: 50, cvesPerEntry: 1, descriptionLength: 10_000);
Assert.True(Serialized(report) > VulnerabilityCorrelationService.MaxReportChars);
VulnerabilityCorrelationService.FitToRow(report);
var kept = Findings(report);
Assert.True(Serialized(report) <= VulnerabilityCorrelationService.MaxReportChars);
Assert.True(Summary(report)["report_trimmed"] is true);
Assert.InRange(kept.Count, 1, 49);
Assert.Equal(7 + (50 - kept.Count), Summary(report)["additional_findings_count"]);
Assert.Equal(
Enumerable.Range(1, kept.Count).Select(n => $"Contoso Widget {n:00}"),
kept.Select(f => (string)f["softwareName"]));
Assert.All(kept, finding => Assert.Equal(100, finding["totalCves"]));
Assert.Empty((List<Dictionary<string, object>>)report["matched_clean_software"]);
}
// -----------------------------------------------------------------------
// Placeholder versions
// -----------------------------------------------------------------------
[Theory]
[InlineData("AppX", "1.0.0.0", true)]
[InlineData("AppX", "1.0.0.1", true)]
[InlineData("appx", "1.0.0.0", true)]
[InlineData("AppX", "1.0.0.2", false)]
[InlineData("AppX", "11.2408.9.0", false)]
[InlineData("HKLM_64", "1.0.0.0", false)] // a classic installer may really ship 1.0.0.0
[InlineData("", "1.0.0.0", false)]
public void IsPlaceholderVersion_OnlyTheStubVersionsOfStoreApps(string registrySource, string version, bool placeholder)
{
Assert.Equal(placeholder, VulnerabilityCorrelationService.IsPlaceholderVersion(registrySource, version));
}
}
@@ -0,0 +1,107 @@
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Moq;
using static AutopilotMonitor.Functions.Tests.ReportFixture;
using static AutopilotMonitor.Functions.Tests.VulnerabilityScanRig;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="VulnerabilityReportWriter"/> is the one place a session's vulnerability result is
/// written, for the queue handler and the manual re-scan alike: the report row is replaced or
/// removed, and the CveIndex is told both the new findings and those of the report it replaces,
/// in the native shape the index reads.
/// </summary>
public class VulnerabilityReportWriterTests
{
private static Dictionary<string, object> Report(params string[] cveIds) => new()
{
["scan_summary"] = new Dictionary<string, object> { ["overall_risk"] = "high" },
["findings"] = new List<Dictionary<string, object>>
{
new()
{
["softwareName"] = "Contoso Widget",
["installedVersion"] = "3.2.0",
["vulnerabilities"] = cveIds.Select(id => new Dictionary<string, object>
{
["cveId"] = id, ["cvssScore"] = 9.8, ["isKev"] = true,
}).ToList(),
},
},
};
private static SessionResultRecorder Recorder() => new(new Mock<IVulnerabilityRepository>());
[Fact]
public async Task NewReport_ReplacesTheStoredOne_ThenTheIndexGetsBothSetsOfFindings()
{
var result = Recorder();
var report = Report("CVE-2026-0001", "CVE-2026-0003");
await result.Writer.StoreAsync(TenantId, SessionId, report, AsStored(Report("CVE-2026-0001", "CVE-2026-0002")));
Assert.Equal(new[] { "store report", "replace index" }, result.Steps);
Assert.Same(report, Assert.Single(result.Stored));
var index = Assert.Single(result.IndexUpdates);
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0003" }, CveIds(index.Findings!));
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
}
/// <summary>
/// The stored report comes back with Newtonsoft tokens inside. The index only reads native
/// dictionaries and a real <c>true</c>, so tokens would hide every row the old report had.
/// </summary>
[Fact]
public async Task PreviousReportFromStorage_ReachesTheIndexInItsNativeShape()
{
var result = Recorder();
await result.Writer.StoreAsync(TenantId, SessionId, Report("CVE-2026-0001"), AsStored(Report("CVE-2026-0002")));
var previous = Assert.Single(Assert.Single(result.IndexUpdates).Previous!);
var vuln = Assert.IsType<Dictionary<string, object>>(Assert.Single((IEnumerable<object>)previous["vulnerabilities"]));
Assert.Equal("CVE-2026-0002", vuln["cveId"]);
Assert.True(vuln["isKev"] is true);
Assert.Equal(9.8, vuln["cvssScore"]);
}
[Fact]
public async Task NoReportAnyMore_RemovesTheStoredOne_AndItsFindingsNameTheRowsToRemove()
{
var result = Recorder();
await result.Writer.StoreAsync(TenantId, SessionId, report: null, AsStored(Report("CVE-2026-0001", "CVE-2026-0002")));
Assert.Equal(new[] { "delete report", "replace index" }, result.Steps);
var index = Assert.Single(result.IndexUpdates);
Assert.Null(index.Findings);
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
}
[Fact]
public async Task FirstReport_HasNothingToCompareWith()
{
var result = Recorder();
await result.Writer.StoreAsync(TenantId, SessionId, Report("CVE-2026-0001"), previousReport: null);
Assert.Equal(new[] { "store report", "replace index" }, result.Steps);
var index = Assert.Single(result.IndexUpdates);
Assert.Equal(new[] { "CVE-2026-0001" }, CveIds(index.Findings!));
Assert.Null(index.Previous);
}
[Fact]
public async Task NeitherReportNorStoredOne_NeitherStoresNorDeletes()
{
var result = Recorder();
await result.Writer.StoreAsync(TenantId, SessionId, report: null, previousReport: null);
Assert.Equal(new[] { "replace index" }, result.Steps);
var index = Assert.Single(result.IndexUpdates);
Assert.Null(index.Findings);
Assert.Null(index.Previous);
}
}
@@ -0,0 +1,153 @@
using System.Net;
using System.Text.Json;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Deletion;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using Azure.Data.Tables;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using static AutopilotMonitor.Functions.Tests.ReportFixture;
using static AutopilotMonitor.Functions.Tests.VulnerabilityScanRig;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// The manual re-scan (<c>GET sessions/{id}/vulnerability-report?rescan=true</c>) writes through
/// the same writer as the queue handler: the stored report is replaced or removed and the
/// session's CveIndex rows follow it. It answers in the same call, so a product without CVE data
/// is reported as not evaluated right away, and mappings that cannot be read are a 503 that
/// changes nothing.
/// </summary>
[Collection(NvdTestState.Collection)]
public sealed class VulnerabilityRescanTests : IDisposable
{
public VulnerabilityRescanTests() => NvdTestState.Reset();
public void Dispose() => NvdTestState.Reset();
private sealed class Rig
{
private Rig(VulnerabilityScanRig scan)
{
Scan = scan;
Result = new SessionResultRecorder(scan.Repo);
var inventoryLoader = new Mock<IVulnerabilityInventoryLoader>();
// The re-scan reads every phase's inventory, whatever the WhiteGlove part.
inventoryLoader.Setup(l => l.LoadShutdownInventoryAsync(TenantId, SessionId, null, true, It.IsAny<CancellationToken>()))
.Returns(() => Task.FromResult<IReadOnlyList<Dictionary<string, object>>>(Inventory.ToList()));
var sessionRows = new Mock<ISessionDeletionInventoryReader>();
sessionRows.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new TableEntity(TenantId, SessionId));
Function = new GetVulnerabilityReportFunction(
NullLogger<GetVulnerabilityReportFunction>.Instance,
scan.Repo.Object,
Result.Sessions.Object,
scan.Service,
inventoryLoader.Object,
new SessionDeletionGuard(sessionRows.Object, NullLogger<SessionDeletionGuard>.Instance),
Result.Writer);
}
public static async Task<Rig> CreateAsync() => new(await VulnerabilityScanRig.CreateAsync());
public VulnerabilityScanRig Scan { get; }
public SessionResultRecorder Result { get; }
public GetVulnerabilityReportFunction Function { get; }
public List<Dictionary<string, object>> Inventory { get; } = new();
/// <summary>Runs the re-scan as an admin of the session's own tenant.</summary>
public async Task<(HttpResponseData Response, JsonElement Body)> RescanAsync()
{
var response = await Function.Run(EndpointHarness.Request(TenantId, queryString: "?rescan=true").Req, SessionId);
response.Body.Position = 0;
using var body = JsonDocument.Parse(response.Body);
return (response, body.RootElement.Clone());
}
}
private static Dictionary<string, object> Chrome(string version) => Installed("google", "Google Chrome", version);
[Fact]
public async Task Rescan_WithFindings_ReplacesTheStoredReportAndBringsTheIndexInLine()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001", "CVE-2026-0003");
rig.Inventory.Add(Chrome("120.0.6099.130"));
rig.Result.StoredReport = StoredChromeReport(phase: null, "CVE-2026-0001", "CVE-2026-0002");
var (response, body) = await rig.RescanAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.True(body.GetProperty("success").GetBoolean());
Assert.True(body.GetProperty("rescanned").GetBoolean());
Assert.Equal(SessionId, body.GetProperty("sessionId").GetString());
var finding = Assert.Single(body.GetProperty("report").GetProperty("findings").EnumerateArray());
Assert.Equal("120.0.6099.130", finding.GetProperty("installedVersion").GetString());
Assert.False(body.TryGetProperty("message", out _));
Assert.Equal(new[] { "store report", "replace index" }, rig.Result.Steps);
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0003" }, CveIds(index.Findings!));
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
}
[Fact]
public async Task Rescan_NothingFound_RemovesTheStoredReportAndItsIndexRows()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001", "CVE-2026-0002");
rig.Inventory.Add(Chrome("121.0.6167.85"));
rig.Result.StoredReport = StoredChromeReport(phase: null, "CVE-2026-0001", "CVE-2026-0002");
var (response, body) = await rig.RescanAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
Assert.False(body.TryGetProperty("report", out _));
Assert.True(body.GetProperty("rescanned").GetBoolean());
Assert.Equal(new[] { "delete report", "replace index" }, rig.Result.Steps);
var index = Assert.Single(rig.Result.IndexUpdates);
Assert.Null(index.Findings);
Assert.Equal(new[] { "CVE-2026-0001", "CVE-2026-0002" }, CveIds(index.Previous!));
}
[Fact]
public async Task Rescan_ProductWithoutCveData_StoresTheReportThatNamesItAndSaysSo()
{
var rig = await Rig.CreateAsync();
rig.Scan.ChromeHas("CVE-2026-0001");
rig.Inventory.AddRange(new[] { Chrome("120.0.6099.130"), Installed("contoso", "Contoso Widget", "3.2.0") });
var (response, body) = await rig.RescanAsync();
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var notEvaluated = Assert.Single(body.GetProperty("report").GetProperty("not_evaluated").EnumerateArray());
Assert.Equal("Contoso Widget", notEvaluated.GetProperty("softwareName").GetString());
Assert.Equal("cve_data_pending", notEvaluated.GetProperty("reason").GetString());
Assert.False(string.IsNullOrWhiteSpace(body.GetProperty("message").GetString()));
Assert.Equal(new[] { "store report", "replace index" }, rig.Result.Steps);
// The product is noted, so the nightly refresh fetches it and a later re-scan finds it.
Assert.Equal(WidgetCpe, Assert.Single(rig.Scan.PendingMarkers)["CpeUri"]);
}
[Fact]
public async Task Rescan_MappingsUnreadable_Answers503AndChangesNothing()
{
var rig = await Rig.CreateAsync();
rig.Scan.MappingFailure = new InvalidOperationException("storage down");
rig.Inventory.Add(Chrome("120.0.6099.130"));
rig.Result.StoredReport = StoredChromeReport(phase: null, "CVE-2026-0001");
var (response, _) = await rig.RescanAsync();
Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
Assert.Equal(Constants.ApiErrorCodes.ServiceUnavailable, EndpointHarness.ErrorCode(response));
Assert.Equal("300", Assert.Single(response.Headers.GetValues("Retry-After")));
Assert.Empty(rig.Result.Steps);
Assert.Empty(rig.Scan.InventoryWrites);
}
}
@@ -0,0 +1,932 @@
using System.Globalization;
using System.Net;
using System.Security.Claims;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Web;
using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Shared.Models.Deletion;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// <see cref="NvdApiClient"/> keeps its 429 cooldown and its request pacing in process-wide
/// statics. Every test class that sends through a real client joins <see cref="Collection"/>,
/// so those classes never run next to each other, and calls <see cref="Reset"/> around each test.
/// </summary>
internal static class NvdTestState
{
public const string Collection = "NvdApiClient static state";
public static void Reset()
{
NvdApiClient.ResetThrottleState();
SkipPacing();
}
// The client waits 2 s (8 s without an API key) after its previous request; a test would
// pay that wait for every request it sends.
public static void SkipPacing() => NvdApiClient.ResetPacing();
}
/// <summary>One request the NVD client sent, as the stub saw it.</summary>
internal sealed record NvdRequest(Uri Uri, string Cpe, int StartIndex, int ResultsPerPage);
/// <summary>Stands in for the NVD CVE API and records what was asked.</summary>
internal sealed class NvdStubHandler : HttpMessageHandler
{
private readonly Func<NvdRequest, CancellationToken, Task<HttpResponseMessage>> _respond;
public NvdStubHandler(Func<NvdRequest, HttpResponseMessage> respond)
: this((request, _) => Task.FromResult(respond(request)))
{
}
public NvdStubHandler(Func<NvdRequest, CancellationToken, Task<HttpResponseMessage>> respond) => _respond = respond;
public List<NvdRequest> Requests { get; } = new();
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var query = HttpUtility.ParseQueryString(request.RequestUri!.Query);
var seen = new NvdRequest(
request.RequestUri,
query["virtualMatchString"] ?? string.Empty,
int.Parse(query["startIndex"] ?? "-1", CultureInfo.InvariantCulture),
int.Parse(query["resultsPerPage"] ?? "-1", CultureInfo.InvariantCulture));
Requests.Add(seen);
// The client stamps its pacing clock before it sends, so clearing it here lets the next
// request of the same test go out at once.
NvdTestState.SkipPacing();
return await _respond(seen, cancellationToken);
}
}
/// <summary>NVD answers and clients over a stub.</summary>
internal static class NvdFixture
{
// NVD writes camelCase and leaves absent fields out.
private static readonly JsonSerializerOptions WireJson = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
public static NvdApiClient Client(HttpMessageHandler handler)
{
var adminConfig = new Mock<AdminConfigurationService>(
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
new MemoryCache(new MemoryCacheOptions()));
adminConfig.Setup(a => a.GetConfigurationAsync())
.ReturnsAsync(new AdminConfiguration { NvdApiKey = "synthetic-key" });
return new NvdApiClient(new HttpClient(handler), adminConfig.Object, NullLogger<NvdApiClient>.Instance);
}
public static HttpResponseMessage Page(int totalResults, IEnumerable<NvdVulnerabilityItem> items) =>
Json(JsonSerializer.Serialize(
new NvdCveResponse { TotalResults = totalResults, Vulnerabilities = items.ToList() }, WireJson));
public static HttpResponseMessage Page(params NvdVulnerabilityItem[] items) => Page(items.Length, items);
public static HttpResponseMessage Json(string body) => new(HttpStatusCode.OK)
{
Content = new StringContent(body, Encoding.UTF8, "application/json"),
};
public static HttpResponseMessage Status(HttpStatusCode status) => new(status);
/// <summary>A CVE published a month ago, so the cache budget's date window keeps it.</summary>
public static NvdVulnerabilityItem Cve(string id, double score, params NvdCpeMatch[] ranges) =>
Cve(id, score, DateTime.UtcNow.AddMonths(-1), ranges);
public static NvdVulnerabilityItem Cve(string id, double score, DateTime publishedUtc, params NvdCpeMatch[] ranges) => new()
{
Cve = new NvdCve
{
Id = id,
Published = publishedUtc.ToString("yyyy-MM-ddTHH:mm:ss.fff", CultureInfo.InvariantCulture),
Descriptions = { new NvdDescription { Lang = "en", Value = $"Synthetic description of {id}." } },
Metrics = new NvdMetrics
{
CvssMetricV31 =
{
new NvdCvssMetricV31
{
CvssData = new NvdCvssData
{
BaseScore = score,
BaseSeverity = Severity(score),
VectorString = "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H",
},
},
},
},
Configurations = { new NvdConfiguration { Nodes = { new NvdNode { Operator = "OR", CpeMatch = ranges.ToList() } } } },
},
};
/// <summary>A vulnerable range of <paramref name="productCpe"/> (version-less form) that ends before <paramref name="fixedIn"/>.</summary>
public static NvdCpeMatch Below(string productCpe, string fixedIn) => new()
{
Vulnerable = true,
Criteria = $"{productCpe}:*:*:*:*:*:*:*:*",
VersionEndExcluding = fixedIn,
};
/// <summary>
/// A vulnerable range that names <paramref name="productCpe"/> (version-less form) and no
/// version at all, which NVD means as every version; <paramref name="targetSw"/> is the
/// platform the range is for.
/// </summary>
public static NvdCpeMatch EveryVersion(string productCpe, string targetSw = "*") => new()
{
Vulnerable = true,
Criteria = $"{productCpe}:*:*:*:*:*:{targetSw}:*:*",
};
public static string Severity(double score) =>
score >= 9.0 ? "CRITICAL" : score >= 7.0 ? "HIGH" : score >= 4.0 ? "MEDIUM" : "LOW";
}
/// <summary>Rows of the VulnerabilityCache table as the repository hands them out.</summary>
internal static class VulnerabilityCacheRows
{
public static Dictionary<string, object> Mapping(
string vendor, string product, string cpeUri, params string[] displayNamePatterns) => new()
{
["PartitionKey"] = CpeMappingStore.SeedPartition,
["RowKey"] = $"{vendor}:{product}",
["NormalizedVendor"] = vendor,
["NormalizedProduct"] = product,
["CpeUri"] = cpeUri,
["DisplayNamePatternsJson"] = JsonSerializer.Serialize(displayNamePatterns),
};
/// <summary>
/// A mapping row outside the seed: an operator's (<c>manual</c>), an automatic one
/// (<c>auto</c>) or one of the community list, keyed the way the writers of those rows key them.
/// </summary>
public static Dictionary<string, object> MappingIn(
string partition, string source, string vendor, string product, string cpeUri, params string[] displayNamePatterns)
{
var row = Mapping(vendor, product, cpeUri, displayNamePatterns);
row["PartitionKey"] = partition;
row["RowKey"] = CpeSeedFile.RowKeyOf(vendor, product);
row["Source"] = source;
return row;
}
public static Dictionary<string, object> SelfUpdating(string productKey) => new()
{
["PartitionKey"] = CpeMappingStore.ProductFlagsPartition,
["RowKey"] = productKey.Replace(':', '_'),
["Product"] = productKey,
["SeedSelfUpdating"] = true,
};
public static Dictionary<string, object> Filled(
string cpeUri, IEnumerable<CachedCve> cves, DateTime cachedAtUtc, int nvdTotal = 0) =>
WithKeys(CveCacheFiller.BuildRow(cpeUri, cves.ToList(), cachedAtUtc, nvdTotal), cpeUri);
/// <summary>A row as it was written before the shape was versioned.</summary>
public static Dictionary<string, object> OlderShape(string cpeUri, IEnumerable<CachedCve> cves, DateTime cachedAtUtc)
{
var row = Filled(cpeUri, cves, cachedAtUtc);
row.Remove("SchemaVersion");
row.Remove("State");
row.Remove("NvdTotal");
return row;
}
public static Dictionary<string, object> Pending(string cpeUri, DateTime requestedAtUtc) => WithKeys(new Dictionary<string, object>
{
["CpeUri"] = cpeUri,
["State"] = CveCacheFiller.StatePending,
["RequestedAt"] = requestedAtUtc.ToString("o"),
["SchemaVersion"] = CveCacheFiller.SchemaVersion,
}, cpeUri);
public static Dictionary<string, object> WithKeys(Dictionary<string, object> row, string cpeUri)
{
row["PartitionKey"] = CveCacheFiller.CacheKey(cpeUri);
row["RowKey"] = CveCacheFiller.RowKey;
return row;
}
/// <summary>A cached CVE of <paramref name="productCpe"/> that affects every version below <paramref name="fixedIn"/>.</summary>
public static CachedCve Cve(string id, double score, string productCpe, string fixedIn) => new()
{
CveId = id,
CvssScore = score,
CvssSeverity = NvdFixture.Severity(score),
Description = $"Synthetic description of {id}.",
PublishedDate = "2026-06-01T00:00:00.000",
AffectedVersions =
{
new CachedVersionRange { Criteria = $"{productCpe}:*:*:*:*:*:*:*:*", VersionEndExcluding = fixedIn },
},
};
/// <summary>A cached CVE of <paramref name="productCpe"/> whose range names no version at all: it affects every install.</summary>
public static CachedCve CveOfEveryVersion(string id, double score, string productCpe) => new()
{
CveId = id,
CvssScore = score,
CvssSeverity = NvdFixture.Severity(score),
Description = $"Synthetic description of {id}.",
PublishedDate = "2026-06-01T00:00:00.000",
AffectedVersions =
{
new CachedVersionRange { Criteria = $"{productCpe}:*:*:*:*:*:*:*:*" },
},
};
}
/// <summary>
/// The KEV catalog and the MSRC index over mocked blob storage and a stub HTTP handler: nothing
/// leaves the process.
/// </summary>
internal static class ReferenceDataFixture
{
public static BlobStorageService BlobStorage(Mock<BlobClient>? blob = null)
{
if (blob == null)
{
blob = new Mock<BlobClient>();
blob.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(Response.FromValue(false, Mock.Of<Response>()));
}
var container = new Mock<BlobContainerClient>();
container.Setup(c => c.GetBlobClient(It.IsAny<string>())).Returns(blob.Object);
var service = new Mock<BlobServiceClient>();
service.Setup(s => s.GetBlobContainerClient(It.IsAny<string>())).Returns(container.Object);
return new BlobStorageService(service.Object, NullLogger<BlobStorageService>.Instance);
}
public static KevDataService Kev(HttpMessageHandler handler, Mock<BlobClient>? blob = null) =>
new(new HttpClient(handler), BlobStorage(blob), NullLogger<KevDataService>.Instance);
public static MsrcCorrelationService Msrc() => new(
new MsrcApiClient(new HttpClient(new FailingHandler()), null, NullLogger<MsrcApiClient>.Instance),
BlobStorage(),
NullLogger<MsrcCorrelationService>.Instance);
/// <summary>The CISA feed with one entry per CVE id.</summary>
public static string KevCatalogJson(params string[] cveIds) => JsonSerializer.Serialize(new
{
title = "CISA Catalog of Known Exploited Vulnerabilities",
catalogVersion = "2026.11.12",
count = cveIds.Length,
vulnerabilities = cveIds.Select(id => new
{
cveID = id,
vendorProject = "Contoso",
product = "Widget",
dateAdded = "2026-10-01",
requiredAction = "Apply mitigations per vendor instructions.",
dueDate = "2026-10-22",
knownRansomwareCampaignUse = "Unknown",
}),
});
/// <summary>Answers every request with a server error, for clients a test must not need.</summary>
internal sealed class FailingHandler : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError));
}
/// <summary>Answers every request with the same body.</summary>
internal sealed class FixedHandler : HttpMessageHandler
{
private readonly string _body;
public FixedHandler(string body) => _body = body;
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) =>
Task.FromResult(NvdFixture.Json(_body));
}
}
/// <summary>
/// A <see cref="VulnerabilityCorrelationService"/> with everything real except storage and HTTP:
/// the mapping store and the cache filler run over one mocked repository whose cache rows live in
/// <see cref="CacheRows"/>, NVD is a stub, and the KEV catalog holds the CVEs a test names.
/// </summary>
internal sealed class VulnerabilityScanRig
{
public const string TenantId = "11111111-1111-1111-1111-111111111111";
public const string SessionId = "22222222-2222-2222-2222-222222222222";
public const string ChromeCpe = "cpe:2.3:a:google:chrome";
public const string SevenZipCpe = "cpe:2.3:a:7-zip:7-zip";
public const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private VulnerabilityScanRig(string[] kevListedCves)
{
Repo.Setup(r => r.GetAllCpeMappingsAsync()).Returns(() => MappingFailure != null
? Task.FromException<List<Dictionary<string, object>>>(MappingFailure)
: Task.FromResult(Mappings.ToList()));
Repo.Setup(r => r.GetCpeMappingsByPartitionAsync(It.IsAny<string>())).Returns<string>(partition =>
Task.FromResult(partition == CpeMappingStore.ProductFlagsPartition
? ProductFlags.ToList()
: new List<Dictionary<string, object>>()));
Repo.Setup(r => r.GetCveCacheEntryAsync(It.IsAny<string>(), CveCacheFiller.RowKey))
.Returns<string, string>((cacheKey, _) =>
{
CacheReads.Add(cacheKey);
return Task.FromResult(CacheRows.TryGetValue(cacheKey, out var row) ? row : null);
});
// The index is the same rows without their payload; handing out whole rows is a superset.
Repo.Setup(r => r.GetCveCacheIndexAsync()).Returns(() => Task.FromResult(CacheRows.Values.ToList()));
Repo.Setup(r => r.UpsertCveCacheEntryAsync(It.IsAny<string>(), CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()))
.Returns<string, string, Dictionary<string, object>>((cacheKey, rowKey, row) =>
{
CacheWrites.Add(cacheKey);
CacheRows[cacheKey] = new Dictionary<string, object>(row) { ["PartitionKey"] = cacheKey, ["RowKey"] = rowKey };
return Task.CompletedTask;
});
Repo.Setup(r => r.TryAddCveCacheEntryAsync(It.IsAny<string>(), CveCacheFiller.RowKey, It.IsAny<Dictionary<string, object>>()))
.Returns<string, string, Dictionary<string, object>>((cacheKey, rowKey, row) =>
{
var stored = new Dictionary<string, object>(row) { ["PartitionKey"] = cacheKey, ["RowKey"] = rowKey };
PendingMarkers.Add(stored);
return Task.FromResult(CacheRows.TryAdd(cacheKey, stored));
});
Repo.Setup(r => r.DeleteCveCacheEntryAsync(It.IsAny<string>(), CveCacheFiller.RowKey))
.Returns<string, string>((cacheKey, _) =>
{
CacheDeletes.Add(cacheKey);
CacheRows.Remove(cacheKey);
return Task.CompletedTask;
});
Repo.Setup(r => r.UpsertSessionInventoryContributionsAsync(
TenantId, SessionId, It.IsAny<IReadOnlyList<SoftwareInventoryItem>>(), It.IsAny<CancellationToken>()))
.Returns<string, string, IReadOnlyList<SoftwareInventoryItem>, CancellationToken>((_, _, items, _) =>
{
InventoryWrites.Add(items);
return Task.FromResult(new InventoryContributionsDelta());
});
Nvd = new NvdStubHandler(request => NvdAnswer(request));
MappingStore = new CpeMappingStore(Repo.Object, NullLogger<CpeMappingStore>.Instance);
Filler = new CveCacheFiller(Repo.Object, NvdFixture.Client(Nvd), NullLogger<CveCacheFiller>.Instance);
Kev = ReferenceDataFixture.Kev(new ReferenceDataFixture.FixedHandler(ReferenceDataFixture.KevCatalogJson(kevListedCves)));
Msrc = ReferenceDataFixture.Msrc();
Service = new VulnerabilityCorrelationService(
Repo.Object, Filler, Kev, Msrc, telemetryClient: null,
NullLogger<VulnerabilityCorrelationService>.Instance, MappingStore);
}
/// <summary>
/// Builds the rig with the reference data loaded, as the cache warmer leaves it at start:
/// the KEV catalog lists <paramref name="kevListedCves"/>, the MSRC index is empty.
/// </summary>
public static async Task<VulnerabilityScanRig> CreateAsync(params string[] kevListedCves)
{
var rig = new VulnerabilityScanRig(kevListedCves);
await rig.Msrc.EnsureHydratedAsync();
if (kevListedCves.Length > 0)
Assert.True(await rig.Kev.RefreshAsync());
else
await rig.Kev.EnsureHydratedAsync();
return rig;
}
public Mock<IVulnerabilityRepository> Repo { get; } = new();
/// <summary>Mapping rows; Chrome, 7-Zip and the Contoso widget unless a test replaces them.</summary>
public List<Dictionary<string, object>> Mappings { get; } = new()
{
VulnerabilityCacheRows.Mapping("google", "google chrome", ChromeCpe, "google chrome"),
VulnerabilityCacheRows.Mapping("7-zip", "7-zip", SevenZipCpe, "7-zip"),
VulnerabilityCacheRows.Mapping("contoso", "contoso widget", WidgetCpe, "contoso widget"),
};
public List<Dictionary<string, object>> ProductFlags { get; } = new();
/// <summary>When set, reading the mappings fails with it.</summary>
public Exception? MappingFailure { get; set; }
/// <summary>The per-product cache rows by partition key; writes of the code under test land here.</summary>
public Dictionary<string, Dictionary<string, object>> CacheRows { get; } = new(StringComparer.Ordinal);
public List<string> CacheReads { get; } = new();
public List<string> CacheWrites { get; } = new();
public List<string> CacheDeletes { get; } = new();
public List<Dictionary<string, object>> PendingMarkers { get; } = new();
/// <summary>The typed inventory handed to the contributions side-row, one entry per run.</summary>
public List<IReadOnlyList<SoftwareInventoryItem>> InventoryWrites { get; } = new();
public NvdStubHandler Nvd { get; }
/// <summary>What NVD answers; a server error unless a test says otherwise.</summary>
public Func<NvdRequest, HttpResponseMessage> NvdAnswer { get; set; } =
_ => NvdFixture.Status(HttpStatusCode.InternalServerError);
public CpeMappingStore MappingStore { get; }
public CveCacheFiller Filler { get; }
public KevDataService Kev { get; }
public MsrcCorrelationService Msrc { get; }
public VulnerabilityCorrelationService Service { get; }
public void Cache(Dictionary<string, object> row) => CacheRows[(string)row["PartitionKey"]] = row;
/// <summary>Caches CVE data for Chrome that reports <paramref name="cveIds"/> for every version below 121.</summary>
public void ChromeHas(params string[] cveIds) => Cache(VulnerabilityCacheRows.Filled(
ChromeCpe, cveIds.Select(id => VulnerabilityCacheRows.Cve(id, 8.8, ChromeCpe, "121.0")), DateTime.UtcNow.AddDays(-1)));
public Task<VulnerabilityCorrelationOutcome> CorrelateAsync(
IEnumerable<Dictionary<string, object>> inventory, CancellationToken ct = default) =>
Service.CorrelateAsync(SessionId, TenantId, inventory.ToList(), ct);
/// <summary>One installed program as the inventory events carry it.</summary>
public static Dictionary<string, object> Installed(
string vendor, string displayName, string version, string registrySource = "HKLM_64") => new()
{
["normalizedPublisher"] = vendor,
["normalizedName"] = displayName.ToLowerInvariant(),
["normalizedVersion"] = version,
["displayName"] = displayName,
["publisher"] = vendor,
["registrySource"] = registrySource,
};
}
/// <summary>
/// Stands in for the two places a session's result is written, the report row and the CveIndex,
/// and keeps what was written in order. The session is the one of <see cref="VulnerabilityScanRig"/>.
/// </summary>
internal sealed class SessionResultRecorder
{
public SessionResultRecorder(Mock<IVulnerabilityRepository> reports)
{
const string tenant = VulnerabilityScanRig.TenantId;
const string session = VulnerabilityScanRig.SessionId;
reports.Setup(r => r.GetVulnerabilityReportAsync(tenant, session)).Returns(() => Task.FromResult(StoredReport));
reports.Setup(r => r.StoreVulnerabilityReportAsync(tenant, session, It.IsAny<Dictionary<string, object>>()))
.Returns<string, string, Dictionary<string, object>>((_, _, report) =>
{
Steps.Add("store report");
Stored.Add(report);
return Task.CompletedTask;
});
reports.Setup(r => r.DeleteVulnerabilityReportAsync(tenant, session))
.Returns(() =>
{
Steps.Add("delete report");
return Task.CompletedTask;
});
Sessions.Setup(s => s.ReplaceCveIndexEntriesAsync(
tenant, session, It.IsAny<List<Dictionary<string, object>>?>(), It.IsAny<List<Dictionary<string, object>>?>()))
.Returns<string, string, List<Dictionary<string, object>>?, List<Dictionary<string, object>>?>((_, _, findings, previous) =>
{
Steps.Add("replace index");
IndexUpdates.Add((findings, previous));
return Task.CompletedTask;
});
Writer = new VulnerabilityReportWriter(reports.Object, Sessions.Object);
}
public Mock<ISessionRepository> Sessions { get; } = new();
public VulnerabilityReportWriter Writer { get; }
/// <summary>The report an earlier run stored, in the shape the table gives it back.</summary>
public Dictionary<string, object>? StoredReport { get; set; }
/// <summary>Everything written for the session, in order.</summary>
public List<string> Steps { get; } = new();
public List<Dictionary<string, object>> Stored { get; } = new();
public List<(List<Dictionary<string, object>>? Findings, List<Dictionary<string, object>>? Previous)> IndexUpdates { get; } = new();
}
/// <summary>Reports as they come back from storage, and what tests read out of them.</summary>
internal static class ReportFixture
{
/// <summary>The report as its table row gives it back: every nested value is a Newtonsoft token.</summary>
public static Dictionary<string, object> AsStored(Dictionary<string, object> report) =>
Newtonsoft.Json.JsonConvert.DeserializeObject<Dictionary<string, object>>(
Newtonsoft.Json.JsonConvert.SerializeObject(report))!;
/// <summary>A stored report with one entry for Chrome 119 that lists <paramref name="cveIds"/>.</summary>
public static Dictionary<string, object> StoredChromeReport(string? phase, params string[] cveIds)
{
var vulns = cveIds.Select(id => new Dictionary<string, object>
{
["cveId"] = id,
["cvssScore"] = 8.8,
["cvssSeverity"] = "HIGH",
["isKev"] = false,
["priority"] = CvePriority.Track,
}).ToList();
var finding = VulnerabilityCorrelationService.BuildFinding(
"Google Chrome", "119.0.6045.200", VulnerabilityScanRig.ChromeCpe, "high", selfUpdating: false, vulns);
if (phase != null) finding["phase"] = phase;
return AsStored(new Dictionary<string, object>
{
["scan_summary"] = new Dictionary<string, object>
{
["total_software_scanned"] = 1,
["matched_to_cpe"] = 1,
["total_cves_found"] = cveIds.Length,
["overall_risk"] = "high",
},
["findings"] = new List<Dictionary<string, object>> { finding },
});
}
/// <summary>The CVE ids the entries list, in order.</summary>
public static IEnumerable<string> CveIds(IEnumerable<Dictionary<string, object>> findings) => findings
.SelectMany(f => ((IEnumerable<object>)f["vulnerabilities"]).Cast<Dictionary<string, object>>())
.Select(v => (string)v["cveId"]);
}
/// <summary>
/// The <c>cpe_*</c> partitions of the VulnerabilityCache table (mappings, ignore list, product
/// flags) and the SoftwareInventory table in memory, behind a mocked repository. A write to a
/// partition replaces its row as a whole, as the table does in replace mode: a column that is
/// not passed again is gone. Rows are handed out as copies, so only a write changes the table.
/// </summary>
internal sealed class MappingTables
{
private readonly object _gate = new();
private readonly SortedDictionary<string, SortedDictionary<string, Dictionary<string, object>>> _partitions = new(StringComparer.Ordinal);
private readonly List<Dictionary<string, object>> _inventory = new();
public MappingTables() : this(new Mock<IVulnerabilityRepository>())
{
}
/// <summary>Takes over the mapping and inventory calls of <paramref name="repo"/>; what it answered for them before no longer applies.</summary>
public MappingTables(Mock<IVulnerabilityRepository> repo)
{
Repo = repo;
// The table repository reads the mappings with a range over the partition key: every
// partition that starts with cpe_map, which leaves the ignore list and the flags out.
repo.Setup(r => r.GetAllCpeMappingsAsync())
.Returns(() => ReadAsync(partition => partition.StartsWith("cpe_map", StringComparison.Ordinal)));
repo.Setup(r => r.GetCpeMappingsByPartitionAsync(It.IsAny<string>()))
.Returns<string>(wanted => ReadAsync(partition => partition == wanted));
repo.Setup(r => r.UpsertCpeMappingAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Dictionary<string, object>>()))
.Returns<string, string, Dictionary<string, object>>((partition, rowKey, properties) =>
{
lock (_gate)
{
var write = $"upsert {partition}/{rowKey}";
if (WriteFailure?.Invoke(write) is { } failure) return Task.FromException(failure);
Writes.Add(write);
Store(partition, rowKey, properties);
}
return Task.CompletedTask;
});
repo.Setup(r => r.DeleteCpeMappingAsync(It.IsAny<string>(), It.IsAny<string>()))
.Returns<string, string>((partition, rowKey) =>
{
lock (_gate)
{
var write = $"delete {partition}/{rowKey}";
if (WriteFailure?.Invoke(write) is { } failure) return Task.FromException(failure);
Writes.Add(write);
if (_partitions.TryGetValue(partition, out var rows)) rows.Remove(rowKey);
}
return Task.CompletedTask;
});
repo.Setup(r => r.GetAllSoftwareInventoryAsync(It.IsAny<IEnumerable<string>?>()))
.Returns<IEnumerable<string>?>(columns =>
{
List<Dictionary<string, object>> rows;
lock (_gate) rows = _inventory.Select(row => Project(row, columns)).ToList();
AfterInventoryRead?.Invoke();
return Task.FromResult(rows);
});
repo.Setup(r => r.SetSoftwareInventoryCpeAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
.Returns<string, string, string>((tenantId, rowKey, cpeUri) =>
{
lock (_gate)
{
if (InventoryWriteFailure?.Invoke(rowKey) is { } failure) return Task.FromException<bool>(failure);
var row = _inventory.FirstOrDefault(r =>
string.Equals((string)r["PartitionKey"], tenantId, StringComparison.Ordinal)
&& string.Equals((string)r["RowKey"], rowKey, StringComparison.Ordinal));
if (row == null) return Task.FromResult(false);
// A merge of the one column: everything else on the row stays.
row["CpeUri"] = cpeUri;
InventoryWrites.Add(rowKey);
return Task.FromResult(true);
}
});
}
public Mock<IVulnerabilityRepository> Repo { get; }
/// <summary>Every write to a <c>cpe_*</c> partition that went through, in order: <c>upsert partition/rowKey</c> or <c>delete partition/rowKey</c>.</summary>
public List<string> Writes { get; } = new();
/// <summary>Decides per write, named as in <see cref="Writes"/>, whether it fails, and with what.</summary>
public Func<string, Exception?>? WriteFailure { get; set; }
/// <summary>When set, reading any <c>cpe_*</c> partition fails with it.</summary>
public Exception? ReadFailure { get; set; }
/// <summary>Row keys of the inventory rows whose CPE was written, in the order the writes arrived.</summary>
public List<string> InventoryWrites { get; } = new();
/// <summary>Runs after the inventory was read, before the reader gets its rows: the place to change the table behind a reader's back.</summary>
public Action? AfterInventoryRead { get; set; }
/// <summary>Decides per row key whether writing its CPE fails, and with what.</summary>
public Func<string, Exception?>? InventoryWriteFailure { get; set; }
/// <summary>Stores a row as the test's arrangement; it does not show up in <see cref="Writes"/>.</summary>
public void Put(string partition, string rowKey, Dictionary<string, object> properties)
{
lock (_gate) Store(partition, rowKey, properties);
}
/// <summary>Stores a row that carries its own keys, as <see cref="VulnerabilityCacheRows"/> builds them.</summary>
public void Put(Dictionary<string, object> row) => Put((string)row["PartitionKey"], (string)row["RowKey"], row);
/// <summary>The rows of one partition in row key order.</summary>
public List<Dictionary<string, object>> Rows(string partition)
{
lock (_gate)
{
return _partitions.TryGetValue(partition, out var rows)
? rows.Values.Select(row => new Dictionary<string, object>(row)).ToList()
: new List<Dictionary<string, object>>();
}
}
public Dictionary<string, object>? Row(string partition, string rowKey) =>
Rows(partition).SingleOrDefault(row => (string)row["RowKey"] == rowKey);
public IEnumerable<string> RowKeys(string partition) => Rows(partition).Select(row => (string)row["RowKey"]);
/// <summary>
/// Adds one inventory row and returns it; the returned row is the stored one, so a test reads
/// the CPE the code under test left on it, or takes a column away before the run.
/// </summary>
public Dictionary<string, object> AddInventory(
string vendor, string displayName, string version = "1.0", int sessions = 1, string cpeUri = "",
string? publisher = null, DateTime? lastSeenUtc = null, string? lastSessionId = null,
string tenantId = VulnerabilityScanRig.TenantId, string registrySource = "HKLM_64")
{
var row = new Dictionary<string, object>
{
["PartitionKey"] = tenantId,
["RowKey"] = $"{vendor}:{displayName.ToLowerInvariant()}:{version}",
["DisplayName"] = displayName,
["NormalizedName"] = displayName.ToLowerInvariant(),
["NormalizedVendor"] = vendor,
["NormalizedVersion"] = version,
["Publisher"] = publisher ?? vendor,
["SessionCount"] = sessions,
["LastSeenAt"] = (lastSeenUtc ?? DateTime.UtcNow).ToString("o"),
["LastSessionId"] = lastSessionId ?? VulnerabilityScanRig.SessionId,
["CpeUri"] = cpeUri,
["RegistrySource"] = registrySource,
};
lock (_gate) _inventory.Add(row);
return row;
}
/// <summary>Takes an inventory row out of the table, as the session cleanup does when its last session went.</summary>
public void RemoveInventory(Dictionary<string, object> row)
{
lock (_gate) _inventory.Remove(row);
}
private void Store(string partition, string rowKey, Dictionary<string, object> properties)
{
if (!_partitions.TryGetValue(partition, out var rows))
_partitions[partition] = rows = new SortedDictionary<string, Dictionary<string, object>>(StringComparer.Ordinal);
rows[rowKey] = new Dictionary<string, object>(properties) { ["PartitionKey"] = partition, ["RowKey"] = rowKey };
}
private Task<List<Dictionary<string, object>>> ReadAsync(Func<string, bool> partitionWanted)
{
lock (_gate)
{
if (ReadFailure != null) return Task.FromException<List<Dictionary<string, object>>>(ReadFailure);
return Task.FromResult(_partitions
.Where(partition => partitionWanted(partition.Key))
.SelectMany(partition => partition.Value.Values)
.Select(row => new Dictionary<string, object>(row))
.ToList());
}
}
// A projected read hands out the named columns only; a row that lacks one simply has no entry for it.
private static Dictionary<string, object> Project(Dictionary<string, object> row, IEnumerable<string>? columns) =>
columns == null
? new Dictionary<string, object>(row)
: columns.Where(row.ContainsKey).ToDictionary(column => column, column => row[column]);
}
/// <summary>Mapping list files as the rules repository serves them.</summary>
internal static class MappingListFixture
{
private static readonly JsonSerializerOptions FileJson = new(JsonSerializerDefaults.Web);
/// <summary>An entry that maps the program by its key and by its name as a pattern.</summary>
public static CpeSeedEntry Entry(string vendor, string product, string cpeUri) => new()
{
NormalizedVendor = vendor,
NormalizedProduct = product,
CpeUri = cpeUri,
DisplayNamePatterns = new List<string> { product },
};
/// <summary><paramref name="count"/> entries of different programs, each with a product of its own.</summary>
public static IEnumerable<CpeSeedEntry> Tools(int count) => Enumerable.Range(0, count)
.Select(n => Entry("contoso", $"contoso tool {(char)('a' + n)}", $"cpe:2.3:a:contoso:tool_{(char)('a' + n)}"));
public static string Json(IEnumerable<CpeSeedEntry> mappings, params string[] selfUpdatingProducts) =>
JsonSerializer.Serialize(new CpeSeedDocument
{
Version = "2.0.0",
LastUpdated = "2026-11-12",
SelfUpdatingProducts = selfUpdatingProducts.ToList(),
Mappings = mappings.ToList(),
}, FileJson);
}
/// <summary>
/// Stands in for the rules repository on GitHub and serves the two mapping list files; a list
/// that is not set is not there (404).
/// </summary>
internal sealed class RulesRepositoryStub : HttpMessageHandler
{
public const string SeedListPath = "/rules/dist/cpe-mappings.json";
public const string CommunityListPath = "/rules/dist/cpe-community-mappings.json";
public string? SeedList { get; set; }
public string? CommunityList { get; set; }
/// <summary>When set, every request fails with it before an answer comes.</summary>
public Exception? Failure { get; set; }
public List<string> RequestedPaths { get; } = new();
/// <summary>The repository client over this stub, pointed at a synthetic address.</summary>
public GitHubRuleRepository Repository() => new(
new HttpClient(this),
new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?> { ["GitHub:RulesBaseUrl"] = "https://rules.contoso.example/rules" })
.Build(),
NullLogger<GitHubRuleRepository>.Instance);
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
var path = request.RequestUri!.AbsolutePath;
RequestedPaths.Add(path);
if (Failure != null) throw Failure;
var body = path switch
{
SeedListPath => SeedList,
CommunityListPath => CommunityList,
_ => null,
};
return Task.FromResult(body == null ? new HttpResponseMessage(HttpStatusCode.NotFound) : NvdFixture.Json(body));
}
}
/// <summary>A blob that keeps what was uploaded to it and hands it out again; without content it does not exist.</summary>
internal sealed class StoredBlob
{
public StoredBlob(string? content = null)
{
Content = content;
Client.Setup(b => b.ExistsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(() => Response.FromValue(Content != null, Mock.Of<Response>()));
Client.Setup(b => b.DownloadContentAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync(() =>
{
Downloads++;
return Response.FromValue(
BlobsModelFactory.BlobDownloadResult(content: BinaryData.FromString(Content!)), Mock.Of<Response>());
});
Client.Setup(b => b.UploadAsync(It.IsAny<Stream>(), true, It.IsAny<CancellationToken>()))
.Returns<Stream, bool, CancellationToken>((content, _, _) =>
{
Uploads++;
using var reader = new StreamReader(content, Encoding.UTF8, leaveOpen: true);
Content = reader.ReadToEnd();
return Task.FromResult(Mock.Of<Response<BlobContentInfo>>());
});
}
public Mock<BlobClient> Client { get; } = new();
public string? Content { get; set; }
public int Uploads { get; private set; }
public int Downloads { get; private set; }
}
/// <summary>The reduced NVD product dictionary as its blob holds it, and services over it.</summary>
internal static class CpeDictionaryFixture
{
/// <summary>The stored snapshot: products by <c>vendor:product</c> with their title.</summary>
public static string SnapshotJson(
IEnumerable<(string ProductKey, string Title)> products, string? feedVersion = null, string lastRefresh = "2026-11-11T03:00:00.0000000Z") =>
JsonSerializer.Serialize(new BlobBackedReferenceCache<CpeDictionaryProduct>.Snapshot
{
LastRefresh = lastRefresh,
CoveredDocuments = feedVersion == null ? new List<string>() : new List<string> { feedVersion },
Entries = products.ToDictionary(p => p.ProductKey, p => new CpeDictionaryProduct { T = p.Title }, StringComparer.Ordinal),
});
public static CpeDictionaryService Service(HttpMessageHandler nvdFeed, StoredBlob blob) =>
new(new HttpClient(nvdFeed), ReferenceDataFixture.BlobStorage(blob.Client), NullLogger<CpeDictionaryService>.Instance);
/// <summary>
/// A service whose stored dictionary holds <paramref name="products"/>; without any, no
/// dictionary was ever stored. The feed is not reachable: a test that reads the stored copy
/// must not need it.
/// </summary>
public static CpeDictionaryService Stored(params (string ProductKey, string Title)[] products) =>
Service(new ReferenceDataFixture.FailingHandler(), new StoredBlob(products.Length == 0 ? null : SnapshotJson(products)));
}
/// <summary>
/// The mapping maintenance with everything real except storage and HTTP: reconciler, product
/// flags and automatic mapper run over the repository, mapping store and cache filler of one
/// <see cref="VulnerabilityScanRig"/>. The mappings, the ignore list, the flags and the inventory
/// live in <see cref="Tables"/>; the CVE cache rows and the NVD stub stay with <see cref="Scan"/>.
/// </summary>
internal sealed class MappingRig
{
private MappingRig(VulnerabilityScanRig scan, IEnumerable<Dictionary<string, object>> mappings)
{
Scan = scan;
Tables = new MappingTables(scan.Repo);
foreach (var mapping in mappings) Tables.Put(mapping);
Flags = new CpeProductFlagsService(scan.Repo.Object, scan.MappingStore);
Reconciler = new SoftwareInventoryCpeReconciler(
scan.Repo.Object, scan.MappingStore, scan.Filler, NullLogger<SoftwareInventoryCpeReconciler>.Instance);
}
/// <summary>A rig whose mappings in service are exactly <paramref name="mappings"/>.</summary>
public static async Task<MappingRig> CreateAsync(params Dictionary<string, object>[] mappings) =>
new(await VulnerabilityScanRig.CreateAsync(), mappings);
public VulnerabilityScanRig Scan { get; }
public MappingTables Tables { get; }
public CpeMappingStore Store => Scan.MappingStore;
public CpeProductFlagsService Flags { get; }
public SoftwareInventoryCpeReconciler Reconciler { get; }
/// <summary>The automatic mapper; without <paramref name="runBudget"/> a nightly pass has the time it has in service.</summary>
public CpeDictionaryResolver Resolver(CpeDictionaryService dictionary, TimeSpan? runBudget = null) => runBudget is { } budget
? new(dictionary, Store, Scan.Repo.Object, Scan.Filler, NullLogger<CpeDictionaryResolver>.Instance) { RunBudget = budget }
: new(dictionary, Store, Scan.Repo.Object, Scan.Filler, NullLogger<CpeDictionaryResolver>.Instance);
}
/// <summary>Requests to the mapping endpoints as the signed-in operator, and what a function answered.</summary>
internal static class OperatorCall
{
public const string Upn = "admin@contoso.com";
/// <summary>A request whose body is <paramref name="body"/> as the portal sends it; without one, a request without a body.</summary>
public static HttpRequestData Request(object? body = null) =>
RequestWithJson(body == null ? null : TestWire.Serialize(body));
public static HttpRequestData RequestWithJson(string? json)
{
var (request, context) = EndpointHarness.Request(VulnerabilityScanRig.TenantId, Upn, json);
// The functions take the operator's name from the principal the authentication step left.
context.Items["ClaimsPrincipal"] = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim("upn", Upn) }, "test"));
return request;
}
public static JsonElement BodyOf(HttpResponseData response)
{
response.Body.Position = 0;
using var document = JsonDocument.Parse(response.Body);
return document.RootElement.Clone();
}
}
@@ -410,4 +410,202 @@ public class VulnerabilitySummaryAggregatorTests
[Fact]
public void DisplayName_OfNoTitlesIsEmpty()
=> Assert.Equal("", VulnerabilitySummaryAggregator.DisplayName(Array.Empty<string>()));
// ---- Self-updating products ----------------------------------------------------------
// Software that updates itself is left out of the summary unless the caller asks for it;
// how much was left out is reported either way.
private const string Browser = "contoso:browser";
private const string Gadget = "fabrikam:gadget";
/// <summary>
/// The browser updates itself and carries a known-exploited CVE in two sessions and a second
/// CVE in one of them; the gadget carries one CVE in two sessions, one shared with the browser.
/// </summary>
private static CveExposureEntry[] BrowserAndGadgetRows()
{
var exploited1 = Row("CVE-2026-10001", "s1", "Critical", 9.8, kev: true, tenant: "t1", software: "Contoso Browser 131.0", product: Browser);
var exploited2 = Row("CVE-2026-10001", "s2", "Critical", 9.8, kev: true, tenant: "t2", software: "Contoso Browser 131.0", product: Browser);
var high = Row("CVE-2026-10002", "s2", "High", 7.5, tenant: "t2", software: "Contoso Browser 131.0", product: Browser);
var gadget1 = Row("CVE-2026-10003", "s2", "Medium", 5.0, tenant: "t2", software: "Fabrikam Gadget 2.0", product: Gadget);
var gadget2 = Row("CVE-2026-10003", "s3", "Medium", 5.0, tenant: "t3", software: "Fabrikam Gadget 2.0", product: Gadget);
exploited1.Priority = exploited2.Priority = "act";
high.Priority = gadget1.Priority = gadget2.Priority = "track";
return new[] { exploited1, exploited2, high, gadget1, gadget2 };
}
[Fact]
public void SelfUpdatingProducts_AreLeftOutOfEveryNumberAndList_AndReportedOnTheirOwn()
{
var r = VulnerabilitySummaryAggregator.Aggregate(
BrowserAndGadgetRows(), false, 20, 30, crossTenant: true, Now, selfUpdatingProducts: new[] { Browser });
Assert.False(r.SelfUpdatingIncluded);
Assert.Equal(1, r.SelfUpdatingSoftware);
Assert.Equal(2, r.SelfUpdatingAffectedSessions);
Assert.Equal(2, r.TotalAffectedSessions); // s2 through the gadget, s3
Assert.Equal(2, r.TotalAffectedTenants);
Assert.Equal(1, r.DistinctCves);
Assert.Equal(1, r.DistinctSoftware);
Assert.Equal(0, r.KevCves);
Assert.Equal(0, r.KevAffectedSessions);
Assert.Equal((0, 0, 1, 0), (r.SeverityBreakdown.Critical, r.SeverityBreakdown.High, r.SeverityBreakdown.Medium, r.SeverityBreakdown.Low));
Assert.Equal((0, 0, 1), (r.PriorityBreakdown.Act, r.PriorityBreakdown.Attend, r.PriorityBreakdown.Track));
Assert.Equal(new[] { "CVE-2026-10003" }, r.TopCves.Select(c => c.CveId));
var gadget = Assert.Single(r.TopSoftware);
Assert.Equal(Gadget, gadget.Product);
Assert.Null(gadget.SelfUpdating);
}
[Fact]
public void SelfUpdatingProducts_AreCountedWhenAskedFor_AndTheirEntriesAreMarked()
{
var r = VulnerabilitySummaryAggregator.Aggregate(
BrowserAndGadgetRows(), false, 20, 30, crossTenant: true, Now,
selfUpdatingProducts: new[] { Browser }, includeSelfUpdating: true);
Assert.True(r.SelfUpdatingIncluded);
// What concerns self-updating software is reported the same either way.
Assert.Equal(1, r.SelfUpdatingSoftware);
Assert.Equal(2, r.SelfUpdatingAffectedSessions);
Assert.Equal(3, r.TotalAffectedSessions);
Assert.Equal(3, r.TotalAffectedTenants);
Assert.Equal(3, r.DistinctCves);
Assert.Equal(2, r.DistinctSoftware);
Assert.Equal(1, r.KevCves);
Assert.Equal(2, r.KevAffectedSessions);
Assert.Equal((1, 1, 1, 0), (r.SeverityBreakdown.Critical, r.SeverityBreakdown.High, r.SeverityBreakdown.Medium, r.SeverityBreakdown.Low));
Assert.Equal((1, 0, 2), (r.PriorityBreakdown.Act, r.PriorityBreakdown.Attend, r.PriorityBreakdown.Track));
Assert.Equal(new[] { "CVE-2026-10001", "CVE-2026-10003", "CVE-2026-10002" }, r.TopCves.Select(c => c.CveId));
Assert.Equal(new[] { Browser, Gadget }, r.TopSoftware.Select(s => s.Product));
Assert.True(r.TopSoftware[0].SelfUpdating);
Assert.Null(r.TopSoftware[1].SelfUpdating);
}
[Fact]
public void SelfUpdatingProducts_ACveTheyShareWithOtherSoftware_IsCountedThroughTheOtherSoftwareOnly()
{
var rows = new[]
{
Row("CVE-2026-10004", "s1", "High", 8.1, software: "Contoso Browser 131.0", product: Browser),
Row("CVE-2026-10004", "s2", "High", 8.1, software: "Contoso Browser 131.0", product: Browser),
Row("CVE-2026-10004", "s2", "High", 8.1, software: "Fabrikam Gadget 2.0", product: Gadget),
Row("CVE-2026-10004", "s3", "High", 8.1, software: "Fabrikam Gadget 2.0", product: Gadget),
};
var leftOut = VulnerabilitySummaryAggregator.Aggregate(rows, false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser });
var included = VulnerabilitySummaryAggregator.Aggregate(
rows, false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser }, includeSelfUpdating: true);
var cve = Assert.Single(leftOut.TopCves);
Assert.Equal(2, cve.AffectedSessions);
Assert.Equal(new[] { "Fabrikam Gadget 2.0" }, cve.SampleSoftware);
Assert.Equal(2, leftOut.TotalAffectedSessions);
// s2 has the CVE in both products: it is among the counted sessions and among those of
// self-updating software, so the two numbers do not add up to the sessions affected.
Assert.Equal(2, leftOut.SelfUpdatingAffectedSessions);
Assert.Equal(3, Assert.Single(included.TopCves).AffectedSessions);
Assert.Equal(new[] { "Contoso Browser 131.0", "Fabrikam Gadget 2.0" }, included.TopCves[0].SampleSoftware);
Assert.Equal(3, included.TotalAffectedSessions);
}
[Fact]
public void SelfUpdatingProducts_AreRecognizedWhateverTheCaseOfTheProductKey()
{
var rows = new[]
{
Row("CVE-2026-10005", "s1", "High", 8.1, software: "Contoso Browser 131.0", product: "Contoso:Browser"),
Row("CVE-2026-10005", "s2", "High", 8.1, software: "Contoso Browser 132.0", product: Browser),
};
var r = VulnerabilitySummaryAggregator.Aggregate(rows, false, 20, 30, false, Now, selfUpdatingProducts: new[] { "CONTOSO:browser" });
Assert.Equal(0, r.DistinctCves);
Assert.Empty(r.TopSoftware);
Assert.Equal(1, r.SelfUpdatingSoftware);
Assert.Equal(2, r.SelfUpdatingAffectedSessions);
}
[Fact]
public void SelfUpdatingProducts_OutsideTheWindow_AreNotReported()
{
var rows = new[]
{
Row("CVE-2026-10006", "s1", "High", 8.1, software: "Contoso Browser 120.0", product: Browser, detectedAt: Now.AddDays(-90)),
Row("CVE-2026-10007", "s2", "High", 8.1, software: "Fabrikam Gadget 2.0", product: Gadget),
};
var r = VulnerabilitySummaryAggregator.Aggregate(rows, false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser });
Assert.Equal(0, r.SelfUpdatingSoftware);
Assert.Equal(0, r.SelfUpdatingAffectedSessions);
Assert.Equal(1, r.DistinctCves);
}
[Fact]
public void WithoutSelfUpdatingProducts_NothingIsLeftOut()
{
var unknown = VulnerabilitySummaryAggregator.Aggregate(BrowserAndGadgetRows(), false, 20, 30, false, Now);
var none = VulnerabilitySummaryAggregator.Aggregate(
BrowserAndGadgetRows(), false, 20, 30, false, Now, selfUpdatingProducts: Array.Empty<string>());
Assert.All(new[] { unknown, none }, r =>
{
Assert.False(r.SelfUpdatingIncluded);
Assert.Equal(0, r.SelfUpdatingSoftware);
Assert.Equal(0, r.SelfUpdatingAffectedSessions);
Assert.Equal(3, r.DistinctCves);
Assert.Equal(1, r.KevCves);
Assert.Equal(3, r.TotalAffectedSessions);
Assert.All(r.TopSoftware, s => Assert.Null(s.SelfUpdating));
});
}
/// <summary>
/// The browser updates itself; one of its rows was written before the index carried the
/// product and is known as the browser's only by its title.
/// </summary>
private static CveExposureEntry[] BrowserRowsOneWithoutItsProduct() => new[]
{
Row("CVE-2026-10008", "s1", "High", 8.1, software: "Contoso Browser 131.0", product: Browser),
Row("CVE-2026-10008", "s2", "High", 8.1, software: "Contoso Browser 131.0"),
};
[Fact]
public void SelfUpdatingProducts_RowThatBorrowsItsProductFromItsTitle_IsLeftOutWithItsSiblings()
{
var leftOut = VulnerabilitySummaryAggregator.Aggregate(
BrowserRowsOneWithoutItsProduct(), false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser });
var included = VulnerabilitySummaryAggregator.Aggregate(
BrowserRowsOneWithoutItsProduct(), false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser }, includeSelfUpdating: true);
Assert.Equal(0, leftOut.DistinctCves);
Assert.Equal(0, leftOut.DistinctSoftware);
Assert.Equal(0, leftOut.TotalAffectedSessions);
Assert.Empty(leftOut.TopCves);
Assert.Empty(leftOut.TopSoftware);
Assert.Equal(2, leftOut.SelfUpdatingAffectedSessions);
var browser = Assert.Single(included.TopSoftware);
Assert.Equal(Browser, browser.Product);
Assert.True(browser.SelfUpdating);
Assert.Equal(2, browser.AffectedSessions);
Assert.Equal(2, included.TotalAffectedSessions);
Assert.Equal(2, included.SelfUpdatingAffectedSessions);
}
[Fact]
public void SelfUpdatingProducts_RowThatBorrowsItsProductFromItsTitle_IsNotCountedAsAProductOfItsOwn()
{
var r = VulnerabilitySummaryAggregator.Aggregate(
BrowserRowsOneWithoutItsProduct(), false, 20, 30, false, Now, selfUpdatingProducts: new[] { Browser });
// Both rows are the browser's: one product updates itself, in two sessions.
Assert.Equal(1, r.SelfUpdatingSoftware);
Assert.Equal(2, r.SelfUpdatingAffectedSessions);
}
}
@@ -0,0 +1,124 @@
using System.Net;
using System.Security.Claims;
using System.Text.Json;
using AutopilotMonitor.Functions.Functions.Vulnerability;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models.Vulnerability;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// GET metrics/vulnerability applies the self-updating flags as they are in service when the
/// summary is read: software that updates itself is left out unless the caller asks for it. When
/// the flags cannot be read nothing can be told apart, so everything is counted and the answer
/// says that it is.
/// </summary>
public class VulnerabilitySummaryFunctionTests
{
private const string Browser = "contoso:browser";
private const string Gadget = "fabrikam:gadget";
private sealed class Rig
{
public Rig()
{
Tables.Put(VulnerabilityCacheRows.Mapping("contoso", "contoso browser", "cpe:2.3:a:contoso:browser", "contoso browser"));
Tables.Put(VulnerabilityCacheRows.Mapping("fabrikam", "fabrikam gadget", "cpe:2.3:a:fabrikam:gadget", "fabrikam gadget"));
Tables.Put(VulnerabilityCacheRows.SelfUpdating(Browser));
var sessions = new Mock<ISessionRepository>();
sessions.Setup(s => s.ScanCveIndexAsync(It.IsAny<string?>(), It.IsAny<int>(), It.IsAny<DateTime?>(), It.IsAny<CancellationToken>()))
.ReturnsAsync((Rows, false));
Function = new VulnerabilitySummaryFunction(
NullLogger<VulnerabilitySummaryFunction>.Instance,
sessions.Object,
new CpeMappingStore(Tables.Repo.Object, NullLogger<CpeMappingStore>.Instance));
}
public MappingTables Tables { get; } = new();
public VulnerabilitySummaryFunction Function { get; }
/// <summary>The browser carries one CVE in two sessions, the gadget another in two, one session shared.</summary>
private static IReadOnlyList<CveExposureEntry> Rows { get; } = new[]
{
Row("CVE-2026-10001", "s1", "Contoso Browser 131.0", Browser),
Row("CVE-2026-10001", "s2", "Contoso Browser 131.0", Browser),
Row("CVE-2026-10003", "s2", "Fabrikam Gadget 2.0", Gadget),
Row("CVE-2026-10003", "s3", "Fabrikam Gadget 2.0", Gadget),
};
public async Task<JsonElement> GetAsync(string query = "")
{
var (request, context) = EndpointHarness.Request(VulnerabilityScanRig.TenantId, queryString: query);
// The tenant endpoint reads the caller's tenant from the principal the authentication step left.
context.Items["ClaimsPrincipal"] = new ClaimsPrincipal(
new ClaimsIdentity(new[] { new Claim("tid", VulnerabilityScanRig.TenantId) }, "test"));
var response = await Function.GetTenant(request);
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
return OperatorCall.BodyOf(response);
}
private static CveExposureEntry Row(string cve, string session, string software, string product) => new()
{
TenantId = VulnerabilityScanRig.TenantId,
CveId = cve,
SessionId = session,
SoftwareName = software,
Product = product,
CvssScore = 8.1,
CvssSeverity = "High",
Priority = "track",
OverallRisk = "high",
DetectedAt = DateTime.UtcNow.AddDays(-1),
};
}
private static IEnumerable<string> ProductsOf(JsonElement body) =>
body.GetProperty("topSoftware").EnumerateArray().Select(s => s.GetProperty("product").GetString()!);
[Fact]
public async Task Get_LeavesOutSoftwareThatUpdatesItself_AndSaysHowMuchThatIs()
{
var body = await new Rig().GetAsync();
Assert.False(body.GetProperty("selfUpdatingIncluded").GetBoolean());
Assert.Equal(1, body.GetProperty("distinctCves").GetInt32());
Assert.Equal(2, body.GetProperty("totalAffectedSessions").GetInt32());
Assert.Equal(new[] { Gadget }, ProductsOf(body));
Assert.Equal(1, body.GetProperty("selfUpdatingSoftware").GetInt32());
Assert.Equal(2, body.GetProperty("selfUpdatingAffectedSessions").GetInt32());
}
[Fact]
public async Task Get_CountsSoftwareThatUpdatesItself_WhenTheCallerAsksForIt_AndMarksIt()
{
var body = await new Rig().GetAsync("?includeSelfUpdating=true");
Assert.True(body.GetProperty("selfUpdatingIncluded").GetBoolean());
Assert.Equal(2, body.GetProperty("distinctCves").GetInt32());
Assert.Equal(3, body.GetProperty("totalAffectedSessions").GetInt32());
var software = body.GetProperty("topSoftware").EnumerateArray().ToDictionary(s => s.GetProperty("product").GetString()!);
Assert.True(software[Browser].GetProperty("selfUpdating").GetBoolean());
Assert.False(software[Gadget].TryGetProperty("selfUpdating", out _));
Assert.Equal(1, body.GetProperty("selfUpdatingSoftware").GetInt32());
}
[Fact]
public async Task Get_FlagsCannotBeRead_CountsEverything_AndSaysThatItDoes()
{
var rig = new Rig();
rig.Tables.ReadFailure = new InvalidOperationException("table storage is not reachable");
var body = await rig.GetAsync();
// Without the flags a row of self-updating software cannot be told from any other one.
Assert.True(body.GetProperty("selfUpdatingIncluded").GetBoolean());
Assert.Equal(2, body.GetProperty("distinctCves").GetInt32());
Assert.Equal(3, body.GetProperty("totalAffectedSessions").GetInt32());
Assert.Equal(new[] { Browser, Gadget }, ProductsOf(body).OrderBy(p => p, StringComparer.Ordinal));
Assert.Equal(0, body.GetProperty("selfUpdatingSoftware").GetInt32());
}
}
@@ -228,7 +228,7 @@ public class VulnerabilityWireParityTests
nvdLastRun = new
{
stale = 40, refreshed = 38, failed = 2,
skippedThrottled = 0, error = (string?)"NVD returned 503 for 2 CPEs",
skippedThrottled = 0, skippedUnavailable = 0, error = (string?)"NVD returned 503 for 2 CPEs",
startedUtc = "2026-08-30T04:00:10.0000000Z", finishedUtc = (string?)"2026-08-30T04:10:00.0000000Z",
},
epssLastSyncUtc = (string?)"2026-08-30T04:11:00.0000000Z",
@@ -253,7 +253,7 @@ public class VulnerabilityWireParityTests
NvdLastRun = new NvdRefreshRunWire
{
Stale = 40, Refreshed = 38, Failed = 2,
SkippedThrottled = 0, Error = "NVD returned 503 for 2 CPEs",
SkippedThrottled = 0, SkippedUnavailable = 0, Error = "NVD returned 503 for 2 CPEs",
StartedUtc = "2026-08-30T04:00:10.0000000Z", FinishedUtc = "2026-08-30T04:10:00.0000000Z",
},
EpssLastSyncUtc = "2026-08-30T04:11:00.0000000Z",
@@ -322,7 +322,7 @@ public class VulnerabilityWireParityTests
nvdLastRun = new
{
stale = 40, refreshed = 12, failed = 0,
skippedThrottled = 0, error = (string?)null,
skippedThrottled = 0, skippedUnavailable = 0, error = (string?)null,
startedUtc = "2026-08-30T12:00:00.0000000Z", finishedUtc = (string?)null,
},
epssLastSyncUtc = (string?)null,
@@ -347,7 +347,7 @@ public class VulnerabilityWireParityTests
NvdLastRun = new NvdRefreshRunWire
{
Stale = 40, Refreshed = 12, Failed = 0,
SkippedThrottled = 0, Error = null,
SkippedThrottled = 0, SkippedUnavailable = 0, Error = null,
StartedUtc = "2026-08-30T12:00:00.0000000Z", FinishedUtc = null,
},
EpssLastSyncUtc = null,
@@ -418,9 +418,84 @@ public class VulnerabilityWireParityTests
{
var rowKey = "igor pavlov:7-zip";
// The count is omitted when the inventory rows could not be reconciled right away.
AssertParity(
new { success = true, message = "CPE mapping saved", rowKey },
new SaveCustomCpeMappingResponse { Success = true, Message = "CPE mapping saved", RowKey = rowKey });
AssertParity(
new { success = true, message = "CPE mapping saved", rowKey, inventoryRowsChanged = 12 },
new SaveCustomCpeMappingResponse { Success = true, Message = "CPE mapping saved", RowKey = rowKey, InventoryRowsChanged = 12 });
}
// ---- SetCpeProductFlags --------------------------------------------------------------
[Fact]
public void SetCpeProductFlagsResponse_matches_the_flag_shape()
{
AssertParity(
new { success = true, product = "microsoft:visual_studio_code", selfUpdating = true, selfUpdatingDefault = false },
new SetCpeProductFlagsResponse { Success = true, Product = "microsoft:visual_studio_code", SelfUpdating = true, SelfUpdatingDefault = false });
}
[Fact]
public void CpeMappingItem_carries_its_row_key_and_the_product_flags_after_createdAt()
{
var mappings = new List<CpeMappingItem>
{
new CpeMappingItem
{
NormalizedVendor = "google", NormalizedProduct = "google chrome", CpeVendor = "google", CpeProduct = "chrome",
CpeUri = "cpe:2.3:a:google:chrome", Category = "browser",
DisplayNamePatterns = new List<string> { "google chrome" }, PublisherPatterns = new List<string> { "google" },
ExcludePatterns = new List<string>(), Source = "seed", CreatedAt = "2026-11-12T03:00:00.0000000Z",
RowKey = "google:google chrome", SelfUpdating = false, SelfUpdatingDefault = true,
},
};
AssertParity(
new
{
mappings = new object[]
{
new
{
normalizedVendor = "google", normalizedProduct = "google chrome", cpeVendor = "google", cpeProduct = "chrome",
cpeUri = "cpe:2.3:a:google:chrome", category = "browser",
displayNamePatterns = new[] { "google chrome" }, publisherPatterns = new[] { "google" },
excludePatterns = Array.Empty<string>(), source = "seed", createdAt = "2026-11-12T03:00:00.0000000Z",
rowKey = "google:google chrome", selfUpdating = false, selfUpdatingDefault = true,
},
},
total = 1,
},
new GetCpeMappingsResponse { Mappings = mappings, Total = 1 });
}
[Fact]
public void UnmatchedSoftwareItem_carries_the_canonical_name_last()
{
var item = new UnmatchedSoftwareItem
{
SoftwareName = "Contoso Widget 4.2 (x64)", Publisher = "Contoso Ltd", Frequency = 12,
LastSeenAt = "2026-11-10T08:00:00.0000000Z", ExampleSessionId = "22222222-2222-2222-2222-222222222222",
NormalizedVendor = "contoso", NormalizedVersion = "4.2", CanonicalName = "contoso widget",
};
AssertParity(
new
{
software = new object[]
{
new
{
softwareName = "Contoso Widget 4.2 (x64)", publisher = "Contoso Ltd", frequency = 12,
lastSeenAt = "2026-11-10T08:00:00.0000000Z", exampleSessionId = "22222222-2222-2222-2222-222222222222",
normalizedVendor = "contoso", normalizedVersion = "4.2", canonicalName = "contoso widget",
},
},
total = 1, skip = 0, take = 50,
},
new GetUnmatchedSoftwareResponse { Software = new List<UnmatchedSoftwareItem> { item }, Total = 1, Skip = 0, Take = 50 });
}
// ---- TriggerEpssSync (dual-purpose success) ------------------------------------------
@@ -713,6 +788,9 @@ public class VulnerabilityWireParityTests
},
},
truncated = false,
selfUpdatingIncluded = false,
selfUpdatingSoftware = 2,
selfUpdatingAffectedSessions = 9,
},
new CveExposureSummary
{
@@ -747,6 +825,9 @@ public class VulnerabilityWireParityTests
},
},
Truncated = false,
SelfUpdatingIncluded = false,
SelfUpdatingSoftware = 2,
SelfUpdatingAffectedSessions = 9,
});
}
@@ -786,6 +867,9 @@ public class VulnerabilityWireParityTests
bandCves,
bandSoftware,
truncated = false,
selfUpdatingIncluded = false,
selfUpdatingSoftware = 2,
selfUpdatingAffectedSessions = 9,
},
new CveExposureSummary
{
@@ -803,6 +887,9 @@ public class VulnerabilityWireParityTests
BandCves = bandCves,
BandSoftware = bandSoftware,
Truncated = false,
SelfUpdatingIncluded = false,
SelfUpdatingSoftware = 2,
SelfUpdatingAffectedSessions = 9,
});
}
@@ -829,6 +916,9 @@ public class VulnerabilityWireParityTests
topCves,
topSoftware,
truncated = true,
selfUpdatingIncluded = false,
selfUpdatingSoftware = 2,
selfUpdatingAffectedSessions = 9,
},
new CveExposureSummary
{
@@ -844,6 +934,85 @@ public class VulnerabilityWireParityTests
TopCves = topCves,
TopSoftware = topSoftware,
Truncated = true,
SelfUpdatingIncluded = false,
SelfUpdatingSoftware = 2,
SelfUpdatingAffectedSessions = 9,
});
}
[Fact]
public void TopSoftware_carries_selfUpdating_last_and_omits_it_for_software_that_does_not_update_itself()
{
var breakdown = new SeverityBreakdown { Critical = 1, High = 0, Medium = 1, Low = 0 };
var priorities = new PriorityBreakdown { Act = 1, Attend = 0, Track = 1 };
var topCves = new List<TopCve>();
AssertParity(
new
{
windowDays = 30,
totalAffectedSessions = 3,
distinctCves = 2,
distinctSoftware = 2,
kevCves = 1,
kevAffectedSessions = 2,
severityBreakdown = breakdown,
priorityBreakdown = priorities,
topCves,
topSoftware = new object[]
{
new
{
product = "contoso:browser", name = "Contoso Browser", affectedSessions = 2, cveCount = 1, kevCount = 1,
cvssSeverity = "CRITICAL", cvssScore = 9.8, priority = "act",
titles = new[] { "Contoso Browser 131.0" }, cves = Array.Empty<object>(),
selfUpdating = true,
},
new
{
product = "fabrikam:gadget", name = "Fabrikam Gadget", affectedSessions = 2, cveCount = 1, kevCount = 0,
cvssSeverity = "MEDIUM", cvssScore = 5.5, priority = "track",
titles = new[] { "Fabrikam Gadget 2.0" }, cves = Array.Empty<object>(),
},
},
truncated = false,
selfUpdatingIncluded = true,
selfUpdatingSoftware = 1,
selfUpdatingAffectedSessions = 2,
},
new CveExposureSummary
{
WindowDays = 30,
TotalAffectedSessions = 3,
TotalAffectedTenants = null,
DistinctCves = 2,
DistinctSoftware = 2,
KevCves = 1,
KevAffectedSessions = 2,
SeverityBreakdown = breakdown,
PriorityBreakdown = priorities,
TopCves = topCves,
TopSoftware = new List<TopSoftware>
{
new TopSoftware
{
Product = "contoso:browser", Name = "Contoso Browser", AffectedSessions = 2, CveCount = 1, KevCount = 1,
CvssSeverity = "CRITICAL", CvssScore = 9.8, Priority = "act",
Titles = new List<string> { "Contoso Browser 131.0" }, Cves = new List<SoftwareCve>(),
SelfUpdating = true,
},
new TopSoftware
{
Product = "fabrikam:gadget", Name = "Fabrikam Gadget", AffectedSessions = 2, CveCount = 1, KevCount = 0,
CvssSeverity = "MEDIUM", CvssScore = 5.5, Priority = "track",
Titles = new List<string> { "Fabrikam Gadget 2.0" }, Cves = new List<SoftwareCve>(),
SelfUpdating = null,
},
},
Truncated = false,
SelfUpdatingIncluded = true,
SelfUpdatingSoftware = 1,
SelfUpdatingAffectedSessions = 2,
});
}
@@ -0,0 +1,247 @@
using Azure;
using Azure.Data.Tables;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
namespace AutopilotMonitor.Functions.Tests;
/// <summary>
/// A CVE cache row and a report row hold their payload in one column or split over several,
/// whichever its size needs. Both are written in replace mode: a merge keeps the other form of an
/// earlier write next to the new one, and the reader returns the old single column. The marker
/// for missing CVE data is an insert that never overwrites, and removing a row that is already
/// gone is not an error.
/// </summary>
public class VulnerabilityWriteModeTests
{
private const string Tenant = "11111111-1111-1111-1111-111111111111";
private const string Session = "22222222-2222-2222-2222-222222222222";
private const string WidgetCpe = "cpe:2.3:a:contoso:widget";
private static readonly string WidgetKey = CveCacheFiller.CacheKey(WidgetCpe);
/// <summary>A table that keeps rows the way the service does: replace swaps the row, merge adds to it.</summary>
private sealed class FakeTable
{
public Mock<TableClient> Client { get; } = new();
public Dictionary<(string, string), TableEntity> Rows { get; } = new();
public List<TableUpdateMode> UpsertModes { get; } = new();
public List<(string? Filter, List<string> Select)> Queries { get; } = new();
/// <summary>Status the next insert or delete answers with; none means success.</summary>
public int? Fail { get; set; }
public FakeTable()
{
Client.Setup(t => t.UpsertEntityAsync(It.IsAny<TableEntity>(), It.IsAny<TableUpdateMode>(), It.IsAny<CancellationToken>()))
.Returns<TableEntity, TableUpdateMode, CancellationToken>((entity, mode, _) =>
{
UpsertModes.Add(mode);
var key = (entity.PartitionKey, entity.RowKey);
if (mode == TableUpdateMode.Replace || !Rows.TryGetValue(key, out var existing))
Rows[key] = new TableEntity(entity);
else
foreach (var property in entity) existing[property.Key] = property.Value;
return Task.FromResult(Mock.Of<Response>());
});
Client.Setup(t => t.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()))
.Returns<TableEntity, CancellationToken>((entity, _) =>
{
if (Fail.HasValue) return Task.FromException<Response>(new RequestFailedException(Fail.Value, "synthetic failure"));
Rows[(entity.PartitionKey, entity.RowKey)] = new TableEntity(entity);
return Task.FromResult(Mock.Of<Response>());
});
Client.Setup(t => t.DeleteEntityAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<ETag>(), It.IsAny<CancellationToken>()))
.Returns<string, string, ETag, CancellationToken>((partitionKey, rowKey, _, _) =>
{
if (Fail.HasValue) return Task.FromException<Response>(new RequestFailedException(Fail.Value, "synthetic failure"));
Rows.Remove((partitionKey, rowKey));
return Task.FromResult(Mock.Of<Response>());
});
Client.Setup(t => t.GetEntityAsync<TableEntity>(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.Returns<string, string, IEnumerable<string>, CancellationToken>((partitionKey, rowKey, _, _) =>
Rows.TryGetValue((partitionKey, rowKey), out var row)
? Task.FromResult(Response.FromValue(row, Mock.Of<Response>()))
: Task.FromException<Response<TableEntity>>(new RequestFailedException(404, "ResourceNotFound")));
Client.Setup(t => t.QueryAsync<TableEntity>(It.IsAny<string>(), It.IsAny<int?>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
.Returns((string? filter, int? _, IEnumerable<string> select, CancellationToken _) =>
{
Queries.Add((filter, select.ToList()));
return AsyncPageable<TableEntity>.FromPages(new[] { Page<TableEntity>.FromValues(Rows.Values.ToList(), null, Mock.Of<Response>()) });
});
}
}
private sealed class Harness
{
public FakeTable Cache { get; } = new();
public FakeTable Reports { get; } = new();
public TableStorageService Storage { get; }
public TableVulnerabilityRepository Repository { get; }
public Harness()
{
var service = new Mock<TableServiceClient>();
service.Setup(s => s.GetTableClient(Constants.TableNames.VulnerabilityCache)).Returns(Cache.Client.Object);
service.Setup(s => s.GetTableClient(Constants.TableNames.VulnerabilityReports)).Returns(Reports.Client.Object);
Storage = new TableStorageService(service.Object, NullLogger<TableStorageService>.Instance);
Repository = new TableVulnerabilityRepository(Storage, Mock.Of<IDataEventPublisher>());
}
}
private static List<CachedCve> Cves(int count, int descriptionLength) => Enumerable.Range(1, count).Select(n =>
{
var cve = VulnerabilityCacheRows.Cve($"CVE-2026-{n:0000}", 7.5, WidgetCpe, "2.0");
cve.Description = new string('d', descriptionLength);
return cve;
}).ToList();
private static Dictionary<string, object> Report(int findings, int descriptionLength) => new()
{
["scan_summary"] = new Dictionary<string, object>
{
["overall_risk"] = "high", ["total_cves_found"] = findings, ["high_cves"] = findings, ["total_software_scanned"] = 40,
},
["findings"] = Enumerable.Range(1, findings).Select(n => new Dictionary<string, object>
{
["softwareName"] = $"Contoso Widget {n:00}",
["description"] = new string('d', descriptionLength),
}).ToList(),
};
// -----------------------------------------------------------------------
// CVE cache row
// -----------------------------------------------------------------------
[Fact]
public async Task UpsertCveCacheEntry_ReplacesTheRow_SoAnEarlierSingleColumnDoesNotOutliveASplitPayload()
{
var harness = new Harness();
var filler = new CveCacheFiller(harness.Repository, nvdApi: null!, NullLogger<CveCacheFiller>.Instance);
var small = Cves(2, descriptionLength: 50);
var large = Cves(90, descriptionLength: 500);
await harness.Repository.UpsertCveCacheEntryAsync(
WidgetKey, CveCacheFiller.RowKey, CveCacheFiller.BuildRow(WidgetCpe, small, DateTime.UtcNow.AddDays(-8), nvdTotal: 2));
await harness.Repository.UpsertCveCacheEntryAsync(
WidgetKey, CveCacheFiller.RowKey, CveCacheFiller.BuildRow(WidgetCpe, large, DateTime.UtcNow, nvdTotal: 90));
Assert.Equal(new[] { TableUpdateMode.Replace, TableUpdateMode.Replace }, harness.Cache.UpsertModes);
var row = harness.Cache.Rows[(WidgetKey, CveCacheFiller.RowKey)];
Assert.False(row.ContainsKey("CveDataJson"));
Assert.True(row.ContainsKey("CveDataJson_ChunkCount"));
Assert.Equal(90, row.GetInt32("TotalCves"));
var entry = await filler.ReadAsync(WidgetCpe);
Assert.Equal(large.Select(c => c.CveId), entry!.Cves.Select(c => c.CveId));
}
[Fact]
public async Task TryAddCveCacheEntry_Inserts_AndLeavesAnExistingRowAlone()
{
var harness = new Harness();
var marker = new Dictionary<string, object> { ["State"] = "pending", ["CpeUri"] = WidgetCpe };
Assert.True(await harness.Repository.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, marker));
Assert.Equal("pending", harness.Cache.Rows[(WidgetKey, CveCacheFiller.RowKey)].GetString("State"));
harness.Cache.Fail = 409; // EntityAlreadyExists
Assert.False(await harness.Repository.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, marker));
harness.Cache.Fail = 503;
await Assert.ThrowsAsync<RequestFailedException>(
() => harness.Repository.TryAddCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey, marker));
// An insert, never an upsert: the marker must not replace a filled row.
Assert.Empty(harness.Cache.UpsertModes);
harness.Cache.Client.Verify(t => t.AddEntityAsync(It.IsAny<TableEntity>(), It.IsAny<CancellationToken>()), Times.Exactly(3));
}
[Fact]
public async Task DeleteCveCacheEntry_RowAlreadyGone_IsNotAnError()
{
var harness = new Harness();
harness.Cache.Fail = 404;
await harness.Repository.DeleteCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey);
harness.Cache.Fail = 503;
await Assert.ThrowsAsync<RequestFailedException>(
() => harness.Repository.DeleteCveCacheEntryAsync(WidgetKey, CveCacheFiller.RowKey));
harness.Cache.Client.Verify(
t => t.DeleteEntityAsync(WidgetKey, CveCacheFiller.RowKey, It.IsAny<ETag>(), It.IsAny<CancellationToken>()), Times.Exactly(2));
}
[Fact]
public async Task GetCveCacheIndex_ReadsTheColumnsTheRefreshWalkDecidesBy()
{
var harness = new Harness();
harness.Cache.Rows[(WidgetKey, CveCacheFiller.RowKey)] = new TableEntity(WidgetKey, CveCacheFiller.RowKey)
{
["CpeUri"] = WidgetCpe, ["State"] = "pending", ["RequestedAt"] = "2026-11-10T08:00:00.0000000Z", ["SchemaVersion"] = 2,
};
harness.Cache.Rows[("cpe_map_seed", CveCacheFiller.RowKey)] = new TableEntity("cpe_map_seed", CveCacheFiller.RowKey);
var index = await harness.Repository.GetCveCacheIndexAsync();
var query = Assert.Single(harness.Cache.Queries);
Assert.Equal("RowKey eq 'cache_result'", query.Filter);
// Without State, RequestedAt and SchemaVersion every row would read as filled and of the older shape.
Assert.Superset(
new HashSet<string> { "PartitionKey", "CpeUri", "CachedAt", "SchemaVersion", "State", "RequestedAt" },
query.Select.ToHashSet());
Assert.DoesNotContain("CveDataJson", query.Select);
var row = Assert.Single(index);
Assert.Equal(WidgetKey, row["PartitionKey"]);
Assert.Equal(CveCacheFiller.RowKey, row["RowKey"]);
Assert.True(CveCacheFiller.IsPending(row));
Assert.Equal("2026-11-10T08:00:00.0000000Z", row["RequestedAt"]);
Assert.Equal(2, row["SchemaVersion"]);
}
// -----------------------------------------------------------------------
// Report row
// -----------------------------------------------------------------------
[Fact]
public async Task StoreVulnerabilityReport_ReplacesTheRow_SoAnEarlierSingleColumnDoesNotOutliveASplitReport()
{
var harness = new Harness();
var small = Report(findings: 2, descriptionLength: 50);
var large = Report(findings: 80, descriptionLength: 500);
await harness.Storage.StoreVulnerabilityReportAsync(Tenant, Session, small);
var smallRow = harness.Reports.Rows[($"{Tenant}_{Session}", "report")];
Assert.True(smallRow.ContainsKey("ReportJson"));
await harness.Storage.StoreVulnerabilityReportAsync(Tenant, Session, large);
Assert.Equal(new[] { TableUpdateMode.Replace, TableUpdateMode.Replace }, harness.Reports.UpsertModes);
var row = harness.Reports.Rows[($"{Tenant}_{Session}", "report")];
Assert.False(row.ContainsKey("ReportJson"));
Assert.True(row.ContainsKey("ReportJson_ChunkCount"));
Assert.Equal("high", row.GetString("OverallRisk"));
Assert.Equal(80, row.GetInt32("TotalCvesFound"));
Assert.Equal(80, row.GetInt32("HighCves"));
Assert.Equal(40, row.GetInt32("TotalScanned"));
var readBack = await harness.Storage.GetVulnerabilityReportAsync(Tenant, Session);
Assert.Equal(80, VulnerabilityCorrelationService.FindingsOf(readBack!).Count);
}
[Fact]
public async Task StoreVulnerabilityReport_ThroughTheRepository_ReplacesToo()
{
var harness = new Harness();
await harness.Repository.StoreVulnerabilityReportAsync(Tenant, Session, Report(findings: 1, descriptionLength: 10));
Assert.Equal(TableUpdateMode.Replace, Assert.Single(harness.Reports.UpsertModes));
}
}
@@ -75,8 +75,8 @@
<EmbeddedResource Include="..\..\..\rules\dist\ime-log-patterns.json" LogicalName="AutopilotMonitor.Functions.Rules.ime-log-patterns.json" />
<!-- The OOBE go-script embeds the installer's payload-verification block verbatim (D-354). -->
<EmbeddedResource Include="..\..\..\scripts\Bootstrap\Install-AutopilotMonitor.ps1" LogicalName="AutopilotMonitor.Functions.Bootstrap.Install-AutopilotMonitor.ps1" />
<!-- CPE mapping seed + community files are loaded from GitHub via reseed, not embedded -->
<!-- Files at rules/dist/cpe-mapping-seed.json and rules/dist/cpe-community-mappings.json -->
<!-- The CPE mapping lists are loaded from GitHub via reseed, not embedded: -->
<!-- rules/dist/cpe-mappings.json and rules/dist/cpe-community-mappings.json -->
</ItemGroup>
</Project>
@@ -284,8 +284,8 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
=> _storage.SearchSessionsByCvePageAsync(tenantId, cveId, minCvssScore, overallRisk, pageSize, continuation);
public Task<(IReadOnlyList<CveExposureEntry> Rows, bool Truncated)> ScanCveIndexAsync(
string? tenantId, int maxRows, CancellationToken ct = default)
=> _storage.ScanCveIndexAsync(tenantId, maxRows, ct);
string? tenantId, int maxRows, DateTime? detectedSinceUtc = null, CancellationToken ct = default)
=> _storage.ScanCveIndexAsync(tenantId, maxRows, detectedSinceUtc, ct);
public Task UpsertEventTypeIndexBatchAsync(string tenantId, string sessionId, IEnumerable<EnrollmentEvent> events)
=> _storage.UpsertEventTypeIndexBatchAsync(tenantId, sessionId, events);
@@ -293,7 +293,10 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
public Task UpsertDeviceSnapshotAsync(string tenantId, string sessionId, IEnumerable<EnrollmentEvent> events)
=> _storage.UpsertDeviceSnapshotAsync(tenantId, sessionId, events);
public Task UpsertCveIndexEntriesAsync(string tenantId, string sessionId, List<Dictionary<string, object>> findings)
=> _storage.UpsertCveIndexEntriesAsync(tenantId, sessionId, findings);
public Task ReplaceCveIndexEntriesAsync(
string tenantId, string sessionId,
List<Dictionary<string, object>>? findings,
List<Dictionary<string, object>>? previousFindings)
=> _storage.ReplaceCveIndexEntriesAsync(tenantId, sessionId, findings, previousFindings);
}
}
@@ -86,28 +86,23 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
string tenantId, string normalizedVendor, string normalizedName, string normalizedVersion, CancellationToken ct = default)
=> _storage.DecrementSoftwareInventoryEntryAsync(tenantId, normalizedVendor, normalizedName, normalizedVersion, ct);
public async Task UpdateSoftwareInventoryCpeAsync(string normalizedName, string cpeUri)
public async Task<bool> SetSoftwareInventoryCpeAsync(string tenantId, string rowKey, string cpeUri)
{
var tableClient = _storage.GetTableClient(Constants.TableNames.SoftwareInventory);
await foreach (var entity in tableClient.QueryAsync<TableEntity>(maxPerPage: 1000))
try
{
var name = entity.GetString("NormalizedName") ?? string.Empty;
if (string.Equals(name, normalizedName, StringComparison.OrdinalIgnoreCase))
{
entity["CpeUri"] = cpeUri;
await tableClient.UpsertEntityAsync(entity);
}
await tableClient.UpdateEntityAsync(
new TableEntity(tenantId, rowKey) { ["CpeUri"] = cpeUri }, ETag.All, TableUpdateMode.Merge);
return true;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
return false;
}
}
// --- CPE Mappings ---
public Task<int> ImportCpeMappingSeedFromJsonAsync(string json)
=> _storage.ImportCpeMappingSeedFromJsonAsync(json);
public Task<int> ImportCpeCommunityMappingsFromJsonAsync(string json)
=> _storage.ImportCpeCommunityMappingsFromJsonAsync(json);
public async Task<List<Dictionary<string, object>>> GetAllCpeMappingsAsync()
{
var tableClient = _storage.GetTableClient(Constants.TableNames.VulnerabilityCache);
@@ -164,23 +159,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
}
}
public async Task<int> DeleteCpeMappingsByPartitionAsync(string partitionKey)
{
var tableClient = _storage.GetTableClient(Constants.TableNames.VulnerabilityCache);
var count = 0;
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
filter: $"PartitionKey eq '{partitionKey}'"))
{
try
{
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
count++;
}
catch (RequestFailedException ex) when (ex.Status == 404) { }
}
return count;
}
// --- CVE Cache ---
public async Task<Dictionary<string, object>?> GetCveCacheEntryAsync(string partitionKey, string rowKey)
@@ -205,13 +183,13 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
}
}
public async Task<List<Dictionary<string, object>>> GetStaleCveCacheEntriesAsync()
public async Task<List<Dictionary<string, object>>> GetCveCacheIndexAsync()
{
var tableClient = _storage.GetTableClient(Constants.TableNames.VulnerabilityCache);
var results = new List<Dictionary<string, object>>();
await foreach (var entity in tableClient.QueryAsync<TableEntity>(
filter: "RowKey eq 'cache_result'",
select: new[] { "PartitionKey", "RowKey", "CachedAt", "CveId", "CpeUri" }))
select: new[] { "PartitionKey", "RowKey", "CachedAt", "CpeUri", "TotalCves", "SchemaVersion", "State", "RequestedAt" }))
{
if (!entity.PartitionKey.StartsWith("cve_")) continue;
var item = new Dictionary<string, object>();
@@ -230,7 +208,37 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
var entity = new TableEntity(partitionKey, rowKey);
foreach (var kv in data)
entity[kv.Key] = kv.Value;
await tableClient.UpsertEntityAsync(entity);
await tableClient.UpsertEntityAsync(entity, TableUpdateMode.Replace);
}
public async Task<bool> TryAddCveCacheEntryAsync(string partitionKey, string rowKey, Dictionary<string, object> data)
{
var tableClient = _storage.GetTableClient(Constants.TableNames.VulnerabilityCache);
var entity = new TableEntity(partitionKey, rowKey);
foreach (var kv in data)
entity[kv.Key] = kv.Value;
try
{
await tableClient.AddEntityAsync(entity);
return true;
}
catch (RequestFailedException ex) when (ex.Status == 409)
{
return false;
}
}
public async Task DeleteCveCacheEntryAsync(string partitionKey, string rowKey)
{
var tableClient = _storage.GetTableClient(Constants.TableNames.VulnerabilityCache);
try
{
await tableClient.DeleteEntityAsync(partitionKey, rowKey);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Already gone.
}
}
}
}
@@ -22,23 +22,26 @@ namespace AutopilotMonitor.Functions.Functions.Admin
private readonly GitHubRuleRepository _gitHubRepo;
private readonly GatherRuleService _gatherRuleService;
private readonly AnalyzeRuleService _analyzeRuleService;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly IRuleRepository _ruleRepo;
private readonly CpeMappingImporter _cpeImporter;
private readonly SoftwareInventoryCpeReconciler _reconciler;
public ReseedFromGitHubFunction(
ILogger<ReseedFromGitHubFunction> logger,
GitHubRuleRepository gitHubRepo,
GatherRuleService gatherRuleService,
AnalyzeRuleService analyzeRuleService,
IVulnerabilityRepository vulnRepo,
IRuleRepository ruleRepo)
IRuleRepository ruleRepo,
CpeMappingImporter cpeImporter,
SoftwareInventoryCpeReconciler reconciler)
{
_logger = logger;
_gitHubRepo = gitHubRepo;
_gatherRuleService = gatherRuleService;
_analyzeRuleService = analyzeRuleService;
_vulnRepo = vulnRepo;
_ruleRepo = ruleRepo;
_cpeImporter = cpeImporter;
_reconciler = reconciler;
}
[Function("ReseedFromGitHub")]
@@ -88,29 +91,60 @@ namespace AutopilotMonitor.Functions.Functions.Admin
}
var cpeMappingsResult = new ReseedTableCountsNode { Deleted = 0, Written = 0 };
if (typeParam == "all" || typeParam == "cpe")
{
var (d, w) = await ReseedCpeCommunityMappingsAsync();
cpeMappingsResult = new ReseedTableCountsNode { Deleted = d, Written = w };
}
var cpeSeedResult = new ReseedTableCountsNode { Deleted = 0, Written = 0 };
ReseedInventoryNode? cpeInventory = null;
List<string>? cpeProblems = null;
if (typeParam == "all" || typeParam == "cpe")
{
var (d, w) = await ReseedCpeSeedMappingsAsync();
cpeSeedResult = new ReseedTableCountsNode { Deleted = d, Written = w };
// Each list is fetched and checked before anything is written; a list that is
// refused leaves the mappings in service as they were.
var community = await _cpeImporter.ImportCommunityAsync();
var seed = await _cpeImporter.ImportSeedAsync();
cpeMappingsResult = new ReseedTableCountsNode { Deleted = community.Removed, Written = community.Written };
cpeSeedResult = new ReseedTableCountsNode { Deleted = seed.Removed, Written = seed.Written };
var problems = community.Problems.Select(p => "community list: " + p)
.Concat(seed.Problems.Select(p => "seed list: " + p))
.ToList();
try
{
var reconciled = await _reconciler.ReconcileAsync("reseed");
cpeInventory = new ReseedInventoryNode
{
Rows = reconciled.Rows,
Set = reconciled.Set,
Cleared = reconciled.Cleared,
Failed = reconciled.Failed,
ProductsNoted = reconciled.ProductsNoted,
};
}
catch (CpeMappingUnavailableException)
{
// After a refused first list no mapping is in service: there is nothing to
// bring the rows in line with, and the answer still has to say why the list
// was refused. After an import the same exception means the table could
// not be read back, and that is said.
if (seed.Written + community.Written > 0)
problems.Add("inventory rows not brought in line yet: the mappings could not be read back; the nightly run does it");
}
if (problems.Count > 0) cpeProblems = problems;
}
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new ReseedFromGitHubResponse
{
Success = true,
Message = "Reseed from GitHub complete",
Success = cpeProblems == null,
Message = cpeProblems == null
? "Reseed from GitHub complete"
: "Reseed from GitHub complete, except for the CPE mappings",
Gather = gatherResult,
Analyze = analyzeResult,
Ime = imeResult,
CpeCommunityMappings = cpeMappingsResult,
CpeSeedMappings = cpeSeedResult
CpeSeedMappings = cpeSeedResult,
CpeInventory = cpeInventory,
CpeProblems = cpeProblems
});
return response;
}
@@ -304,38 +338,5 @@ namespace AutopilotMonitor.Functions.Functions.Admin
AutopilotMonitor.Shared.Models.RuleCatalogStamp.KindIme, AutopilotMonitor.Shared.Models.RuleCatalogStamp.SourceGitHub, patterns.Count));
return (deleted, patterns.Count);
}
private async Task<(int deleted, int written)> ReseedCpeCommunityMappingsAsync()
{
// Delete existing community CPE mapping entries
var deleted = await _vulnRepo.DeleteCpeMappingsByPartitionAsync("cpe_map_community");
// Fetch community CPE mappings from GitHub and import
var json = await _gitHubRepo.FetchCpeCommunityMappingsAsync();
var written = await _vulnRepo.ImportCpeCommunityMappingsFromJsonAsync(json);
// Reset the static seed-loaded flag so the next correlation picks up fresh data
VulnerabilityCorrelationService.ResetMappingsCache();
_logger.LogInformation("GitHub reseed community CPE mappings: {Deleted} deleted, {Written} written", deleted, written);
return (deleted, written);
}
private async Task<(int deleted, int written)> ReseedCpeSeedMappingsAsync()
{
// Count existing seed entries before deletion (for reporting)
var existingEntries = await _vulnRepo.GetCpeMappingsByPartitionAsync("cpe_map_seed");
var deleted = existingEntries.Count;
// Fetch CPE seed mappings from GitHub and import (deletes + writes in one call)
var json = await _gitHubRepo.FetchCpeSeedMappingsAsync();
var written = await _vulnRepo.ImportCpeMappingSeedFromJsonAsync(json);
// Reset the static seed-loaded flag so the next correlation picks up fresh data
VulnerabilityCorrelationService.ResetMappingsCache();
_logger.LogInformation("GitHub reseed CPE seed mappings: {Deleted} deleted, {Written} written", deleted, written);
return (deleted, written);
}
}
}
@@ -2,11 +2,10 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -15,25 +14,33 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Bulk auto-resolve unmapped software by searching the NVD CPE dictionary.
/// Accepts up to 25 items per request. For each item, searches NVD and saves
/// the best CPE match as a custom mapping if the score meets the threshold.
/// Tries to map the given programs automatically: each name is compared with the stored copy
/// of the NVD product dictionary and mapped only when it is exactly a product name there and
/// the product has CVEs (see <see cref="CpeDictionaryResolver"/>). The nightly run does the
/// same for every unmapped name; this is the operator's way to get it for a selection now.
/// Accepts up to 25 items per request, because each new mapping costs one paced NVD request.
/// Once NVD throttles or gives no answer, NVD is not asked about the remaining items: each
/// would wait for the same timeout while the operator waits for the response. A program that
/// occurs only as a Store package is not mapped here either; its version is the package's.
/// </summary>
public class AutoResolveCpeMappingFunction
{
private readonly NvdApiClient _nvdApi;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly CpeDictionaryResolver _resolver;
private readonly CpeMappingStore _mappingStore;
private readonly SoftwareInventoryCpeReconciler _reconciler;
private readonly ILogger<AutoResolveCpeMappingFunction> _logger;
private const int MaxItemsPerRequest = 25;
public AutoResolveCpeMappingFunction(
NvdApiClient nvdApi,
IVulnerabilityRepository vulnRepo,
CpeDictionaryResolver resolver,
CpeMappingStore mappingStore,
SoftwareInventoryCpeReconciler reconciler,
ILogger<AutoResolveCpeMappingFunction> logger)
{
_nvdApi = nvdApi;
_vulnRepo = vulnRepo;
_resolver = resolver;
_mappingStore = mappingStore;
_reconciler = reconciler;
_logger = logger;
}
@@ -44,131 +51,47 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
try
{
// Global Admin only — enforced by PolicyEnforcementMiddleware
var userIdentifier = TenantHelper.GetUserIdentifier(req);
var read = await req.ReadAsync<AutoResolveCpeMappingRequest>();
if (read.Error != null) return read.Error;
var request = read.Value!;
if (request.Items == null || request.Items.Count == 0)
{
return await req.BadRequestAsync("items array is required");
}
if (request.Items.Count > MaxItemsPerRequest)
{
return await req.BadRequestAsync($"Maximum {MaxItemsPerRequest} items per request");
}
// Deduplicate by softwareName
var uniqueItems = request.Items
.GroupBy(i => (i.SoftwareName ?? "").ToLowerInvariant())
.Select(g => g.First())
.ToList();
var resolved = new List<AutoResolveResultItem>();
var failed = new List<AutoResolveFailedItem>();
var nvdDown = false;
var storeOnly = await _resolver.StoreOnlyKeysAsync();
foreach (var item in uniqueItems)
foreach (var item in request.Items)
{
if (string.IsNullOrWhiteSpace(item.SoftwareName))
{
_logger.LogWarning("Bulk auto-resolve: skipping item with empty name");
failed.Add(new AutoResolveFailedItem { SoftwareName = item.SoftwareName ?? "", Reason = "empty_name" });
failed.Add(new AutoResolveFailedItem { SoftwareName = item.SoftwareName ?? string.Empty, Reason = "empty_name" });
continue;
}
try
{
var keyword = CpeMatchScorer.BuildSearchKeyword(item.SoftwareName, item.NormalizedVendor ?? item.Publisher ?? "");
if (string.IsNullOrWhiteSpace(keyword))
{
failed.Add(new AutoResolveFailedItem { SoftwareName = item.SoftwareName, Reason = "empty_keyword" });
continue;
}
var cpeFetch = await _nvdApi.SearchCpeAsync(keyword);
if (!cpeFetch.Succeeded)
{
failed.Add(new AutoResolveFailedItem
{
SoftwareName = item.SoftwareName,
Reason = cpeFetch.Throttled ? "nvd_throttled" : "nvd_unavailable"
});
continue;
}
var cpeResponse = cpeFetch.Data;
var bestMatch = CpeMatchScorer.SelectBestMatch(
cpeResponse.Products,
item.NormalizedVendor ?? item.Publisher ?? "",
item.SoftwareName);
if (bestMatch == null)
{
failed.Add(new AutoResolveFailedItem
{
SoftwareName = item.SoftwareName,
Reason = cpeResponse.Products?.Count > 0 ? "no_match" : "no_results"
});
continue;
}
var (cpeEntry, score) = bestMatch.Value;
var mapping = CpeMatchScorer.BuildMappingFromNvdResult(
cpeEntry,
item.NormalizedVendor ?? item.Publisher ?? "",
item.SoftwareName,
item.SoftwareName);
// Save as custom mapping
var rowKey = TableKeySanitizer.Sanitize(
$"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant());
var properties = new Dictionary<string, object>
{
{ "NormalizedVendor", mapping.NormalizedVendor ?? "" },
{ "NormalizedProduct", mapping.NormalizedProduct ?? "" },
{ "CpeVendor", mapping.CpeVendor ?? "" },
{ "CpeProduct", mapping.CpeProduct ?? "" },
{ "CpeUri", mapping.CpeUri ?? "" },
{ "Category", "auto" },
{ "DisplayNamePatternsJson", JsonSerializer.Serialize(mapping.DisplayNamePatterns) },
{ "PublisherPatternsJson", JsonSerializer.Serialize(mapping.PublisherPatterns) },
{ "ExcludePatternsJson", JsonSerializer.Serialize(new List<string>()) },
{ "Source", "auto" },
{ "CreatedAt", DateTime.UtcNow.ToString("o") }
};
await _vulnRepo.UpsertCpeMappingAsync("cpe_map_custom", rowKey, properties);
await _vulnRepo.UpdateSoftwareInventoryCpeAsync(mapping.NormalizedProduct ?? "", mapping.CpeUri ?? "");
resolved.Add(new AutoResolveResultItem
{
SoftwareName = item.SoftwareName,
CpeUri = mapping.CpeUri ?? "",
Confidence = Math.Round(score, 2)
});
_logger.LogInformation("Bulk auto-resolved: '{SoftwareName}' -> {CpeUri} (score={Score:F2})",
item.SoftwareName, mapping.CpeUri, score);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Bulk auto-resolve failed for '{SoftwareName}'", item.SoftwareName);
failed.Add(new AutoResolveFailedItem { SoftwareName = item.SoftwareName, Reason = "nvd_api_error" });
}
var result = await _resolver.ResolveAsync(
item.NormalizedVendor, item.SoftwareName, item.Publisher, req.FunctionContext.CancellationToken,
askNvd: !nvdDown,
storePackage: storeOnly.Contains(CpeMappingMatcher.SoftwareKey(item.NormalizedVendor, item.SoftwareName)));
nvdDown |= result.NvdDown;
if (result.Outcome == CpeAutoMapOutcome.Mapped)
resolved.Add(new AutoResolveResultItem { SoftwareName = item.SoftwareName, CpeUri = result.CpeUri!, Confidence = 1 });
else
failed.Add(new AutoResolveFailedItem { SoftwareName = item.SoftwareName, Reason = ReasonOf(result.Outcome) });
}
// Reset mappings cache so future correlations pick up new mappings
if (resolved.Count > 0)
{
VulnerabilityCorrelationService.ResetMappingsCache();
_mappingStore.Invalidate();
await _reconciler.ReconcileAsync("auto_map");
}
_logger.LogInformation(
"Bulk auto-resolve complete: processed={Processed}, resolved={Resolved}, failed={Failed} " +
"(reasons: {FailedReasons})",
resolved.Count + failed.Count, resolved.Count, failed.Count,
"Auto-map complete: resolved={Resolved}, failed={Failed} (reasons: {FailedReasons})",
resolved.Count, failed.Count,
string.Join(", ", failed.GroupBy(f => f.Reason).Select(g => $"{g.Key}={g.Count()}")));
var response = req.CreateResponse(HttpStatusCode.OK);
@@ -182,10 +105,30 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
});
return response;
}
catch (CpeMappingUnavailableException)
{
return await req.ErrorAsync(
HttpStatusCode.ServiceUnavailable, Constants.ApiErrorCodes.ServiceUnavailable,
"The software mappings cannot be read right now. Nothing was changed; try again in a few minutes.",
retryAfterSeconds: 60);
}
catch (Exception ex)
{
return await req.InternalServerErrorAsync(_logger, ex, "AutoResolveCpeMapping");
}
}
internal static string ReasonOf(CpeAutoMapOutcome outcome) => outcome switch
{
CpeAutoMapOutcome.AlreadyMapped => "already_mapped",
CpeAutoMapOutcome.Ignored => "ignored",
CpeAutoMapOutcome.NoExactProduct => "no_exact_product",
CpeAutoMapOutcome.NoCves => "no_cves",
CpeAutoMapOutcome.UnversionedCves => "unversioned_cves",
CpeAutoMapOutcome.StorePackage => "store_package",
CpeAutoMapOutcome.NvdUnavailable => "nvd_unavailable",
CpeAutoMapOutcome.NoDictionary => "no_dictionary",
_ => "not_mapped",
};
}
}
@@ -1,28 +1,36 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Functions.Helpers;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Deletes an operator's mapping or an automatic one. Removing an automatic mapping also puts
/// the program on the ignore list: its name is still an NVD product name, so the next nightly
/// run would create the same mapping again.
/// </summary>
public class DeleteCustomCpeMappingFunction
{
private readonly IVulnerabilityRepository _vulnRepo;
private readonly SoftwareInventoryCpeReconciler _reconciler;
private readonly ILogger<DeleteCustomCpeMappingFunction> _logger;
public DeleteCustomCpeMappingFunction(
IVulnerabilityRepository vulnRepo,
SoftwareInventoryCpeReconciler reconciler,
ILogger<DeleteCustomCpeMappingFunction> logger)
{
_vulnRepo = vulnRepo;
_reconciler = reconciler;
_logger = logger;
}
@@ -32,44 +40,73 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
try
{
var authHeader = req.Headers.Contains("Authorization")
? req.Headers.GetValues("Authorization").FirstOrDefault()
: null;
if (string.IsNullOrEmpty(authHeader))
{
return req.CreateResponse(HttpStatusCode.Unauthorized);
}
// Global Admin only — enforced by PolicyEnforcementMiddleware
var read = await req.ReadAsync<DeleteCustomCpeMappingRequest>();
if (read.Error != null) return read.Error;
var request = read.Value!;
if (string.IsNullOrWhiteSpace(request.NormalizedProduct))
var source = (string.IsNullOrWhiteSpace(request.Source) ? "custom" : request.Source!).Trim().ToLowerInvariant();
if (source != "custom" && source != "auto")
return await req.BadRequestAsync("Only custom and automatic mappings can be deleted. The seed and the community list are changed in the rules repository.");
if (string.IsNullOrWhiteSpace(request.RowKey) && string.IsNullOrWhiteSpace(request.NormalizedProduct))
return await req.BadRequestAsync("rowKey or normalizedProduct is required");
var rowKey = !string.IsNullOrWhiteSpace(request.RowKey)
? request.RowKey!
: CpeSeedFile.RowKeyOf(
CpeMappingMatcher.VendorKey(request.NormalizedVendor),
SoftwareNameCanonicalizer.Canonicalize(request.NormalizedProduct));
// A mapping is listed as automatic wherever it is stored: in the automatic mapper's
// partition, or among the operator's rows, where the retired name search put its own.
// The repository's delete does not tell a missing row from a deleted one, so the
// row is looked up first.
var wanted = source == "auto" ? CpeMappingSource.Auto : CpeMappingSource.Custom;
var partitions = source == "auto"
? new[] { CpeMappingStore.AutoPartition, CpeMappingStore.CustomPartition }
: new[] { CpeMappingStore.CustomPartition };
string? partition = null;
Dictionary<string, object>? row = null;
foreach (var candidate in partitions)
{
return await req.BadRequestAsync("normalizedProduct is required");
row = (await _vulnRepo.GetCpeMappingsByPartitionAsync(candidate)).FirstOrDefault(m =>
string.Equals(m.GetValueOrDefault("RowKey")?.ToString(), rowKey, StringComparison.Ordinal)
&& CpeMappingStore.SourceOf(candidate, m.GetValueOrDefault("Source")?.ToString() ?? string.Empty) == wanted);
if (row == null) continue;
partition = candidate;
break;
}
var rowKey = TableKeySanitizer.Sanitize(
$"{(request.NormalizedVendor ?? "unknown")}:{request.NormalizedProduct}".ToLowerInvariant());
// Check if mapping exists before deleting (repository silently ignores 404)
var existingMappings = await _vulnRepo.GetCpeMappingsByPartitionAsync("cpe_map_custom");
var exists = existingMappings.Any(m =>
m.ContainsKey("RowKey") && string.Equals(m["RowKey"]?.ToString(), rowKey, StringComparison.Ordinal));
if (!exists)
{
if (row == null || partition == null)
return await req.NotFoundAsync("Mapping not found");
// Only what the automatic mapper created comes back by itself. The ignore entry is
// written first: a failure between the two writes then leaves the mapping in place
// instead of letting the next night recreate it.
var vendor = row.GetValueOrDefault("NormalizedVendor")?.ToString() ?? string.Empty;
var product = row.GetValueOrDefault("NormalizedProduct")?.ToString() ?? string.Empty;
var wasAutomatic = partition == CpeMappingStore.AutoPartition;
if (wasAutomatic && CpeIgnoreEntry.Build(vendor, product, "automatic mapping removed") is { } ignore)
await _vulnRepo.UpsertCpeMappingAsync(CpeMappingStore.IgnoredPartition, ignore.RowKey, ignore.Properties);
await _vulnRepo.DeleteCpeMappingAsync(partition, rowKey);
_logger.LogInformation("CPE mapping deleted: {Partition}/{RowKey} (automatic={Automatic})", partition, rowKey, wasAutomatic);
try
{
await _reconciler.ReconcileAsync("mapping_deleted");
}
catch (CpeMappingUnavailableException ex)
{
_logger.LogWarning(ex, "Inventory rows not reconciled after deleting {RowKey}", rowKey);
}
await _vulnRepo.DeleteCpeMappingAsync("cpe_map_custom", rowKey);
VulnerabilityCorrelationService.ResetMappingsCache();
_logger.LogInformation("Custom CPE mapping deleted: {RowKey}", rowKey);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new SuccessMessageResponse { Success = true, Message = "CPE mapping deleted" });
await response.WriteAsJsonAsync(new SuccessMessageResponse
{
Success = true,
Message = wasAutomatic ? "CPE mapping deleted; the program is on the ignore list now" : "CPE mapping deleted",
});
return response;
}
catch (Exception ex)
@@ -1,11 +1,10 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -14,20 +13,18 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Admin endpoint that returns all CPE mappings from all sources (seed, custom, community).
/// Used by the Admin UI Software Mapping section to browse, search, and edit existing mappings.
/// Every mapping in service with its source and, per product, whether it counts as
/// self-updating. Global Reader or Global Admin.
/// </summary>
public class GetCpeMappingsFunction
{
private readonly CpeMappingStore _mappingStore;
private readonly ILogger<GetCpeMappingsFunction> _logger;
private readonly IVulnerabilityRepository _vulnRepo;
public GetCpeMappingsFunction(
ILogger<GetCpeMappingsFunction> logger,
IVulnerabilityRepository vulnRepo)
public GetCpeMappingsFunction(CpeMappingStore mappingStore, ILogger<GetCpeMappingsFunction> logger)
{
_mappingStore = mappingStore;
_logger = logger;
_vulnRepo = vulnRepo;
}
[Function("GetCpeMappings")]
@@ -36,85 +33,69 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
try
{
// Global Admin only — enforced by PolicyEnforcementMiddleware
var allEntities = await _vulnRepo.GetAllCpeMappingsAsync();
// Global Reader or Global Admin — enforced by PolicyEnforcementMiddleware.
// Read fresh: the operator who just changed a mapping or a flag must see it here,
// whichever instance took the change.
var snapshot = await _mappingStore.ReloadAsync();
var mappings = new List<CpeMappingItem>();
foreach (var entity in allEntities)
{
var partitionKey = entity.ContainsKey("PartitionKey") ? entity["PartitionKey"]?.ToString() ?? "" : "";
var source = partitionKey switch
{
"cpe_map_seed" => "seed",
"cpe_map_custom" => "custom",
"cpe_map_community" => "community",
_ => entity.ContainsKey("Source") ? entity["Source"]?.ToString() ?? "unknown" : "unknown"
};
List<string> displayNamePatterns;
List<string> publisherPatterns;
try
{
displayNamePatterns = JsonSerializer.Deserialize<List<string>>(
entity.ContainsKey("DisplayNamePatternsJson") ? entity["DisplayNamePatternsJson"]?.ToString() ?? "[]" : "[]") ?? new List<string>();
}
catch
{
displayNamePatterns = new List<string>();
}
try
{
publisherPatterns = JsonSerializer.Deserialize<List<string>>(
entity.ContainsKey("PublisherPatternsJson") ? entity["PublisherPatternsJson"]?.ToString() ?? "[]" : "[]") ?? new List<string>();
}
catch
{
publisherPatterns = new List<string>();
}
List<string> excludePatterns;
try
{
excludePatterns = JsonSerializer.Deserialize<List<string>>(
entity.ContainsKey("ExcludePatternsJson") ? entity["ExcludePatternsJson"]?.ToString() ?? "[]" : "[]") ?? new List<string>();
}
catch
{
excludePatterns = new List<string>();
}
mappings.Add(new CpeMappingItem
{
NormalizedVendor = entity.ContainsKey("NormalizedVendor") ? entity["NormalizedVendor"]?.ToString() ?? "" : "",
NormalizedProduct = entity.ContainsKey("NormalizedProduct") ? entity["NormalizedProduct"]?.ToString() ?? "" : "",
CpeVendor = entity.ContainsKey("CpeVendor") ? entity["CpeVendor"]?.ToString() ?? "" : "",
CpeProduct = entity.ContainsKey("CpeProduct") ? entity["CpeProduct"]?.ToString() ?? "" : "",
CpeUri = entity.ContainsKey("CpeUri") ? entity["CpeUri"]?.ToString() ?? "" : "",
Category = entity.ContainsKey("Category") ? entity["Category"]?.ToString() ?? "" : "",
DisplayNamePatterns = displayNamePatterns,
PublisherPatterns = publisherPatterns,
ExcludePatterns = excludePatterns,
Source = source,
CreatedAt = entity.ContainsKey("CreatedAt") ? entity["CreatedAt"]?.ToString() ?? "" :
entity.ContainsKey("ImportedAt") ? entity["ImportedAt"]?.ToString() ?? "" : ""
});
}
// Sort by source (custom first for visibility), then by product name
var sorted = mappings
.OrderBy(m => m.Source == "custom" ? 0 : m.Source == "community" ? 1 : 2)
var mappings = snapshot.Mappings
.Select(m => ToItem(m, snapshot))
// The operator's own entries first, then the automatic ones to look over.
.OrderBy(m => SourceOrder(m.Source))
.ThenBy(m => m.NormalizedProduct, StringComparer.OrdinalIgnoreCase)
.ToList();
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new GetCpeMappingsResponse { Mappings = sorted, Total = sorted.Count });
await response.WriteAsJsonAsync(new GetCpeMappingsResponse { Mappings = mappings, Total = mappings.Count });
return response;
}
catch (CpeMappingUnavailableException)
{
return await req.ErrorAsync(
HttpStatusCode.ServiceUnavailable, Constants.ApiErrorCodes.ServiceUnavailable,
"The software mappings cannot be read right now. Try again in a few minutes.", retryAfterSeconds: 60);
}
catch (Exception ex)
{
return await req.InternalServerErrorAsync(_logger, ex, "GetCpeMappings");
}
}
internal static CpeMappingItem ToItem(CpeMapping mapping, CpeMappingSnapshot snapshot)
{
var flags = snapshot.FlagsOf(mapping.ProductKey);
var separator = mapping.ProductKey.IndexOf(':');
return new CpeMappingItem
{
NormalizedVendor = mapping.Vendor,
NormalizedProduct = mapping.Product,
CpeVendor = separator > 0 ? mapping.ProductKey.Substring(0, separator) : string.Empty,
CpeProduct = separator > 0 ? mapping.ProductKey.Substring(separator + 1) : string.Empty,
CpeUri = mapping.CpeUri,
Category = mapping.Category,
DisplayNamePatterns = mapping.DisplayNamePatterns,
PublisherPatterns = mapping.PublisherPatterns,
ExcludePatterns = mapping.ExcludePatterns,
Source = mapping.Source switch
{
CpeMappingSource.Seed => "seed",
CpeMappingSource.Community => "community",
CpeMappingSource.Auto => "auto",
_ => "custom",
},
CreatedAt = mapping.CreatedAt,
RowKey = mapping.RowKey,
SelfUpdating = flags?.SelfUpdating ?? false,
SelfUpdatingDefault = flags?.SeedSelfUpdating ?? false,
};
}
private static int SourceOrder(string source) => source switch
{
"custom" => 0,
"auto" => 1,
"community" => 2,
_ => 3,
};
}
}
@@ -4,8 +4,8 @@ using System.Linq;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Security;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
@@ -15,25 +15,27 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Admin endpoint that returns software without CPE mappings from the SoftwareInventory table.
/// Used by the /update-cpe-seed skill and Admin UI to identify gaps in the CPE mapping.
/// Entries in SoftwareInventory with an empty CpeUri are considered "unmatched".
/// The programs across all tenants that no mapping covers (see <see cref="UnmappedSoftware"/>),
/// most used first. Global Reader or Global Admin.
/// </summary>
public class GetUnmatchedSoftwareFunction
{
private readonly ILogger<GetUnmatchedSoftwareFunction> _logger;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly CpeMappingStore _mappingStore;
private readonly ILogger<GetUnmatchedSoftwareFunction> _logger;
public GetUnmatchedSoftwareFunction(
ILogger<GetUnmatchedSoftwareFunction> logger,
IVulnerabilityRepository vulnRepo)
IVulnerabilityRepository vulnRepo,
CpeMappingStore mappingStore,
ILogger<GetUnmatchedSoftwareFunction> logger)
{
_logger = logger;
_vulnRepo = vulnRepo;
_mappingStore = mappingStore;
_logger = logger;
}
// Server-side pagination bounds. Aggregation runs on the full inventory; only the response
// slice is paginated, so callers can iterate through every unmatched entry.
// Paging: the full unmatched set is aggregated in memory (deduped across tenants), then a
// slice is returned, so callers can iterate through every unmatched entry.
private const int DefaultTake = 50;
private const int MaxTake = 200;
@@ -44,97 +46,36 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
try
{
// Global Reader or Global Admin (GlobalReadOrAdmin) — enforced by PolicyEnforcementMiddleware
var userIdentifier = TenantHelper.GetUserIdentifier(req);
var query = req.Query;
var skip = QueryParams.Int(query["skip"], @default: 0, min: 0, max: int.MaxValue);
var take = QueryParams.Int(query["take"], DefaultTake, 1, MaxTake);
// Query all SoftwareInventory entries across tenants and filter for unmatched (no CpeUri).
// Server-side filtering on CpeUri is NOT possible: rows re-created by the cascade-delete
// 404 path carry no CpeUri property at all, and Table Storage cannot express
// "property missing" — such rows must still count as unmatched. Project to the columns
// this aggregation reads instead of materializing full entities.
var allEntities = await _vulnRepo.GetAllSoftwareInventoryAsync(new[]
{
"CpeUri", "NormalizedName", "DisplayName", "Publisher", "SessionCount",
"LastSeenAt", "LastSessionId", "NormalizedVendor", "NormalizedVersion"
});
// Load the persistent ignore list to exclude ignored software
var ignoredEntities = await _vulnRepo.GetCpeMappingsByPartitionAsync("cpe_ignored");
var ignoredNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var ie in ignoredEntities)
{
var name = ie.GetValueOrDefault("NormalizedProduct")?.ToString() ?? "";
if (!string.IsNullOrWhiteSpace(name))
ignoredNames.Add(name.ToLowerInvariant());
}
var unmatchedAgg = new Dictionary<string, UnmatchedSoftwareItem>(StringComparer.OrdinalIgnoreCase);
foreach (var entity in allEntities)
{
var cpeUri = entity.ContainsKey("CpeUri") ? entity["CpeUri"]?.ToString() ?? "" : "";
if (!string.IsNullOrEmpty(cpeUri))
continue; // Has a CPE mapping — not unmatched
var normalizedName = entity.ContainsKey("NormalizedName") ? entity["NormalizedName"]?.ToString() ?? "" : "";
if (string.IsNullOrWhiteSpace(normalizedName))
continue;
// Skip software on the persistent ignore list
if (ignoredNames.Contains(normalizedName.ToLowerInvariant()))
continue;
var key = normalizedName.ToLowerInvariant();
var sessionCount = entity.ContainsKey("SessionCount") && entity["SessionCount"] is int sc ? sc : 1;
var lastSeenAt = entity.ContainsKey("LastSeenAt") ? entity["LastSeenAt"]?.ToString() ?? "" : "";
var lastSessionId = entity.ContainsKey("LastSessionId") ? entity["LastSessionId"]?.ToString() ?? "" : "";
if (!unmatchedAgg.TryGetValue(key, out var existing))
{
unmatchedAgg[key] = new UnmatchedSoftwareItem
{
SoftwareName = entity.ContainsKey("DisplayName") ? entity["DisplayName"]?.ToString() ?? normalizedName : normalizedName,
Publisher = entity.ContainsKey("Publisher") ? entity["Publisher"]?.ToString() ?? "" : "",
Frequency = sessionCount,
LastSeenAt = lastSeenAt,
ExampleSessionId = lastSessionId,
NormalizedVendor = entity.ContainsKey("NormalizedVendor") ? entity["NormalizedVendor"]?.ToString() ?? "" : "",
NormalizedVersion = entity.ContainsKey("NormalizedVersion") ? entity["NormalizedVersion"]?.ToString() ?? "" : ""
};
}
else
{
// Aggregate across tenants: sum frequency, keep most recent
existing.Frequency += sessionCount;
if (string.Compare(lastSeenAt, existing.LastSeenAt, StringComparison.Ordinal) > 0)
{
existing.LastSeenAt = lastSeenAt;
existing.ExampleSessionId = lastSessionId;
}
}
}
// Read fresh: a program the operator just mapped or ignored must be gone from the
// list, whichever instance took the change.
var mappings = await _mappingStore.ReloadAsync();
var rows = await _vulnRepo.GetAllSoftwareInventoryAsync(UnmappedSoftware.Columns);
var unmatched = UnmappedSoftware.Aggregate(rows, mappings);
// Stable order (Frequency desc, then case-insensitive name) so paging is deterministic.
var ordered = unmatchedAgg.Values
var page = unmatched
.OrderByDescending(x => x.Frequency)
.ThenBy(x => x.SoftwareName, StringComparer.OrdinalIgnoreCase);
var total = unmatchedAgg.Count;
var page = ordered.Skip(skip).Take(take).ToList();
.ThenBy(x => x.SoftwareName, StringComparer.OrdinalIgnoreCase)
.ThenBy(x => x.NormalizedVendor, StringComparer.OrdinalIgnoreCase)
.Skip(skip).Take(take).ToList();
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new GetUnmatchedSoftwareResponse { Software = page, Total = total, Skip = skip, Take = take });
await response.WriteAsJsonAsync(new GetUnmatchedSoftwareResponse { Software = page, Total = unmatched.Count, Skip = skip, Take = take });
return response;
}
catch (CpeMappingUnavailableException)
{
return await req.ErrorAsync(
HttpStatusCode.ServiceUnavailable, Constants.ApiErrorCodes.ServiceUnavailable,
"The software mappings cannot be read right now. Try again in a few minutes.", retryAfterSeconds: 60);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get unmatched software report");
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
await errorResponse.WriteStringAsync("Internal server error");
return errorResponse;
return await req.InternalServerErrorAsync(_logger, ex, "GetUnmatchedSoftware");
}
}
}
@@ -25,6 +25,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
private readonly VulnerabilityCorrelationService _correlationService;
private readonly IVulnerabilityInventoryLoader _inventoryLoader;
private readonly Services.Deletion.SessionDeletionGuard _deletionGuard;
private readonly VulnerabilityReportWriter _reportWriter;
public GetVulnerabilityReportFunction(
ILogger<GetVulnerabilityReportFunction> logger,
@@ -32,7 +33,8 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
ISessionRepository sessionRepo,
VulnerabilityCorrelationService correlationService,
IVulnerabilityInventoryLoader inventoryLoader,
Services.Deletion.SessionDeletionGuard deletionGuard)
Services.Deletion.SessionDeletionGuard deletionGuard,
VulnerabilityReportWriter reportWriter)
{
_logger = logger;
_vulnRepo = vulnRepo;
@@ -40,6 +42,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
_correlationService = correlationService;
_inventoryLoader = inventoryLoader;
_deletionGuard = deletionGuard;
_reportWriter = reportWriter;
}
[Function("GetVulnerabilityReport")]
@@ -96,31 +99,45 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
return noInventory;
}
var storedReport = await _vulnRepo.GetVulnerabilityReportAsync(effectiveTenantId, sessionId);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(60));
var freshReport = await _correlationService.CorrelateAsync(
sessionId, effectiveTenantId, inventoryItems.ToList(), cts.Token);
_logger.LogInformation("Vulnerability re-scan: CorrelateAsync returned {Result} for session {SessionId}",
freshReport != null ? "report with findings" : "null (no findings)", sessionId);
if (freshReport != null)
VulnerabilityCorrelationOutcome outcome;
try
{
await _vulnRepo.StoreVulnerabilityReportAsync(effectiveTenantId, sessionId, freshReport);
_logger.LogInformation("Vulnerability re-scan complete for session {SessionId}: stored fresh report", sessionId);
var rescanResponse = req.CreateResponse(HttpStatusCode.OK);
await rescanResponse.WriteAsJsonAsync(new GetVulnerabilityReportResponse { Success = true, SessionId = sessionId, Report = freshReport, Rescanned = true });
return rescanResponse;
outcome = await _correlationService.CorrelateAsync(
sessionId, effectiveTenantId, inventoryItems.ToList(), cts.Token);
}
catch (CpeMappingUnavailableException)
{
return await req.ErrorAsync(
HttpStatusCode.ServiceUnavailable, Constants.ApiErrorCodes.ServiceUnavailable,
"The software mappings cannot be read right now. Nothing was changed; try the re-scan again in a few minutes.",
retryAfterSeconds: 300);
}
// Rescan found no vulnerabilities — clear the stale report so the
// UI no longer shows outdated findings from a previous scan.
await _vulnRepo.DeleteVulnerabilityReportAsync(effectiveTenantId, sessionId);
_logger.LogInformation("Vulnerability re-scan complete for session {SessionId}: no findings, deleted old report", sessionId);
// The same writer as the queue path: the stored report is replaced or removed
// and the session's CveIndex rows follow it.
await _reportWriter.StoreAsync(effectiveTenantId, sessionId, outcome.Report, storedReport);
var noFindingsResponse = req.CreateResponse(HttpStatusCode.OK);
await noFindingsResponse.WriteAsJsonAsync(new GetVulnerabilityReportResponse { Success = true, SessionId = sessionId, Report = null, Rescanned = true, Message = "Re-scan complete: no vulnerabilities found" });
return noFindingsResponse;
_logger.LogInformation(
"Vulnerability re-scan complete for session {SessionId}: {Result}, {Pending} products without CVE data",
sessionId, outcome.Report != null ? "report stored" : "no findings, stored report removed", outcome.PendingProducts.Count);
var rescanResponse = req.CreateResponse(HttpStatusCode.OK);
await rescanResponse.WriteAsJsonAsync(new GetVulnerabilityReportResponse
{
Success = true,
SessionId = sessionId,
Report = outcome.Report,
Rescanned = true,
Message = outcome.Report == null
? "Re-scan complete: no vulnerabilities found"
: outcome.PendingProducts.Count > 0
? $"CVE data for {outcome.PendingProducts.Count} products is still being fetched. They are listed as not evaluated; re-scan again later."
: null,
});
return rescanResponse;
}
catch (Exception ex)
{
@@ -71,7 +71,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
NvdLastRun = nvd == null ? null : new NvdRefreshRunWire
{
Stale = nvd.Stale, Refreshed = nvd.Refreshed, Failed = nvd.Failed,
SkippedThrottled = nvd.SkippedThrottled, Error = nvd.Error,
SkippedThrottled = nvd.SkippedThrottled, SkippedUnavailable = nvd.SkippedUnavailable, Error = nvd.Error,
StartedUtc = nvd.StartedUtc.ToString("o"), FinishedUtc = nvd.FinishedUtc?.ToString("o"),
},
EpssLastSyncUtc = config?.EpssLastSyncUtc,
@@ -15,21 +15,25 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Manage the persistent ignore list for software that should never be auto-resolved.
/// Manage the persistent ignore list: programs an operator decided need no mapping. They stay
/// out of the unmatched list and get no automatic mapping.
/// Entries stored in VulnerabilityCache table under partition "cpe_ignored".
/// </summary>
public class IgnoreSoftwareFunction
{
private readonly IVulnerabilityRepository _vulnRepo;
private readonly CpeMappingStore _mappingStore;
private readonly ILogger<IgnoreSoftwareFunction> _logger;
private const string IgnorePartition = "cpe_ignored";
private const string IgnorePartition = CpeMappingStore.IgnoredPartition;
public IgnoreSoftwareFunction(
IVulnerabilityRepository vulnRepo,
CpeMappingStore mappingStore,
ILogger<IgnoreSoftwareFunction> logger)
{
_vulnRepo = vulnRepo;
_mappingStore = mappingStore;
_logger = logger;
}
@@ -83,23 +87,16 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
int added = 0;
foreach (var item in request.Items)
{
if (string.IsNullOrWhiteSpace(item.SoftwareName)) continue;
var entry = CpeIgnoreEntry.Build(
string.IsNullOrWhiteSpace(item.NormalizedVendor) ? item.Publisher : item.NormalizedVendor,
item.SoftwareName, item.Reason ?? "manual");
if (entry == null) continue;
var rowKey = TableKeySanitizer.Sanitize(
$"{(item.Publisher ?? "unknown")}:{item.SoftwareName}".ToLowerInvariant());
var properties = new Dictionary<string, object>
{
{ "NormalizedProduct", item.SoftwareName },
{ "NormalizedVendor", item.Publisher ?? "" },
{ "Reason", item.Reason ?? "manual" },
{ "CreatedAt", DateTime.UtcNow.ToString("o") }
};
await _vulnRepo.UpsertCpeMappingAsync(IgnorePartition, rowKey, properties);
await _vulnRepo.UpsertCpeMappingAsync(IgnorePartition, entry.Value.RowKey, entry.Value.Properties);
added++;
}
if (added > 0) _mappingStore.Invalidate();
_logger.LogInformation("Added {Count} software items to ignore list", added);
var response = req.CreateResponse(HttpStatusCode.OK);
@@ -130,10 +127,27 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
return await req.BadRequestAsync("softwareName is required");
}
var rowKey = TableKeySanitizer.Sanitize(
$"{(request.Publisher ?? "unknown")}:{request.SoftwareName}".ToLowerInvariant());
// What counts as ignored is decided by an entry's stored vendor and name, not by its
// row key: an entry written before programs were filed under their canonical name
// has another key and is found the same way.
// An entry whose name is no program name (a bare version, a KB number) has no key
// and ignores nothing, but it is listed: it is found by what the list shows for it.
var key = CpeMappingMatcher.SoftwareKey(request.Publisher, request.SoftwareName);
foreach (var entry in await _vulnRepo.GetCpeMappingsByPartitionAsync(IgnorePartition))
{
var storedVendor = entry.GetValueOrDefault("NormalizedVendor")?.ToString() ?? string.Empty;
var storedName = entry.GetValueOrDefault("NormalizedProduct")?.ToString() ?? string.Empty;
var rowKey = entry.GetValueOrDefault("RowKey")?.ToString() ?? string.Empty;
if (rowKey.Length == 0) continue;
await _vulnRepo.DeleteCpeMappingAsync(IgnorePartition, rowKey);
var sameKey = key.Length > 0
&& string.Equals(CpeMappingMatcher.SoftwareKey(storedVendor, storedName), key, StringComparison.Ordinal);
var asListed = string.Equals(storedName, request.SoftwareName, StringComparison.Ordinal)
&& string.Equals(storedVendor, request.Publisher ?? string.Empty, StringComparison.Ordinal);
if (sameKey || asListed)
await _vulnRepo.DeleteCpeMappingAsync(IgnorePartition, rowKey);
}
_mappingStore.Invalidate();
_logger.LogInformation("Removed '{SoftwareName}' from ignore list", request.SoftwareName);
@@ -4,26 +4,34 @@ using System.Linq;
using System.Net;
using System.Text.Json;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using AutopilotMonitor.Shared.Models;
using AutopilotMonitor.Functions.Helpers;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Saves an operator's mapping for one program. The program is filed under its vendor and its
/// canonical name (no version, architecture or language), so one mapping covers every variant
/// of it, and the inventory rows take the new CPE right away.
/// </summary>
public class SaveCustomCpeMappingFunction
{
private readonly IVulnerabilityRepository _vulnRepo;
private readonly SoftwareInventoryCpeReconciler _reconciler;
private readonly ILogger<SaveCustomCpeMappingFunction> _logger;
public SaveCustomCpeMappingFunction(
IVulnerabilityRepository vulnRepo,
SoftwareInventoryCpeReconciler reconciler,
ILogger<SaveCustomCpeMappingFunction> logger)
{
_vulnRepo = vulnRepo;
_reconciler = reconciler;
_logger = logger;
}
@@ -33,61 +41,80 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
try
{
// Global Admin only - require Authorization header
var authHeader = req.Headers.Contains("Authorization")
? req.Headers.GetValues("Authorization").FirstOrDefault()
: null;
if (string.IsNullOrEmpty(authHeader))
{
var unauthorized = req.CreateResponse(HttpStatusCode.Unauthorized);
return unauthorized;
}
// Global Admin only — enforced by PolicyEnforcementMiddleware
var read = await req.ReadAsync<SaveCustomCpeMappingRequest>();
if (read.Error != null) return read.Error;
var mapping = read.Value!;
if (string.IsNullOrWhiteSpace(mapping.NormalizedProduct) || string.IsNullOrWhiteSpace(mapping.CpeUri))
{
return await req.BadRequestAsync("normalizedProduct and cpeUri are required");
}
// Normalize CPE 2.3 quoting (%2b/%2b/bare '+' -> '\+'). See CpeUriNormalizer for rationale.
mapping.CpeUri = CpeUriNormalizer.Normalize(mapping.CpeUri);
mapping.CpeProduct = CpeUriNormalizer.Normalize(mapping.CpeProduct);
var rowKey = TableKeySanitizer.Sanitize(
$"{(mapping.NormalizedVendor ?? "unknown")}:{mapping.NormalizedProduct}".ToLowerInvariant());
var vendor = CpeMappingMatcher.VendorKey(mapping.NormalizedVendor);
var product = SoftwareNameCanonicalizer.Canonicalize(mapping.NormalizedProduct);
// Normalize CPE 2.3 quoting (%2b / bare '+' -> '\+') before reducing it to its product.
var cpeUri = CpeName.ToProductUri(CpeUriNormalizer.Normalize(mapping.CpeUri));
if (product.Length == 0 || !CpeName.TryParse(cpeUri, out var cpe) || cpe.ProductKey.Length == 0)
return await req.BadRequestAsync("normalizedProduct and a cpeUri of the form cpe:2.3:a:vendor:product are required");
var rowKey = CpeSeedFile.RowKeyOf(vendor, product);
var properties = new Dictionary<string, object>
{
{ "NormalizedVendor", mapping.NormalizedVendor ?? "" },
{ "NormalizedProduct", mapping.NormalizedProduct },
{ "CpeVendor", mapping.CpeVendor ?? "" },
{ "CpeProduct", mapping.CpeProduct ?? "" },
{ "CpeUri", mapping.CpeUri },
{ "NormalizedVendor", vendor },
{ "NormalizedProduct", product },
{ "CpeVendor", cpe.Vendor },
{ "CpeProduct", cpe.Product },
{ "CpeUri", cpeUri },
{ "Category", mapping.Category ?? "custom" },
{ "DisplayNamePatternsJson", JsonSerializer.Serialize(mapping.DisplayNamePatterns ?? new List<string>()) },
{ "PublisherPatternsJson", JsonSerializer.Serialize(mapping.PublisherPatterns ?? new List<string>()) },
{ "ExcludePatternsJson", JsonSerializer.Serialize(mapping.ExcludePatterns ?? new List<string>()) },
{ "DisplayNamePatternsJson", JsonSerializer.Serialize(Words(mapping.DisplayNamePatterns)) },
{ "PublisherPatternsJson", JsonSerializer.Serialize(Words(mapping.PublisherPatterns)) },
{ "ExcludePatternsJson", JsonSerializer.Serialize(Words(mapping.ExcludePatterns)) },
{ "Source", "manual" },
{ "CreatedAt", DateTime.UtcNow.ToString("o") }
{ "CreatedAt", DateTime.UtcNow.ToString("o") },
{ "CreatedBy", TenantHelper.GetUserIdentifier(req) }
};
await _vulnRepo.UpsertCpeMappingAsync("cpe_map_custom", rowKey, properties);
await _vulnRepo.UpsertCpeMappingAsync(CpeMappingStore.CustomPartition, rowKey, properties);
// Reset the static mappings cache so future correlations use the new mapping
VulnerabilityCorrelationService.ResetMappingsCache();
// A mapping saved before programs were filed under their canonical name sits in
// another row with the same key. Among equals the matcher keeps the first row, so
// the earlier one would go on winning over the correction.
try
{
var key = CpeMappingMatcher.SoftwareKey(vendor, product);
foreach (var earlier in await _vulnRepo.GetCpeMappingsByPartitionAsync(CpeMappingStore.CustomPartition))
{
var earlierRowKey = earlier.GetValueOrDefault("RowKey")?.ToString() ?? string.Empty;
if (earlierRowKey.Length == 0 || string.Equals(earlierRowKey, rowKey, StringComparison.Ordinal)) continue;
var earlierKey = CpeMappingMatcher.SoftwareKey(
earlier.GetValueOrDefault("NormalizedVendor")?.ToString(), earlier.GetValueOrDefault("NormalizedProduct")?.ToString());
if (string.Equals(earlierKey, key, StringComparison.Ordinal))
await _vulnRepo.DeleteCpeMappingAsync(CpeMappingStore.CustomPartition, earlierRowKey);
}
}
catch (Exception ex)
{
// The mapping is stored. An earlier row that is still there is removed by the next save.
_logger.LogWarning(ex, "Earlier rows for {RowKey} could not be looked up after the save", rowKey);
}
_logger.LogInformation("Custom CPE mapping saved: {RowKey} -> {CpeUri}", rowKey, cpeUri);
// Update SoftwareInventory entries that match this product so they no longer appear as "unmatched"
await _vulnRepo.UpdateSoftwareInventoryCpeAsync(mapping.NormalizedProduct, mapping.CpeUri);
_logger.LogInformation(
"Custom CPE mapping saved: {RowKey} -> {CpeUri}",
rowKey, mapping.CpeUri);
int? inventoryRowsChanged = null;
try
{
var reconciled = await _reconciler.ReconcileAsync("mapping_saved");
inventoryRowsChanged = reconciled.Set + reconciled.Cleared;
}
catch (CpeMappingUnavailableException ex)
{
// The mapping is stored; the nightly run brings the inventory rows in line.
_logger.LogWarning(ex, "Inventory rows not reconciled after saving {RowKey}", rowKey);
}
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new SaveCustomCpeMappingResponse { Success = true, Message = "CPE mapping saved", RowKey = rowKey });
await response.WriteAsJsonAsync(new SaveCustomCpeMappingResponse
{
Success = true,
Message = "CPE mapping saved",
RowKey = rowKey,
InventoryRowsChanged = inventoryRowsChanged,
});
return response;
}
catch (Exception ex)
@@ -95,5 +122,9 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
return await req.InternalServerErrorAsync(_logger, ex, "SaveCustomCpeMapping");
}
}
// A pattern without a word can never match and would only fail the next list check.
private static List<string> Words(List<string>? patterns) =>
(patterns ?? new List<string>()).Where(p => CpeMappingMatcher.Tokenize(p).Length > 0).ToList();
}
}
@@ -0,0 +1,83 @@
using System;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared;
using AutopilotMonitor.Shared.Models;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Sets whether a product counts as self-updating, for all tenants. Such software is listed
/// with all its findings, but they are kept out of the counts that make a device an urgent
/// case. The seed list ships the default; this is the operator's way to deviate from it or
/// to return to it.
/// </summary>
public class SetCpeProductFlagsFunction
{
private readonly CpeProductFlagsService _productFlags;
private readonly ILogger<SetCpeProductFlagsFunction> _logger;
public SetCpeProductFlagsFunction(CpeProductFlagsService productFlags, ILogger<SetCpeProductFlagsFunction> logger)
{
_productFlags = productFlags;
_logger = logger;
}
[Function("SetCpeProductFlags")]
public async Task<HttpResponseData> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "vulnerability/cpe-product-flags")] HttpRequestData req)
{
try
{
// Global Admin only — enforced by PolicyEnforcementMiddleware
var read = await req.ReadAsync<SetCpeProductFlagsRequest>();
if (read.Error != null) return read.Error;
var request = read.Value!;
if (string.IsNullOrWhiteSpace(request.Product) || request.SelfUpdating == null)
return await req.BadRequestAsync("product (vendor:product) and selfUpdating are required");
CpeProductFlags flags;
try
{
flags = await _productFlags.SetSelfUpdatingAsync(
request.Product, request.SelfUpdating.Value, TenantHelper.GetUserIdentifier(req));
}
catch (ArgumentException)
{
return await req.NotFoundAsync("No mapping points to this product.");
}
_logger.LogInformation(
"Product flag set: {Product} selfUpdating={SelfUpdating} (default {Default})",
flags.Product, flags.SelfUpdating, flags.SeedSelfUpdating ?? false);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new SetCpeProductFlagsResponse
{
Success = true,
Product = flags.Product,
SelfUpdating = flags.SelfUpdating,
SelfUpdatingDefault = flags.SeedSelfUpdating ?? false,
});
return response;
}
catch (CpeMappingUnavailableException)
{
return await req.ErrorAsync(
HttpStatusCode.ServiceUnavailable, Constants.ApiErrorCodes.ServiceUnavailable,
"The software mappings cannot be read right now. Nothing was changed; try again in a few minutes.",
retryAfterSeconds: 60);
}
catch (Exception ex)
{
return await req.InternalServerErrorAsync(_logger, ex, "SetCpeProductFlags");
}
}
}
}
@@ -1,4 +1,5 @@
using System;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services;
@@ -21,21 +22,21 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
private readonly KevDataService _kevService;
private readonly AdminConfigurationService _adminConfigService;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly GitHubRuleRepository _gitHubRepo;
private readonly CpeMappingImporter _cpeImporter;
private readonly SoftwareInventoryCpeReconciler _reconciler;
private readonly ILogger<TriggerVulnerabilityDataSyncFunction> _logger;
public TriggerVulnerabilityDataSyncFunction(
KevDataService kevService,
AdminConfigurationService adminConfigService,
IVulnerabilityRepository vulnRepo,
GitHubRuleRepository gitHubRepo,
CpeMappingImporter cpeImporter,
SoftwareInventoryCpeReconciler reconciler,
ILogger<TriggerVulnerabilityDataSyncFunction> logger)
{
_kevService = kevService;
_adminConfigService = adminConfigService;
_vulnRepo = vulnRepo;
_gitHubRepo = gitHubRepo;
_cpeImporter = cpeImporter;
_reconciler = reconciler;
_logger = logger;
}
@@ -46,7 +47,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
try
{
// Refresh KEV catalog
await _kevService.RefreshAsync();
var kevRefreshed = await _kevService.RefreshAsync();
// Reseed CPE mappings from GitHub: ?reseed=seed | ?reseed=community | ?reseed=true (both)
int cpeSeedEntries = 0;
@@ -58,13 +59,15 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
bool reseedSeed = reseedParam == "true" || reseedParam == "seed";
bool reseedCommunity = reseedParam == "true" || reseedParam == "community";
// A list is fetched and checked before anything is written; a list that cannot be
// fetched or is refused leaves the mappings in service as they were.
if (reseedSeed)
{
try
{
var seedJson = await _gitHubRepo.FetchCpeSeedMappingsAsync();
cpeSeedEntries = await _vulnRepo.ImportCpeMappingSeedFromJsonAsync(seedJson);
_logger.LogInformation("CPE seed mappings reseeded from GitHub: {Count} entries", cpeSeedEntries);
var seed = await _cpeImporter.ImportSeedAsync();
cpeSeedEntries = seed.Written;
if (seed.Refused) cpeSeedError = "The seed list was refused: " + string.Join("; ", seed.Problems.Take(3));
}
catch (Exception ex)
{
@@ -77,33 +80,45 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
try
{
var communityJson = await _gitHubRepo.FetchCpeCommunityMappingsAsync();
cpeCommunityEntries = await _vulnRepo.ImportCpeCommunityMappingsFromJsonAsync(communityJson);
_logger.LogInformation("CPE community mappings reseeded from GitHub: {Count} entries", cpeCommunityEntries);
var community = await _cpeImporter.ImportCommunityAsync();
cpeCommunityEntries = community.Written;
if (community.Refused) cpeCommunityError = "The community list was refused: " + string.Join("; ", community.Problems.Take(3));
}
catch (Exception ex)
{
cpeCommunityError = "No community mappings available: " + ex.Message;
_logger.LogInformation("No CPE community mappings available on GitHub (this is normal if none have been contributed yet)");
_logger.LogWarning(ex, "Failed to reseed CPE community mappings from GitHub");
}
}
if (reseedSeed || reseedCommunity)
{
VulnerabilityCorrelationService.ResetMappingsCache();
try
{
await _reconciler.ReconcileAsync("reseed");
}
catch (CpeMappingUnavailableException ex)
{
_logger.LogWarning(ex, "Inventory rows not reconciled after the reseed");
}
}
// Update sync timestamp
var syncedAt = DateTime.UtcNow.ToString("o");
await _adminConfigService.UpdateAsync(
c => { c.VulnerabilityDataLastSyncUtc = syncedAt; return null; },
"System (vulnerability sync)", "vuln-sync");
// The timestamp says when the KEV catalog was last refreshed, so a failed fetch leaves it.
if (kevRefreshed)
{
var syncedAt = DateTime.UtcNow.ToString("o");
await _adminConfigService.UpdateAsync(
c => { c.VulnerabilityDataLastSyncUtc = syncedAt; return null; },
"System (vulnerability sync)", "vuln-sync");
}
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(new TriggerVulnerabilityDataSyncResponse
{
Success = true,
Message = "Vulnerability data sync completed",
Success = kevRefreshed,
Message = kevRefreshed
? "Vulnerability data sync completed"
: "The KEV catalog could not be refreshed; the previous copy stays in use",
KevCatalogEntries = _kevService.CatalogSize,
CpeSeedEntries = cpeSeedEntries,
CpeSeedError = cpeSeedError,
@@ -1,8 +1,10 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Functions.Telemetry;
using Microsoft.ApplicationInsights;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;
@@ -10,25 +12,30 @@ using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
/// <summary>
/// Timer-triggered function that refreshes vulnerability data caches daily.
/// - CISA KEV catalog (full in-memory refresh)
/// - MSRC CVRF index (last 6 Patch Tuesdays)
/// - Stale NVD CVE cache entries in VulnerabilityCache table (older than 7 days)
/// - FIRST EPSS re-score of every cached CVE
///
/// Nightly refresh of everything the vulnerability scanner reads: the CISA KEV catalog, the
/// MSRC index, the copy of the NVD product dictionary, automatic mappings and the CPE on the
/// inventory rows, the per-product CVE cache and the EPSS scores.
/// <para>
/// The NVD and EPSS walks live in <see cref="VulnerabilityCacheRefreshService"/>, shared with
/// the admin's manual buttons, so both paths behave identically.
///
/// All operations are logged to ILogger → visible in Application Insights under
/// "traces" with customDimensions.Category = "VulnerabilityDataSyncFunction".
/// Query: traces | where customDimensions.Category contains "VulnerabilityDataSync"
/// </para>
/// <para>
/// A worker's information logs do not reach Application Insights. What a run did is in the
/// event <c>VulnerabilityDataSyncCompleted</c>, sent unsampled once per run, so a night
/// without it is a run that did not finish; a run with a part that fell short raises the ops
/// event <c>VulnerabilityDataSyncDegraded</c>.
/// </para>
/// </summary>
public class VulnerabilityDataSyncFunction
{
private readonly KevDataService _kevService;
private readonly MsrcCorrelationService _msrcCorrelation;
private readonly VulnerabilityCacheRefreshService _refresh;
private readonly CpeDictionaryService _dictionary;
private readonly CpeDictionaryResolver _resolver;
private readonly SoftwareInventoryCpeReconciler _reconciler;
private readonly AdminConfigurationService _adminConfigService;
private readonly OpsEventService _opsEvents;
private readonly TelemetryClient? _telemetryClient;
private readonly ILogger<VulnerabilityDataSyncFunction> _logger;
@@ -36,13 +43,21 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
KevDataService kevService,
MsrcCorrelationService msrcCorrelation,
VulnerabilityCacheRefreshService refresh,
CpeDictionaryService dictionary,
CpeDictionaryResolver resolver,
SoftwareInventoryCpeReconciler reconciler,
AdminConfigurationService adminConfigService,
OpsEventService opsEvents,
TelemetryClient? telemetryClient,
ILogger<VulnerabilityDataSyncFunction> logger)
{
_opsEvents = opsEvents;
_kevService = kevService;
_msrcCorrelation = msrcCorrelation;
_refresh = refresh;
_dictionary = dictionary;
_resolver = resolver;
_reconciler = reconciler;
_adminConfigService = adminConfigService;
_telemetryClient = telemetryClient;
_logger = logger;
@@ -70,10 +85,17 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
// 1. Refresh CISA KEV catalog
try
{
await _kevService.RefreshAsync();
kevCount = _kevService.CatalogSize;
kevSyncedAt = DateTime.UtcNow.ToString("o");
_logger.LogInformation("[VulnSync] KEV catalog refreshed: {Count} entries", kevCount);
if (await _kevService.RefreshAsync())
{
kevCount = _kevService.CatalogSize;
kevSyncedAt = DateTime.UtcNow.ToString("o");
_logger.LogInformation("[VulnSync] KEV catalog refreshed: {Count} entries", kevCount);
}
else
{
kevError = "KEV catalog could not be refreshed; the loaded copy stays in service";
}
}
catch (Exception ex)
{
@@ -101,18 +123,48 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
_logger.LogWarning(ex, "[VulnSync] MSRC index refresh failed");
}
// 3. Refresh stale CVE cache entries (null = a manual walk is in flight; skip, it does the same work).
// 3. NVD product dictionary: downloaded only when NVD published a new one. Never throws.
var dictionary = await _dictionary.RefreshAsync();
// 4. Automatic mappings for names that are exactly an NVD product name, then every
// inventory row is brought in line with the mappings. Both run before the cache
// refresh, so a product that got its first mapping tonight gets its CVE data tonight.
CpeAutoMapRunResult? autoMap = null;
InventoryReconcileResult? reconciled = null;
string? mappingError = null;
try
{
autoMap = await _resolver.ResolveInventoryAsync();
}
catch (Exception ex)
{
mappingError = ex.Message;
_logger.LogWarning(ex, "[VulnSync] Automatic mappings failed");
}
// On its own: a failed mapper must not cost the night's reconcile.
try
{
reconciled = await _reconciler.ReconcileAsync("timer");
}
catch (Exception ex)
{
mappingError = mappingError == null ? ex.Message : $"{mappingError}; {ex.Message}";
_logger.LogWarning(ex, "[VulnSync] Inventory reconcile failed");
}
// 5. Refresh the CVE cache (null = a manual walk is in flight; skip, it does the same work).
var cveStats = await _refresh.RefreshStaleCveCacheAsync("timer");
if (cveStats == null)
_logger.LogInformation("[VulnSync] CVE cache refresh skipped: a manual refresh is already running");
// 3b. Re-score EVERY cached CVE row with today's EPSS. Independent of the 7-day NVD
// 6. Re-score EVERY cached CVE row with today's EPSS. Independent of the 7-day NVD
// TTL: EPSS is republished daily and the calls are cheap (one batch per 100 ids).
var epssStats = await _refresh.RefreshEpssScoresAsync("timer");
if (epssStats == null)
_logger.LogInformation("[VulnSync] EPSS refresh skipped: a manual sync is already running");
// 4. Update per-source sync timestamps. Only the sources that succeeded get
// 7. Update per-source sync timestamps. Only the sources that succeeded get
// bumped — a failed phase keeps its prior timestamp so the UI can surface
// "Last KEV/MSRC Sync" accurately. (NVD/EPSS stamps are written by the service.)
try
@@ -132,12 +184,34 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
_logger.LogWarning(ex, "[VulnSync] Failed to update sync timestamp");
}
// Every automatic mapping is one nobody has looked at yet.
if (autoMap is { Created.Count: > 0 })
await _opsEvents.RecordCpeAutoMappingsCreatedAsync(autoMap.Created);
var fellShort = PartsThatFellShort(kevError, msrcError, dictionary, mappingError, reconciled, cveStats, epssStats);
fellShort.AddRange(await StalePartsAsync());
if (fellShort.Count > 0)
{
await _opsEvents.RecordVulnerabilityDataSyncDegradedAsync(fellShort, new
{
kevError,
msrcError,
dictionaryError = dictionary.Error,
mappingError,
inventoryFailed = reconciled?.Failed ?? 0,
cveError = cveStats?.Error,
cveFailed = cveStats?.Failed ?? 0,
cveSkippedUnavailable = cveStats?.SkippedUnavailable ?? 0,
cvePendingOverdue = cveStats?.PendingOverdue ?? 0,
epssError = epssStats?.Error,
});
}
var durationMs = (DateTime.UtcNow - startedAt).TotalMilliseconds;
_logger.LogInformation("[VulnSync] Vulnerability data sync completed in {DurationMs:F0}ms", durationMs);
// Sampling-bypass observability: host.json adds "Event" to excludedTypes so this
// survives even when default trace sampling is aggressive.
_telemetryClient?.TrackEvent("VulnerabilityDataSyncCompleted", new Dictionary<string, string>
// One event per night whose presence and numbers are the signal: sent unsampled.
_telemetryClient.TrackEventUnsampled("VulnerabilityDataSyncCompleted", new Dictionary<string, string>
{
{ "kevCount", kevCount.ToString() },
{ "kevError", kevError ?? "" },
@@ -148,14 +222,94 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{ "cveRefreshed", (cveStats?.Refreshed ?? 0).ToString() },
{ "cveFailed", (cveStats?.Failed ?? 0).ToString() },
{ "cveSkippedThrottled", (cveStats?.SkippedThrottled ?? 0).ToString() },
{ "cveSkippedUnavailable", (cveStats?.SkippedUnavailable ?? 0).ToString() },
{ "cveSkippedBusy", (cveStats == null).ToString() },
{ "cveRemoved", (cveStats?.Removed ?? 0).ToString() },
{ "cvePendingOverdue", (cveStats?.PendingOverdue ?? 0).ToString() },
{ "cveError", cveStats?.Error ?? "" },
{ "dictionaryChanged", dictionary.Changed.ToString() },
{ "dictionaryProducts", dictionary.Products.ToString() },
{ "dictionaryError", dictionary.Error ?? "" },
{ "unmappedNames", (autoMap?.UnmappedNames ?? 0).ToString() },
{ "autoMapped", (autoMap?.Mapped ?? 0).ToString() },
{ "autoNoCves", (autoMap?.NoCves ?? 0).ToString() },
{ "autoUnversioned", (autoMap?.Unversioned ?? 0).ToString() },
{ "autoStorePackages", (autoMap?.StorePackages ?? 0).ToString() },
{ "autoNvdUnavailable", (autoMap?.NvdUnavailable ?? 0).ToString() },
{ "autoPostponed", (autoMap?.Postponed ?? 0).ToString() },
{ "inventoryRows", (reconciled?.Rows ?? 0).ToString() },
{ "inventorySet", (reconciled?.Set ?? 0).ToString() },
{ "inventoryCleared", (reconciled?.Cleared ?? 0).ToString() },
{ "inventoryFailed", (reconciled?.Failed ?? 0).ToString() },
{ "mappingError", mappingError ?? "" },
{ "epssRows", (epssStats?.Rows ?? 0).ToString() },
{ "epssCves", (epssStats?.Cves ?? 0).ToString() },
{ "epssScored", (epssStats?.Scored ?? 0).ToString() },
{ "epssRowsRewritten", (epssStats?.RowsRewritten ?? 0).ToString() },
{ "epssError", epssStats?.Error ?? "" },
{ "degradedParts", fellShort.Count.ToString() },
{ "durationMs", durationMs.ToString("F0", System.Globalization.CultureInfo.InvariantCulture) }
});
}
/// <summary>
/// What one run cannot see about itself: a walk that is cut short every night, each time for
/// a reason that is harmless on its own, and a dictionary that has not been replaced for days.
/// Read from what earlier runs stored, so it also shows after a night whose run never finished.
/// </summary>
private async Task<List<string>> StalePartsAsync()
{
try
{
var config = await _adminConfigService.GetConfigurationAsync();
var lastWalk = QueryParams.UtcInstant(config?.NvdCacheLastRefreshUtc);
return StaleParts(lastWalk, await _dictionary.LastRefreshAsync(), DateTime.UtcNow);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[VulnSync] Stored sync state could not be read");
return new List<string>();
}
}
internal static readonly TimeSpan WalkOverdueAfter = TimeSpan.FromDays(2);
internal static readonly TimeSpan DictionaryOverdueAfter = TimeSpan.FromDays(7);
internal static List<string> StaleParts(DateTime? lastCompletedWalkUtc, DateTime? dictionaryRefreshedUtc, DateTime nowUtc)
{
var parts = new List<string>();
if (lastCompletedWalkUtc.HasValue && nowUtc - lastCompletedWalkUtc.Value > WalkOverdueAfter)
parts.Add($"CVE cache walk has not reached its end since {lastCompletedWalkUtc.Value:yyyy-MM-dd}");
if (dictionaryRefreshedUtc.HasValue && nowUtc - dictionaryRefreshedUtc.Value > DictionaryOverdueAfter)
parts.Add($"NVD product dictionary not replaced since {dictionaryRefreshedUtc.Value:yyyy-MM-dd}");
return parts;
}
/// <summary>
/// The parts of a run an operator has to look at, in plain words. A walk that NVD throttled
/// part of the way is not one of them: its rows are due again the next night. A walk that
/// refreshed nothing is, and so are products that waited for their first data for a day.
/// </summary>
internal static List<string> PartsThatFellShort(
string? kevError, string? msrcError, CpeDictionaryRefreshResult dictionary, string? mappingError,
InventoryReconcileResult? reconciled, NvdCacheRefreshResult? cveStats, EpssRefreshResult? epssStats)
{
var parts = new List<string>();
if (!string.IsNullOrEmpty(kevError)) parts.Add("KEV catalog not refreshed");
if (!string.IsNullOrEmpty(msrcError)) parts.Add("MSRC index not refreshed");
if (!string.IsNullOrEmpty(dictionary.Error)) parts.Add("NVD product dictionary not refreshed");
if (!string.IsNullOrEmpty(mappingError)) parts.Add("automatic mappings or inventory reconcile failed");
if (reconciled is { Failed: > 0 }) parts.Add($"{reconciled.Failed} inventory rows did not take their CPE");
if (!string.IsNullOrEmpty(cveStats?.Error))
parts.Add("CVE cache walk failed");
else if (cveStats is { SkippedUnavailable: > 0 })
parts.Add($"CVE cache walk cut off with {cveStats.SkippedUnavailable} rows left (NVD not answering or out of time)");
else if (cveStats is { Stale: > 0, Refreshed: 0 })
parts.Add($"CVE cache walk refreshed none of {cveStats.Stale} due rows (NVD refused or failed every request)");
if (cveStats is { PendingOverdue: > 0 })
parts.Add($"{cveStats.PendingOverdue} products without CVE data for more than a day");
if (!string.IsNullOrEmpty(epssStats?.Error)) parts.Add("EPSS scores not refreshed");
return parts;
}
}
}
@@ -80,6 +80,12 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
/// <see cref="CveExposureSummary.BandSoftware"/>: per severity and priority band its top
/// entries outside the lists, so a band filter never comes up empty while the band has entries.
/// </param>
/// <param name="selfUpdatingProducts">
/// Products (<c>vendor:product</c>) that count as self-updating right now. Their rows are
/// left out of every number and list unless <paramref name="includeSelfUpdating"/> is set;
/// how many products and sessions that concerns is reported either way. The set is applied
/// here and not stamped on the rows, so a changed flag shows on the next read.
/// </param>
public static CveExposureSummary Aggregate(
IReadOnlyList<CveExposureEntry> rows,
bool truncated,
@@ -87,7 +93,9 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
int days,
bool crossTenant,
DateTime nowUtc,
int perBand = 0)
int perBand = 0,
IReadOnlyCollection<string>? selfUpdatingProducts = null,
bool includeSelfUpdating = false)
{
if (topN < 1) topN = 1;
perBand = Math.Clamp(perBand, 0, MaxPerBand);
@@ -97,9 +105,19 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
// DetectedAt is the correlation write time (~session time). A null is
// treated as in-window rather than dropped — better to over-include a
// legacy row than to silently undercount exposure.
var inWindow = rows
var allInWindow = rows
.Where(r => !r.DetectedAt.HasValue || r.DetectedAt.Value >= cutoff)
.ToList();
var selfUpdating = new HashSet<string>(selfUpdatingProducts ?? Array.Empty<string>(), StringComparer.OrdinalIgnoreCase);
// A row written before the index carried the product borrows it from a row with the
// same title before anything is left out. Otherwise such a row of a self-updating
// product would stay in the numbers its siblings were taken out of.
var productByTitle = ProductByTitle(allInWindow);
bool IsSelfUpdating(CveExposureEntry r) => selfUpdating.Contains(ProductOf(r, productByTitle));
var selfUpdatingRows = allInWindow.Where(r => !string.IsNullOrEmpty(r.CveId) && IsSelfUpdating(r)).ToList();
var inWindow = includeSelfUpdating ? allInWindow : allInWindow.Where(r => !IsSelfUpdating(r)).ToList();
var withCve = inWindow.Where(r => !string.IsNullOrEmpty(r.CveId)).ToList();
var cves = withCve
@@ -115,7 +133,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
var listedCves = WithKevAdditions(rankedCves, topN, c => c.IsKev, MaxKevAdditions);
var software = FoldSoftware(withCve);
var software = FoldSoftware(withCve, selfUpdating, productByTitle);
var rankedSoftware = software
.OrderByDescending(s => s.AffectedSessions)
.ThenByDescending(s => CvePriority.Rank(s.Priority))
@@ -178,6 +196,10 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
BandCves = bandCves,
BandSoftware = bandSoftware,
Truncated = truncated,
SelfUpdatingIncluded = includeSelfUpdating,
SelfUpdatingSoftware = selfUpdatingRows
.Select(r => ProductOf(r, productByTitle)).Distinct(StringComparer.OrdinalIgnoreCase).Count(),
SelfUpdatingAffectedSessions = DistinctSessions(selfUpdatingRows),
};
}
@@ -331,34 +353,39 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
Rows: rows);
}
/// <summary>
/// The product each title is known under: the one most rows with that title carry.
/// </summary>
private static Dictionary<string, string> ProductByTitle(IEnumerable<CveExposureEntry> rows) => rows
.Where(r => !string.IsNullOrEmpty(r.Product) && !string.IsNullOrWhiteSpace(r.SoftwareName))
.GroupBy(r => r.SoftwareName.Trim(), StringComparer.OrdinalIgnoreCase)
.ToDictionary(
g => g.Key,
g => g.GroupBy(r => r.Product, StringComparer.OrdinalIgnoreCase)
.OrderByDescending(p => p.Count())
.ThenBy(p => p.Key, StringComparer.OrdinalIgnoreCase)
.First().Key,
StringComparer.OrdinalIgnoreCase);
/// <summary>The product a row is counted under: its own, else its title's, else the title itself.</summary>
private static string ProductOf(CveExposureEntry r, Dictionary<string, string> productByTitle)
{
if (!string.IsNullOrEmpty(r.Product)) return r.Product;
var title = r.SoftwareName?.Trim() ?? string.Empty;
return title.Length > 0 && productByTitle.TryGetValue(title, out var product) ? product : title;
}
/// <summary>
/// Folds the rows per product. A row without a product (written before the index carried
/// it, or a finding without a CPE) borrows the product of another row with the same title —
/// so the transition converges as new sessions are correlated — and otherwise is keyed by
/// its title. Rows with neither are left out of the product view.
/// </summary>
private static List<TopSoftware> FoldSoftware(List<CveExposureEntry> rows)
private static List<TopSoftware> FoldSoftware(
List<CveExposureEntry> rows, HashSet<string> selfUpdating, Dictionary<string, string> productByTitle)
{
var productByTitle = rows
.Where(r => !string.IsNullOrEmpty(r.Product) && !string.IsNullOrWhiteSpace(r.SoftwareName))
.GroupBy(r => r.SoftwareName.Trim(), StringComparer.OrdinalIgnoreCase)
.ToDictionary(
g => g.Key,
g => g.GroupBy(r => r.Product, StringComparer.OrdinalIgnoreCase)
.OrderByDescending(p => p.Count())
.ThenBy(p => p.Key, StringComparer.OrdinalIgnoreCase)
.First().Key,
StringComparer.OrdinalIgnoreCase);
string ProductKey(CveExposureEntry r)
{
if (!string.IsNullOrEmpty(r.Product)) return r.Product;
var title = r.SoftwareName?.Trim() ?? string.Empty;
return title.Length > 0 && productByTitle.TryGetValue(title, out var product) ? product : title;
}
return rows
.Select(r => (Row: r, Key: ProductKey(r)))
.Select(r => (Row: r, Key: ProductOf(r, productByTitle)))
.Where(x => x.Key.Length > 0)
.GroupBy(x => x.Key, x => x.Row, StringComparer.OrdinalIgnoreCase)
.Select(g =>
@@ -411,6 +438,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
AffectedSessions = c.AffectedSessions,
})
.ToList(),
SelfUpdating = selfUpdating.Contains(g.Key) ? true : null,
};
})
.ToList();
@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.Net;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Services.Vulnerability;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
@@ -19,6 +21,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
{
private readonly ILogger<VulnerabilitySummaryFunction> _logger;
private readonly ISessionRepository _sessionRepo;
private readonly CpeMappingStore _mappingStore;
// Safety cap on the CveIndex scan. A row is one (tenant, CVE, session)
// triple; 50k comfortably covers large tenants while bounding a cross-
@@ -27,14 +30,16 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
public VulnerabilitySummaryFunction(
ILogger<VulnerabilitySummaryFunction> logger,
ISessionRepository sessionRepo)
ISessionRepository sessionRepo,
CpeMappingStore mappingStore)
{
_logger = logger;
_sessionRepo = sessionRepo;
_mappingStore = mappingStore;
}
/// <summary>
/// GET /api/metrics/vulnerability?days=30&topN=20&perBand=10
/// GET /api/metrics/vulnerability?days=30&topN=20&perBand=10&includeSelfUpdating=false
/// Fleet vulnerability summary for the caller's tenant (MemberRead).
/// </summary>
[Function("GetVulnerabilitySummary")]
@@ -47,7 +52,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
}
/// <summary>
/// GET /api/global/metrics/vulnerability?days=30&topN=20&perBand=10&tenantId=...
/// GET /api/global/metrics/vulnerability?days=30&topN=20&perBand=10&includeSelfUpdating=false&tenantId=...
/// Cross-tenant summary (Global Admin). With tenantId, scopes to one tenant.
/// </summary>
[Function("GetGlobalVulnerabilitySummary")]
@@ -75,9 +80,26 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
"Vulnerability summary requested (scope={Scope}, days={Days}, topN={TopN}, perBand={PerBand})",
crossTenant ? "cross-tenant" : tenantId, days, topN, perBand);
var (rows, truncated) = await _sessionRepo.ScanCveIndexAsync(tenantId, MaxScanRows);
// Software that updates itself is left out unless the caller asks for it.
var includeSelfUpdating = string.Equals(query["includeSelfUpdating"], "true", StringComparison.OrdinalIgnoreCase);
IReadOnlyCollection<string> selfUpdatingProducts = Array.Empty<string>();
try
{
selfUpdatingProducts = (await _mappingStore.GetAsync()).SelfUpdatingProducts();
}
catch (CpeMappingUnavailableException ex)
{
// Without the flags nothing can be told apart: everything is counted, and the
// answer says so.
includeSelfUpdating = true;
_logger.LogWarning(ex, "Vulnerability summary without the self-updating flags");
}
var (rows, truncated) = await _sessionRepo.ScanCveIndexAsync(
tenantId, MaxScanRows, DateTime.UtcNow.AddDays(-days));
var summary = VulnerabilitySummaryAggregator.Aggregate(
rows, truncated, topN, days, crossTenant, DateTime.UtcNow, perBand);
rows, truncated, topN, days, crossTenant, DateTime.UtcNow, perBand,
selfUpdatingProducts, includeSelfUpdating);
var response = req.CreateResponse(HttpStatusCode.OK);
await response.WriteAsJsonAsync(summary);
@@ -22,7 +22,16 @@ public sealed class ResiliencePolicies
];
/// <summary>
/// Policy for external data APIs (GitHub, NVD, CISA, MSRC):
/// Policy for the NVD API: one retry for a server error or a broken connection, behind its
/// own circuit breaker. A 429 is not retried, because the client paces its requests and opens
/// its own cooldown; a timeout is not retried, because a page of several megabytes that timed
/// out once is not fetched again inside a run with a time budget. The breaker is separate so
/// an NVD outage does not cut off the other sources.
/// </summary>
public IAsyncPolicy<HttpResponseMessage> NvdApi { get; }
/// <summary>
/// Policy for external data APIs (GitHub, CISA, MSRC, FIRST):
/// Retry 3× with exponential backoff + jitter, wrapped in an advanced circuit breaker
/// (≥50% failures in 30s → 60s open).
/// </summary>
@@ -39,6 +48,7 @@ public sealed class ResiliencePolicies
{
var logger = loggerFactory.CreateLogger(nameof(ResiliencePolicies));
ExternalDataApi = BuildExternalDataApi(logger);
NvdApi = BuildNvdApi(logger);
Notification = BuildNotification(logger);
}
@@ -87,6 +97,40 @@ public sealed class ResiliencePolicies
return Policy.WrapAsync(retry, circuitBreaker);
}
private static IAsyncPolicy<HttpResponseMessage> BuildNvdApi(ILogger logger)
{
var retry = Policy<HttpResponseMessage>
.HandleResult(r => (int)r.StatusCode >= 500)
.Or<HttpRequestException>()
.WaitAndRetryAsync(
retryCount: 1,
sleepDurationProvider: _ => TimeSpan.FromSeconds(2),
onRetry: (outcome, _, _, _) =>
logger.LogWarning(outcome.Exception,
"NVD request failed ({Reason}), retrying once",
outcome.Exception?.GetType().Name ?? outcome.Result?.StatusCode.ToString()));
var circuitBreaker = Policy<HttpResponseMessage>
.HandleResult(r => (int)r.StatusCode >= 500)
.Or<HttpRequestException>()
.AdvancedCircuitBreakerAsync(
failureThreshold: 0.5,
samplingDuration: TimeSpan.FromSeconds(30),
minimumThroughput: 5,
durationOfBreak: TimeSpan.FromSeconds(60),
onBreak: (outcome, duration) =>
logger.LogWarning(
"NVD circuit breaker opened for {DurationSec}s — {Reason}",
(int)duration.TotalSeconds,
outcome.Exception?.Message ?? outcome.Result?.StatusCode.ToString()),
onReset: () => logger.LogInformation("NVD circuit breaker reset"),
onHalfOpen: () => logger.LogInformation("NVD circuit breaker half-open"));
// Retry outside, breaker inside: an open breaker throws BrokenCircuitException, which
// the retry does not handle.
return Policy.WrapAsync(retry, circuitBreaker);
}
private static IAsyncPolicy<HttpResponseMessage> BuildNotification(ILogger logger)
=> Policy<HttpResponseMessage>
.HandleResult(r => TransientCodes.Contains(r.StatusCode))
@@ -436,7 +436,7 @@ builder.Services.AddSingleton<ISignalRNotificationService>(sp => sp.GetRequiredS
// Vulnerability correlation services
builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability.NvdApiClient>()
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().NvdApi);
// KevDataService keeps the hydrated catalog as instance state, but AddHttpClient<T> registers T
// transient: the cache warmer hydrated one instance while the correlation service and the sync
// functions each held a cold one (audit 2026-09-05 F12). The typed-client registration stays for
@@ -454,6 +454,32 @@ builder.Services.AddHttpClient<AutopilotMonitor.Functions.Services.Vulnerability
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi);
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.MsrcCorrelationService>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityCacheRefreshService>();
// The store does not know the ops events; a failed load reaches them through this hook.
builder.Services.AddSingleton(sp =>
{
var store = new AutopilotMonitor.Functions.Services.Vulnerability.CpeMappingStore(
sp.GetRequiredService<AutopilotMonitor.Shared.DataAccess.IVulnerabilityRepository>(),
sp.GetRequiredService<ILogger<AutopilotMonitor.Functions.Services.Vulnerability.CpeMappingStore>>());
var opsEvents = sp.GetRequiredService<OpsEventService>();
store.LoadFailed = (error, servingEarlierCopy) =>
opsEvents.RecordCpeMappingUnavailableAsync($"{error.GetType().Name}: {error.Message}", servingEarlierCopy);
return store;
});
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.CveCacheFiller>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityReportWriter>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.CpeProductFlagsService>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.SoftwareInventoryCpeReconciler>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.CpeMappingImporter>();
// The dictionary keeps its loaded copy as instance state, but AddHttpClient<T> registers T as
// transient: the named client goes into one singleton, like KevDataService above.
builder.Services.AddHttpClient(nameof(AutopilotMonitor.Functions.Services.Vulnerability.CpeDictionaryService))
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().ExternalDataApi)
.SetHandlerLifetime(Timeout.InfiniteTimeSpan);
builder.Services.AddSingleton(sp => new AutopilotMonitor.Functions.Services.Vulnerability.CpeDictionaryService(
sp.GetRequiredService<IHttpClientFactory>().CreateClient(nameof(AutopilotMonitor.Functions.Services.Vulnerability.CpeDictionaryService)),
sp.GetRequiredService<BlobStorageService>(),
sp.GetRequiredService<ILogger<AutopilotMonitor.Functions.Services.Vulnerability.CpeDictionaryService>>()));
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.CpeDictionaryResolver>();
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityCorrelationService>();
// Hydrate MSRC + KEV in-memory caches from blob snapshots at app startup (fire-and-forget;
// keeps cold-start fast for re-deploys, see VulnerabilityCacheWarmer for the contract).
@@ -691,6 +691,7 @@ public static class EndpointAccessPolicyCatalog
new("POST", "vulnerability/cpe-mapping/auto-resolve", EndpointPolicy.GlobalAdminOnly),
new("DELETE", "vulnerability/cpe-mapping", EndpointPolicy.GlobalAdminOnly),
new("GET", "vulnerability/cpe-mappings", EndpointPolicy.GlobalReadOrAdmin),
new("POST", "vulnerability/cpe-product-flags", EndpointPolicy.GlobalAdminOnly),
new("GET", "vulnerability/ignored-software", EndpointPolicy.GlobalReadOrAdmin),
new("POST", "vulnerability/ignored-software", EndpointPolicy.GlobalAdminOnly),
new("DELETE", "vulnerability/ignored-software", EndpointPolicy.GlobalAdminOnly),
@@ -72,7 +72,7 @@ namespace AutopilotMonitor.Functions.Services
public async Task<string> FetchCpeSeedMappingsAsync()
{
var url = $"{_baseUrl}/dist/cpe-mapping-seed.json";
var url = $"{_baseUrl}/dist/cpe-mappings.json";
_logger.LogInformation("Fetching CPE seed mappings from {Url}", url);
var json = await FetchAsync(url);
_logger.LogInformation("Fetched CPE seed mappings JSON from GitHub ({Length} bytes)", json.Length);
@@ -312,6 +312,90 @@ namespace AutopilotMonitor.Functions.Services
tenantId, "System.VulnerabilityCorrelation",
new { sessionId, failedIncrements, failedDecrements, sideRowFailed, error, sampleKeys });
/// <summary>
/// The nightly vulnerability data sync finished with parts that fell short. One event per
/// run; <paramref name="parts"/> names them in plain words, <paramref name="details"/>
/// carries the counts and error texts of the run.
/// </summary>
public virtual Task RecordVulnerabilityDataSyncDegradedAsync(IReadOnlyList<string> parts, object details)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.VulnerabilityDataSyncDegraded, OpsEventSeverity.Warning,
$"Vulnerability data sync degraded: {string.Join("; ", parts)}",
null, "System.VulnerabilityDataSync", details);
private const string CpeMappingsUnreadable = "CPE mappings could not be";
private const string CpeMappingListRefused = "CPE mapping list refused";
/// <summary>
/// The software-to-CPE mappings could not be read. Without an earlier copy every scan on
/// this instance waits, which is the Error; with one the scans go on over older mappings.
/// Every instance runs into the same outage and there are dozens of them, so today's
/// events are read first: one per UTC day and severity, unless two instances read at the
/// same moment. The mapping store adds its own hourly pause per instance.
/// </summary>
public virtual async Task RecordCpeMappingUnavailableAsync(string error, bool servingEarlierCopy)
{
var severity = servingEarlierCopy ? OpsEventSeverity.Warning : OpsEventSeverity.Error;
if (await RecordedTodayAsync(OpsEventTypes.CpeMappingUnavailable, severity, CpeMappingsUnreadable).ConfigureAwait(false))
return;
await WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CpeMappingUnavailable, severity,
servingEarlierCopy
? $"{CpeMappingsUnreadable} reloaded; an earlier copy keeps serving"
: $"{CpeMappingsUnreadable} loaded; vulnerability scans are postponed",
null, "System.VulnerabilityCorrelation", new { error, servingEarlierCopy }).ConfigureAwait(false);
}
/// <summary>
/// An import fetched a mapping list the check refused; the mappings in service were kept.
/// It shares its type with an unreadable mapping table, because either way the mappings are
/// not what the rules repository says they should be. Error, once per UTC day and list.
/// </summary>
public virtual async Task RecordCpeMappingListRefusedAsync(string list, IReadOnlyList<string> problems)
{
if (problems.Count == 0) return;
var refused = $"{CpeMappingListRefused} ({list})";
if (await RecordedTodayAsync(OpsEventTypes.CpeMappingUnavailable, OpsEventSeverity.Error, refused).ConfigureAwait(false))
return;
await WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CpeMappingUnavailable, OpsEventSeverity.Error,
$"{refused}, the mappings in service were kept: {problems[0]}",
null, "System.VulnerabilityCorrelation", new { list, problems = problems.Take(20).ToList() }).ConfigureAwait(false);
}
/// <summary>
/// True when a Maintenance event of this type, at least this severity and this message
/// start was recorded today (UTC). A failed read counts as "not recorded": one event too
/// many is the smaller harm.
/// </summary>
private async Task<bool> RecordedTodayAsync(string eventType, string severity, string messageStart)
{
try
{
var today = DateTime.UtcNow.Date;
var events = await _repository.GetOpsEventsAsync(OpsEventCategory.Maintenance, today, today.AddDays(1)).ConfigureAwait(false);
return events.Any(e =>
string.Equals(e.EventType, eventType, StringComparison.Ordinal)
&& OpsEventSeverity.Rank(e.Severity) >= OpsEventSeverity.Rank(severity)
&& (e.Message ?? string.Empty).StartsWith(messageStart, StringComparison.Ordinal));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Today's {EventType} events could not be read", eventType);
return false;
}
}
/// <summary>
/// The nightly run created automatic CPE mappings. One event per run; the message names up
/// to ten as "program -> vendor:product", the details all of them. A program gets such a
/// mapping only when its name is an NVD product name, so the names are public ones.
/// </summary>
public virtual Task RecordCpeAutoMappingsCreatedAsync(IReadOnlyList<string> created)
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.CpeAutoMappingsCreated, OpsEventSeverity.Info,
$"{created.Count} automatic CPE mapping(s) created: {string.Join("; ", created.Take(10))}"
+ (created.Count > 10 ? "; ..." : string.Empty),
null, "System.VulnerabilityDataSync", new { created });
// ── Cascade-Delete Maintenance (Plan §5 PR6 / §16 R14) ─────────────────
// Event types dispatched by SessionDeletionMaintenanceFunction: Started, LongRunning,
// LongRunningSevere, BudgetExceeded, SkippedLocked, Failed, Completed, FanoutSkipped
@@ -242,23 +242,39 @@ namespace AutopilotMonitor.Functions.Services
// ===== CVE INDEX =====
/// <summary>CVEs a software entry contributes to the index next to every known-exploited one.</summary>
internal const int MaxIndexedCvesPerFinding = 5;
/// <summary>
/// Upserts CVE index entries so sessions can be searched by CVE identifier.
/// PartitionKey = {tenantId}_{cveId}, RowKey = sessionId
/// One CVE commonly appears under several findings of the same report (e.g. one
/// Remote Desktop CVE across 14 installed RD components), so occurrences are merged
/// per CVE first — otherwise concurrent upserts race on the same entity and the
/// winner is arbitrary. Writes go out with bounded parallelism (PK differs per CVE,
/// so batch transactions are not an option). Per-entity failures are counted, not
/// fatal — the index is a search aid, the report itself is stored separately.
/// Brings a session's CveIndex rows (PartitionKey = {tenantId}_{cveId}, RowKey = sessionId)
/// in line with its report: rows for CVEs the report no longer names are deleted, the others
/// written. A row that existed keeps its DetectedAt, so a later evaluation of an old session
/// does not move it into today's window.
/// <para>
/// <paramref name="previousFindings"/> are the findings of the report this one replaces;
/// they are what tells which rows exist, because the index cannot be read by session.
/// One CVE commonly appears under several findings of the same report (e.g. one Remote
/// Desktop CVE across 14 installed RD components), so occurrences are merged per CVE first.
/// Writes go out with bounded parallelism (PK differs per CVE, so batch transactions are not
/// an option). Per-entity failures are counted, not fatal — the index is a search aid, the
/// report itself is stored separately.
/// </para>
/// </summary>
public async Task UpsertCveIndexEntriesAsync(string tenantId, string sessionId, List<Dictionary<string, object>> findings)
public async Task ReplaceCveIndexEntriesAsync(
string tenantId, string sessionId,
List<Dictionary<string, object>>? findings,
List<Dictionary<string, object>>? previousFindings)
{
try
{
var entities = BuildCveIndexEntities(tenantId, sessionId, findings);
if (entities.Count == 0) return;
var entities = BuildCveIndexEntities(tenantId, sessionId, findings ?? new List<Dictionary<string, object>>());
var previousIds = new HashSet<string>(
BuildCveIndexEntities(tenantId, sessionId, previousFindings ?? new List<Dictionary<string, object>>(), mostUrgentOnly: false)
.Select(e => e.GetString("CveId")),
StringComparer.OrdinalIgnoreCase);
if (entities.Count == 0 && previousIds.Count == 0) return;
var currentIds = new HashSet<string>(entities.Select(e => e.GetString("CveId")), StringComparer.OrdinalIgnoreCase);
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.CveIndex);
var failed = 0;
@@ -269,23 +285,55 @@ namespace AutopilotMonitor.Functions.Services
{
try
{
await tableClient.UpsertEntityAsync(entity, cancellationToken: ct);
if (previousIds.Contains(entity.GetString("CveId")))
{
var detectedAt = entity["DetectedAt"];
entity.Remove("DetectedAt");
try
{
await tableClient.UpdateEntityAsync(entity, ETag.All, TableUpdateMode.Merge, ct);
return;
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// The earlier write of this row never landed.
entity["DetectedAt"] = detectedAt;
}
}
await tableClient.UpsertEntityAsync(entity, TableUpdateMode.Replace, ct);
}
catch (Exception ex)
{
Interlocked.Increment(ref failed);
_logger.LogWarning(ex, "Failed to upsert CveIndex entry {CveId} for session {SessionId}",
_logger.LogWarning(ex, "Failed to write CveIndex entry {CveId} for session {SessionId}",
entity.GetString("CveId"), sessionId);
}
});
foreach (var cveId in previousIds.Where(id => !currentIds.Contains(id)))
{
try
{
await tableClient.DeleteEntityAsync($"{tenantId}_{cveId}", sessionId);
}
catch (RequestFailedException ex) when (ex.Status == 404)
{
// Already gone.
}
catch (Exception ex)
{
failed++;
_logger.LogWarning(ex, "Failed to delete CveIndex entry {CveId} for session {SessionId}", cveId, sessionId);
}
}
if (failed > 0)
_logger.LogWarning("CveIndex upsert for session {SessionId}: {Failed}/{Total} entries failed",
sessionId, failed, entities.Count);
_logger.LogWarning("CveIndex update for session {SessionId}: {Failed} of {Total} entries failed",
sessionId, failed, entities.Count + previousIds.Count(id => !currentIds.Contains(id)));
}
catch (Exception ex)
{
_logger.LogWarning(ex, "Failed to upsert CveIndex entries for session {SessionId}", sessionId);
_logger.LogWarning(ex, "Failed to update CveIndex entries for session {SessionId}", sessionId);
}
}
@@ -293,9 +341,16 @@ namespace AutopilotMonitor.Functions.Services
/// Collapses report findings to one entity per CVE. Merge rule: the occurrence with the
/// highest CVSS score supplies score/severity/software name/product (ties keep report
/// order), IsKev is OR-ed, OverallRisk is the highest risk level seen across occurrences.
/// Product is the finding's CPE "vendor:product" (<see cref="Vulnerability.CpeMatchScorer.ProductKey"/>).
/// Product is the finding's CPE "vendor:product" (<see cref="Vulnerability.CpeName.ProductKeyOf"/>).
/// <para>
/// With <paramref name="mostUrgentOnly"/> a finding contributes its first
/// <see cref="MaxIndexedCvesPerFinding"/> CVEs (the report lists them most urgent first) and
/// every known-exploited one. One row per CVE and session for all of a browser's CVEs would
/// outgrow what the fleet summary reads; known-exploited counts stay complete.
/// </para>
/// </summary>
internal static List<TableEntity> BuildCveIndexEntities(string tenantId, string sessionId, List<Dictionary<string, object>> findings)
internal static List<TableEntity> BuildCveIndexEntities(
string tenantId, string sessionId, List<Dictionary<string, object>> findings, bool mostUrgentOnly = true)
{
var byCve = new Dictionary<string, TableEntity>(StringComparer.OrdinalIgnoreCase);
var now = DateTime.UtcNow;
@@ -303,12 +358,13 @@ namespace AutopilotMonitor.Functions.Services
foreach (var finding in findings)
{
var softwareName = finding.TryGetValue("softwareName", out var sn) ? sn?.ToString() ?? "" : "";
var product = Vulnerability.CpeMatchScorer.ProductKey(finding.TryGetValue("cpeUri", out var cu) ? cu?.ToString() : null);
var product = Vulnerability.CpeName.ProductKeyOf(finding.TryGetValue("cpeUri", out var cu) ? cu?.ToString() : null);
var overallRisk = finding.TryGetValue("riskLevel", out var rl) ? rl?.ToString() ?? "" : "";
if (!finding.TryGetValue("vulnerabilities", out var vulnsObj) || vulnsObj == null) continue;
if (vulnsObj is not System.Collections.IEnumerable vulnList) continue;
var position = 0;
foreach (var vulnObj in vulnList)
{
if (vulnObj is not Dictionary<string, object> vuln) continue;
@@ -316,6 +372,9 @@ namespace AutopilotMonitor.Functions.Services
var cveId = vuln.TryGetValue("cveId", out var cid) ? cid?.ToString() : null;
if (string.IsNullOrEmpty(cveId)) continue;
var withinMostUrgent = position++ < MaxIndexedCvesPerFinding;
if (mostUrgentOnly && !withinMostUrgent && !(vuln.TryGetValue("isKev", out var kevFlag) && kevFlag is true)) continue;
double cvssScore = 0;
try { if (vuln.TryGetValue("cvssScore", out var cs) && cs != null) cvssScore = Convert.ToDouble(cs); } catch { }
@@ -1279,11 +1338,12 @@ namespace AutopilotMonitor.Functions.Services
/// targets only that tenant's CVE partitions; cross-tenant (tenantId null) is
/// a bounded full-table scan — each row carries TenantId + CveId, so the
/// aggregate is correct without the loose suffix-range the per-CVE search
/// has to use. Reads at most <paramref name="maxRows"/> rows and reports
/// whether the cap was hit so the caller can flag partial results.
/// has to use. Reads at most <paramref name="maxRows"/> rows detected since
/// <paramref name="detectedSinceUtc"/> and reports whether the cap was hit so the caller
/// can flag partial results.
/// </summary>
public async Task<(IReadOnlyList<CveExposureEntry> Rows, bool Truncated)> ScanCveIndexAsync(
string? tenantId, int maxRows, CancellationToken ct = default)
string? tenantId, int maxRows, DateTime? detectedSinceUtc = null, CancellationToken ct = default)
{
if (maxRows < 1) maxRows = 1;
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.CveIndex);
@@ -1295,6 +1355,14 @@ namespace AutopilotMonitor.Functions.Services
filter = $"PartitionKey ge '{safe}_' and PartitionKey lt '{safe}_~'";
}
// The window is applied by the table, not after the read: the row cap would otherwise
// be spent on rows outside it. Every row has carried DetectedAt since the table exists.
if (detectedSinceUtc.HasValue)
{
var since = $"DetectedAt ge datetime'{detectedSinceUtc.Value.ToUniversalTime():yyyy-MM-ddTHH:mm:ssZ}'";
filter = filter == null ? since : $"({filter}) and {since}";
}
var select = new[]
{
"SessionId", "TenantId", "CveId", "SoftwareName", "Product",
@@ -272,9 +272,10 @@ namespace AutopilotMonitor.Functions.Services
existing["SessionCount"] = (existing.GetInt32("SessionCount") ?? 0) + 1;
existing["LastSeenAt"] = nowIso;
existing["LastSessionId"] = sessionId;
// Stamp a CpeUri if we just got one and the row didn't have one yet — preserves
// existing behaviour from UpsertSoftwareInventoryAsync.
if (!string.IsNullOrEmpty(item.CpeUri) && string.IsNullOrEmpty(existing.GetString("CpeUri")))
// A computed CPE, including the empty "no mapping", replaces what the row holds, so
// a corrected or withdrawn mapping reaches rows written before it. Null means the
// caller did not run the matching.
if (item.CpeUri != null)
{
existing["CpeUri"] = item.CpeUri;
}
@@ -1012,7 +1012,9 @@ namespace AutopilotMonitor.Functions.Services
foreach (var chunk in TableStorageChunking.ChunkProperty("ReportJson", reportJson))
entity[chunk.Key] = chunk.Value;
await tableClient.UpsertEntityAsync(entity);
// Replace, not merge: a report that shrinks from the split form to one column would
// otherwise keep the old parts next to it.
await tableClient.UpsertEntityAsync(entity, TableUpdateMode.Replace);
_logger.LogInformation("Stored vulnerability report for session {SessionId} (risk={Risk})", sessionId,
scanSummary?.GetValueOrDefault("overall_risk")?.ToString() ?? "none");
}
@@ -1120,128 +1122,5 @@ namespace AutopilotMonitor.Functions.Services
return new List<Dictionary<string, object>>();
}
}
// ===== CPE MAPPING SEED IMPORT =====
/// <summary>
/// <summary>
/// Imports CPE seed mappings from a JSON string (fetched from GitHub).
/// Deletes existing cpe_map_seed entries, then writes the new ones.
/// Returns the number of entries imported.
/// </summary>
public async Task<int> ImportCpeMappingSeedFromJsonAsync(string json)
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VulnerabilityCache);
// Delete existing seed entries
var existingEntities = tableClient.QueryAsync<TableEntity>(
filter: "PartitionKey eq 'cpe_map_seed'",
select: new[] { "PartitionKey", "RowKey" });
await foreach (var entity in existingEntities)
{
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
}
// Parse the JSON
var seed = System.Text.Json.JsonSerializer.Deserialize<CpeMappingSeed>(json,
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
if (seed?.Mappings == null || seed.Mappings.Count == 0)
return 0;
// Build entities
var entities = new List<TableEntity>();
foreach (var mapping in seed.Mappings)
{
var rowKey = TableKeySanitizer.Sanitize(
$"{(mapping.NormalizedVendor ?? "unknown")}:{(mapping.NormalizedProduct ?? "unknown")}".ToLowerInvariant());
if (rowKey.Length > 512)
rowKey = rowKey.Substring(0, 512);
entities.Add(new TableEntity("cpe_map_seed", rowKey)
{
{ "NormalizedVendor", mapping.NormalizedVendor ?? "" },
{ "NormalizedProduct", mapping.NormalizedProduct ?? "" },
{ "CpeVendor", mapping.CpeVendor ?? "" },
{ "CpeProduct", CpeUriNormalizer.Normalize(mapping.CpeProduct) },
{ "CpeUri", CpeUriNormalizer.Normalize(mapping.CpeUri) },
{ "Category", mapping.Category ?? "" },
{ "DisplayNamePatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.DisplayNamePatterns ?? new List<string>()) },
{ "ExcludePatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.ExcludePatterns ?? new List<string>()) },
{ "PublisherPatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.PublisherPatterns ?? new List<string>()) },
{ "Source", "seed" },
{ "ImportedAt", DateTime.UtcNow.ToString("o") }
});
}
// Batch write (byte-aware, all same PK)
foreach (var batch in TableTransactionBatcher.Split(entities, TableTransactionActionType.UpsertReplace))
await tableClient.SubmitTransactionAsync(batch);
_logger.LogInformation("Imported {Count} CPE seed mapping entries from GitHub JSON into VulnerabilityCache table", entities.Count);
return entities.Count;
}
/// <summary>
/// Imports CPE community mappings from a JSON string (fetched from GitHub).
/// Deletes existing cpe_map_community entries, then writes the new ones.
/// Same format as the seed file (CpeMappingSeed with Mappings list).
/// </summary>
public async Task<int> ImportCpeCommunityMappingsFromJsonAsync(string json)
{
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VulnerabilityCache);
// Delete existing community entries
var existingEntities = tableClient.QueryAsync<TableEntity>(
filter: "PartitionKey eq 'cpe_map_community'",
select: new[] { "PartitionKey", "RowKey" });
await foreach (var entity in existingEntities)
{
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
}
// Parse the JSON (same format as seed)
var seed = System.Text.Json.JsonSerializer.Deserialize<CpeMappingSeed>(json,
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
if (seed?.Mappings == null || seed.Mappings.Count == 0)
return 0;
// Build entities
var entities = new List<TableEntity>();
foreach (var mapping in seed.Mappings)
{
var rowKey = TableKeySanitizer.Sanitize(
$"{(mapping.NormalizedVendor ?? "unknown")}:{(mapping.NormalizedProduct ?? "unknown")}".ToLowerInvariant());
if (rowKey.Length > 512)
rowKey = rowKey.Substring(0, 512);
entities.Add(new TableEntity("cpe_map_community", rowKey)
{
{ "NormalizedVendor", mapping.NormalizedVendor ?? "" },
{ "NormalizedProduct", mapping.NormalizedProduct ?? "" },
{ "CpeVendor", mapping.CpeVendor ?? "" },
{ "CpeProduct", mapping.CpeProduct ?? "" },
{ "CpeUri", mapping.CpeUri ?? "" },
{ "Category", mapping.Category ?? "community" },
{ "DisplayNamePatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.DisplayNamePatterns ?? new List<string>()) },
{ "ExcludePatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.ExcludePatterns ?? new List<string>()) },
{ "PublisherPatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.PublisherPatterns ?? new List<string>()) },
{ "Source", "community" },
{ "ImportedAt", DateTime.UtcNow.ToString("o") }
});
}
// Batch write (byte-aware, all same PK)
foreach (var batch in TableTransactionBatcher.Split(entities, TableTransactionActionType.UpsertReplace))
await tableClient.SubmitTransactionAsync(batch);
_logger.LogInformation("Imported {Count} CPE community mapping entries from GitHub JSON into VulnerabilityCache table", entities.Count);
return entities.Count;
}
}
}
@@ -69,6 +69,18 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
}
}
public async Task EnqueueDelayedAsync(VulnerabilityCorrelateEnvelope envelope, TimeSpan delay, CancellationToken cancellationToken = default)
{
if (envelope is null) throw new ArgumentNullException(nameof(envelope));
await EnsureQueueExistsAsync(cancellationToken).ConfigureAwait(false);
var body = JsonConvert.SerializeObject(envelope);
await _queueClient.SendMessageAsync(body, visibilityTimeout: delay, cancellationToken: cancellationToken).ConfigureAwait(false);
_logger.LogWarning(
"VulnerabilityCorrelate deferred by {DelaySeconds:0}s (tenant={Tenant} session={Session} reason={Reason} deferrals={Deferrals})",
delay.TotalSeconds, envelope.TenantId, envelope.SessionId, envelope.Reason, envelope.Deferrals);
}
private async Task EnsureQueueExistsAsync(CancellationToken cancellationToken)
{
if (_queueEnsured == 1) return;
@@ -0,0 +1,297 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>Why a program name got an automatic mapping, or did not.</summary>
public enum CpeAutoMapOutcome
{
Mapped,
/// <summary>A mapping already covers the name.</summary>
AlreadyMapped,
/// <summary>An operator put the name on the ignore list.</summary>
Ignored,
/// <summary>The name is not an NVD product name, or more than one product carries it.</summary>
NoExactProduct,
/// <summary>NVD lists the product but no CVE for it: a mapping would only make it look checked.</summary>
NoCves,
/// <summary>
/// NVD lists a CVE for the product without any version. It would be reported for every
/// install, whatever its version, and nobody has looked at this product: left to the operator.
/// </summary>
UnversionedCves,
/// <summary>
/// The program occurs only as a Store package, whose version is the package's. Comparing it
/// with the product's version ranges can report every CVE the product ever had.
/// </summary>
StorePackage,
/// <summary>NVD did not answer; the name is tried again next time.</summary>
NvdUnavailable,
/// <summary>No dictionary has been stored yet.</summary>
NoDictionary,
}
public sealed class CpeAutoMapResult
{
public CpeAutoMapOutcome Outcome { get; set; }
public string? CpeUri { get; set; }
public string? ProductTitle { get; set; }
/// <summary>True when the attempt sent a request to NVD.</summary>
internal bool AskedNvd { get; set; }
/// <summary>True when NVD throttled or gave no answer: the next request would fare the same.</summary>
internal bool NvdDown { get; set; }
}
/// <summary>Totals of one nightly pass over the unmapped inventory names.</summary>
public sealed class CpeAutoMapRunResult
{
public int UnmappedNames { get; set; }
public int Mapped { get; set; }
/// <summary>The mappings created, each as "program -> vendor:product".</summary>
public List<string> Created { get; } = new();
public int NoCves { get; set; }
/// <summary>Exact products left unmapped because NVD lists a CVE for them without any version.</summary>
public int Unversioned { get; set; }
public int NvdUnavailable { get; set; }
/// <summary>Exact products left unmapped because the program occurs only as a Store package.</summary>
public int StorePackages { get; set; }
/// <summary>
/// Exact products not tried because the run had used its requests or NVD was not
/// answering; they are taken up next time.
/// </summary>
public int Postponed { get; set; }
public bool NoDictionary { get; set; }
}
/// <summary>
/// Gives a program an automatic mapping when its name is exactly an NVD product name (see
/// <see cref="CpeDictionary"/>), NVD lists at least one CVE for that product and none of the
/// cached ones lacks a version. Runs against the stored copy of the dictionary; the only
/// request it sends carries the CPE identifier of the product found, to read its CVEs.
/// Automatic mappings rank below every other source.
/// <para>
/// The version rule keeps one kind of false alarm out: a record that names a product without
/// a version is often a product NVD listed in a hurry and never revisited, and it would make
/// every device with the program an urgent case for good. A curated mapping can accept that
/// knowingly; an automatic one cannot.
/// </para>
/// </summary>
public class CpeDictionaryResolver
{
/// <summary>
/// Requests one nightly pass may send to NVD, one per product it has not asked about in
/// the last week. A larger backlog is worked off over the following nights; the answer
/// "no CVEs" is kept for a week, so the pass moves on instead of asking the same again.
/// </summary>
internal const int MaxNvdRequestsPerRun = 50;
/// <summary>Requests NVD may leave unanswered before the pass stops asking.</summary>
internal const int MaxUnansweredPerRun = 2;
/// <summary>
/// Time the nightly pass may spend asking. Fifty answers that come, but slowly, would
/// otherwise take the time the dictionary download and the cache walk need to fit into
/// the function timeout together with it.
/// </summary>
internal TimeSpan RunBudget { get; init; } = TimeSpan.FromMinutes(5);
private readonly CpeDictionaryService _dictionary;
private readonly CpeMappingStore _mappingStore;
private readonly IVulnerabilityRepository _vulnRepo;
private readonly CveCacheFiller _cveCache;
private readonly ILogger<CpeDictionaryResolver> _logger;
public CpeDictionaryResolver(
CpeDictionaryService dictionary,
CpeMappingStore mappingStore,
IVulnerabilityRepository vulnRepo,
CveCacheFiller cveCache,
ILogger<CpeDictionaryResolver> logger)
{
_dictionary = dictionary;
_mappingStore = mappingStore;
_vulnRepo = vulnRepo;
_cveCache = cveCache;
_logger = logger;
}
/// <summary>
/// The programs that occur only as Store packages (see <see cref="UnmappedSoftware.StoreOnlyKeys"/>),
/// by software key. The nightly pass skips them itself; a caller that resolves single
/// names asks here first.
/// </summary>
public async Task<HashSet<string>> StoreOnlyKeysAsync()
{
var rows = await _vulnRepo.GetAllSoftwareInventoryAsync(UnmappedSoftware.Columns).ConfigureAwait(false);
return UnmappedSoftware.StoreOnlyKeys(rows);
}
/// <summary>
/// Tries one program name. The caller reconciles the inventory after its last call.
/// With <paramref name="askNvd"/> off, everything that can be said without NVD is said and
/// a product whose CVEs would have to be fetched comes back as not answered.
/// <paramref name="storePackage"/> says the program occurs only as a Store package
/// (see <see cref="StoreOnlyKeysAsync"/>); such a program gets no mapping here.
/// </summary>
public async Task<CpeAutoMapResult> ResolveAsync(
string? normalizedVendor, string? displayName, string? publisher, CancellationToken ct = default,
bool askNvd = true, bool storePackage = false)
{
var dictionary = await _dictionary.GetAsync(ct).ConfigureAwait(false);
if (dictionary == null) return new CpeAutoMapResult { Outcome = CpeAutoMapOutcome.NoDictionary };
var mappings = await _mappingStore.GetAsync().ConfigureAwait(false);
return await ResolveAsync(
dictionary, mappings, normalizedVendor, displayName, publisher, askNvd, storePackage, ct).ConfigureAwait(false);
}
/// <summary>
/// Tries every inventory name without a mapping, most used first.
/// </summary>
/// <exception cref="CpeMappingUnavailableException">The mappings cannot be read.</exception>
public async Task<CpeAutoMapRunResult> ResolveInventoryAsync(CancellationToken ct = default)
{
var run = new CpeAutoMapRunResult();
var dictionary = await _dictionary.GetAsync(ct).ConfigureAwait(false);
if (dictionary == null)
{
run.NoDictionary = true;
return run;
}
var mappings = await _mappingStore.GetAsync().ConfigureAwait(false);
var rows = await _vulnRepo.GetAllSoftwareInventoryAsync(UnmappedSoftware.Columns).ConfigureAwait(false);
var names = UnmappedSoftware.Aggregate(rows, mappings);
var storeOnly = UnmappedSoftware.StoreOnlyKeys(rows);
run.UnmappedNames = names.Count;
var asked = 0;
var unanswered = 0;
var pass = Stopwatch.StartNew();
// The most used names first, so a limited run spends its requests where they count.
foreach (var name in names.OrderByDescending(n => n.Frequency).ThenBy(n => n.CanonicalName, StringComparer.Ordinal))
{
ct.ThrowIfCancellationRequested();
if (dictionary.Resolve(name.CanonicalName ?? string.Empty, name.NormalizedVendor, name.Publisher) == null) continue;
var storePackage = storeOnly.Contains(CpeMappingMatcher.SoftwareKey(name.NormalizedVendor, name.SoftwareName));
if (!storePackage && (asked >= MaxNvdRequestsPerRun || unanswered >= MaxUnansweredPerRun || pass.Elapsed >= RunBudget))
{
run.Postponed++;
continue;
}
var result = await ResolveAsync(
dictionary, mappings, name.NormalizedVendor, name.SoftwareName, name.Publisher, true, storePackage, ct).ConfigureAwait(false);
if (result.AskedNvd) asked++;
if (result.NvdDown) unanswered++;
switch (result.Outcome)
{
case CpeAutoMapOutcome.Mapped:
run.Mapped++;
run.Created.Add($"{name.CanonicalName} -> {CpeName.ProductKeyOf(result.CpeUri ?? string.Empty)}");
break;
case CpeAutoMapOutcome.NoCves: run.NoCves++; break;
case CpeAutoMapOutcome.UnversionedCves: run.Unversioned++; break;
case CpeAutoMapOutcome.StorePackage: run.StorePackages++; break;
case CpeAutoMapOutcome.NvdUnavailable: run.NvdUnavailable++; break;
}
}
if (run.Mapped > 0) _mappingStore.Invalidate();
_logger.LogInformation(
"[VulnSync] Automatic mappings: {Mapped} added, {NoCves} products without CVEs, {Unversioned} with CVEs that name no version, {Unavailable} without an answer from NVD, {Postponed} not tried this run ({Names} unmapped names)",
run.Mapped, run.NoCves, run.Unversioned, run.NvdUnavailable, run.Postponed, run.UnmappedNames);
return run;
}
private async Task<CpeAutoMapResult> ResolveAsync(
CpeDictionary dictionary, CpeMappingSnapshot mappings,
string? normalizedVendor, string? displayName, string? publisher, bool askNvd, bool storePackage, CancellationToken ct)
{
if (mappings.Matcher.Match(normalizedVendor, displayName, displayName, publisher) is { } existing)
return new CpeAutoMapResult { Outcome = CpeAutoMapOutcome.AlreadyMapped, CpeUri = existing.Mapping.CpeUri };
if (mappings.IsIgnored(normalizedVendor, publisher, displayName))
return new CpeAutoMapResult { Outcome = CpeAutoMapOutcome.Ignored };
var canonical = SoftwareNameCanonicalizer.Canonicalize(displayName);
if (dictionary.Resolve(canonical, normalizedVendor, publisher) is not { } product)
return new CpeAutoMapResult { Outcome = CpeAutoMapOutcome.NoExactProduct };
var cpeUri = CpeName.ToProductUri(product.CpeUri);
var result = new CpeAutoMapResult { CpeUri = cpeUri, ProductTitle = product.Title };
if (storePackage)
{
result.Outcome = CpeAutoMapOutcome.StorePackage;
return result;
}
// The fetch doubles as the product's first cache fill. A row of the older shape is
// not an answer to "does this product have CVEs": it is fetched again.
var cached = await _cveCache.ReadAsync(cpeUri).ConfigureAwait(false);
if (cached is { CurrentShape: false }) cached = null;
if (cached == null)
{
if (!askNvd)
{
result.Outcome = CpeAutoMapOutcome.NvdUnavailable;
return result;
}
var fill = await _cveCache.FillAsync(cpeUri, ct).ConfigureAwait(false);
result.AskedNvd = true;
if (fill.Outcome != CveCacheFillOutcome.Filled)
{
result.Outcome = CpeAutoMapOutcome.NvdUnavailable;
result.NvdDown = fill.Outcome != CveCacheFillOutcome.Failed;
return result;
}
cached = fill.Entry;
}
// What the cache holds is what a scan can report: the product's own ranges inside the
// time window. NVD's total also counts CVEs that name the product only as a platform.
if (cached == null || cached.Cves.Count == 0)
{
result.Outcome = CpeAutoMapOutcome.NoCves;
return result;
}
// Without an installed version the filter reports exactly the records that name none.
if (CveVersionFilter.Filter(cached.Cves, cpeUri, string.Empty).Vulnerabilities.Count > 0)
{
result.Outcome = CpeAutoMapOutcome.UnversionedCves;
return result;
}
var vendorKey = CpeMappingMatcher.VendorKey(normalizedVendor);
await _vulnRepo.UpsertCpeMappingAsync(
CpeMappingStore.AutoPartition,
CpeSeedFile.RowKeyOf(vendorKey, canonical),
new Dictionary<string, object>
{
{ "NormalizedVendor", vendorKey },
{ "NormalizedProduct", canonical },
{ "CpeVendor", product.Vendor },
{ "CpeProduct", product.Product },
{ "CpeUri", cpeUri },
{ "Category", "auto" },
{ "DisplayNamePatternsJson", JsonSerializer.Serialize(Array.Empty<string>()) },
{ "PublisherPatternsJson", JsonSerializer.Serialize(Array.Empty<string>()) },
{ "ExcludePatternsJson", JsonSerializer.Serialize(Array.Empty<string>()) },
{ "Source", "auto" },
{ "ProductTitle", product.Title },
{ "CreatedAt", DateTime.UtcNow.ToString("o") },
}).ConfigureAwait(false);
result.Outcome = CpeAutoMapOutcome.Mapped;
return result;
}
}
}
@@ -0,0 +1,481 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics;
using System.Formats.Tar;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>One product of the reduced NVD dictionary. The key it is stored under is <c>vendor:product</c>.</summary>
public sealed class CpeDictionaryProduct
{
/// <summary>English title of the product; shown to the operator next to an automatic mapping.</summary>
public string T { get; set; } = string.Empty;
}
public sealed class CpeDictionaryRefreshResult
{
/// <summary>False when the feed still carries the version already stored; nothing was downloaded.</summary>
public bool Changed { get; set; }
public int Products { get; set; }
public string? FeedVersion { get; set; }
public string? Error { get; set; }
public long DurationMs { get; set; }
}
/// <summary>
/// A product name from the dictionary that a program name was resolved to.
/// </summary>
internal readonly struct CpeDictionaryMatch
{
public CpeDictionaryMatch(string vendor, string product, string title)
{
Vendor = vendor;
Product = product;
Title = title;
}
public string Vendor { get; }
public string Product { get; }
public string Title { get; }
public string CpeUri => $"cpe:2.3:a:{Vendor}:{Product}";
}
/// <summary>
/// The reduced NVD product dictionary in memory, indexed for the one question asked of it:
/// is this program name an NVD product name? There is no similarity search. A name resolves
/// only when its words ARE the product's words and the vendor agrees, or when they are the
/// vendor's words followed by the product's, and only when exactly one product fits.
/// </summary>
internal sealed class CpeDictionary
{
private readonly Dictionary<string, List<CpeDictionaryMatch>> _byProductWords = new(StringComparer.Ordinal);
private readonly Dictionary<string, List<CpeDictionaryMatch>> _byVendorAndProductWords = new(StringComparer.Ordinal);
public CpeDictionary(IReadOnlyDictionary<string, CpeDictionaryProduct> products, string lastRefresh)
{
LastRefresh = lastRefresh;
Count = products.Count;
foreach (var pair in products)
{
var separator = pair.Key.IndexOf(':');
if (separator <= 0 || separator == pair.Key.Length - 1) continue;
var match = new CpeDictionaryMatch(pair.Key.Substring(0, separator), pair.Key.Substring(separator + 1), pair.Value.T);
var productWords = WordsOf(match.Product);
if (productWords.Length == 0) continue;
Add(_byProductWords, Join(productWords), match);
Add(_byVendorAndProductWords, Join(WordsOf(match.Vendor).Concat(productWords)), match);
}
}
public int Count { get; }
public string LastRefresh { get; }
/// <param name="canonicalName">The program name without version, architecture and language.</param>
public CpeDictionaryMatch? Resolve(string canonicalName, string? normalizedVendor, string? publisher)
{
var nameWords = CpeMappingMatcher.Tokenize(canonicalName);
if (nameWords.Length == 0) return null;
var key = Join(nameWords);
var candidates = new Dictionary<string, CpeDictionaryMatch>(StringComparer.Ordinal);
if (_byProductWords.TryGetValue(key, out var sameName))
{
var programVendors = new[] { CpeMappingMatcher.VendorTokens(normalizedVendor), CpeMappingMatcher.VendorTokens(publisher) };
foreach (var match in sameName)
{
var productVendor = new List<string[]> { CpeMappingMatcher.VendorTokens(match.Vendor.Replace('_', ' ')) };
if (CpeMappingMatcher.VendorAgrees(productVendor, programVendors))
candidates[match.Vendor + ":" + match.Product] = match;
}
}
if (_byVendorAndProductWords.TryGetValue(key, out var vendorAndName))
{
foreach (var match in vendorAndName)
candidates[match.Vendor + ":" + match.Product] = match;
}
return candidates.Count == 1 ? candidates.Values.First() : null;
}
// A CPE field escapes punctuation with a backslash and joins words with an underscore.
private static string[] WordsOf(string cpeField) =>
CpeMappingMatcher.Tokenize(cpeField.Replace("\\", string.Empty).Replace('_', ' '));
private static string Join(IEnumerable<string> words) => string.Join(' ', words);
private static void Add(Dictionary<string, List<CpeDictionaryMatch>> index, string key, CpeDictionaryMatch match)
{
if (!index.TryGetValue(key, out var list)) index[key] = list = new List<CpeDictionaryMatch>(1);
list.Add(match);
}
}
/// <summary>
/// Keeps a reduced copy of the NVD CPE dictionary as a blob next to the other vulnerability
/// sources. Program names are compared with this copy inside the backend; no name leaves it.
/// <para>
/// The feed is about 85 MB compressed and 0.9 GB unpacked. It is read as a stream, one
/// product name at a time, and reduced to the application products that are not withdrawn
/// and can be Windows software, which is a few megabytes. The copy is loaded only by the
/// nightly run and the operator's Auto-Map, never at instance start.
/// </para>
/// </summary>
public class CpeDictionaryService
{
public const string BlobName = "cpe-dictionary.json";
internal const string MetaUrl = "https://nvd.nist.gov/feeds/json/cpe/2.0/nvdcpe-2.0.meta";
internal const string FeedUrl = "https://nvd.nist.gov/feeds/json/cpe/2.0/nvdcpe-2.0.tar.gz";
/// <summary>
/// One entry of a chunk file is a few kilobytes. The read buffer grows to hold an entry,
/// but not beyond this: a file whose entry never ends is refused, not loaded.
/// </summary>
internal const int MaxEntryBytes = 16 * 1024 * 1024;
/// <summary>A reduced dictionary this small is a broken download, not a new edition.</summary>
internal const int MinProducts = 10_000;
/// <summary>
/// Covers the whole refresh. The nightly run goes on to the CVE cache afterwards and has to
/// fit into the function timeout with it.
/// </summary>
internal static readonly TimeSpan DownloadTimeout = TimeSpan.FromMinutes(10);
/// <summary>
/// An instance that loaded the copy keeps it; after this time it looks at the blob again,
/// which another instance's nightly run may have replaced.
/// </summary>
internal static readonly TimeSpan LoadedCopyMaxAge = TimeSpan.FromHours(12);
private readonly HttpClient _http;
private readonly BlobBackedReferenceCache<CpeDictionaryProduct> _blob;
private readonly ILogger<CpeDictionaryService> _logger;
private readonly SemaphoreSlim _loadLock = new(1, 1);
private CpeDictionary? _loaded;
private DateTime _loadedAtUtc;
public CpeDictionaryService(HttpClient http, BlobStorageService blobs, ILogger<CpeDictionaryService> logger)
{
_http = http ?? throw new ArgumentNullException(nameof(http));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_http.Timeout = Timeout.InfiniteTimeSpan; // bounded by DownloadTimeout, which covers the body
_blob = new BlobBackedReferenceCache<CpeDictionaryProduct>(blobs, BlobName, logger);
}
/// <summary>The stored dictionary, or null when none was ever stored.</summary>
internal async Task<CpeDictionary?> GetAsync(CancellationToken ct = default)
{
if (IsCurrent()) return _loaded;
await _loadLock.WaitAsync(ct).ConfigureAwait(false);
try
{
if (IsCurrent()) return _loaded;
var snapshot = await _blob.TryLoadAsync(ct).ConfigureAwait(false);
// A blob that cannot be read leaves the copy this instance holds in service.
if (snapshot == null || snapshot.Entries.Count == 0) return _loaded;
_loaded = new CpeDictionary(snapshot.Entries, snapshot.LastRefresh);
_loadedAtUtc = DateTime.UtcNow;
return _loaded;
}
finally
{
_loadLock.Release();
}
}
/// <summary>
/// Downloads the feed when NVD published a new one and stores the reduced copy. Never throws:
/// a failure is reported in the result and the stored copy stays in service.
/// </summary>
internal async Task<CpeDictionaryRefreshResult> RefreshAsync(CancellationToken ct = default)
{
var elapsed = Stopwatch.StartNew();
var result = new CpeDictionaryRefreshResult();
try
{
using var budget = CancellationTokenSource.CreateLinkedTokenSource(ct);
budget.CancelAfter(DownloadTimeout);
var feedVersion = ParseFeedVersion(await GetStringAsync(MetaUrl, budget.Token).ConfigureAwait(false));
result.FeedVersion = feedVersion;
var stored = await _blob.TryLoadAsync(budget.Token).ConfigureAwait(false);
if (feedVersion != null && stored?.CoveredDocuments?.Contains(feedVersion) == true && stored.Entries.Count >= MinProducts)
{
result.Products = stored.Entries.Count;
return result;
}
var products = await DownloadAndReduceAsync(budget.Token).ConfigureAwait(false);
if (products.Count < MinProducts)
throw new InvalidOperationException($"The reduced dictionary holds only {products.Count} products; the stored copy is kept.");
var snapshot = new BlobBackedReferenceCache<CpeDictionaryProduct>.Snapshot
{
LastRefresh = DateTime.UtcNow.ToString("o"),
CoveredDocuments = feedVersion == null ? new List<string>() : new List<string> { feedVersion },
Entries = products,
};
await _blob.SaveAsync(snapshot, budget.Token).ConfigureAwait(false);
_loaded = new CpeDictionary(products, snapshot.LastRefresh);
_loadedAtUtc = DateTime.UtcNow;
result.Changed = true;
result.Products = products.Count;
}
catch (Exception ex)
{
result.Error = ex.Message;
_logger.LogWarning(ex, "[VulnSync] CPE dictionary refresh failed");
}
finally
{
result.DurationMs = elapsed.ElapsedMilliseconds;
}
return result;
}
private bool IsCurrent() => _loaded != null && DateTime.UtcNow - _loadedAtUtc < LoadedCopyMaxAge;
// NVD writes "windows" for most Windows software and names a release for some ("windows_10").
// The phone, embedded and ARM-only lines share the prefix and are other platforms.
private static bool IsWindowsCapable(string targetSw) =>
targetSw is "*" or "-"
|| (targetSw.StartsWith("windows", StringComparison.Ordinal) && !CveVersionFilter.IsOtherPlatform(targetSw));
/// <summary>When the stored dictionary was last replaced; null when there is none.</summary>
internal async Task<DateTime?> LastRefreshAsync(CancellationToken ct = default)
{
var dictionary = await GetAsync(ct).ConfigureAwait(false);
return dictionary != null && DateTime.TryParse(
dictionary.LastRefresh, System.Globalization.CultureInfo.InvariantCulture,
System.Globalization.DateTimeStyles.RoundtripKind, out var refreshed)
? refreshed.ToUniversalTime()
: null;
}
private async Task<Dictionary<string, CpeDictionaryProduct>> DownloadAndReduceAsync(CancellationToken ct)
{
using var request = new HttpRequestMessage(HttpMethod.Get, FeedUrl);
request.Headers.Add("User-Agent", "AutopilotMonitor/1.0");
using var response = await _http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
await using var body = await response.Content.ReadAsStreamAsync(ct).ConfigureAwait(false);
return await ReduceAsync(body, ct).ConfigureAwait(false);
}
/// <summary>Reads the feed archive as a stream and returns the reduced dictionary.</summary>
internal static async Task<Dictionary<string, CpeDictionaryProduct>> ReduceAsync(Stream feed, CancellationToken ct)
{
await using var unzipped = new GZipStream(feed, CompressionMode.Decompress);
await using var tar = new TarReader(unzipped);
var reduced = new Dictionary<string, ProductState>(StringComparer.Ordinal);
while (await tar.GetNextEntryAsync(copyData: false, ct).ConfigureAwait(false) is { } entry)
{
if (entry.DataStream == null || !entry.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) continue;
await ReadChunkAsync(entry.DataStream, reduced, ct).ConfigureAwait(false);
}
return reduced
.Where(pair => pair.Value.WindowsCapable)
.ToDictionary(pair => pair.Key, pair => new CpeDictionaryProduct { T = pair.Value.Title }, StringComparer.Ordinal);
}
internal sealed class ProductState
{
public string Title = string.Empty;
public bool WindowsCapable;
}
/// <summary>Where the reader stands in a chunk file between two buffer fills.</summary>
private sealed class ChunkCursor
{
public JsonReaderState Json;
/// <summary>The <c>products</c> property name was read; its value comes next.</summary>
public bool AtProducts;
public bool InProducts;
}
/// <summary>
/// Folds one chunk file into the per-product state without loading the file: the stream
/// goes through a small buffer and every entry of the <c>products</c> array is parsed on
/// its own. A chunk is about 50 MB of JSON; parsed as one document, the feed took several
/// hundred megabytes of heap on an instance that serves requests at the same time.
/// </summary>
internal static async Task ReadChunkAsync(
Stream chunk, Dictionary<string, ProductState> reduced, CancellationToken ct, int bufferBytes = 256 * 1024)
{
var buffer = ArrayPool<byte>.Shared.Rent(bufferBytes);
try
{
var cursor = new ChunkCursor();
var filled = 0;
var ended = false;
while (!ended)
{
var read = await chunk.ReadAsync(buffer.AsMemory(filled), ct).ConfigureAwait(false);
ended = read == 0;
filled += read;
var used = FoldEntries(buffer.AsSpan(0, filled), ended, cursor, reduced);
if (used == 0 && filled == buffer.Length)
{
// The entry at the start of the buffer does not fit into it.
if (buffer.Length >= MaxEntryBytes)
throw new InvalidDataException($"A dictionary entry is larger than {MaxEntryBytes} bytes.");
var larger = ArrayPool<byte>.Shared.Rent(buffer.Length * 2);
buffer.AsSpan(0, filled).CopyTo(larger);
ArrayPool<byte>.Shared.Return(buffer);
buffer = larger;
continue;
}
buffer.AsSpan(used, filled - used).CopyTo(buffer);
filled -= used;
}
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}
/// <summary>
/// Reads the whole tokens and <c>products</c> entries <paramref name="data"/> holds and
/// returns the number of bytes they took. What follows is cut off by the end of the
/// buffer; the caller reads on and passes it again from its first byte.
/// </summary>
private static int FoldEntries(
ReadOnlySpan<byte> data, bool isFinalBlock, ChunkCursor cursor, Dictionary<string, ProductState> reduced)
{
var reader = new Utf8JsonReader(data, isFinalBlock, cursor.Json);
while (true)
{
// A value that is cut off is read again from its first token.
var before = reader;
if (!reader.Read()) break;
if (cursor.InProducts)
{
if (reader.TokenType == JsonTokenType.EndArray)
{
cursor.InProducts = false;
}
else if (reader.TokenType == JsonTokenType.StartObject)
{
if (!JsonDocument.TryParseValue(ref reader, out var entry))
{
reader = before;
break;
}
using (entry) Fold(entry!.RootElement, reduced);
}
else if (reader.TokenType == JsonTokenType.StartArray && !reader.TrySkip())
{
reader = before;
break;
}
continue;
}
if (cursor.AtProducts)
{
if (reader.TokenType == JsonTokenType.StartArray)
{
cursor.InProducts = true;
}
else if (reader.TokenType == JsonTokenType.StartObject && !reader.TrySkip())
{
reader = before;
break;
}
cursor.AtProducts = false;
continue;
}
if (reader.CurrentDepth != 1) continue; // the root object's own braces
if (reader.TokenType == JsonTokenType.PropertyName)
{
cursor.AtProducts = reader.ValueTextEquals("products"u8);
}
else if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray && !reader.TrySkip())
{
reader = before;
break;
}
}
cursor.Json = reader.CurrentState;
return (int)reader.BytesConsumed;
}
/// <summary>
/// Folds one entry into the per-product state: application products that are not
/// deprecated; a product counts as possible Windows software when one of its names targets
/// Windows or no platform. What exists only as a plug-in for another system, a package of a
/// language ecosystem or a mobile app carries that system as its target and stays out.
/// </summary>
private static void Fold(JsonElement entry, Dictionary<string, ProductState> reduced)
{
if (entry.ValueKind != JsonValueKind.Object) return;
if (!entry.TryGetProperty("cpe", out var cpe) || cpe.ValueKind != JsonValueKind.Object) return;
if (cpe.TryGetProperty("deprecated", out var deprecated) && deprecated.ValueKind == JsonValueKind.True) return;
if (!cpe.TryGetProperty("cpeName", out var nameElement) || nameElement.ValueKind != JsonValueKind.String) return;
if (!CpeName.TryParse(nameElement.GetString(), out var name) || name.Part != "a" || name.ProductKey.Length == 0) return;
if (!reduced.TryGetValue(name.ProductKey, out var state))
reduced[name.ProductKey] = state = new ProductState();
if (IsWindowsCapable(name.TargetSw)) state.WindowsCapable = true;
if (!cpe.TryGetProperty("titles", out var titles) || titles.ValueKind != JsonValueKind.Array) return;
foreach (var title in titles.EnumerateArray())
{
if (title.ValueKind != JsonValueKind.Object) continue;
if (!title.TryGetProperty("lang", out var lang) || lang.ValueKind != JsonValueKind.String || lang.GetString() != "en") continue;
var text = title.TryGetProperty("title", out var titleText) && titleText.ValueKind == JsonValueKind.String
? titleText.GetString()
: null;
if (!string.IsNullOrEmpty(text) && (state.Title.Length == 0 || text!.Length < state.Title.Length))
state.Title = text!;
}
}
/// <summary>The feed's content hash from its .meta file; the marker for "nothing new".</summary>
internal static string? ParseFeedVersion(string? meta)
{
if (string.IsNullOrEmpty(meta)) return null;
foreach (var line in meta!.Split('\n'))
{
var trimmed = line.Trim();
if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
var value = trimmed.Substring("sha256:".Length).Trim();
return value.Length > 0 ? value.ToUpperInvariant() : null;
}
}
return null;
}
private async Task<string> GetStringAsync(string url, CancellationToken ct)
{
using var request = new HttpRequestMessage(HttpMethod.Get, url);
request.Headers.Add("User-Agent", "AutopilotMonitor/1.0");
using var response = await _http.SendAsync(request, ct).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
return await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false);
}
}
}
@@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>
/// An entry of the ignore list. It is filed like a mapping, under the vendor and the program's
/// canonical name, so one entry covers every version of the program.
/// </summary>
internal static class CpeIgnoreEntry
{
/// <summary>Row key and columns of the entry; null when the name canonicalizes to nothing.</summary>
public static (string RowKey, Dictionary<string, object> Properties)? Build(string? vendor, string? softwareName, string reason)
{
var product = SoftwareNameCanonicalizer.Canonicalize(softwareName);
if (product.Length == 0) return null;
var vendorKey = CpeMappingMatcher.VendorKey(vendor);
return (CpeSeedFile.RowKeyOf(vendorKey, product), new Dictionary<string, object>
{
{ "NormalizedProduct", product },
{ "NormalizedVendor", vendorKey },
{ "Reason", reason },
{ "CreatedAt", DateTime.UtcNow.ToString("o") },
});
}
}
}
@@ -0,0 +1,97 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
public sealed class CpeMappingImportResult
{
public int Written { get; set; }
/// <summary>Mappings in service that the list no longer holds and that were removed.</summary>
public int Removed { get; set; }
/// <summary>Why the list was refused; empty when it was imported.</summary>
public IReadOnlyList<string> Problems { get; set; } = Array.Empty<string>();
public bool Refused => Problems.Count > 0;
}
/// <summary>
/// Imports a mapping list from the rules repository: the maintained seed or the community list.
/// The order is fetch, check, write, then remove what the list no longer holds. Whatever fails
/// before the first write leaves the mappings in service exactly as they were, so a list that
/// cannot be fetched or does not pass the check never empties them. A refused list is recorded
/// as an ops event here, so every way of asking for an import reports it.
/// </summary>
public class CpeMappingImporter
{
private readonly IVulnerabilityRepository _vulnRepo;
private readonly GitHubRuleRepository _gitHubRepo;
private readonly CpeProductFlagsService _productFlags;
private readonly CpeMappingStore _mappingStore;
private readonly OpsEventService _opsEvents;
private readonly ILogger<CpeMappingImporter> _logger;
public CpeMappingImporter(
IVulnerabilityRepository vulnRepo,
GitHubRuleRepository gitHubRepo,
CpeProductFlagsService productFlags,
CpeMappingStore mappingStore,
OpsEventService opsEvents,
ILogger<CpeMappingImporter> logger)
{
_vulnRepo = vulnRepo;
_gitHubRepo = gitHubRepo;
_productFlags = productFlags;
_mappingStore = mappingStore;
_opsEvents = opsEvents;
_logger = logger;
}
/// <summary>The maintained seed; it also carries the default set of self-updating products.</summary>
public Task<CpeMappingImportResult> ImportSeedAsync() =>
ImportAsync(_gitHubRepo.FetchCpeSeedMappingsAsync, CpeMappingStore.SeedPartition, "seed", isSeed: true);
public Task<CpeMappingImportResult> ImportCommunityAsync() =>
ImportAsync(_gitHubRepo.FetchCpeCommunityMappingsAsync, CpeMappingStore.CommunityPartition, "community", isSeed: false);
private async Task<CpeMappingImportResult> ImportAsync(
Func<Task<string>> fetch, string partition, string source, bool isSeed)
{
var json = await fetch().ConfigureAwait(false);
var inService = await _vulnRepo.GetCpeMappingsByPartitionAsync(partition).ConfigureAwait(false);
var (document, problems) = CpeSeedFile.Read(json, inService.Count, mayBeEmpty: !isSeed);
if (problems.Count > 0 || document == null)
{
_logger.LogError(
"CPE mapping list '{Source}' refused, the mappings in service stay: {Problems}",
source, string.Join("; ", problems.Take(10)));
await _opsEvents.RecordCpeMappingListRefusedAsync(source, problems).ConfigureAwait(false);
return new CpeMappingImportResult { Problems = problems };
}
var rows = CpeSeedFile.ToRows(document, source);
foreach (var (rowKey, properties) in rows)
await _vulnRepo.UpsertCpeMappingAsync(partition, rowKey, properties).ConfigureAwait(false);
var kept = new HashSet<string>(rows.Select(r => r.RowKey), StringComparer.Ordinal);
var removed = 0;
foreach (var rowKey in inService.Select(r => r.GetValueOrDefault("RowKey")?.ToString() ?? string.Empty))
{
if (rowKey.Length == 0 || kept.Contains(rowKey)) continue;
await _vulnRepo.DeleteCpeMappingAsync(partition, rowKey).ConfigureAwait(false);
removed++;
}
if (isSeed)
await _productFlags.ApplySeedDefaultsAsync(document.SelfUpdatingProducts ?? new List<string>()).ConfigureAwait(false);
_mappingStore.Invalidate();
_logger.LogInformation(
"CPE mapping list '{Source}' imported: {Written} written, {Removed} removed", source, rows.Count, removed);
return new CpeMappingImportResult { Written = rows.Count, Removed = removed };
}
}
}
@@ -0,0 +1,284 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>Where a mapping comes from; the higher value wins when two mappings tie.</summary>
internal enum CpeMappingSource
{
/// <summary>Taken over from the NVD product dictionary without a human looking at it.</summary>
Auto = 0,
Seed = 1,
Community = 2,
/// <summary>Entered by an operator.</summary>
Custom = 3,
}
/// <summary>One software-name-to-CPE mapping as the matcher sees it. Immutable.</summary>
internal sealed class CpeMapping
{
public string PartitionKey { get; init; } = string.Empty;
public string RowKey { get; init; } = string.Empty;
public CpeMappingSource Source { get; init; }
/// <summary>Vendor and product as stored on the row (the exact-match key before canonicalizing).</summary>
public string Vendor { get; init; } = string.Empty;
public string Product { get; init; } = string.Empty;
/// <summary>Version-less CPE (<c>cpe:2.3:a:vendor:product</c>).</summary>
public string CpeUri { get; init; } = string.Empty;
/// <summary>Lower-case <c>vendor:product</c> of <see cref="CpeUri"/>.</summary>
public string ProductKey { get; init; } = string.Empty;
public string Category { get; init; } = string.Empty;
public IReadOnlyList<string> DisplayNamePatterns { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> PublisherPatterns { get; init; } = Array.Empty<string>();
public IReadOnlyList<string> ExcludePatterns { get; init; } = Array.Empty<string>();
/// <summary>When set, only CVE ranges of this software edition count (e.g. <c>esr</c>).</summary>
public string? Edition { get; init; }
/// <summary>CVE ranges of these software editions never count for this mapping.</summary>
public IReadOnlyList<string> ExcludeEditions { get; init; } = Array.Empty<string>();
public string CreatedAt { get; init; } = string.Empty;
}
internal readonly struct CpeMatch
{
public CpeMatch(CpeMapping mapping, string confidence)
{
Mapping = mapping;
Confidence = confidence;
}
public CpeMapping Mapping { get; }
/// <summary><c>high</c> for a key match, <c>medium</c> for a name pattern.</summary>
public string Confidence { get; }
}
/// <summary>
/// Maps an installed program to a CPE product. Rules, in order:
/// <list type="number">
/// <item>Key match: the program's vendor and canonical name equal a mapping's.</item>
/// <item>Name pattern: the pattern's words appear in the display name as whole words, in order.
/// The pattern with the most words wins, then the longest, then the higher source.</item>
/// </list>
/// A pattern never matches as part of a longer word ("rust" in "BeyondTrust"), never when it
/// follows "for" ("Add-in for Microsoft Office" is not Office), and a one-word pattern only
/// when the vendor agrees or the word is the whole name. There is no fuzzy step: an unknown
/// name stays unmapped rather than landing on a neighbouring product.
/// </summary>
internal sealed class CpeMappingMatcher
{
private static readonly Regex Token = new(@"[\p{L}\p{N}]+[+#]*|[+#]+", RegexOptions.CultureInvariant);
private static readonly Regex Marks = new(@"\((?:r|tm|c)\)|[®™©]", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
private static readonly HashSet<string> ForWords = new(StringComparer.Ordinal)
{
"for", "für", "fuer", "pour", "para", "per", "voor", "för", "til", "dla",
};
private static readonly HashSet<string> LegalForms = new(StringComparer.Ordinal)
{
"inc", "llc", "ltd", "gmbh", "ag", "corp", "corporation", "co", "limited", "group", "sa", "se",
"bv", "ab", "pty", "plc", "srl", "oy", "as", "kg", "sro", "lp", "the", "company",
};
private sealed class PatternEntry
{
public PatternEntry(string[] tokens, int textLength, MappingEntry owner)
{
Tokens = tokens;
TextLength = textLength;
Owner = owner;
}
public string[] Tokens { get; }
public int TextLength { get; }
public MappingEntry Owner { get; }
}
private sealed class MappingEntry
{
public MappingEntry(CpeMapping mapping, List<string[]> excludes, List<string[]> vendorKeys)
{
Mapping = mapping;
Excludes = excludes;
VendorKeys = vendorKeys;
}
public CpeMapping Mapping { get; }
public List<string[]> Excludes { get; }
public List<string[]> VendorKeys { get; }
}
private readonly Dictionary<string, CpeMapping> _byKey = new(StringComparer.Ordinal);
private readonly Dictionary<string, List<PatternEntry>> _byFirstToken = new(StringComparer.Ordinal);
public CpeMappingMatcher(IEnumerable<CpeMapping> mappings)
{
foreach (var mapping in mappings)
{
var key = SoftwareKey(mapping.Vendor, mapping.Product);
if (key.Length > 0 && (!_byKey.TryGetValue(key, out var held) || mapping.Source > held.Source))
_byKey[key] = mapping;
var entry = new MappingEntry(
mapping,
mapping.ExcludePatterns.Select(Tokenize).Where(t => t.Length > 0).ToList(),
VendorKeysOf(mapping));
foreach (var pattern in mapping.DisplayNamePatterns)
{
var tokens = Tokenize(pattern);
if (tokens.Length == 0) continue;
if (!_byFirstToken.TryGetValue(tokens[0], out var bucket))
_byFirstToken[tokens[0]] = bucket = new List<PatternEntry>();
bucket.Add(new PatternEntry(tokens, pattern.Length, entry));
}
}
}
/// <summary>
/// The key a program is filed under: its vendor and canonical name. Empty when the name
/// canonicalizes to nothing. "unknown" is the placeholder older rows carry for no vendor.
/// </summary>
public static string SoftwareKey(string? vendor, string? name)
{
var canonical = SoftwareNameCanonicalizer.Canonicalize(name);
return canonical.Length == 0 ? string.Empty : $"{VendorKey(vendor)}|{canonical}";
}
public static string VendorKey(string? vendor)
{
if (string.IsNullOrWhiteSpace(vendor)) return string.Empty;
var key = string.Join(" ", vendor!.Replace(' ', ' ').ToLowerInvariant()
.Split(' ', StringSplitOptions.RemoveEmptyEntries));
return key == "unknown" ? string.Empty : key;
}
public CpeMatch? Match(string? normalizedVendor, string? normalizedName, string? displayName, string? publisher)
{
var shownName = string.IsNullOrWhiteSpace(displayName) ? normalizedName : displayName;
var key = SoftwareKey(normalizedVendor, shownName);
if (key.Length > 0 && _byKey.TryGetValue(key, out var keyed))
return new CpeMatch(keyed, "high");
var display = Tokenize(shownName);
if (display.Length == 0) return null;
var name = Tokenize(normalizedName);
string[][]? programVendors = null;
HashSet<PatternEntry>? judged = null;
PatternEntry? best = null;
for (var i = 0; i < display.Length; i++)
{
if (!_byFirstToken.TryGetValue(display[i], out var bucket)) continue;
foreach (var candidate in bucket)
{
if (!MatchesAt(display, i, candidate.Tokens)) continue;
// A pattern is judged at its first occurrence only.
judged ??= new HashSet<PatternEntry>();
if (!judged.Add(candidate)) continue;
if (i > 0 && ForWords.Contains(display[i - 1])) continue;
if (candidate.Owner.Excludes.Any(exclude => IndexOf(display, exclude) >= 0)) continue;
if (candidate.Tokens.Length == 1
&& !SameTokens(name, candidate.Tokens)
&& !SameTokens(display, candidate.Tokens))
{
programVendors ??= new[] { VendorTokens(normalizedVendor), VendorTokens(publisher) };
if (!VendorAgrees(candidate.Owner.VendorKeys, programVendors)) continue;
}
if (best == null || IsMoreSpecific(candidate, best))
best = candidate;
}
}
return best == null ? null : new CpeMatch(best.Owner.Mapping, "medium");
}
internal static string[] Tokenize(string? text)
{
if (string.IsNullOrWhiteSpace(text)) return Array.Empty<string>();
var cleaned = Marks.Replace(text!, " ").ToLowerInvariant();
var matches = Token.Matches(cleaned);
if (matches.Count == 0) return Array.Empty<string>();
var tokens = new string[matches.Count];
for (var i = 0; i < tokens.Length; i++) tokens[i] = matches[i].Value;
return tokens;
}
private static bool IsMoreSpecific(PatternEntry candidate, PatternEntry held)
{
if (candidate.Tokens.Length != held.Tokens.Length) return candidate.Tokens.Length > held.Tokens.Length;
if (candidate.TextLength != held.TextLength) return candidate.TextLength > held.TextLength;
return candidate.Owner.Mapping.Source > held.Owner.Mapping.Source;
}
private static bool MatchesAt(string[] haystack, int start, string[] needle)
{
if (start + needle.Length > haystack.Length) return false;
for (var j = 0; j < needle.Length; j++)
{
if (!string.Equals(haystack[start + j], needle[j], StringComparison.Ordinal)) return false;
}
return true;
}
private static int IndexOf(string[] haystack, string[] needle)
{
for (var i = 0; i + needle.Length <= haystack.Length; i++)
{
if (MatchesAt(haystack, i, needle)) return i;
}
return -1;
}
private static bool SameTokens(string[] a, string[] b) => a.Length == b.Length && MatchesAt(a, 0, b);
internal static string[] VendorTokens(string? vendor) =>
Tokenize(vendor).Where(t => !LegalForms.Contains(t)).ToArray();
private static List<string[]> VendorKeysOf(CpeMapping mapping)
{
var keys = new List<string[]> { VendorTokens(mapping.Vendor) };
if (CpeName.TryParse(mapping.CpeUri, out var cpe))
keys.Add(VendorTokens(cpe.Vendor.Replace('_', ' ')));
keys.AddRange(mapping.PublisherPatterns.Select(VendorTokens));
return keys.Where(k => k.Length > 0).ToList();
}
/// <summary>
/// True when a vendor of the program and a vendor of the mapping are the same up to a legal
/// form or a trailing word ("google" and "google llc", "mozilla" and "mozilla corporation").
/// </summary>
internal static bool VendorAgrees(List<string[]> mappingVendors, string[][] programVendors)
{
foreach (var program in programVendors)
{
if (program.Length == 0) continue;
foreach (var mapping in mappingVendors)
{
if (StartsWith(program, mapping) || StartsWith(mapping, program)) return true;
if (string.Equals(string.Concat(program), string.Concat(mapping), StringComparison.Ordinal)) return true;
}
}
return false;
}
private static bool StartsWith(string[] tokens, string[] prefix) =>
prefix.Length <= tokens.Length && MatchesAt(tokens, 0, prefix);
}
}
@@ -0,0 +1,304 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Services.Caching;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>Per-product settings kept next to the mappings, keyed by <c>vendor:product</c>.</summary>
internal sealed class CpeProductFlags
{
public string Product { get; init; } = string.Empty;
/// <summary>Default shipped with the seed list; null when the seed does not name the product.</summary>
public bool? SeedSelfUpdating { get; init; }
/// <summary>An operator's deviation from the default; null when there is none.</summary>
public bool? AdminSelfUpdating { get; init; }
public string UpdatedBy { get; init; } = string.Empty;
public string UpdatedAt { get; init; } = string.Empty;
public bool SelfUpdating => AdminSelfUpdating ?? SeedSelfUpdating ?? false;
}
/// <summary>
/// One consistent copy of everything the matching needs: mappings, ignore list and product
/// flags, read together and never changed afterwards.
/// </summary>
internal sealed class CpeMappingSnapshot
{
private readonly HashSet<string> _ignoredKeys;
private readonly Dictionary<string, CpeProductFlags> _flags;
public CpeMappingSnapshot(
IReadOnlyList<CpeMapping> mappings,
IEnumerable<string> ignoredKeys,
IEnumerable<CpeProductFlags> flags,
DateTime loadedAtUtc)
{
Mappings = mappings;
Matcher = new CpeMappingMatcher(mappings);
_ignoredKeys = new HashSet<string>(ignoredKeys, StringComparer.Ordinal);
_flags = new Dictionary<string, CpeProductFlags>(StringComparer.Ordinal);
foreach (var flag in flags)
{
if (flag.Product.Length > 0) _flags[flag.Product] = flag;
}
LoadedAtUtc = loadedAtUtc;
}
public IReadOnlyList<CpeMapping> Mappings { get; }
public CpeMappingMatcher Matcher { get; }
public DateTime LoadedAtUtc { get; }
public int IgnoredCount => _ignoredKeys.Count;
public IReadOnlyCollection<CpeProductFlags> ProductFlags => _flags.Values;
/// <summary>
/// True when an operator put this program on the ignore list. The stored vendor is compared
/// with both vendor forms a program carries, because older entries hold the raw publisher.
/// </summary>
public bool IsIgnored(string? normalizedVendor, string? publisher, string? name)
{
if (_ignoredKeys.Count == 0) return false;
var key = CpeMappingMatcher.SoftwareKey(normalizedVendor, name);
if (key.Length == 0) return false;
return _ignoredKeys.Contains(key) || _ignoredKeys.Contains(CpeMappingMatcher.SoftwareKey(publisher, name));
}
public CpeProductFlags? FlagsOf(string? productKey) =>
!string.IsNullOrEmpty(productKey) && _flags.TryGetValue(productKey!, out var flags) ? flags : null;
public bool IsSelfUpdating(string? productKey) => FlagsOf(productKey)?.SelfUpdating ?? false;
/// <summary>Products that currently count as self-updating.</summary>
public HashSet<string> SelfUpdatingProducts() =>
new(_flags.Values.Where(f => f.SelfUpdating).Select(f => f.Product), StringComparer.Ordinal);
}
/// <summary>
/// Thrown when the mappings cannot be read and this instance holds no earlier copy. Callers
/// postpone their work: correlating without mappings would file every program as unknown and
/// take the CPE off its inventory row.
/// </summary>
public sealed class CpeMappingUnavailableException : Exception
{
public CpeMappingUnavailableException(string message, Exception? inner = null) : base(message, inner) { }
}
/// <summary>
/// Reads the CPE mappings and hands out an immutable snapshot. Concurrent first readers share
/// one load; a failed reload keeps the last good snapshot in service.
/// <para>
/// An instance sees another instance's change after <see cref="Ttl"/> at the latest. The
/// writer does not wait for that: it calls <see cref="ReloadAsync"/> and brings the inventory
/// rows in line itself.
/// </para>
/// </summary>
public class CpeMappingStore
{
internal const string SeedPartition = "cpe_map_seed";
internal const string CommunityPartition = "cpe_map_community";
internal const string CustomPartition = "cpe_map_custom";
internal const string AutoPartition = "cpe_map_auto";
internal const string IgnoredPartition = "cpe_ignored";
internal const string ProductFlagsPartition = "cpe_product_flags";
internal static readonly TimeSpan Ttl = TimeSpan.FromMinutes(5);
internal static readonly TimeSpan RetryPause = TimeSpan.FromMinutes(1);
/// <summary>A failing load is retried every minute; it is reported once in this time.</summary>
internal static readonly TimeSpan ReportPause = TimeSpan.FromHours(1);
private const string CacheKey = "snapshot";
private readonly IVulnerabilityRepository _vulnRepo;
private readonly ILogger<CpeMappingStore> _logger;
private readonly SingleFlightCache<CpeMappingSnapshot> _cache = new(maxEntries: 1);
private CpeMappingSnapshot? _lastGood;
private long _retryNotBeforeTicks;
private long _reportNotBeforeTicks;
public CpeMappingStore(IVulnerabilityRepository vulnRepo, ILogger<CpeMappingStore> logger)
{
_vulnRepo = vulnRepo ?? throw new ArgumentNullException(nameof(vulnRepo));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>
/// Raised for a failed load with the error and whether an earlier snapshot is served
/// instead, at most once per <see cref="ReportPause"/>; a successful load ends the pause.
/// Wired to the ops event by the composition root.
/// </summary>
internal Func<Exception, bool, Task>? LoadFailed { get; set; }
/// <exception cref="CpeMappingUnavailableException">No snapshot can be served.</exception>
internal async Task<CpeMappingSnapshot> GetAsync()
{
// While a reload keeps failing, the held copy is served without paying for another
// attempt on every call.
var earlier = Volatile.Read(ref _lastGood);
if (earlier != null && DateTime.UtcNow.Ticks < Interlocked.Read(ref _retryNotBeforeTicks))
return earlier;
try
{
var snapshot = await _cache.GetOrAddAsync(CacheKey, Ttl, LoadAsync).ConfigureAwait(false);
Volatile.Write(ref _lastGood, snapshot);
Interlocked.Exchange(ref _reportNotBeforeTicks, 0);
return snapshot;
}
catch (Exception ex)
{
Interlocked.Exchange(ref _retryNotBeforeTicks, DateTime.UtcNow.Add(RetryPause).Ticks);
var held = Volatile.Read(ref _lastGood);
_logger.LogError(ex,
"CPE mappings could not be loaded; {Outcome}",
held != null ? $"serving the copy read at {held.LoadedAtUtc:o}" : "no earlier copy is held");
if (LoadFailed != null && DateTime.UtcNow.Ticks >= Interlocked.Read(ref _reportNotBeforeTicks))
{
Interlocked.Exchange(ref _reportNotBeforeTicks, DateTime.UtcNow.Add(ReportPause).Ticks);
try { await LoadFailed(ex, held != null).ConfigureAwait(false); }
catch (Exception reportEx) { _logger.LogWarning(reportEx, "Reporting the CPE mapping load failure failed"); }
}
if (held != null) return held;
throw new CpeMappingUnavailableException("CPE mappings could not be loaded and no earlier copy is held.", ex);
}
}
/// <summary>Drops this instance's copy; the next reader loads again. For the path that just wrote.</summary>
internal void Invalidate()
{
_cache.Invalidate(CacheKey);
Interlocked.Exchange(ref _retryNotBeforeTicks, 0);
}
/// <summary><see cref="Invalidate"/>, then the fresh snapshot.</summary>
internal Task<CpeMappingSnapshot> ReloadAsync()
{
Invalidate();
return GetAsync();
}
private async Task<CpeMappingSnapshot> LoadAsync(CancellationToken budget)
{
var mappingRows = _vulnRepo.GetAllCpeMappingsAsync();
var ignoredRows = _vulnRepo.GetCpeMappingsByPartitionAsync(IgnoredPartition);
var flagRows = _vulnRepo.GetCpeMappingsByPartitionAsync(ProductFlagsPartition);
await Task.WhenAll(mappingRows, ignoredRows, flagRows).WaitAsync(budget).ConfigureAwait(false);
var unusable = 0;
var mappings = new List<CpeMapping>();
foreach (var row in await mappingRows.ConfigureAwait(false))
{
var mapping = MappingFromRow(row);
if (mapping == null) unusable++;
else mappings.Add(mapping);
}
// Zero mappings is never a real state once the seed is imported; serving it would
// unmap the whole inventory.
if (mappings.Count == 0)
throw new InvalidOperationException("The mapping table returned no usable CPE mapping.");
if (unusable > 0)
_logger.LogWarning("{Count} CPE mapping rows carry no usable CPE and are skipped", unusable);
mappings.Sort(static (a, b) =>
{
var byPartition = string.CompareOrdinal(a.PartitionKey, b.PartitionKey);
return byPartition != 0 ? byPartition : string.CompareOrdinal(a.RowKey, b.RowKey);
});
var ignoredKeys = (await ignoredRows.ConfigureAwait(false))
.Select(row => CpeMappingMatcher.SoftwareKey(Text(row, "NormalizedVendor"), Text(row, "NormalizedProduct")))
.Where(key => key.Length > 0);
var flags = (await flagRows.ConfigureAwait(false)).Select(FlagsFromRow);
return new CpeMappingSnapshot(mappings, ignoredKeys, flags, DateTime.UtcNow);
}
internal static CpeMapping? MappingFromRow(Dictionary<string, object> row)
{
var partitionKey = Text(row, "PartitionKey");
var cpeUri = CpeName.ToProductUri(CpeUriNormalizer.Normalize(Text(row, "CpeUri")));
var productKey = CpeName.ProductKeyOf(cpeUri);
if (productKey.Length == 0)
{
// Older rows may carry only the two columns.
cpeUri = CpeName.ToProductUri(CpeUriNormalizer.Normalize(
$"cpe:2.3:a:{Text(row, "CpeVendor")}:{Text(row, "CpeProduct")}"));
productKey = CpeName.ProductKeyOf(cpeUri);
if (productKey.Length == 0) return null;
}
var edition = Text(row, "Edition");
return new CpeMapping
{
PartitionKey = partitionKey,
RowKey = Text(row, "RowKey"),
Source = SourceOf(partitionKey, Text(row, "Source")),
Vendor = Text(row, "NormalizedVendor"),
Product = Text(row, "NormalizedProduct"),
CpeUri = cpeUri,
ProductKey = productKey,
Category = Text(row, "Category"),
DisplayNamePatterns = StringList(row, "DisplayNamePatternsJson"),
PublisherPatterns = StringList(row, "PublisherPatternsJson"),
ExcludePatterns = StringList(row, "ExcludePatternsJson"),
Edition = edition.Length > 0 ? edition.ToLowerInvariant() : null,
ExcludeEditions = StringList(row, "ExcludeEditionsJson").Select(e => e.ToLowerInvariant()).ToList(),
CreatedAt = Text(row, "CreatedAt") is { Length: > 0 } createdAt ? createdAt : Text(row, "ImportedAt"),
};
}
/// <summary>
/// The custom partition also holds the rows the retired name search wrote on its own;
/// their Source column tells them apart from an operator's entries.
/// </summary>
internal static CpeMappingSource SourceOf(string partitionKey, string sourceColumn) => partitionKey switch
{
SeedPartition => CpeMappingSource.Seed,
CommunityPartition => CpeMappingSource.Community,
AutoPartition => CpeMappingSource.Auto,
_ => string.Equals(sourceColumn, "auto", StringComparison.OrdinalIgnoreCase)
? CpeMappingSource.Auto
: CpeMappingSource.Custom,
};
internal static CpeProductFlags FlagsFromRow(Dictionary<string, object> row) => new()
{
Product = Text(row, "Product").ToLowerInvariant(),
SeedSelfUpdating = row.GetValueOrDefault("SeedSelfUpdating") as bool?,
AdminSelfUpdating = row.GetValueOrDefault("AdminSelfUpdating") as bool?,
UpdatedBy = Text(row, "UpdatedBy"),
UpdatedAt = Text(row, "UpdatedAt"),
};
private static string Text(Dictionary<string, object> row, string column) =>
row.GetValueOrDefault(column)?.ToString() ?? string.Empty;
private static IReadOnlyList<string> StringList(Dictionary<string, object> row, string column)
{
var json = Text(row, column);
if (json.Length == 0) return Array.Empty<string>();
try
{
return JsonSerializer.Deserialize<List<string>>(json)?.Where(s => !string.IsNullOrWhiteSpace(s)).ToList()
?? (IReadOnlyList<string>)Array.Empty<string>();
}
catch (JsonException)
{
return Array.Empty<string>();
}
}
}
}
@@ -1,360 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>
/// Static scoring/matching utilities for NVD CPE auto-resolution.
/// Uses a hybrid approach: exact token overlap + Jaro-Winkler fuzzy matching
/// to handle typos, repackaging errors, and minor naming differences.
/// </summary>
public static class CpeMatchScorer
{
private const double MatchThreshold = 0.6;
private static readonly Regex VersionPattern = new Regex(
@"\b\d+\.\d+[\.\d]*\b", RegexOptions.Compiled);
private static readonly Regex ArchPattern = new Regex(
@"\s*\(?(x64|x86|64-bit|32-bit|arm64|amd64|ia64)\)?\s*",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
/// <summary>
/// Build a clean search keyword from a software display name and vendor.
/// Strips versions, architecture tokens, and normalizes whitespace.
/// </summary>
public static string BuildSearchKeyword(string softwareName, string vendor)
{
var cleaned = softwareName ?? "";
cleaned = VersionPattern.Replace(cleaned, " ");
cleaned = ArchPattern.Replace(cleaned, " ");
cleaned = cleaned.Replace("_", " ");
cleaned = Regex.Replace(cleaned, @"\s+", " ").Trim();
var vendorClean = (vendor ?? "").Replace("_", " ").Trim();
if (!string.IsNullOrWhiteSpace(vendorClean)
&& cleaned.IndexOf(vendorClean, StringComparison.OrdinalIgnoreCase) < 0
&& vendorClean.Length <= 30)
{
cleaned = $"{vendorClean} {cleaned}";
}
if (cleaned.Length > 60)
cleaned = cleaned.Substring(0, 60).TrimEnd();
return cleaned;
}
/// <summary>
/// Score a single NVD CPE entry against the original software metadata.
/// Returns 0.0 – 1.0. Uses hybrid exact + fuzzy matching.
/// </summary>
public static double ScoreCpeMatch(
NvdCpeEntry entry,
string normalizedVendor,
string cleanedProductName)
{
if (entry?.CpeName == null) return 0;
var parts = ParseCpeComponents(entry.CpeName);
if (parts == null) return 0;
var (cpeType, cpeVendor, cpeProduct) = parts.Value;
// --- Type check (0.05) ---
double typeScore = cpeType == "a" ? 1.0 : 0.0;
// --- Vendor match (0.35) — hybrid: token overlap + fuzzy ---
double vendorScore = HybridTokenScore(
NormalizeForComparison(normalizedVendor),
NormalizeForComparison(cpeVendor));
// --- Product match (0.45) — hybrid: token overlap + fuzzy ---
double productScore = HybridTokenScore(
NormalizeForComparison(cleanedProductName),
NormalizeForComparison(cpeProduct));
// --- Title match (0.15) ---
double titleScore = 0;
var title = entry.Titles?.FirstOrDefault(t => t.Lang == "en")?.Title
?? entry.Titles?.FirstOrDefault()?.Title;
if (!string.IsNullOrWhiteSpace(title))
{
var titleNorm = NormalizeForComparison(title);
var productNorm = NormalizeForComparison(cleanedProductName);
if (titleNorm.Contains(productNorm) || productNorm.Contains(titleNorm))
titleScore = 1.0;
else
titleScore = HybridTokenScore(productNorm, titleNorm);
}
return (vendorScore * 0.35)
+ (productScore * 0.45)
+ (titleScore * 0.15)
+ (typeScore * 0.05);
}
/// <summary>
/// Select the best CPE match from NVD results. Returns null if no match meets threshold.
/// </summary>
public static (NvdCpeEntry entry, double score)? SelectBestMatch(
List<NvdCpeProduct> products,
string normalizedVendor,
string cleanedProductName)
{
if (products == null || products.Count == 0) return null;
NvdCpeEntry? bestEntry = null;
double bestScore = 0;
foreach (var product in products)
{
if (product.Cpe == null) continue;
var score = ScoreCpeMatch(product.Cpe, normalizedVendor, cleanedProductName);
if (score > bestScore)
{
bestScore = score;
bestEntry = product.Cpe;
}
}
if (bestEntry == null || bestScore < MatchThreshold) return null;
return (bestEntry, bestScore);
}
/// <summary>
/// Extract a base CPE URI (without version) from a full CPE 2.3 name.
/// e.g. "cpe:2.3:a:google:chrome:120.0:*:*:*:*:*:*:*" -> "cpe:2.3:a:google:chrome"
/// </summary>
public static string ExtractBaseCpeUri(string cpeName)
{
if (string.IsNullOrEmpty(cpeName)) return "";
var parts = cpeName.Split(':');
if (parts.Length < 5) return cpeName;
return $"{parts[0]}:{parts[1]}:{parts[2]}:{parts[3]}:{parts[4]}";
}
/// <summary>
/// Parse type, vendor, product from a CPE 2.3 name.
/// </summary>
public static (string type, string vendor, string product)? ParseCpeComponents(string cpeName)
{
if (string.IsNullOrEmpty(cpeName)) return null;
var parts = cpeName.Split(':');
if (parts.Length < 5) return null;
return (parts[2], parts[3], parts[4]);
}
/// <summary>
/// Product identity of a CPE 2.3 name as "vendor:product" in lower case
/// ("cpe:2.3:a:7-zip:7-zip:24.08" → "7-zip:7-zip"); empty when the name has no concrete
/// vendor and product. CveIndex rows carry it so the exposure summary can fold every title
/// and version of one product together.
/// </summary>
public static string ProductKey(string? cpeName)
{
var parts = ParseCpeComponents(cpeName ?? "");
if (parts == null) return "";
var (_, vendor, product) = parts.Value;
if (!IsConcrete(vendor) || !IsConcrete(product)) return "";
return $"{vendor}:{product}".ToLowerInvariant();
static bool IsConcrete(string value) => value.Length > 0 && value != "*" && value != "-";
}
/// <summary>
/// Build a CpeMappingEntry from an NVD CPE entry for saving as a custom mapping.
/// </summary>
public static CpeMappingEntry BuildMappingFromNvdResult(
NvdCpeEntry cpeEntry,
string normalizedVendor,
string normalizedProduct,
string displayName)
{
var components = ParseCpeComponents(cpeEntry.CpeName ?? "");
var cpeVendor = components?.vendor ?? "";
var cpeProduct = components?.product ?? "";
var baseCpeUri = ExtractBaseCpeUri(cpeEntry.CpeName ?? "");
return new CpeMappingEntry
{
NormalizedVendor = normalizedVendor,
NormalizedProduct = normalizedProduct,
CpeVendor = cpeVendor,
CpeProduct = cpeProduct,
CpeUri = baseCpeUri,
Category = "auto",
DisplayNamePatterns = new List<string> { displayName },
PublisherPatterns = string.IsNullOrWhiteSpace(normalizedVendor)
? new List<string>()
: new List<string> { normalizedVendor },
ExcludePatterns = new List<string>(),
MatchConfidence = "auto"
};
}
// -----------------------------------------------------------------------
// Hybrid scoring: combines exact token overlap with fuzzy matching
// -----------------------------------------------------------------------
/// <summary>
/// Hybrid score: max of (exact token overlap, fuzzy token match, whole-string Jaro-Winkler).
/// This catches typos ("Microsft" vs "Microsoft"), repackaging errors, and minor variations.
/// </summary>
internal static double HybridTokenScore(string a, string b)
{
if (string.IsNullOrWhiteSpace(a) || string.IsNullOrWhiteSpace(b)) return 0;
// Score 1: exact token overlap (fast, handles multi-word names)
double exactScore = ExactTokenOverlapScore(a, b);
// Score 2: fuzzy token matching (handles typos within individual tokens)
double fuzzyTokenScore = FuzzyTokenOverlapScore(a, b);
// Score 3: whole-string Jaro-Winkler (handles single-word names with typos)
double jwScore = JaroWinkler(a, b);
return Math.Max(exactScore, Math.Max(fuzzyTokenScore, jwScore));
}
/// <summary>
/// Exact token overlap: count of shared tokens / max tokens.
/// e.g. "google chrome" vs "chrome" = 1/2 = 0.5
/// </summary>
private static double ExactTokenOverlapScore(string a, string b)
{
var tokensA = a.Split(' ', StringSplitOptions.RemoveEmptyEntries).ToHashSet();
var tokensB = b.Split(' ', StringSplitOptions.RemoveEmptyEntries).ToHashSet();
if (tokensA.Count == 0 || tokensB.Count == 0) return 0;
int overlap = tokensA.Intersect(tokensB).Count();
int maxTokens = Math.Max(tokensA.Count, tokensB.Count);
return (double)overlap / maxTokens;
}
/// <summary>
/// Fuzzy token matching: for each token in A, find the best Jaro-Winkler match in B.
/// Average the best matches. Catches typos like "Microsft" → "Microsoft".
/// </summary>
private static double FuzzyTokenOverlapScore(string a, string b)
{
var tokensA = a.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var tokensB = b.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if (tokensA.Length == 0 || tokensB.Length == 0) return 0;
// For each token in the shorter set, find best fuzzy match in the longer set
var shorter = tokensA.Length <= tokensB.Length ? tokensA : tokensB;
var longer = tokensA.Length <= tokensB.Length ? tokensB : tokensA;
double totalScore = 0;
foreach (var token in shorter)
{
double bestMatch = 0;
foreach (var candidate in longer)
{
var jw = JaroWinkler(token, candidate);
if (jw > bestMatch) bestMatch = jw;
}
// Only count as a match if fuzzy score is above 0.85 (high confidence)
totalScore += bestMatch >= 0.85 ? bestMatch : 0;
}
int maxTokens = Math.Max(tokensA.Length, tokensB.Length);
return totalScore / maxTokens;
}
// -----------------------------------------------------------------------
// Jaro-Winkler distance — standard string similarity for name matching.
// Preferred over Levenshtein for names because:
// - Favors strings that match from the beginning (common in product names)
// - Handles transpositions naturally (Buchstabendreher)
// - Returns 0-1 directly (no normalization needed)
// -----------------------------------------------------------------------
/// <summary>
/// Jaro-Winkler similarity between two strings. Returns 0.0 (no match) – 1.0 (identical).
/// Handles transpositions (e.g. "Chorme" vs "Chrome") and prefix matches well.
/// </summary>
internal static double JaroWinkler(string s1, string s2)
{
if (string.IsNullOrEmpty(s1) || string.IsNullOrEmpty(s2)) return 0;
if (string.Equals(s1, s2, StringComparison.Ordinal)) return 1.0;
double jaro = JaroSimilarity(s1, s2);
// Winkler modification: boost score for common prefix (up to 4 chars)
int prefixLength = 0;
int maxPrefix = Math.Min(4, Math.Min(s1.Length, s2.Length));
for (int i = 0; i < maxPrefix; i++)
{
if (s1[i] == s2[i]) prefixLength++;
else break;
}
// Standard Winkler scaling factor p = 0.1
return jaro + (prefixLength * 0.1 * (1.0 - jaro));
}
private static double JaroSimilarity(string s1, string s2)
{
int len1 = s1.Length;
int len2 = s2.Length;
int matchWindow = Math.Max(len1, len2) / 2 - 1;
if (matchWindow < 0) matchWindow = 0;
var matched1 = new bool[len1];
var matched2 = new bool[len2];
int matches = 0;
int transpositions = 0;
// Find matching characters
for (int i = 0; i < len1; i++)
{
int start = Math.Max(0, i - matchWindow);
int end = Math.Min(i + matchWindow + 1, len2);
for (int j = start; j < end; j++)
{
if (matched2[j] || s1[i] != s2[j]) continue;
matched1[i] = true;
matched2[j] = true;
matches++;
break;
}
}
if (matches == 0) return 0;
// Count transpositions
int k = 0;
for (int i = 0; i < len1; i++)
{
if (!matched1[i]) continue;
while (!matched2[k]) k++;
if (s1[i] != s2[k]) transpositions++;
k++;
}
double m = matches;
return (m / len1 + m / len2 + (m - transpositions / 2.0) / m) / 3.0;
}
private static string NormalizeForComparison(string input)
{
if (string.IsNullOrWhiteSpace(input)) return "";
return input
.ToLowerInvariant()
.Replace("_", " ")
.Replace("-", " ")
.Replace(".", " ")
.Trim();
}
}
}
@@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>
/// One parsed CPE 2.3 formatted string
/// (<c>cpe:2.3:part:vendor:product:version:update:edition:language:sw_edition:target_sw:target_hw:other</c>).
/// The only place that knows the field layout: a field may carry a backslash-escaped colon,
/// and a prefix compare on the raw string treats <c>acrobat</c> as <c>acrobat_reader_dc</c>,
/// so nothing else splits or prefix-matches these strings.
/// </summary>
internal readonly struct CpeName
{
private const string Any = "*";
private CpeName(string part, string vendor, string product, string version, string swEdition, string targetSw, string targetHw)
{
Part = part;
Vendor = vendor;
Product = product;
Version = version;
SwEdition = swEdition;
TargetSw = targetSw;
TargetHw = targetHw;
}
public string Part { get; }
public string Vendor { get; }
public string Product { get; }
/// <summary>Version field; <c>*</c> when the string ends before it.</summary>
public string Version { get; }
/// <summary>Software edition (e.g. <c>esr</c>, <c>continuous</c>); <c>*</c> when absent.</summary>
public string SwEdition { get; }
public string TargetSw { get; }
public string TargetHw { get; }
/// <summary>Lower-case <c>vendor:product</c>; empty when either is missing or a wildcard.</summary>
public string ProductKey =>
IsConcrete(Vendor) && IsConcrete(Product) ? $"{Vendor}:{Product}".ToLowerInvariant() : string.Empty;
/// <summary>The version-less form mappings and cache rows are keyed by.</summary>
public string ProductUri => $"cpe:2.3:{Part}:{Vendor}:{Product}".ToLowerInvariant();
public bool IsSameProduct(in CpeName other) =>
ProductKey.Length > 0 && string.Equals(ProductKey, other.ProductKey, StringComparison.Ordinal);
public static bool TryParse(string? value, out CpeName name)
{
name = default;
if (string.IsNullOrWhiteSpace(value)) return false;
var fields = SplitFields(value!.Trim());
if (fields.Count < 5 || !string.Equals(fields[0], "cpe", StringComparison.OrdinalIgnoreCase)) return false;
name = new CpeName(
fields[2], fields[3], fields[4],
Field(fields, 5), Field(fields, 9), Field(fields, 10), Field(fields, 11));
return true;
}
/// <summary>Lower-case <c>vendor:product</c> of any CPE string; empty when it has none.</summary>
public static string ProductKeyOf(string? value) =>
TryParse(value, out var name) ? name.ProductKey : string.Empty;
/// <summary>
/// The version-less form of any CPE string. A value that does not parse is returned
/// trimmed, so a hand-typed mapping stays visible instead of silently becoming empty.
/// </summary>
public static string ToProductUri(string? value)
{
if (TryParse(value, out var name) && name.ProductKey.Length > 0) return name.ProductUri;
return value?.Trim() ?? string.Empty;
}
private static bool IsConcrete(string value) => value.Length > 0 && value != Any && value != "-";
private static string Field(List<string> fields, int index) =>
index < fields.Count && fields[index].Length > 0 ? fields[index] : Any;
private static List<string> SplitFields(string value)
{
var fields = new List<string>(13);
var current = new StringBuilder();
for (var i = 0; i < value.Length; i++)
{
var c = value[i];
if (c == '\\' && i + 1 < value.Length)
{
current.Append(c).Append(value[++i]);
}
else if (c == ':')
{
fields.Add(current.ToString());
current.Clear();
}
else
{
current.Append(c);
}
}
fields.Add(current.ToString());
return fields;
}
}
}
@@ -0,0 +1,123 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Shared.DataAccess;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>
/// Writes the per-product flags. A product's row holds the default shipped with the seed and,
/// only while it differs from that default, an operator's own value. The operator's value is
/// dropped as soon as it equals the default again, so a later change of the default reaches
/// every product nobody deliberately set otherwise.
/// </summary>
public class CpeProductFlagsService
{
private readonly IVulnerabilityRepository _vulnRepo;
private readonly CpeMappingStore _mappingStore;
public CpeProductFlagsService(IVulnerabilityRepository vulnRepo, CpeMappingStore mappingStore)
{
_vulnRepo = vulnRepo;
_mappingStore = mappingStore;
}
/// <summary>
/// Makes <paramref name="selfUpdatingProducts"/> the default set: the listed products get
/// the default, every other product loses it. Returns the number of rows written or removed.
/// </summary>
internal async Task<int> ApplySeedDefaultsAsync(IEnumerable<string> selfUpdatingProducts)
{
var defaults = new HashSet<string>(
selfUpdatingProducts.Select(NormalizeKey).Where(p => p.Length > 0), StringComparer.Ordinal);
var held = await ReadAsync().ConfigureAwait(false);
var changed = 0;
foreach (var product in defaults.Union(held.Keys).ToList())
{
held.TryGetValue(product, out var current);
bool? seed = defaults.Contains(product) ? true : null;
var admin = current?.AdminSelfUpdating;
if (admin.HasValue && admin.Value == (seed ?? false)) admin = null;
if (current != null && current.SeedSelfUpdating == seed && current.AdminSelfUpdating == admin) continue;
await WriteAsync(product, seed, admin, current?.UpdatedBy ?? string.Empty, current?.UpdatedAt ?? string.Empty).ConfigureAwait(false);
changed++;
}
if (changed > 0) _mappingStore.Invalidate();
return changed;
}
/// <summary>
/// Sets whether a product counts as self-updating and returns its flags afterwards.
/// </summary>
/// <exception cref="ArgumentException">No mapping points to the product.</exception>
internal async Task<CpeProductFlags> SetSelfUpdatingAsync(string productKey, bool selfUpdating, string updatedBy)
{
var product = NormalizeKey(productKey);
var snapshot = await _mappingStore.GetAsync().ConfigureAwait(false);
// This instance's copy can be minutes old; a mapping saved elsewhere a moment ago is
// read before the product is refused.
if (product.Length > 0 && !PointsTo(snapshot, product))
snapshot = await _mappingStore.ReloadAsync().ConfigureAwait(false);
if (product.Length == 0 || !PointsTo(snapshot, product))
throw new ArgumentException($"No mapping points to the product '{productKey}'.", nameof(productKey));
var held = await ReadAsync().ConfigureAwait(false);
held.TryGetValue(product, out var current);
var seed = current?.SeedSelfUpdating;
bool? admin = selfUpdating == (seed ?? false) ? null : selfUpdating;
var updatedAt = DateTime.UtcNow.ToString("o");
await WriteAsync(product, seed, admin, updatedBy, updatedAt).ConfigureAwait(false);
_mappingStore.Invalidate();
return new CpeProductFlags
{
Product = product,
SeedSelfUpdating = seed,
AdminSelfUpdating = admin,
UpdatedBy = admin.HasValue ? updatedBy : string.Empty,
UpdatedAt = admin.HasValue ? updatedAt : string.Empty,
};
}
private async Task<Dictionary<string, CpeProductFlags>> ReadAsync()
{
var rows = await _vulnRepo.GetCpeMappingsByPartitionAsync(CpeMappingStore.ProductFlagsPartition).ConfigureAwait(false);
var flags = new Dictionary<string, CpeProductFlags>(StringComparer.Ordinal);
foreach (var row in rows)
{
var entry = CpeMappingStore.FlagsFromRow(row);
if (entry.Product.Length > 0) flags[entry.Product] = entry;
}
return flags;
}
private Task WriteAsync(string product, bool? seed, bool? admin, string updatedBy, string updatedAt)
{
var rowKey = TableKeySanitizer.Sanitize(product);
if (seed == null && admin == null)
return _vulnRepo.DeleteCpeMappingAsync(CpeMappingStore.ProductFlagsPartition, rowKey);
// Replace mode: a column that is not passed is removed, which is how a value is unset.
var properties = new Dictionary<string, object> { { "Product", product } };
if (seed.HasValue) properties["SeedSelfUpdating"] = seed.Value;
if (admin.HasValue)
{
properties["AdminSelfUpdating"] = admin.Value;
properties["UpdatedBy"] = updatedBy;
properties["UpdatedAt"] = updatedAt;
}
return _vulnRepo.UpsertCpeMappingAsync(CpeMappingStore.ProductFlagsPartition, rowKey, properties);
}
private static bool PointsTo(CpeMappingSnapshot snapshot, string product) =>
snapshot.Mappings.Any(m => string.Equals(m.ProductKey, product, StringComparison.Ordinal));
private static string NormalizeKey(string? productKey) => (productKey ?? string.Empty).Trim().ToLowerInvariant();
}
}
@@ -0,0 +1,183 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using AutopilotMonitor.Functions.Helpers;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>One mapping as written in a mapping list file.</summary>
public sealed class CpeSeedEntry
{
public string? NormalizedVendor { get; set; }
public string? NormalizedProduct { get; set; }
public List<string>? DisplayNamePatterns { get; set; }
public List<string>? PublisherPatterns { get; set; }
public List<string>? ExcludePatterns { get; set; }
public string? CpeUri { get; set; }
public string? Category { get; set; }
public string? Edition { get; set; }
public List<string>? ExcludeEditions { get; set; }
}
/// <summary>A mapping list file: the maintained seed or the community list.</summary>
public sealed class CpeSeedDocument
{
public string? Version { get; set; }
public string? LastUpdated { get; set; }
/// <summary>Products (<c>vendor:product</c>) that update themselves by default.</summary>
public List<string>? SelfUpdatingProducts { get; set; }
public List<CpeSeedEntry>? Mappings { get; set; }
}
/// <summary>
/// Reads and checks a mapping list file. The same check runs before an import and as a unit
/// test over the committed file, so a list that would be refused in production fails the build
/// first. An import that fails the check writes nothing.
/// </summary>
internal static class CpeSeedFile
{
/// <summary>
/// A list that lost more than this share of the mappings in service is refused: a
/// truncated download or a merge accident, not an edit.
/// </summary>
internal const double MaxShrink = 0.5;
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
};
private static readonly Regex EditionForm = new(@"^[a-z0-9_\-\.]+$", RegexOptions.CultureInvariant);
/// <summary>
/// Parses and checks a list. <paramref name="mappingsInService"/> is the number of mappings
/// the list would replace; 0 when there are none yet. Returns the document and the problems
/// found; the list may be imported only when there are none.
/// <paramref name="mayBeEmpty"/> is for the community list, which holds nothing until
/// someone contributes; an empty list is still refused when it would remove mappings.
/// </summary>
public static (CpeSeedDocument? Document, List<string> Problems) Read(string? json, int mappingsInService, bool mayBeEmpty = false)
{
var problems = new List<string>();
CpeSeedDocument? document = null;
try
{
document = string.IsNullOrWhiteSpace(json) ? null : JsonSerializer.Deserialize<CpeSeedDocument>(json!, JsonOptions);
}
catch (JsonException ex)
{
problems.Add($"not valid JSON: {ex.Message}");
return (null, problems);
}
var mappings = document?.Mappings;
if (document == null || mappings == null || mappings.Count == 0)
{
if (document == null || !mayBeEmpty || mappingsInService > 0)
problems.Add("the list holds no mapping");
return (document, problems);
}
if (mappingsInService > 0 && mappings.Count < mappingsInService * (1 - MaxShrink))
problems.Add($"the list holds {mappings.Count} mappings, the one in service {mappingsInService}; a list that shrinks this much is refused");
var keys = new HashSet<string>(StringComparer.Ordinal);
var rowKeys = new HashSet<string>(StringComparer.Ordinal);
var products = new HashSet<string>(StringComparer.Ordinal);
for (var i = 0; i < mappings.Count; i++)
{
var entry = mappings[i];
var label = $"mapping {i + 1} ({entry?.NormalizedVendor}:{entry?.NormalizedProduct})";
if (entry == null)
{
problems.Add($"mapping {i + 1} is empty");
continue;
}
var productUri = CpeName.ToProductUri(CpeUriNormalizer.Normalize(entry.CpeUri));
var productKey = CpeName.ProductKeyOf(productUri);
if (productKey.Length == 0)
problems.Add($"{label}: '{entry.CpeUri}' names no CPE product");
else if (!string.Equals(productUri, CpeUriNormalizer.Normalize(entry.CpeUri), StringComparison.Ordinal))
problems.Add($"{label}: write the CPE as '{productUri}' (vendor and product only, lower case)");
else
products.Add(productKey);
var key = CpeMappingMatcher.SoftwareKey(entry.NormalizedVendor, entry.NormalizedProduct);
if (key.Length == 0)
problems.Add($"{label}: normalizedProduct is missing");
else if (!keys.Add(key))
problems.Add($"{label}: vendor and product name occur twice");
else if (!rowKeys.Add(RowKeyOf(entry.NormalizedVendor, entry.NormalizedProduct)))
problems.Add($"{label}: would be stored in the row of another mapping; the two differ only in characters a row key cannot hold");
foreach (var pattern in (entry.DisplayNamePatterns ?? new List<string>()).Concat(entry.ExcludePatterns ?? new List<string>()))
{
if (CpeMappingMatcher.Tokenize(pattern).Length == 0)
problems.Add($"{label}: the pattern '{pattern}' holds no word");
}
foreach (var edition in (entry.ExcludeEditions ?? new List<string>()).Append(entry.Edition).Where(e => e != null))
{
if (!EditionForm.IsMatch(edition!))
problems.Add($"{label}: '{edition}' is not a CPE software edition (lower case, no spaces)");
}
}
foreach (var product in document.SelfUpdatingProducts ?? new List<string>())
{
if (!products.Contains((product ?? string.Empty).Trim().ToLowerInvariant()))
problems.Add($"selfUpdatingProducts names '{product}', which no mapping in the list points to");
}
return (document, problems);
}
/// <summary>The table rows of a checked list, keyed like the rows written before.</summary>
public static List<(string RowKey, Dictionary<string, object> Properties)> ToRows(CpeSeedDocument document, string source)
{
var importedAt = DateTime.UtcNow.ToString("o");
var rows = new List<(string, Dictionary<string, object>)>();
foreach (var entry in document.Mappings ?? new List<CpeSeedEntry>())
{
var cpeUri = CpeName.ToProductUri(CpeUriNormalizer.Normalize(entry.CpeUri));
CpeName.TryParse(cpeUri, out var cpe);
var properties = new Dictionary<string, object>
{
{ "NormalizedVendor", entry.NormalizedVendor ?? string.Empty },
{ "NormalizedProduct", entry.NormalizedProduct ?? string.Empty },
{ "CpeVendor", cpe.Vendor ?? string.Empty },
{ "CpeProduct", cpe.Product ?? string.Empty },
{ "CpeUri", cpeUri },
{ "Category", entry.Category ?? string.Empty },
{ "DisplayNamePatternsJson", JsonSerializer.Serialize(entry.DisplayNamePatterns ?? new List<string>()) },
{ "PublisherPatternsJson", JsonSerializer.Serialize(entry.PublisherPatterns ?? new List<string>()) },
{ "ExcludePatternsJson", JsonSerializer.Serialize(entry.ExcludePatterns ?? new List<string>()) },
{ "Source", source },
{ "ImportedAt", importedAt },
};
if (!string.IsNullOrWhiteSpace(entry.Edition))
properties["Edition"] = entry.Edition!.Trim().ToLowerInvariant();
if (entry.ExcludeEditions is { Count: > 0 })
properties["ExcludeEditionsJson"] = JsonSerializer.Serialize(entry.ExcludeEditions.Select(e => e.Trim().ToLowerInvariant()));
rows.Add((RowKeyOf(entry.NormalizedVendor, entry.NormalizedProduct), properties));
}
return rows;
}
internal static string RowKeyOf(string? vendor, string? product)
{
var rowKey = TableKeySanitizer.Sanitize(
$"{(string.IsNullOrEmpty(vendor) ? "unknown" : vendor)}:{(string.IsNullOrEmpty(product) ? "unknown" : product)}".ToLowerInvariant());
return rowKey.Length > 512 ? rowKey.Substring(0, 512) : rowKey;
}
}
}
@@ -2,7 +2,6 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using static AutopilotMonitor.Functions.Services.Vulnerability.VulnerabilityCorrelationService;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
@@ -16,7 +15,9 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
/// 1. Date filter — keep CVEs published in the last <see cref="RecentMonths"/>,
/// or HIGH+ (CVSS ≥ <see cref="HighSevFloor"/>) in the last <see cref="HighSevMonths"/>.
/// Unparseable PublishedDate is kept conservatively.
/// 2. Sort by CvssScore desc, PublishedDate desc; cap at <see cref="MaxCachedCves"/>.
/// 2. Sort: CVEs CISA lists as exploited first, then CvssScore desc, PublishedDate desc;
/// cap at <see cref="MaxCachedCves"/>. Without the first key a product with more than
/// that many high scores in the window would lose every exploited CVE to the cap.
/// 3. Per-CVE: cap <see cref="CachedCve.AffectedVersions"/> at <see cref="MaxRangesPerCve"/>
/// (some single CVEs carry hundreds of CPE matches — Windows components,
/// Chromium-style mass releases).
@@ -67,7 +68,8 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
.ToList();
var capped = dateFiltered
.OrderByDescending(c => c.CvssScore)
.OrderByDescending(c => c.KevAdded != null)
.ThenByDescending(c => c.CvssScore)
.ThenByDescending(c => ParsePublished(c.PublishedDate) ?? DateTime.MinValue)
.Take(MaxCachedCves)
.ToList();
@@ -0,0 +1,383 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using AutopilotMonitor.Functions.DataAccess.TableStorage;
using AutopilotMonitor.Functions.Helpers;
using AutopilotMonitor.Functions.Telemetry;
using AutopilotMonitor.Shared.DataAccess;
using Microsoft.ApplicationInsights;
using Microsoft.Extensions.Logging;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
/// <summary>One CVE as kept in a product's cache row.</summary>
internal class CachedCve
{
public string CveId { get; set; } = "";
public double CvssScore { get; set; }
public string CvssSeverity { get; set; } = "UNKNOWN";
/// <summary>CVSS v3.x vector string (e.g. <c>CVSS:3.1/AV:N/AC:L/…</c>); null on legacy rows.</summary>
public string? CvssVector { get; set; }
public string Description { get; set; } = "";
public string PublishedDate { get; set; } = "";
/// <summary>FIRST EPSS probability (0..1); null = not scored / not yet fetched.</summary>
public double? EpssScore { get; set; }
public double? EpssPercentile { get; set; }
/// <summary>EPSS scoring date (yyyy-MM-dd) the score belongs to.</summary>
public string? EpssDate { get; set; }
/// <summary>
/// Date NVD says CISA listed the CVE as exploited; null when it is not listed. Only decides
/// which CVEs get a place in the row. The verdict shown to users comes from the KEV catalog.
/// </summary>
public string? KevAdded { get; set; }
public List<CachedVersionRange> AffectedVersions { get; set; } = new();
}
internal class CachedVersionRange
{
public string? Criteria { get; set; }
public string? VersionStartIncluding { get; set; }
public string? VersionStartExcluding { get; set; }
public string? VersionEndIncluding { get; set; }
public string? VersionEndExcluding { get; set; }
}
/// <summary>What the cache holds for one product.</summary>
internal sealed class CveCacheEntry
{
public List<CachedCve> Cves { get; init; } = new();
public DateTime CachedAt { get; init; }
/// <summary>How many CVEs NVD lists for the product; null on rows written before it was recorded.</summary>
public int? NvdTotal { get; init; }
/// <summary>
/// False for a row written before only the product's own ranges were kept: it can hold CVEs
/// that merely name the product next to another one.
/// </summary>
public bool CurrentShape { get; init; } = true;
}
internal enum CveCacheFillOutcome
{
Filled,
/// <summary>NVD refused with 429 or its cooldown is running.</summary>
Throttled,
/// <summary>NVD gave no answer: no connection, a timeout, a server error. The next request is likely to fare the same.</summary>
Unanswered,
/// <summary>NVD answered and turned this request down; it may well serve the next product.</summary>
Failed,
}
internal sealed class CveCacheFillResult
{
public CveCacheFillOutcome Outcome { get; init; }
public CveCacheEntry? Entry { get; init; }
public int NvdTotal { get; init; }
public int InWindow { get; init; }
public int Kept { get; init; }
}
/// <summary>
/// The one path that turns an NVD answer into a product's cache row: fetch, keep the product's
/// own version ranges, apply the size budget, add EPSS, write. The scan and the nightly refresh
/// both go through it, so they cannot drift apart.
/// </summary>
public class CveCacheFiller
{
internal const string RowKey = "cache_result";
/// <summary>
/// Shape of the row. A row without this value was written before the fetch asked for the
/// newest CVEs and before foreign products' ranges were dropped; it stays in service until
/// the refresh rewrites it.
/// </summary>
internal const int SchemaVersion = 2;
internal const string StatePending = "pending";
private const string StateFilled = "filled";
internal static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private readonly IVulnerabilityRepository _vulnRepo;
private readonly NvdApiClient _nvdApi;
private readonly EpssApiClient? _epssApi;
private readonly TelemetryClient? _telemetryClient;
private readonly ILogger<CveCacheFiller> _logger;
public CveCacheFiller(
IVulnerabilityRepository vulnRepo,
NvdApiClient nvdApi,
ILogger<CveCacheFiller> logger,
EpssApiClient? epssApi = null,
TelemetryClient? telemetryClient = null)
{
_vulnRepo = vulnRepo;
_nvdApi = nvdApi;
_logger = logger;
_epssApi = epssApi;
_telemetryClient = telemetryClient;
}
internal static string CacheKey(string cpeUri) => TableKeySanitizer.Sanitize($"cve_{cpeUri}");
/// <summary>
/// The cached CVEs of a product; null when there is no row, the row only says that data is
/// wanted, or it cannot be read.
/// </summary>
internal async Task<CveCacheEntry?> ReadAsync(string cpeUri)
{
var cacheKey = CacheKey(cpeUri);
try
{
var row = await _vulnRepo.GetCveCacheEntryAsync(cacheKey, RowKey).ConfigureAwait(false);
return row == null ? null : EntryFromRow(row, cacheKey);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "CVE cache row {CacheKey} could not be read", cacheKey);
return null;
}
}
internal CveCacheEntry? EntryFromRow(Dictionary<string, object> row, string cacheKey)
{
if (IsPending(row)) return null;
var cachedAtText = row.GetValueOrDefault("CachedAt")?.ToString();
if (!DateTime.TryParse(cachedAtText, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var cachedAt))
return null;
var json = TableStorageChunking.ReassembleProperty(row, "CveDataJson", _logger, $"{cacheKey}/{RowKey}") ?? "[]";
var cves = JsonSerializer.Deserialize<List<CachedCve>>(json, JsonOptions) ?? new List<CachedCve>();
return new CveCacheEntry
{
Cves = cves,
CachedAt = cachedAt,
NvdTotal = int.TryParse(row.GetValueOrDefault("NvdTotal")?.ToString(), out var nvdTotal) ? nvdTotal : null,
CurrentShape = int.TryParse(row.GetValueOrDefault("SchemaVersion")?.ToString(), out var shape) && shape == SchemaVersion,
};
}
internal static bool IsPending(Dictionary<string, object> row) =>
string.Equals(row.GetValueOrDefault("State")?.ToString(), StatePending, StringComparison.Ordinal);
/// <summary>
/// Notes that a product needs CVE data. Only ever creates the row: an existing one, filled
/// or pending, is left alone. The nightly refresh fills what is noted here.
/// </summary>
internal async Task MarkPendingAsync(string cpeUri)
{
try
{
await _vulnRepo.TryAddCveCacheEntryAsync(CacheKey(cpeUri), RowKey, new Dictionary<string, object>
{
{ "CpeUri", cpeUri },
{ "State", StatePending },
{ "RequestedAt", DateTime.UtcNow.ToString("o") },
{ "SchemaVersion", SchemaVersion },
}).ConfigureAwait(false);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "CVE cache marker for {CpeUri} could not be written", cpeUri);
}
}
/// <summary>
/// Fetches a product's CVEs and writes its cache row. A refusal or failure writes nothing:
/// the cache never learns "no CVEs" from an answer NVD did not give.
/// </summary>
internal async Task<CveCacheFillResult> FillAsync(string cpeUri, CancellationToken ct)
{
var startedAt = DateTime.UtcNow;
var fetch = await _nvdApi.SearchCvesByCpeAsync(cpeUri, ct).ConfigureAwait(false);
if (!fetch.Succeeded)
{
var outcome = fetch.Throttled ? CveCacheFillOutcome.Throttled
: fetch.Unanswered ? CveCacheFillOutcome.Unanswered
: CveCacheFillOutcome.Failed;
Track(cpeUri, outcome.ToString().ToLowerInvariant(), startedAt, 0, 0, 0, 0);
return new CveCacheFillResult { Outcome = outcome };
}
var projected = Project(fetch.Data, cpeUri);
var budget = CveCacheBudget.Apply(projected, DateTime.UtcNow);
var cves = budget.Cves;
// One EPSS batch per product. Best effort: the nightly run scores every row again.
if (_epssApi != null && cves.Count > 0)
{
try
{
var scores = await _epssApi.GetScoresAsync(cves.Select(c => c.CveId), ct).ConfigureAwait(false);
ApplyEpssScores(cves, scores);
}
catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; }
catch (Exception ex)
{
_logger.LogWarning(ex, "EPSS enrichment failed for {CpeUri} (cached without scores)", cpeUri);
}
}
var nvdTotal = fetch.Data.TotalResults;
var entry = new CveCacheEntry { Cves = cves, CachedAt = DateTime.UtcNow, NvdTotal = nvdTotal };
var written = true;
try
{
await _vulnRepo.UpsertCveCacheEntryAsync(
CacheKey(cpeUri), RowKey, BuildRow(cpeUri, cves, entry.CachedAt, nvdTotal)).ConfigureAwait(false);
}
catch (Exception ex)
{
// The caller still gets the data; the next reader fetches again.
written = false;
_logger.LogWarning(ex, "CVE cache row for {CpeUri} could not be written", cpeUri);
}
Track(cpeUri, written ? "filled" : "not_stored", startedAt, nvdTotal, budget.KeptAfterDateFilter, cves.Count,
cves.Count(c => c.KevAdded != null));
return new CveCacheFillResult
{
Outcome = CveCacheFillOutcome.Filled,
Entry = entry,
NvdTotal = nvdTotal,
InWindow = budget.KeptAfterDateFilter,
Kept = cves.Count,
};
}
/// <summary>
/// Projects an NVD 2.0 response onto the cached shape for <paramref name="cpeUri"/>'s
/// product. A CVE that names several products carries a version range per product; only the
/// ranges of this product are kept, and a CVE without one is dropped. Comparing an installed
/// version against another product's range reports CVEs the product never had.
/// CVSS prefers v3.1 and falls back to v3.0.
/// </summary>
internal static List<CachedCve> Project(NvdCveResponse? response, string cpeUri)
{
var cves = new List<CachedCve>();
if (response?.Vulnerabilities == null || !CpeName.TryParse(cpeUri, out var product)) return cves;
foreach (var item in response.Vulnerabilities)
{
var cve = item?.Cve;
if (cve == null) continue;
var ranges = new List<CachedVersionRange>();
foreach (var match in (cve.Configurations ?? new List<NvdConfiguration>())
.SelectMany(c => c.Nodes ?? new List<NvdNode>())
.SelectMany(n => n.CpeMatch ?? new List<NvdCpeMatch>()))
{
if (!match.Vulnerable) continue;
if (!CpeName.TryParse(match.Criteria, out var criteria) || !criteria.IsSameProduct(product)) continue;
ranges.Add(new CachedVersionRange
{
Criteria = match.Criteria,
VersionStartIncluding = match.VersionStartIncluding,
VersionStartExcluding = match.VersionStartExcluding,
VersionEndIncluding = match.VersionEndIncluding,
VersionEndExcluding = match.VersionEndExcluding,
});
}
if (ranges.Count == 0) continue;
double cvssScore = 0;
var cvssSeverity = "UNKNOWN";
string? cvssVector = null;
var metric = cve.Metrics?.CvssMetricV31?.FirstOrDefault() ?? cve.Metrics?.CvssMetricV30?.FirstOrDefault();
if (metric?.CvssData != null)
{
cvssScore = metric.CvssData.BaseScore;
cvssSeverity = metric.CvssData.BaseSeverity ?? "UNKNOWN";
cvssVector = string.IsNullOrWhiteSpace(metric.CvssData.VectorString) ? null : metric.CvssData.VectorString;
}
var description = cve.Descriptions?.FirstOrDefault(d =>
string.Equals(d.Lang, "en", StringComparison.OrdinalIgnoreCase))?.Value ?? "";
if (description.Length > 500)
description = description.Substring(0, 497) + "...";
cves.Add(new CachedCve
{
CveId = cve.Id ?? "",
CvssScore = cvssScore,
CvssSeverity = cvssSeverity,
CvssVector = cvssVector,
Description = description,
PublishedDate = cve.Published ?? "",
KevAdded = string.IsNullOrWhiteSpace(cve.CisaExploitAdd) ? null : cve.CisaExploitAdd,
AffectedVersions = ranges,
});
}
return cves;
}
/// <summary>
/// Copies EPSS scores onto cached CVEs. Returns how many entries changed (new score, or a
/// different score/date than before) so the nightly refresh can skip unchanged rows.
/// A CVE absent from <paramref name="scores"/> keeps whatever it had: a failed batch must
/// not erase yesterday's score.
/// </summary>
internal static int ApplyEpssScores(List<CachedCve> cves, IReadOnlyDictionary<string, EpssScore> scores)
{
var changed = 0;
foreach (var cve in cves)
{
if (!scores.TryGetValue(cve.CveId, out var s)) continue;
if (cve.EpssScore == s.Score && cve.EpssPercentile == s.Percentile && cve.EpssDate == s.Date) continue;
cve.EpssScore = s.Score;
cve.EpssPercentile = s.Percentile;
cve.EpssDate = s.Date;
changed++;
}
return changed;
}
/// <summary>
/// The cache row for one product, CveDataJson chunked at 30K chars. The row is written in
/// replace mode, so every column a rewrite must keep has to be passed in again:
/// <paramref name="cachedAt"/> is the NVD freshness stamp and <paramref name="nvdTotal"/> the
/// size NVD reported, and an EPSS-only rewrite hands both back unchanged, together with the
/// shape version the row already had.
/// </summary>
internal static Dictionary<string, object> BuildRow(
string cpeUri, List<CachedCve> cves, DateTime cachedAt, int nvdTotal, int schemaVersion = SchemaVersion)
{
var row = new Dictionary<string, object>
{
{ "CachedAt", cachedAt.ToString("o") },
{ "CpeUri", cpeUri },
{ "TotalCves", cves.Count },
{ "NvdTotal", nvdTotal },
{ "SchemaVersion", schemaVersion },
{ "State", StateFilled },
};
foreach (var chunk in TableStorageChunking.ChunkProperty("CveDataJson", JsonSerializer.Serialize(cves, JsonOptions)))
row[chunk.Key] = chunk.Value;
return row;
}
private void Track(string cpeUri, string outcome, DateTime startedAt, int nvdTotal, int inWindow, int kept, int kevKept)
=> _telemetryClient.TrackEventUnsampled("CveCacheFilled", new Dictionary<string, string>
{
{ "cpeUri", cpeUri },
{ "outcome", outcome },
{ "nvdTotal", nvdTotal.ToString(CultureInfo.InvariantCulture) },
{ "inWindow", inWindow.ToString(CultureInfo.InvariantCulture) },
{ "kept", kept.ToString(CultureInfo.InvariantCulture) },
{ "kevKept", kevKept.ToString(CultureInfo.InvariantCulture) },
{ "durationMs", (DateTime.UtcNow - startedAt).TotalMilliseconds.ToString("F0", CultureInfo.InvariantCulture) },
});
}
}
@@ -0,0 +1,187 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
namespace AutopilotMonitor.Functions.Services.Vulnerability
{
internal sealed class CveFilterResult
{
/// <summary>The CVEs the installed version is affected by, in the report's per-CVE shape.</summary>
public List<Dictionary<string, object>> Vulnerabilities { get; } = new();
/// <summary>
/// CVEs left unjudged because their ranges count versions in another scheme than the
/// installed one (years against releases).
/// </summary>
public int IncomparableCves { get; set; }
}
/// <summary>
/// Decides which of a product's cached CVEs apply to one installed version.
/// A range counts only when it belongs to the product itself, targets Windows (or no platform),
/// fits the edition the mapping asks for, and counts versions the way the installed one does.
/// </summary>
internal static class CveVersionFilter
{
// CPE target_sw / target_hw values that rule a Windows desktop out. The last four share
// the prefix "windows" with it and are the phone, embedded and ARM-only lines.
private static readonly HashSet<string> NonWindowsPlatforms = new(StringComparer.OrdinalIgnoreCase)
{
"android", "ios", "iphone_os", "macos", "mac_os_x", "linux",
"linux_kernel", "chrome_os", "chromeos", "tvos", "watchos",
"ipados", "freebsd", "openbsd", "netbsd", "solaris", "aix",
"hp-ux", "hpux", "webos",
"windows_phone", "windows_mobile", "windows_ce", "windows_rt",
};
/// <summary>
/// True for a target platform that rules a Windows desktop out. The one list for the scan
/// and for the product dictionary: a product the dictionary keeps out for its platform is
/// not judged by that platform's ranges either.
/// </summary>
internal static bool IsOtherPlatform(string platform) => NonWindowsPlatforms.Contains(platform);
private const RegexOptions WordOptions = RegexOptions.CultureInvariant | RegexOptions.IgnoreCase;
// Whole words only: "ios" is also the end of "scenarios", "unix" the end of a product name.
private static readonly Regex NonWindowsWord = new(
@"(?<![\p{L}\p{N}_])(?:linux|macos|mac os|mac_os|mac_os_x|macintosh|android|ios|iphone|ipad|freebsd|openbsd|netbsd|solaris|aix|hp-ux|hpux|chrome os|chromeos|tvos|watchos|ipados|unix)(?![\p{L}\p{N}_])",
WordOptions, TimeSpan.FromMilliseconds(250));
private static readonly Regex WindowsWord = new(
@"(?<![\p{L}\p{N}_])(?:windows|win32|win64)(?![\p{L}\p{N}_])",
WordOptions, TimeSpan.FromMilliseconds(250));
/// <param name="edition">When set, only ranges of this software edition count.</param>
/// <param name="excludeEditions">Ranges of these software editions never count.</param>
public static CveFilterResult Filter(
IReadOnlyList<CachedCve> cves,
string cpeUri,
string installedVersion,
string? edition = null,
IReadOnlyCollection<string>? excludeEditions = null)
{
var result = new CveFilterResult();
if (!CpeName.TryParse(cpeUri, out var product)) return result;
var hasVersion = !string.IsNullOrWhiteSpace(installedVersion);
foreach (var cve in cves)
{
var affected = false;
var everyVersion = false;
var incomparable = false;
foreach (var range in cve.AffectedVersions)
{
if (!CpeName.TryParse(range.Criteria, out var criteria)) continue;
// Rows written before the projection kept only the product's own ranges still
// carry the ranges of every other product a CVE names.
if (!criteria.IsSameProduct(product)) continue;
if (IsNonWindowsPlatform(criteria)) continue;
if (!EditionCounts(criteria.SwEdition, edition, excludeEditions)) continue;
var bounds = new[]
{
range.VersionStartIncluding, range.VersionStartExcluding,
range.VersionEndIncluding, range.VersionEndExcluding,
};
if (bounds.All(string.IsNullOrWhiteSpace))
{
if (criteria.Version == "*")
{
// No version named at all: reported, but as a weak match.
affected = true;
everyVersion = true;
}
else if (criteria.Version != "-" && hasVersion)
{
// "-" means NVD gave no version; without a range that says nothing.
if (!VersionComparer.AreComparable(installedVersion, criteria.Version)) incomparable = true;
else affected = VersionComparer.Compare(installedVersion, criteria.Version) == 0;
}
}
else if (hasVersion)
{
if (bounds.Any(b => !VersionComparer.AreComparable(installedVersion, b))) incomparable = true;
else affected = VersionComparer.IsVersionAffected(installedVersion, bounds[0], bounds[1], bounds[2], bounds[3]);
}
if (affected) break;
}
if (!affected)
{
if (incomparable) result.IncomparableCves++;
continue;
}
// Second line of defence for CVEs whose ranges name no platform.
if (IsDescriptionNonWindowsOnly(cve.Description)) continue;
var entry = new Dictionary<string, object>
{
{ "cveId", cve.CveId },
{ "cvssScore", cve.CvssScore },
{ "cvssSeverity", cve.CvssSeverity },
{ "description", cve.Description },
{ "publishedDate", cve.PublishedDate },
{ "source", "NVD" },
{ "isKev", false },
{ "kevDueDate", "" },
{ "kevRequiredAction", "" },
{ "knownRansomware", "" },
};
// Optional enrichment: an absent key means "unknown", never a defaulted 0 or empty.
if (!string.IsNullOrEmpty(cve.CvssVector))
entry["cvssVector"] = cve.CvssVector!;
if (cve.EpssScore.HasValue)
{
entry["epssScore"] = cve.EpssScore.Value;
if (cve.EpssPercentile.HasValue) entry["epssPercentile"] = cve.EpssPercentile.Value;
if (!string.IsNullOrEmpty(cve.EpssDate)) entry["epssDate"] = cve.EpssDate!;
}
if (everyVersion)
entry["versionMatchConfidence"] = "low";
result.Vulnerabilities.Add(entry);
}
return result;
}
/// <summary>True when a CPE string targets a platform other than Windows.</summary>
public static bool IsNonWindowsPlatform(string? criteria) =>
CpeName.TryParse(criteria, out var name) && IsNonWindowsPlatform(name);
private static bool IsNonWindowsPlatform(in CpeName criteria) =>
NonWindowsPlatforms.Contains(criteria.TargetSw) || NonWindowsPlatforms.Contains(criteria.TargetHw);
/// <summary>
/// True when the description names another operating system and never Windows. A description
/// that names no platform passes.
/// </summary>
public static bool IsDescriptionNonWindowsOnly(string? description)
{
if (string.IsNullOrWhiteSpace(description)) return false;
try
{
return NonWindowsWord.IsMatch(description!) && !WindowsWord.IsMatch(description!);
}
catch (RegexMatchTimeoutException)
{
return false;
}
}
private static bool EditionCounts(string rangeEdition, string? required, IReadOnlyCollection<string>? excluded)
{
if (!string.IsNullOrEmpty(required))
return string.Equals(rangeEdition, required, StringComparison.OrdinalIgnoreCase);
return excluded == null || !excluded.Contains(rangeEdition, StringComparer.OrdinalIgnoreCase);
}
}
}
@@ -18,5 +18,13 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
public interface IVulnerabilityCorrelateProducer
{
Task EnqueueAsync(VulnerabilityCorrelateEnvelope envelope, CancellationToken cancellationToken = default);
/// <summary>
/// Puts a session back for a later run: the message stays invisible for
/// <paramref name="delay"/>. Called by the queue consumer while it still holds the
/// original message, so unlike <see cref="EnqueueAsync"/> this one throws when the send
/// fails: the host then retries the original instead of losing the session.
/// </summary>
Task EnqueueDelayedAsync(VulnerabilityCorrelateEnvelope envelope, System.TimeSpan delay, CancellationToken cancellationToken = default);
}
}

Some files were not shown because too many files have changed in this diff Show More