mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-24 04:32:32 +00:00
CPE Mappings simplified
This commit is contained in:
Oliver Kieselbach
@@ -109,7 +109,6 @@ tmpclaude-*-cwd
|
||||
.claude/commands/ux-insights.md
|
||||
.claude/commands/diag-agent-log.md
|
||||
.claude/commands/diag-ime-log.md
|
||||
.claude/commands/update-cpe-seed.md
|
||||
.claude/scripts/query-appinsights.sh
|
||||
.claude/scripts/query-table.sh
|
||||
.claude/scripts/fetch-diag-zip.sh
|
||||
|
||||
@@ -1 +1 @@
|
||||
557
|
||||
562
|
||||
|
||||
+76
@@ -0,0 +1,76 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Web;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
{
|
||||
/// <summary>
|
||||
/// Returns the full software inventory for a tenant from the SoftwareInventory table.
|
||||
/// Galactic Admin only. Query param: ?tenantId=...
|
||||
/// </summary>
|
||||
public class GetSoftwareInventoryFunction
|
||||
{
|
||||
private readonly ILogger<GetSoftwareInventoryFunction> _logger;
|
||||
private readonly TableStorageService _storageService;
|
||||
private readonly GalacticAdminService _galacticAdminService;
|
||||
|
||||
public GetSoftwareInventoryFunction(
|
||||
ILogger<GetSoftwareInventoryFunction> logger,
|
||||
TableStorageService storageService,
|
||||
GalacticAdminService galacticAdminService)
|
||||
{
|
||||
_logger = logger;
|
||||
_storageService = storageService;
|
||||
_galacticAdminService = galacticAdminService;
|
||||
}
|
||||
|
||||
[Function("GetSoftwareInventory")]
|
||||
public async Task<HttpResponseData> Run(
|
||||
[HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "vulnerability/software-inventory")] HttpRequestData req)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Galactic Admin only — enforced by PolicyEnforcementMiddleware
|
||||
var userIdentifier = TenantHelper.GetUserIdentifier(req);
|
||||
|
||||
var query = HttpUtility.ParseQueryString(req.Url.Query);
|
||||
var tenantId = query["tenantId"];
|
||||
|
||||
if (string.IsNullOrEmpty(tenantId))
|
||||
{
|
||||
var badRequest = req.CreateResponse(HttpStatusCode.BadRequest);
|
||||
await badRequest.WriteAsJsonAsync(new { success = false, message = "tenantId query parameter is required" });
|
||||
return badRequest;
|
||||
}
|
||||
|
||||
var inventory = await _storageService.GetSoftwareInventoryAsync(tenantId);
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
await response.WriteAsJsonAsync(new
|
||||
{
|
||||
success = true,
|
||||
tenantId,
|
||||
total = inventory.Count,
|
||||
matched = inventory.Count(i => !string.IsNullOrEmpty(i["cpeUri"]?.ToString())),
|
||||
unmatched = inventory.Count(i => string.IsNullOrEmpty(i["cpeUri"]?.ToString())),
|
||||
inventory
|
||||
});
|
||||
return response;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to get software inventory");
|
||||
var errorResponse = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await errorResponse.WriteStringAsync("Internal server error");
|
||||
return errorResponse;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+44
-63
@@ -2,12 +2,11 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
@@ -16,8 +15,9 @@ using Microsoft.Extensions.Logging;
|
||||
namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
{
|
||||
/// <summary>
|
||||
/// Admin endpoint that aggregates frequently unmatched software across all sessions.
|
||||
/// Admin endpoint that returns software without CPE mappings from the SoftwareInventory table.
|
||||
/// Used by the /update-cpe-seed skill and Admin UI to identify gaps in the CPE mapping.
|
||||
/// Entries in SoftwareInventory with an empty CpeUri are considered "unmatched".
|
||||
/// </summary>
|
||||
public class GetUnmatchedSoftwareFunction
|
||||
{
|
||||
@@ -41,74 +41,53 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
{
|
||||
try
|
||||
{
|
||||
// Galactic Admin only
|
||||
var authHeader = req.Headers.Contains("Authorization")
|
||||
? req.Headers.GetValues("Authorization").FirstOrDefault()
|
||||
: null;
|
||||
if (string.IsNullOrEmpty(authHeader))
|
||||
{
|
||||
var unauthorized = req.CreateResponse(HttpStatusCode.Unauthorized);
|
||||
return unauthorized;
|
||||
}
|
||||
// Galactic Admin only — enforced by PolicyEnforcementMiddleware
|
||||
var userIdentifier = TenantHelper.GetUserIdentifier(req);
|
||||
|
||||
// Query all SoftwareInventory entries across tenants and filter for unmatched (no CpeUri)
|
||||
var tableClient = _storageService.GetTableClient(Constants.TableNames.SoftwareInventory);
|
||||
var allEntities = tableClient.QueryAsync<TableEntity>(maxPerPage: 1000);
|
||||
|
||||
// Scan vulnerability reports in VulnerabilityReports table for unmatched_software lists
|
||||
var unmatchedAgg = new Dictionary<string, UnmatchedSoftwareAgg>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var tableClient = _storageService.GetTableClient(Constants.TableNames.VulnerabilityReports);
|
||||
var reportEntities = tableClient.QueryAsync<TableEntity>(
|
||||
filter: $"RowKey eq 'report'",
|
||||
maxPerPage: 200);
|
||||
|
||||
await foreach (var entity in reportEntities)
|
||||
await foreach (var entity in allEntities)
|
||||
{
|
||||
var reportJson = entity.GetString("ReportJson");
|
||||
if (string.IsNullOrEmpty(reportJson)) continue;
|
||||
var cpeUri = entity.GetString("CpeUri") ?? "";
|
||||
if (!string.IsNullOrEmpty(cpeUri))
|
||||
continue; // Has a CPE mapping — not unmatched
|
||||
|
||||
try
|
||||
var normalizedName = entity.GetString("NormalizedName") ?? "";
|
||||
if (string.IsNullOrWhiteSpace(normalizedName))
|
||||
continue;
|
||||
|
||||
var key = normalizedName.ToLowerInvariant();
|
||||
var sessionCount = entity.GetInt32("SessionCount") ?? 1;
|
||||
var lastSeenAt = entity.GetString("LastSeenAt") ?? "";
|
||||
var lastSessionId = entity.GetString("LastSessionId") ?? "";
|
||||
|
||||
if (!unmatchedAgg.TryGetValue(key, out var existing))
|
||||
{
|
||||
var data = JsonSerializer.Deserialize<Dictionary<string, object>>(reportJson,
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
|
||||
|
||||
if (data == null || !data.ContainsKey("unmatched_software")) continue;
|
||||
|
||||
var unmatchedJson = data["unmatched_software"]?.ToString();
|
||||
if (string.IsNullOrEmpty(unmatchedJson)) continue;
|
||||
|
||||
var unmatchedList = JsonSerializer.Deserialize<List<Dictionary<string, string>>>(unmatchedJson,
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
|
||||
|
||||
if (unmatchedList == null) continue;
|
||||
|
||||
var sessionId = entity.GetString("SessionId") ?? "";
|
||||
|
||||
foreach (var item in unmatchedList)
|
||||
unmatchedAgg[key] = new UnmatchedSoftwareAgg
|
||||
{
|
||||
var softwareName = item.GetValueOrDefault("softwareName", "");
|
||||
if (string.IsNullOrWhiteSpace(softwareName)) continue;
|
||||
|
||||
var key = softwareName.ToLowerInvariant();
|
||||
if (!unmatchedAgg.ContainsKey(key))
|
||||
{
|
||||
unmatchedAgg[key] = new UnmatchedSoftwareAgg
|
||||
{
|
||||
SoftwareName = softwareName,
|
||||
Publisher = item.GetValueOrDefault("publisher", ""),
|
||||
Frequency = 0,
|
||||
LastSeenAt = entity.GetDateTimeOffset("Timestamp")?.UtcDateTime ?? DateTime.UtcNow,
|
||||
ExampleSessionId = sessionId
|
||||
};
|
||||
}
|
||||
|
||||
unmatchedAgg[key].Frequency++;
|
||||
var ts = entity.GetDateTimeOffset("Timestamp")?.UtcDateTime ?? DateTime.MinValue;
|
||||
if (ts > unmatchedAgg[key].LastSeenAt)
|
||||
{
|
||||
unmatchedAgg[key].LastSeenAt = ts;
|
||||
unmatchedAgg[key].ExampleSessionId = sessionId;
|
||||
}
|
||||
SoftwareName = entity.GetString("DisplayName") ?? normalizedName,
|
||||
Publisher = entity.GetString("Publisher") ?? "",
|
||||
Frequency = sessionCount,
|
||||
LastSeenAt = lastSeenAt,
|
||||
ExampleSessionId = lastSessionId,
|
||||
NormalizedVendor = entity.GetString("NormalizedVendor") ?? "",
|
||||
NormalizedVersion = entity.GetString("NormalizedVersion") ?? ""
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
// Aggregate across tenants: sum frequency, keep most recent
|
||||
existing.Frequency += sessionCount;
|
||||
if (string.Compare(lastSeenAt, existing.LastSeenAt, StringComparison.Ordinal) > 0)
|
||||
{
|
||||
existing.LastSeenAt = lastSeenAt;
|
||||
existing.ExampleSessionId = lastSessionId;
|
||||
}
|
||||
}
|
||||
catch { /* skip malformed reports */ }
|
||||
}
|
||||
|
||||
var result = unmatchedAgg.Values
|
||||
@@ -134,8 +113,10 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
public string? SoftwareName { get; set; } = "";
|
||||
public string? Publisher { get; set; } = "";
|
||||
public int Frequency { get; set; }
|
||||
public DateTime LastSeenAt { get; set; }
|
||||
public string? LastSeenAt { get; set; } = "";
|
||||
public string? ExampleSessionId { get; set; } = "";
|
||||
public string? NormalizedVendor { get; set; } = "";
|
||||
public string? NormalizedVersion { get; set; } = "";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+17
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Web;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Vulnerability;
|
||||
@@ -11,6 +12,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
{
|
||||
/// <summary>
|
||||
/// HTTP-triggered manual sync for vulnerability data (KEV catalog refresh).
|
||||
/// Supports ?reseed=true to force re-import of CPE mapping seed from embedded resource.
|
||||
/// Galactic Admin only.
|
||||
/// </summary>
|
||||
public class TriggerVulnerabilityDataSyncFunction
|
||||
@@ -18,17 +20,20 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
private readonly KevDataService _kevService;
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly GalacticAdminService _galacticAdminService;
|
||||
private readonly TableStorageService _storageService;
|
||||
private readonly ILogger<TriggerVulnerabilityDataSyncFunction> _logger;
|
||||
|
||||
public TriggerVulnerabilityDataSyncFunction(
|
||||
KevDataService kevService,
|
||||
AdminConfigurationService adminConfigService,
|
||||
GalacticAdminService galacticAdminService,
|
||||
TableStorageService storageService,
|
||||
ILogger<TriggerVulnerabilityDataSyncFunction> logger)
|
||||
{
|
||||
_kevService = kevService;
|
||||
_adminConfigService = adminConfigService;
|
||||
_galacticAdminService = galacticAdminService;
|
||||
_storageService = storageService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -41,6 +46,17 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
// Refresh KEV catalog
|
||||
await _kevService.RefreshAsync();
|
||||
|
||||
// If reseed requested, re-import the CPE mapping seed
|
||||
int cpeSeedEntries = 0;
|
||||
var query = HttpUtility.ParseQueryString(req.Url.Query);
|
||||
if (string.Equals(query["reseed"], "true", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("CPE mapping seed re-import requested");
|
||||
cpeSeedEntries = await _storageService.ImportCpeMappingSeedAsync(force: true);
|
||||
VulnerabilityCorrelationService.ResetMappingsCache();
|
||||
_logger.LogInformation("CPE mapping seed re-imported: {Count} entries", cpeSeedEntries);
|
||||
}
|
||||
|
||||
// Update sync timestamp
|
||||
var config = await _adminConfigService.GetConfigurationAsync();
|
||||
if (config != null)
|
||||
@@ -55,6 +71,7 @@ namespace AutopilotMonitor.Functions.Functions.Vulnerability
|
||||
success = true,
|
||||
message = "Vulnerability data sync completed",
|
||||
kevCatalogEntries = _kevService.CatalogSize,
|
||||
cpeSeedEntries,
|
||||
syncedAt = DateTime.UtcNow.ToString("o")
|
||||
});
|
||||
return response;
|
||||
|
||||
@@ -183,6 +183,7 @@ public static class EndpointAccessPolicyCatalog
|
||||
new("GET", "health/detailed", EndpointPolicy.AuthenticatedUser),
|
||||
new("POST", "rules/reseed-from-github", EndpointPolicy.GalacticAdminOnly),
|
||||
new("GET", "vulnerability/unmatched-software", EndpointPolicy.GalacticAdminOnly),
|
||||
new("GET", "vulnerability/software-inventory", EndpointPolicy.GalacticAdminOnly),
|
||||
new("POST", "vulnerability/sync", EndpointPolicy.GalacticAdminOnly),
|
||||
new("POST", "vulnerability/cpe-mapping", EndpointPolicy.GalacticAdminOnly),
|
||||
new("POST", "rules/ime-log-patterns/reseed", EndpointPolicy.GalacticAdminOnly),
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services.Vulnerability;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -578,5 +582,276 @@ namespace AutopilotMonitor.Functions.Services
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== SOFTWARE INVENTORY METHODS =====
|
||||
|
||||
/// <summary>
|
||||
/// Upserts software inventory entries for a tenant.
|
||||
/// PK = tenantId, RK = {normalizedVendor}:{normalizedProduct}:{normalizedVersion} (sanitized).
|
||||
/// If an entry already exists, increments SessionCount and updates LastSeenAt/LastSessionId.
|
||||
/// </summary>
|
||||
public async Task UpsertSoftwareInventoryAsync(string tenantId, List<Dictionary<string, object>> inventoryItems, string sessionId, Dictionary<string, string?>? cpeMappings = null)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.SoftwareInventory);
|
||||
|
||||
// Read all existing entities for this tenant to merge SessionCount
|
||||
var existingEntities = new Dictionary<string, TableEntity>(StringComparer.OrdinalIgnoreCase);
|
||||
var existingQuery = tableClient.QueryAsync<TableEntity>(filter: $"PartitionKey eq '{tenantId}'");
|
||||
await foreach (var entity in existingQuery)
|
||||
{
|
||||
existingEntities[entity.RowKey] = entity;
|
||||
}
|
||||
|
||||
// Build the batch of entities to upsert
|
||||
var entitiesToUpsert = new List<TableEntity>();
|
||||
|
||||
foreach (var item in inventoryItems)
|
||||
{
|
||||
var normalizedName = item.ContainsKey("normalizedName") ? item["normalizedName"]?.ToString() ?? "" : "";
|
||||
var normalizedVendor = item.ContainsKey("normalizedPublisher") ? item["normalizedPublisher"]?.ToString() ?? "" : "";
|
||||
var normalizedVersion = item.ContainsKey("normalizedVersion") ? item["normalizedVersion"]?.ToString() ?? "" : "";
|
||||
var displayName = item.ContainsKey("displayName") ? item["displayName"]?.ToString() ?? "" : "";
|
||||
var publisher = item.ContainsKey("publisher") ? item["publisher"]?.ToString() ?? "" : "";
|
||||
var registrySource = item.ContainsKey("registrySource") ? item["registrySource"]?.ToString() ?? "" : "";
|
||||
var normalizationConfidence = item.ContainsKey("normalizationConfidence") ? item["normalizationConfidence"]?.ToString() ?? "" : "";
|
||||
|
||||
var rowKey = SanitizeTableKey($"{normalizedVendor}:{normalizedName}:{normalizedVersion}");
|
||||
if (string.IsNullOrWhiteSpace(rowKey) || rowKey == "::")
|
||||
continue;
|
||||
|
||||
// Truncate RowKey to Table Storage limit (1KB)
|
||||
if (rowKey.Length > 512)
|
||||
rowKey = rowKey.Substring(0, 512);
|
||||
|
||||
var now = DateTime.UtcNow;
|
||||
TableEntity entity;
|
||||
|
||||
if (existingEntities.TryGetValue(rowKey, out var existing))
|
||||
{
|
||||
// Update existing: increment SessionCount, update LastSeenAt
|
||||
existing["SessionCount"] = (existing.GetInt32("SessionCount") ?? 0) + 1;
|
||||
existing["LastSeenAt"] = now.ToString("o");
|
||||
existing["LastSessionId"] = sessionId;
|
||||
|
||||
// Update CpeUri if we have a mapping and current is empty
|
||||
if (cpeMappings != null && cpeMappings.TryGetValue(normalizedName, out var cpeUri) && !string.IsNullOrEmpty(cpeUri))
|
||||
{
|
||||
existing["CpeUri"] = cpeUri;
|
||||
}
|
||||
|
||||
entity = existing;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Create new entry
|
||||
entity = new TableEntity(tenantId, rowKey)
|
||||
{
|
||||
["DisplayName"] = displayName,
|
||||
["NormalizedName"] = normalizedName,
|
||||
["NormalizedVendor"] = normalizedVendor,
|
||||
["NormalizedVersion"] = normalizedVersion,
|
||||
["Publisher"] = publisher,
|
||||
["RegistrySource"] = registrySource,
|
||||
["NormalizationConfidence"] = normalizationConfidence,
|
||||
["FirstSeenAt"] = now.ToString("o"),
|
||||
["LastSeenAt"] = now.ToString("o"),
|
||||
["FirstSessionId"] = sessionId,
|
||||
["LastSessionId"] = sessionId,
|
||||
["SessionCount"] = 1,
|
||||
["CpeUri"] = cpeMappings != null && cpeMappings.TryGetValue(normalizedName, out var cpeUri) && !string.IsNullOrEmpty(cpeUri) ? cpeUri : ""
|
||||
};
|
||||
}
|
||||
|
||||
entitiesToUpsert.Add(entity);
|
||||
}
|
||||
|
||||
// Batch write (max 100 per batch, all same PK)
|
||||
for (int i = 0; i < entitiesToUpsert.Count; i += 100)
|
||||
{
|
||||
var batch = entitiesToUpsert.Skip(i).Take(100)
|
||||
.Select(e => new TableTransactionAction(TableTransactionActionType.UpsertReplace, e))
|
||||
.ToList();
|
||||
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Upserted {Count} software inventory entries for tenant {TenantId}", entitiesToUpsert.Count, tenantId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to upsert software inventory for tenant {TenantId}", tenantId);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets all software inventory entries for a tenant.
|
||||
/// PK = tenantId. Returns all rows.
|
||||
/// </summary>
|
||||
public async Task<List<Dictionary<string, object>>> GetSoftwareInventoryAsync(string tenantId)
|
||||
{
|
||||
SecurityValidator.EnsureValidGuid(tenantId, nameof(tenantId));
|
||||
|
||||
try
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.SoftwareInventory);
|
||||
var query = tableClient.QueryAsync<TableEntity>(filter: $"PartitionKey eq '{tenantId}'");
|
||||
|
||||
var results = new List<Dictionary<string, object>>();
|
||||
await foreach (var entity in query)
|
||||
{
|
||||
results.Add(new Dictionary<string, object>
|
||||
{
|
||||
{ "displayName", entity.GetString("DisplayName") ?? "" },
|
||||
{ "normalizedName", entity.GetString("NormalizedName") ?? "" },
|
||||
{ "normalizedVendor", entity.GetString("NormalizedVendor") ?? "" },
|
||||
{ "normalizedVersion", entity.GetString("NormalizedVersion") ?? "" },
|
||||
{ "publisher", entity.GetString("Publisher") ?? "" },
|
||||
{ "registrySource", entity.GetString("RegistrySource") ?? "" },
|
||||
{ "normalizationConfidence", entity.GetString("NormalizationConfidence") ?? "" },
|
||||
{ "cpeUri", entity.GetString("CpeUri") ?? "" },
|
||||
{ "firstSeenAt", entity.GetString("FirstSeenAt") ?? "" },
|
||||
{ "lastSeenAt", entity.GetString("LastSeenAt") ?? "" },
|
||||
{ "firstSessionId", entity.GetString("FirstSessionId") ?? "" },
|
||||
{ "lastSessionId", entity.GetString("LastSessionId") ?? "" },
|
||||
{ "sessionCount", entity.GetInt32("SessionCount") ?? 0 }
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Failed to get software inventory for tenant {TenantId}", tenantId);
|
||||
return new List<Dictionary<string, object>>();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sanitize a string for use as an Azure Table Storage key.
|
||||
/// Replaces /, \, #, ? with underscores.
|
||||
/// </summary>
|
||||
private static string SanitizeTableKey(string key)
|
||||
{
|
||||
return key
|
||||
.Replace("/", "_")
|
||||
.Replace("\\", "_")
|
||||
.Replace("#", "_")
|
||||
.Replace("?", "_");
|
||||
}
|
||||
|
||||
// ===== CPE MAPPING SEED IMPORT =====
|
||||
|
||||
/// <summary>
|
||||
/// Imports CPE mappings from the embedded seed JSON into the VulnerabilityCache table
|
||||
/// with PK = "cpe_map_seed". Only imports if no seed entries exist yet (idempotent).
|
||||
/// When force=true, deletes existing seed entries first and re-imports.
|
||||
/// Returns the number of entries imported (0 if already seeded and force=false).
|
||||
/// </summary>
|
||||
public async Task<int> ImportCpeMappingSeedAsync(bool force = false)
|
||||
{
|
||||
var tableClient = _tableServiceClient.GetTableClient(Constants.TableNames.VulnerabilityCache);
|
||||
|
||||
// Check if seed entries already exist
|
||||
if (!force)
|
||||
{
|
||||
var existingQuery = tableClient.QueryAsync<TableEntity>(
|
||||
filter: "PartitionKey eq 'cpe_map_seed'",
|
||||
select: new[] { "PartitionKey" },
|
||||
maxPerPage: 1);
|
||||
|
||||
await foreach (var _ in existingQuery)
|
||||
{
|
||||
// At least one seed entry exists — already seeded
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Force mode: delete all existing seed entries first
|
||||
var existingEntities = tableClient.QueryAsync<TableEntity>(
|
||||
filter: "PartitionKey eq 'cpe_map_seed'",
|
||||
select: new[] { "PartitionKey", "RowKey" });
|
||||
|
||||
await foreach (var entity in existingEntities)
|
||||
{
|
||||
await tableClient.DeleteEntityAsync(entity.PartitionKey, entity.RowKey);
|
||||
}
|
||||
}
|
||||
|
||||
// Load the embedded seed JSON
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var resourceName = assembly.GetManifestResourceNames()
|
||||
.FirstOrDefault(n => n.EndsWith("cpe-mapping-seed.json", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (resourceName == null)
|
||||
{
|
||||
_logger.LogWarning("CPE mapping seed resource not found in assembly");
|
||||
return 0;
|
||||
}
|
||||
|
||||
string json;
|
||||
using (var stream = assembly.GetManifestResourceStream(resourceName))
|
||||
{
|
||||
if (stream == null) return 0;
|
||||
using var reader = new StreamReader(stream);
|
||||
json = await reader.ReadToEndAsync();
|
||||
}
|
||||
|
||||
var seed = System.Text.Json.JsonSerializer.Deserialize<CpeMappingSeed>(json,
|
||||
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
|
||||
|
||||
if (seed?.Mappings == null || seed.Mappings.Count == 0)
|
||||
return 0;
|
||||
|
||||
// Build entities
|
||||
var entities = new List<TableEntity>();
|
||||
foreach (var mapping in seed.Mappings)
|
||||
{
|
||||
var rowKey = SanitizeTableKey(
|
||||
$"{(mapping.NormalizedVendor ?? "unknown")}:{(mapping.NormalizedProduct ?? "unknown")}".ToLowerInvariant());
|
||||
|
||||
// Truncate RowKey to Table Storage limit
|
||||
if (rowKey.Length > 512)
|
||||
rowKey = rowKey.Substring(0, 512);
|
||||
|
||||
entities.Add(new TableEntity("cpe_map_seed", rowKey)
|
||||
{
|
||||
{ "NormalizedVendor", mapping.NormalizedVendor ?? "" },
|
||||
{ "NormalizedProduct", mapping.NormalizedProduct ?? "" },
|
||||
{ "CpeVendor", mapping.CpeVendor ?? "" },
|
||||
{ "CpeProduct", mapping.CpeProduct ?? "" },
|
||||
{ "CpeUri", mapping.CpeUri ?? "" },
|
||||
{ "Category", mapping.Category ?? "" },
|
||||
{ "DisplayNamePatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.DisplayNamePatterns ?? new List<string>()) },
|
||||
{ "PublisherPatternsJson", System.Text.Json.JsonSerializer.Serialize(mapping.PublisherPatterns ?? new List<string>()) },
|
||||
{ "Source", "seed" },
|
||||
{ "ImportedAt", DateTime.UtcNow.ToString("o") }
|
||||
});
|
||||
}
|
||||
|
||||
// Batch write (max 100 per batch, all same PK)
|
||||
for (int i = 0; i < entities.Count; i += 100)
|
||||
{
|
||||
var batch = entities.Skip(i).Take(100)
|
||||
.Select(e => new TableTransactionAction(TableTransactionActionType.UpsertReplace, e))
|
||||
.ToList();
|
||||
|
||||
if (batch.Count > 0)
|
||||
{
|
||||
await tableClient.SubmitTransactionAsync(batch);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation("Imported {Count} CPE mapping seed entries into VulnerabilityCache table", entities.Count);
|
||||
return entities.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
stopwatch.Stop();
|
||||
_logger.LogInformation($"Table initialization completed in {stopwatch.ElapsedMilliseconds}ms: {successCount} succeeded, {failCount} failed");
|
||||
|
||||
// Import CPE mapping seed data if not already present
|
||||
try
|
||||
{
|
||||
var seedCount = await ImportCpeMappingSeedAsync();
|
||||
if (seedCount > 0)
|
||||
_logger.LogInformation("Imported {Count} CPE mapping seed entries", seedCount);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to import CPE mapping seed (non-fatal)");
|
||||
}
|
||||
|
||||
lock (_initLock)
|
||||
{
|
||||
_tablesInitialized = failCount == 0;
|
||||
|
||||
+43
-114
@@ -1,8 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
@@ -76,6 +74,8 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
int kevMatches = 0;
|
||||
string overallRisk = "none";
|
||||
|
||||
var cpeMappingsDict = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var item in inventoryItems)
|
||||
{
|
||||
if (ct.IsCancellationRequested) break;
|
||||
@@ -99,6 +99,12 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
continue;
|
||||
}
|
||||
|
||||
// Track CPE mapping for software inventory persistence
|
||||
if (!string.IsNullOrEmpty(normalizedName) && !cpeMappingsDict.ContainsKey(normalizedName))
|
||||
{
|
||||
cpeMappingsDict[normalizedName] = cpeMapping.CpeUri;
|
||||
}
|
||||
|
||||
matchedCount++;
|
||||
var cpeUri = cpeMapping.CpeUri ?? "";
|
||||
|
||||
@@ -188,10 +194,24 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
return ra.CompareTo(rb);
|
||||
});
|
||||
|
||||
// Persist software inventory (fire-and-forget, non-blocking)
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
await _storage.UpsertSoftwareInventoryAsync(tenantId, inventoryItems, sessionId, cpeMappingsDict);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Failed to upsert software inventory (non-fatal)");
|
||||
}
|
||||
});
|
||||
|
||||
// No findings = no report (same as RuleResults: don't store empty results)
|
||||
if (findings.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("Vulnerability correlation: no findings for {Count} scanned products", inventoryItems.Count);
|
||||
_logger.LogInformation("Vulnerability correlation: no findings for {Count} scanned, {Unmatched} unmatched",
|
||||
inventoryItems.Count, unmatchedSoftware.Count);
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -594,7 +614,7 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Seed Loading
|
||||
// Mapping Loading (table-only — no embedded resource fallback)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
private async Task EnsureMappingsLoadedAsync()
|
||||
@@ -610,46 +630,17 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
|
||||
_cpeMappings = new List<CpeMappingEntry>();
|
||||
|
||||
// Load from embedded seed resource
|
||||
try
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var resourceName = assembly.GetManifestResourceNames()
|
||||
.FirstOrDefault(n => n.EndsWith("cpe-mapping-seed.json", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (resourceName != null)
|
||||
{
|
||||
using var stream = assembly.GetManifestResourceStream(resourceName);
|
||||
if (stream != null)
|
||||
{
|
||||
using var reader = new StreamReader(stream);
|
||||
var json = await reader.ReadToEndAsync();
|
||||
var seed = JsonSerializer.Deserialize<CpeMappingSeed>(json, JsonOptions);
|
||||
if (seed?.Mappings != null)
|
||||
{
|
||||
_cpeMappings.AddRange(seed.Mappings);
|
||||
_logger.LogInformation("Loaded {Count} CPE mappings from seed", seed.Mappings.Count);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("CPE mapping seed resource not found in assembly");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load CPE mapping seed");
|
||||
}
|
||||
|
||||
// Load custom mappings from VulnerabilityCache table
|
||||
// Load ALL CPE mappings from VulnerabilityCache table
|
||||
// (seed + custom + community — all in one table, distinguished by PK prefix)
|
||||
try
|
||||
{
|
||||
var tableClient = _storage.GetTableClient(AutopilotMonitor.Shared.Constants.TableNames.VulnerabilityCache);
|
||||
var customMappings = tableClient.QueryAsync<TableEntity>(
|
||||
filter: $"PartitionKey eq 'cpe_map_custom'");
|
||||
|
||||
await foreach (var entity in customMappings)
|
||||
// Query all cpe_map_* entries (seed, custom, community)
|
||||
var allMappings = tableClient.QueryAsync<TableEntity>(
|
||||
filter: "PartitionKey ge 'cpe_map' and PartitionKey lt 'cpe_mar'");
|
||||
|
||||
await foreach (var entity in allMappings)
|
||||
{
|
||||
_cpeMappings.Add(new CpeMappingEntry
|
||||
{
|
||||
@@ -658,94 +649,32 @@ namespace AutopilotMonitor.Functions.Services.Vulnerability
|
||||
CpeVendor = entity.GetString("CpeVendor") ?? "",
|
||||
CpeProduct = entity.GetString("CpeProduct") ?? "",
|
||||
CpeUri = entity.GetString("CpeUri") ?? "",
|
||||
Category = entity.GetString("Category") ?? "custom",
|
||||
DisplayNamePatterns = JsonSerializer.Deserialize<List<string>>(
|
||||
entity.GetString("DisplayNamePatternsJson") ?? "[]") ?? new List<string>(),
|
||||
PublisherPatterns = JsonSerializer.Deserialize<List<string>>(
|
||||
entity.GetString("PublisherPatternsJson") ?? "[]") ?? new List<string>()
|
||||
Category = entity.GetString("Category") ?? "",
|
||||
DisplayNamePatterns = DeserializeStringList(entity.GetString("DisplayNamePatternsJson")),
|
||||
PublisherPatterns = DeserializeStringList(entity.GetString("PublisherPatternsJson"))
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "No custom CPE mappings loaded (table may not exist yet)");
|
||||
}
|
||||
|
||||
// Load community CPE mappings (table storage from reseed, falling back to embedded resource)
|
||||
int communityCount = 0;
|
||||
try
|
||||
{
|
||||
var vulnTable = _storage.GetTableClient(AutopilotMonitor.Shared.Constants.TableNames.VulnerabilityCache);
|
||||
var communityMappings = vulnTable.QueryAsync<TableEntity>(
|
||||
filter: $"PartitionKey eq 'cpe_map_community'");
|
||||
|
||||
await foreach (var entity in communityMappings)
|
||||
{
|
||||
_cpeMappings.Add(new CpeMappingEntry
|
||||
{
|
||||
NormalizedVendor = entity.GetString("NormalizedVendor") ?? "",
|
||||
NormalizedProduct = entity.GetString("NormalizedProduct") ?? "",
|
||||
CpeVendor = entity.GetString("CpeVendor") ?? "",
|
||||
CpeProduct = entity.GetString("CpeProduct") ?? "",
|
||||
CpeUri = entity.GetString("CpeUri") ?? "",
|
||||
Category = entity.GetString("Category") ?? "community",
|
||||
DisplayNamePatterns = JsonSerializer.Deserialize<List<string>>(
|
||||
entity.GetString("DisplayNamePatternsJson") ?? "[]") ?? new List<string>(),
|
||||
PublisherPatterns = JsonSerializer.Deserialize<List<string>>(
|
||||
entity.GetString("PublisherPatternsJson") ?? "[]") ?? new List<string>()
|
||||
});
|
||||
communityCount++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "No community CPE mappings in table storage, falling back to embedded resource");
|
||||
}
|
||||
|
||||
// Fallback: load from embedded resource if table had no community entries
|
||||
if (communityCount == 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
var assembly = Assembly.GetExecutingAssembly();
|
||||
var communityResourceName = assembly.GetManifestResourceNames()
|
||||
.FirstOrDefault(n => n.EndsWith("cpe-community-mappings.json", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (communityResourceName != null)
|
||||
{
|
||||
using var communityStream = assembly.GetManifestResourceStream(communityResourceName);
|
||||
if (communityStream != null)
|
||||
{
|
||||
using var communityReader = new StreamReader(communityStream);
|
||||
var communityJson = await communityReader.ReadToEndAsync();
|
||||
var communitySeed = JsonSerializer.Deserialize<CpeMappingSeed>(communityJson, JsonOptions);
|
||||
if (communitySeed?.Mappings != null && communitySeed.Mappings.Count > 0)
|
||||
{
|
||||
_cpeMappings.AddRange(communitySeed.Mappings);
|
||||
communityCount = communitySeed.Mappings.Count;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to load community CPE mappings from embedded resource");
|
||||
}
|
||||
}
|
||||
|
||||
if (communityCount > 0)
|
||||
{
|
||||
_logger.LogInformation("Loaded {Count} community CPE mappings", communityCount);
|
||||
_logger.LogWarning(ex, "Failed to load CPE mappings from table");
|
||||
}
|
||||
|
||||
_seedLoaded = true;
|
||||
_logger.LogInformation("CPE mappings ready: {Count} total entries", _cpeMappings.Count);
|
||||
_logger.LogInformation("CPE mappings ready: {Count} total entries (from table)", _cpeMappings.Count);
|
||||
}
|
||||
finally
|
||||
{
|
||||
_initLock.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static List<string> DeserializeStringList(string? json)
|
||||
{
|
||||
if (string.IsNullOrEmpty(json)) return new List<string>();
|
||||
try { return JsonSerializer.Deserialize<List<string>>(json, JsonOptions) ?? new List<string>(); }
|
||||
catch { return new List<string>(); }
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -259,6 +259,9 @@ namespace AutopilotMonitor.Shared
|
||||
// Vulnerability reports per session (like RuleResults but for CVE correlation)
|
||||
public const string VulnerabilityReports = "VulnerabilityReports";
|
||||
|
||||
// Persistent software inventory per tenant (aggregated from enrollment snapshots)
|
||||
public const string SoftwareInventory = "SoftwareInventory";
|
||||
|
||||
/// <summary>
|
||||
/// Returns all table names for initialization
|
||||
/// </summary>
|
||||
@@ -289,7 +292,8 @@ namespace AutopilotMonitor.Shared
|
||||
BootstrapSessions,
|
||||
GalacticNotifications,
|
||||
VulnerabilityCache,
|
||||
VulnerabilityReports
|
||||
VulnerabilityReports,
|
||||
SoftwareInventory
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user