mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-11-03 09:03:29 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6866fe79af | |||
| 6e34fa517c | |||
| dc05428dae | |||
| cd0c750e80 | |||
| cede4be654 | |||
| 0d63057725 | |||
| 7dd151b714 | |||
| 778eb44a34 | |||
| 00f37d869a | |||
| d88a118903 | |||
| 0ed3258d4f | |||
| 3941c3b984 | |||
| 3d831d16d7 | |||
| 46b895a70d | |||
| 77fae059ed | |||
| fd309da51d | |||
| 9c4039b3c3 | |||
| b0fb8eb5fa | |||
| dd2432fa62 |
+1
-1
Submodule .claude/commands updated: 45a18760a0...f8941093f4
+1
-1
Submodule internal updated: 70adb72c87...f161dac6f8
@@ -72,7 +72,8 @@ public class AppHomingServiceTests
|
||||
Mock.Of<System.Net.Http.IHttpClientFactory>(),
|
||||
cache,
|
||||
_registry,
|
||||
_tenantConfigMock.Object)
|
||||
_tenantConfigMock.Object,
|
||||
Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
_detectorMock = new Mock<IGraphFeatureDetector>();
|
||||
_maintenanceMock = new Mock<IMaintenanceRepository>();
|
||||
@@ -287,6 +288,32 @@ public class AppHomingServiceTests
|
||||
Assert.True((await _sut.ProbePrimaryConsentAsync(TenantId)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_succeeds_when_the_budget_cuts_the_retries_of_a_missing_legacy_service_principal()
|
||||
{
|
||||
// The access check runs on 4 s and the first retry waits 5 s, so the answer arrives as
|
||||
// transient. The Entra code still says the SP is gone: nothing left to lose.
|
||||
SetupProbe(GraphTokenResult.Success(Jwt(ValidationRole)));
|
||||
SetupLegacyToken(GraphTokenResult.TransientFailure(
|
||||
new EntraTokenRejection(401, EntraTokenRejection.ServicePrincipalMissingCode, "no service principal")));
|
||||
|
||||
Assert.True((await _sut.ProbePrimaryConsentAsync(TenantId)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_stays_transient_when_the_budget_cuts_the_retries_of_any_other_legacy_rejection()
|
||||
{
|
||||
// Any other refusal proves nothing about the roles the tenant would lose with the flip.
|
||||
SetupProbe(GraphTokenResult.Success(Jwt(ValidationRole)));
|
||||
SetupLegacyToken(GraphTokenResult.TransientFailure(
|
||||
new EntraTokenRejection(400, "AADSTS53003", "blocked by Conditional Access")));
|
||||
|
||||
var probe = await _sut.ProbePrimaryConsentAsync(TenantId);
|
||||
|
||||
Assert.False(probe.Succeeded);
|
||||
Assert.True(probe.IsTransient);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_never_mints_a_legacy_token_when_the_primary_token_fails()
|
||||
{
|
||||
@@ -349,6 +376,21 @@ public class AppHomingServiceTests
|
||||
Assert.DoesNotContain(_savedOpsEvents, e => e.EventType == "AppHomingFlippedWithEntraRoles");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoFlip_clears_the_graph_access_rejection_of_the_app_the_tenant_leaves()
|
||||
{
|
||||
_store.Row!.GraphAccessRejectedSince = DateTime.UtcNow.AddHours(-3);
|
||||
_store.Row.GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode;
|
||||
_store.Row.GraphAccessRejectionConfirmedAt = DateTime.UtcNow.AddHours(-2);
|
||||
|
||||
var outcome = await _sut.TryAutoFlipToPrimaryAsync(LegacyHomedConfig(), Actor);
|
||||
|
||||
Assert.Equal(AppHomingAutoFlipOutcome.Flipped, outcome.Outcome);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoFlip_does_not_flip_on_permanent_probe_failure()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A radar pass reads one tenant at a time so its memory follows the largest tenant, and it never
|
||||
/// evaluates an empty list: re-arm would read that as "version drained" and delete open episodes.
|
||||
/// </summary>
|
||||
public class AppVersionRadarPassTests
|
||||
{
|
||||
private static List<AppInstallSummary> Rows(string tenantId, int count)
|
||||
{
|
||||
var rows = new List<AppInstallSummary>();
|
||||
for (var i = 0; i < count; i++)
|
||||
rows.Add(new AppInstallSummary { TenantId = tenantId });
|
||||
return rows;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Each_tenant_is_finished_before_the_next_one_is_loaded()
|
||||
{
|
||||
var calls = new List<string>();
|
||||
|
||||
await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "a", "b" },
|
||||
tenantId =>
|
||||
{
|
||||
calls.Add($"load {tenantId}");
|
||||
return Task.FromResult(Rows(tenantId, 2));
|
||||
},
|
||||
(tenantId, summaries) =>
|
||||
{
|
||||
calls.Add($"evaluate {tenantId}");
|
||||
Assert.All(summaries, s => Assert.Equal(tenantId, s.TenantId));
|
||||
return Task.FromResult((0, 0, 0));
|
||||
},
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal(new[] { "load a", "evaluate a", "load b", "evaluate b" }, calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_rows_is_not_evaluated()
|
||||
{
|
||||
var evaluated = new List<string>();
|
||||
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "empty", "full" },
|
||||
tenantId => Task.FromResult(tenantId == "empty" ? new List<AppInstallSummary>() : Rows(tenantId, 3)),
|
||||
(tenantId, _) =>
|
||||
{
|
||||
evaluated.Add(tenantId);
|
||||
return Task.FromResult((0, 0, 0));
|
||||
},
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal(new[] { "full" }, evaluated);
|
||||
Assert.Equal(1, pass.Tenants);
|
||||
Assert.Equal(3, pass.Rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_tenant_does_not_cost_the_others_their_pass()
|
||||
{
|
||||
var failed = new List<string>();
|
||||
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "load-fails", "evaluate-fails", "ok" },
|
||||
tenantId => tenantId == "load-fails"
|
||||
? throw new InvalidOperationException("load")
|
||||
: Task.FromResult(Rows(tenantId, 1)),
|
||||
(tenantId, _) => tenantId == "evaluate-fails"
|
||||
? throw new InvalidOperationException("evaluate")
|
||||
: Task.FromResult((1, 2, 3)),
|
||||
(tenantId, _) => failed.Add(tenantId));
|
||||
|
||||
Assert.Equal(new[] { "load-fails", "evaluate-fails" }, failed);
|
||||
Assert.Equal((1, 2, 3), (pass.Fired, pass.Refreshed, pass.Rearmed));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_totals_add_up_over_the_tenants()
|
||||
{
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "a", "b" },
|
||||
tenantId => Task.FromResult(Rows(tenantId, 2)),
|
||||
(_, _) => Task.FromResult((1, 2, 3)),
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal((2, 4L, 2, 4, 6), pass);
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,6 @@ public class AuthFunctionSideEffectTests
|
||||
private readonly Mock<TenantAdminsService> _tenantAdminsMock;
|
||||
private readonly Mock<PreviewWhitelistService> _previewMock;
|
||||
private readonly Mock<McpUserService> _mcpUserMock;
|
||||
private readonly Mock<TelegramNotificationService> _telegramMock;
|
||||
private readonly Mock<OpsEventService> _opsEventsMock;
|
||||
private readonly Mock<GlobalNotificationService> _globalNotificationMock;
|
||||
private readonly Mock<IMetricsRepository> _metricsRepoMock;
|
||||
@@ -84,10 +83,6 @@ public class AuthFunctionSideEffectTests
|
||||
configRepo, cache, Mock.Of<ILogger<PreviewWhitelistService>>(), _tenantConfigMock.Object)
|
||||
{ CallBase = false };
|
||||
|
||||
_telegramMock = new Mock<TelegramNotificationService>(
|
||||
new HttpClient(), configRepo, Mock.Of<ILogger<TelegramNotificationService>>())
|
||||
{ CallBase = false };
|
||||
|
||||
_globalNotificationMock = new Mock<GlobalNotificationService>(
|
||||
notificationRepo, new FakeSignalRNotificationService(), Mock.Of<ILogger<GlobalNotificationService>>())
|
||||
{ CallBase = false };
|
||||
@@ -117,7 +112,6 @@ public class AuthFunctionSideEffectTests
|
||||
tenantAdmins ?? _tenantAdminsMock.Object,
|
||||
_metricsRepoMock.Object,
|
||||
preview ?? _previewMock.Object,
|
||||
_telegramMock.Object,
|
||||
_globalNotificationMock.Object,
|
||||
_mcpUserMock.Object,
|
||||
_autoApproveEnqueuerMock.Object,
|
||||
@@ -136,9 +130,6 @@ public class AuthFunctionSideEffectTests
|
||||
// Writes go through UpdateAsync / CreateOrUpdateAsync over one stored row (none by default:
|
||||
// a brand-new tenant). Default: all fire-and-forget calls succeed.
|
||||
_store = _tenantConfigMock.StubWrites(TenantId, row: null);
|
||||
_telegramMock
|
||||
.Setup(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
_globalNotificationMock
|
||||
.Setup(x => x.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
@@ -233,7 +224,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.True(string.IsNullOrEmpty(config.DomainName));
|
||||
Assert.Null(config.OnboardedBy);
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -261,8 +252,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.Null(written.DpaAcceptedBy);
|
||||
Assert.Null(written.DpaAcceptedAt);
|
||||
Assert.Equal("auth", source);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(TenantId, Upn), Times.Once);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(TenantId, Upn), Times.Once); // the routable record next to the ping
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(TenantId, Upn), Times.Once); // the routable record
|
||||
_globalNotificationMock.Verify(x => x.CreateNotificationAsync(
|
||||
"preview_signup", "New Tenant Signup",
|
||||
It.Is<string>(m => m.Contains(TenantId) && m.Contains("contoso.com") && m.Contains(Upn)),
|
||||
@@ -350,7 +340,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.True(string.IsNullOrEmpty(config.DomainName));
|
||||
Assert.Null(config.OnboardedBy);
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_autoApproveEnqueuerMock.Verify(x => x.EnqueueAsync(
|
||||
It.IsAny<AutopilotMonitor.Functions.Services.Activation.TenantAutoApproveEnvelope>(),
|
||||
It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
@@ -371,7 +361,7 @@ public class AuthFunctionSideEffectTests
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Equal("first@contoso.com", _store.Row!.OnboardedBy);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_autoApproveEnqueuerMock.Verify(x => x.EnqueueAsync(
|
||||
It.IsAny<AutopilotMonitor.Functions.Services.Activation.TenantAutoApproveEnvelope>(),
|
||||
It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
@@ -389,7 +379,7 @@ public class AuthFunctionSideEffectTests
|
||||
await _sut.HandleNewTenantDomainAsync(DefaultConfig(), TenantId, Upn);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -624,7 +614,7 @@ public class AuthFunctionSideEffectTests
|
||||
await _sut.HandleNewTenantDomainAsync(config, TenantId, Upn);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -650,16 +640,17 @@ public class AuthFunctionSideEffectTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleNewTenantDomain_TelegramFailure_DoesNotThrow()
|
||||
public async Task HandleNewTenantDomain_BellFailure_DoesNotThrow()
|
||||
{
|
||||
var config = DefaultConfig();
|
||||
config.DomainName = null!;
|
||||
|
||||
_telegramMock
|
||||
.Setup(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ThrowsAsync(new HttpRequestException("Telegram unreachable"));
|
||||
_globalNotificationMock
|
||||
.Setup(x => x.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
.ThrowsAsync(new HttpRequestException("Notification store unreachable"));
|
||||
|
||||
// Should not throw — Telegram is fire-and-forget
|
||||
// Should not throw — the bell is fire-and-forget
|
||||
await _sut.HandleNewTenantDomainAsync(config, TenantId, Upn);
|
||||
|
||||
// The write still landed before the fire-and-forget
|
||||
|
||||
@@ -69,7 +69,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
@@ -201,7 +201,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
CancellationToken observed = default;
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
|
||||
@@ -181,6 +181,27 @@ public class ConfigBackupHookTests
|
||||
b => b.UpsertAsync(It.IsAny<ConfigBackupEntry>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoiseOnlyChange_GraphAccessState_NoSnapshot()
|
||||
{
|
||||
// Token side effect: stamped and cleared without anyone changing the configuration.
|
||||
var stored = StoredConfig();
|
||||
var harness = new Harness(TableConfigRepository.ConvertToTenantTableEntity(stored));
|
||||
|
||||
var incoming = RoundtripClone(stored);
|
||||
incoming.GraphAccessRejectedSince = DateTime.UtcNow.AddMinutes(-20);
|
||||
incoming.GraphAccessRejectedCode = "AADSTS7000229";
|
||||
incoming.GraphAccessRejectionConfirmedAt = DateTime.UtcNow;
|
||||
incoming.LastUpdated = DateTime.UtcNow;
|
||||
|
||||
var saved = await harness.Sut.TryReplaceTenantConfigurationAsync(incoming, "\"etag\"", "graph-access", null);
|
||||
|
||||
Assert.True(saved);
|
||||
Assert.Single(harness.Upserts);
|
||||
harness.Backup.Verify(
|
||||
b => b.UpsertAsync(It.IsAny<ConfigBackupEntry>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoOpSave_IdenticalContent_NoSnapshot()
|
||||
{
|
||||
|
||||
@@ -65,7 +65,7 @@ public class CorporateIdentifierValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -4,6 +4,7 @@ using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -12,8 +13,8 @@ using Moq;
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// E-mail as a notification-channel provider (D-332): dispatch routing (never the webhook path,
|
||||
/// never the bot), recipient validation, the Global-Admin gate, the rendered message, and the
|
||||
/// E-mail as a notification-channel provider (D-332): dispatch routing (never the webhook path),
|
||||
/// recipient validation, the Global-Admin gate, the rendered message, and the
|
||||
/// send against a stubbed provider endpoint.
|
||||
/// </summary>
|
||||
public class EmailChannelProviderTests
|
||||
@@ -30,15 +31,6 @@ public class EmailChannelProviderTests
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<IEmailChannelSender> EmailMock()
|
||||
{
|
||||
var mock = new Mock<IEmailChannelSender>();
|
||||
@@ -50,6 +42,9 @@ public class EmailChannelProviderTests
|
||||
private static NotificationChannel Channel(string? url, string id = "mail", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = Email, Url = url, Enabled = enabled };
|
||||
|
||||
private static NotificationChannel Webhook(WebhookProviderType provider, string url)
|
||||
=> new() { Id = provider.ToString(), Name = provider.ToString(), ProviderType = (int)provider, Url = url, Enabled = true };
|
||||
|
||||
private static NotificationAlert Alert() => new()
|
||||
{
|
||||
EventType = "PushDeliveryFailed",
|
||||
@@ -67,9 +62,8 @@ public class EmailChannelProviderTests
|
||||
public async Task Email_channel_reaches_the_email_sender_with_its_recipients_and_never_the_webhook_path()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid; two@example.invalid") }, alert, Platform);
|
||||
@@ -77,7 +71,6 @@ public class EmailChannelProviderTests
|
||||
email.Verify(e => e.SendAlertAsync("ops@example.invalid; two@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -86,7 +79,7 @@ public class EmailChannelProviderTests
|
||||
var email = EmailMock();
|
||||
email.Setup(e => e.SendAlertAsync("ops@example.invalid", It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "Sent to 1 recipient(s)." });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel(""), Channel(null, "n") }, Alert(), Platform);
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
@@ -99,7 +92,7 @@ public class EmailChannelProviderTests
|
||||
[Fact]
|
||||
public async Task Dispatcher_without_an_email_transport_skips_the_channel_and_says_so_on_test()
|
||||
{
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>());
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid") }, Alert(), Platform); // no throw
|
||||
var result = await dispatcher.SendTestAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
@@ -108,6 +101,54 @@ public class EmailChannelProviderTests
|
||||
Assert.Contains("not registered", result.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Webhook_channel_reaches_the_webhook_path_and_never_the_email_sender()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(WebhookProviderType.Slack, "https://hooks.slack.example/x") }, alert, Platform);
|
||||
|
||||
webhook.Verify(w => w.SendAsync("https://hooks.slack.example/x", WebhookProviderType.Slack, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Mixed_list_each_channel_reaches_its_own_transport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel("ops@example.invalid"),
|
||||
Webhook(WebhookProviderType.GenericJson, "https://sales.example/hook"),
|
||||
}, alert, Platform);
|
||||
|
||||
email.Verify(e => e.SendAlertAsync("ops@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync("https://sales.example/hook", WebhookProviderType.GenericJson, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Channels_without_a_destination_are_skipped_on_every_transport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel(null), Webhook(WebhookProviderType.Slack, "") }, Alert(), Platform);
|
||||
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
// ── Validation and gate ───────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
@@ -138,6 +179,10 @@ public class EmailChannelProviderTests
|
||||
|
||||
var bad = NotificationChannel.SerializeList(new[] { Channel("https://hooks.example.invalid/x") });
|
||||
Assert.Contains("not a valid e-mail address", TenantConfigValidation.ValidateNotificationChannels(bad));
|
||||
|
||||
// The branch is per provider: a recipient list on a webhook channel still meets the URL gate.
|
||||
var onWebhook = NotificationChannel.SerializeList(new[] { Webhook(WebhookProviderType.Slack, "ops@example.invalid") });
|
||||
Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(onWebhook));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -146,14 +191,45 @@ public class EmailChannelProviderTests
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider(Email));
|
||||
var candidate = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid") });
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(candidate, null, isGlobalAdmin: true));
|
||||
var refused = TenantConfigValidation.ValidateTelegramChannelGate(candidate, null, isGlobalAdmin: false);
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: true));
|
||||
var refused = TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: false);
|
||||
Assert.Equal("E-mail channels can only be configured by a Global Administrator.", refused);
|
||||
|
||||
// A GA-created channel survives a tenant admin's unrelated save; retargeting it does not.
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(candidate, candidate, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, candidate, isGlobalAdmin: false));
|
||||
var retargeted = NotificationChannel.SerializeList(new[] { Channel("other@example.invalid") });
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(retargeted, candidate, isGlobalAdmin: false));
|
||||
Assert.NotNull(TenantConfigValidation.ValidatePlatformProviderGate(retargeted, candidate, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_refuses_a_tenant_admin_who_flips_the_enabled_state()
|
||||
{
|
||||
var existing = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid", enabled: true) });
|
||||
var disabled = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid", enabled: false) });
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidatePlatformProviderGate(disabled, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_ignores_webhook_channels()
|
||||
{
|
||||
var candidate = NotificationChannel.SerializeList(new[] { Webhook(WebhookProviderType.GenericJson, "https://sales.example/hook") });
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateModel_enforces_the_gate_for_every_client_write()
|
||||
{
|
||||
var candidate = new TenantConfiguration
|
||||
{
|
||||
TenantId = "t1",
|
||||
NotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid") }),
|
||||
};
|
||||
var existing = new TenantConfiguration { TenantId = "t1" };
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: true));
|
||||
}
|
||||
|
||||
// ── Rendering ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the AADSTS code out of Entra's token error body. The body below has the shape Entra
|
||||
/// really sends for a tenant whose enterprise application was removed; the ids are synthetic.
|
||||
/// </summary>
|
||||
public class EntraTokenRejectionTests
|
||||
{
|
||||
private const string MissingServicePrincipalBody =
|
||||
"{\"error\":\"invalid_client\"," +
|
||||
"\"error_description\":\"AADSTS7000229: The client application aaaaaaaa-0000-0000-0000-000000000001 is missing service principal in the tenant 11111111-1111-1111-1111-111111111111. See instructions here: https://go.microsoft.com/fwlink/?linkid=2225119\\r\\nTrace ID: 00000000-0000-0000-0000-000000000002\\r\\nCorrelation ID: 00000000-0000-0000-0000-000000000003\\r\\nTimestamp: 2026-11-01 08:00:00Z\"," +
|
||||
"\"error_codes\":[7000229]," +
|
||||
"\"timestamp\":\"2026-11-01 08:00:00Z\"," +
|
||||
"\"trace_id\":\"00000000-0000-0000-0000-000000000002\"," +
|
||||
"\"correlation_id\":\"00000000-0000-0000-0000-000000000003\"," +
|
||||
"\"error_uri\":\"https://login.microsoftonline.com/error?code=7000229\"}";
|
||||
|
||||
[Fact]
|
||||
public void Parse_reads_the_code_and_keeps_only_the_first_line_of_the_description()
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(401, MissingServicePrincipalBody);
|
||||
|
||||
Assert.Equal(401, rejection.StatusCode);
|
||||
Assert.Equal("AADSTS7000229", rejection.ErrorCode);
|
||||
Assert.True(rejection.IsServicePrincipalMissing);
|
||||
Assert.False(rejection.IsOurCredential);
|
||||
Assert.StartsWith("AADSTS7000229: The client application", rejection.Description);
|
||||
Assert.DoesNotContain("Trace ID", rejection.Description);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_falls_back_to_the_description_without_error_codes()
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(400, "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: Access has been blocked by Conditional Access policies.\"}");
|
||||
|
||||
Assert.Equal("AADSTS53003", rejection.ErrorCode);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData("<html>Bad Request</html>")]
|
||||
[InlineData("[1,2]")]
|
||||
[InlineData("{\"error\":\"invalid_request\"}")]
|
||||
public void Parse_keeps_only_the_status_when_the_body_carries_no_code(string? body)
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(400, body);
|
||||
|
||||
Assert.Equal(400, rejection.StatusCode);
|
||||
Assert.Null(rejection.ErrorCode);
|
||||
Assert.Null(rejection.Description);
|
||||
Assert.False(rejection.IsServicePrincipalMissing);
|
||||
Assert.False(rejection.IsOurCredential);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_cuts_a_long_description()
|
||||
{
|
||||
var body = "{\"error_description\":\"AADSTS50000: " + new string('x', 1000) + "\"}";
|
||||
|
||||
Assert.Equal(300, EntraTokenRejection.Parse(400, body).Description!.Length);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("AADSTS7000215", true)]
|
||||
[InlineData("AADSTS7000222", true)]
|
||||
[InlineData("AADSTS7000218", true)]
|
||||
[InlineData("aadsts7000222", true)]
|
||||
[InlineData("AADSTS7000229", false)]
|
||||
[InlineData("AADSTS7000112", false)]
|
||||
[InlineData("AADSTS53003", false)]
|
||||
public void IsOurCredential_only_blames_our_own_secret(string code, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, new EntraTokenRejection(401, code, null).IsOurCredential);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Feedback;
|
||||
@@ -19,8 +18,8 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// The rating bubble's answer route, driven through <c>ProcessSubmitAsync</c> (the HTTP shell only
|
||||
/// reads the body and maps the validation error). Pins that a recorded rating, and nothing else,
|
||||
/// raises <c>FeedbackRatingReceived</c>, the Global Admin bell and the Telegram ping, and that the
|
||||
/// alert carries neither the comment nor the sender.
|
||||
/// raises <c>FeedbackRatingReceived</c> and the Global Admin bell, and that the alert carries neither
|
||||
/// the comment nor the sender.
|
||||
/// </summary>
|
||||
public sealed class FeedbackFunctionSubmitTests
|
||||
{
|
||||
@@ -34,7 +33,7 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
private static readonly string EntryPath = Constants.PortalUserFeedbackEntryPath(FeedbackId);
|
||||
|
||||
[Fact]
|
||||
public async Task Rating_IsStored_RaisesTheEvent_RingsTheBell_AndPings()
|
||||
public async Task Rating_IsStored_RaisesTheEvent_AndRingsTheBell()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
@@ -64,13 +63,12 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
|
||||
h.Bell.Verify(b => b.CreateNotificationAsync(
|
||||
"feedback", "New Rating", $"{Upn} (contoso.invalid) · 4/5", EntryPath), Times.Once);
|
||||
h.Telegram.Verify(t => t.SendFeedbackAsync(TenantId, Upn, DisplayName, 4, Comment), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TheAlert_CarriesNeitherCommentNorSender()
|
||||
{
|
||||
// Channels bound to FeedbackRatingReceived (Telegram, Slack, webhooks, e-mail) receive message
|
||||
// Channels bound to FeedbackRatingReceived (Slack, webhooks, e-mail) receive message
|
||||
// and details; the comment is read in the portal. Personal data must never ride along.
|
||||
var h = new Harness();
|
||||
|
||||
@@ -160,7 +158,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
Assert.Empty(h.OpsEvents);
|
||||
h.Bell.Verify(b => b.CreateNotificationAsync(
|
||||
"feedback", "New Rating", It.IsAny<string>(), EntryPath), Times.Once);
|
||||
h.Telegram.Verify(t => t.SendFeedbackAsync(TenantId, Upn, DisplayName, 3, Comment), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -183,7 +180,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
public FakeFeedbackRepository Repo { get; }
|
||||
public List<OpsEventEntry> OpsEvents { get; } = new();
|
||||
public Mock<GlobalNotificationService> Bell { get; }
|
||||
public Mock<TelegramNotificationService> Telegram { get; }
|
||||
|
||||
public Harness(FeedbackEntry? current = null, bool configExists = true, bool opsEventThrows = false)
|
||||
{
|
||||
@@ -203,9 +199,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
Mock.Of<INotificationRepository>(), new FakeSignalRNotificationService(),
|
||||
Mock.Of<ILogger<GlobalNotificationService>>())
|
||||
{ CallBase = false };
|
||||
Telegram = new Mock<TelegramNotificationService>(
|
||||
new HttpClient(), configRepo, Mock.Of<ILogger<TelegramNotificationService>>())
|
||||
{ CallBase = false };
|
||||
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
configRepo, NullLogger<AdminConfigurationService>.Instance, cache)
|
||||
@@ -216,7 +209,7 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
|
||||
Sut = new FeedbackFunction(
|
||||
NullLogger<FeedbackFunction>.Instance, tenantConfig.Object, adminConfig.Object, tenantAdmins.Object,
|
||||
Telegram.Object, Mock.Of<ISessionRepository>(), Repo,
|
||||
Mock.Of<ISessionRepository>(), Repo,
|
||||
BuildOpsEventService(adminConfig.Object, opsEventThrows), Bell.Object);
|
||||
}
|
||||
|
||||
@@ -228,8 +221,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
{
|
||||
Bell.Verify(b => b.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()), Times.Never);
|
||||
Telegram.Verify(t => t.SendFeedbackAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
/// <summary>A real OpsEventService over a capturing repository, so the actual record method runs.</summary>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Config;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
@@ -55,6 +56,9 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
"enableIntegrityBypassAnalyzer",
|
||||
"enableSoftwareInventoryAnalyzer",
|
||||
"entitlements",
|
||||
// Non-sensitive by review: booleans only; the Entra code stays in the admin-gated config.
|
||||
"graphAccessAppMissing",
|
||||
"graphAccessRejected",
|
||||
"isTrial",
|
||||
"showScriptOutput",
|
||||
"trialAvailable",
|
||||
@@ -65,6 +69,54 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
}, fieldNames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_GraphAccess_SuspicionAloneShowsNothing()
|
||||
{
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
ValidateAutopilotDevice = true,
|
||||
GraphAccessRejectedSince = Now.AddMinutes(-5),
|
||||
GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode,
|
||||
});
|
||||
|
||||
Assert.False(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.False(element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(EntraTokenRejection.ServicePrincipalMissingCode, true)]
|
||||
[InlineData("AADSTS53003", false)]
|
||||
[InlineData(null, false)]
|
||||
public void Payload_GraphAccess_ConfirmedShowsTheNotice_AndNamesOnlyTheMissingApp(string? code, bool appMissing)
|
||||
{
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
ValidateAutopilotDevice = true,
|
||||
GraphAccessRejectedSince = Now.AddHours(-1),
|
||||
GraphAccessRejectedCode = code,
|
||||
GraphAccessRejectionConfirmedAt = Now.AddMinutes(-45),
|
||||
});
|
||||
|
||||
Assert.True(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.Equal(appMissing, element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
Assert.DoesNotContain("AADSTS", element.GetRawText());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_GraphAccess_StaysSilentWithoutDeviceValidation()
|
||||
{
|
||||
// The switch being off has its own banner; the agent does not depend on Graph then.
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
GraphAccessRejectedSince = Now.AddHours(-1),
|
||||
GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode,
|
||||
GraphAccessRejectionConfirmedAt = Now.AddMinutes(-45),
|
||||
});
|
||||
|
||||
Assert.False(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.False(element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_PassesThroughExplicitFieldValues()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The two stages of a lost Graph access: one refusal only stamps a suspicion, a refusal at least
|
||||
/// 15 minutes later confirms it and alerts once, a freshly issued token ends the episode. Runs on
|
||||
/// the real configuration service over a stored row, so the storage round trip of the three fields
|
||||
/// and the decision on the fresh row are part of what is proven.
|
||||
/// </summary>
|
||||
public class GraphAccessHealthRecorderTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private static readonly DateTimeOffset Start = new(2026, 11, 2, 8, 0, 0, TimeSpan.Zero);
|
||||
private static readonly EntraTokenRejection MissingApp = new(401, EntraTokenRejection.ServicePrincipalMissingCode, "missing");
|
||||
private static readonly EntraTokenRejection ConditionalAccess = new(400, "AADSTS53003", "blocked");
|
||||
private static readonly EntraTokenRejection ExpiredSecret = new(401, "AADSTS7000222", "expired");
|
||||
|
||||
private readonly Mock<IConfigRepository> _repo = new();
|
||||
private readonly StoredTenantConfig _store;
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly TestTimeProvider _time = new(Start);
|
||||
private readonly GraphAccessHealthRecorder _sut;
|
||||
|
||||
public GraphAccessHealthRecorderTests()
|
||||
{
|
||||
_store = _repo.BackWithRow(ValidatedRow());
|
||||
_sut = NewInstance();
|
||||
}
|
||||
|
||||
private static TenantConfiguration ValidatedRow()
|
||||
{
|
||||
var row = TenantConfiguration.CreateDefault(TenantId);
|
||||
row.ValidateAutopilotDevice = true;
|
||||
return row;
|
||||
}
|
||||
|
||||
/// <summary>A backend instance of its own: its own config cache over the same stored row.</summary>
|
||||
private GraphAccessHealthRecorder NewInstance()
|
||||
{
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfig = new TenantConfigurationService(_repo.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => _events.Add(e))
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
_repo.Object, NullLogger<AdminConfigurationService>.Instance, cache) { CallBase = false };
|
||||
var opsEvents = new OpsEventService(
|
||||
opsRepo.Object, NullLogger<OpsEventService>.Instance, TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
|
||||
return new GraphAccessHealthRecorder(
|
||||
tenantConfig, new Lazy<OpsEventService>(() => opsEvents), NullLogger<GraphAccessHealthRecorder>.Instance, _time);
|
||||
}
|
||||
|
||||
private void Advance(TimeSpan by) => _time.SetUtcNow(_time.GetUtcNow() + by);
|
||||
|
||||
// ── Rejected: the two stages ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task First_rejection_only_stamps_the_suspicion()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(Start.UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Equal(EntraTokenRejection.ServicePrincipalMissingCode, _store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Further_rejections_inside_the_window_write_nothing()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(TimeSpan.FromMinutes(14));
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Single(_store.Writes);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rejection_after_the_window_confirms_and_alerts_once()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, ConditionalAccess);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(TimeSpan.FromMinutes(5));
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(2, _store.Writes.Count);
|
||||
Assert.Equal(Start.UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Equal(Start.UtcDateTime + GraphAccessHealthRecorder.ConfirmAfter, _store.Row.GraphAccessRejectionConfirmedAt);
|
||||
// The code of the latest refusal counts: it selects the text and the repair.
|
||||
Assert.Equal(EntraTokenRejection.ServicePrincipalMissingCode, _store.Row.GraphAccessRejectedCode);
|
||||
var alert = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, alert.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Error, alert.Severity);
|
||||
Assert.Equal(TenantId, alert.TenantId);
|
||||
Assert.Contains(EntraTokenRejection.ServicePrincipalMissingCode, alert.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Across_instances_only_one_confirms_and_alerts()
|
||||
{
|
||||
var other = NewInstance();
|
||||
// The other instance holds a view from before the suspicion.
|
||||
await other.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await other.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Single(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_lost_write_alerts_nothing_and_the_next_rejection_decides_again()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
_store.RejectWrites = true;
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
|
||||
_store.RejectWrites = false;
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Single(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_runs_through_the_same_two_stages()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, EntraTokenRejection.NoApplicationRole);
|
||||
Assert.Equal("NoApplicationRole", _store.Row!.GraphAccessRejectedCode);
|
||||
Assert.Empty(_events);
|
||||
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, EntraTokenRejection.NoApplicationRole);
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_device_validation_stays_untouched()
|
||||
{
|
||||
_store.Row = TenantConfiguration.CreateDefault(TenantId);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_a_row_stays_without_one()
|
||||
{
|
||||
_store.Row = null;
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
// ── Our own credentials ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Our_expired_secret_never_touches_the_tenant_and_alerts_once_per_hour()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
Advance(TimeSpan.FromMinutes(59));
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
var alert = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAppCredentialRejected, alert.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Critical, alert.Severity);
|
||||
|
||||
Advance(TimeSpan.FromMinutes(2));
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
|
||||
Assert.Equal(2, _events.Count);
|
||||
}
|
||||
|
||||
// ── Minted ──────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_writes_nothing_without_a_state()
|
||||
{
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_clears_a_suspicion_silently()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_ends_a_confirmed_episode_with_one_recovery_event()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
_events.Clear();
|
||||
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
var recovery = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRecovered, recovery.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, recovery.Severity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task After_a_recovery_a_new_episode_starts_at_the_first_stage()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(_time.GetUtcNow().UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
// ── Fail-soft: the agent route must not notice ──────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task A_storage_failure_never_reaches_the_caller()
|
||||
{
|
||||
_store.ThrowOnRead = new InvalidOperationException("storage down");
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_write_never_reaches_the_caller()
|
||||
{
|
||||
_store.ThrowOnWrite = new InvalidOperationException("storage down");
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Tests.GraphResolution;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The maintenance run asks Entra for every tenant with a suspicion, so the state does not depend on
|
||||
/// device traffic. Runs the real token service and recorder; only Entra's answer is scripted.
|
||||
/// </summary>
|
||||
public class GraphAccessRecheckMaintenanceTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string OtherTenantId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string MissingAppBody = "{\"error\":\"invalid_client\",\"error_description\":\"AADSTS7000229: missing service principal\",\"error_codes\":[7000229]}";
|
||||
private const string ConditionalAccessBody = "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: blocked\",\"error_codes\":[53003]}";
|
||||
private static readonly DateTimeOffset Now = new(2026, 11, 2, 8, 0, 0, TimeSpan.Zero);
|
||||
private static readonly string TokenBody = TestGraphToken.Body(TestGraphToken.ValidationRole);
|
||||
|
||||
private readonly ScriptedEntra _entra = new();
|
||||
private readonly Mock<IConfigRepository> _repo = new();
|
||||
private readonly StoredTenantConfig _store;
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly GraphTokenService _tokens;
|
||||
private readonly GraphAccessRecheckMaintenance _sut;
|
||||
|
||||
public GraphAccessRecheckMaintenanceTests()
|
||||
{
|
||||
_store = _repo.BackWithRow(ValidatedRow());
|
||||
_repo.Setup(r => r.GetAllTenantConfigurationsAsync())
|
||||
.ReturnsAsync(() => _store.Row == null
|
||||
? new List<TenantConfiguration>()
|
||||
: new List<TenantConfiguration> { ConfigRepoMockExtensions.Clone(_store.Row) });
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfig = new TenantConfigurationService(_repo.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => _events.Add(e))
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
_repo.Object, NullLogger<AdminConfigurationService>.Instance, cache) { CallBase = false };
|
||||
var opsEvents = new OpsEventService(
|
||||
opsRepo.Object, NullLogger<OpsEventService>.Instance, TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
var recorder = new GraphAccessHealthRecorder(
|
||||
tenantConfig, new Lazy<OpsEventService>(() => opsEvents), NullLogger<GraphAccessHealthRecorder>.Instance, new TestTimeProvider(Now));
|
||||
|
||||
_tokens = NewTokenService(new SingleHandlerFactory(_entra), recorder, tenantConfig);
|
||||
_sut = new GraphAccessRecheckMaintenance(_repo.Object, _tokens, NullLogger<GraphAccessRecheckMaintenance>.Instance);
|
||||
}
|
||||
|
||||
private static GraphTokenService NewTokenService(
|
||||
IHttpClientFactory factory, IGraphAccessHealthRecorder recorder, TenantConfigurationService tenantConfig)
|
||||
{
|
||||
var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "aaaaaaaa-0000-0000-0000-000000000001",
|
||||
["EntraId:ClientSecret"] = "secret",
|
||||
}).Build();
|
||||
return new GraphTokenService(
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
new EntraAppRegistry(configuration, NullLogger<EntraAppRegistry>.Instance),
|
||||
tenantConfig,
|
||||
recorder);
|
||||
}
|
||||
|
||||
private static TenantConfiguration ValidatedRow()
|
||||
{
|
||||
var row = TenantConfiguration.CreateDefault(TenantId);
|
||||
row.ValidateAutopilotDevice = true;
|
||||
return row;
|
||||
}
|
||||
|
||||
private void Suspect(TimeSpan age, string code = EntraTokenRejection.ServicePrincipalMissingCode)
|
||||
{
|
||||
_store.Row!.GraphAccessRejectedSince = Now.UtcDateTime - age;
|
||||
_store.Row.GraphAccessRejectedCode = code;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Confirms_an_old_suspicion_without_device_traffic()
|
||||
{
|
||||
// A real 401 goes into the retry delay, which the budget cuts: the refusal must survive it.
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.Unauthorized, MissingAppBody);
|
||||
|
||||
Assert.Equal(1, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(Now.UtcDateTime, _store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_young_suspicion_stays_a_suspicion()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(10));
|
||||
_entra.Answer(HttpStatusCode.BadRequest, ConditionalAccessBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Clears_the_notice_after_a_repair_made_directly_in_Entra()
|
||||
{
|
||||
Suspect(TimeSpan.FromHours(5));
|
||||
_store.Row!.GraphAccessRejectionConfirmedAt = Now.UtcDateTime - TimeSpan.FromHours(4);
|
||||
_entra.Answer(HttpStatusCode.OK, TokenBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRecovered, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_keeps_the_notice_up()
|
||||
{
|
||||
// A user's sign-in re-created the removed enterprise application without any permission.
|
||||
Suspect(TimeSpan.FromHours(5));
|
||||
_store.Row!.GraphAccessRejectionConfirmedAt = Now.UtcDateTime - TimeSpan.FromHours(4);
|
||||
_entra.Answer(HttpStatusCode.OK, TestGraphToken.Body());
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.NotNull(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_confirms_an_old_suspicion()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.OK, TestGraphToken.Body());
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal("NoApplicationRole", _store.Row.GraphAccessRejectedCode);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cached_token_cannot_answer_the_question()
|
||||
{
|
||||
// The token this instance still holds stays valid after the enterprise app is gone.
|
||||
_entra.Answer(HttpStatusCode.OK, TokenBody);
|
||||
await _tokens.GetAccessTokenAsync(TenantId);
|
||||
Assert.Equal(1, _entra.Posts);
|
||||
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.BadRequest, ConditionalAccessBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Equal(2, _entra.Posts);
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task No_answer_from_Entra_decides_nothing()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.ServiceUnavailable, "{}");
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_a_suspicion_is_not_asked()
|
||||
{
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(0, _entra.Posts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_that_switched_device_validation_off_is_not_asked()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_store.Row!.ValidateAutopilotDevice = false;
|
||||
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(0, _entra.Posts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_read_ends_the_run_quietly()
|
||||
{
|
||||
_repo.Setup(r => r.GetAllTenantConfigurationsAsync()).ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task One_failing_tenant_does_not_stop_the_next()
|
||||
{
|
||||
// The stub throws on the unscripted first tenant, which the token service does not catch.
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
var first = ValidatedRow();
|
||||
first.GraphAccessRejectedSince = Now.UtcDateTime;
|
||||
var second = ValidatedRow();
|
||||
second.TenantId = OtherTenantId;
|
||||
second.GraphAccessRejectedSince = Now.UtcDateTime;
|
||||
repo.Setup(r => r.GetAllTenantConfigurationsAsync()).ReturnsAsync(new List<TenantConfiguration> { first, second });
|
||||
var recorder = new Mock<IGraphAccessHealthRecorder>();
|
||||
var handler = new StubHttpMessageHandler().When($"/{OtherTenantId}/oauth2", HttpStatusCode.OK, TokenBody);
|
||||
var tenantConfig = new TenantConfigurationService(
|
||||
repo.Object, NullLogger<TenantConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
var sut = new GraphAccessRecheckMaintenance(
|
||||
repo.Object, NewTokenService(new StubHttpClientFactory(handler), recorder.Object, tenantConfig), NullLogger<GraphAccessRecheckMaintenance>.Instance);
|
||||
|
||||
Assert.Equal(2, await sut.RunAsync());
|
||||
|
||||
recorder.Verify(r => r.RecordMintedAsync(OtherTenantId), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>Entra's token endpoint with an answer that can change between calls.</summary>
|
||||
private sealed class ScriptedEntra : HttpMessageHandler
|
||||
{
|
||||
private HttpStatusCode _status = HttpStatusCode.OK;
|
||||
private string _body = "{}";
|
||||
|
||||
public int Posts { get; private set; }
|
||||
|
||||
public void Answer(HttpStatusCode status, string body)
|
||||
{
|
||||
_status = status;
|
||||
_body = body;
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
Posts++;
|
||||
return Task.FromResult(new HttpResponseMessage(_status) { Content = new StringContent(_body) });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ public class GraphAuthFailureTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
return new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -342,7 +342,8 @@ public class GraphFeatureDetectorTests
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
var telemetry = new TelemetryClient(new TelemetryConfiguration { DisableTelemetry = true });
|
||||
var detector = new GraphFeatureDetector(
|
||||
tokenService, cache, NullLogger<GraphFeatureDetector>.Instance, telemetry);
|
||||
@@ -497,7 +498,8 @@ public class GraphFeatureDetectorTests
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
}
|
||||
|
||||
@@ -529,7 +531,8 @@ public class GraphFeatureDetectorTests
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
_firstToken = firstToken;
|
||||
_laterToken = laterToken;
|
||||
@@ -567,7 +570,8 @@ public class GraphFeatureDetectorTests
|
||||
factory,
|
||||
cache,
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
_onInvalidate = onInvalidate;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.IdentityModel.Tokens.Jwt;
|
||||
using System.Security.Claims;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.GraphResolution;
|
||||
|
||||
/// <summary>What Entra's token endpoint answers on success, with the application roles of the tenant's service principal.</summary>
|
||||
internal static class TestGraphToken
|
||||
{
|
||||
public const string ValidationRole = "DeviceManagementServiceConfig.Read.All";
|
||||
|
||||
public static string Body(params string[] roles)
|
||||
{
|
||||
var jwt = new JwtSecurityToken(
|
||||
issuer: "https://sts.windows.net/test",
|
||||
audience: "https://graph.microsoft.com",
|
||||
claims: roles.Select(r => new Claim("roles", r)),
|
||||
notBefore: null,
|
||||
expires: DateTime.UtcNow.AddHours(1));
|
||||
return $"{{\"access_token\":\"{new JwtSecurityTokenHandler().WriteToken(jwt)}\",\"expires_in\":3599}}";
|
||||
}
|
||||
}
|
||||
@@ -43,16 +43,21 @@ public class GraphTokenServiceTests
|
||||
/// these tests ⇒ the registry always resolves the primary credentials and the tenant-config
|
||||
/// lookup is never consulted for the decision — a null-returning repo is fine.
|
||||
/// </summary>
|
||||
private static GraphTokenService NewService(IConfiguration config, System.Net.Http.IHttpClientFactory factory) =>
|
||||
private static GraphTokenService NewService(
|
||||
IConfiguration config,
|
||||
System.Net.Http.IHttpClientFactory factory,
|
||||
IGraphAccessHealthRecorder? accessHealth = null,
|
||||
Microsoft.Extensions.Logging.ILogger<GraphTokenService>? logger = null) =>
|
||||
new GraphTokenService(
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
logger ?? NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
new EntraAppRegistry(config, NullLogger<EntraAppRegistry>.Instance),
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
accessHealth ?? Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
|
||||
private static int TokenPosts(StubHttpMessageHandler handler, string tenantId)
|
||||
=> handler.Requests.Count(u => u.Contains($"/{tenantId}/oauth2/v2.0/token"));
|
||||
@@ -240,7 +245,8 @@ public class GraphTokenServiceTests
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
|
||||
var probe1 = await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
var probe2 = await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
@@ -260,6 +266,195 @@ public class GraphTokenServiceTests
|
||||
Assert.Equal(3, TokenPosts(handler, TenantA));
|
||||
}
|
||||
|
||||
// ── Entra's refusal ─────────────────────────────────────────────────────
|
||||
|
||||
private const string MissingAppBody = "{\"error\":\"invalid_client\",\"error_description\":\"AADSTS7000229: missing service principal\",\"error_codes\":[7000229]}";
|
||||
private const string ConditionalAccessBody = "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: blocked\",\"error_codes\":[53003]}";
|
||||
|
||||
private static GraphTokenService ServiceFor(
|
||||
StubHttpMessageHandler handler,
|
||||
Moq.Mock<IGraphAccessHealthRecorder> accessHealth,
|
||||
Microsoft.Extensions.Logging.ILogger<GraphTokenService>? logger = null)
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "test-client",
|
||||
["EntraId:ClientSecret"] = "test-secret",
|
||||
})
|
||||
.Build();
|
||||
return NewService(config, new StubHttpClientFactory(handler), accessHealth.Object, logger);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rejection_survives_a_budget_that_ends_the_retries()
|
||||
{
|
||||
// The validators run on 8 s and the recheck on 3 s; the chain's first retry waits 5 s.
|
||||
var handler = new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.Unauthorized, MissingAppBody);
|
||||
var (svc, _) = Create(handler);
|
||||
using var budget = new System.Threading.CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
var result = await svc.GetAccessTokenAsync(TenantA, budget.Token);
|
||||
|
||||
Assert.Null(result.AccessToken);
|
||||
Assert.True(result.IsTransient);
|
||||
Assert.NotNull(result.Rejection);
|
||||
Assert.Equal(401, result.Rejection!.StatusCode);
|
||||
Assert.True(result.Rejection.IsServicePrincipalMissing);
|
||||
Assert.Equal(1, TokenPosts(handler, TenantA));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.ServiceUnavailable)]
|
||||
[InlineData(HttpStatusCode.TooManyRequests)]
|
||||
[InlineData(HttpStatusCode.RequestTimeout)]
|
||||
public async Task A_server_error_or_throttle_is_no_rejection(HttpStatusCode status)
|
||||
{
|
||||
var handler = new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", status, MissingAppBody);
|
||||
var (svc, _) = Create(handler);
|
||||
using var budget = new System.Threading.CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
|
||||
var result = await svc.GetAccessTokenAsync(TenantA, budget.Token);
|
||||
|
||||
Assert.True(result.IsTransient);
|
||||
Assert.Null(result.Rejection);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_reports_a_rejection()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody), health);
|
||||
|
||||
var result = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.False(result.IsTransient);
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, Moq.It.Is<EntraTokenRejection>(r => r.ErrorCode == "AADSTS53003")), Moq.Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_portal_path_does_not_report_a_rejection()
|
||||
{
|
||||
// The consent flows see refusals as a matter of course while a consent propagates.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody), health);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Only_a_fresh_token_counts_as_proof_of_access()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body(TestGraphToken.ValidationRole)), health);
|
||||
|
||||
var minted = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
var cached = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.True(minted.IsFreshlyMinted);
|
||||
Assert.False(cached.IsFreshlyMinted);
|
||||
health.Verify(h => h.RecordMintedAsync(TenantA), Moq.Times.Once);
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_is_no_proof_of_access()
|
||||
{
|
||||
// A sign-in consent re-created the removed enterprise application without any permission.
|
||||
// The notice must survive it: this is the moment an admin signs in to read it.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordMintedAsync(Moq.It.IsAny<string>()), Moq.Times.Never);
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_counts_a_token_without_a_role_as_a_refusal()
|
||||
{
|
||||
// Graph refuses every call made with it, so the agent is locked out all the same.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
var result = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.NotNull(result.AccessToken);
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, EntraTokenRejection.NoApplicationRole), Moq.Times.Once);
|
||||
health.Verify(h => h.RecordMintedAsync(Moq.It.IsAny<string>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_does_not_judge_a_token_from_the_cache()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, EntraTokenRejection.NoApplicationRole), Moq.Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_probe_under_explicit_credentials_reports_nothing()
|
||||
{
|
||||
// A token of the app the tenant is not homed on says nothing about the tenant's access.
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "test-client",
|
||||
["EntraId:ClientSecret"] = "test-secret",
|
||||
})
|
||||
.Build();
|
||||
var registry = new EntraAppRegistry(config, NullLogger<EntraAppRegistry>.Instance);
|
||||
|
||||
foreach (var (status, body) in new[]
|
||||
{
|
||||
(HttpStatusCode.OK, TestGraphToken.Body(TestGraphToken.ValidationRole)),
|
||||
(HttpStatusCode.OK, TestGraphToken.Body()),
|
||||
(HttpStatusCode.BadRequest, ConditionalAccessBody),
|
||||
})
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = NewService(config, new StubHttpClientFactory(new StubHttpMessageHandler().When($"/{TenantA}/oauth2", status, body)), health.Object);
|
||||
|
||||
await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
|
||||
health.VerifyNoOtherCalls();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Entra_refusal_is_logged_once_per_window()
|
||||
{
|
||||
var logger = new Moq.Mock<Microsoft.Extensions.Logging.ILogger<GraphTokenService>>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody),
|
||||
new Moq.Mock<IGraphAccessHealthRecorder>(), logger.Object);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
logger.Verify(l => l.Log(
|
||||
Microsoft.Extensions.Logging.LogLevel.Warning,
|
||||
Moq.It.IsAny<Microsoft.Extensions.Logging.EventId>(),
|
||||
Moq.It.Is<Moq.It.IsAnyType>((state, _) => state.ToString()!.Contains("rejected by Entra") && state.ToString()!.Contains("AADSTS53003")),
|
||||
Moq.It.IsAny<Exception?>(),
|
||||
Moq.It.IsAny<Func<Moq.It.IsAnyType, Exception?, string>>()), Moq.Times.Once);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
|
||||
@@ -79,7 +79,7 @@ public class IntuneEnrollmentValidationTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokens = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokens.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using Xunit;
|
||||
@@ -7,6 +8,18 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class MaintenanceStepTimerTests
|
||||
{
|
||||
private const long Mb = 1024 * 1024;
|
||||
|
||||
private static MaintenanceStepTimer TimerOver(params (long PeakMb, long AllocatedMb)[] readings)
|
||||
{
|
||||
var queue = new Queue<(long PeakMb, long AllocatedMb)>(readings);
|
||||
return new MaintenanceStepTimer(() =>
|
||||
{
|
||||
var (peakMb, allocatedMb) = queue.Dequeue();
|
||||
return new MaintenanceMemoryReading(peakMb * Mb, allocatedMb * Mb, 1740 * Mb, 2088 * Mb);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Records_every_step_in_order_and_returns_the_body_result()
|
||||
{
|
||||
@@ -30,4 +43,49 @@ public class MaintenanceStepTimerTests
|
||||
|
||||
Assert.True(timer.Steps.ContainsKey("boom"));
|
||||
}
|
||||
|
||||
// The memory columns have to name the step that set the peak: each step records the process
|
||||
// peak after it ran and only the allocation of its own span.
|
||||
[Fact]
|
||||
public async Task Each_step_reports_the_process_peak_after_it_and_its_own_allocation()
|
||||
{
|
||||
var timer = TimerOver((400, 100), (400, 150), (400, 150), (1900, 1450), (1900, 1450));
|
||||
|
||||
await timer.RunAsync("light", () => Task.CompletedTask);
|
||||
await timer.RunAsync<int>("heavy", () => Task.FromResult(7));
|
||||
|
||||
var report = timer.MemoryReport();
|
||||
Assert.Equal(400, report.StepsPeakMb["light"]);
|
||||
Assert.Equal(50, report.StepsAllocMb["light"]);
|
||||
Assert.Equal(1900, report.StepsPeakMb["heavy"]);
|
||||
Assert.Equal(1300, report.StepsAllocMb["heavy"]);
|
||||
Assert.Equal(1740, report.GcHeapLimitMb);
|
||||
Assert.Equal(2088, report.GcHighLoadMb);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_throwing_step_still_reports_its_memory()
|
||||
{
|
||||
var timer = TimerOver((300, 10), (800, 60), (800, 60));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => timer.RunAsync("boom", () => throw new InvalidOperationException("x")));
|
||||
|
||||
var report = timer.MemoryReport();
|
||||
Assert.Equal(800, report.StepsPeakMb["boom"]);
|
||||
Assert.Equal(50, report.StepsAllocMb["boom"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_real_reading_sees_the_process_peak_and_the_gc_limits()
|
||||
{
|
||||
GC.Collect();
|
||||
|
||||
var reading = MaintenanceMemoryReading.Capture();
|
||||
|
||||
Assert.True(reading.PeakWorkingSetBytes > 0);
|
||||
Assert.True(reading.AllocatedBytes > 0);
|
||||
Assert.True(reading.GcHeapLimitBytes > 0);
|
||||
Assert.True(reading.GcHighLoadBytes > 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -435,10 +435,10 @@ public class NotificationChannelHealthTests
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(webhookResult);
|
||||
}
|
||||
var telegram = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance) { CallBase = false };
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>())).ReturnsAsync(Ok());
|
||||
return new NotificationChannelDispatcher(webhook.Object, telegram.Object, (push ?? new Mock<IPushChannelSender>()).Object,
|
||||
Mock.Of<IEmailChannelSender>(), recorder.Object, NullLogger<NotificationChannelDispatcher>.Instance);
|
||||
var email = new Mock<IEmailChannelSender>();
|
||||
email.Setup(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>())).ReturnsAsync(Ok());
|
||||
return new NotificationChannelDispatcher(webhook.Object, (push ?? new Mock<IPushChannelSender>()).Object,
|
||||
email.Object, recorder.Object, NullLogger<NotificationChannelDispatcher>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -447,13 +447,13 @@ public class NotificationChannelHealthTests
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = Dispatcher(recorder, push: push);
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
var mail = new NotificationChannel { Id = "mail", Name = "mail", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
var phones = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Push, Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram, phones }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), mail, phones }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"), It.Is<NotificationSendResult>(x => !x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.Is<NotificationSendResult>(x => x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "mail"), It.Is<NotificationSendResult>(x => x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(It.IsAny<NotificationScope>(), It.Is<NotificationChannel>(c => c.Id == "push"), It.IsAny<NotificationSendResult>(), It.IsAny<ChannelSendKind>()), Times.Never);
|
||||
push.Verify(p => p.SendAsync(Tenant, It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
@@ -474,13 +474,13 @@ public class NotificationChannelHealthTests
|
||||
{
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var dispatcher = Dispatcher(recorder, webhookResult: () => throw new InvalidOperationException("boom"));
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
var mail = new NotificationChannel { Id = "mail", Name = "mail", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), mail }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"),
|
||||
It.Is<NotificationSendResult>(x => !x.Success && x.Message == "Unexpected error: boom"), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.IsAny<NotificationSendResult>(), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "mail"), It.IsAny<NotificationSendResult>(), ChannelSendKind.Delivery), Times.Once);
|
||||
}
|
||||
|
||||
// ── Read endpoint projection ──────────────────────────────────────────
|
||||
|
||||
@@ -36,6 +36,7 @@ public class NotificationChannelsTests
|
||||
"{\"name\":\"no-id\",\"providerType\":20,\"url\":\"https://x.example\"}",
|
||||
ChannelJson("none-provider", providerType: 0),
|
||||
ChannelJson("retired-teams-connector", providerType: 1),
|
||||
ChannelJson("retired-provider", providerType: 40),
|
||||
ChannelJson("unknown-provider", providerType: 99)) + "]";
|
||||
|
||||
var parsed = NotificationChannel.ParseList(json);
|
||||
@@ -153,6 +154,20 @@ public class NotificationChannelsTests
|
||||
Assert.Null(redacted[1].SigningSecret);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RedactedCopy_MasksTheRecipientsOfAnEmailChannel()
|
||||
{
|
||||
var cfg = new TenantConfiguration
|
||||
{
|
||||
NotificationChannelsJson = "[" + ChannelJson("mail", providerType: 60, url: "ops@example.invalid") + "]",
|
||||
};
|
||||
|
||||
var redacted = NotificationChannel.ParseList(cfg.RedactedCopyForReader().NotificationChannelsJson);
|
||||
|
||||
Assert.Equal("mail", Assert.Single(redacted).Id);
|
||||
Assert.Equal(Constants.RedactedSecretPlaceholder, redacted[0].Url);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RedactedCopy_UnparseableChannelsJson_FallsBackToWholeStringRedaction()
|
||||
{
|
||||
@@ -218,6 +233,7 @@ public class NotificationChannelsTests
|
||||
[InlineData("[{\"name\":\"no-id\",\"providerType\":20}]", "needs an id")]
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":99}]", "invalid provider type")]
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":1}]", "invalid provider type")] // retired Teams Office 365 Connector
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":40}]", "invalid provider type")] // retired, never reused
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":20,\"url\":\"http://plain.example\"}]", "")] // non-https rejected by SsrfGuard format check
|
||||
public void ValidateNotificationChannels_RejectsInvalidEntries(string json, string expectedFragment)
|
||||
{
|
||||
@@ -238,6 +254,7 @@ public class NotificationChannelsTests
|
||||
|
||||
[Theory]
|
||||
[InlineData(1, false)] // retired Teams Office 365 Connector
|
||||
[InlineData(40, false)] // retired, never reused
|
||||
[InlineData(99, false)]
|
||||
[InlineData(0, true)]
|
||||
[InlineData(2, true)]
|
||||
@@ -257,8 +274,6 @@ public class NotificationChannelsTests
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(40, false)] // Telegram
|
||||
[InlineData(40, true)]
|
||||
[InlineData(50, false)] // Push
|
||||
[InlineData(50, true)]
|
||||
[InlineData(60, false)] // E-mail
|
||||
@@ -275,7 +290,6 @@ public class NotificationChannelsTests
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(40)]
|
||||
[InlineData(50)]
|
||||
[InlineData(60)]
|
||||
public void ValidateModel_LegacyWebhook_KeepsAnUnchangedStoredNonWebhookProviderSaveable(int providerType)
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event payload delivery: the structured Details object reaches outbound channels, both as
|
||||
/// readable facts (card + Telegram formats) and verbatim under "data" (generic JSON consumers),
|
||||
/// readable facts (card + e-mail formats) and verbatim under "data" (generic JSON consumers),
|
||||
/// and only the channels whose rule opted in get it — on the platform scope, like every ops send.
|
||||
/// Before this, an alert carried only category/event/severity and a tenant GUID.
|
||||
/// </summary>
|
||||
@@ -117,10 +117,10 @@ public class OpsAlertPayloadTests
|
||||
=> Assert.Equal("1.0", JObject.Parse(new GenericJsonRenderer().RenderToJson(
|
||||
new NotificationAlert { Title = "t", Summary = "s", DataJson = """{"a":1}""" }))["schemaVersion"]!.Value<string>());
|
||||
|
||||
// ── Telegram text carries the same information ────────────────────────
|
||||
// ── E-mail text carries the same information ──────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TelegramRendering_IncludesFlattenedFacts()
|
||||
public void EmailRendering_IncludesFlattenedFacts()
|
||||
{
|
||||
var alert = new NotificationAlert
|
||||
{
|
||||
@@ -130,7 +130,7 @@ public class OpsAlertPayloadTests
|
||||
Facts = OpsAlertDispatchService.FlattenDetails("""{"domainName":"contoso.example"}"""),
|
||||
};
|
||||
|
||||
var text = TelegramNotificationService.RenderAlertText(alert);
|
||||
var text = EmailAlertRenderer.Text(alert);
|
||||
|
||||
Assert.Contains("Domain Name: contoso.example", text);
|
||||
Assert.Contains("Pro trial started", text);
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The ops events behind a customer submission are the only signal an alert channel gets. They
|
||||
/// name ids and the portal path, never the submitter or a comment; the submitter stays in the
|
||||
/// row's user id, which the dispatch never forwards.
|
||||
/// </summary>
|
||||
public class OpsEventSubmissionSignalsTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string Submitter = "someone@contoso.invalid";
|
||||
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly OpsEventService _sut;
|
||||
|
||||
public OpsEventSubmissionSignalsTests()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(_events.Add)
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()))
|
||||
{ CallBase = false };
|
||||
|
||||
_sut = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
}
|
||||
|
||||
private OpsEventEntry AssertSingleSignal(string eventType, string portalPath)
|
||||
{
|
||||
var entry = Assert.Single(_events);
|
||||
Assert.Equal(eventType, entry.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, entry.Severity);
|
||||
Assert.Equal(TenantId, entry.TenantId);
|
||||
Assert.Contains(portalPath, entry.Details!);
|
||||
|
||||
Assert.Equal(Submitter, entry.UserId);
|
||||
Assert.DoesNotContain(Submitter, entry.Message);
|
||||
Assert.DoesNotContain(Submitter, entry.Details!);
|
||||
return entry;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantSignup_names_the_tenant_page()
|
||||
{
|
||||
await _sut.RecordTenantSignupAsync(TenantId, Submitter);
|
||||
|
||||
AssertSingleSignal(OpsEventTypes.TenantSignup, $"/admin/tenants/management?tenantId={TenantId}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionReport_names_the_session_and_the_report_only()
|
||||
{
|
||||
await _sut.RecordSessionReportSubmittedAsync(TenantId, Submitter, "session-1", "report-1");
|
||||
|
||||
var entry = AssertSingleSignal(OpsEventTypes.SessionReportSubmitted, "/admin/reports/session-reports?reportId=report-1");
|
||||
Assert.Contains("session-1", entry.Details!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiagFilesReport_names_the_report_only()
|
||||
{
|
||||
await _sut.RecordDiagFilesReportSubmittedAsync(TenantId, Submitter, "report-2");
|
||||
|
||||
AssertSingleSignal(OpsEventTypes.DiagFilesReportSubmitted, "/admin/reports/session-reports?reportId=report-2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RuleSubmission_names_the_submission_and_the_rule_only()
|
||||
{
|
||||
await _sut.RecordRuleSubmissionReceivedAsync(TenantId, Submitter, "abc123def456", "analyze", "ANALYZE-X-1");
|
||||
|
||||
var entry = AssertSingleSignal(OpsEventTypes.RuleSubmissionReceived, "/admin/reports/rule-submissions?submissionId=abc123def456");
|
||||
Assert.Contains("ANALYZE-X-1", entry.Details!);
|
||||
}
|
||||
}
|
||||
@@ -12,28 +12,12 @@ using Moq;
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Platform (ops) notification channels: legacy-slot synthesis so the pre-channels dispatch
|
||||
/// behavior survives the migration untouched, per-rule channel routing, the platform dispatch
|
||||
/// scope (an ops alert's tenant is its subject, never the channel owner), and redaction of the
|
||||
/// new secret-bearing field.
|
||||
/// Platform (ops) notification channels: the stored list, per-rule channel routing, the platform
|
||||
/// dispatch scope (an ops alert's tenant is its subject, never the channel owner), and redaction of
|
||||
/// the secret-bearing field.
|
||||
/// </summary>
|
||||
public class OpsNotificationChannelsTests
|
||||
{
|
||||
private static AdminConfiguration LegacyConfig(
|
||||
bool telegramEnabled = true, string telegramChat = "-1003785642894",
|
||||
bool teamsEnabled = true, string teamsUrl = "https://teams.example/hook",
|
||||
bool slackEnabled = false, string slackUrl = "")
|
||||
=> new()
|
||||
{
|
||||
UpdatedBy = "test",
|
||||
OpsAlertTelegramEnabled = telegramEnabled,
|
||||
OpsAlertTelegramChatId = telegramChat,
|
||||
OpsAlertTeamsEnabled = teamsEnabled,
|
||||
OpsAlertTeamsWebhookUrl = teamsUrl,
|
||||
OpsAlertSlackEnabled = slackEnabled,
|
||||
OpsAlertSlackWebhookUrl = slackUrl,
|
||||
};
|
||||
|
||||
private static NotificationChannel Channel(string id, bool enabled = true, int providerType = 20)
|
||||
=> new() { Id = id, Name = id, ProviderType = providerType, Url = "https://x.example/" + id, Enabled = enabled };
|
||||
|
||||
@@ -61,44 +45,20 @@ public class OpsNotificationChannelsTests
|
||||
private static IEnumerable<string> WithPayload(IReadOnlyList<OpsAlertRule> rules, IReadOnlyList<NotificationChannel> channels)
|
||||
=> OpsAlertDispatchService.ResolveTargets(rules, channels).WithPayload.Select(c => c.Id);
|
||||
|
||||
// ── Legacy synthesis ──────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_SynthesizesConfiguredLegacySlots()
|
||||
{
|
||||
var channels = LegacyConfig().GetOpsNotificationChannels();
|
||||
|
||||
Assert.Equal(2, channels.Count); // Slack slot is empty → not synthesized
|
||||
var telegram = channels[0];
|
||||
Assert.Equal(AdminConfiguration.LegacyTelegramChannelId, telegram.Id);
|
||||
Assert.Equal(AdminConfiguration.LegacyTelegramChannelName, telegram.Name);
|
||||
Assert.Equal((int)WebhookProviderType.Telegram, telegram.ProviderType);
|
||||
Assert.Equal("-1003785642894", telegram.Url);
|
||||
Assert.True(telegram.Enabled);
|
||||
|
||||
Assert.Equal(AdminConfiguration.LegacyTeamsChannelId, channels[1].Id);
|
||||
Assert.Equal((int)WebhookProviderType.TeamsWorkflowWebhook, channels[1].ProviderType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_CarriesLegacyEnabledFlags()
|
||||
{
|
||||
// A configured-but-disabled provider must stay disabled, or the migration would start
|
||||
// sending to a destination the operator had deliberately switched off.
|
||||
var channels = LegacyConfig(telegramEnabled: false).GetOpsNotificationChannels();
|
||||
|
||||
Assert.False(channels.Single(c => c.Id == AdminConfiguration.LegacyTelegramChannelId).Enabled);
|
||||
}
|
||||
// ── Stored list ───────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_EmptyWhenNothingConfigured()
|
||||
=> Assert.Empty(new AdminConfiguration { UpdatedBy = "t" }.GetOpsNotificationChannels());
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_StoredListWinsOverLegacySlots()
|
||||
public void GetOpsNotificationChannels_ReturnsTheStoredList()
|
||||
{
|
||||
var config = LegacyConfig();
|
||||
config.OpsNotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("sales") });
|
||||
var config = new AdminConfiguration
|
||||
{
|
||||
UpdatedBy = "t",
|
||||
OpsNotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("sales") }),
|
||||
};
|
||||
|
||||
var channels = config.GetOpsNotificationChannels();
|
||||
|
||||
|
||||
@@ -46,6 +46,25 @@ public class OrphanSweepOpsEventTests
|
||||
Assert.Contains("\"aggregateMetrics\":90000", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MaintenanceCompleted_carries_the_step_memory_and_the_gc_limits()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var memory = new MaintenanceMemoryReport(
|
||||
new Dictionary<string, long> { ["appVersionRadar"] = 1900 },
|
||||
new Dictionary<string, long> { ["appVersionRadar"] = 1300 },
|
||||
GcHeapLimitMb: 1740,
|
||||
GcHighLoadMb: 2088);
|
||||
|
||||
await service.RecordMaintenanceCompletedAsync(123456, "Timer", memory: memory);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Contains("\"stepsPeakMb\":{\"appVersionRadar\":1900}", evt.Details);
|
||||
Assert.Contains("\"stepsAllocMb\":{\"appVersionRadar\":1300}", evt.Details);
|
||||
Assert.Contains("\"gcHeapLimitMb\":1740", evt.Details);
|
||||
Assert.Contains("\"gcHighLoadMb\":2088", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Functions.Telemetry;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The memory metric is a percentage of the limits the GC works against, so a reading keeps its
|
||||
/// meaning when the instance size changes, and an alert can name one number for every size.
|
||||
/// </summary>
|
||||
public class ProcessMemoryPressureTests
|
||||
{
|
||||
[Fact]
|
||||
public void Heap_is_committed_memory_against_the_heap_limit()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { CommittedBytes = 1305, TotalAvailableMemoryBytes = 1740 };
|
||||
|
||||
Assert.Equal(75.0, ProcessMemoryPressure.HeapPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Load_is_memory_load_against_the_high_load_threshold()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { MemoryLoadBytes = 2088, HighMemoryLoadThresholdBytes = 2088 };
|
||||
|
||||
Assert.Equal(100.0, ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_value_past_the_limit_is_not_capped()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { MemoryLoadBytes = 2320, HighMemoryLoadThresholdBytes = 2088 };
|
||||
|
||||
Assert.Equal(111.1, ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Before_the_first_gc_there_is_no_reading()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { WorkingSetBytes = 500 };
|
||||
|
||||
Assert.Null(ProcessMemoryPressure.HeapPercent(snapshot));
|
||||
Assert.Null(ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_captured_snapshot_after_a_gc_has_both_readings()
|
||||
{
|
||||
GC.Collect();
|
||||
|
||||
var snapshot = ProcessSnapshot.Capture();
|
||||
|
||||
Assert.True(ProcessMemoryPressure.HeapPercent(snapshot) > 0);
|
||||
Assert.True(ProcessMemoryPressure.LoadPercent(snapshot) > 0);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public class ProcessStallClassifierTests
|
||||
PendingWorkItems: pending,
|
||||
CompletedWorkItems: completed,
|
||||
WorkingSetBytes: 0, HeapSizeBytes: 0, CommittedBytes: 0,
|
||||
TotalAvailableMemoryBytes: 0, MemoryLoadBytes: 0, HighMemoryLoadThresholdBytes: 0,
|
||||
LastGcGeneration: 0, LastGcPause: TimeSpan.Zero, LastGcCompacted: false);
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -17,7 +17,7 @@ namespace AutopilotMonitor.Functions.Tests.Push;
|
||||
/// <summary>
|
||||
/// Push as a channel provider: the dispatcher's Push branch (scope passed through, never the
|
||||
/// webhook path, one fan-out per send), the validation rules (no destination, open to tenant
|
||||
/// admins — only Telegram and E-mail stay behind the GA gate — and one Push channel per scope),
|
||||
/// admins — only E-mail stays behind the GA gate — and one Push channel per scope),
|
||||
/// the raw-table deny-list and the dependency redaction.
|
||||
/// </summary>
|
||||
public class PushChannelRoutingTests
|
||||
@@ -35,15 +35,6 @@ public class PushChannelRoutingTests
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel PushChannel(string id = "push", bool enabled = true) =>
|
||||
new() { Id = id, Name = id, ProviderType = (int)WebhookProviderType.Push, Url = null, Enabled = enabled };
|
||||
|
||||
@@ -56,9 +47,8 @@ public class PushChannelRoutingTests
|
||||
public async Task Push_channel_reaches_the_push_sender_with_the_callers_scope_and_never_the_webhook_path()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, push.Object);
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { PushChannel() }, alert, Tenant);
|
||||
@@ -66,7 +56,6 @@ public class PushChannelRoutingTests
|
||||
push.Verify(p => p.SendAsync(Tenant, alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -75,7 +64,7 @@ public class PushChannelRoutingTests
|
||||
// A pair stored before validation allowed only one: each would reach every paired device.
|
||||
var webhook = WebhookMock();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, TelegramMock().Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, push.Object);
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
var slack = new NotificationChannel { Id = "slack", Name = "slack", ProviderType = (int)WebhookProviderType.Slack, Url = "https://hooks.slack.example/x", Enabled = true };
|
||||
|
||||
@@ -94,7 +83,7 @@ public class PushChannelRoutingTests
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
push.Setup(p => p.SendWithResultAsync(It.IsAny<NotificationScope>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, push.Object);
|
||||
|
||||
var result = await dispatcher.SendTestAsync(PushChannel(), new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
@@ -109,7 +98,6 @@ public class PushChannelRoutingTests
|
||||
Assert.False(new NotificationChannel { Id = "s", ProviderType = (int)WebhookProviderType.Slack, Url = "" }.HasDestination());
|
||||
Assert.True(new NotificationChannel { Id = "s", ProviderType = (int)WebhookProviderType.Slack, Url = "https://hooks.slack.example/x" }.HasDestination());
|
||||
Assert.False(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Push));
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Telegram));
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Email));
|
||||
Assert.False(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Slack));
|
||||
}
|
||||
@@ -127,23 +115,23 @@ public class PushChannelRoutingTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_leaves_a_push_channel_to_the_tenant_admin_but_not_its_change_into_telegram()
|
||||
public void Gate_leaves_a_push_channel_to_the_tenant_admin_but_not_its_change_into_email()
|
||||
{
|
||||
var json = Json(PushChannel());
|
||||
|
||||
// Add, disable, rename and toggle: all the tenant admin's own.
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(json, existingJson: null, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(PushChannel(enabled: false)), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(json, existingJson: null, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(PushChannel(enabled: false)), json, isGlobalAdmin: false));
|
||||
var renamed = PushChannel();
|
||||
renamed.Name = "Phones";
|
||||
renamed.NotifyOnFailure = true;
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(renamed), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(renamed), json, isGlobalAdmin: false));
|
||||
|
||||
// The same channel id turned into Telegram still needs a Global Administrator.
|
||||
var retyped = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
Assert.Equal("Telegram channels can only be configured by a Global Administrator.",
|
||||
TenantConfigValidation.ValidateTelegramChannelGate(Json(retyped), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(retyped), json, isGlobalAdmin: true));
|
||||
// The same channel id turned into E-mail still needs a Global Administrator.
|
||||
var retyped = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
Assert.Equal("E-mail channels can only be configured by a Global Administrator.",
|
||||
TenantConfigValidation.ValidatePlatformProviderGate(Json(retyped), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(retyped), json, isGlobalAdmin: true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -72,7 +72,7 @@ public class SessionDeletionWorkerTests
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// PR-B audit consolidation: deletion_poisoned audit moved to SessionDeletionPoisoned
|
||||
// OpsEvent (operator-bound, Telegram-routable). Tenant-audit row is no longer written.
|
||||
// OpsEvent (operator-bound, routable through alert rules). Tenant-audit row is no longer written.
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Equal(TenantId, poisoned.TenantId);
|
||||
Assert.Equal(SessionId, poisoned.SessionId);
|
||||
|
||||
@@ -1013,7 +1013,7 @@ public class SlaTenantStatusLifecycleTests
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var channelDispatcher = new NotificationChannelDispatcher(webhook, new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance), Mock.Of<IPushChannelSender>());
|
||||
var channelDispatcher = new NotificationChannelDispatcher(webhook, Mock.Of<IPushChannelSender>());
|
||||
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
|
||||
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ public sealed class SubmitGeneralFeedbackFunctionTests
|
||||
[Fact]
|
||||
public async Task TheAlert_CarriesNeitherTextNorSenderNorContactAddress()
|
||||
{
|
||||
// Channels bound to FeedbackReceived (Telegram, Slack, webhooks) receive message + details;
|
||||
// Channels bound to FeedbackReceived (Slack, webhooks, e-mail) receive message + details;
|
||||
// the feedback itself is read in the portal. Personal data must never ride along.
|
||||
var h = new Harness();
|
||||
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ public sealed class SubmitOffboardingFeedbackFunctionTests
|
||||
Assert.Equal("Pricing was too high for our use case.", h.Repo.PersistedOffboardingFeedback.Comment);
|
||||
Assert.NotNull(h.Repo.PersistedOffboardingFeedback.InteractedAt);
|
||||
|
||||
// OpsEvent emitted with the canonical type catalog-registered for Telegram-routing.
|
||||
// OpsEvent emitted with the canonical type catalog-registered for alert routing.
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal("OffboardingFeedbackReceived", ops.EventType);
|
||||
Assert.Equal(TenantId, ops.TenantId);
|
||||
@@ -283,7 +283,7 @@ public sealed class SubmitOffboardingFeedbackFunctionTests
|
||||
|
||||
// Alert-dispatch dependencies are inert: empty AdminConfig service + bare HTTP
|
||||
// clients. The feedback path emits Info-severity so even if alerts ARE evaluated,
|
||||
// no Telegram/webhook fires for an unconfigured rule set.
|
||||
// no channel fires for an unconfigured rule set.
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(),
|
||||
|
||||
@@ -45,7 +45,7 @@ public class TableLifecycleBucketTests
|
||||
[Constants.TableNames.GlobalAdmins] = "platform operator identities",
|
||||
[Constants.TableNames.McpUsers] = "platform MCP identities (PK='McpUsers', RK=UPN)",
|
||||
[Constants.TableNames.TenantGroupAssignments] = "per-UPN rows without tenant anchor; offboarding never touches it (see Constants doc)",
|
||||
[Constants.TableNames.PreviewConfig] = "global TelegramBot config row",
|
||||
[Constants.TableNames.PreviewConfig] = "global e-mail template overrides",
|
||||
[Constants.TableNames.Feedback] = "product feedback deliberately survives offboarding (TableFeedbackRepositoryTests contract)",
|
||||
[Constants.TableNames.BlockedVersions] = "platform-wide agent version blocks",
|
||||
[Constants.TableNames.GlobalNotifications] = "GA notifications; time-retention only",
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Telegram as a notification-channel provider: dispatch routing (it must never reach the
|
||||
/// SSRF-guarded webhook path), chat-ID validation, and the Global-Admin gate that keeps a
|
||||
/// tenant admin from sending through the platform-owned bot.
|
||||
/// </summary>
|
||||
public class TelegramChannelProviderTests
|
||||
{
|
||||
private static Mock<WebhookNotificationService> WebhookMock()
|
||||
{
|
||||
var mock = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
mock.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel Channel(int providerType, string? url, string id = "c1", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = providerType, Url = url, Enabled = enabled };
|
||||
|
||||
private static string ChannelJson(int providerType, string url, string id = "c1", bool enabled = true)
|
||||
=> $"[{{\"id\":\"{id}\",\"name\":\"{id}\",\"providerType\":{providerType},\"url\":\"{url}\",\"enabled\":{(enabled ? "true" : "false")}}}]";
|
||||
|
||||
// ── Dispatch routing ──────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_RoutesTelegramToBotAndNeverToWebhook()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Telegram, "-1003785642894") }, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync("-1003785642894", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_RoutesWebhookProvidersToWebhookService()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Slack, "https://hooks.slack.example/x") }, alert, NotificationScope.Platform);
|
||||
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://hooks.slack.example/x", WebhookProviderType.Slack, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_MixedList_EachChannelReachesItsOwnTransport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel((int)WebhookProviderType.Telegram, "-100123", "tg"),
|
||||
Channel((int)WebhookProviderType.GenericJson, "https://sales.example/hook", "sales"),
|
||||
}, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync("-100123", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://sales.example/hook", WebhookProviderType.GenericJson, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_SkipsChannelsWithoutDestination()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel((int)WebhookProviderType.Telegram, null, "tg"),
|
||||
Channel((int)WebhookProviderType.Slack, "", "slack"),
|
||||
}, new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendTest_RoutesTelegramToBot()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
var result = await dispatcher.SendTestAsync(
|
||||
Channel((int)WebhookProviderType.Telegram, "@salesdesk"),
|
||||
new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
Assert.True(result.Success);
|
||||
telegram.Verify(t => t.SendAlertAsync("@salesdesk", It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
|
||||
// ── Chat-ID validation ────────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("-1003785642894")]
|
||||
[InlineData("123456789")]
|
||||
[InlineData("@salesdesk")]
|
||||
[InlineData("@Sales_Desk_01")]
|
||||
public void ValidateTelegramChatId_AcceptsNumericAndUsernameForms(string chatId)
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChatId(chatId));
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("https://api.telegram.org/bot123/sendMessage")]
|
||||
[InlineData("-100abc")]
|
||||
[InlineData("@abc")] // too short
|
||||
[InlineData("@1sales")] // must start with a letter
|
||||
[InlineData("@sales-desk")] // hyphen not allowed
|
||||
[InlineData("123456789012345678901")] // 21 digits
|
||||
public void ValidateTelegramChatId_RejectsEverythingElse(string? chatId)
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateTelegramChatId(chatId));
|
||||
|
||||
[Fact]
|
||||
public void ValidateNotificationChannels_TelegramChatIdBypassesSsrfUrlGate()
|
||||
{
|
||||
// A chat ID is not a URL — the webhook SSRF gate would reject it. Proves the
|
||||
// provider-specific destination branch is wired, not just the format helper.
|
||||
Assert.Null(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-1003785642894")));
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Slack, "-1003785642894")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateNotificationChannels_RejectsTelegramChannelWithUrlDestination()
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "https://hooks.example/x")));
|
||||
|
||||
// ── Global-Admin gate ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_GlobalAdminMayAddChannel()
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-100123"), null, isGlobalAdmin: true));
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotAddChannel()
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-100123"), null, isGlobalAdmin: false));
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotRetargetExistingChannel()
|
||||
{
|
||||
var existing = ChannelJson((int)WebhookProviderType.Telegram, "-100123");
|
||||
var retargeted = ChannelJson((int)WebhookProviderType.Telegram, "-100999");
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(retargeted, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotFlipEnabledState()
|
||||
{
|
||||
var existing = ChannelJson((int)WebhookProviderType.Telegram, "-100123", enabled: true);
|
||||
var disabled = ChannelJson((int)WebhookProviderType.Telegram, "-100123", enabled: false);
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(disabled, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMaySaveWithUnchangedGaCreatedChannel()
|
||||
{
|
||||
// The gate must not turn a GA-created Telegram channel into a permanent save block
|
||||
// for the tenant admin's unrelated configuration changes.
|
||||
var json = ChannelJson((int)WebhookProviderType.Telegram, "-100123");
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(json, json, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_IgnoresNonTelegramChannels()
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.GenericJson, "https://sales.example/hook"), null, isGlobalAdmin: false));
|
||||
|
||||
// ── Both write paths (PUT + field patch) go through ValidateModel ─────
|
||||
|
||||
[Fact]
|
||||
public void ValidateModel_EnforcesTelegramGateForNonGlobalAdmin()
|
||||
{
|
||||
var candidate = new TenantConfiguration
|
||||
{
|
||||
TenantId = "t1",
|
||||
NotificationChannelsJson = ChannelJson((int)WebhookProviderType.Telegram, "-100123"),
|
||||
};
|
||||
var existing = new TenantConfiguration { TenantId = "t1" };
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: true));
|
||||
}
|
||||
}
|
||||
@@ -310,6 +310,9 @@ public class TenantConfigPatchServiceTests
|
||||
[InlineData("domainName")]
|
||||
[InlineData("lastUpdated")]
|
||||
[InlineData("updatedBy")]
|
||||
[InlineData("graphAccessRejectedSince")]
|
||||
[InlineData("graphAccessRejectedCode")]
|
||||
[InlineData("graphAccessRejectionConfirmedAt")]
|
||||
public async Task Patch_DeniedField_Returns400WithFieldName(string field)
|
||||
{
|
||||
var harness = new Harness(Stored());
|
||||
|
||||
@@ -33,6 +33,7 @@ public class TenantConfigProjectionTests
|
||||
"onboardedAt", "onboardedBy", "dpaVersion", "dpaAcceptancePending", "dpaAcceptedBy", "dpaAcceptedAt",
|
||||
"lastUpdated", "dataRetentionDays",
|
||||
"homedAppClientId", "lastAuthClientId", "lastAuthClientIdSince",
|
||||
"graphAccessRejectedSince", "graphAccessRejectedCode", "graphAccessRejectionConfirmedAt",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -213,6 +213,40 @@ public class TenantConfigTableSerializationTests
|
||||
Assert.Null(mapped.LastAuthClientIdSince);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Roundtrip_GraphAccessFields_SurviveStoreAndMap()
|
||||
{
|
||||
var since = new DateTime(2026, 11, 2, 8, 0, 0, DateTimeKind.Utc);
|
||||
var config = new TenantConfiguration
|
||||
{
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
UpdatedBy = "admin@contoso.com",
|
||||
GraphAccessRejectedSince = since,
|
||||
GraphAccessRejectedCode = "AADSTS7000229",
|
||||
GraphAccessRejectionConfirmedAt = since.AddMinutes(20),
|
||||
};
|
||||
|
||||
var mapped = TableConfigRepository.ConvertFromTenantTableEntity(
|
||||
TableConfigRepository.ConvertToTenantTableEntity(config));
|
||||
|
||||
Assert.Equal(since, mapped.GraphAccessRejectedSince);
|
||||
Assert.Equal("AADSTS7000229", mapped.GraphAccessRejectedCode);
|
||||
Assert.Equal(since.AddMinutes(20), mapped.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Map_LegacyRow_WithoutGraphAccessColumns_ReadsNull()
|
||||
{
|
||||
var entity = new TableEntity(TenantId, "config") { { "DomainName", "fabrikam.com" } };
|
||||
|
||||
var mapped = TableConfigRepository.ConvertFromTenantTableEntity(entity);
|
||||
|
||||
Assert.Null(mapped.GraphAccessRejectedSince);
|
||||
Assert.Null(mapped.GraphAccessRejectedCode);
|
||||
Assert.Null(mapped.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData(0)]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
@@ -10,11 +9,10 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Inert notification plumbing for tests that need an <see cref="OpsEventService"/> but do not
|
||||
/// assert on anything being sent. Nothing leaves the process: the webhook transport gets a bare
|
||||
/// HttpClient it never uses (no channel is configured in these stubs), and the Telegram transport
|
||||
/// resolves its bot URL from a mocked repository that returns nothing.
|
||||
/// HttpClient it never uses (no channel is configured in these stubs).
|
||||
/// <para>
|
||||
/// Tests that DO assert on dispatch should mock the transports directly — see
|
||||
/// <c>TelegramChannelProviderTests</c> — rather than extending this helper.
|
||||
/// <c>EmailChannelProviderTests</c> — rather than extending this helper.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class TestNotifications
|
||||
@@ -22,8 +20,6 @@ internal static class TestNotifications
|
||||
internal static NotificationChannelDispatcher InertDispatcher()
|
||||
=> new(
|
||||
new WebhookNotificationService(new HttpClient()),
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
Mock.Of<IPushChannelSender>());
|
||||
|
||||
internal static OpsAlertDispatchService InertOpsAlertDispatch(AdminConfigurationService adminConfigService)
|
||||
|
||||
@@ -376,6 +376,8 @@ public class TypedContractCompletionWireParityTests
|
||||
bootstrapTokenEnabled = true,
|
||||
diagnosticsUploadConfigured = false,
|
||||
deviceValidationEnabled = true,
|
||||
graphAccessRejected = true,
|
||||
graphAccessAppMissing = false,
|
||||
appHomingFunnelActive = false,
|
||||
showScriptOutput = true,
|
||||
enableSoftwareInventoryAnalyzer = false,
|
||||
@@ -402,6 +404,8 @@ public class TypedContractCompletionWireParityTests
|
||||
BootstrapTokenEnabled = true,
|
||||
DiagnosticsUploadConfigured = false,
|
||||
DeviceValidationEnabled = true,
|
||||
GraphAccessRejected = true,
|
||||
GraphAccessAppMissing = false,
|
||||
AppHomingFunnelActive = false,
|
||||
ShowScriptOutput = true,
|
||||
EnableSoftwareInventoryAnalyzer = false,
|
||||
|
||||
@@ -88,8 +88,7 @@ public class WhatsNewNotificationServiceTests
|
||||
public Harness()
|
||||
{
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var telegram = new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance);
|
||||
Dispatcher = new Mock<NotificationChannelDispatcher>(webhook, telegram, Mock.Of<IPushChannelSender>());
|
||||
Dispatcher = new Mock<NotificationChannelDispatcher>(webhook, Mock.Of<IPushChannelSender>());
|
||||
Dispatcher
|
||||
.Setup(d => d.SendToChannelsAsync(It.IsAny<IEnumerable<NotificationChannel>>(), It.IsAny<NotificationAlert>(), It.IsAny<NotificationScope>()))
|
||||
.Returns<IEnumerable<NotificationChannel>, NotificationAlert, NotificationScope>((channels, alert, scope) =>
|
||||
@@ -434,14 +433,14 @@ public class WhatsNewNotificationServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildWhatsNewAlert_TelegramText_CarriesSectionsAndLinks()
|
||||
public void BuildWhatsNewAlert_EmailText_CarriesSectionsAndLinks()
|
||||
{
|
||||
var alert = NotificationAlertBuilder.BuildWhatsNewAlert(
|
||||
new List<WhatsNewFeedEntry> { Entry("platform", "p", title: "**Lead**", body: "Body.") },
|
||||
null,
|
||||
WhatsNewNotificationService.PortalWhatsNewUrl("platform"));
|
||||
|
||||
var text = TelegramNotificationService.RenderAlertText(alert);
|
||||
var text = EmailAlertRenderer.Text(alert);
|
||||
|
||||
Assert.Contains("What's new in Autopilot Monitor", text);
|
||||
Assert.Contains("Platform \u00b7 Sep 8, 2026 \u2014 Lead", text);
|
||||
|
||||
+10
-70
@@ -32,12 +32,14 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
/// <summary>
|
||||
/// Changes to ONLY these properties do not snapshot the tenant-config row: the
|
||||
/// LastAuthClientId pair flips as an auth-flow side effect and would flood the
|
||||
/// two backup slots with states nobody ever wants to revert to.
|
||||
/// LastAuthClientId pair flips as an auth-flow side effect and the Graph access state as
|
||||
/// a token side effect; both would flood the two backup slots with states nobody ever
|
||||
/// wants to revert to.
|
||||
/// </summary>
|
||||
private static readonly HashSet<string> TenantBackupNoiseProperties = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"LastUpdated", "UpdatedBy", "LastAuthClientId", "LastAuthClientIdSince",
|
||||
"GraphAccessRejectedSince", "GraphAccessRejectedCode", "GraphAccessRejectionConfirmedAt",
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> AdminBackupNoiseProperties = new(StringComparer.OrdinalIgnoreCase)
|
||||
@@ -541,62 +543,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
}
|
||||
}
|
||||
|
||||
// --- Preview Config ---
|
||||
|
||||
public async Task<Dictionary<string, string>> GetPreviewConfigAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var config = new Dictionary<string, string>();
|
||||
var entity = await _previewConfigTableClient.GetEntityAsync<TableEntity>("TelegramBot", "config");
|
||||
|
||||
foreach (var kvp in entity.Value)
|
||||
{
|
||||
if (kvp.Key == "odata.etag" || kvp.Key == "PartitionKey" || kvp.Key == "RowKey" || kvp.Key == "Timestamp")
|
||||
continue;
|
||||
config[kvp.Key] = kvp.Value?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return new Dictionary<string, string>();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error loading preview config");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> SavePreviewConfigAsync(string key, string value)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Get existing entity or create new one
|
||||
TableEntity entity;
|
||||
try
|
||||
{
|
||||
var existing = await _previewConfigTableClient.GetEntityAsync<TableEntity>("TelegramBot", "config");
|
||||
entity = existing.Value;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
entity = new TableEntity("TelegramBot", "config");
|
||||
}
|
||||
|
||||
entity[key] = value;
|
||||
await _previewConfigTableClient.UpsertEntityAsync(entity);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error saving preview config key {Key}", key);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Email Template Overrides ---
|
||||
|
||||
private const string EmailTemplatesPartition = "EmailTemplates";
|
||||
@@ -871,6 +817,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "HomedAppClientId", config.HomedAppClientId },
|
||||
{ "LastAuthClientId", config.LastAuthClientId },
|
||||
{ "LastAuthClientIdSince", config.LastAuthClientIdSince },
|
||||
{ "GraphAccessRejectedSince", config.GraphAccessRejectedSince },
|
||||
{ "GraphAccessRejectedCode", config.GraphAccessRejectedCode },
|
||||
{ "GraphAccessRejectionConfirmedAt", config.GraphAccessRejectionConfirmedAt },
|
||||
{ "PlanTier", config.PlanTier },
|
||||
{ "TrialExpiresUtc", config.TrialExpiresUtc },
|
||||
{ "TrialStartedUtc", config.TrialStartedUtc },
|
||||
@@ -994,6 +943,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
HomedAppClientId = entity.GetString("HomedAppClientId"),
|
||||
LastAuthClientId = entity.GetString("LastAuthClientId"),
|
||||
LastAuthClientIdSince = entity.GetDateTime("LastAuthClientIdSince"),
|
||||
GraphAccessRejectedSince = entity.GetDateTime("GraphAccessRejectedSince"),
|
||||
GraphAccessRejectedCode = entity.GetString("GraphAccessRejectedCode"),
|
||||
GraphAccessRejectionConfirmedAt = entity.GetDateTime("GraphAccessRejectionConfirmedAt"),
|
||||
PlanTier = entity.GetString("PlanTier") ?? "free",
|
||||
TrialExpiresUtc = entity.GetDateTime("TrialExpiresUtc"),
|
||||
TrialStartedUtc = entity.GetDateTime("TrialStartedUtc"),
|
||||
@@ -1054,12 +1006,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "SlaNotificationCooldownHours", config.SlaNotificationCooldownHours },
|
||||
// Ops Alert settings
|
||||
{ "OpsAlertRulesJson", config.OpsAlertRulesJson ?? string.Empty },
|
||||
{ "OpsAlertTelegramEnabled", config.OpsAlertTelegramEnabled },
|
||||
{ "OpsAlertTelegramChatId", config.OpsAlertTelegramChatId ?? string.Empty },
|
||||
{ "OpsAlertTeamsEnabled", config.OpsAlertTeamsEnabled },
|
||||
{ "OpsAlertTeamsWebhookUrl", config.OpsAlertTeamsWebhookUrl ?? string.Empty },
|
||||
{ "OpsAlertSlackEnabled", config.OpsAlertSlackEnabled },
|
||||
{ "OpsAlertSlackWebhookUrl", config.OpsAlertSlackWebhookUrl ?? string.Empty },
|
||||
{ "OpsNotificationChannelsJson", config.OpsNotificationChannelsJson ?? string.Empty },
|
||||
// Per-line agent binary integrity: written by the release pipeline via Merge.
|
||||
// The backend never changes them (the PATCH gate rejects them), so the
|
||||
@@ -1130,12 +1076,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
SlaNotificationCooldownHours = entity.GetInt32("SlaNotificationCooldownHours") ?? 24,
|
||||
// Ops Alert settings
|
||||
OpsAlertRulesJson = entity.GetString("OpsAlertRulesJson"),
|
||||
OpsAlertTelegramEnabled = entity.GetBoolean("OpsAlertTelegramEnabled") ?? false,
|
||||
OpsAlertTelegramChatId = entity.GetString("OpsAlertTelegramChatId"),
|
||||
OpsAlertTeamsEnabled = entity.GetBoolean("OpsAlertTeamsEnabled") ?? false,
|
||||
OpsAlertTeamsWebhookUrl = entity.GetString("OpsAlertTeamsWebhookUrl"),
|
||||
OpsAlertSlackEnabled = entity.GetBoolean("OpsAlertSlackEnabled") ?? false,
|
||||
OpsAlertSlackWebhookUrl = entity.GetString("OpsAlertSlackWebhookUrl"),
|
||||
OpsNotificationChannelsJson = entity.GetString("OpsNotificationChannelsJson"),
|
||||
// Per-line agent binary integrity. V2 is the only wired line (V1 retired).
|
||||
AllowAgentDowngrade = entity.GetBoolean("AllowAgentDowngrade") ?? false,
|
||||
|
||||
+3
-1
@@ -302,13 +302,15 @@ public class AutopilotDeviceValidationConsentFunction
|
||||
/// <summary>
|
||||
/// Known consent triggers — frontend flows that initiate the AAD admin-consent dialog:
|
||||
/// one per serial-based validation (Autopilot, corporate identifier, and
|
||||
/// "device-preparation" for Device Association Validation).
|
||||
/// "device-preparation" for Device Association Validation), plus "restore" for repairing the
|
||||
/// Graph access of validations that are already on.
|
||||
/// </summary>
|
||||
internal static readonly HashSet<string> KnownConsentTriggers = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"autopilot",
|
||||
"corporate",
|
||||
"device-preparation",
|
||||
"restore",
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -14,17 +14,20 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
private readonly MaintenanceService _maintenanceService;
|
||||
private readonly Services.Push.PushMaintenanceService _pushMaintenance;
|
||||
private readonly Services.Notifications.NotificationChannelHealthMaintenance _channelHealthMaintenance;
|
||||
private readonly GraphAccessRecheckMaintenance _graphAccessRecheck;
|
||||
private readonly ILogger<MaintenanceFunction> _logger;
|
||||
|
||||
public MaintenanceFunction(
|
||||
MaintenanceService maintenanceService,
|
||||
Services.Push.PushMaintenanceService pushMaintenance,
|
||||
Services.Notifications.NotificationChannelHealthMaintenance channelHealthMaintenance,
|
||||
GraphAccessRecheckMaintenance graphAccessRecheck,
|
||||
ILogger<MaintenanceFunction> logger)
|
||||
{
|
||||
_maintenanceService = maintenanceService;
|
||||
_pushMaintenance = pushMaintenance;
|
||||
_channelHealthMaintenance = channelHealthMaintenance;
|
||||
_graphAccessRecheck = graphAccessRecheck;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -45,6 +48,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
await _pushMaintenance.RunAsync();
|
||||
// Health rows of deleted notification channels (D-342); fail-soft inside.
|
||||
await _channelHealthMaintenance.RunAsync();
|
||||
// Confirms or clears Graph access suspicions without waiting for device traffic; fail-soft inside.
|
||||
await _graphAccessRecheck.RunAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
/// Timer-triggered watcher for the Azure SignalR plan limits (Standard S1
|
||||
/// x1 unit: 1,000 concurrent connections / 1M messages per UTC day). Runs
|
||||
/// every full hour and emits OpsEvents when the configured thresholds are
|
||||
/// crossed so operators get a Telegram ping before clients start getting
|
||||
/// crossed so operators get an alert before clients start getting
|
||||
/// 429'd or message overage is billed.
|
||||
///
|
||||
/// Separate from the 2h MaintenanceFunction: the SignalR limits can be
|
||||
|
||||
+1
-2
@@ -178,8 +178,7 @@ public class SubmitOffboardingFeedbackFunction
|
||||
}
|
||||
|
||||
// Best-effort OpsEvent so operators see "feedback received" on the Ops dashboard.
|
||||
// Telegram-wiring done by the admin via OpsAlertRules UI (memory:
|
||||
// feedback_ops_event_types_dual_register).
|
||||
// Alert channels are bound by the admin in the Ops Alert Rules UI.
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordOffboardingFeedbackReceivedAsync(
|
||||
|
||||
+2
-19
@@ -3,7 +3,6 @@ using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -37,24 +36,8 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
|
||||
var config = await _adminConfigService.GetConfigurationAsync();
|
||||
|
||||
// Ops channels: a tenant that predates named channels stores an empty list, and the
|
||||
// dispatcher synthesizes one channel per configured legacy provider slot. Project the
|
||||
// SAME synthesis into the read model so the editor shows what actually receives
|
||||
// alerts today — the alternative is a second copy of the synthesis rules in the web
|
||||
// app, drifting from the one the dispatcher uses. The first save then persists
|
||||
// exactly the list the operator was shown. Never mutates the cached instance.
|
||||
if (string.IsNullOrWhiteSpace(config.OpsNotificationChannelsJson))
|
||||
{
|
||||
var synthesized = config.GetOpsNotificationChannels();
|
||||
if (synthesized.Count > 0)
|
||||
{
|
||||
config = config.ShallowCopy();
|
||||
config.OpsNotificationChannelsJson = NotificationChannel.SerializeList(synthesized);
|
||||
}
|
||||
}
|
||||
|
||||
// A read-only GlobalReader gets secrets (NvdApiKey, SAS, ops webhook URLs and the
|
||||
// per-channel destinations) redacted; a Global Admin gets the full config. Redaction
|
||||
// A read-only GlobalReader gets secrets (NvdApiKey, SAS and the per-channel
|
||||
// destinations) redacted; a Global Admin gets the full config. Redaction
|
||||
// returns a copy — the cached instance is never mutated.
|
||||
if (requestCtx.IsGlobalReader)
|
||||
config = config.RedactedCopyForReader();
|
||||
|
||||
+6
@@ -74,6 +74,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
var resolution = FeatureEntitlementCatalog.Resolve(config, nowUtc);
|
||||
var entitlements = FeatureEntitlementCatalog.Get(resolution);
|
||||
var isTrial = resolution.IsTrial;
|
||||
var graphAccessRejected = config.HasAnyDeviceValidation() && config.GraphAccessRejectionConfirmedAt != null;
|
||||
|
||||
return new TenantFeatureFlagsResponse
|
||||
{
|
||||
@@ -87,6 +88,11 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
DiagnosticsUploadConfigured = DiagnosticsUploadConfigChange.IsConfigured(config),
|
||||
// Drives the "no device validation enabled" dashboard banner (useTenantSecurityConfig).
|
||||
DeviceValidationEnabled = config.HasAnyDeviceValidation(),
|
||||
// Drives the "enrollment monitoring is interrupted" notice. A suspicion alone shows
|
||||
// nothing; the error code itself stays in the admin-gated full config.
|
||||
GraphAccessRejected = graphAccessRejected,
|
||||
GraphAccessAppMissing = graphAccessRejected
|
||||
&& string.Equals(config.GraphAccessRejectedCode, EntraTokenRejection.ServicePrincipalMissingCode, StringComparison.OrdinalIgnoreCase),
|
||||
// Dual app-reg self-service migration: when true, running the consent flow (or
|
||||
// "Detect existing access") targets the NEW app registration and auto-flips this
|
||||
// tenant's homing after verification — drives the explanatory banner in the
|
||||
|
||||
+2
-2
@@ -207,12 +207,12 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
nameof(AdminConfiguration.OpsNotificationChannelsJson), StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
// A GlobalReader is served destinations as the ***REDACTED*** sentinel; persisting it as a
|
||||
// chat ID or webhook URL would silently break every alert, so restore per channel id.
|
||||
// recipient or webhook URL would silently break every alert, so restore per channel id.
|
||||
draft.OpsNotificationChannelsJson = NotificationChannel.RestoreRedactedList(
|
||||
draft.OpsNotificationChannelsJson, storedChannels);
|
||||
|
||||
// Same structural gate and one-Push-channel rule the tenant channel list passes. No GA
|
||||
// gate on Telegram here: this endpoint is GlobalAdminOnly by policy.
|
||||
// gate on platform providers here: this endpoint is GlobalAdminOnly by policy.
|
||||
var channelsError = TenantConfigValidation.ValidateNotificationChannels(draft.OpsNotificationChannelsJson)
|
||||
?? TenantConfigValidation.ValidateSinglePushChannel(draft.OpsNotificationChannelsJson, storedChannels);
|
||||
if (channelsError != null)
|
||||
|
||||
@@ -21,7 +21,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
private readonly TenantConfigurationService _tenantConfigService;
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly TenantAdminsService _tenantAdminsService;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly IFeedbackRepository _feedbackRepo;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
@@ -32,7 +31,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
TenantConfigurationService tenantConfigService,
|
||||
AdminConfigurationService adminConfigService,
|
||||
TenantAdminsService tenantAdminsService,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
ISessionRepository sessionRepo,
|
||||
IFeedbackRepository feedbackRepo,
|
||||
OpsEventService opsEvents,
|
||||
@@ -42,7 +40,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_adminConfigService = adminConfigService;
|
||||
_tenantAdminsService = tenantAdminsService;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_sessionRepo = sessionRepo;
|
||||
_feedbackRepo = feedbackRepo;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -185,7 +182,7 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
|
||||
// Only a rating that was actually recorded is announced — at most once per person.
|
||||
if (stored is { Submitted: true } && stored.Rating is int rating)
|
||||
await AnnounceRatingAsync(tenantId, upn, displayName, stored.FeedbackId!, rating, comment);
|
||||
await AnnounceRatingAsync(tenantId, upn, stored.FeedbackId!, rating, comment);
|
||||
|
||||
_logger.LogInformation("Feedback {Action} by {Upn} (tenant {TenantId})",
|
||||
stored == null ? "unchanged" : body.Dismissed ? "dismissed" : "submitted", upn, tenantId);
|
||||
@@ -196,13 +193,11 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
/// <summary>
|
||||
/// The signals of a recorded rating, all best effort: a stored rating never fails because of
|
||||
/// them. The ops event carries the stars and the comment length only, so an alert channel
|
||||
/// receives no personal data; the Telegram ping keeps sender and comment until it is retired.
|
||||
/// receives no personal data.
|
||||
/// </summary>
|
||||
private async Task AnnounceRatingAsync(
|
||||
string tenantId, string upn, string displayName, string feedbackId, int rating, string? comment)
|
||||
string tenantId, string upn, string feedbackId, int rating, string? comment)
|
||||
{
|
||||
_ = _telegramNotificationService.SendFeedbackAsync(tenantId, upn, displayName, rating, comment);
|
||||
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
// The domain makes the alert and the bell readable. The lookup never creates a
|
||||
// configuration row; an unknown tenant simply keeps its GUID label.
|
||||
|
||||
@@ -31,7 +31,6 @@ public class AuthFunction
|
||||
private readonly TenantAdminsService _tenantAdminsService;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly McpUserService _mcpUserService;
|
||||
private readonly Services.Activation.ITenantAutoApproveEnqueuer _tenantAutoApproveEnqueuer;
|
||||
@@ -41,7 +40,7 @@ public class AuthFunction
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly IOffboardingAuditRepository _offboardingRepo;
|
||||
private readonly Services.Push.PushOwnerSignInService? _pushSignIn;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public AuthFunction(
|
||||
ILogger<AuthFunction> logger,
|
||||
@@ -51,7 +50,6 @@ public class AuthFunction
|
||||
TenantAdminsService tenantAdminsService,
|
||||
IMetricsRepository metricsRepo,
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
McpUserService mcpUserService,
|
||||
Services.Activation.ITenantAutoApproveEnqueuer tenantAutoApproveEnqueuer,
|
||||
@@ -60,8 +58,8 @@ public class AuthFunction
|
||||
LiveStreamRevoker streams,
|
||||
AdminConfigurationService adminConfigService,
|
||||
IOffboardingAuditRepository offboardingRepo,
|
||||
Services.Push.PushOwnerSignInService? pushSignIn = null,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents,
|
||||
Services.Push.PushOwnerSignInService? pushSignIn = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_pushSignIn = pushSignIn;
|
||||
@@ -76,7 +74,6 @@ public class AuthFunction
|
||||
_tenantAdminsService = tenantAdminsService;
|
||||
_metricsRepo = metricsRepo;
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_mcpUserService = mcpUserService;
|
||||
_tenantAutoApproveEnqueuer = tenantAutoApproveEnqueuer;
|
||||
@@ -444,7 +441,7 @@ public class AuthFunction
|
||||
|
||||
/// <summary>
|
||||
/// If the tenant has no domain name yet, extracts it from the UPN, persists it,
|
||||
/// and fires best-effort notifications (Telegram + global notification). The write lands
|
||||
/// and fires the signup signals (ops event + global notification). The write lands
|
||||
/// only while the STORED row still has no domain (conditional create or update), and only
|
||||
/// the login that wrote it fires the signup side effects.
|
||||
/// </summary>
|
||||
@@ -559,15 +556,8 @@ public class AuthFunction
|
||||
Seed(tenantConfig);
|
||||
|
||||
// The ops event is the routable record (alert rules → any channel, Push included); it is
|
||||
// written before the response and never throws. The Telegram ping below stays until retired.
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordTenantSignupAsync(tenantId, upn);
|
||||
|
||||
// Fire-and-forget: Telegram
|
||||
_ = _telegramNotificationService.SendNewTenantSignupAsync(tenantId, upn)
|
||||
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
|
||||
"Fire-and-forget Telegram notification failed for tenant {TenantId}", tenantId),
|
||||
TaskContinuationOptions.OnlyOnFaulted);
|
||||
// written before the response and never throws.
|
||||
await _opsEvents.RecordTenantSignupAsync(tenantId, upn);
|
||||
|
||||
// Fire-and-forget: Global notification
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ namespace AutopilotMonitor.Functions.Functions.Infrastructure
|
||||
/// <summary>
|
||||
/// Receives Azure Monitor alert notifications (Common Alert Schema) from an Action Group
|
||||
/// webhook receiver and re-emits them as ops events, so platform alerts flow through the
|
||||
/// same OpsEventService → OpsAlertDispatchService fan-out (Telegram/Teams/Slack providers +
|
||||
/// same OpsEventService → OpsAlertDispatchService fan-out (the notification channels +
|
||||
/// the portal ops-events list) as every internally raised operational event.
|
||||
///
|
||||
/// The delivery path is deliberately independent of the portal: Azure Monitor → this
|
||||
|
||||
+2
-2
@@ -29,12 +29,12 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
/// Independent cadence from the 2h <c>Maintenance</c> function so cascade-lifecycle
|
||||
/// work cannot starve rule-stats aggregation / cert-expiry checks, and operators get a
|
||||
/// dedicated OpsEvent stream (<see cref="OpsEventService.RecordSessionDeletionMaintenanceLongRunningAsync"/>
|
||||
/// + siblings) wired to Telegram via the standard alert-rules UI.
|
||||
/// + siblings) wired to channels via the standard alert-rules UI.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// 30-minute watchdog emits a Warning OpsEvent; 60-minute watchdog escalates to Error. The
|
||||
/// 60-minute mark matches the Flex Consumption hard <c>functionTimeout</c> (host.json) so the
|
||||
/// operator gets a Telegram alert before the Azure runtime abort. The retention fanout stops
|
||||
/// operator gets an alert before the Azure runtime abort. The retention fanout stops
|
||||
/// itself at the <see cref="RunBudget"/> (50min) deadline so a growing backlog degrades into
|
||||
/// a clean <c>BudgetExceeded</c> + partial sweep instead of a host abort. Cron
|
||||
/// <c>0 0 */12 * * *</c> (every 12h at 00:00 / 12:00 UTC) — const per repo convention;
|
||||
|
||||
+3
-11
@@ -22,24 +22,21 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
private readonly ILogger<SubmitDiagFilesReportFunction> _logger;
|
||||
private readonly SessionReportService _sessionReportService;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly ReportUploadGate _uploadGate;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitDiagFilesReportFunction(
|
||||
ILogger<SubmitDiagFilesReportFunction> logger,
|
||||
SessionReportService sessionReportService,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
ReportUploadGate uploadGate,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_sessionReportService = sessionReportService;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_uploadGate = uploadGate;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -122,12 +119,7 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
}
|
||||
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordDiagFilesReportSubmittedAsync(request.TenantId, userIdentifier, metadata.ReportId);
|
||||
|
||||
// Telegram notification — best effort
|
||||
_ = _telegramNotificationService.SendDiagFilesReportAsync(
|
||||
request.TenantId, userIdentifier, metadata.ReportId, request.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordDiagFilesReportSubmittedAsync(request.TenantId, userIdentifier, metadata.ReportId);
|
||||
|
||||
// Persistent in-app notification for Global Admins — best effort
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+3
-11
@@ -16,24 +16,21 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
private readonly ILogger<SubmitSessionReportFunction> _logger;
|
||||
private readonly SessionReportService _sessionReportService;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly ReportUploadGate _uploadGate;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitSessionReportFunction(
|
||||
ILogger<SubmitSessionReportFunction> logger,
|
||||
SessionReportService sessionReportService,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
ReportUploadGate uploadGate,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_sessionReportService = sessionReportService;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_uploadGate = uploadGate;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -131,12 +128,7 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
}
|
||||
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordSessionReportSubmittedAsync(request.TenantId, userIdentifier, sessionId, metadata.ReportId);
|
||||
|
||||
// Telegram notification — best effort
|
||||
_ = _telegramNotificationService.SendSessionReportAsync(
|
||||
request.TenantId, userIdentifier, sessionId, metadata.ReportId, request.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordSessionReportSubmittedAsync(request.TenantId, userIdentifier, sessionId, metadata.ReportId);
|
||||
|
||||
// Persistent in-app notification for Global Admins — best effort
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+3
-8
@@ -18,22 +18,19 @@ namespace AutopilotMonitor.Functions.Functions.Rules.Submissions
|
||||
private readonly ILogger<SubmitRuleSubmissionsFunction> _logger;
|
||||
private readonly RuleSubmissionService _service;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegram;
|
||||
private readonly GlobalNotificationService _globalNotifications;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitRuleSubmissionsFunction(
|
||||
ILogger<SubmitRuleSubmissionsFunction> logger,
|
||||
RuleSubmissionService service,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegram,
|
||||
GlobalNotificationService globalNotifications,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_service = service;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegram = telegram;
|
||||
_globalNotifications = globalNotifications;
|
||||
_opsEvents = opsEvents;
|
||||
}
|
||||
@@ -112,9 +109,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules.Submissions
|
||||
foreach (var s in created)
|
||||
{
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordRuleSubmissionReceivedAsync(request.TenantId, userIdentifier, s.SubmissionId, s.RuleKind, s.SourceRuleId);
|
||||
_ = _telegram.SendRuleSubmissionAsync(s.SubmissionId, request.TenantId, userIdentifier, s.RuleKind, s.SourceRuleId, s.Title, s.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordRuleSubmissionReceivedAsync(request.TenantId, userIdentifier, s.SubmissionId, s.RuleKind, s.SourceRuleId);
|
||||
_ = _globalNotifications.CreateNotificationAsync(
|
||||
"rule_submission",
|
||||
$"Rule submission {s.SubmissionId}",
|
||||
|
||||
@@ -29,7 +29,7 @@ public sealed class ResiliencePolicies
|
||||
public IAsyncPolicy<HttpResponseMessage> ExternalDataApi { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Policy for notification clients (webhook, Telegram):
|
||||
/// Policy for notification clients (webhook):
|
||||
/// 2 retries with 2s fixed delay. No circuit breaker — a single failing endpoint
|
||||
/// should not block all notifications across tenants.
|
||||
/// </summary>
|
||||
|
||||
@@ -52,6 +52,10 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
("homedAppClientId", c => c.HomedAppClientId),
|
||||
("lastAuthClientId", c => c.LastAuthClientId),
|
||||
("lastAuthClientIdSince", c => c.LastAuthClientIdSince),
|
||||
// Graph access state: two timestamps and a Microsoft error code, not secrets.
|
||||
("graphAccessRejectedSince", c => c.GraphAccessRejectedSince),
|
||||
("graphAccessRejectedCode", c => c.GraphAccessRejectedCode),
|
||||
("graphAccessRejectionConfirmedAt", c => c.GraphAccessRejectionConfirmedAt),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -187,8 +187,8 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
return "Invalid webhook provider type.";
|
||||
|
||||
// ...and only a webhook provider: its destination passed the webhook URL gate above, never
|
||||
// the channel gates of Telegram, Push or E-mail. Only a CHANGED value is checked, so a
|
||||
// stored one stays saveable.
|
||||
// the channel gates of Push or E-mail. Only a CHANGED value is checked, so a stored one
|
||||
// stays saveable.
|
||||
if (candidate.WebhookProviderType != existing.WebhookProviderType
|
||||
&& candidate.WebhookProviderType != 0
|
||||
&& !Shared.Models.Notifications.NotificationChannel.IsWebhookUrlProvider(candidate.WebhookProviderType))
|
||||
@@ -202,10 +202,10 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
if (channelsError != null)
|
||||
return $"Invalid notification channels: {channelsError}";
|
||||
|
||||
var telegramGateError = ValidateTelegramChannelGate(
|
||||
var platformProviderError = ValidatePlatformProviderGate(
|
||||
candidate.NotificationChannelsJson, existing.NotificationChannelsJson, isGlobalAdmin);
|
||||
if (telegramGateError != null)
|
||||
return telegramGateError;
|
||||
if (platformProviderError != null)
|
||||
return platformProviderError;
|
||||
|
||||
var pushCountError = ValidateSinglePushChannel(candidate.NotificationChannelsJson, existing.NotificationChannelsJson);
|
||||
if (pushCountError != null)
|
||||
@@ -365,15 +365,11 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
if (!Shared.Models.Notifications.NotificationChannel.IsSupportedProviderType(channel.ProviderType))
|
||||
return $"channel \"{label}\" has an invalid provider type.";
|
||||
|
||||
// Telegram's destination is a chat ID, not a URL — the SSRF gate does not apply
|
||||
// (there is no caller-supplied endpoint; the bot URL is platform-owned) and would
|
||||
// reject every valid value. Format-check the chat ID instead. A Push channel has
|
||||
// no destination at all: a URL on it is refused so it can never fall through to
|
||||
// the webhook path should a future branch forget the provider.
|
||||
// A Push channel has no destination at all: a URL on it is refused so it can never
|
||||
// fall through to the webhook path should a future branch forget the provider. An
|
||||
// e-mail channel names recipients, not a URL, so the SSRF gate does not apply.
|
||||
var destinationError =
|
||||
channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Telegram
|
||||
? ValidateTelegramChatId(channel.Url)
|
||||
: channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Push
|
||||
channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Push
|
||||
? ValidatePushDestination(channel.Url)
|
||||
: channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Email
|
||||
? ValidateEmailDestination(channel.Url)
|
||||
@@ -393,45 +389,6 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a Telegram destination chat ID. Returns an error message, or null when valid.
|
||||
/// Accepts a numeric id (negative for groups/channels, e.g. -1003785642894) or an @username.
|
||||
/// Empty is rejected: unlike a webhook URL, a Telegram channel with no chat ID cannot be
|
||||
/// saved as "configured but idle" — it would silently drop every message.
|
||||
/// </summary>
|
||||
internal static string? ValidateTelegramChatId(string? chatId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(chatId))
|
||||
return "a Telegram chat ID is required.";
|
||||
|
||||
var trimmed = chatId!.Trim();
|
||||
|
||||
if (trimmed.StartsWith("@", StringComparison.Ordinal))
|
||||
{
|
||||
// Telegram public usernames: 5-32 chars, letters/digits/underscore, letter first.
|
||||
var name = trimmed.Substring(1);
|
||||
if (name.Length < 5 || name.Length > 32 || !char.IsLetter(name[0]))
|
||||
return "the @username must be 5-32 characters and start with a letter.";
|
||||
foreach (var ch in name)
|
||||
{
|
||||
if (!char.IsLetterOrDigit(ch) && ch != '_')
|
||||
return "the @username may only contain letters, digits and underscores.";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
var digits = trimmed.StartsWith("-", StringComparison.Ordinal) ? trimmed.Substring(1) : trimmed;
|
||||
if (digits.Length == 0 || digits.Length > 20)
|
||||
return "must be a numeric chat ID (up to 20 digits) or an @username.";
|
||||
foreach (var ch in digits)
|
||||
{
|
||||
if (!char.IsDigit(ch))
|
||||
return "must be a numeric chat ID (up to 20 digits) or an @username.";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A Push channel carries no destination (its recipients are the scope's paired devices,
|
||||
/// resolved at send time); a URL on it is refused rather than ignored.
|
||||
@@ -467,9 +424,8 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
/// <summary>
|
||||
/// Global-Admin gate for the providers that send through PLATFORM-owned infrastructure
|
||||
/// (<see cref="Shared.Models.Notifications.NotificationChannel.IsGlobalAdminOnlyProvider"/>):
|
||||
/// Telegram through the platform bot (the token lives in PreviewConfig, not in tenant
|
||||
/// config), E-mail through the platform's sender identity and provider key. A tenant admin
|
||||
/// must not be able to create or retarget one — hiding the provider in the UI is not a control.
|
||||
/// E-mail through the platform's sender identity and provider key. A tenant admin must not be
|
||||
/// able to create or retarget one — hiding the provider in the UI is not a control.
|
||||
/// <para>
|
||||
/// Compares against the stored config so a non-GA caller can still save unrelated changes
|
||||
/// while a GA-created gated channel is present: only ADDING one, changing an existing
|
||||
@@ -477,7 +433,7 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
/// deletion stay with the tenant admin. Same shape as the retention-cap check.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static string? ValidateTelegramChannelGate(string? candidateJson, string? existingJson, bool isGlobalAdmin)
|
||||
internal static string? ValidatePlatformProviderGate(string? candidateJson, string? existingJson, bool isGlobalAdmin)
|
||||
{
|
||||
if (isGlobalAdmin)
|
||||
return null;
|
||||
@@ -501,12 +457,7 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
|| !string.Equals(stored.Url ?? string.Empty, channel.Url ?? string.Empty, StringComparison.Ordinal)
|
||||
|| stored.Enabled != channel.Enabled)
|
||||
{
|
||||
var provider = channel.ProviderType switch
|
||||
{
|
||||
(int)Shared.Models.Notifications.WebhookProviderType.Email => "E-mail",
|
||||
_ => "Telegram",
|
||||
};
|
||||
return $"{provider} channels can only be configured by a Global Administrator.";
|
||||
return "E-mail channels can only be configured by a Global Administrator.";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -458,6 +458,10 @@ builder.Services.AddSingleton<RegisterSessionFunction>();
|
||||
builder.Services.AddSingleton<GetAgentConfigFunction>();
|
||||
builder.Services.AddSingleton<ReportAgentErrorFunction>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.EntraAppRegistry>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Security.IGraphAccessHealthRecorder,
|
||||
AutopilotMonitor.Functions.Security.GraphAccessHealthRecorder>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphAccessRecheckMaintenance>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.GraphTokenService>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.AutopilotDeviceValidator>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.CorporateIdentifierValidator>();
|
||||
@@ -473,10 +477,8 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphResolutio
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphResolution.IScriptDisplayNameResolver,
|
||||
AutopilotMonitor.Functions.Services.GraphResolution.ScriptDisplayNameResolver>();
|
||||
builder.Services.AddWebhookNotificationHttpClient();
|
||||
builder.Services.AddHttpClient<TelegramNotificationService>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().Notification);
|
||||
// Channel-level send API — routes each NotificationChannel to its transport (webhook renderer
|
||||
// vs. the platform Telegram bot). Transient: both transports are typed HttpClients.
|
||||
// Channel-level send API — routes each NotificationChannel to its transport. Transient: the
|
||||
// webhook transport is a typed HttpClient.
|
||||
builder.Services.AddTransient<AutopilotMonitor.Functions.Services.Notifications.NotificationChannelDispatcher>();
|
||||
// Every channel send lands in the channel's health row (status dot in the portal, failing bell).
|
||||
builder.Services.AddSingleton<
|
||||
|
||||
@@ -114,7 +114,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new AutopilotDeviceValidationResult
|
||||
|
||||
@@ -119,7 +119,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new CloudPcValidationResult
|
||||
|
||||
@@ -121,7 +121,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new CorporateIdentifierValidationResult
|
||||
|
||||
@@ -111,7 +111,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new DeviceAssociationResult
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
/// <summary>
|
||||
/// Microsoft Entra's answer when it refuses an app token: the HTTP status plus the AADSTS code
|
||||
/// from the response body. Only a client error is a rejection; a 5xx, a throttle or a network
|
||||
/// failure says nothing about the tenant and never becomes one.
|
||||
/// </summary>
|
||||
public sealed record EntraTokenRejection(int StatusCode, string? ErrorCode, string? Description)
|
||||
{
|
||||
/// <summary>The app has no service principal in the tenant: its enterprise application was removed.</summary>
|
||||
public const string ServicePrincipalMissingCode = "AADSTS7000229";
|
||||
|
||||
/// <summary>
|
||||
/// Not a refusal in Entra's words but the same lockout: a token without any application role,
|
||||
/// which Graph refuses on every call. Typically a removed enterprise application that a user's
|
||||
/// sign-in consent re-created. The repair is the same admin consent.
|
||||
/// </summary>
|
||||
public static readonly EntraTokenRejection NoApplicationRole =
|
||||
new(200, "NoApplicationRole", "The app token carries no application permission");
|
||||
|
||||
private const string CodePrefix = "AADSTS";
|
||||
private const int MaxDescriptionLength = 300;
|
||||
|
||||
// Wrong, expired or missing client secret: our configuration, never the customer's tenant.
|
||||
private static readonly string[] OurCredentialCodes = { "AADSTS7000215", "AADSTS7000222", "AADSTS7000218" };
|
||||
|
||||
public bool IsServicePrincipalMissing =>
|
||||
string.Equals(ErrorCode, ServicePrincipalMissingCode, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public bool IsOurCredential =>
|
||||
ErrorCode != null && OurCredentialCodes.Contains(ErrorCode, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Reads the code and the description from a token error body; a body that is not JSON keeps only the status.</summary>
|
||||
public static EntraTokenRejection Parse(int statusCode, string? body)
|
||||
{
|
||||
var json = TryParseObject(body);
|
||||
var description = (json?["error_description"] as JValue)?.Value as string;
|
||||
return new EntraTokenRejection(statusCode, ReadCode(json, description), Shorten(description));
|
||||
}
|
||||
|
||||
private static JObject? TryParseObject(string? body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body)) return null;
|
||||
try
|
||||
{
|
||||
return JsonConvert.DeserializeObject<JToken>(body) as JObject;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ReadCode(JObject? json, string? description)
|
||||
{
|
||||
var first = (json?["error_codes"] as JArray)?.FirstOrDefault(t => t.Type == JTokenType.Integer);
|
||||
if (first != null) return CodePrefix + first.Value<long>();
|
||||
|
||||
// Fallback for a body without error_codes: the description starts with the code.
|
||||
var start = description?.IndexOf(CodePrefix, StringComparison.Ordinal) ?? -1;
|
||||
if (start < 0) return null;
|
||||
var digits = new string(description!.Skip(start + CodePrefix.Length).TakeWhile(char.IsDigit).ToArray());
|
||||
return digits.Length == 0 ? null : CodePrefix + digits;
|
||||
}
|
||||
|
||||
private static string? Shorten(string? description)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(description)) return null;
|
||||
var line = description.Split('\n')[0].Trim();
|
||||
return line.Length <= MaxDescriptionLength ? line : line.Substring(0, MaxDescriptionLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
/// <summary>Folds what Entra answers for a tenant's app token into the tenant's Graph access state.</summary>
|
||||
public interface IGraphAccessHealthRecorder
|
||||
{
|
||||
/// <summary>Entra refused the token on an agent request. Never throws.</summary>
|
||||
Task RecordRejectedAsync(string tenantId, EntraTokenRejection rejection);
|
||||
|
||||
/// <summary>A token with an application role was freshly minted for the app acting on this tenant. Never throws.</summary>
|
||||
Task RecordMintedAsync(string tenantId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Two stages, so one failed request never reaches a customer. The first rejection only stamps
|
||||
/// the suspicion. A rejection at least <see cref="ConfirmAfter"/> later confirms it: only then
|
||||
/// does the portal show its notice and the operator get an alert. A freshly minted token with an
|
||||
/// application role ends the episode. A token from the cache proves nothing and never reaches
|
||||
/// this class: it stays valid for up to 55 minutes after the enterprise application is gone.
|
||||
/// Neither does a token without a role, or the notice would vanish the moment an admin signs in
|
||||
/// to read it: that sign-in re-creates the application without any permission.
|
||||
/// <para>
|
||||
/// Every decision runs on the fresh row, so across instances only one write wins and the ops
|
||||
/// event fires once. Codes that blame our own credentials never touch the tenant: an expired
|
||||
/// secret would otherwise tell every customer to repair something on their side.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class GraphAccessHealthRecorder : IGraphAccessHealthRecorder
|
||||
{
|
||||
public static readonly TimeSpan ConfirmAfter = TimeSpan.FromMinutes(15);
|
||||
|
||||
private const string BackupSource = "graph-access";
|
||||
private static readonly TimeSpan CredentialAlertInterval = TimeSpan.FromHours(1);
|
||||
|
||||
private readonly TenantConfigurationService _tenantConfig;
|
||||
private readonly Lazy<OpsEventService> _opsEvents;
|
||||
private readonly ILogger<GraphAccessHealthRecorder> _logger;
|
||||
private readonly TimeProvider _time;
|
||||
private long _lastCredentialAlertTicks;
|
||||
|
||||
public GraphAccessHealthRecorder(
|
||||
TenantConfigurationService tenantConfig,
|
||||
Lazy<OpsEventService> opsEvents,
|
||||
ILogger<GraphAccessHealthRecorder> logger,
|
||||
TimeProvider? time = null)
|
||||
{
|
||||
_tenantConfig = tenantConfig;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task RecordRejectedAsync(string tenantId, EntraTokenRejection rejection)
|
||||
{
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (rejection.IsOurCredential)
|
||||
{
|
||||
await AlertOurCredentialAsync(tenantId, rejection, now).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cheap pre-filter on the cached view; the write decides again on the fresh row.
|
||||
var cached = await _tenantConfig.GetConfigurationIfExistsAsync(tenantId).ConfigureAwait(false);
|
||||
if (cached == null || !cached.HasAnyDeviceValidation() || cached.GraphAccessRejectionConfirmedAt != null)
|
||||
return;
|
||||
if (cached.GraphAccessRejectedSince is { } cachedSince && now - cachedSince < ConfirmAfter)
|
||||
return;
|
||||
|
||||
var confirmed = false;
|
||||
var update = await _tenantConfig.UpdateAsync(tenantId, row =>
|
||||
{
|
||||
confirmed = false;
|
||||
if (row.GraphAccessRejectionConfirmedAt != null)
|
||||
return false;
|
||||
if (row.GraphAccessRejectedSince is not { } since)
|
||||
{
|
||||
row.GraphAccessRejectedSince = now;
|
||||
row.GraphAccessRejectedCode = rejection.ErrorCode;
|
||||
return true;
|
||||
}
|
||||
if (now - since < ConfirmAfter)
|
||||
return false;
|
||||
|
||||
row.GraphAccessRejectionConfirmedAt = now;
|
||||
row.GraphAccessRejectedCode = rejection.ErrorCode;
|
||||
confirmed = true;
|
||||
return true;
|
||||
}, BackupSource, "Entra rejected the app token").ConfigureAwait(false);
|
||||
|
||||
if (update.Status != TenantConfigUpdateStatus.Updated)
|
||||
return;
|
||||
|
||||
if (!confirmed)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Graph access for tenant {TenantId} is suspect: Entra rejected the app token ({ErrorCode}, status {StatusCode})",
|
||||
tenantId, rejection.ErrorCode, rejection.StatusCode);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Graph access for tenant {TenantId} is confirmed lost: Entra still rejects the app token ({ErrorCode}, status {StatusCode})",
|
||||
tenantId, rejection.ErrorCode, rejection.StatusCode);
|
||||
await _opsEvents.Value.RecordGraphAccessRejectedAsync(tenantId, rejection.ErrorCode, rejection.StatusCode).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Recording the Graph token rejection of tenant {TenantId} failed", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RecordMintedAsync(string tenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cached = await _tenantConfig.GetConfigurationIfExistsAsync(tenantId).ConfigureAwait(false);
|
||||
if (cached?.GraphAccessRejectedSince == null)
|
||||
return;
|
||||
|
||||
var wasConfirmed = false;
|
||||
string? code = null;
|
||||
var update = await _tenantConfig.UpdateAsync(tenantId, row =>
|
||||
{
|
||||
if (row.GraphAccessRejectedSince == null)
|
||||
return false;
|
||||
wasConfirmed = row.GraphAccessRejectionConfirmedAt != null;
|
||||
code = row.GraphAccessRejectedCode;
|
||||
row.GraphAccessRejectedSince = null;
|
||||
row.GraphAccessRejectedCode = null;
|
||||
row.GraphAccessRejectionConfirmedAt = null;
|
||||
return true;
|
||||
}, BackupSource, "Entra issued an app token again").ConfigureAwait(false);
|
||||
|
||||
if (update.Status != TenantConfigUpdateStatus.Updated || !wasConfirmed)
|
||||
return;
|
||||
|
||||
_logger.LogWarning("Graph access for tenant {TenantId} is back: Entra issued an app token again (was {ErrorCode})", tenantId, code);
|
||||
await _opsEvents.Value.RecordGraphAccessRecoveredAsync(tenantId, code).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Recording the recovered Graph access of tenant {TenantId} failed", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AlertOurCredentialAsync(string tenantId, EntraTokenRejection rejection, DateTime now)
|
||||
{
|
||||
// Hits every tenant at once: one alert per instance and hour says everything.
|
||||
var last = Interlocked.Read(ref _lastCredentialAlertTicks);
|
||||
if (now.Ticks - last < CredentialAlertInterval.Ticks)
|
||||
return;
|
||||
if (Interlocked.CompareExchange(ref _lastCredentialAlertTicks, now.Ticks, last) != last)
|
||||
return;
|
||||
|
||||
_logger.LogError(
|
||||
"Entra rejects our app credentials ({ErrorCode}, status {StatusCode}, seen on tenant {TenantId}); device validation fails for every tenant on that app",
|
||||
rejection.ErrorCode, rejection.StatusCode, tenantId);
|
||||
await _opsEvents.Value.RecordGraphAppCredentialRejectedAsync(tenantId, rejection.ErrorCode, rejection.StatusCode).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using AutopilotMonitor.Functions.Services.GraphResolution;
|
||||
using AutopilotMonitor.Shared;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
@@ -30,6 +31,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
private static readonly TimeSpan MaxTokenTtl = TimeSpan.FromMinutes(55);
|
||||
private static readonly TimeSpan DefaultTokenLifetime = TimeSpan.FromMinutes(35); // when AAD omits expires_in
|
||||
private static readonly TimeSpan TokenGateWait = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan RejectionLogWindow = TimeSpan.FromMinutes(15);
|
||||
|
||||
// One coalescing gate per tenant (see GetAccessTokenAsync). A SemaphoreSlim is tiny and
|
||||
// tenant cardinality is bounded + validated upstream, so these are never evicted.
|
||||
@@ -52,19 +54,22 @@ namespace AutopilotMonitor.Functions.Security
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly EntraAppRegistry _appRegistry;
|
||||
private readonly Services.TenantConfigurationService _tenantConfigService;
|
||||
private readonly IGraphAccessHealthRecorder _accessHealth;
|
||||
|
||||
public GraphTokenService(
|
||||
ILogger<GraphTokenService> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IMemoryCache cache,
|
||||
EntraAppRegistry appRegistry,
|
||||
Services.TenantConfigurationService tenantConfigService)
|
||||
Services.TenantConfigurationService tenantConfigService,
|
||||
IGraphAccessHealthRecorder accessHealth)
|
||||
{
|
||||
_logger = logger;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_cache = cache;
|
||||
_appRegistry = appRegistry;
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_accessHealth = accessHealth;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -138,9 +143,37 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// the resolved client id, so a tenant flipped between apps can never be served a token
|
||||
// minted under the other app.
|
||||
var credentials = await ResolveCredentialsAsync(tenantId);
|
||||
return await GetAccessTokenWithCredentialsAsync(tenantId, credentials, ct);
|
||||
var result = await GetAccessTokenWithCredentialsAsync(tenantId, credentials, ct);
|
||||
if (result.IsFreshlyMinted && CarriesApplicationRole(result.AccessToken))
|
||||
await _accessHealth.RecordMintedAsync(tenantId);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GetAccessTokenAsync"/> for the device validators of the agent routes and the
|
||||
/// maintenance recheck, the only callers whose rejection counts as evidence of lost access.
|
||||
/// The consent flows in the portal see 401s as a matter of course while a consent propagates.
|
||||
/// </summary>
|
||||
public async Task<GraphTokenResult> GetAccessTokenForAgentValidationAsync(string tenantId, CancellationToken ct = default)
|
||||
{
|
||||
var result = await GetAccessTokenAsync(tenantId, ct);
|
||||
var refusal = result.Rejection
|
||||
?? (result.IsFreshlyMinted && !CarriesApplicationRole(result.AccessToken) ? EntraTokenRejection.NoApplicationRole : null);
|
||||
if (refusal != null)
|
||||
await _accessHealth.RecordRejectedAsync(tenantId, refusal);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a token can do anything at Graph. A user's sign-in consent re-creates a removed
|
||||
/// enterprise application without any application permission: Entra issues tokens again, and
|
||||
/// Graph refuses every call made with them. Only a role proves access.
|
||||
/// </summary>
|
||||
private static bool CarriesApplicationRole(string? accessToken) =>
|
||||
!string.IsNullOrEmpty(accessToken)
|
||||
&& GraphFeatureDetector.TryParseToken(accessToken, out var roles, out _)
|
||||
&& roles.Count > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Acquires an app-only token for this tenant under EXPLICIT app credentials, skipping the
|
||||
/// homing resolution. Used by the app-homing consent probe: proving that the PRIMARY app
|
||||
@@ -238,6 +271,8 @@ namespace AutopilotMonitor.Functions.Security
|
||||
string? responseBody = null;
|
||||
int attempt = 0;
|
||||
bool lastAttemptWasTransient = false;
|
||||
// Survives the budget: a caller cut off between retries still learns what Entra said.
|
||||
EntraTokenRejection? rejection = null;
|
||||
|
||||
while (true)
|
||||
{
|
||||
@@ -245,7 +280,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
// Caller's budget exhausted — treat as transient so we never cache "no consent"
|
||||
// for a tenant just because the optional path timed out.
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
try
|
||||
{
|
||||
@@ -271,7 +306,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// the token's own expires_in, minus a skew margin, clamped to a safe band.
|
||||
CacheToken(tenantId, clientId, accessToken!, genAtStart, (long?)tokenJson?["expires_in"]);
|
||||
}
|
||||
return GraphTokenResult.Success(accessToken);
|
||||
return GraphTokenResult.Minted(accessToken);
|
||||
}
|
||||
|
||||
// Classify the error: consent-propagation errors are retryable but ultimately permanent (no consent).
|
||||
@@ -287,6 +322,9 @@ namespace AutopilotMonitor.Functions.Security
|
||||
|| isConsentError || isServerError;
|
||||
|
||||
lastAttemptWasTransient = isServerError;
|
||||
rejection = isServerError ? null : EntraTokenRejection.Parse(statusCode, responseBody);
|
||||
if (rejection != null)
|
||||
LogRejectionOncePerWindow(tenantId, clientId, rejection);
|
||||
|
||||
if (!isRetryable || attempt >= retryDelays.Length)
|
||||
{
|
||||
@@ -300,7 +338,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// Server errors are transient (infrastructure issue), consent errors are permanent (no consent granted)
|
||||
return lastAttemptWasTransient
|
||||
? GraphTokenResult.TransientFailure()
|
||||
: GraphTokenResult.PermanentFailure();
|
||||
: GraphTokenResult.PermanentFailure(rejection);
|
||||
}
|
||||
|
||||
var delay = retryDelays[attempt];
|
||||
@@ -328,7 +366,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Caller budget exhausted between retries — transient by definition.
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
@@ -338,7 +376,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
_logger.LogWarning(ex,
|
||||
"Graph token acquisition for tenant {TenantId} failed with network error after {Attempts} attempt(s)",
|
||||
tenantId, attempt + 1);
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
|
||||
var delay = retryDelays[attempt];
|
||||
@@ -349,13 +387,29 @@ namespace AutopilotMonitor.Functions.Security
|
||||
try { await Task.Delay(delay, ct); }
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The retry chain's own failure line only comes after its 50 s, which the validation budget
|
||||
/// never reaches, so without this line Entra's reason would not reach App Insights at all.
|
||||
/// </summary>
|
||||
private void LogRejectionOncePerWindow(string tenantId, string clientId, EntraTokenRejection rejection)
|
||||
{
|
||||
var key = $"graph-token-rejection-logged:{tenantId}:{clientId.ToLowerInvariant()}";
|
||||
if (_cache.TryGetValue(key, out _))
|
||||
return;
|
||||
_cache.Set(key, true, RejectionLogWindow);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Graph token rejected by Entra for tenant {TenantId}, app {ClientId}: status {StatusCode}, {ErrorCode}. {Description}",
|
||||
tenantId, clientId, rejection.StatusCode, rejection.ErrorCode, rejection.Description);
|
||||
}
|
||||
|
||||
// Keys include the resolved client id: during the dual app-reg window a tenant flip
|
||||
// (HomedAppClientId change) must never be served a token/verdict minted under the other
|
||||
// app. Normalized to lowercase so casing differences in config can't split the cache.
|
||||
@@ -440,9 +494,16 @@ namespace AutopilotMonitor.Functions.Security
|
||||
/// </summary>
|
||||
public bool IsTransient { get; private set; }
|
||||
|
||||
/// <summary>True when Entra issued this token just now; false for one from the cache.</summary>
|
||||
public bool IsFreshlyMinted { get; private set; }
|
||||
|
||||
/// <summary>What Entra last refused with, also when the caller's budget ended the retries; null without a refusal.</summary>
|
||||
public EntraTokenRejection? Rejection { get; private set; }
|
||||
|
||||
public static GraphTokenResult Success(string? token) => new() { AccessToken = token };
|
||||
public static GraphTokenResult PermanentFailure() => new() { AccessToken = null, IsTransient = false };
|
||||
public static GraphTokenResult TransientFailure() => new() { AccessToken = null, IsTransient = true };
|
||||
public static GraphTokenResult Minted(string? token) => new() { AccessToken = token, IsFreshlyMinted = !string.IsNullOrEmpty(token) };
|
||||
public static GraphTokenResult PermanentFailure(EntraTokenRejection? rejection = null) => new() { AccessToken = null, IsTransient = false, Rejection = rejection };
|
||||
public static GraphTokenResult TransientFailure(EntraTokenRejection? rejection = null) => new() { AccessToken = null, IsTransient = true, Rejection = rejection };
|
||||
}
|
||||
|
||||
public class GraphConsentStatusResult
|
||||
|
||||
@@ -247,7 +247,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new IntuneDeviceBindingResult
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ namespace AutopilotMonitor.Functions.Services.Activation
|
||||
/// <summary>
|
||||
/// Queue message contract for <c>tenant-auto-approve</c>. One envelope per new tenant
|
||||
/// signup, enqueued with <see cref="ActivationDelay"/> visibility delay so the operator's
|
||||
/// Telegram / portal notification lands before the activation happens.
|
||||
/// alert and portal notification land before the activation happens.
|
||||
/// </summary>
|
||||
public sealed class TenantAutoApproveEnvelope
|
||||
{
|
||||
|
||||
@@ -144,7 +144,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
var legacy = await _graphTokenService.GetAccessTokenForAppAsync(tenantId, legacyApp, ct);
|
||||
if (string.IsNullOrWhiteSpace(legacy.AccessToken) && legacy.IsTransient)
|
||||
// A missing service principal is a fact even when the caller's budget cut the retries
|
||||
// short: the access check runs on 4 s, its first retry waits 5 s.
|
||||
if (string.IsNullOrWhiteSpace(legacy.AccessToken) && legacy.IsTransient
|
||||
&& legacy.Rejection?.IsServicePrincipalMissing != true)
|
||||
{
|
||||
return new AppHomingProbeResult(Succeeded: false, IsTransient: true);
|
||||
}
|
||||
@@ -238,6 +241,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
oldValue = row.HomedAppClientId;
|
||||
row.HomedAppClientId = normalizedTarget;
|
||||
row.UpdatedBy = actorUpn;
|
||||
// A Graph access rejection belongs to the app the tenant just left.
|
||||
row.GraphAccessRejectedSince = null;
|
||||
row.GraphAccessRejectedCode = null;
|
||||
row.GraphAccessRejectionConfirmedAt = null;
|
||||
return true;
|
||||
}, "app-homing", reason);
|
||||
|
||||
|
||||
+1
-1
@@ -336,7 +336,7 @@ namespace AutopilotMonitor.Functions.Services.Backup.Queue
|
||||
await _jobs.TryUpdateWithCasAsync(lostJob, lostETag.Value, ct).ConfigureAwait(false);
|
||||
}
|
||||
// Codex-Hotfix Wave2 #2: emit OpsEvent for the lease-loss terminal path so
|
||||
// Telegram alert rules fire on this manual-queue failure mode the same way
|
||||
// alert rules fire on this manual-queue failure mode the same way
|
||||
// they do for the timer path.
|
||||
await TryRecordOpsEventAsync(() => _opsEvents.RecordCriticalTableBackupFailedAsync(
|
||||
backupIdForEvent, leaseLossReason, runningJobRequestedBy)).ConfigureAwait(false);
|
||||
|
||||
@@ -136,7 +136,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
}
|
||||
|
||||
// PR-B audit consolidation: poisoned cascades surface as a global-scope OpsEvent
|
||||
// (operator-bound, Telegram-routable) rather than a per-tenant audit. Tenant admins
|
||||
// (operator-bound, routable through alert rules) rather than a per-tenant audit. Tenant admins
|
||||
// observe deletion via the lifecycle audits (started / completed / restored).
|
||||
//
|
||||
// Codex F4 follow-up: pull the handler's last-failure breadcrumb out of the
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Asks Entra again for every tenant with a Graph access suspicion, folded into the 2-hourly
|
||||
/// maintenance run. Without it the state would depend on device traffic: a suspicion from a
|
||||
/// Friday evening would stay unconfirmed until Monday, and a repair made directly in Entra would
|
||||
/// keep the notice up until the next device sends.
|
||||
/// </summary>
|
||||
public class GraphAccessRecheckMaintenance
|
||||
{
|
||||
/// <summary>One token request answers the question; a rejection then only waits for its retry delay.</summary>
|
||||
internal static readonly TimeSpan AttemptBudget = TimeSpan.FromSeconds(3);
|
||||
|
||||
private readonly IConfigRepository _configRepository;
|
||||
private readonly GraphTokenService _graphTokenService;
|
||||
private readonly ILogger<GraphAccessRecheckMaintenance> _logger;
|
||||
|
||||
public GraphAccessRecheckMaintenance(
|
||||
IConfigRepository configRepository,
|
||||
GraphTokenService graphTokenService,
|
||||
ILogger<GraphAccessRecheckMaintenance> logger)
|
||||
{
|
||||
_configRepository = configRepository;
|
||||
_graphTokenService = graphTokenService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>Never throws; returns how many tenants it asked about.</summary>
|
||||
public async Task<int> RunAsync()
|
||||
{
|
||||
string[] tenantIds;
|
||||
try
|
||||
{
|
||||
var configs = await _configRepository.GetAllTenantConfigurationsAsync().ConfigureAwait(false);
|
||||
// Without device validation the state means nothing: it waits for the switch to come back.
|
||||
tenantIds = configs
|
||||
.Where(c => c.GraphAccessRejectedSince != null && c.HasAnyDeviceValidation())
|
||||
.Select(c => c.TenantId)
|
||||
.ToArray();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Graph access recheck: reading the tenant configurations failed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
foreach (var tenantId in tenantIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
// A cached token proves nothing either way: only a fresh answer may decide.
|
||||
_graphTokenService.InvalidateTenant(tenantId);
|
||||
using var budget = new CancellationTokenSource(AttemptBudget);
|
||||
await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, budget.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Graph access recheck failed for tenant {TenantId}", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
return tenantIds.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,7 +503,7 @@ public class HealthCheckService
|
||||
/// overall status to at least <c>warning</c> so a flaky Storage call cannot silently
|
||||
/// mask a real backlog; an enumeration failure does the same. Thresholds live in
|
||||
/// <see cref="MaintenanceService"/> so the live health card and the timer-driven
|
||||
/// Telegram alert classify identically.
|
||||
/// alert classify identically.
|
||||
/// </summary>
|
||||
internal async Task<HealthCheck> CheckPoisonQueuesAsync()
|
||||
{
|
||||
|
||||
+74
-3
@@ -6,7 +6,7 @@ using System.Threading.Tasks;
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-step wall-clock of one maintenance run, reported as <c>stepsMs</c> on the
|
||||
/// Per-step wall-clock and memory of one maintenance run, reported on the
|
||||
/// <c>MaintenanceCompleted</c> ops event. Worker-side timing logs never reach App Insights,
|
||||
/// and the run's composition is what decides which steps leave the 2h timer next.
|
||||
/// A step that throws is still recorded (its time counts), then the exception propagates.
|
||||
@@ -14,12 +14,31 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
public sealed class MaintenanceStepTimer
|
||||
{
|
||||
private readonly Dictionary<string, long> _steps = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _peakMb = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _allocMb = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Func<MaintenanceMemoryReading> _readMemory;
|
||||
|
||||
public MaintenanceStepTimer() : this(MaintenanceMemoryReading.Capture)
|
||||
{
|
||||
}
|
||||
|
||||
internal MaintenanceStepTimer(Func<MaintenanceMemoryReading> readMemory)
|
||||
{
|
||||
_readMemory = readMemory;
|
||||
}
|
||||
|
||||
/// <summary>Durations in declaration order (ms, rounded).</summary>
|
||||
public IReadOnlyDictionary<string, long> Steps => _steps;
|
||||
|
||||
public MaintenanceMemoryReport MemoryReport()
|
||||
{
|
||||
var reading = _readMemory();
|
||||
return new MaintenanceMemoryReport(_peakMb, _allocMb, reading.GcHeapLimitBytes >> 20, reading.GcHighLoadBytes >> 20);
|
||||
}
|
||||
|
||||
public async Task RunAsync(string step, Func<Task> body)
|
||||
{
|
||||
var before = _readMemory();
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
@@ -27,12 +46,13 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
Record(step, sw, before);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<T> RunAsync<T>(string step, Func<Task<T>> body)
|
||||
{
|
||||
var before = _readMemory();
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
@@ -40,7 +60,58 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
Record(step, sw, before);
|
||||
}
|
||||
}
|
||||
|
||||
private void Record(string step, Stopwatch sw, MaintenanceMemoryReading before)
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
|
||||
var after = _readMemory();
|
||||
_peakMb[step] = after.PeakWorkingSetBytes >> 20;
|
||||
_allocMb[step] = (after.AllocatedBytes - before.AllocatedBytes) >> 20;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StepsPeakMb"/> is the process's peak working set after each step: the step where
|
||||
/// it jumps set the peak, without depending on when a counter happened to sample.
|
||||
/// <see cref="StepsAllocMb"/> is what the process allocated during each step.
|
||||
/// </summary>
|
||||
public sealed record MaintenanceMemoryReport(
|
||||
IReadOnlyDictionary<string, long> StepsPeakMb,
|
||||
IReadOnlyDictionary<string, long> StepsAllocMb,
|
||||
long GcHeapLimitMb,
|
||||
long GcHighLoadMb);
|
||||
|
||||
internal readonly record struct MaintenanceMemoryReading(
|
||||
long PeakWorkingSetBytes,
|
||||
long AllocatedBytes,
|
||||
long GcHeapLimitBytes,
|
||||
long GcHighLoadBytes)
|
||||
{
|
||||
internal static MaintenanceMemoryReading Capture()
|
||||
{
|
||||
var info = GC.GetGCMemoryInfo();
|
||||
return new MaintenanceMemoryReading(
|
||||
ReadPeakWorkingSet(),
|
||||
GC.GetTotalAllocatedBytes(precise: false),
|
||||
info.TotalAvailableMemoryBytes,
|
||||
info.HighMemoryLoadThresholdBytes);
|
||||
}
|
||||
|
||||
// A maintenance run must not fail on its own telemetry.
|
||||
private static long ReadPeakWorkingSet()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var process = Process.GetCurrentProcess();
|
||||
return process.PeakWorkingSet64;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+50
-25
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
@@ -26,9 +27,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// or median back under 1.5×) deletes the row so a future regression rings again.
|
||||
/// The tracker's 30d retention sweep re-arms long-burning episodes by design.
|
||||
/// <para>
|
||||
/// One cross-tenant install-summary read over the trailing
|
||||
/// <see cref="AppVersionRegressionRadar.HorizonDays"/> days (the same query the global
|
||||
/// apps dashboard runs), then grouped by tenant — mirrors the rule radar's enumeration.
|
||||
/// Reads the trailing <see cref="AppVersionRegressionRadar.HorizonDays"/> days of install
|
||||
/// summaries tenant by tenant (see <see cref="RunAppVersionRadarPassAsync"/>).
|
||||
/// Idempotent per pass via the tracker dedup. Fail-soft per tenant and overall.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
@@ -44,33 +44,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var sinceUtc = DateTime.UtcNow.AddDays(-AppVersionRegressionRadar.HorizonDays);
|
||||
var summaries = await _metricsRepo.GetAppsDashboardSummariesAsync(sinceUtc, tenantId: null);
|
||||
var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
|
||||
|
||||
var byTenant = summaries
|
||||
.Where(s => !string.IsNullOrEmpty(s.TenantId))
|
||||
.GroupBy(s => s.TenantId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
int fired = 0, refreshed = 0, rearmed = 0, tenants = 0;
|
||||
foreach (var tenantGroup in byTenant)
|
||||
{
|
||||
tenants++;
|
||||
try
|
||||
{
|
||||
var (f, r, a) = await EvaluateTenantAppVersionRegressionsAsync(tenantGroup.Key, tenantGroup.ToList());
|
||||
fired += f;
|
||||
refreshed += r;
|
||||
rearmed += a;
|
||||
}
|
||||
catch (Exception tenantEx)
|
||||
{
|
||||
_logger.LogWarning(tenantEx, "App-version regression radar failed for tenant {TenantId} (non-fatal)", tenantGroup.Key);
|
||||
}
|
||||
}
|
||||
var pass = await RunAppVersionRadarPassAsync(
|
||||
tenantIds,
|
||||
tenantId => _metricsRepo.GetAppsDashboardSummariesAsync(sinceUtc, tenantId),
|
||||
EvaluateTenantAppVersionRegressionsAsync,
|
||||
(tenantId, tenantEx) => _logger.LogWarning(tenantEx, "App-version regression radar failed for tenant {TenantId} (non-fatal)", tenantId));
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"App-version regression radar: {Tenants} tenants over {Rows} install summaries — {Fired} fired, {Refreshed} refreshed, {Rearmed} re-armed in {Ms}ms",
|
||||
tenants, summaries.Count, fired, refreshed, rearmed, sw.ElapsedMilliseconds);
|
||||
pass.Tenants, pass.Rows, pass.Fired, pass.Refreshed, pass.Rearmed, sw.ElapsedMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -78,6 +63,46 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads and evaluates one tenant at a time, so a pass holds at most one tenant's rows and
|
||||
/// its memory follows the largest tenant, not the fleet.
|
||||
/// </summary>
|
||||
internal static async Task<(int Tenants, long Rows, int Fired, int Refreshed, int Rearmed)> RunAppVersionRadarPassAsync(
|
||||
IEnumerable<string> tenantIds,
|
||||
Func<string, Task<List<AppInstallSummary>>> loadTenant,
|
||||
Func<string, List<AppInstallSummary>, Task<(int Fired, int Refreshed, int Rearmed)>> evaluateTenant,
|
||||
Action<string, Exception> onTenantFailed)
|
||||
{
|
||||
int tenants = 0, fired = 0, refreshed = 0, rearmed = 0;
|
||||
long rows = 0;
|
||||
|
||||
foreach (var tenantId in tenantIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
var summaries = await loadTenant(tenantId);
|
||||
|
||||
// Never evaluate an empty list: re-arm reads "no rows" as "version drained" and
|
||||
// would delete the tenant's open episodes, and a failed read also returns empty.
|
||||
if (summaries.Count == 0)
|
||||
continue;
|
||||
|
||||
tenants++;
|
||||
rows += summaries.Count;
|
||||
var (f, r, a) = await evaluateTenant(tenantId, summaries);
|
||||
fired += f;
|
||||
refreshed += r;
|
||||
rearmed += a;
|
||||
}
|
||||
catch (Exception tenantEx)
|
||||
{
|
||||
onTenantFailed(tenantId, tenantEx);
|
||||
}
|
||||
}
|
||||
|
||||
return (tenants, rows, fired, refreshed, rearmed);
|
||||
}
|
||||
|
||||
private async Task<(int Fired, int Refreshed, int Rearmed)> EvaluateTenantAppVersionRegressionsAsync(
|
||||
string tenantId, System.Collections.Generic.List<AppInstallSummary> tenantSummaries)
|
||||
{
|
||||
|
||||
@@ -13,7 +13,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// Cert-expiry monitoring leg of MaintenanceService. Runs on the existing 2h
|
||||
/// timer and emits tiered OpsEvents for embedded Intune CA certs nearing
|
||||
/// NotAfter, so the operator gets a Telegram ping months before agent mTLS
|
||||
/// NotAfter, so the operator gets an alert months before agent mTLS
|
||||
/// would break - bundle rotation is the only fix and needs lead time to
|
||||
/// source new PEMs from a corporate-enrolled device.
|
||||
/// </summary>
|
||||
|
||||
@@ -209,7 +209,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
maintenanceStart.Stop();
|
||||
_logger.LogInformation($"Daily maintenance completed in {maintenanceStart.ElapsedMilliseconds}ms");
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer", stepsMs: steps.Steps);
|
||||
await _opsEventService.RecordMaintenanceCompletedAsync((int)maintenanceStart.ElapsedMilliseconds, "Timer", stepsMs: steps.Steps, memory: steps.MemoryReport());
|
||||
|
||||
// Soft watchdog: the run completed, but if it is climbing toward the host's 60min
|
||||
// functionTimeout (e.g. a large first-time retention backlog deleting row-by-row),
|
||||
|
||||
+5
-15
@@ -14,22 +14,19 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
/// in exactly one place — and so does the outcome: every non-Push send, real or test, is
|
||||
/// recorded in the channel's health row by <see cref="INotificationChannelHealthRecorder"/>.
|
||||
/// <para>
|
||||
/// Four transports: <see cref="WebhookProviderType.Email"/> goes to the
|
||||
/// Three transports: <see cref="WebhookProviderType.Email"/> goes to the
|
||||
/// <see cref="IEmailChannelSender"/> (the platform's own sender, recipients in the channel's
|
||||
/// Url); <see cref="WebhookProviderType.Telegram"/> goes to
|
||||
/// <see cref="TelegramNotificationService"/> (plain text through the platform bot, the channel's
|
||||
/// Url field carrying the chat ID); <see cref="WebhookProviderType.Push"/> goes to the
|
||||
/// Url); <see cref="WebhookProviderType.Push"/> goes to the
|
||||
/// <see cref="IPushChannelSender"/> (encrypted Web Push to the paired devices of the
|
||||
/// <see cref="NotificationScope"/>, the channel carrying no destination at all, health kept
|
||||
/// per device); everything else is a rendered webhook POST via
|
||||
/// <see cref="WebhookNotificationService"/>. Neither Telegram nor Push is a renderer — they
|
||||
/// <see cref="WebhookNotificationService"/>. Neither E-mail nor Push is a renderer — they
|
||||
/// have no caller-supplied endpoint and must never reach the SSRF-guarded webhook path.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public class NotificationChannelDispatcher
|
||||
{
|
||||
private readonly WebhookNotificationService _webhook;
|
||||
private readonly TelegramNotificationService _telegram;
|
||||
private readonly IPushChannelSender _push;
|
||||
private readonly IEmailChannelSender? _email;
|
||||
private readonly INotificationChannelHealthRecorder? _health;
|
||||
@@ -37,11 +34,9 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
|
||||
public NotificationChannelDispatcher(
|
||||
WebhookNotificationService webhook,
|
||||
TelegramNotificationService telegram,
|
||||
IPushChannelSender push)
|
||||
{
|
||||
_webhook = webhook;
|
||||
_telegram = telegram;
|
||||
_push = push;
|
||||
_logger = NullLogger<NotificationChannelDispatcher>.Instance;
|
||||
}
|
||||
@@ -49,10 +44,9 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
/// <summary>With the e-mail transport; the shorter overload stays for the existing mocks.</summary>
|
||||
public NotificationChannelDispatcher(
|
||||
WebhookNotificationService webhook,
|
||||
TelegramNotificationService telegram,
|
||||
IPushChannelSender push,
|
||||
IEmailChannelSender email)
|
||||
: this(webhook, telegram, push)
|
||||
: this(webhook, push)
|
||||
{
|
||||
_email = email;
|
||||
}
|
||||
@@ -60,12 +54,11 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
/// <summary>The full dispatcher (DI picks this one): outcomes are logged and recorded.</summary>
|
||||
public NotificationChannelDispatcher(
|
||||
WebhookNotificationService webhook,
|
||||
TelegramNotificationService telegram,
|
||||
IPushChannelSender push,
|
||||
IEmailChannelSender email,
|
||||
INotificationChannelHealthRecorder health,
|
||||
ILogger<NotificationChannelDispatcher> logger)
|
||||
: this(webhook, telegram, push, email)
|
||||
: this(webhook, push, email)
|
||||
{
|
||||
_health = health;
|
||||
_logger = logger;
|
||||
@@ -143,9 +136,6 @@ namespace AutopilotMonitor.Functions.Services.Notifications
|
||||
: await _email.SendAlertAsync(channel.Url!, alert);
|
||||
}
|
||||
|
||||
if (channel.ProviderType == (int)WebhookProviderType.Telegram)
|
||||
return await _telegram.SendAlertAsync(channel.Url!, alert);
|
||||
|
||||
return await _webhook.SendAsync(
|
||||
channel.Url!,
|
||||
(WebhookProviderType)channel.ProviderType,
|
||||
|
||||
+1
-1
@@ -1079,7 +1079,7 @@ namespace AutopilotMonitor.Functions.Services.Offboarding
|
||||
// Side-effect 2: Pointer → Failed.
|
||||
await UpdatePointerWithRetryAsync(tenantId, history.RowKey, "Failed", ct);
|
||||
|
||||
// Side-effect 3: OpsEvent (Telegram-routable).
|
||||
// Side-effect 3: OpsEvent (routable through alert rules).
|
||||
await _opsEvents.RecordTenantOffboardingFailedAsync(
|
||||
tenantId, history.InitiatedBy, failedPhase, errorMessage, history.RetryCount, history.DomainName);
|
||||
|
||||
|
||||
@@ -202,7 +202,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// Turns the ops event's structured details into readable facts, so card channels (Teams,
|
||||
/// Slack, Discord) and the plain-text Telegram message carry the same information the
|
||||
/// Slack, Discord) and the e-mail carry the same information the
|
||||
/// generic JSON consumer gets in <c>data</c>. Reached only for rules that opted in via
|
||||
/// <see cref="OpsAlertRule.IncludePayload"/> — the caller passes null otherwise, so the
|
||||
/// default alert stays the category/event/severity/tenant baseline.
|
||||
|
||||
@@ -62,6 +62,23 @@ namespace AutopilotMonitor.Functions.Services
|
||||
$"App-reg homing flipped {oldApp} -> {newApp} for a tenant with Entra app roles — re-assign roles on the new enterprise app",
|
||||
tenantId, userId, new { oldApp, newApp });
|
||||
|
||||
/// <summary>Entra still refuses the tenant's app token 15 minutes after the first refusal: its agents get no validation.</summary>
|
||||
public Task RecordGraphAccessRejectedAsync(string tenantId, string? errorCode, int statusCode)
|
||||
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.GraphAccessRejected, OpsEventSeverity.Error,
|
||||
$"Microsoft Entra keeps rejecting the app token of this tenant ({errorCode ?? $"status {statusCode}"}): agents get no validation, the portal shows a notice",
|
||||
tenantId, "System.GraphAccess", new { errorCode, statusCode });
|
||||
|
||||
public Task RecordGraphAccessRecoveredAsync(string tenantId, string? errorCode)
|
||||
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.GraphAccessRecovered, OpsEventSeverity.Info,
|
||||
$"Microsoft Entra issues the app token of this tenant again (was {errorCode ?? "no code"})",
|
||||
tenantId, "System.GraphAccess", new { errorCode });
|
||||
|
||||
/// <summary>Entra refuses our own client secret: device validation fails for every tenant on that app.</summary>
|
||||
public Task RecordGraphAppCredentialRejectedAsync(string tenantId, string? errorCode, int statusCode)
|
||||
=> WriteAsync(OpsEventCategory.Consent, OpsEventTypes.GraphAppCredentialRejected, OpsEventSeverity.Critical,
|
||||
$"Microsoft Entra rejects our app credentials ({errorCode ?? $"status {statusCode}"}): device validation fails for every tenant on that app",
|
||||
tenantId, "System.GraphAccess", new { errorCode, statusCode });
|
||||
|
||||
// ── Maintenance ────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
@@ -85,12 +102,20 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// before the run starts, so this event is where the operator reads what the run did.
|
||||
/// </summary>
|
||||
public Task RecordMaintenanceCompletedAsync(int durationMs, string triggeredBy, MaintenanceResult? report = null,
|
||||
IReadOnlyDictionary<string, long>? stepsMs = null)
|
||||
IReadOnlyDictionary<string, long>? stepsMs = null, Maintenance.MaintenanceMemoryReport? memory = null)
|
||||
=> WriteAsync(OpsEventCategory.Maintenance, OpsEventTypes.MaintenanceCompleted, OpsEventSeverity.Info,
|
||||
$"Maintenance completed in {durationMs}ms (triggered by {triggeredBy})",
|
||||
null, triggeredBy,
|
||||
report == null
|
||||
? (object)new { durationMs, stepsMs }
|
||||
? (object)new
|
||||
{
|
||||
durationMs,
|
||||
stepsMs,
|
||||
stepsPeakMb = memory?.StepsPeakMb,
|
||||
stepsAllocMb = memory?.StepsAllocMb,
|
||||
gcHeapLimitMb = memory?.GcHeapLimitMb,
|
||||
gcHighLoadMb = memory?.GcHighLoadMb,
|
||||
}
|
||||
: new
|
||||
{
|
||||
durationMs,
|
||||
@@ -331,7 +356,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// the poison queue and CAS-transitioned the Sessions row to <see cref="SessionDeletionState.Poisoned"/>.
|
||||
/// Replaces the prior <c>deletion_poisoned</c> tenant audit — tenant admins see only the
|
||||
/// lifecycle endpoints (<c>deletion_started</c>, <c>deletion_completed</c>, <c>deletion_restored</c>),
|
||||
/// while operators get this OpsEvent for the Session Cleanup admin page + Telegram routing.
|
||||
/// while operators get this OpsEvent for the Session Cleanup admin page + alert routing.
|
||||
/// <para>
|
||||
/// Codex follow-ups F4 + F2: <paramref name="failureType"/> / <paramref name="failureMessage"/>
|
||||
/// / <paramref name="observedResidualCount"/> / <paramref name="residualSamplePreviewJson"/>
|
||||
@@ -465,7 +490,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// routes a non-GA attempt at a GA-only tool through the same path (access-probe route), so
|
||||
/// one event type covers both the direct-HTTP probe and the tool-catalog probe.
|
||||
/// Critical unless the caller is a Global Reader (a known operator, not an intruder) — the
|
||||
/// severity split lets one rule push Critical to Telegram while Reader slips stay on the page.
|
||||
/// severity split lets one rule alert on Critical while Reader slips stay on the page.
|
||||
/// Recorded against the caller's HOME tenant (JWT tid); the identity is in the details.
|
||||
/// </summary>
|
||||
public Task RecordPrivilegedRouteDeniedAsync(PrivilegedDenial d)
|
||||
@@ -528,7 +553,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// Fired by <see cref="KillSwitchEvaluator"/> when a Kill signal was actually SERVED to
|
||||
/// an agent (as opposed to DeviceBlocked/VersionBlocked, which fire when the admin
|
||||
/// creates the rule). This is the delivery confirmation operators wire Telegram rules
|
||||
/// creates the rule). This is the delivery confirmation operators wire alert rules
|
||||
/// on — throttled at the evaluator (24h per tenant+serial+pattern) because a kill-blind
|
||||
/// old agent keeps hitting the endpoint every few seconds. Details carry
|
||||
/// <c>serialNumber</c> so the Ops Events detail modal's Block/Kill shortcuts deep-link.
|
||||
@@ -637,7 +662,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// A GA operator restored a single row from a backup (plan §PR2). Warning-level
|
||||
/// because the restore overwrote (or created) live data via ETag-CAS — operators
|
||||
/// frequently want a Telegram ping for this event so a parallel admin sees the
|
||||
/// frequently want an alert for this event so a parallel admin sees the
|
||||
/// audit trail in near-real-time. Payload carries the canonical
|
||||
/// <c>{ container, blobName }</c> only — no SAS URL, per plan §Medium #6.
|
||||
/// </summary>
|
||||
@@ -751,7 +776,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// Fired by the tenant-offboarding worker when the cascade fails closed (kill-switch
|
||||
/// active mid-enqueue, drain timeout, expectations blob corruption, ETag/CAS exhaustion,
|
||||
/// SafeWipe verify abort, …). Marker stays Failed until operator action; this event is
|
||||
/// the Telegram-routable signal that something needs human attention. Plan Rev-4 Q2.
|
||||
/// the routable signal that something needs human attention. Plan Rev-4 Q2.
|
||||
/// </summary>
|
||||
public Task RecordTenantOffboardingFailedAsync(
|
||||
string tenantId, string performedBy, string failedPhase, string errorMessage,
|
||||
@@ -767,7 +792,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
/// <summary>
|
||||
/// Fired by the tenant-auto-approve queue worker when a new signup was activated
|
||||
/// automatically (AutoApproveNewTenants enabled). Info-tier audit + Telegram-routable
|
||||
/// automatically (AutoApproveNewTenants enabled). Info-tier audit + routable
|
||||
/// signal so the operator sees which tenants entered without a manual approval click.
|
||||
/// Dual-registered in OpsAlertRulesSection.tsx OPS_EVENT_TYPES
|
||||
/// (memory feedback_ops_event_types_dual_register).
|
||||
@@ -901,7 +926,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// A Pro trial expired within the last sweep window — the tenant silently degraded
|
||||
/// to Community at read time (retention cap, rate limits, MSP delegation, MCP plan).
|
||||
/// Warning-tier: a conversion moment an operator likely wants a Telegram ping for.
|
||||
/// Warning-tier: a conversion moment an operator likely wants an alert for.
|
||||
/// </summary>
|
||||
public Task RecordTenantTrialExpiredAsync(string tenantId, string? domainName, DateTime trialExpiredUtc)
|
||||
{
|
||||
@@ -1118,7 +1143,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// once per break report (for a session with a row, guarded by the timeline-event
|
||||
/// idempotency check). Severity, <paramref name="sessionStatusAtBreak"/> and
|
||||
/// <paramref name="context"/> come from <c>ReportAgentErrorFunction.ClassifyEmergencyBreak</c>:
|
||||
/// Warning only when the session was still open (the break IS the verdict — Telegram-rule
|
||||
/// Warning only when the session was still open (the break IS the verdict — alert-rule
|
||||
/// worthy) or its row could not be read. The 2026-09-03 audit found 224 of 269 breaks in
|
||||
/// three days were late cleanups on terminal sessions, the 2026-09-28 sweep that 115 of 160
|
||||
/// remaining Warnings named sessions that never registered; as Warnings both buried the real
|
||||
@@ -1165,8 +1190,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// (tamper, stale blob, or a build that never came from a committed tree). Session
|
||||
/// e9753578 (2026-08-20) carried exactly this report, but it landed only in App
|
||||
/// Insights and was invisible to every product surface while the field damage was
|
||||
/// mis-attributed to committed code for hours. Warning-tier so operators can wire a
|
||||
/// Telegram rule; the message carries both hashes verbatim.
|
||||
/// mis-attributed to committed code for hours. Warning-tier so operators can wire an
|
||||
/// alert rule; the message carries both hashes verbatim.
|
||||
/// </summary>
|
||||
public Task RecordAgentBinaryIntegrityMismatchAsync(string tenantId, string? sessionId, string? agentVersion, string message)
|
||||
=> WriteAsync(OpsEventCategory.Agent, OpsEventTypes.AgentBinaryIntegrityMismatch, OpsEventSeverity.Warning,
|
||||
@@ -1211,7 +1236,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// Fired when maintenance auto-blocks or auto-kills a device after its session crossed
|
||||
/// <see cref="AutopilotMonitor.Shared.Models.AdminConfiguration.ExcessiveEventAutoActionThreshold"/>.
|
||||
/// Critical-severity so operators can wire a Telegram rule independent of the warn-tier
|
||||
/// Critical-severity so operators can wire an alert rule independent of the warn-tier
|
||||
/// <c>ExcessiveSessionEvents</c> rule. Details carry the resolved <c>serialNumber</c>
|
||||
/// so the Ops Events detail modal's Block/Kill shortcuts deep-link correctly.
|
||||
/// </summary>
|
||||
@@ -1314,11 +1339,11 @@ namespace AutopilotMonitor.Functions.Services
|
||||
null, "System.Push",
|
||||
new { host, scope = scopeKind });
|
||||
|
||||
// ── Tenant: customer-initiated submissions (plan push-relay, Phase 0) ─────
|
||||
// Info events next to the direct Telegram pings, so an alert rule can route them to any
|
||||
// channel. Like FeedbackReceived, message and details carry ids only — never the submitter
|
||||
// or a comment; the portal page behind portalPath has them. The user id column keeps the
|
||||
// submitter for the admin view and never reaches a channel.
|
||||
// ── Tenant: customer-initiated submissions ────────────────────────────────
|
||||
// Info events, so an alert rule can route them to any channel. Like FeedbackReceived, message
|
||||
// and details carry ids only — never the submitter or a comment; the portal page behind
|
||||
// portalPath has them. The user id column keeps the submitter for the admin view and never
|
||||
// reaches a channel.
|
||||
|
||||
public virtual Task RecordTenantSignupAsync(string tenantId, string upn)
|
||||
=> WriteAsync(OpsEventCategory.Tenant, OpsEventTypes.TenantSignup, OpsEventSeverity.Info,
|
||||
|
||||
@@ -1,361 +0,0 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Sends Telegram notifications for tenant signup events.
|
||||
/// Webhook URL is read from the PreviewConfig table via IConfigRepository.
|
||||
/// </summary>
|
||||
public class TelegramNotificationService
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly IConfigRepository _configRepo;
|
||||
private readonly ILogger<TelegramNotificationService> _logger;
|
||||
|
||||
public TelegramNotificationService(
|
||||
HttpClient http,
|
||||
IConfigRepository configRepo,
|
||||
ILogger<TelegramNotificationService> logger)
|
||||
{
|
||||
_http = http;
|
||||
_configRepo = configRepo;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a Telegram message to the configured channel when a new tenant signs up.
|
||||
/// No-op if the webhook URL is not configured in the PreviewConfig table.
|
||||
/// </summary>
|
||||
public virtual async Task SendNewTenantSignupAsync(string tenantId, string upn)
|
||||
{
|
||||
try
|
||||
{
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
_logger.LogDebug("Telegram webhook URL not configured — skipping tenant signup notification");
|
||||
return;
|
||||
}
|
||||
|
||||
var payload = new
|
||||
{
|
||||
chat_id = "-1003632442830", // Telegram channel ID (as string)
|
||||
text = $"New tenant signup!\nTenantID: {tenantId}\nUPN: {upn}"
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
_logger.LogInformation(
|
||||
"Telegram tenant signup notification sent for tenant {TenantId}, UPN {Upn}",
|
||||
tenantId, upn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Failed to send Telegram tenant signup notification for tenant {TenantId}",
|
||||
tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a Telegram notification when a Tenant Admin submits a session report.
|
||||
/// Best-effort — silently no-ops if the webhook URL is not configured.
|
||||
/// </summary>
|
||||
public async Task SendSessionReportAsync(string tenantId, string submittedBy, string sessionId, string reportId, string comment)
|
||||
{
|
||||
try
|
||||
{
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
_logger.LogDebug("Telegram webhook URL not configured — skipping session report notification");
|
||||
return;
|
||||
}
|
||||
|
||||
var commentLine = string.IsNullOrWhiteSpace(comment) ? "" : $"\nComment: {comment}";
|
||||
var payload = new
|
||||
{
|
||||
chat_id = "-1003632442830",
|
||||
text = $"New Session Report submitted!\nTenantID: {tenantId}\nBy: {submittedBy}\nSessionID: {sessionId}\nReportID: {reportId}{commentLine}"
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
_logger.LogInformation(
|
||||
"Telegram session report notification sent for report {ReportId}", reportId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to send Telegram session report notification for {ReportId}", reportId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a Telegram notification when a tenant admin submits a custom rule for the
|
||||
/// community pool. The submission id comes first — it is the handle for the review.
|
||||
/// Best-effort — silently no-ops if the webhook URL is not configured.
|
||||
/// </summary>
|
||||
public async Task SendRuleSubmissionAsync(string submissionId, string tenantId, string submittedBy, string ruleKind, string ruleId, string title, string comment)
|
||||
{
|
||||
try
|
||||
{
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
_logger.LogDebug("Telegram webhook URL not configured — skipping rule submission notification");
|
||||
return;
|
||||
}
|
||||
|
||||
var commentLine = string.IsNullOrWhiteSpace(comment) ? "" : $"\nComment: {comment}";
|
||||
var payload = new
|
||||
{
|
||||
chat_id = "-1003632442830",
|
||||
text = $"New Rule Submission {submissionId}\nRule: {ruleId} \"{title}\" ({ruleKind})\nTenantID: {tenantId}\nBy: {submittedBy}{commentLine}"
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
_logger.LogInformation("Telegram rule submission notification sent for {SubmissionId}", submissionId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to send Telegram rule submission notification for {SubmissionId}", submissionId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a Telegram notification when a tenant admin submits a diag-files-only report
|
||||
/// (no session context). Best-effort — silently no-ops if the webhook URL is not configured.
|
||||
/// </summary>
|
||||
public async Task SendDiagFilesReportAsync(string tenantId, string submittedBy, string reportId, string comment)
|
||||
{
|
||||
try
|
||||
{
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
_logger.LogDebug("Telegram webhook URL not configured — skipping diag-files report notification");
|
||||
return;
|
||||
}
|
||||
|
||||
var commentLine = string.IsNullOrWhiteSpace(comment) ? "" : $"\nComment: {comment}";
|
||||
var payload = new
|
||||
{
|
||||
chat_id = "-1003632442830",
|
||||
text = $"New Diag Files Report submitted!\nTenantID: {tenantId}\nBy: {submittedBy}\nReportID: {reportId}{commentLine}"
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
_logger.LogInformation(
|
||||
"Telegram diag-files report notification sent for report {ReportId}", reportId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to send Telegram diag-files report notification for {ReportId}", reportId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Posts content to a webhook URL with a single retry on transient failures (429, 5xx, network errors).
|
||||
/// </summary>
|
||||
private async Task<bool> PostWithRetryAsync(string webhookUrl, StringContent content, string context)
|
||||
{
|
||||
const int maxAttempts = 2;
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
return true;
|
||||
|
||||
var statusCode = (int)response.StatusCode;
|
||||
var isTransient = statusCode == 429 || statusCode >= 500 && statusCode <= 599;
|
||||
|
||||
if (!isTransient || attempt >= maxAttempts)
|
||||
{
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
_logger.LogWarning("Telegram webhook returned {StatusCode} for {Context}: {Body}", statusCode, context, body);
|
||||
return false;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Telegram webhook returned {StatusCode} for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
|
||||
statusCode, context, attempt, maxAttempts);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
catch (Exception ex) when (attempt < maxAttempts && (ex is HttpRequestException or TaskCanceledException))
|
||||
{
|
||||
_logger.LogWarning(ex,
|
||||
"Telegram webhook network error for {Context} (attempt {Attempt}/{MaxAttempts}). Retrying in 2s",
|
||||
context, attempt, maxAttempts);
|
||||
await Task.Delay(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends a Telegram notification when a user submits feedback via the in-app feedback bubble.
|
||||
/// Best-effort — silently no-ops if the webhook URL is not configured.
|
||||
/// </summary>
|
||||
public virtual async Task SendFeedbackAsync(string tenantId, string upn, string displayName, int rating, string? comment)
|
||||
{
|
||||
try
|
||||
{
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
_logger.LogDebug("Telegram webhook URL not configured — skipping feedback notification");
|
||||
return;
|
||||
}
|
||||
|
||||
var stars = new string('\u2605', rating) + new string('\u2606', 5 - rating);
|
||||
var commentLine = string.IsNullOrWhiteSpace(comment) ? "" : $"\nComment: {comment}";
|
||||
var payload = new
|
||||
{
|
||||
chat_id = "-1003632442830",
|
||||
text = $"New Feedback!\nRating: {stars} ({rating}/5)\nUser: {displayName} ({upn})\nTenant: {tenantId}{commentLine}"
|
||||
};
|
||||
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var response = await _http.PostAsync(webhookUrl, content);
|
||||
|
||||
if (response.IsSuccessStatusCode)
|
||||
_logger.LogInformation("Telegram feedback notification sent for user {Upn}", upn);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to send Telegram feedback notification for {Upn}", upn);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends an alert to the specified Telegram chat as a plain-text message with severity
|
||||
/// emoji and reports the outcome; never throws. The one Telegram channel send path — real
|
||||
/// deliveries and the "send test" endpoints alike (the dispatcher records the outcome).
|
||||
/// <para>
|
||||
/// Virtual: <see cref="Notifications.NotificationChannelDispatcher"/> routes Telegram
|
||||
/// channels here, and its routing tests need a seam.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public virtual async Task<Notifications.NotificationSendResult> SendAlertAsync(string chatId, NotificationAlert alert)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(chatId))
|
||||
return new Notifications.NotificationSendResult { Success = false, Message = "No Telegram chat ID configured." };
|
||||
|
||||
var webhookUrl = await GetWebhookUrlAsync();
|
||||
if (string.IsNullOrWhiteSpace(webhookUrl))
|
||||
{
|
||||
return new Notifications.NotificationSendResult
|
||||
{
|
||||
Success = false,
|
||||
Message = "The platform Telegram bot is not configured — no message can be delivered."
|
||||
};
|
||||
}
|
||||
|
||||
var payload = new { chat_id = chatId, text = RenderAlertText(alert) };
|
||||
var json = JsonConvert.SerializeObject(payload);
|
||||
var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
var sent = await PostWithRetryAsync(webhookUrl!, content, $"Alert:{alert.Title}");
|
||||
|
||||
return sent
|
||||
? new Notifications.NotificationSendResult { Success = true, Message = "Test notification sent successfully." }
|
||||
: new Notifications.NotificationSendResult
|
||||
{
|
||||
Success = false,
|
||||
Message = "Telegram rejected the message — check the chat ID and that the bot is a member of that chat."
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new Notifications.NotificationSendResult { Success = false, Message = $"Connection error: {ex.Message}" };
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flattens an alert into the plain-text form Telegram accepts. Telegram has no card
|
||||
/// format, so facts become "Name: Value" lines — that is also what carries an ops event's
|
||||
/// structured payload into the message. Sections (rule explanations, What's new entries)
|
||||
/// follow as title + text blocks, and openUrl actions as trailing "Title: url" lines.
|
||||
/// </summary>
|
||||
internal static string RenderAlertText(NotificationAlert alert)
|
||||
{
|
||||
var icon = alert.Severity switch
|
||||
{
|
||||
NotificationSeverity.Error => "\U0001f7e0", // orange circle
|
||||
NotificationSeverity.Warning => "\U0001f7e1", // yellow circle
|
||||
NotificationSeverity.Info => "\u2139\ufe0f", // info
|
||||
_ => "\U0001f534" // red circle (fallback / success)
|
||||
};
|
||||
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine($"{icon} {alert.Title}");
|
||||
sb.AppendLine(alert.Summary);
|
||||
|
||||
foreach (var fact in alert.Facts)
|
||||
sb.AppendLine($"{fact.Name}: {fact.Value}");
|
||||
|
||||
foreach (var section in alert.Sections)
|
||||
{
|
||||
if (string.IsNullOrEmpty(section.Title) && string.IsNullOrEmpty(section.Text))
|
||||
continue;
|
||||
sb.AppendLine();
|
||||
if (!string.IsNullOrEmpty(section.Title))
|
||||
sb.AppendLine(section.Title);
|
||||
if (!string.IsNullOrEmpty(section.Text))
|
||||
sb.AppendLine(section.Text);
|
||||
}
|
||||
|
||||
var links = alert.Actions.Where(a => a.Type == "openUrl" && !string.IsNullOrEmpty(a.Url)).ToList();
|
||||
if (links.Count > 0)
|
||||
{
|
||||
sb.AppendLine();
|
||||
foreach (var action in links)
|
||||
sb.AppendLine($"{action.Title}: {action.Url}");
|
||||
}
|
||||
|
||||
return sb.ToString().TrimEnd();
|
||||
}
|
||||
|
||||
private async Task<string?> GetWebhookUrlAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var config = await _configRepo.GetPreviewConfigAsync();
|
||||
config.TryGetValue("WebhookUrl", out var value);
|
||||
return value;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to read Telegram webhook URL from PreviewConfig table");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -112,6 +112,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
"LastUpdated", "UpdatedBy", "OnboardedAt", "OnboardedBy", "DpaVersion",
|
||||
"DpaAcceptancePending", "DpaAcceptedBy", "DpaAcceptedAt",
|
||||
"HomedAppClientId", "LastAuthClientId", "LastAuthClientIdSince",
|
||||
"GraphAccessRejectedSince", "GraphAccessRejectedCode", "GraphAccessRejectionConfirmedAt",
|
||||
"PlanTier", "TrialExpiresUtc", "TrialStartedUtc", "TrialConsumed", "TrialGrantedBy",
|
||||
"ProDowngradedUtc", "MaxDelegatedTenantsOverride", "McpUsagePlanOverride", "PayingCustomer",
|
||||
"ManagedByProTenantId",
|
||||
@@ -161,6 +162,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
internal static readonly HashSet<string> RevertProtectedFields = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"HomedAppClientId", "LastAuthClientId", "LastAuthClientIdSince",
|
||||
"GraphAccessRejectedSince", "GraphAccessRejectedCode", "GraphAccessRejectionConfirmedAt",
|
||||
"OnboardedBy", "OnboardedAt", "DpaVersion",
|
||||
"DpaAcceptancePending", "DpaAcceptedBy", "DpaAcceptedAt",
|
||||
"PlanTier", "TrialExpiresUtc", "TrialStartedUtc", "TrialConsumed", "TrialGrantedBy",
|
||||
@@ -514,6 +516,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
"DpaAcceptancePending" or "DpaAcceptedBy" or "DpaAcceptedAt" => "system-owned — written by the onboarding and the Terms + DPA acceptance",
|
||||
"HomedAppClientId" => "system-owned — written only by the app-homing flow",
|
||||
"LastAuthClientId" or "LastAuthClientIdSince" => "system-owned auth provenance",
|
||||
"GraphAccessRejectedSince" or "GraphAccessRejectedCode" or "GraphAccessRejectionConfirmedAt"
|
||||
=> "system-owned — follows what Microsoft Entra answers for the tenant's app token",
|
||||
"PlanTier" or "TrialExpiresUtc" or "TrialStartedUtc" or "TrialConsumed" or "TrialGrantedBy"
|
||||
or "MaxDelegatedTenantsOverride" or "McpUsagePlanOverride" or "PayingCustomer"
|
||||
=> "mutable only via the dedicated plan/trial endpoints",
|
||||
|
||||
@@ -31,6 +31,9 @@ public readonly record struct ProcessSnapshot(
|
||||
long WorkingSetBytes,
|
||||
long HeapSizeBytes,
|
||||
long CommittedBytes,
|
||||
long TotalAvailableMemoryBytes,
|
||||
long MemoryLoadBytes,
|
||||
long HighMemoryLoadThresholdBytes,
|
||||
int LastGcGeneration,
|
||||
TimeSpan LastGcPause,
|
||||
bool LastGcCompacted)
|
||||
@@ -53,6 +56,9 @@ public readonly record struct ProcessSnapshot(
|
||||
WorkingSetBytes: Environment.WorkingSet,
|
||||
HeapSizeBytes: info.HeapSizeBytes,
|
||||
CommittedBytes: info.TotalCommittedBytes,
|
||||
TotalAvailableMemoryBytes: info.TotalAvailableMemoryBytes,
|
||||
MemoryLoadBytes: info.MemoryLoadBytes,
|
||||
HighMemoryLoadThresholdBytes: info.HighMemoryLoadThresholdBytes,
|
||||
LastGcGeneration: info.Generation,
|
||||
LastGcPause: lastPause,
|
||||
LastGcCompacted: info.Compacted);
|
||||
@@ -120,6 +126,32 @@ public static class ProcessStallClassifier
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Memory pressure as a percentage of the limits the GC itself works against, so a reading keeps
|
||||
/// its meaning when the instance size changes. The inputs date from the last GC; before the first
|
||||
/// one there is no reading.
|
||||
/// </summary>
|
||||
public static class ProcessMemoryPressure
|
||||
{
|
||||
public const string MetricName = "BackendMemoryPct";
|
||||
public const string KindDimension = "kind";
|
||||
|
||||
/// <summary>Committed heap against the GC's heap limit; at 100 allocations start to fail.</summary>
|
||||
public const string HeapKind = "heap";
|
||||
|
||||
/// <summary>Memory load against the GC's high-load threshold; at 100 the GC collects aggressively.</summary>
|
||||
public const string LoadKind = "load";
|
||||
|
||||
public static double? HeapPercent(ProcessSnapshot snapshot)
|
||||
=> Percent(snapshot.CommittedBytes, snapshot.TotalAvailableMemoryBytes);
|
||||
|
||||
public static double? LoadPercent(ProcessSnapshot snapshot)
|
||||
=> Percent(snapshot.MemoryLoadBytes, snapshot.HighMemoryLoadThresholdBytes);
|
||||
|
||||
private static double? Percent(long value, long limit)
|
||||
=> limit > 0 ? Math.Round(100.0 * value / limit, 1) : null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detects process-wide pauses in the worker and reports what caused them. Production showed
|
||||
/// episodes where every in-flight request on ONE instance finished at the same instant after
|
||||
@@ -141,6 +173,11 @@ public static class ProcessStallClassifier
|
||||
/// much of one slow request was GC pause. Query:
|
||||
/// <c>customEvents | where name == 'ProcessStall' | project timestamp, cloud_RoleInstance, customDimensions</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The same tick feeds the pre-aggregated metric <see cref="ProcessMemoryPressure.MetricName"/>
|
||||
/// (dimension <c>kind</c>), so a peak shorter than the platform's per-minute counters still shows
|
||||
/// as that minute's <c>valueMax</c>.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ProcessStallMonitor : BackgroundService
|
||||
{
|
||||
@@ -154,6 +191,7 @@ public sealed class ProcessStallMonitor : BackgroundService
|
||||
|
||||
private readonly TelemetryClient? _telemetry;
|
||||
private readonly Metric? _metric;
|
||||
private readonly Metric? _memoryMetric;
|
||||
private readonly BackendBuildInfo _buildInfo;
|
||||
private readonly ILogger<ProcessStallMonitor> _logger;
|
||||
|
||||
@@ -163,6 +201,7 @@ public sealed class ProcessStallMonitor : BackgroundService
|
||||
_logger = logger;
|
||||
_telemetry = telemetry;
|
||||
_metric = telemetry?.GetMetric(MetricName, "cause");
|
||||
_memoryMetric = telemetry?.GetMetric(ProcessMemoryPressure.MetricName, ProcessMemoryPressure.KindDimension);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
@@ -185,22 +224,32 @@ public sealed class ProcessStallMonitor : BackgroundService
|
||||
var drift = stopwatch.Elapsed - Tick;
|
||||
var current = ProcessSnapshot.Capture();
|
||||
|
||||
if (drift >= StallThreshold)
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
TrackMemoryPressure(current);
|
||||
|
||||
if (drift >= StallThreshold)
|
||||
Report(ProcessStallClassifier.Classify(drift, previous, current));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ProcessStallMonitor: failed to report a stall");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "ProcessStallMonitor: failed to report");
|
||||
}
|
||||
|
||||
previous = current;
|
||||
}
|
||||
}
|
||||
|
||||
private void TrackMemoryPressure(ProcessSnapshot snapshot)
|
||||
{
|
||||
if (_memoryMetric == null) return;
|
||||
|
||||
if (ProcessMemoryPressure.HeapPercent(snapshot) is { } heap)
|
||||
_memoryMetric.TrackValue(heap, ProcessMemoryPressure.HeapKind);
|
||||
if (ProcessMemoryPressure.LoadPercent(snapshot) is { } load)
|
||||
_memoryMetric.TrackValue(load, ProcessMemoryPressure.LoadKind);
|
||||
}
|
||||
|
||||
private void Report(StallReport report)
|
||||
{
|
||||
var driftMs = Math.Round(report.Drift.TotalMilliseconds);
|
||||
|
||||
@@ -114,7 +114,7 @@ export interface AdminConfiguration {
|
||||
collectorIdleTimeoutMinutes: number;
|
||||
/** DAD liveness threshold in minutes. When the V2 DesktopArrivalDetector polls for this duration without detecting either an excluded-user explorer.exe or a real user desktop, it emits a single desktop_detector_no_candidate observability event. State-change-only — NOT a periodic heartbeat; max 1 event per detector lifetime. Used to distinguish "user never logged in" from "detector wiring dead post-reboot" in sessions where desktop_arrived is missing. 0 = disabled (event never fires). Default: 10 minutes. */
|
||||
desktopDetectorNoCandidateTimeoutMinutes: number;
|
||||
/** Maintenance alarm threshold: sessions with more events than this value trigger an ExcessiveSessionEvents ops alert (dispatched to Telegram/Teams). 0 = disabled. Default: 2000 (largest real sessions observed are ~500). */
|
||||
/** Maintenance alarm threshold: sessions with more events than this value trigger an ExcessiveSessionEvents ops alert (dispatched through the alert rules). 0 = disabled. Default: 2000 (largest real sessions observed are ~500). */
|
||||
excessiveEventCountThreshold: number;
|
||||
/** Auto-action mode for runaway sessions whose EventCount exceeds ExcessiveEventAutoActionThreshold. One of "Off", "Block", or "Kill". Off (default) keeps the warn-only behaviour; Block stops the device's uploads for ExcessiveEventAutoActionDurationHours; Kill issues a remote self-destruct signal. Action runs in the 2h maintenance batch, is idempotent per session via its block record (SessionSummary.BlockCause), and never downgrades an existing Kill to Block. */
|
||||
excessiveEventAutoActionMode: string;
|
||||
@@ -148,19 +148,7 @@ export interface AdminConfiguration {
|
||||
nvdApiKey: string;
|
||||
/** JSON-serialized list of OpsAlertRule objects defining which event types trigger notifications. Provider-agnostic — rules apply to all enabled providers. */
|
||||
opsAlertRulesJson: string;
|
||||
/** Whether the Telegram alert provider is enabled. Default: false. */
|
||||
opsAlertTelegramEnabled: boolean;
|
||||
/** Telegram chat ID for ops alerts (e.g. ITEngineer channel). */
|
||||
opsAlertTelegramChatId: string;
|
||||
/** Whether the Teams alert provider is enabled. Default: false. */
|
||||
opsAlertTeamsEnabled: boolean;
|
||||
/** Teams Workflow webhook URL for ops alerts. */
|
||||
opsAlertTeamsWebhookUrl: string;
|
||||
/** Whether the Slack alert provider is enabled. Default: false. */
|
||||
opsAlertSlackEnabled: boolean;
|
||||
/** Slack Incoming Webhook URL for ops alerts. */
|
||||
opsAlertSlackWebhookUrl: string;
|
||||
/** JSON array of platform notification channels (NotificationChannel), the destinations an OpsAlertRule can target by id. Supersedes the three fixed provider slots above: each channel carries its own provider, destination and name, so an operator can keep e.g. an "IT Engineer" push channel and a "Sales" webhook apart and bind each event type to only the channels that should see it. Null/empty = not migrated yet — then synthesizes the list from the legacy slots, so dispatch behavior is unchanged until an operator saves the section once. */
|
||||
/** JSON array of platform notification channels (NotificationChannel), the destinations an OpsAlertRule can target by id. Each channel carries its own provider, destination and name, so an operator can keep e.g. an "IT Engineer" push channel and a "Sales" webhook apart and bind each event type to only the channels that should see it. */
|
||||
opsNotificationChannelsJson: string;
|
||||
/** Version string of the latest published V2 agent (e.g. "2.0.647"). */
|
||||
latestAgentV2Version: string;
|
||||
@@ -5383,6 +5371,12 @@ export interface TenantConfiguration {
|
||||
lastAuthClientId?: string | null;
|
||||
/** When LastAuthClientId last changed (i.e. logins arrive via that app since this instant). Null on rows that pre-date the field. */
|
||||
lastAuthClientIdSince?: string | null;
|
||||
/** When Microsoft Entra first refused this tenant's app token on an agent request, in the current episode. Only a suspicion: nothing is shown until GraphAccessRejectionConfirmedAt is set. System-owned, cleared by the next freshly issued token that carries an application permission. */
|
||||
graphAccessRejectedSince?: string | null;
|
||||
/** The Entra error of the rejection (for example AADSTS7000229), or NoApplicationRole for a token without any application permission; null when the response carried no code. System-owned. */
|
||||
graphAccessRejectedCode?: string | null;
|
||||
/** When a later rejection confirmed the suspicion: the tenant's agents get no validation and the portal shows its notice. System-owned, cleared by the next freshly issued token that carries an application permission. */
|
||||
graphAccessRejectionConfirmedAt?: string | null;
|
||||
/** Whether this tenant is disabled/suspended If true, users from this tenant cannot log in Default: false */
|
||||
disabled: boolean;
|
||||
/** Optional reason why the tenant was disabled Displayed to users attempting to log in */
|
||||
@@ -5599,6 +5593,10 @@ export interface TenantFeatureFlagsResponse {
|
||||
diagnosticsUploadConfigured: boolean;
|
||||
/** At least one device-validation method is enabled (HasAnyDeviceValidation); false drives the "agent ingestion is blocked" dashboard banner. */
|
||||
deviceValidationEnabled: boolean;
|
||||
/** Microsoft Entra has kept refusing the tenant's app token, so agents get no validation; drives the "enrollment monitoring is interrupted" notice. Boolean only, no error detail. */
|
||||
graphAccessRejected: boolean;
|
||||
/** Only with GraphAccessRejected: the refusal says the enterprise application is missing from the tenant, which selects the concrete notice text. */
|
||||
graphAccessAppMissing: boolean;
|
||||
/** Dual app-reg self-service migration: consent flow targets the NEW app registration. Non-sensitive — exposes no client ids. */
|
||||
appHomingFunnelActive: boolean;
|
||||
showScriptOutput: boolean;
|
||||
|
||||
@@ -61,6 +61,9 @@ export const OPS_EVENT_TYPES = [
|
||||
"ConsentRedirectUriMismatch",
|
||||
"AppHomingFlipped",
|
||||
"AppHomingFlippedWithEntraRoles",
|
||||
"GraphAccessRejected",
|
||||
"GraphAccessRecovered",
|
||||
"GraphAppCredentialRejected",
|
||||
"MaintenanceCompleted",
|
||||
"MaintenanceFailed",
|
||||
"MaintenanceLongRunning",
|
||||
|
||||
@@ -105,10 +105,6 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
Task<bool> RemoveFromPreviewWhitelistAsync(string tenantId);
|
||||
Task<List<string>> GetPreviewWhitelistAsync();
|
||||
|
||||
// --- Preview Config ---
|
||||
Task<Dictionary<string, string>> GetPreviewConfigAsync();
|
||||
Task<bool> SavePreviewConfigAsync(string key, string value);
|
||||
|
||||
// --- Email Template Overrides (operator-level, PreviewConfig table, partition "EmailTemplates") ---
|
||||
/// <summary>Returns the stored override for the given kind, or null when the built-in template applies.</summary>
|
||||
Task<EmailTemplateOverride?> GetEmailTemplateOverrideAsync(string kind);
|
||||
|
||||
@@ -27,6 +27,9 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
public const string ConsentRedirectUriMismatch = "ConsentRedirectUriMismatch";
|
||||
public const string AppHomingFlipped = "AppHomingFlipped";
|
||||
public const string AppHomingFlippedWithEntraRoles = "AppHomingFlippedWithEntraRoles";
|
||||
public const string GraphAccessRejected = "GraphAccessRejected";
|
||||
public const string GraphAccessRecovered = "GraphAccessRecovered";
|
||||
public const string GraphAppCredentialRejected = "GraphAppCredentialRejected";
|
||||
|
||||
// ── Maintenance ── Scheduled maintenance, cascade deletion, critical-table backup.
|
||||
public const string MaintenanceCompleted = "MaintenanceCompleted";
|
||||
@@ -122,8 +125,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// </summary>
|
||||
public const string FeedbackRatingReceived = "FeedbackRatingReceived";
|
||||
/// <summary>
|
||||
/// A new tenant signed up: its first sign-in seeded the domain (<c>auth/me</c>). Info. Recorded
|
||||
/// next to the direct Telegram ping until that is retired (plan push-relay, Phase 0); message and
|
||||
/// A new tenant signed up: its first sign-in seeded the domain (<c>auth/me</c>). Info. Message and
|
||||
/// details carry no personal data — the Tenants page has the rest.
|
||||
/// </summary>
|
||||
public const string TenantSignup = "TenantSignup";
|
||||
@@ -204,7 +206,7 @@ namespace AutopilotMonitor.Shared.DataAccess
|
||||
/// <summary>Every declared type, declaration order (grouped by category).</summary>
|
||||
public static readonly IReadOnlyList<string> All = new[]
|
||||
{
|
||||
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles,
|
||||
ConsentFlowStarted, ConsentFlowSuccess, ConsentFlowFailed, ConsentRedirectUriMismatch, AppHomingFlipped, AppHomingFlippedWithEntraRoles, GraphAccessRejected, GraphAccessRecovered, GraphAppCredentialRejected,
|
||||
MaintenanceCompleted, MaintenanceFailed, MaintenanceLongRunning, MaintenanceStarted, MaintenanceSkippedLocked, SessionSweepCompleted, SessionSweepFailed, OpsEventCleanup, OrphanEventsCleaned, OrphanSweepCompleted, OrphanSweepFailed, OrphanSweepSkippedLocked, OrphanReconcileCompleted, SessionDeletionMaintenanceStarted, SessionDeletionMaintenanceBudgetExceeded, SessionDeletionMaintenanceSkippedLocked, SessionDeletionMaintenanceLongRunning, SessionDeletionMaintenanceLongRunningSevere, SessionDeletionMaintenanceFailed, SessionDeletionStrandedQueued, SessionDeletionPoisoned, SessionDeletionMaintenanceCompleted, SessionDeletionMaintenanceFanoutSkipped, CriticalTableBackupCompleted, CriticalTableBackupPartial, CriticalTableBackupFailed, CriticalTableBackupSkippedLocked, BackupRowRestored, VerdictCalibrationDrift, SoftwareInventoryCounterWriteFailed,
|
||||
DeviceBlocked, VersionBlocked, SessionTenantConflict, SessionOwnerMismatch, KillSignalDelivered, EmbeddedCertExpiringSoon, EmbeddedCertExpiringUrgent, EmbeddedCertExpired, EmbeddedCertBundleEmpty, SignalRConnectionsHigh, SignalRConnectionsCritical, SignalRMessagesHigh, SignalRMessagesCritical, PoisonQueueBacklogHigh, PoisonQueueBacklogCritical, ExcessiveSessionEventsAutoActioned, PrivilegedRouteDenied, McpServicePrincipalFirstSeen, ReportUploadLimitReached,
|
||||
OffboardingFeedbackReceived, FeedbackReceived, FeedbackRatingReceived, TenantSignup, SessionReportSubmitted, DiagFilesReportSubmitted, RuleSubmissionReceived, TenantOffboarded, TenantOffboardingFailed, TenantAutoApproved, WelcomeEmailSent, WelcomeEmailSkipped, WelcomeEmailFailed, FarewellEmailSent, FarewellEmailSkipped, FarewellEmailFailed, TenantTrialStarted, TenantTrialExpiring, TenantTrialExpired, TenantPlanDowngraded, TenantRetentionGraceExpiring, TenantRetentionGraceEnded, RuleFrequencyRegression, AppVersionDurationRegression, CollectLogsQuickConfigEnabled, DiagnosticsUploadEnabled, DiagnosticsUploadDisabled,
|
||||
|
||||
@@ -96,7 +96,7 @@ namespace AutopilotMonitor.Shared.Models
|
||||
|
||||
/// <summary>
|
||||
/// Maintenance alarm threshold: sessions with more events than this value trigger an
|
||||
/// ExcessiveSessionEvents ops alert (dispatched to Telegram/Teams).
|
||||
/// ExcessiveSessionEvents ops alert (dispatched through the alert rules).
|
||||
/// 0 = disabled. Default: 2000 (largest real sessions observed are ~500).
|
||||
/// </summary>
|
||||
public int ExcessiveEventCountThreshold { get; set; } = 2000;
|
||||
@@ -276,109 +276,17 @@ namespace AutopilotMonitor.Shared.Models
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Whether the Telegram alert provider is enabled. Default: false.</summary>
|
||||
public bool OpsAlertTelegramEnabled { get; set; }
|
||||
|
||||
/// <summary>Telegram chat ID for ops alerts (e.g. ITEngineer channel).</summary>
|
||||
public string OpsAlertTelegramChatId { get; set; } = default!;
|
||||
|
||||
/// <summary>Whether the Teams alert provider is enabled. Default: false.</summary>
|
||||
public bool OpsAlertTeamsEnabled { get; set; }
|
||||
|
||||
/// <summary>Teams Workflow webhook URL for ops alerts.</summary>
|
||||
public string OpsAlertTeamsWebhookUrl { get; set; } = default!;
|
||||
|
||||
/// <summary>Whether the Slack alert provider is enabled. Default: false.</summary>
|
||||
public bool OpsAlertSlackEnabled { get; set; }
|
||||
|
||||
/// <summary>Slack Incoming Webhook URL for ops alerts.</summary>
|
||||
public string OpsAlertSlackWebhookUrl { get; set; } = default!;
|
||||
|
||||
/// <summary>
|
||||
/// JSON array of platform notification channels (<see cref="Notifications.NotificationChannel"/>),
|
||||
/// the destinations an <see cref="OpsAlertRule"/> can target by id. Supersedes the three
|
||||
/// fixed provider slots above: each channel carries its own provider, destination and name,
|
||||
/// so an operator can keep e.g. an "IT Engineer" push channel and a "Sales" webhook apart
|
||||
/// and bind each event type to only the channels that should see it.
|
||||
/// <para>
|
||||
/// Null/empty = not migrated yet — <see cref="GetOpsNotificationChannels"/> then synthesizes
|
||||
/// the list from the legacy slots, so dispatch behavior is unchanged until an operator
|
||||
/// saves the section once.
|
||||
/// </para>
|
||||
/// the destinations an <see cref="OpsAlertRule"/> can target by id. Each channel carries its own
|
||||
/// provider, destination and name, so an operator can keep e.g. an "IT Engineer" push channel and
|
||||
/// a "Sales" webhook apart and bind each event type to only the channels that should see it.
|
||||
/// </summary>
|
||||
public string OpsNotificationChannelsJson { get; set; } = default!;
|
||||
|
||||
/// <summary>Stable id of the channel synthesized from the legacy Telegram slot.</summary>
|
||||
public const string LegacyTelegramChannelId = "legacy-telegram";
|
||||
|
||||
/// <summary>Stable id of the channel synthesized from the legacy Teams slot.</summary>
|
||||
public const string LegacyTeamsChannelId = "legacy-teams";
|
||||
|
||||
/// <summary>Stable id of the channel synthesized from the legacy Slack slot.</summary>
|
||||
public const string LegacySlackChannelId = "legacy-slack";
|
||||
|
||||
/// <summary>
|
||||
/// Display name for the synthesized Telegram channel — the operator push channel that
|
||||
/// every ops alert went to before channels existed.
|
||||
/// </summary>
|
||||
public const string LegacyTelegramChannelName = "Autopilot Monitor IT Engineer";
|
||||
|
||||
/// <summary>
|
||||
/// Returns the platform notification channels. Prefers <see cref="OpsNotificationChannelsJson"/>;
|
||||
/// when that is empty, synthesizes one channel per configured legacy slot (stable ids, so a
|
||||
/// rule's channel binding survives the later migration to the stored list) — same
|
||||
/// forward-compatibility trick as <c>TenantConfiguration.GetNotificationChannels</c>.
|
||||
/// <para>
|
||||
/// The synthesized channels carry the legacy enabled flags, so a disabled provider stays
|
||||
/// disabled and the pre-channels dispatch behavior is reproduced exactly.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <summary>Returns the platform notification channels stored in <see cref="OpsNotificationChannelsJson"/>.</summary>
|
||||
public IReadOnlyList<Notifications.NotificationChannel> GetOpsNotificationChannels()
|
||||
{
|
||||
var channels = Notifications.NotificationChannel.ParseList(OpsNotificationChannelsJson);
|
||||
if (channels.Count > 0)
|
||||
return channels;
|
||||
|
||||
var synthesized = new List<Notifications.NotificationChannel>();
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(OpsAlertTelegramChatId))
|
||||
{
|
||||
synthesized.Add(new Notifications.NotificationChannel
|
||||
{
|
||||
Id = LegacyTelegramChannelId,
|
||||
Name = LegacyTelegramChannelName,
|
||||
ProviderType = (int)Notifications.WebhookProviderType.Telegram,
|
||||
Url = OpsAlertTelegramChatId,
|
||||
Enabled = OpsAlertTelegramEnabled,
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(OpsAlertTeamsWebhookUrl))
|
||||
{
|
||||
synthesized.Add(new Notifications.NotificationChannel
|
||||
{
|
||||
Id = LegacyTeamsChannelId,
|
||||
Name = "Microsoft Teams",
|
||||
ProviderType = (int)Notifications.WebhookProviderType.TeamsWorkflowWebhook,
|
||||
Url = OpsAlertTeamsWebhookUrl,
|
||||
Enabled = OpsAlertTeamsEnabled,
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(OpsAlertSlackWebhookUrl))
|
||||
{
|
||||
synthesized.Add(new Notifications.NotificationChannel
|
||||
{
|
||||
Id = LegacySlackChannelId,
|
||||
Name = "Slack",
|
||||
ProviderType = (int)Notifications.WebhookProviderType.Slack,
|
||||
Url = OpsAlertSlackWebhookUrl,
|
||||
Enabled = OpsAlertSlackEnabled,
|
||||
});
|
||||
}
|
||||
|
||||
return synthesized;
|
||||
}
|
||||
=> Notifications.NotificationChannel.ParseList(OpsNotificationChannelsJson);
|
||||
|
||||
// ===== PER-LINE AGENT HASH ORACLE =====
|
||||
// Per-major-line schema. GetAgentConfigFunction parses X-Agent-Version-Major and
|
||||
@@ -661,15 +569,11 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// Shallow copy for read-model projections that must not mutate the cached instance
|
||||
/// (the configuration service hands out the same object to every caller).
|
||||
/// </summary>
|
||||
public AdminConfiguration ShallowCopy() => (AdminConfiguration)MemberwiseClone();
|
||||
|
||||
public AdminConfiguration RedactedCopyForReader()
|
||||
{
|
||||
var copy = (AdminConfiguration)MemberwiseClone();
|
||||
copy.PlatformStatsBlobSasUrl = Redact(copy.PlatformStatsBlobSasUrl);
|
||||
copy.NvdApiKey = Redact(copy.NvdApiKey);
|
||||
copy.OpsAlertTeamsWebhookUrl = Redact(copy.OpsAlertTeamsWebhookUrl);
|
||||
copy.OpsAlertSlackWebhookUrl = Redact(copy.OpsAlertSlackWebhookUrl);
|
||||
// Per-channel: destinations and secrets are redacted, the list structure (ids, names,
|
||||
// providers) stays readable so a reader still sees which channels exist.
|
||||
copy.OpsNotificationChannelsJson =
|
||||
|
||||
@@ -137,6 +137,27 @@ namespace AutopilotMonitor.Shared.Models
|
||||
/// </summary>
|
||||
public DateTime? LastAuthClientIdSince { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When Microsoft Entra first refused this tenant's app token on an agent request, in the
|
||||
/// current episode. Only a suspicion: nothing is shown until
|
||||
/// <see cref="GraphAccessRejectionConfirmedAt"/> is set. System-owned, cleared by the next
|
||||
/// freshly issued token that carries an application permission.
|
||||
/// </summary>
|
||||
public DateTime? GraphAccessRejectedSince { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The Entra error of the rejection (for example AADSTS7000229), or NoApplicationRole for a
|
||||
/// token without any application permission; null when the response carried no code. System-owned.
|
||||
/// </summary>
|
||||
public string? GraphAccessRejectedCode { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// When a later rejection confirmed the suspicion: the tenant's agents get no validation
|
||||
/// and the portal shows its notice. System-owned, cleared by the next freshly issued token
|
||||
/// that carries an application permission.
|
||||
/// </summary>
|
||||
public DateTime? GraphAccessRejectionConfirmedAt { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this tenant is disabled/suspended
|
||||
/// If true, users from this tenant cannot log in
|
||||
|
||||
@@ -18,6 +18,10 @@ namespace AutopilotMonitor.Shared.Models
|
||||
public bool DiagnosticsUploadConfigured { get; set; }
|
||||
/// <summary>At least one device-validation method is enabled (<see cref="TenantConfiguration.HasAnyDeviceValidation"/>); false drives the "agent ingestion is blocked" dashboard banner.</summary>
|
||||
public bool DeviceValidationEnabled { get; set; }
|
||||
/// <summary>Microsoft Entra has kept refusing the tenant's app token, so agents get no validation; drives the "enrollment monitoring is interrupted" notice. Boolean only, no error detail.</summary>
|
||||
public bool GraphAccessRejected { get; set; }
|
||||
/// <summary>Only with <see cref="GraphAccessRejected"/>: the refusal says the enterprise application is missing from the tenant, which selects the concrete notice text.</summary>
|
||||
public bool GraphAccessAppMissing { get; set; }
|
||||
/// <summary>Dual app-reg self-service migration: consent flow targets the NEW app registration. Non-sensitive — exposes no client ids.</summary>
|
||||
public bool AppHomingFunnelActive { get; set; }
|
||||
public bool ShowScriptOutput { get; set; }
|
||||
|
||||
@@ -134,7 +134,7 @@ namespace AutopilotMonitor.Shared.Models.Deletion
|
||||
/// <c>Details</c> column at 4096 chars (<c>TableOpsEventRepository.cs</c>); the full
|
||||
/// 50-entry sample plus the other JSON-payload fields can blow past that and leave the
|
||||
/// OpsEvent details mid-string-corrupt, which makes the admin UI fall back to raw text
|
||||
/// rendering. The OpsEvent is a Telegram-routable summary, not the forensic record —
|
||||
/// rendering. The OpsEvent is a summary routable through alert rules, not the forensic record —
|
||||
/// operators get the full sample from <c>DeletionProgress.LastResidualSampleJson</c>
|
||||
/// via the Session Cleanup page's stored-manifest modal.
|
||||
/// </summary>
|
||||
|
||||
@@ -44,19 +44,16 @@ namespace AutopilotMonitor.Shared.Models.Notifications
|
||||
/// <summary>
|
||||
/// Provider type (int form of <see cref="WebhookProviderType"/>):
|
||||
/// 2=TeamsWorkflowWebhook, 10=Slack, 20=GenericJson, 30=Discord,
|
||||
/// 40=Telegram (Global-Admin only — see the enum member), 50=Push (paired devices, no
|
||||
/// destination field; open to tenant Admins, at most one Push channel per scope),
|
||||
/// 60=Email (Global-Admin only).
|
||||
/// 50=Push (paired devices, no destination field; open to tenant Admins, at most one Push
|
||||
/// channel per scope), 60=Email (Global-Admin only).
|
||||
/// </summary>
|
||||
public int ProviderType { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Destination (secret — redacted for read-only viewers). For every webhook provider this
|
||||
/// is the endpoint URL; for <see cref="WebhookProviderType.Telegram"/> it is the CHAT ID
|
||||
/// instead, because the endpoint (the bot) is platform-owned. Deliberately one field: the
|
||||
/// redaction and restore-by-id machinery already covers it, so Telegram needs no new
|
||||
/// secret plumbing. Empty for <see cref="WebhookProviderType.Push"/> (see
|
||||
/// <see cref="HasDestination"/>).
|
||||
/// is the endpoint URL; for <see cref="WebhookProviderType.Email"/> it is the recipient
|
||||
/// list. Deliberately one field: the redaction and restore-by-id machinery already covers
|
||||
/// it. Empty for <see cref="WebhookProviderType.Push"/> (see <see cref="HasDestination"/>).
|
||||
/// </summary>
|
||||
public string? Url { get; set; }
|
||||
|
||||
@@ -72,24 +69,22 @@ namespace AutopilotMonitor.Shared.Models.Notifications
|
||||
|
||||
/// <summary>
|
||||
/// Providers a tenant admin may not create or retarget: they deliver to a destination the
|
||||
/// caller names (a chat, mail recipients) through PLATFORM-owned infrastructure (the Telegram
|
||||
/// bot; the platform's e-mail sender and identity). Enforced server-side in
|
||||
/// TenantConfigValidation, mirrored by the <c>gaOnly</c> flag in the web ChannelEditor. Push
|
||||
/// sends through the platform too, but is open to tenant Admins: its only recipients are the
|
||||
/// scope's own paired devices, and a scope holds at most one Push channel.
|
||||
/// caller names (mail recipients) through PLATFORM-owned infrastructure (the platform's e-mail
|
||||
/// sender and identity). Enforced server-side in TenantConfigValidation, mirrored by the
|
||||
/// <c>gaOnly</c> flag in the web ChannelEditor. Push sends through the platform too, but is
|
||||
/// open to tenant Admins: its only recipients are the scope's own paired devices, and a scope
|
||||
/// holds at most one Push channel.
|
||||
/// </summary>
|
||||
public static bool IsGlobalAdminOnlyProvider(int providerType)
|
||||
=> providerType == (int)WebhookProviderType.Telegram
|
||||
|| providerType == (int)WebhookProviderType.Email;
|
||||
=> providerType == (int)WebhookProviderType.Email;
|
||||
|
||||
/// <summary>
|
||||
/// True for a provider whose destination is a caller-supplied webhook URL, rendered and posted
|
||||
/// through the SSRF-guarded webhook path: every supported provider except Telegram, Push and
|
||||
/// E-mail. The legacy single-webhook field accepts only these.
|
||||
/// through the SSRF-guarded webhook path: every supported provider except Push and E-mail. The
|
||||
/// legacy single-webhook field accepts only these.
|
||||
/// </summary>
|
||||
public static bool IsWebhookUrlProvider(int providerType)
|
||||
=> IsSupportedProviderType(providerType)
|
||||
&& providerType != (int)WebhookProviderType.Telegram
|
||||
&& providerType != (int)WebhookProviderType.Push
|
||||
&& providerType != (int)WebhookProviderType.Email;
|
||||
|
||||
|
||||
@@ -31,15 +31,7 @@ namespace AutopilotMonitor.Shared.Models.Notifications
|
||||
/// </summary>
|
||||
Discord = 30,
|
||||
|
||||
/// <summary>
|
||||
/// Telegram chat (plain-text message via the platform bot). The odd one out: this is not
|
||||
/// a webhook. The channel's <c>Url</c> carries the destination CHAT ID, and the bot token
|
||||
/// belongs to the platform (PreviewConfig <c>WebhookUrl</c>), not to the tenant — a caller
|
||||
/// configuring one sends through OUR bot. That is why Telegram channels are Global-Admin
|
||||
/// only (enforced in TenantConfigValidation, not just hidden in the UI), and why they are
|
||||
/// dispatched by TelegramNotificationService instead of a renderer.
|
||||
/// </summary>
|
||||
Telegram = 40,
|
||||
// 40 is retired as well. Never reuse it, for the same reason as value 1.
|
||||
|
||||
/// <summary>
|
||||
/// Web Push to devices the channel's recipients paired in the portal (RFC 8030/8291/8292,
|
||||
@@ -56,8 +48,8 @@ namespace AutopilotMonitor.Shared.Models.Notifications
|
||||
/// E-mail through the platform's own sender (the same provider path as the welcome and
|
||||
/// farewell mails, <c>noreply@autopilotmonitor.com</c>). Not a webhook: <c>Url</c> carries
|
||||
/// the recipients — one to five addresses, ';' or ',' separated — and the message is rendered
|
||||
/// by the backend. Global-Admin only (same gate as Telegram, enforced in
|
||||
/// TenantConfigValidation) because it sends under the platform's identity and API key.
|
||||
/// by the backend. Global-Admin only (enforced in TenantConfigValidation) because it sends
|
||||
/// under the platform's identity and API key.
|
||||
/// </summary>
|
||||
Email = 60,
|
||||
}
|
||||
|
||||
@@ -91,18 +91,6 @@ interface AdminConfigContextValue {
|
||||
// Ops Alert settings
|
||||
opsAlertRules: OpsAlertRule[];
|
||||
setOpsAlertRules: React.Dispatch<React.SetStateAction<OpsAlertRule[]>>;
|
||||
opsAlertTelegramEnabled: boolean;
|
||||
setOpsAlertTelegramEnabled: (value: boolean) => void;
|
||||
opsAlertTelegramChatId: string;
|
||||
setOpsAlertTelegramChatId: (value: string) => void;
|
||||
opsAlertTeamsEnabled: boolean;
|
||||
setOpsAlertTeamsEnabled: (value: boolean) => void;
|
||||
opsAlertTeamsWebhookUrl: string;
|
||||
setOpsAlertTeamsWebhookUrl: (value: string) => void;
|
||||
opsAlertSlackEnabled: boolean;
|
||||
setOpsAlertSlackEnabled: (value: boolean) => void;
|
||||
opsAlertSlackWebhookUrl: string;
|
||||
setOpsAlertSlackWebhookUrl: (value: string) => void;
|
||||
excessiveEventCountThreshold: number;
|
||||
excessiveEventAutoActionMode: "Off" | "Block" | "Kill";
|
||||
excessiveEventAutoActionThreshold: number;
|
||||
@@ -197,12 +185,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
const [testingOpsChannelId, setTestingOpsChannelId] = useState<string | null>(null);
|
||||
const [testOpsChannelResult, setTestOpsChannelResult] =
|
||||
useState<{ channelId: string; success: boolean; message: string } | null>(null);
|
||||
const [opsAlertTelegramEnabled, setOpsAlertTelegramEnabled] = useState(false);
|
||||
const [opsAlertTelegramChatId, setOpsAlertTelegramChatId] = useState("");
|
||||
const [opsAlertTeamsEnabled, setOpsAlertTeamsEnabled] = useState(false);
|
||||
const [opsAlertTeamsWebhookUrl, setOpsAlertTeamsWebhookUrl] = useState("");
|
||||
const [opsAlertSlackEnabled, setOpsAlertSlackEnabled] = useState(false);
|
||||
const [opsAlertSlackWebhookUrl, setOpsAlertSlackWebhookUrl] = useState("");
|
||||
const [excessiveEventCountThreshold, setExcessiveEventCountThreshold] = useState(2000);
|
||||
const [excessiveEventAutoActionMode, setExcessiveEventAutoActionMode] = useState<"Off" | "Block" | "Kill">("Off");
|
||||
const [excessiveEventAutoActionThreshold, setExcessiveEventAutoActionThreshold] = useState(2500);
|
||||
@@ -274,12 +256,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
} catch {
|
||||
setOpsNotificationChannels([]);
|
||||
}
|
||||
setOpsAlertTelegramEnabled(data.opsAlertTelegramEnabled ?? false);
|
||||
setOpsAlertTelegramChatId(data.opsAlertTelegramChatId ?? "");
|
||||
setOpsAlertTeamsEnabled(data.opsAlertTeamsEnabled ?? false);
|
||||
setOpsAlertTeamsWebhookUrl(data.opsAlertTeamsWebhookUrl ?? "");
|
||||
setOpsAlertSlackEnabled(data.opsAlertSlackEnabled ?? false);
|
||||
setOpsAlertSlackWebhookUrl(data.opsAlertSlackWebhookUrl ?? "");
|
||||
setExcessiveEventCountThreshold(data.excessiveEventCountThreshold ?? 2000);
|
||||
setExcessiveEventAutoActionMode(data.excessiveEventAutoActionMode ?? "Off");
|
||||
setExcessiveEventAutoActionThreshold(data.excessiveEventAutoActionThreshold ?? 2500);
|
||||
@@ -467,7 +443,7 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
|
||||
/**
|
||||
* Sends a test notification to one SAVED ops channel. Deliberately server-side: only the
|
||||
* backend holds the Telegram bot token and the webhook destinations, and only it applies the
|
||||
* backend holds the webhook destinations and the e-mail provider key, and only it applies the
|
||||
* SSRF gate — the browser must never post to a configured endpoint directly.
|
||||
*/
|
||||
const handleTestOpsChannel = useCallback(async (channelId: string) => {
|
||||
@@ -521,12 +497,6 @@ export function AdminConfigProvider({ children }: { children: React.ReactNode })
|
||||
maxImeMsiDownloadSizeMB, setMaxImeMsiDownloadSizeMB,
|
||||
globalDiagPaths, setGlobalDiagPaths, savingDiagPaths,
|
||||
opsAlertRules, setOpsAlertRules,
|
||||
opsAlertTelegramEnabled, setOpsAlertTelegramEnabled,
|
||||
opsAlertTelegramChatId, setOpsAlertTelegramChatId,
|
||||
opsAlertTeamsEnabled, setOpsAlertTeamsEnabled,
|
||||
opsAlertTeamsWebhookUrl, setOpsAlertTeamsWebhookUrl,
|
||||
opsAlertSlackEnabled, setOpsAlertSlackEnabled,
|
||||
opsAlertSlackWebhookUrl, setOpsAlertSlackWebhookUrl,
|
||||
excessiveEventCountThreshold,
|
||||
excessiveEventAutoActionMode,
|
||||
excessiveEventAutoActionThreshold,
|
||||
|
||||
@@ -28,6 +28,12 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// the operator to re-assign app roles on the new enterprise app.
|
||||
"AppHomingFlipped",
|
||||
"AppHomingFlippedWithEntraRoles",
|
||||
// Entra keeps refusing a tenant's app token (Error, after 15 min) and issues it again (Info).
|
||||
// Credential: our own client secret is refused (Critical, hits every tenant on that app).
|
||||
// Backend helpers RecordGraphAccess{Rejected,Recovered}Async, RecordGraphAppCredentialRejectedAsync.
|
||||
"GraphAccessRejected",
|
||||
"GraphAccessRecovered",
|
||||
"GraphAppCredentialRejected",
|
||||
],
|
||||
Maintenance: [
|
||||
// Verdict-calibration drift radar (operator-only classifier diagnostics): a verdict path's
|
||||
@@ -78,19 +84,19 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
"SessionDeletionStrandedQueued",
|
||||
// PR-B audit consolidation: the per-session deletion_poisoned tenant-audit moved here so
|
||||
// tenant admins see only the lifecycle endpoints (deletion_started/completed/restored).
|
||||
// Operators wire Telegram on this event + read it in the Session Cleanup admin page.
|
||||
// Operators wire an alert rule on this event + read it in the Session Cleanup admin page.
|
||||
"SessionDeletionPoisoned",
|
||||
// Critical-table backup (plan §PR1) — dual-register per memory feedback_ops_event_types_dual_register.
|
||||
// Dispatched by CriticalTableBackupQueueWorker + CriticalTableBackupTimerFunction.
|
||||
// Backend helpers: RecordCriticalTableBackup{Completed,Partial,Failed,SkippedLocked}Async.
|
||||
// Partial is a distinct type (Warning severity) so operators can wire a softer
|
||||
// Telegram rule than for Failed (Error severity, fatal manifest-write failure).
|
||||
// alert rule than for Failed (Error severity, fatal manifest-write failure).
|
||||
"CriticalTableBackupCompleted",
|
||||
"CriticalTableBackupPartial",
|
||||
"CriticalTableBackupFailed",
|
||||
"CriticalTableBackupSkippedLocked",
|
||||
// PR2: GA restored a single row from a backup. Warning severity so operators
|
||||
// can wire a Telegram rule and see the audit in near-real-time.
|
||||
// can wire an alert rule and see the audit in near-real-time.
|
||||
"BackupRowRestored",
|
||||
// Orphan-session sweep (OrphanSessionSweepFunction, every 4h at :45) — dual-register per
|
||||
// memory feedback_ops_event_types_dual_register. Backend helpers RecordOrphanSweep{Completed,
|
||||
@@ -119,13 +125,13 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// Critical-tier auto-action: emitted when maintenance auto-blocks/kills a device
|
||||
// after its session crosses `excessiveEventAutoActionThreshold`. Dual-register
|
||||
// per memory feedback_ops_event_types_dual_register so operators can wire a
|
||||
// dedicated Telegram rule independent of the warn-tier ExcessiveSessionEvents.
|
||||
// dedicated alert rule independent of the warn-tier ExcessiveSessionEvents.
|
||||
"ExcessiveSessionEventsAutoActioned",
|
||||
// Assume-breach layer for the Global-Admin-only surface: an authenticated caller without the
|
||||
// GlobalAdmin role hit a GlobalAdminOnly route (or, via the MCP probe, a GA-only tool).
|
||||
// Critical for callers without any platform role, Warning for a Global Reader. Flood-capped
|
||||
// per instance: one event per caller per hour and at most 5 per hour in total, then ONE
|
||||
// Critical "storm" marker (PrivilegedDenialReporter). Route it to Telegram at MinSeverity Critical.
|
||||
// Critical "storm" marker (PrivilegedDenialReporter). Route it to a channel at MinSeverity Critical.
|
||||
"PrivilegedRouteDenied",
|
||||
// A service principal (app-only token, key app:<client-id>) opened its first MCP session — once per
|
||||
// application and home tenant, so new automation is visible the moment it starts.
|
||||
@@ -161,9 +167,9 @@ const OPS_EVENT_TYPES: Record<string, string[]> = {
|
||||
// TenantOffboardingFailed is dispatched by TenantOffboardingWorker when the async cascade fails
|
||||
// closed (drain timeout, KillSwitch active, Poisoned session, ETag/CAS exhaustion, SafeWipe verify
|
||||
// abort, etc.). Marker remains in Failed state until operator action — this event is the
|
||||
// Telegram-routable signal that something needs human attention. Plan Rev-4 Q2 + Rev-9 §11 PR3.
|
||||
// routable signal that something needs human attention. Plan Rev-4 Q2 + Rev-9 §11 PR3.
|
||||
// OffboardingFeedbackReceived fires when a departing admin submits free-form feedback
|
||||
// in the drain-barrier banner. Information-tier — wire Telegram to get pinged when
|
||||
// in the drain-barrier banner. Information-tier — wire a channel to get pinged when
|
||||
// someone leaves a comment so you can read it promptly.
|
||||
// TenantTrialExpiring/Expired — dual-register per memory feedback_ops_event_types_dual_register.
|
||||
// Dispatched by TrialExpirySweepFunction (daily 03:30 UTC, informational — enforcement is
|
||||
@@ -559,7 +565,7 @@ export function OpsAlertRulesSection({
|
||||
)}
|
||||
{et === "ExcessiveSessionEventsAutoActioned" && (
|
||||
// Auto-action controls always render under the rule row (independent of `rule.enabled`,
|
||||
// which only governs Telegram routing — the auto-block/kill itself fires from
|
||||
// which only governs channel routing — the auto-block/kill itself fires from
|
||||
// maintenance regardless). Operators can opt in/out via the Off mode here.
|
||||
<div className="ml-12 mt-1 mb-2 space-y-1.5">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
@@ -639,7 +645,7 @@ export function OpsAlertRulesSection({
|
||||
onTest={() => onTestChannel(channel.id)}
|
||||
testing={testingChannelId === channel.id}
|
||||
testResult={testChannelResult?.channelId === channel.id ? testChannelResult : null}
|
||||
showTelegramProvider
|
||||
showPlatformProviders
|
||||
showEventToggles={false}
|
||||
pushDestinationHint={PUSH_DESTINATION_HINT_OPS}
|
||||
health={channelHealth?.get(channel.id) ?? null}
|
||||
|
||||
+1
-1
@@ -34,6 +34,6 @@ describe("filterOpsEventCatalog", () => {
|
||||
["Consent", ["ConsentFlowFailed"]],
|
||||
["Platform", ["PushDeliveryFailed"]],
|
||||
]);
|
||||
expect(filterOpsEventCatalog(CATALOG, "telegram")).toEqual([]);
|
||||
expect(filterOpsEventCatalog(CATALOG, "no-such-event")).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ const SERVER_DENIED_FIELDS = new Set([
|
||||
"lastUpdated", "updatedBy", "onboardedAt", "onboardedBy", "dpaVersion",
|
||||
"dpaAcceptancePending", "dpaAcceptedBy", "dpaAcceptedAt",
|
||||
"homedAppClientId", "lastAuthClientId", "lastAuthClientIdSince",
|
||||
"graphAccessRejectedSince", "graphAccessRejectedCode", "graphAccessRejectionConfirmedAt",
|
||||
"planTier", "trialExpiresUtc", "trialStartedUtc", "trialConsumed", "trialGrantedBy",
|
||||
"proDowngradedUtc", "maxDelegatedTenantsOverride", "mcpUsagePlanOverride", "payingCustomer",
|
||||
"managedByProTenantId",
|
||||
|
||||
@@ -406,7 +406,7 @@ function ActivityTab() {
|
||||
(Stranded tab),
|
||||
<code className="mx-1 px-1 bg-gray-100 dark:bg-gray-700 rounded text-xs">SessionDeletionMaintenanceFanoutSkipped</code>
|
||||
(kill-switch active during a retention sweep). All available on the Ops Events page and
|
||||
wirable for Telegram alerts.
|
||||
wirable for alert rules.
|
||||
</p>
|
||||
<Link
|
||||
href="/audit"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user