mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-11-03 09:03:29 +00:00
Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6866fe79af | |||
| 6e34fa517c | |||
| dc05428dae | |||
| cd0c750e80 | |||
| cede4be654 | |||
| 0d63057725 | |||
| 7dd151b714 | |||
| 778eb44a34 | |||
| 00f37d869a | |||
| d88a118903 | |||
| 0ed3258d4f | |||
| 3941c3b984 | |||
| 3d831d16d7 | |||
| 46b895a70d | |||
| 77fae059ed | |||
| fd309da51d | |||
| 9c4039b3c3 | |||
| b0fb8eb5fa | |||
| dd2432fa62 | |||
| d9b67e444a | |||
| 47b91f76e7 | |||
| 9c6b55a8bc | |||
| f00b475f59 | |||
| f8743dcae0 | |||
| bd94382edb | |||
| 8abd3a8848 |
+1
-1
Submodule .claude/commands updated: d937abd228...f8941093f4
@@ -105,7 +105,7 @@ jobs:
|
||||
# the local install so the gate and the pre-commit run agree on the finding set.
|
||||
# Informational findings (template expansion of our own step outputs in pwsh) are
|
||||
# visible locally but do not fail the gate.
|
||||
- uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
|
||||
- uses: zizmorcore/zizmor-action@53290badfa369cb0bf7ad16342cb62c2fa47ac7b # v0.8.0
|
||||
with:
|
||||
version: 1.30.1
|
||||
min-severity: low
|
||||
|
||||
@@ -247,7 +247,7 @@ jobs:
|
||||
Get-ChildItem $outDir | ForEach-Object { Write-Host "$($_.Name): $([math]::Round($_.Length / 1MB, 2)) MB" }
|
||||
|
||||
- name: Upload unsigned build output
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: agent-unsigned
|
||||
path: ${{ runner.temp }}/agent-unsigned
|
||||
@@ -293,7 +293,7 @@ jobs:
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download unsigned build output
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: agent-unsigned
|
||||
path: ${{ runner.temp }}/agent-unsigned
|
||||
@@ -790,13 +790,13 @@ jobs:
|
||||
staging/agent/AutopilotMonitor.SummaryDialog.exe
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: AutopilotMonitor-Agent-v${{ needs.reserve.outputs.version }}
|
||||
path: ${{ steps.artifact.outputs.versioned_zip_release }}
|
||||
|
||||
- name: Upload version metadata
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: version-metadata
|
||||
path: ${{ env.MANIFEST_NAME }}
|
||||
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
Write-Host "$env:RUNNER_SCRIPT : Valid, timestamped, signer $($sig.SignerCertificate.Subject.Split(',')[0])"
|
||||
|
||||
- name: Upload signed loader
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: bootstrap-loader-signed
|
||||
path: ${{ env.RUNNER_SCRIPT }}
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download signed loader
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: bootstrap-loader-signed
|
||||
path: ${{ runner.temp }}/loader
|
||||
@@ -187,7 +187,7 @@ jobs:
|
||||
Write-Host "Built $env:STABLE_MSI ($([math]::Round((Get-Item $env:STABLE_MSI).Length / 1KB)) KB)"
|
||||
|
||||
- name: Upload unsigned MSI
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: bootstrap-msi-unsigned
|
||||
path: ${{ env.STABLE_MSI }}
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download unsigned MSI
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: bootstrap-msi-unsigned
|
||||
path: ${{ runner.temp }}/msi
|
||||
@@ -310,7 +310,7 @@ jobs:
|
||||
"version=$version" >> $env:GITHUB_OUTPUT
|
||||
|
||||
- name: Download unsigned MSI
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: bootstrap-msi-unsigned
|
||||
path: ${{ runner.temp }}/msi
|
||||
@@ -442,7 +442,7 @@ jobs:
|
||||
subject-path: ${{ env.STABLE_MSI }}
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: AutopilotMonitor-Bootstrap-v${{ steps.version.outputs.version }}
|
||||
path: ${{ env.STABLE_MSI }}
|
||||
|
||||
@@ -134,7 +134,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -176,7 +176,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -245,7 +245,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -276,7 +276,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
dotnet-version: '10.0.x'
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
uses: github/codeql-action/init@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
build-mode: ${{ matrix.build-mode }}
|
||||
@@ -123,6 +123,6 @@ jobs:
|
||||
run: dotnet build AutopilotMonitor.sln --configuration Release
|
||||
|
||||
- name: Analyze
|
||||
uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
|
||||
uses: github/codeql-action/analyze@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1
|
||||
with:
|
||||
category: /language:${{ matrix.language }}
|
||||
|
||||
@@ -32,7 +32,7 @@ jobs:
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
# `git push` needs.
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ jobs:
|
||||
run: tar -czf "$RUNNER_TEMP/backend-publish.tgz" -C publish .
|
||||
|
||||
- name: Upload publish output
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: backend-publish
|
||||
path: ${{ runner.temp }}/backend-publish.tgz
|
||||
@@ -128,7 +128,7 @@ jobs:
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download publish output
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: backend-publish
|
||||
path: ${{ runner.temp }}/backend-artifact
|
||||
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
id: docs
|
||||
run: echo "sha=$(git -C whats-new-docs rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
sha256sum "$archive"
|
||||
|
||||
- name: Upload build artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ steps.package.outputs.archive }}
|
||||
@@ -170,7 +170,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/portal
|
||||
@@ -215,7 +215,7 @@ jobs:
|
||||
run: ./.github/scripts/Get-Endpoints.ps1 -ExportToGitHubEnv
|
||||
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
||||
with:
|
||||
name: portal-${{ github.sha }}
|
||||
path: ${{ runner.temp }}/portal
|
||||
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
# when someone wants to look at the file.
|
||||
- name: Upload signed scripts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
||||
with:
|
||||
name: signed-scripts
|
||||
path: ${{ runner.temp }}/signed-scripts
|
||||
|
||||
+1
-1
Submodule internal updated: af17a42883...f161dac6f8
+162
@@ -0,0 +1,162 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
{
|
||||
/// <summary>The watchers only listen on the real keys; settings are played in, none is changed.</summary>
|
||||
public sealed class ProxySettingsHostTests : IDisposable
|
||||
{
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly FakeSignalIngressSink _ingress = new FakeSignalIngressSink();
|
||||
private readonly VirtualClock _clock = new VirtualClock(new DateTime(2026, 11, 2, 6, 53, 43, DateTimeKind.Utc));
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly ProxyObserver _observer;
|
||||
private ProxySettings _settings = new ProxySettings(true, null, null, null, null);
|
||||
private string _route = "proxy.example.test:8080";
|
||||
|
||||
public ProxySettingsHostTests()
|
||||
{
|
||||
_logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
_observer = new ProxyObserver(() => _settings, () => _route, _logger, _clock, TimeSpan.FromSeconds(30));
|
||||
_observer.CaptureBaseline();
|
||||
}
|
||||
|
||||
private ProxySettingsHost NewHost() => new ProxySettingsHost(_observer, _ingress, _clock, _logger);
|
||||
|
||||
private void SwitchToNoProxy()
|
||||
{
|
||||
_settings = new ProxySettings(false, null, null, null, null);
|
||||
_route = "direct";
|
||||
}
|
||||
|
||||
private List<FakeSignalIngressSink.PostedSignal> ProxyEvents() =>
|
||||
_ingress.Posted
|
||||
.Where(p => p.Kind == DecisionSignalKind.InformationalEvent
|
||||
&& p.Payload != null
|
||||
&& p.Payload.TryGetValue(SignalPayloadKeys.EventType, out var eventType)
|
||||
&& eventType == Constants.EventTypes.ProxyConfiguration)
|
||||
.ToList();
|
||||
|
||||
private string Log()
|
||||
{
|
||||
var text = string.Empty;
|
||||
foreach (var file in Directory.EnumerateFiles(_tmp.Path, "*.log", SearchOption.AllDirectories))
|
||||
{
|
||||
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
||||
using var reader = new StreamReader(stream);
|
||||
text += reader.ReadToEnd();
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
[Fact]
|
||||
public void A_change_goes_on_the_timeline_as_a_warning_with_what_moved()
|
||||
{
|
||||
using var host = NewHost();
|
||||
host.Start();
|
||||
SwitchToNoProxy();
|
||||
|
||||
_observer.CheckForChange();
|
||||
|
||||
var signal = Assert.Single(ProxyEvents());
|
||||
Assert.Equal(nameof(EventSeverity.Warning), signal.Payload![SignalPayloadKeys.Severity]);
|
||||
Assert.Equal("Proxy settings changed: IE settings of this account: auto-detect -> no proxy", signal.Payload[SignalPayloadKeys.Message]);
|
||||
|
||||
var data = Assert.IsAssignableFrom<IReadOnlyDictionary<string, object>>(signal.TypedPayload);
|
||||
Assert.Equal("changed", data["trigger"]);
|
||||
Assert.Equal("IE settings of this account: auto-detect -> no proxy", data["change"]);
|
||||
Assert.Equal("direct", data["route"]);
|
||||
Assert.Equal("Direct", data["proxyType"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_between_the_baseline_and_the_start_is_noticed_at_the_start()
|
||||
{
|
||||
SwitchToNoProxy();
|
||||
|
||||
using var host = NewHost();
|
||||
host.Start();
|
||||
|
||||
Assert.Single(ProxyEvents());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void No_change_means_no_event()
|
||||
{
|
||||
using var host = NewHost();
|
||||
host.Start();
|
||||
|
||||
_observer.CheckForChange();
|
||||
|
||||
Assert.Empty(ProxyEvents());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_stopped_ingress_does_not_reach_the_upload_path()
|
||||
{
|
||||
using var host = NewHost();
|
||||
host.Start();
|
||||
_ingress.ThrowOnPost = new InvalidOperationException("SignalIngress has been stopped.");
|
||||
SwitchToNoProxy();
|
||||
|
||||
_observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
Assert.Contains("uploads are failing", Log());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopping_while_uploads_fail_says_so_once()
|
||||
{
|
||||
var host = NewHost();
|
||||
host.Start();
|
||||
_observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
host.Stop();
|
||||
host.Stop();
|
||||
host.Dispose();
|
||||
|
||||
var log = Log();
|
||||
var first = log.IndexOf("stopping with uploads still failing", StringComparison.Ordinal);
|
||||
Assert.True(first >= 0);
|
||||
Assert.Equal(-1, log.IndexOf("stopping with uploads still failing", first + 1, StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_after_the_stop_no_longer_posts()
|
||||
{
|
||||
var host = NewHost();
|
||||
host.Start();
|
||||
host.Stop();
|
||||
SwitchToNoProxy();
|
||||
|
||||
_observer.CheckForChange();
|
||||
|
||||
Assert.Empty(ProxyEvents());
|
||||
host.Dispose();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_host_that_never_started_stays_silent_on_stop()
|
||||
{
|
||||
var host = NewHost();
|
||||
_observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
host.Dispose();
|
||||
|
||||
Assert.DoesNotContain("stopping with uploads still failing", Log());
|
||||
}
|
||||
}
|
||||
}
|
||||
+130
@@ -6,6 +6,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport.Telemetry;
|
||||
using Xunit;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
@@ -35,6 +36,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
requiresImmediateFlush: false,
|
||||
enqueuedAtUtc: At);
|
||||
|
||||
private static ProxyObserver NewProxyObserver(AgentLogger logger) =>
|
||||
new ProxyObserver(
|
||||
() => new ProxySettings(false, null, null, null, null),
|
||||
() => "direct",
|
||||
logger,
|
||||
new VirtualClock(At),
|
||||
TimeSpan.FromSeconds(30));
|
||||
|
||||
private static string ReadLogs(string path)
|
||||
{
|
||||
// AgentLogger writes to a dated file; scan the whole dir to stay robust to format.
|
||||
@@ -235,5 +244,126 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.DoesNotContain("flushing", logContent);
|
||||
Assert.DoesNotContain("ingest", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Network_failure_names_the_route_and_opens_a_series()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueThrow(new HttpRequestException("An error occurred while sending the request."));
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger, proxy: NewProxyObserver(logger));
|
||||
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains("ingest NETWORK fail after", logContent);
|
||||
Assert.Contains("items=1, route=direct): ", logContent);
|
||||
Assert.Contains("ProxyObserver: uploads are failing since", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Timeout_names_the_route_and_opens_a_series()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueThrow(new TaskCanceledException("A task was canceled."));
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger, proxy: NewProxyObserver(logger));
|
||||
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains("ingest TIMEOUT after", logContent);
|
||||
Assert.Contains("items=1, route=direct): ", logContent);
|
||||
Assert.Contains("ProxyObserver: uploads are failing since", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Interception_names_the_route_and_opens_a_series()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueForeign(HttpStatusCode.Forbidden, InterceptionFixtures.CertificateRequiredPage);
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger, proxy: NewProxyObserver(logger));
|
||||
|
||||
var result = await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
Assert.True(result.IsIntercepted);
|
||||
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains(", route=direct).", logContent);
|
||||
Assert.Contains("ProxyObserver: uploads are failing since", logContent);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_successful_upload_ends_the_series()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueThrow(new HttpRequestException("An error occurred while sending the request."));
|
||||
handler.QueueStatus(HttpStatusCode.OK);
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger, proxy: NewProxyObserver(logger));
|
||||
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.Contains("ProxyObserver: uploads work again after 1 failed attempt(s)", ReadLogs(tmp.Path));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_answer_from_the_backend_that_is_not_a_success_leaves_the_series_alone()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueStatus(HttpStatusCode.InternalServerError);
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger, proxy: NewProxyObserver(logger));
|
||||
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.DoesNotContain("ProxyObserver: uploads", ReadLogs(tmp.Path));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Without_an_observer_the_failure_line_stays_as_it_was()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var logger = new AgentLogger(tmp.Path, AgentLogLevel.Info);
|
||||
|
||||
var handler = new RecordingHttpMessageHandler();
|
||||
handler.QueueThrow(new HttpRequestException("An error occurred while sending the request."));
|
||||
using var http = new HttpClient(handler) { Timeout = TimeSpan.FromSeconds(30) };
|
||||
|
||||
var sut = new BackendTelemetryUploader(
|
||||
httpClient: http, baseUrl: "https://backend.test", tenantId: "T1", logger: logger);
|
||||
|
||||
await sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
var logContent = ReadLogs(tmp.Path);
|
||||
Assert.Contains("items=1): ", logContent);
|
||||
Assert.DoesNotContain("route=", logContent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
/// <summary>Settings and route are played in: no test changes a real proxy setting.</summary>
|
||||
public sealed class ProxyObserverTests : IDisposable
|
||||
{
|
||||
private static readonly ProxySettings AutoDetect = new ProxySettings(true, null, null, null, null);
|
||||
private static readonly ProxySettings NoProxy = new ProxySettings(false, null, null, null, null);
|
||||
|
||||
private readonly TempDirectory _tmp = new TempDirectory();
|
||||
private readonly VirtualClock _clock = new VirtualClock(new DateTime(2026, 11, 2, 6, 29, 13, DateTimeKind.Utc));
|
||||
private ProxySettings _settings = AutoDetect;
|
||||
private string _route = "proxy.example.test:8080";
|
||||
|
||||
private ProxyObserver NewObserver(Func<string>? readRoute = null, TimeSpan? routeBound = null, Func<ProxySettings>? readSettings = null) =>
|
||||
new ProxyObserver(
|
||||
readSettings ?? (() => _settings),
|
||||
readRoute ?? (() => _route),
|
||||
new AgentLogger(_tmp.Path, AgentLogLevel.Info),
|
||||
_clock,
|
||||
routeBound ?? TimeSpan.FromSeconds(30));
|
||||
|
||||
private ProxyObserver StartedObserver()
|
||||
{
|
||||
var observer = NewObserver();
|
||||
observer.CaptureBaseline();
|
||||
return observer;
|
||||
}
|
||||
|
||||
private void SwitchToNoProxy()
|
||||
{
|
||||
_settings = NoProxy;
|
||||
_route = "direct";
|
||||
}
|
||||
|
||||
private List<string> LogLines()
|
||||
{
|
||||
var lines = new List<string>();
|
||||
foreach (var file in Directory.EnumerateFiles(_tmp.Path, "*.log", SearchOption.AllDirectories))
|
||||
{
|
||||
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
|
||||
using var reader = new StreamReader(stream);
|
||||
lines.AddRange(reader.ReadToEnd().Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries));
|
||||
}
|
||||
return lines.Where(l => l.Contains("ProxyObserver:")).ToList();
|
||||
}
|
||||
|
||||
public void Dispose() => _tmp.Dispose();
|
||||
|
||||
[Fact]
|
||||
public void The_baseline_names_settings_and_route_at_info()
|
||||
{
|
||||
StartedObserver();
|
||||
|
||||
var line = Assert.Single(LogLines());
|
||||
Assert.Contains("[INFO]", line);
|
||||
Assert.Contains("start - IE settings of this account: auto-detect; WinHTTP: direct; route to backend: proxy.example.test:8080", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_rebuild_of_the_clients_does_not_repeat_the_baseline()
|
||||
{
|
||||
var logger = new AgentLogger(_tmp.Path, AgentLogLevel.Info);
|
||||
foreach (var _ in new[] { 1, 2 })
|
||||
{
|
||||
new ProxyObserver(() => _settings, () => _route, logger, _clock, TimeSpan.FromSeconds(30)).CaptureBaseline();
|
||||
}
|
||||
|
||||
Assert.Single(LogLines());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_warns_once_with_both_sides()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
SwitchToNoProxy();
|
||||
|
||||
observer.CheckForChange();
|
||||
observer.CheckForChange();
|
||||
|
||||
var line = Assert.Single(LogLines().Skip(1));
|
||||
Assert.Contains("[WARN]", line);
|
||||
Assert.Contains("proxy settings changed - IE settings of this account: auto-detect -> no proxy; route to backend: proxy.example.test:8080 -> direct.", line);
|
||||
Assert.Contains("bitsadmin /util /getieproxy localsystem", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Rewriting_the_same_content_says_nothing()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
_settings = new ProxySettings(true, null, null, null, null);
|
||||
|
||||
observer.CheckForChange();
|
||||
|
||||
Assert.Single(LogLines());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_winhttp_change_says_uploads_do_not_use_it()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
_settings = new ProxySettings(true, null, null, null, "p:8080");
|
||||
|
||||
observer.CheckForChange();
|
||||
|
||||
var line = LogLines().Last();
|
||||
Assert.Contains("[WARN]", line);
|
||||
Assert.Contains("WinHTTP: direct -> proxy p:8080", line);
|
||||
Assert.Contains("do not use WinHTTP", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_winhttp_change_is_not_blamed_for_failing_uploads()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
_settings = new ProxySettings(true, null, null, null, "p:8080");
|
||||
observer.CheckForChange();
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
Assert.Contains("Proxy settings of this account unchanged since start: auto-detect.", LogLines().Last());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_reaches_the_subscriber_with_settings_and_route()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
ProxySettings? seen = null;
|
||||
string? seenWhat = null;
|
||||
string? seenRoute = null;
|
||||
observer.SettingsChanged += (settings, what, route) =>
|
||||
{
|
||||
seen = settings;
|
||||
seenWhat = what;
|
||||
seenRoute = route;
|
||||
};
|
||||
SwitchToNoProxy();
|
||||
|
||||
observer.CheckForChange();
|
||||
|
||||
Assert.Equal(NoProxy, seen);
|
||||
Assert.Equal("IE settings of this account: auto-detect -> no proxy", seenWhat);
|
||||
Assert.Equal("direct", seenRoute);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_subscriber_that_throws_does_not_reach_the_caller()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
observer.SettingsChanged += (_, _, _) => throw new InvalidOperationException("SignalIngress has been stopped.");
|
||||
SwitchToNoProxy();
|
||||
|
||||
observer.CheckForChange();
|
||||
|
||||
Assert.Contains(LogLines(), l => l.Contains("proxy settings changed"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_first_failure_ties_the_series_to_the_change()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
_clock.Advance(TimeSpan.FromMinutes(23));
|
||||
observer.OnSendSucceeded();
|
||||
_clock.Advance(TimeSpan.FromSeconds(90));
|
||||
SwitchToNoProxy();
|
||||
observer.CheckForChange();
|
||||
_clock.Advance(TimeSpan.FromSeconds(62));
|
||||
|
||||
observer.OnSendFailed(TimeSpan.FromSeconds(21));
|
||||
|
||||
var line = LogLines().Last();
|
||||
Assert.Contains("[WARN]", line);
|
||||
Assert.Contains("uploads are failing since 06:54:24, 41 s after the proxy settings changed at 06:53:43 (IE settings of this account: auto-detect -> no proxy).", line);
|
||||
Assert.Contains("Route to backend: direct. Last successful upload 06:52:13.", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failure_notices_a_change_the_watcher_did_not_report()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
SwitchToNoProxy();
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
var lines = LogLines();
|
||||
Assert.Contains(lines, l => l.Contains("proxy settings changed - IE settings of this account: auto-detect -> no proxy"));
|
||||
Assert.Contains("after the proxy settings changed", lines.Last());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_series_writes_its_finding_once()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
Assert.Single(LogLines(), l => l.Contains("uploads are failing"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failure_without_a_change_says_the_settings_are_unchanged()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
var line = LogLines().Last();
|
||||
Assert.Contains("Proxy settings of this account unchanged since start: auto-detect.", line);
|
||||
Assert.Contains("Route to backend: proxy.example.test:8080. No successful upload yet.", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_before_the_last_success_is_not_blamed()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
SwitchToNoProxy();
|
||||
observer.CheckForChange();
|
||||
_clock.Advance(TimeSpan.FromMinutes(1));
|
||||
observer.OnSendSucceeded();
|
||||
_clock.Advance(TimeSpan.FromMinutes(1));
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
Assert.Contains("Proxy settings of this account unchanged since the last successful upload: no proxy.", LogLines().Last());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Recovery_is_noted_and_a_new_series_gets_its_own_finding()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
_clock.Advance(TimeSpan.FromMinutes(9));
|
||||
|
||||
observer.OnSendSucceeded();
|
||||
|
||||
var line = LogLines().Last();
|
||||
Assert.Contains("[INFO]", line);
|
||||
Assert.Contains("uploads work again after 2 failed attempt(s) over 9 min; route to backend: proxy.example.test:8080.", line);
|
||||
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
Assert.Equal(2, LogLines().Count(l => l.Contains("uploads are failing")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Recovery_reports_settings_that_came_back()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
SwitchToNoProxy();
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
_settings = AutoDetect;
|
||||
_route = "proxy.example.test:8080";
|
||||
|
||||
observer.OnSendSucceeded();
|
||||
|
||||
Assert.Contains(LogLines(), l => l.Contains("no proxy -> auto-detect"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_success_without_a_series_stays_silent()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
|
||||
observer.OnSendSucceeded();
|
||||
|
||||
Assert.Single(LogLines());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopping_while_failing_sums_it_up()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
observer.OnSendSucceeded();
|
||||
_clock.Advance(TimeSpan.FromMinutes(1));
|
||||
SwitchToNoProxy();
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
observer.LogOpenFailures();
|
||||
|
||||
var line = LogLines().Last();
|
||||
Assert.Contains("[WARN]", line);
|
||||
Assert.Contains("stopping with uploads still failing: 2 failed attempt(s) since 06:30:13, last success 06:29:13, route to backend: direct.", line);
|
||||
Assert.Contains("Proxy settings changed at 06:30:13 (IE settings of this account: auto-detect -> no proxy).", line);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Stopping_without_a_series_stays_silent()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
observer.OnSendSucceeded();
|
||||
var before = LogLines().Count;
|
||||
|
||||
observer.LogOpenFailures();
|
||||
|
||||
Assert.Equal(before, LogLines().Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_settings_read_that_throws_is_not_a_change_and_never_reaches_the_upload()
|
||||
{
|
||||
var throwing = false;
|
||||
var observer = NewObserver(readSettings: () => throwing ? throw new InvalidOperationException("boom") : _settings);
|
||||
observer.CaptureBaseline();
|
||||
throwing = true;
|
||||
|
||||
observer.CheckForChange();
|
||||
observer.OnSendFailed(TimeSpan.Zero);
|
||||
|
||||
Assert.DoesNotContain(LogLines(), l => l.Contains("proxy settings changed"));
|
||||
Assert.Contains("Proxy settings of this account unchanged since start: auto-detect.", LogLines().Last());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unreadable_state_is_not_a_change()
|
||||
{
|
||||
var observer = StartedObserver();
|
||||
_settings = ProxySettings.Unreadable(5);
|
||||
|
||||
observer.CheckForChange();
|
||||
|
||||
Assert.Single(LogLines());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_route_read_that_throws_reads_as_unknown()
|
||||
{
|
||||
var observer = NewObserver(readRoute: () => throw new InvalidOperationException("boom"));
|
||||
|
||||
Assert.Equal("unknown", observer.RouteText());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_hanging_route_read_gives_up_and_does_not_pile_up()
|
||||
{
|
||||
using var release = new ManualResetEventSlim(false);
|
||||
var reads = 0;
|
||||
var observer = NewObserver(
|
||||
readRoute: () =>
|
||||
{
|
||||
Interlocked.Increment(ref reads);
|
||||
release.Wait();
|
||||
return "direct";
|
||||
},
|
||||
routeBound: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
Assert.Equal("unknown", observer.RouteText());
|
||||
Assert.Equal("unknown", observer.RouteText());
|
||||
Assert.Equal(1, Volatile.Read(ref reads));
|
||||
|
||||
release.Set();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class ProxySettingsTests
|
||||
{
|
||||
private static ProxySettings Settings(
|
||||
bool autoDetect = false, string? pac = null, string? proxy = null, string? bypass = null, string? winHttp = null) =>
|
||||
new ProxySettings(autoDetect, pac, proxy, bypass, winHttp);
|
||||
|
||||
[Fact]
|
||||
public void Each_kind_of_setting_describes_itself()
|
||||
{
|
||||
Assert.Equal("no proxy", Settings().Describe());
|
||||
Assert.Equal("auto-detect", Settings(autoDetect: true).Describe());
|
||||
Assert.Equal("PAC https://pac.example.test/proxy.pac", Settings(pac: "https://pac.example.test/proxy.pac").Describe());
|
||||
Assert.Equal("proxy proxy.example.test:8080", Settings(proxy: "proxy.example.test:8080").Describe());
|
||||
Assert.Equal("auto-detect, proxy p:8080", Settings(autoDetect: true, proxy: "p:8080").Describe());
|
||||
Assert.Equal("direct", Settings().DescribeWinHttp());
|
||||
Assert.Equal("proxy p:8080", Settings(winHttp: "p:8080").DescribeWinHttp());
|
||||
Assert.Equal("unreadable (Win32 error 5)", ProxySettings.Unreadable(5).Describe());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(false, null, null, "Direct")]
|
||||
[InlineData(true, null, null, "AutoDetect")]
|
||||
[InlineData(true, "https://pac.example.test/p.pac", null, "PAC")]
|
||||
[InlineData(true, "https://pac.example.test/p.pac", "p:8080", "Proxy")]
|
||||
public void Type_keeps_the_values_the_download_rule_matches(bool autoDetect, string? pac, string? proxy, string expected)
|
||||
{
|
||||
Assert.Equal(expected, Settings(autoDetect, pac, proxy).ProxyType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_unreadable_state_has_its_own_type()
|
||||
{
|
||||
Assert.Equal("Unknown", ProxySettings.Unreadable(5).ProxyType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Equal_content_is_equal_and_any_field_makes_it_different()
|
||||
{
|
||||
Assert.Equal(Settings(autoDetect: true), Settings(autoDetect: true));
|
||||
Assert.NotEqual(Settings(autoDetect: true), Settings());
|
||||
Assert.NotEqual(Settings(pac: "https://a.test/p"), Settings(pac: "https://b.test/p"));
|
||||
Assert.NotEqual(Settings(proxy: "a:1"), Settings(proxy: "b:1"));
|
||||
Assert.NotEqual(Settings(proxy: "a:1", bypass: "x"), Settings(proxy: "a:1", bypass: "y"));
|
||||
Assert.NotEqual(Settings(winHttp: "a:1"), Settings());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void An_empty_string_counts_as_not_set()
|
||||
{
|
||||
Assert.Equal(Settings(), Settings(pac: "", proxy: "", bypass: "", winHttp: ""));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("user:secret@p:8080")]
|
||||
[InlineData("http://user:secret@p:8080")]
|
||||
[InlineData("http=user:secret@p:8080;https=user:secret@q:8443")]
|
||||
[InlineData("user:se=cret@p:8080")]
|
||||
[InlineData("user:se;cret@p:8080")]
|
||||
[InlineData("http=user:se;cret@p:8080")]
|
||||
[InlineData("user:se://cret@p:8080")]
|
||||
public void Credentials_never_reach_logs_or_events(string configured)
|
||||
{
|
||||
var settings = Settings(proxy: configured, winHttp: configured);
|
||||
var data = settings.ToEventData();
|
||||
|
||||
foreach (var text in new[] { settings.DescribeAll(), (string)data["proxyServer"], (string)data["winHttpProxy"] })
|
||||
{
|
||||
Assert.DoesNotContain("user", text);
|
||||
Assert.DoesNotContain("cret", text);
|
||||
}
|
||||
|
||||
Assert.Equal(ProxySettings.CredentialsHidden, settings.Proxy);
|
||||
Assert.Equal(ProxySettings.CredentialsHidden, settings.WinHttpProxy);
|
||||
Assert.True((bool)data["proxyEnabled"]);
|
||||
Assert.True((bool)data["winHttpProxyConfigured"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("p:8080")]
|
||||
[InlineData("http://p:8080")]
|
||||
[InlineData("http=p:8080;https=q:8443")]
|
||||
public void A_value_without_credentials_stays_as_it_is(string configured)
|
||||
{
|
||||
Assert.Equal(configured, Settings(proxy: configured).Proxy);
|
||||
Assert.Equal(configured, Settings(winHttp: configured).WinHttpProxy);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("https://pac.example.test/p.pac?key=secret", "https://pac.example.test/p.pac")]
|
||||
[InlineData("http://user:secret@pac.example.test:8080/p.pac", "http://pac.example.test:8080/p.pac")]
|
||||
[InlineData("http://user:se=cr;et@pac.example.test/p.pac", "http://pac.example.test/p.pac")]
|
||||
[InlineData("not a url with secret", "(unparsable URL)")]
|
||||
public void A_pac_url_keeps_scheme_host_port_and_path(string configured, string expected)
|
||||
{
|
||||
Assert.Equal(expected, Settings(pac: configured).PacUrl);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_change_names_what_moved()
|
||||
{
|
||||
var before = Settings(autoDetect: true);
|
||||
|
||||
Assert.Equal("IE settings of this account: auto-detect -> no proxy", Settings().DescribeChangeFrom(before));
|
||||
Assert.Equal("WinHTTP: direct -> proxy p:1", Settings(autoDetect: true, winHttp: "p:1").DescribeChangeFrom(before));
|
||||
Assert.Equal(
|
||||
"IE settings of this account: auto-detect -> no proxy; WinHTTP: direct -> proxy p:1",
|
||||
Settings(winHttp: "p:1").DescribeChangeFrom(before));
|
||||
Assert.Equal(
|
||||
"IE settings of this account: proxy bypass list changed",
|
||||
Settings(proxy: "p:1", bypass: "b").DescribeChangeFrom(Settings(proxy: "p:1", bypass: "a")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Only_the_account_settings_count_as_a_change_for_uploads()
|
||||
{
|
||||
var before = Settings(autoDetect: true);
|
||||
|
||||
Assert.True(Settings().DiffersForUploadsFrom(before));
|
||||
Assert.False(Settings(autoDetect: true, winHttp: "p:1").DiffersForUploadsFrom(before));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_event_tells_auto_detect_from_direct()
|
||||
{
|
||||
var data = Settings(autoDetect: true).ToEventData();
|
||||
|
||||
Assert.Equal("AutoDetect", data["proxyType"]);
|
||||
Assert.True((bool)data["autoDetect"]);
|
||||
Assert.False((bool)data["proxyEnabled"]);
|
||||
Assert.Null(data["proxyServer"]);
|
||||
Assert.False((bool)data["winHttpProxyConfigured"]);
|
||||
Assert.Null(data["winHttpProxy"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_event_carries_a_real_winhttp_proxy()
|
||||
{
|
||||
var data = Settings(winHttp: "p:8080").ToEventData();
|
||||
|
||||
Assert.True((bool)data["winHttpProxyConfigured"]);
|
||||
Assert.Equal("p:8080", data["winHttpProxy"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_real_settings_of_this_account_can_be_read()
|
||||
{
|
||||
var settings = ProxySettingsReader.Read();
|
||||
|
||||
Assert.False(settings.IsUnreadable, settings.Describe());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_real_route_can_be_read()
|
||||
{
|
||||
var read = Task.Run(() => ProxySettingsReader.ReadRoute(new Uri("https://backend.example.invalid/")));
|
||||
|
||||
Assert.False(string.IsNullOrEmpty(await read));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Interop
|
||||
{
|
||||
internal static class WinHttpNativeMethods
|
||||
{
|
||||
public const int ERROR_FILE_NOT_FOUND = 2;
|
||||
public const uint WINHTTP_ACCESS_TYPE_NAMED_PROXY = 3;
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct WINHTTP_CURRENT_USER_IE_PROXY_CONFIG
|
||||
{
|
||||
[MarshalAs(UnmanagedType.Bool)]
|
||||
public bool fAutoDetect;
|
||||
public IntPtr lpszAutoConfigUrl;
|
||||
public IntPtr lpszProxy;
|
||||
public IntPtr lpszProxyBypass;
|
||||
}
|
||||
|
||||
[StructLayout(LayoutKind.Sequential)]
|
||||
public struct WINHTTP_PROXY_INFO
|
||||
{
|
||||
public uint dwAccessType;
|
||||
public IntPtr lpszProxy;
|
||||
public IntPtr lpszProxyBypass;
|
||||
}
|
||||
|
||||
[DllImport("winhttp.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool WinHttpGetIEProxyConfigForCurrentUser(ref WINHTTP_CURRENT_USER_IE_PROXY_CONFIG config);
|
||||
|
||||
[DllImport("winhttp.dll", SetLastError = true)]
|
||||
[return: MarshalAs(UnmanagedType.Bool)]
|
||||
public static extern bool WinHttpGetDefaultProxyConfiguration(ref WINHTTP_PROXY_INFO info);
|
||||
|
||||
// WinHTTP allocates the strings of both structs with GlobalAlloc; the caller frees them.
|
||||
[DllImport("kernel32.dll", SetLastError = true)]
|
||||
public static extern IntPtr GlobalFree(IntPtr hMem);
|
||||
}
|
||||
}
|
||||
+3
-28
@@ -11,6 +11,7 @@ using AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Security;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.DecisionCore.Signals;
|
||||
using AutopilotMonitor.Shared;
|
||||
@@ -113,34 +114,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.DeviceInfo
|
||||
{
|
||||
try
|
||||
{
|
||||
var data = new Dictionary<string, object>();
|
||||
|
||||
using (var key = Registry.CurrentUser.OpenSubKey(@"Software\Microsoft\Windows\CurrentVersion\Internet Settings"))
|
||||
{
|
||||
if (key != null)
|
||||
{
|
||||
data["proxyEnabled"] = key.GetValue("ProxyEnable")?.ToString() == "1";
|
||||
data["proxyServer"] = key.GetValue("ProxyServer")?.ToString();
|
||||
data["proxyOverride"] = key.GetValue("ProxyOverride")?.ToString();
|
||||
data["autoConfigUrl"] = key.GetValue("AutoConfigURL")?.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
// Also check WinHTTP proxy
|
||||
using (var key = Registry.LocalMachine.OpenSubKey(@"SOFTWARE\Microsoft\Windows\CurrentVersion\Internet Settings\Connections"))
|
||||
{
|
||||
data["winHttpProxyConfigured"] = key?.GetValue("WinHttpSettings") != null;
|
||||
}
|
||||
|
||||
var proxyType = "Direct";
|
||||
if (data.ContainsKey("proxyEnabled") && (bool)data["proxyEnabled"])
|
||||
proxyType = "Proxy";
|
||||
else if (data.ContainsKey("autoConfigUrl") && data["autoConfigUrl"] != null)
|
||||
proxyType = "PAC";
|
||||
|
||||
data["proxyType"] = proxyType;
|
||||
|
||||
EmitDeviceInfoEvent(Constants.EventTypes.ProxyConfiguration, $"Proxy configuration: {proxyType}", data);
|
||||
var settings = ProxySettingsReader.Read();
|
||||
EmitDeviceInfoEvent(Constants.EventTypes.ProxyConfiguration, $"Proxy configuration: {settings.ProxyType}", settings.ToEventData());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -96,6 +96,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
private readonly DateTime? _previousRunLastAliveUtc;
|
||||
|
||||
private readonly Transport.ProxyObserver? _proxyObserver;
|
||||
|
||||
public DefaultComponentFactory(
|
||||
AgentConfiguration agentConfig,
|
||||
AgentConfigResponse remoteConfig,
|
||||
@@ -105,7 +107,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
Persistence.StartupEventGate? startupEventGate = null,
|
||||
DateTime? previousBootUtc = null,
|
||||
string? previousExitType = null,
|
||||
DateTime? previousRunLastAliveUtc = null)
|
||||
DateTime? previousRunLastAliveUtc = null,
|
||||
Transport.ProxyObserver? proxyObserver = null)
|
||||
{
|
||||
_agentConfig = agentConfig ?? throw new ArgumentNullException(nameof(agentConfig));
|
||||
_remoteConfig = remoteConfig ?? throw new ArgumentNullException(nameof(remoteConfig));
|
||||
@@ -116,6 +119,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_previousBootUtc = previousBootUtc;
|
||||
_previousExitType = previousExitType;
|
||||
_previousRunLastAliveUtc = previousRunLastAliveUtc;
|
||||
_proxyObserver = proxyObserver;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -666,6 +670,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
clock: clock,
|
||||
logger: logger));
|
||||
|
||||
// The exact time of a proxy change for the agent log and the timeline. Always on: when
|
||||
// this matters, nothing else from the device reaches the backend.
|
||||
if (_proxyObserver != null)
|
||||
{
|
||||
hosts.Add(new ProxySettingsHost(
|
||||
observer: _proxyObserver,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger));
|
||||
}
|
||||
|
||||
// Live AC/battery watcher — WMI Win32_PowerManagementEvent push (no polling), emits
|
||||
// power_state_change on AC↔battery transitions and 50/30/15 % downward threshold
|
||||
// crossings. Deliberately NOT under PeriodicCollectorLifecycleHost: it is push-based
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Transport;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Win32;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
/// <summary>
|
||||
/// Tells <see cref="ProxyObserver"/> the moment proxy settings change, so the log has the exact
|
||||
/// time. A watcher that fails costs only that time: the observer compares again on every failed
|
||||
/// upload.
|
||||
/// </summary>
|
||||
internal sealed class ProxySettingsHost : ICollectorHost
|
||||
{
|
||||
private const string Source = "ProxySettingsHost";
|
||||
private const int DebounceMs = 1000;
|
||||
|
||||
// The whole subtree: "Connections" does not exist yet in an account nothing has written to.
|
||||
// HKCU of a SYSTEM process is HKU\S-1-5-18.
|
||||
private const string AccountSettingsKey = @"Software\Microsoft\Windows\CurrentVersion\Internet Settings";
|
||||
// Holds WinHttpSettings, which netsh winhttp writes.
|
||||
private const string MachineConnectionsKey = @"SOFTWARE\Microsoft\Windows\CurrentVersion\Internet Settings\Connections";
|
||||
|
||||
private const RegistryNativeMethods.RegChangeNotifyFilter Filter =
|
||||
RegistryNativeMethods.RegChangeNotifyFilter.Name |
|
||||
RegistryNativeMethods.RegChangeNotifyFilter.LastSet;
|
||||
|
||||
private readonly ProxyObserver _observer;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly List<RegistryWatcher> _watchers = new List<RegistryWatcher>();
|
||||
private readonly Timer _debounce;
|
||||
private int _started;
|
||||
private int _stopped;
|
||||
|
||||
public string Name => "ProxySettingsWatcher";
|
||||
|
||||
public ProxySettingsHost(
|
||||
ProxyObserver observer,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_observer = observer ?? throw new ArgumentNullException(nameof(observer));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_post = new InformationalEventPost(ingress, clock);
|
||||
_debounce = new Timer(_ => OnDebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite);
|
||||
|
||||
AddWatcher(RegistryHive.CurrentUser, AccountSettingsKey, watchSubtree: true, "account settings");
|
||||
AddWatcher(RegistryHive.LocalMachine, MachineConnectionsKey, watchSubtree: false, "WinHTTP settings");
|
||||
}
|
||||
|
||||
private void AddWatcher(RegistryHive hive, string subKey, bool watchSubtree, string label)
|
||||
{
|
||||
var watcher = new RegistryWatcher(hive, subKey, watchSubtree, RegistryView.Registry64, Filter);
|
||||
watcher.Changed += (_, _) => _debounce.Change(DebounceMs, Timeout.Infinite);
|
||||
watcher.Error += (_, ex) => _logger.Warning($"ProxySettingsHost: {label} watcher error - {ex.GetType().Name}: {ex.Message}");
|
||||
_watchers.Add(watcher);
|
||||
}
|
||||
|
||||
public void Start()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _started, 1) == 1) return;
|
||||
|
||||
_observer.SettingsChanged += OnSettingsChanged;
|
||||
foreach (var watcher in _watchers)
|
||||
{
|
||||
try { watcher.Start(); }
|
||||
catch (Exception ex) { _logger.Warning($"ProxySettingsHost: a watcher failed to start: {ex.Message}"); }
|
||||
}
|
||||
|
||||
// Whatever moved between the baseline and the start of the watchers.
|
||||
OnDebounceElapsed();
|
||||
}
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _stopped, 1) == 1) return;
|
||||
|
||||
DisposeWatchers();
|
||||
_observer.SettingsChanged -= OnSettingsChanged;
|
||||
if (Volatile.Read(ref _started) == 1) _observer.LogOpenFailures();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Stop();
|
||||
_debounce.Dispose();
|
||||
}
|
||||
|
||||
private void DisposeWatchers()
|
||||
{
|
||||
foreach (var watcher in _watchers)
|
||||
{
|
||||
try { watcher.Dispose(); }
|
||||
catch (Exception ex) { _logger.Debug($"ProxySettingsHost: disposing a watcher threw: {ex.Message}"); }
|
||||
}
|
||||
_watchers.Clear();
|
||||
}
|
||||
|
||||
private void OnDebounceElapsed()
|
||||
{
|
||||
_logger.Trace("ProxySettingsHost: checking the proxy settings.");
|
||||
try { _observer.CheckForChange(); }
|
||||
catch (Exception ex) { _logger.Warning($"ProxySettingsHost: checking the proxy settings failed: {ex.Message}"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits in the spool if uploads are down, so the timeline explains the gap once they work
|
||||
/// again. The portal shows the newest event of this type.
|
||||
/// </summary>
|
||||
private void OnSettingsChanged(ProxySettings settings, string what, string route)
|
||||
{
|
||||
var data = settings.ToEventData();
|
||||
data["trigger"] = "changed";
|
||||
data["change"] = what;
|
||||
data["route"] = route;
|
||||
|
||||
_post.Emit(
|
||||
eventType: Constants.EventTypes.ProxyConfiguration,
|
||||
source: Source,
|
||||
message: $"Proxy settings changed: {what}",
|
||||
severity: EventSeverity.Warning,
|
||||
typedPayload: data);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// Makes the agent log say which way uploads go and what happened to it. Something on the device
|
||||
/// can change the proxy settings of the agent's account; .NET follows at once, and from then on
|
||||
/// no channel may reach the backend, so the log on the device is the only witness.
|
||||
/// <para>
|
||||
/// Compares with what held at the start instead of judging a value: "no proxy" is the right
|
||||
/// state on one network and the broken one on the next. Reads on failures and on registry
|
||||
/// changes, never by polling. Nothing here throws into the upload path.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ProxyObserver
|
||||
{
|
||||
internal const string RouteUnknown = "unknown";
|
||||
|
||||
private static readonly TimeSpan DefaultRouteBound = TimeSpan.FromSeconds(3);
|
||||
|
||||
private readonly Func<ProxySettings> _readSettings;
|
||||
private readonly Func<string> _readRoute;
|
||||
private readonly IClock _clock;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly TimeSpan _routeBound;
|
||||
private readonly object _sync = new object();
|
||||
|
||||
private ProxySettings? _current;
|
||||
private string _lastRoute = RouteUnknown;
|
||||
private Task<string>? _routeRead;
|
||||
private DateTime? _uploadChangeAtUtc;
|
||||
private string? _uploadChange;
|
||||
private DateTime? _failingSinceUtc;
|
||||
private int _failedAttempts;
|
||||
private DateTime? _lastSuccessUtc;
|
||||
|
||||
/// <summary>Settings now, what changed, route now. Raised outside the lock, on the caller's thread.</summary>
|
||||
internal event Action<ProxySettings, string, string>? SettingsChanged;
|
||||
|
||||
public ProxyObserver(string baseUrl, AgentLogger logger, IClock clock)
|
||||
: this(
|
||||
ProxySettingsReader.Read,
|
||||
RouteReaderFor(baseUrl),
|
||||
logger,
|
||||
clock,
|
||||
DefaultRouteBound)
|
||||
{
|
||||
}
|
||||
|
||||
internal ProxyObserver(
|
||||
Func<ProxySettings> readSettings,
|
||||
Func<string> readRoute,
|
||||
AgentLogger logger,
|
||||
IClock clock,
|
||||
TimeSpan routeBound)
|
||||
{
|
||||
_readSettings = readSettings ?? throw new ArgumentNullException(nameof(readSettings));
|
||||
_readRoute = readRoute ?? throw new ArgumentNullException(nameof(readRoute));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
_routeBound = routeBound;
|
||||
}
|
||||
|
||||
private static Func<string> RouteReaderFor(string baseUrl)
|
||||
{
|
||||
if (!Uri.TryCreate(baseUrl, UriKind.Absolute, out var target)) return () => RouteUnknown;
|
||||
return () => ProxySettingsReader.ReadRoute(target);
|
||||
}
|
||||
|
||||
/// <summary>Logs the state to compare against. Runs before the first request of the agent.</summary>
|
||||
public void CaptureBaseline()
|
||||
{
|
||||
var settings = ReadSettingsSafe();
|
||||
var route = ReadRouteBounded();
|
||||
lock (_sync)
|
||||
{
|
||||
if (_current == null && !settings.IsUnreadable) _current = settings;
|
||||
_lastRoute = route;
|
||||
}
|
||||
// Same text on a rebuild of the clients after an endpoint migration: the repeat goes to Trace.
|
||||
_logger.Info($"ProxyObserver: start - {settings.DescribeAll()}; route to backend: {route}", repeatKey: "ProxyObserver.baseline");
|
||||
}
|
||||
|
||||
/// <summary>Re-reads the settings and reports a change of content once, whoever notices it first.</summary>
|
||||
public void CheckForChange()
|
||||
{
|
||||
var settings = ReadSettingsSafe();
|
||||
if (settings.IsUnreadable) return;
|
||||
|
||||
ProxySettings previous;
|
||||
var affectsUploads = false;
|
||||
lock (_sync)
|
||||
{
|
||||
if (_current == null)
|
||||
{
|
||||
_current = settings;
|
||||
return;
|
||||
}
|
||||
if (_current.Equals(settings)) return;
|
||||
|
||||
previous = _current;
|
||||
_current = settings;
|
||||
if (settings.DiffersForUploadsFrom(previous))
|
||||
{
|
||||
affectsUploads = true;
|
||||
_uploadChangeAtUtc = _clock.UtcNow;
|
||||
_uploadChange = settings.DescribeUploadChangeFrom(previous);
|
||||
}
|
||||
}
|
||||
|
||||
var route = ReadRouteBounded();
|
||||
string routeBefore;
|
||||
lock (_sync)
|
||||
{
|
||||
routeBefore = _lastRoute;
|
||||
_lastRoute = route;
|
||||
}
|
||||
|
||||
var what = settings.DescribeChangeFrom(previous);
|
||||
_logger.Warning(affectsUploads
|
||||
? $"ProxyObserver: proxy settings changed - {what}; route to backend: {(routeBefore == route ? route : $"{routeBefore} -> {route}")}. " +
|
||||
"The agent's uploads follow these settings. Check with: bitsadmin /util /getieproxy localsystem"
|
||||
: $"ProxyObserver: proxy settings changed - {what}. The agent's uploads do not use WinHTTP, services that rely on its default do.");
|
||||
|
||||
try
|
||||
{
|
||||
SettingsChanged?.Invoke(settings, what, route);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"ProxyObserver: change handler threw: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The next hop to the backend: <c>direct</c>, <c>host:port</c> or <c>unknown</c>.</summary>
|
||||
public string RouteText() => ReadRouteBounded();
|
||||
|
||||
/// <summary>
|
||||
/// An upload did not reach the backend. The first of a series says what the reader needs to
|
||||
/// know: when it began and whether the proxy settings moved before that.
|
||||
/// </summary>
|
||||
public void OnSendFailed(TimeSpan attemptDuration)
|
||||
{
|
||||
// Also catches a change the registry watcher did not report.
|
||||
CheckForChange();
|
||||
|
||||
DateTime since;
|
||||
DateTime? lastSuccess;
|
||||
DateTime? changedAt;
|
||||
string? change;
|
||||
string? current;
|
||||
lock (_sync)
|
||||
{
|
||||
_failedAttempts++;
|
||||
if (_failingSinceUtc != null) return;
|
||||
|
||||
since = _clock.UtcNow - attemptDuration;
|
||||
_failingSinceUtc = since;
|
||||
lastSuccess = _lastSuccessUtc;
|
||||
changedAt = _uploadChangeAtUtc;
|
||||
change = _uploadChange;
|
||||
current = _current?.Describe();
|
||||
}
|
||||
|
||||
var route = ReadRouteBounded();
|
||||
var lastUpload = lastSuccess == null ? "No successful upload yet." : $"Last successful upload {Time(lastSuccess.Value)}.";
|
||||
|
||||
if (changedAt != null && (lastSuccess == null || changedAt > lastSuccess))
|
||||
{
|
||||
_logger.Warning(
|
||||
$"ProxyObserver: uploads are failing since {Time(since)}, {Span(since - changedAt.Value)} after the proxy settings changed at {Time(changedAt.Value)} ({change}). " +
|
||||
$"Route to backend: {route}. {lastUpload}");
|
||||
}
|
||||
else
|
||||
{
|
||||
var unchangedSince = changedAt == null ? "start" : "the last successful upload";
|
||||
_logger.Warning(
|
||||
$"ProxyObserver: uploads are failing since {Time(since)}. Proxy settings of this account unchanged since {unchangedSince}: {current ?? RouteUnknown}. " +
|
||||
$"Route to backend: {route}. {lastUpload}");
|
||||
}
|
||||
}
|
||||
|
||||
public void OnSendSucceeded()
|
||||
{
|
||||
DateTime since;
|
||||
int attempts;
|
||||
lock (_sync)
|
||||
{
|
||||
_lastSuccessUtc = _clock.UtcNow;
|
||||
if (_failingSinceUtc == null) return;
|
||||
|
||||
since = _failingSinceUtc.Value;
|
||||
attempts = _failedAttempts;
|
||||
_failingSinceUtc = null;
|
||||
_failedAttempts = 0;
|
||||
}
|
||||
|
||||
CheckForChange();
|
||||
_logger.Info($"ProxyObserver: uploads work again after {attempts} failed attempt(s) over {Span(_clock.UtcNow - since)}; route to backend: {ReadRouteBounded()}.");
|
||||
}
|
||||
|
||||
/// <summary>The end of the log is read first, so a run that ends while failing says so there.</summary>
|
||||
public void LogOpenFailures()
|
||||
{
|
||||
DateTime since;
|
||||
int attempts;
|
||||
DateTime? lastSuccess;
|
||||
DateTime? changedAt;
|
||||
string? change;
|
||||
string? current;
|
||||
lock (_sync)
|
||||
{
|
||||
if (_failingSinceUtc == null) return;
|
||||
|
||||
since = _failingSinceUtc.Value;
|
||||
attempts = _failedAttempts;
|
||||
lastSuccess = _lastSuccessUtc;
|
||||
changedAt = _uploadChangeAtUtc;
|
||||
change = _uploadChange;
|
||||
current = _current?.Describe();
|
||||
}
|
||||
|
||||
var lastUpload = lastSuccess == null ? "no successful upload" : $"last success {Time(lastSuccess.Value)}";
|
||||
var settings = changedAt == null
|
||||
? $"Proxy settings of this account unchanged since start: {current ?? RouteUnknown}."
|
||||
: $"Proxy settings changed at {Time(changedAt.Value)} ({change}).";
|
||||
_logger.Warning(
|
||||
$"ProxyObserver: stopping with uploads still failing: {attempts} failed attempt(s) since {Time(since)}, {lastUpload}, route to backend: {ReadRouteBounded()}. {settings}");
|
||||
}
|
||||
|
||||
private ProxySettings ReadSettingsSafe()
|
||||
{
|
||||
try
|
||||
{
|
||||
var settings = _readSettings();
|
||||
if (settings.IsUnreadable) _logger.Debug($"ProxyObserver: proxy settings {settings.Describe()}", repeatKey: "ProxyObserver.unreadable");
|
||||
return settings;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"ProxyObserver: reading the proxy settings threw: {ex.Message}", repeatKey: "ProxyObserver.readThrew");
|
||||
return ProxySettings.Unreadable(-1);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Auto-detect can hang and must hold up neither the start nor an upload. A read still under
|
||||
/// way is joined, so a hanging one does not pile up pool threads.
|
||||
/// </summary>
|
||||
private string ReadRouteBounded()
|
||||
{
|
||||
Task<string> read;
|
||||
lock (_sync)
|
||||
{
|
||||
if (_routeRead == null || _routeRead.IsCompleted) _routeRead = Task.Run(ReadRouteSafe);
|
||||
read = _routeRead;
|
||||
}
|
||||
return read.Wait(_routeBound) ? read.Result : RouteUnknown;
|
||||
}
|
||||
|
||||
private string ReadRouteSafe()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _readRoute();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Debug($"ProxyObserver: reading the route threw: {ex.Message}", repeatKey: "ProxyObserver.routeThrew");
|
||||
return RouteUnknown;
|
||||
}
|
||||
}
|
||||
|
||||
private static string Time(DateTime utc) => utc.ToString("HH:mm:ss", CultureInfo.InvariantCulture);
|
||||
|
||||
private static string Span(TimeSpan span)
|
||||
{
|
||||
var seconds = Math.Max(0, (long)span.TotalSeconds);
|
||||
return seconds < 120 ? $"{seconds} s" : $"{seconds / 60} min";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
/// <summary>
|
||||
/// The proxy settings Windows holds for the account this process runs as (what
|
||||
/// <c>bitsadmin /util /getieproxy localsystem</c> shows for SYSTEM and what .NET uses), plus the
|
||||
/// WinHTTP proxy (<c>netsh winhttp show proxy</c>). Compared by content, because WinINET rewrites
|
||||
/// its settings blob without changing anything. The values end up in logs and events, so
|
||||
/// credentials and PAC query strings are kept out on the way in.
|
||||
/// </summary>
|
||||
internal sealed class ProxySettings : IEquatable<ProxySettings>
|
||||
{
|
||||
internal const string CredentialsHidden = "(value with credentials, not shown)";
|
||||
|
||||
public bool AutoDetect { get; }
|
||||
public string PacUrl { get; }
|
||||
public string Proxy { get; }
|
||||
public string Bypass { get; }
|
||||
public string WinHttpProxy { get; }
|
||||
|
||||
/// <summary>Win32 error of a failed read; null when the settings were read.</summary>
|
||||
public int? ReadError { get; }
|
||||
|
||||
public bool IsUnreadable => ReadError != null;
|
||||
|
||||
public ProxySettings(bool autoDetect, string pacUrl, string proxy, string bypass, string winHttpProxy)
|
||||
{
|
||||
AutoDetect = autoDetect;
|
||||
PacUrl = SanitizeUrl(pacUrl);
|
||||
Proxy = HideCredentials(proxy);
|
||||
Bypass = EmptyToNull(bypass);
|
||||
WinHttpProxy = HideCredentials(winHttpProxy);
|
||||
}
|
||||
|
||||
private ProxySettings(int readError)
|
||||
{
|
||||
ReadError = readError;
|
||||
}
|
||||
|
||||
public static ProxySettings Unreadable(int win32Error) => new ProxySettings(win32Error);
|
||||
|
||||
/// <summary>Kept in step with ANALYZE-CORR-003, which matches <c>Proxy|PAC</c>.</summary>
|
||||
public string ProxyType =>
|
||||
IsUnreadable ? "Unknown"
|
||||
: Proxy != null ? "Proxy"
|
||||
: PacUrl != null ? "PAC"
|
||||
: AutoDetect ? "AutoDetect"
|
||||
: "Direct";
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
if (IsUnreadable) return $"unreadable (Win32 error {ReadError})";
|
||||
|
||||
var parts = new List<string>(3);
|
||||
if (AutoDetect) parts.Add("auto-detect");
|
||||
if (PacUrl != null) parts.Add($"PAC {PacUrl}");
|
||||
if (Proxy != null) parts.Add($"proxy {Proxy}");
|
||||
return parts.Count == 0 ? "no proxy" : string.Join(", ", parts);
|
||||
}
|
||||
|
||||
public string DescribeWinHttp() => WinHttpProxy == null ? "direct" : $"proxy {WinHttpProxy}";
|
||||
|
||||
public string DescribeAll() => $"IE settings of this account: {Describe()}; WinHTTP: {DescribeWinHttp()}";
|
||||
|
||||
/// <summary>The account's own settings, which the agent's uploads follow; WinHTTP is apart.</summary>
|
||||
public bool DiffersForUploadsFrom(ProxySettings other) =>
|
||||
AutoDetect != other.AutoDetect
|
||||
|| PacUrl != other.PacUrl
|
||||
|| Proxy != other.Proxy
|
||||
|| Bypass != other.Bypass;
|
||||
|
||||
public string DescribeUploadChangeFrom(ProxySettings previous) =>
|
||||
Describe() == previous.Describe()
|
||||
? "IE settings of this account: proxy bypass list changed"
|
||||
: $"IE settings of this account: {previous.Describe()} -> {Describe()}";
|
||||
|
||||
public string DescribeChangeFrom(ProxySettings previous)
|
||||
{
|
||||
var parts = new List<string>(2);
|
||||
if (DiffersForUploadsFrom(previous)) parts.Add(DescribeUploadChangeFrom(previous));
|
||||
if (WinHttpProxy != previous.WinHttpProxy) parts.Add($"WinHTTP: {previous.DescribeWinHttp()} -> {DescribeWinHttp()}");
|
||||
return string.Join("; ", parts);
|
||||
}
|
||||
|
||||
/// <summary>Payload of the <c>proxy_configuration</c> event.</summary>
|
||||
public Dictionary<string, object> ToEventData() => new Dictionary<string, object>
|
||||
{
|
||||
{ "proxyType", ProxyType },
|
||||
{ "autoDetect", AutoDetect },
|
||||
{ "proxyEnabled", Proxy != null },
|
||||
{ "proxyServer", Proxy },
|
||||
{ "proxyOverride", Bypass },
|
||||
{ "autoConfigUrl", PacUrl },
|
||||
{ "winHttpProxyConfigured", WinHttpProxy != null },
|
||||
{ "winHttpProxy", WinHttpProxy },
|
||||
};
|
||||
|
||||
public bool Equals(ProxySettings other) =>
|
||||
other != null
|
||||
&& ReadError == other.ReadError
|
||||
&& !DiffersForUploadsFrom(other)
|
||||
&& WinHttpProxy == other.WinHttpProxy;
|
||||
|
||||
public override bool Equals(object obj) => Equals(obj as ProxySettings);
|
||||
|
||||
public override int GetHashCode() => (AutoDetect, PacUrl, Proxy, Bypass, WinHttpProxy, ReadError).GetHashCode();
|
||||
|
||||
private static string EmptyToNull(string value) => string.IsNullOrEmpty(value) ? null : value;
|
||||
|
||||
/// <summary>Scheme, host, port and path. A query string can carry a key; an unparsable URL is not repeated.</summary>
|
||||
private static string SanitizeUrl(string url)
|
||||
{
|
||||
if (string.IsNullOrEmpty(url)) return null;
|
||||
if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || string.IsNullOrEmpty(uri.Host)) return "(unparsable URL)";
|
||||
var port = uri.IsDefaultPort ? string.Empty : $":{uri.Port}";
|
||||
return $"{uri.Scheme}://{uri.Host}{port}{uri.AbsolutePath}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A proxy string has no defined form for credentials and a password can hold <c>=</c>,
|
||||
/// <c>;</c> or <c>://</c>, so a value with an <c>@</c> is replaced as a whole, not taken apart.
|
||||
/// </summary>
|
||||
private static string HideCredentials(string proxy)
|
||||
{
|
||||
if (string.IsNullOrEmpty(proxy)) return null;
|
||||
return proxy.IndexOf('@') < 0 ? proxy : CredentialsHidden;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Runtime.InteropServices;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Interop;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport
|
||||
{
|
||||
internal static class ProxySettingsReader
|
||||
{
|
||||
public const string RouteDirect = "direct";
|
||||
|
||||
/// <summary>Reads the registry through WinHTTP, no network. Does not throw on a failed read.</summary>
|
||||
public static ProxySettings Read()
|
||||
{
|
||||
var ie = new WinHttpNativeMethods.WINHTTP_CURRENT_USER_IE_PROXY_CONFIG();
|
||||
var winHttp = new WinHttpNativeMethods.WINHTTP_PROXY_INFO();
|
||||
try
|
||||
{
|
||||
if (!WinHttpNativeMethods.WinHttpGetIEProxyConfigForCurrentUser(ref ie))
|
||||
{
|
||||
var error = Marshal.GetLastWin32Error();
|
||||
// The account has never had settings written, so nothing configures a proxy.
|
||||
if (error != WinHttpNativeMethods.ERROR_FILE_NOT_FOUND) return ProxySettings.Unreadable(error);
|
||||
}
|
||||
|
||||
if (!WinHttpNativeMethods.WinHttpGetDefaultProxyConfiguration(ref winHttp))
|
||||
{
|
||||
return ProxySettings.Unreadable(Marshal.GetLastWin32Error());
|
||||
}
|
||||
|
||||
return new ProxySettings(
|
||||
autoDetect: ie.fAutoDetect,
|
||||
pacUrl: Marshal.PtrToStringUni(ie.lpszAutoConfigUrl),
|
||||
proxy: Marshal.PtrToStringUni(ie.lpszProxy),
|
||||
bypass: Marshal.PtrToStringUni(ie.lpszProxyBypass),
|
||||
winHttpProxy: winHttp.dwAccessType == WinHttpNativeMethods.WINHTTP_ACCESS_TYPE_NAMED_PROXY
|
||||
? Marshal.PtrToStringUni(winHttp.lpszProxy)
|
||||
: null);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Free(ie.lpszAutoConfigUrl);
|
||||
Free(ie.lpszProxy);
|
||||
Free(ie.lpszProxyBypass);
|
||||
Free(winHttp.lpszProxy);
|
||||
Free(winHttp.lpszProxyBypass);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The next hop .NET chooses for <paramref name="target"/>: the same decision an
|
||||
/// <c>HttpClientHandler</c> with default settings takes, PAC and auto-detect evaluated.
|
||||
/// Auto-detect can take seconds on a network without WPAD.
|
||||
/// </summary>
|
||||
public static string ReadRoute(Uri target)
|
||||
{
|
||||
var hop = WebRequest.DefaultWebProxy?.GetProxy(target);
|
||||
if (hop == null || string.Equals(hop.Host, target.Host, StringComparison.OrdinalIgnoreCase)) return RouteDirect;
|
||||
return hop.IsDefaultPort ? hop.Host : $"{hop.Host}:{hop.Port}";
|
||||
}
|
||||
|
||||
private static void Free(IntPtr memory)
|
||||
{
|
||||
if (memory != IntPtr.Zero) WinHttpNativeMethods.GlobalFree(memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
-4
@@ -52,6 +52,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
private readonly ServerCertificateObserver? _serverCertificates;
|
||||
private readonly NetworkMetrics? _networkMetrics;
|
||||
private readonly ProxyObserver? _proxy;
|
||||
|
||||
/// <param name="serverCertificates">
|
||||
/// Observer installed on the <paramref name="httpClient"/>'s handler; its issuer becomes
|
||||
@@ -60,6 +61,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// <param name="networkMetrics">
|
||||
/// Counts 2xx answers without the correlation echo (<c>agent_metrics_snapshot</c>). Null = not counted.
|
||||
/// </param>
|
||||
/// <param name="proxy">
|
||||
/// Names the route in the failure lines and hears how each upload ended. Null = neither.
|
||||
/// </param>
|
||||
public BackendTelemetryUploader(
|
||||
HttpClient httpClient,
|
||||
string baseUrl,
|
||||
@@ -72,7 +76,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
AuthFailureTracker? authFailureTracker = null,
|
||||
Logging.AgentLogger? logger = null,
|
||||
ServerCertificateObserver? serverCertificates = null,
|
||||
NetworkMetrics? networkMetrics = null)
|
||||
NetworkMetrics? networkMetrics = null,
|
||||
ProxyObserver? proxy = null)
|
||||
{
|
||||
_httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient));
|
||||
if (string.IsNullOrEmpty(baseUrl)) throw new ArgumentException("BaseUrl is mandatory.", nameof(baseUrl));
|
||||
@@ -89,6 +94,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
_logger = logger;
|
||||
_serverCertificates = serverCertificates;
|
||||
_networkMetrics = networkMetrics;
|
||||
_proxy = proxy;
|
||||
}
|
||||
|
||||
public async Task<UploadResult> UploadBatchAsync(
|
||||
@@ -153,7 +159,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
catch (TaskCanceledException ex)
|
||||
{
|
||||
sw.Stop();
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TIMEOUT after {sw.ElapsedMilliseconds}ms (corr={correlationId}, items={items.Count}): {ex.Message}");
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TIMEOUT after {sw.ElapsedMilliseconds}ms (corr={correlationId}, items={items.Count}{RouteSuffix()}): {ex.Message}");
|
||||
_proxy?.OnSendFailed(sw.Elapsed);
|
||||
// HttpClient timeout (not caller cancellation) surfaces as TaskCanceledException
|
||||
// with cancellationToken.IsCancellationRequested == false.
|
||||
return UploadResult.Transient($"timeout: {ex.Message}");
|
||||
@@ -162,7 +169,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
sw.Stop();
|
||||
var cause = TransportErrorCodes.DescribeSendFailure(ex);
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest NETWORK fail after {sw.ElapsedMilliseconds}ms (corr={correlationId}, items={items.Count}): {cause}");
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest NETWORK fail after {sw.ElapsedMilliseconds}ms (corr={correlationId}, items={items.Count}{RouteSuffix()}): {cause}");
|
||||
_proxy?.OnSendFailed(sw.Elapsed);
|
||||
// DNS failure, socket error, etc. — network-layer is retryable.
|
||||
return UploadResult.Transient($"network: {cause}");
|
||||
}
|
||||
@@ -175,6 +183,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
if (result.Success)
|
||||
{
|
||||
_logger?.Verbose($"BackendTelemetryUploader: ingest OK (corr={correlationId}, items={items.Count}, firstId={items[0].TelemetryItemId}, lastId={items[items.Count - 1].TelemetryItemId}, bytes={compressedBody.Length}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}).");
|
||||
_proxy?.OnSendSucceeded();
|
||||
}
|
||||
else if (result.IsTransient)
|
||||
{
|
||||
@@ -189,7 +198,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
$"BackendTelemetryUploader: ingest INTERCEPTED — the answer did not come from the backend " +
|
||||
$"(corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={interception.StatusCode}, " +
|
||||
$"reason={interception.ForeignReason}, pageTitle=\"{interception.PageTitle ?? "(none)"}\", " +
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\").");
|
||||
$"serverCertIssuer=\"{interception.ServerCertIssuer ?? "(unknown)"}\"{RouteSuffix()}).");
|
||||
_proxy?.OnSendFailed(sw.Elapsed);
|
||||
}
|
||||
else if (result.IsSessionGone)
|
||||
{
|
||||
@@ -205,6 +215,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
private string RouteSuffix() => _proxy == null ? string.Empty : $", route={_proxy.RouteText()}";
|
||||
|
||||
private async Task<UploadResult> MapResponseAsync(HttpResponseMessage response, string correlationId)
|
||||
{
|
||||
var statusCode = (int)response.StatusCode;
|
||||
|
||||
@@ -99,7 +99,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// Measured, not guessed: the reducer snapshot is rewritten on every decision step,
|
||||
// so its mtime is the last moment the previous run is known to have been alive —
|
||||
// and unlike LastBootUtc it exists for every exit type, exception_crash included.
|
||||
previousRunLastAliveUtc: DefaultComponentFactory.ReadPreviousRunLastAliveUtc(stateSubdir));
|
||||
previousRunLastAliveUtc: DefaultComponentFactory.ReadPreviousRunLastAliveUtc(stateSubdir),
|
||||
proxyObserver: auth.ProxyObserver);
|
||||
|
||||
var whiteGloveSealingPatternIds = (System.Collections.Generic.IReadOnlyCollection<string>)remoteConfig.WhiteGloveSealingPatternIds
|
||||
?? Array.Empty<string>();
|
||||
|
||||
@@ -52,6 +52,10 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// outbound call (legacy BackendApiClient calls + dominant /api/agent/telemetry POSTs).
|
||||
var networkMetrics = new NetworkMetrics();
|
||||
|
||||
// Before the first request, so a registration that fails has the route in the log too.
|
||||
var proxyObserver = new ProxyObserver(agentConfig.ApiBaseUrl, logger, SystemClock.Instance);
|
||||
proxyObserver.CaptureBaseline();
|
||||
|
||||
var hardware = HardwareInfo.GetHardwareInfo(logger);
|
||||
|
||||
// Resolve the MDM client cert when cert-mode is on. Cert-missing is non-fatal here:
|
||||
@@ -150,6 +154,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
emergencyReporter: emergencyReporter,
|
||||
authFailureTracker: authFailureTracker,
|
||||
serverCertificates: serverCertificates,
|
||||
proxyObserver: proxyObserver,
|
||||
clientCertificateThumbprint: clientCertificate?.Thumbprint);
|
||||
}
|
||||
|
||||
@@ -201,7 +206,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
authFailureTracker: auth.AuthFailureTracker,
|
||||
logger: logger, // Plan §5 Fix 5 — upload-cadence logging
|
||||
serverCertificates: auth.ServerCertificates,
|
||||
networkMetrics: auth.NetworkMetrics);
|
||||
networkMetrics: auth.NetworkMetrics,
|
||||
proxy: auth.ProxyObserver);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -291,6 +297,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// </summary>
|
||||
public ServerCertificateObserver ServerCertificates { get; }
|
||||
|
||||
/// <summary>Shared by the telemetry uploader built in Phase 5 and the collector host.</summary>
|
||||
public ProxyObserver ProxyObserver { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Thumbprint of the client certificate this run uses; null without one. Kept beside a
|
||||
/// failed run's record so the next successful start can tell whether the certificate changed.
|
||||
@@ -308,6 +317,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
EmergencyReporter emergencyReporter,
|
||||
AuthFailureTracker authFailureTracker,
|
||||
ServerCertificateObserver serverCertificates,
|
||||
ProxyObserver proxyObserver,
|
||||
string clientCertificateThumbprint = null)
|
||||
{
|
||||
BackendApiClient = backendApiClient;
|
||||
@@ -320,6 +330,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
EmergencyReporter = emergencyReporter;
|
||||
AuthFailureTracker = authFailureTracker;
|
||||
ServerCertificates = serverCertificates;
|
||||
ProxyObserver = proxyObserver;
|
||||
ClientCertificateThumbprint = clientCertificateThumbprint;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +72,8 @@ public class AppHomingServiceTests
|
||||
Mock.Of<System.Net.Http.IHttpClientFactory>(),
|
||||
cache,
|
||||
_registry,
|
||||
_tenantConfigMock.Object)
|
||||
_tenantConfigMock.Object,
|
||||
Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
_detectorMock = new Mock<IGraphFeatureDetector>();
|
||||
_maintenanceMock = new Mock<IMaintenanceRepository>();
|
||||
@@ -287,6 +288,32 @@ public class AppHomingServiceTests
|
||||
Assert.True((await _sut.ProbePrimaryConsentAsync(TenantId)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_succeeds_when_the_budget_cuts_the_retries_of_a_missing_legacy_service_principal()
|
||||
{
|
||||
// The access check runs on 4 s and the first retry waits 5 s, so the answer arrives as
|
||||
// transient. The Entra code still says the SP is gone: nothing left to lose.
|
||||
SetupProbe(GraphTokenResult.Success(Jwt(ValidationRole)));
|
||||
SetupLegacyToken(GraphTokenResult.TransientFailure(
|
||||
new EntraTokenRejection(401, EntraTokenRejection.ServicePrincipalMissingCode, "no service principal")));
|
||||
|
||||
Assert.True((await _sut.ProbePrimaryConsentAsync(TenantId)).Succeeded);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_stays_transient_when_the_budget_cuts_the_retries_of_any_other_legacy_rejection()
|
||||
{
|
||||
// Any other refusal proves nothing about the roles the tenant would lose with the flip.
|
||||
SetupProbe(GraphTokenResult.Success(Jwt(ValidationRole)));
|
||||
SetupLegacyToken(GraphTokenResult.TransientFailure(
|
||||
new EntraTokenRejection(400, "AADSTS53003", "blocked by Conditional Access")));
|
||||
|
||||
var probe = await _sut.ProbePrimaryConsentAsync(TenantId);
|
||||
|
||||
Assert.False(probe.Succeeded);
|
||||
Assert.True(probe.IsTransient);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Probe_never_mints_a_legacy_token_when_the_primary_token_fails()
|
||||
{
|
||||
@@ -349,6 +376,21 @@ public class AppHomingServiceTests
|
||||
Assert.DoesNotContain(_savedOpsEvents, e => e.EventType == "AppHomingFlippedWithEntraRoles");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoFlip_clears_the_graph_access_rejection_of_the_app_the_tenant_leaves()
|
||||
{
|
||||
_store.Row!.GraphAccessRejectedSince = DateTime.UtcNow.AddHours(-3);
|
||||
_store.Row.GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode;
|
||||
_store.Row.GraphAccessRejectionConfirmedAt = DateTime.UtcNow.AddHours(-2);
|
||||
|
||||
var outcome = await _sut.TryAutoFlipToPrimaryAsync(LegacyHomedConfig(), Actor);
|
||||
|
||||
Assert.Equal(AppHomingAutoFlipOutcome.Flipped, outcome.Outcome);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AutoFlip_does_not_flip_on_permanent_probe_failure()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// A radar pass reads one tenant at a time so its memory follows the largest tenant, and it never
|
||||
/// evaluates an empty list: re-arm would read that as "version drained" and delete open episodes.
|
||||
/// </summary>
|
||||
public class AppVersionRadarPassTests
|
||||
{
|
||||
private static List<AppInstallSummary> Rows(string tenantId, int count)
|
||||
{
|
||||
var rows = new List<AppInstallSummary>();
|
||||
for (var i = 0; i < count; i++)
|
||||
rows.Add(new AppInstallSummary { TenantId = tenantId });
|
||||
return rows;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Each_tenant_is_finished_before_the_next_one_is_loaded()
|
||||
{
|
||||
var calls = new List<string>();
|
||||
|
||||
await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "a", "b" },
|
||||
tenantId =>
|
||||
{
|
||||
calls.Add($"load {tenantId}");
|
||||
return Task.FromResult(Rows(tenantId, 2));
|
||||
},
|
||||
(tenantId, summaries) =>
|
||||
{
|
||||
calls.Add($"evaluate {tenantId}");
|
||||
Assert.All(summaries, s => Assert.Equal(tenantId, s.TenantId));
|
||||
return Task.FromResult((0, 0, 0));
|
||||
},
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal(new[] { "load a", "evaluate a", "load b", "evaluate b" }, calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_rows_is_not_evaluated()
|
||||
{
|
||||
var evaluated = new List<string>();
|
||||
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "empty", "full" },
|
||||
tenantId => Task.FromResult(tenantId == "empty" ? new List<AppInstallSummary>() : Rows(tenantId, 3)),
|
||||
(tenantId, _) =>
|
||||
{
|
||||
evaluated.Add(tenantId);
|
||||
return Task.FromResult((0, 0, 0));
|
||||
},
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal(new[] { "full" }, evaluated);
|
||||
Assert.Equal(1, pass.Tenants);
|
||||
Assert.Equal(3, pass.Rows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_tenant_does_not_cost_the_others_their_pass()
|
||||
{
|
||||
var failed = new List<string>();
|
||||
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "load-fails", "evaluate-fails", "ok" },
|
||||
tenantId => tenantId == "load-fails"
|
||||
? throw new InvalidOperationException("load")
|
||||
: Task.FromResult(Rows(tenantId, 1)),
|
||||
(tenantId, _) => tenantId == "evaluate-fails"
|
||||
? throw new InvalidOperationException("evaluate")
|
||||
: Task.FromResult((1, 2, 3)),
|
||||
(tenantId, _) => failed.Add(tenantId));
|
||||
|
||||
Assert.Equal(new[] { "load-fails", "evaluate-fails" }, failed);
|
||||
Assert.Equal((1, 2, 3), (pass.Fired, pass.Refreshed, pass.Rearmed));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_totals_add_up_over_the_tenants()
|
||||
{
|
||||
var pass = await MaintenanceService.RunAppVersionRadarPassAsync(
|
||||
new[] { "a", "b" },
|
||||
tenantId => Task.FromResult(Rows(tenantId, 2)),
|
||||
(_, _) => Task.FromResult((1, 2, 3)),
|
||||
(_, ex) => throw ex);
|
||||
|
||||
Assert.Equal((2, 4L, 2, 4, 6), pass);
|
||||
}
|
||||
}
|
||||
@@ -35,7 +35,6 @@ public class AuthFunctionSideEffectTests
|
||||
private readonly Mock<TenantAdminsService> _tenantAdminsMock;
|
||||
private readonly Mock<PreviewWhitelistService> _previewMock;
|
||||
private readonly Mock<McpUserService> _mcpUserMock;
|
||||
private readonly Mock<TelegramNotificationService> _telegramMock;
|
||||
private readonly Mock<OpsEventService> _opsEventsMock;
|
||||
private readonly Mock<GlobalNotificationService> _globalNotificationMock;
|
||||
private readonly Mock<IMetricsRepository> _metricsRepoMock;
|
||||
@@ -84,10 +83,6 @@ public class AuthFunctionSideEffectTests
|
||||
configRepo, cache, Mock.Of<ILogger<PreviewWhitelistService>>(), _tenantConfigMock.Object)
|
||||
{ CallBase = false };
|
||||
|
||||
_telegramMock = new Mock<TelegramNotificationService>(
|
||||
new HttpClient(), configRepo, Mock.Of<ILogger<TelegramNotificationService>>())
|
||||
{ CallBase = false };
|
||||
|
||||
_globalNotificationMock = new Mock<GlobalNotificationService>(
|
||||
notificationRepo, new FakeSignalRNotificationService(), Mock.Of<ILogger<GlobalNotificationService>>())
|
||||
{ CallBase = false };
|
||||
@@ -117,7 +112,6 @@ public class AuthFunctionSideEffectTests
|
||||
tenantAdmins ?? _tenantAdminsMock.Object,
|
||||
_metricsRepoMock.Object,
|
||||
preview ?? _previewMock.Object,
|
||||
_telegramMock.Object,
|
||||
_globalNotificationMock.Object,
|
||||
_mcpUserMock.Object,
|
||||
_autoApproveEnqueuerMock.Object,
|
||||
@@ -136,9 +130,6 @@ public class AuthFunctionSideEffectTests
|
||||
// Writes go through UpdateAsync / CreateOrUpdateAsync over one stored row (none by default:
|
||||
// a brand-new tenant). Default: all fire-and-forget calls succeed.
|
||||
_store = _tenantConfigMock.StubWrites(TenantId, row: null);
|
||||
_telegramMock
|
||||
.Setup(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.Returns(Task.CompletedTask);
|
||||
_globalNotificationMock
|
||||
.Setup(x => x.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
@@ -233,7 +224,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.True(string.IsNullOrEmpty(config.DomainName));
|
||||
Assert.Null(config.OnboardedBy);
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -261,8 +252,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.Null(written.DpaAcceptedBy);
|
||||
Assert.Null(written.DpaAcceptedAt);
|
||||
Assert.Equal("auth", source);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(TenantId, Upn), Times.Once);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(TenantId, Upn), Times.Once); // the routable record next to the ping
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(TenantId, Upn), Times.Once); // the routable record
|
||||
_globalNotificationMock.Verify(x => x.CreateNotificationAsync(
|
||||
"preview_signup", "New Tenant Signup",
|
||||
It.Is<string>(m => m.Contains(TenantId) && m.Contains("contoso.com") && m.Contains(Upn)),
|
||||
@@ -350,7 +340,7 @@ public class AuthFunctionSideEffectTests
|
||||
Assert.True(string.IsNullOrEmpty(config.DomainName));
|
||||
Assert.Null(config.OnboardedBy);
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_autoApproveEnqueuerMock.Verify(x => x.EnqueueAsync(
|
||||
It.IsAny<AutopilotMonitor.Functions.Services.Activation.TenantAutoApproveEnvelope>(),
|
||||
It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
@@ -371,7 +361,7 @@ public class AuthFunctionSideEffectTests
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Equal("first@contoso.com", _store.Row!.OnboardedBy);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_autoApproveEnqueuerMock.Verify(x => x.EnqueueAsync(
|
||||
It.IsAny<AutopilotMonitor.Functions.Services.Activation.TenantAutoApproveEnvelope>(),
|
||||
It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
@@ -389,7 +379,7 @@ public class AuthFunctionSideEffectTests
|
||||
await _sut.HandleNewTenantDomainAsync(DefaultConfig(), TenantId, Upn);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -624,7 +614,7 @@ public class AuthFunctionSideEffectTests
|
||||
await _sut.HandleNewTenantDomainAsync(config, TenantId, Upn);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
_telegramMock.Verify(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
_opsEventsMock.Verify(x => x.RecordTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -650,16 +640,17 @@ public class AuthFunctionSideEffectTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleNewTenantDomain_TelegramFailure_DoesNotThrow()
|
||||
public async Task HandleNewTenantDomain_BellFailure_DoesNotThrow()
|
||||
{
|
||||
var config = DefaultConfig();
|
||||
config.DomainName = null!;
|
||||
|
||||
_telegramMock
|
||||
.Setup(x => x.SendNewTenantSignupAsync(It.IsAny<string>(), It.IsAny<string>()))
|
||||
.ThrowsAsync(new HttpRequestException("Telegram unreachable"));
|
||||
_globalNotificationMock
|
||||
.Setup(x => x.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
.ThrowsAsync(new HttpRequestException("Notification store unreachable"));
|
||||
|
||||
// Should not throw — Telegram is fire-and-forget
|
||||
// Should not throw — the bell is fire-and-forget
|
||||
await _sut.HandleNewTenantDomainAsync(config, TenantId, Upn);
|
||||
|
||||
// The write still landed before the fire-and-forget
|
||||
|
||||
@@ -69,7 +69,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
@@ -201,7 +201,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
CancellationToken observed = default;
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
|
||||
@@ -181,6 +181,27 @@ public class ConfigBackupHookTests
|
||||
b => b.UpsertAsync(It.IsAny<ConfigBackupEntry>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoiseOnlyChange_GraphAccessState_NoSnapshot()
|
||||
{
|
||||
// Token side effect: stamped and cleared without anyone changing the configuration.
|
||||
var stored = StoredConfig();
|
||||
var harness = new Harness(TableConfigRepository.ConvertToTenantTableEntity(stored));
|
||||
|
||||
var incoming = RoundtripClone(stored);
|
||||
incoming.GraphAccessRejectedSince = DateTime.UtcNow.AddMinutes(-20);
|
||||
incoming.GraphAccessRejectedCode = "AADSTS7000229";
|
||||
incoming.GraphAccessRejectionConfirmedAt = DateTime.UtcNow;
|
||||
incoming.LastUpdated = DateTime.UtcNow;
|
||||
|
||||
var saved = await harness.Sut.TryReplaceTenantConfigurationAsync(incoming, "\"etag\"", "graph-access", null);
|
||||
|
||||
Assert.True(saved);
|
||||
Assert.Single(harness.Upserts);
|
||||
harness.Backup.Verify(
|
||||
b => b.UpsertAsync(It.IsAny<ConfigBackupEntry>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task NoOpSave_IdenticalContent_NoSnapshot()
|
||||
{
|
||||
|
||||
@@ -65,7 +65,7 @@ public class CorporateIdentifierValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -4,6 +4,7 @@ using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
@@ -12,8 +13,8 @@ using Moq;
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// E-mail as a notification-channel provider (D-332): dispatch routing (never the webhook path,
|
||||
/// never the bot), recipient validation, the Global-Admin gate, the rendered message, and the
|
||||
/// E-mail as a notification-channel provider (D-332): dispatch routing (never the webhook path),
|
||||
/// recipient validation, the Global-Admin gate, the rendered message, and the
|
||||
/// send against a stubbed provider endpoint.
|
||||
/// </summary>
|
||||
public class EmailChannelProviderTests
|
||||
@@ -30,15 +31,6 @@ public class EmailChannelProviderTests
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<IEmailChannelSender> EmailMock()
|
||||
{
|
||||
var mock = new Mock<IEmailChannelSender>();
|
||||
@@ -50,6 +42,9 @@ public class EmailChannelProviderTests
|
||||
private static NotificationChannel Channel(string? url, string id = "mail", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = Email, Url = url, Enabled = enabled };
|
||||
|
||||
private static NotificationChannel Webhook(WebhookProviderType provider, string url)
|
||||
=> new() { Id = provider.ToString(), Name = provider.ToString(), ProviderType = (int)provider, Url = url, Enabled = true };
|
||||
|
||||
private static NotificationAlert Alert() => new()
|
||||
{
|
||||
EventType = "PushDeliveryFailed",
|
||||
@@ -67,9 +62,8 @@ public class EmailChannelProviderTests
|
||||
public async Task Email_channel_reaches_the_email_sender_with_its_recipients_and_never_the_webhook_path()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid; two@example.invalid") }, alert, Platform);
|
||||
@@ -77,7 +71,6 @@ public class EmailChannelProviderTests
|
||||
email.Verify(e => e.SendAlertAsync("ops@example.invalid; two@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -86,7 +79,7 @@ public class EmailChannelProviderTests
|
||||
var email = EmailMock();
|
||||
email.Setup(e => e.SendAlertAsync("ops@example.invalid", It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "Sent to 1 recipient(s)." });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel(""), Channel(null, "n") }, Alert(), Platform);
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
@@ -99,7 +92,7 @@ public class EmailChannelProviderTests
|
||||
[Fact]
|
||||
public async Task Dispatcher_without_an_email_transport_skips_the_channel_and_says_so_on_test()
|
||||
{
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>());
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid") }, Alert(), Platform); // no throw
|
||||
var result = await dispatcher.SendTestAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
@@ -108,6 +101,54 @@ public class EmailChannelProviderTests
|
||||
Assert.Contains("not registered", result.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Webhook_channel_reaches_the_webhook_path_and_never_the_email_sender()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(WebhookProviderType.Slack, "https://hooks.slack.example/x") }, alert, Platform);
|
||||
|
||||
webhook.Verify(w => w.SendAsync("https://hooks.slack.example/x", WebhookProviderType.Slack, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Mixed_list_each_channel_reaches_its_own_transport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel("ops@example.invalid"),
|
||||
Webhook(WebhookProviderType.GenericJson, "https://sales.example/hook"),
|
||||
}, alert, Platform);
|
||||
|
||||
email.Verify(e => e.SendAlertAsync("ops@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync("https://sales.example/hook", WebhookProviderType.GenericJson, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Channels_without_a_destination_are_skipped_on_every_transport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel(null), Webhook(WebhookProviderType.Slack, "") }, Alert(), Platform);
|
||||
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
// ── Validation and gate ───────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
@@ -138,6 +179,10 @@ public class EmailChannelProviderTests
|
||||
|
||||
var bad = NotificationChannel.SerializeList(new[] { Channel("https://hooks.example.invalid/x") });
|
||||
Assert.Contains("not a valid e-mail address", TenantConfigValidation.ValidateNotificationChannels(bad));
|
||||
|
||||
// The branch is per provider: a recipient list on a webhook channel still meets the URL gate.
|
||||
var onWebhook = NotificationChannel.SerializeList(new[] { Webhook(WebhookProviderType.Slack, "ops@example.invalid") });
|
||||
Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(onWebhook));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -146,14 +191,45 @@ public class EmailChannelProviderTests
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider(Email));
|
||||
var candidate = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid") });
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(candidate, null, isGlobalAdmin: true));
|
||||
var refused = TenantConfigValidation.ValidateTelegramChannelGate(candidate, null, isGlobalAdmin: false);
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: true));
|
||||
var refused = TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: false);
|
||||
Assert.Equal("E-mail channels can only be configured by a Global Administrator.", refused);
|
||||
|
||||
// A GA-created channel survives a tenant admin's unrelated save; retargeting it does not.
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(candidate, candidate, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, candidate, isGlobalAdmin: false));
|
||||
var retargeted = NotificationChannel.SerializeList(new[] { Channel("other@example.invalid") });
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(retargeted, candidate, isGlobalAdmin: false));
|
||||
Assert.NotNull(TenantConfigValidation.ValidatePlatformProviderGate(retargeted, candidate, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_refuses_a_tenant_admin_who_flips_the_enabled_state()
|
||||
{
|
||||
var existing = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid", enabled: true) });
|
||||
var disabled = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid", enabled: false) });
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidatePlatformProviderGate(disabled, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_ignores_webhook_channels()
|
||||
{
|
||||
var candidate = NotificationChannel.SerializeList(new[] { Webhook(WebhookProviderType.GenericJson, "https://sales.example/hook") });
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(candidate, null, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateModel_enforces_the_gate_for_every_client_write()
|
||||
{
|
||||
var candidate = new TenantConfiguration
|
||||
{
|
||||
TenantId = "t1",
|
||||
NotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("ops@example.invalid") }),
|
||||
};
|
||||
var existing = new TenantConfiguration { TenantId = "t1" };
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: true));
|
||||
}
|
||||
|
||||
// ── Rendering ─────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Reads the AADSTS code out of Entra's token error body. The body below has the shape Entra
|
||||
/// really sends for a tenant whose enterprise application was removed; the ids are synthetic.
|
||||
/// </summary>
|
||||
public class EntraTokenRejectionTests
|
||||
{
|
||||
private const string MissingServicePrincipalBody =
|
||||
"{\"error\":\"invalid_client\"," +
|
||||
"\"error_description\":\"AADSTS7000229: The client application aaaaaaaa-0000-0000-0000-000000000001 is missing service principal in the tenant 11111111-1111-1111-1111-111111111111. See instructions here: https://go.microsoft.com/fwlink/?linkid=2225119\\r\\nTrace ID: 00000000-0000-0000-0000-000000000002\\r\\nCorrelation ID: 00000000-0000-0000-0000-000000000003\\r\\nTimestamp: 2026-11-01 08:00:00Z\"," +
|
||||
"\"error_codes\":[7000229]," +
|
||||
"\"timestamp\":\"2026-11-01 08:00:00Z\"," +
|
||||
"\"trace_id\":\"00000000-0000-0000-0000-000000000002\"," +
|
||||
"\"correlation_id\":\"00000000-0000-0000-0000-000000000003\"," +
|
||||
"\"error_uri\":\"https://login.microsoftonline.com/error?code=7000229\"}";
|
||||
|
||||
[Fact]
|
||||
public void Parse_reads_the_code_and_keeps_only_the_first_line_of_the_description()
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(401, MissingServicePrincipalBody);
|
||||
|
||||
Assert.Equal(401, rejection.StatusCode);
|
||||
Assert.Equal("AADSTS7000229", rejection.ErrorCode);
|
||||
Assert.True(rejection.IsServicePrincipalMissing);
|
||||
Assert.False(rejection.IsOurCredential);
|
||||
Assert.StartsWith("AADSTS7000229: The client application", rejection.Description);
|
||||
Assert.DoesNotContain("Trace ID", rejection.Description);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_falls_back_to_the_description_without_error_codes()
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(400, "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: Access has been blocked by Conditional Access policies.\"}");
|
||||
|
||||
Assert.Equal("AADSTS53003", rejection.ErrorCode);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData("<html>Bad Request</html>")]
|
||||
[InlineData("[1,2]")]
|
||||
[InlineData("{\"error\":\"invalid_request\"}")]
|
||||
public void Parse_keeps_only_the_status_when_the_body_carries_no_code(string? body)
|
||||
{
|
||||
var rejection = EntraTokenRejection.Parse(400, body);
|
||||
|
||||
Assert.Equal(400, rejection.StatusCode);
|
||||
Assert.Null(rejection.ErrorCode);
|
||||
Assert.Null(rejection.Description);
|
||||
Assert.False(rejection.IsServicePrincipalMissing);
|
||||
Assert.False(rejection.IsOurCredential);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Parse_cuts_a_long_description()
|
||||
{
|
||||
var body = "{\"error_description\":\"AADSTS50000: " + new string('x', 1000) + "\"}";
|
||||
|
||||
Assert.Equal(300, EntraTokenRejection.Parse(400, body).Description!.Length);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("AADSTS7000215", true)]
|
||||
[InlineData("AADSTS7000222", true)]
|
||||
[InlineData("AADSTS7000218", true)]
|
||||
[InlineData("aadsts7000222", true)]
|
||||
[InlineData("AADSTS7000229", false)]
|
||||
[InlineData("AADSTS7000112", false)]
|
||||
[InlineData("AADSTS53003", false)]
|
||||
public void IsOurCredential_only_blames_our_own_secret(string code, bool expected)
|
||||
{
|
||||
Assert.Equal(expected, new EntraTokenRejection(401, code, null).IsOurCredential);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Functions.Feedback;
|
||||
@@ -19,8 +18,8 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// The rating bubble's answer route, driven through <c>ProcessSubmitAsync</c> (the HTTP shell only
|
||||
/// reads the body and maps the validation error). Pins that a recorded rating, and nothing else,
|
||||
/// raises <c>FeedbackRatingReceived</c>, the Global Admin bell and the Telegram ping, and that the
|
||||
/// alert carries neither the comment nor the sender.
|
||||
/// raises <c>FeedbackRatingReceived</c> and the Global Admin bell, and that the alert carries neither
|
||||
/// the comment nor the sender.
|
||||
/// </summary>
|
||||
public sealed class FeedbackFunctionSubmitTests
|
||||
{
|
||||
@@ -34,7 +33,7 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
private static readonly string EntryPath = Constants.PortalUserFeedbackEntryPath(FeedbackId);
|
||||
|
||||
[Fact]
|
||||
public async Task Rating_IsStored_RaisesTheEvent_RingsTheBell_AndPings()
|
||||
public async Task Rating_IsStored_RaisesTheEvent_AndRingsTheBell()
|
||||
{
|
||||
var h = new Harness();
|
||||
|
||||
@@ -64,13 +63,12 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
|
||||
h.Bell.Verify(b => b.CreateNotificationAsync(
|
||||
"feedback", "New Rating", $"{Upn} (contoso.invalid) · 4/5", EntryPath), Times.Once);
|
||||
h.Telegram.Verify(t => t.SendFeedbackAsync(TenantId, Upn, DisplayName, 4, Comment), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TheAlert_CarriesNeitherCommentNorSender()
|
||||
{
|
||||
// Channels bound to FeedbackRatingReceived (Telegram, Slack, webhooks, e-mail) receive message
|
||||
// Channels bound to FeedbackRatingReceived (Slack, webhooks, e-mail) receive message
|
||||
// and details; the comment is read in the portal. Personal data must never ride along.
|
||||
var h = new Harness();
|
||||
|
||||
@@ -160,7 +158,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
Assert.Empty(h.OpsEvents);
|
||||
h.Bell.Verify(b => b.CreateNotificationAsync(
|
||||
"feedback", "New Rating", It.IsAny<string>(), EntryPath), Times.Once);
|
||||
h.Telegram.Verify(t => t.SendFeedbackAsync(TenantId, Upn, DisplayName, 3, Comment), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -183,7 +180,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
public FakeFeedbackRepository Repo { get; }
|
||||
public List<OpsEventEntry> OpsEvents { get; } = new();
|
||||
public Mock<GlobalNotificationService> Bell { get; }
|
||||
public Mock<TelegramNotificationService> Telegram { get; }
|
||||
|
||||
public Harness(FeedbackEntry? current = null, bool configExists = true, bool opsEventThrows = false)
|
||||
{
|
||||
@@ -203,9 +199,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
Mock.Of<INotificationRepository>(), new FakeSignalRNotificationService(),
|
||||
Mock.Of<ILogger<GlobalNotificationService>>())
|
||||
{ CallBase = false };
|
||||
Telegram = new Mock<TelegramNotificationService>(
|
||||
new HttpClient(), configRepo, Mock.Of<ILogger<TelegramNotificationService>>())
|
||||
{ CallBase = false };
|
||||
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
configRepo, NullLogger<AdminConfigurationService>.Instance, cache)
|
||||
@@ -216,7 +209,7 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
|
||||
Sut = new FeedbackFunction(
|
||||
NullLogger<FeedbackFunction>.Instance, tenantConfig.Object, adminConfig.Object, tenantAdmins.Object,
|
||||
Telegram.Object, Mock.Of<ISessionRepository>(), Repo,
|
||||
Mock.Of<ISessionRepository>(), Repo,
|
||||
BuildOpsEventService(adminConfig.Object, opsEventThrows), Bell.Object);
|
||||
}
|
||||
|
||||
@@ -228,8 +221,6 @@ public sealed class FeedbackFunctionSubmitTests
|
||||
{
|
||||
Bell.Verify(b => b.CreateNotificationAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()), Times.Never);
|
||||
Telegram.Verify(t => t.SendFeedbackAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<string?>()), Times.Never);
|
||||
}
|
||||
|
||||
/// <summary>A real OpsEventService over a capturing repository, so the actual record method runs.</summary>
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using AutopilotMonitor.Functions.Functions.Config;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
@@ -55,6 +56,9 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
"enableIntegrityBypassAnalyzer",
|
||||
"enableSoftwareInventoryAnalyzer",
|
||||
"entitlements",
|
||||
// Non-sensitive by review: booleans only; the Entra code stays in the admin-gated config.
|
||||
"graphAccessAppMissing",
|
||||
"graphAccessRejected",
|
||||
"isTrial",
|
||||
"showScriptOutput",
|
||||
"trialAvailable",
|
||||
@@ -65,6 +69,54 @@ public class GetTenantFeatureFlagsPayloadTests
|
||||
}, fieldNames);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_GraphAccess_SuspicionAloneShowsNothing()
|
||||
{
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
ValidateAutopilotDevice = true,
|
||||
GraphAccessRejectedSince = Now.AddMinutes(-5),
|
||||
GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode,
|
||||
});
|
||||
|
||||
Assert.False(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.False(element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(EntraTokenRejection.ServicePrincipalMissingCode, true)]
|
||||
[InlineData("AADSTS53003", false)]
|
||||
[InlineData(null, false)]
|
||||
public void Payload_GraphAccess_ConfirmedShowsTheNotice_AndNamesOnlyTheMissingApp(string? code, bool appMissing)
|
||||
{
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
ValidateAutopilotDevice = true,
|
||||
GraphAccessRejectedSince = Now.AddHours(-1),
|
||||
GraphAccessRejectedCode = code,
|
||||
GraphAccessRejectionConfirmedAt = Now.AddMinutes(-45),
|
||||
});
|
||||
|
||||
Assert.True(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.Equal(appMissing, element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
Assert.DoesNotContain("AADSTS", element.GetRawText());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_GraphAccess_StaysSilentWithoutDeviceValidation()
|
||||
{
|
||||
// The switch being off has its own banner; the agent does not depend on Graph then.
|
||||
var element = Serialize(new TenantConfiguration
|
||||
{
|
||||
GraphAccessRejectedSince = Now.AddHours(-1),
|
||||
GraphAccessRejectedCode = EntraTokenRejection.ServicePrincipalMissingCode,
|
||||
GraphAccessRejectionConfirmedAt = Now.AddMinutes(-45),
|
||||
});
|
||||
|
||||
Assert.False(element.GetProperty("graphAccessRejected").GetBoolean());
|
||||
Assert.False(element.GetProperty("graphAccessAppMissing").GetBoolean());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Payload_PassesThroughExplicitFieldValues()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The two stages of a lost Graph access: one refusal only stamps a suspicion, a refusal at least
|
||||
/// 15 minutes later confirms it and alerts once, a freshly issued token ends the episode. Runs on
|
||||
/// the real configuration service over a stored row, so the storage round trip of the three fields
|
||||
/// and the decision on the fresh row are part of what is proven.
|
||||
/// </summary>
|
||||
public class GraphAccessHealthRecorderTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private static readonly DateTimeOffset Start = new(2026, 11, 2, 8, 0, 0, TimeSpan.Zero);
|
||||
private static readonly EntraTokenRejection MissingApp = new(401, EntraTokenRejection.ServicePrincipalMissingCode, "missing");
|
||||
private static readonly EntraTokenRejection ConditionalAccess = new(400, "AADSTS53003", "blocked");
|
||||
private static readonly EntraTokenRejection ExpiredSecret = new(401, "AADSTS7000222", "expired");
|
||||
|
||||
private readonly Mock<IConfigRepository> _repo = new();
|
||||
private readonly StoredTenantConfig _store;
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly TestTimeProvider _time = new(Start);
|
||||
private readonly GraphAccessHealthRecorder _sut;
|
||||
|
||||
public GraphAccessHealthRecorderTests()
|
||||
{
|
||||
_store = _repo.BackWithRow(ValidatedRow());
|
||||
_sut = NewInstance();
|
||||
}
|
||||
|
||||
private static TenantConfiguration ValidatedRow()
|
||||
{
|
||||
var row = TenantConfiguration.CreateDefault(TenantId);
|
||||
row.ValidateAutopilotDevice = true;
|
||||
return row;
|
||||
}
|
||||
|
||||
/// <summary>A backend instance of its own: its own config cache over the same stored row.</summary>
|
||||
private GraphAccessHealthRecorder NewInstance()
|
||||
{
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfig = new TenantConfigurationService(_repo.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => _events.Add(e))
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
_repo.Object, NullLogger<AdminConfigurationService>.Instance, cache) { CallBase = false };
|
||||
var opsEvents = new OpsEventService(
|
||||
opsRepo.Object, NullLogger<OpsEventService>.Instance, TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
|
||||
return new GraphAccessHealthRecorder(
|
||||
tenantConfig, new Lazy<OpsEventService>(() => opsEvents), NullLogger<GraphAccessHealthRecorder>.Instance, _time);
|
||||
}
|
||||
|
||||
private void Advance(TimeSpan by) => _time.SetUtcNow(_time.GetUtcNow() + by);
|
||||
|
||||
// ── Rejected: the two stages ────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task First_rejection_only_stamps_the_suspicion()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(Start.UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Equal(EntraTokenRejection.ServicePrincipalMissingCode, _store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Further_rejections_inside_the_window_write_nothing()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(TimeSpan.FromMinutes(14));
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Single(_store.Writes);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rejection_after_the_window_confirms_and_alerts_once()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, ConditionalAccess);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(TimeSpan.FromMinutes(5));
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(2, _store.Writes.Count);
|
||||
Assert.Equal(Start.UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Equal(Start.UtcDateTime + GraphAccessHealthRecorder.ConfirmAfter, _store.Row.GraphAccessRejectionConfirmedAt);
|
||||
// The code of the latest refusal counts: it selects the text and the repair.
|
||||
Assert.Equal(EntraTokenRejection.ServicePrincipalMissingCode, _store.Row.GraphAccessRejectedCode);
|
||||
var alert = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, alert.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Error, alert.Severity);
|
||||
Assert.Equal(TenantId, alert.TenantId);
|
||||
Assert.Contains(EntraTokenRejection.ServicePrincipalMissingCode, alert.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Across_instances_only_one_confirms_and_alerts()
|
||||
{
|
||||
var other = NewInstance();
|
||||
// The other instance holds a view from before the suspicion.
|
||||
await other.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await other.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Single(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_lost_write_alerts_nothing_and_the_next_rejection_decides_again()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
_store.RejectWrites = true;
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
|
||||
_store.RejectWrites = false;
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Single(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_runs_through_the_same_two_stages()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, EntraTokenRejection.NoApplicationRole);
|
||||
Assert.Equal("NoApplicationRole", _store.Row!.GraphAccessRejectedCode);
|
||||
Assert.Empty(_events);
|
||||
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, EntraTokenRejection.NoApplicationRole);
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_device_validation_stays_untouched()
|
||||
{
|
||||
_store.Row = TenantConfiguration.CreateDefault(TenantId);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_a_row_stays_without_one()
|
||||
{
|
||||
_store.Row = null;
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
// ── Our own credentials ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task Our_expired_secret_never_touches_the_tenant_and_alerts_once_per_hour()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
Advance(TimeSpan.FromMinutes(59));
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
var alert = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAppCredentialRejected, alert.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Critical, alert.Severity);
|
||||
|
||||
Advance(TimeSpan.FromMinutes(2));
|
||||
await _sut.RecordRejectedAsync(TenantId, ExpiredSecret);
|
||||
|
||||
Assert.Equal(2, _events.Count);
|
||||
}
|
||||
|
||||
// ── Minted ──────────────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_writes_nothing_without_a_state()
|
||||
{
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_clears_a_suspicion_silently()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_minted_token_ends_a_confirmed_episode_with_one_recovery_event()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
_events.Clear();
|
||||
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectedCode);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
var recovery = Assert.Single(_events);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRecovered, recovery.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, recovery.Severity);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task After_a_recovery_a_new_episode_starts_at_the_first_stage()
|
||||
{
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
Advance(GraphAccessHealthRecorder.ConfirmAfter);
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Equal(_time.GetUtcNow().UtcDateTime, _store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
// ── Fail-soft: the agent route must not notice ──────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task A_storage_failure_never_reaches_the_caller()
|
||||
{
|
||||
_store.ThrowOnRead = new InvalidOperationException("storage down");
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
await _sut.RecordMintedAsync(TenantId);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_write_never_reaches_the_caller()
|
||||
{
|
||||
_store.ThrowOnWrite = new InvalidOperationException("storage down");
|
||||
|
||||
await _sut.RecordRejectedAsync(TenantId, MissingApp);
|
||||
|
||||
Assert.Empty(_store.Writes);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Tests.GraphResolution;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The maintenance run asks Entra for every tenant with a suspicion, so the state does not depend on
|
||||
/// device traffic. Runs the real token service and recorder; only Entra's answer is scripted.
|
||||
/// </summary>
|
||||
public class GraphAccessRecheckMaintenanceTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string OtherTenantId = "22222222-2222-2222-2222-222222222222";
|
||||
private const string MissingAppBody = "{\"error\":\"invalid_client\",\"error_description\":\"AADSTS7000229: missing service principal\",\"error_codes\":[7000229]}";
|
||||
private const string ConditionalAccessBody = "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: blocked\",\"error_codes\":[53003]}";
|
||||
private static readonly DateTimeOffset Now = new(2026, 11, 2, 8, 0, 0, TimeSpan.Zero);
|
||||
private static readonly string TokenBody = TestGraphToken.Body(TestGraphToken.ValidationRole);
|
||||
|
||||
private readonly ScriptedEntra _entra = new();
|
||||
private readonly Mock<IConfigRepository> _repo = new();
|
||||
private readonly StoredTenantConfig _store;
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly GraphTokenService _tokens;
|
||||
private readonly GraphAccessRecheckMaintenance _sut;
|
||||
|
||||
public GraphAccessRecheckMaintenanceTests()
|
||||
{
|
||||
_store = _repo.BackWithRow(ValidatedRow());
|
||||
_repo.Setup(r => r.GetAllTenantConfigurationsAsync())
|
||||
.ReturnsAsync(() => _store.Row == null
|
||||
? new List<TenantConfiguration>()
|
||||
: new List<TenantConfiguration> { ConfigRepoMockExtensions.Clone(_store.Row) });
|
||||
|
||||
var cache = new MemoryCache(new MemoryCacheOptions());
|
||||
var tenantConfig = new TenantConfigurationService(_repo.Object, NullLogger<TenantConfigurationService>.Instance, cache);
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => _events.Add(e))
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
_repo.Object, NullLogger<AdminConfigurationService>.Instance, cache) { CallBase = false };
|
||||
var opsEvents = new OpsEventService(
|
||||
opsRepo.Object, NullLogger<OpsEventService>.Instance, TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
var recorder = new GraphAccessHealthRecorder(
|
||||
tenantConfig, new Lazy<OpsEventService>(() => opsEvents), NullLogger<GraphAccessHealthRecorder>.Instance, new TestTimeProvider(Now));
|
||||
|
||||
_tokens = NewTokenService(new SingleHandlerFactory(_entra), recorder, tenantConfig);
|
||||
_sut = new GraphAccessRecheckMaintenance(_repo.Object, _tokens, NullLogger<GraphAccessRecheckMaintenance>.Instance);
|
||||
}
|
||||
|
||||
private static GraphTokenService NewTokenService(
|
||||
IHttpClientFactory factory, IGraphAccessHealthRecorder recorder, TenantConfigurationService tenantConfig)
|
||||
{
|
||||
var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "aaaaaaaa-0000-0000-0000-000000000001",
|
||||
["EntraId:ClientSecret"] = "secret",
|
||||
}).Build();
|
||||
return new GraphTokenService(
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
new EntraAppRegistry(configuration, NullLogger<EntraAppRegistry>.Instance),
|
||||
tenantConfig,
|
||||
recorder);
|
||||
}
|
||||
|
||||
private static TenantConfiguration ValidatedRow()
|
||||
{
|
||||
var row = TenantConfiguration.CreateDefault(TenantId);
|
||||
row.ValidateAutopilotDevice = true;
|
||||
return row;
|
||||
}
|
||||
|
||||
private void Suspect(TimeSpan age, string code = EntraTokenRejection.ServicePrincipalMissingCode)
|
||||
{
|
||||
_store.Row!.GraphAccessRejectedSince = Now.UtcDateTime - age;
|
||||
_store.Row.GraphAccessRejectedCode = code;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Confirms_an_old_suspicion_without_device_traffic()
|
||||
{
|
||||
// A real 401 goes into the retry delay, which the budget cuts: the refusal must survive it.
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.Unauthorized, MissingAppBody);
|
||||
|
||||
Assert.Equal(1, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(Now.UtcDateTime, _store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_young_suspicion_stays_a_suspicion()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(10));
|
||||
_entra.Answer(HttpStatusCode.BadRequest, ConditionalAccessBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Clears_the_notice_after_a_repair_made_directly_in_Entra()
|
||||
{
|
||||
Suspect(TimeSpan.FromHours(5));
|
||||
_store.Row!.GraphAccessRejectionConfirmedAt = Now.UtcDateTime - TimeSpan.FromHours(4);
|
||||
_entra.Answer(HttpStatusCode.OK, TokenBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Null(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRecovered, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_keeps_the_notice_up()
|
||||
{
|
||||
// A user's sign-in re-created the removed enterprise application without any permission.
|
||||
Suspect(TimeSpan.FromHours(5));
|
||||
_store.Row!.GraphAccessRejectionConfirmedAt = Now.UtcDateTime - TimeSpan.FromHours(4);
|
||||
_entra.Answer(HttpStatusCode.OK, TestGraphToken.Body());
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.NotNull(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_confirms_an_old_suspicion()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.OK, TestGraphToken.Body());
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Equal("NoApplicationRole", _store.Row.GraphAccessRejectedCode);
|
||||
Assert.Equal(OpsEventTypes.GraphAccessRejected, Assert.Single(_events).EventType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cached_token_cannot_answer_the_question()
|
||||
{
|
||||
// The token this instance still holds stays valid after the enterprise app is gone.
|
||||
_entra.Answer(HttpStatusCode.OK, TokenBody);
|
||||
await _tokens.GetAccessTokenAsync(TenantId);
|
||||
Assert.Equal(1, _entra.Posts);
|
||||
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.BadRequest, ConditionalAccessBody);
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.Equal(2, _entra.Posts);
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task No_answer_from_Entra_decides_nothing()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_entra.Answer(HttpStatusCode.ServiceUnavailable, "{}");
|
||||
|
||||
await _sut.RunAsync();
|
||||
|
||||
Assert.NotNull(_store.Row!.GraphAccessRejectedSince);
|
||||
Assert.Null(_store.Row.GraphAccessRejectionConfirmedAt);
|
||||
Assert.Empty(_events);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_a_suspicion_is_not_asked()
|
||||
{
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(0, _entra.Posts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_that_switched_device_validation_off_is_not_asked()
|
||||
{
|
||||
Suspect(TimeSpan.FromMinutes(20));
|
||||
_store.Row!.ValidateAutopilotDevice = false;
|
||||
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
|
||||
Assert.Equal(0, _entra.Posts);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_read_ends_the_run_quietly()
|
||||
{
|
||||
_repo.Setup(r => r.GetAllTenantConfigurationsAsync()).ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
Assert.Equal(0, await _sut.RunAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task One_failing_tenant_does_not_stop_the_next()
|
||||
{
|
||||
// The stub throws on the unscripted first tenant, which the token service does not catch.
|
||||
var repo = new Mock<IConfigRepository>();
|
||||
var first = ValidatedRow();
|
||||
first.GraphAccessRejectedSince = Now.UtcDateTime;
|
||||
var second = ValidatedRow();
|
||||
second.TenantId = OtherTenantId;
|
||||
second.GraphAccessRejectedSince = Now.UtcDateTime;
|
||||
repo.Setup(r => r.GetAllTenantConfigurationsAsync()).ReturnsAsync(new List<TenantConfiguration> { first, second });
|
||||
var recorder = new Mock<IGraphAccessHealthRecorder>();
|
||||
var handler = new StubHttpMessageHandler().When($"/{OtherTenantId}/oauth2", HttpStatusCode.OK, TokenBody);
|
||||
var tenantConfig = new TenantConfigurationService(
|
||||
repo.Object, NullLogger<TenantConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()));
|
||||
var sut = new GraphAccessRecheckMaintenance(
|
||||
repo.Object, NewTokenService(new StubHttpClientFactory(handler), recorder.Object, tenantConfig), NullLogger<GraphAccessRecheckMaintenance>.Instance);
|
||||
|
||||
Assert.Equal(2, await sut.RunAsync());
|
||||
|
||||
recorder.Verify(r => r.RecordMintedAsync(OtherTenantId), Times.Once);
|
||||
}
|
||||
|
||||
/// <summary>Entra's token endpoint with an answer that can change between calls.</summary>
|
||||
private sealed class ScriptedEntra : HttpMessageHandler
|
||||
{
|
||||
private HttpStatusCode _status = HttpStatusCode.OK;
|
||||
private string _body = "{}";
|
||||
|
||||
public int Posts { get; private set; }
|
||||
|
||||
public void Answer(HttpStatusCode status, string body)
|
||||
{
|
||||
_status = status;
|
||||
_body = body;
|
||||
}
|
||||
|
||||
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
|
||||
{
|
||||
Posts++;
|
||||
return Task.FromResult(new HttpResponseMessage(_status) { Content = new StringContent(_body) });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,7 +32,7 @@ public class GraphAuthFailureTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
return new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -342,7 +342,8 @@ public class GraphFeatureDetectorTests
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
var telemetry = new TelemetryClient(new TelemetryConfiguration { DisableTelemetry = true });
|
||||
var detector = new GraphFeatureDetector(
|
||||
tokenService, cache, NullLogger<GraphFeatureDetector>.Instance, telemetry);
|
||||
@@ -497,7 +498,8 @@ public class GraphFeatureDetectorTests
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
}
|
||||
|
||||
@@ -529,7 +531,8 @@ public class GraphFeatureDetectorTests
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
_firstToken = firstToken;
|
||||
_laterToken = laterToken;
|
||||
@@ -567,7 +570,8 @@ public class GraphFeatureDetectorTests
|
||||
factory,
|
||||
cache,
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
StubTenantConfigService(),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{
|
||||
_onInvalidate = onInvalidate;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
using System.IdentityModel.Tokens.Jwt;
|
||||
using System.Security.Claims;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests.GraphResolution;
|
||||
|
||||
/// <summary>What Entra's token endpoint answers on success, with the application roles of the tenant's service principal.</summary>
|
||||
internal static class TestGraphToken
|
||||
{
|
||||
public const string ValidationRole = "DeviceManagementServiceConfig.Read.All";
|
||||
|
||||
public static string Body(params string[] roles)
|
||||
{
|
||||
var jwt = new JwtSecurityToken(
|
||||
issuer: "https://sts.windows.net/test",
|
||||
audience: "https://graph.microsoft.com",
|
||||
claims: roles.Select(r => new Claim("roles", r)),
|
||||
notBefore: null,
|
||||
expires: DateTime.UtcNow.AddHours(1));
|
||||
return $"{{\"access_token\":\"{new JwtSecurityTokenHandler().WriteToken(jwt)}\",\"expires_in\":3599}}";
|
||||
}
|
||||
}
|
||||
@@ -43,16 +43,21 @@ public class GraphTokenServiceTests
|
||||
/// these tests ⇒ the registry always resolves the primary credentials and the tenant-config
|
||||
/// lookup is never consulted for the decision — a null-returning repo is fine.
|
||||
/// </summary>
|
||||
private static GraphTokenService NewService(IConfiguration config, System.Net.Http.IHttpClientFactory factory) =>
|
||||
private static GraphTokenService NewService(
|
||||
IConfiguration config,
|
||||
System.Net.Http.IHttpClientFactory factory,
|
||||
IGraphAccessHealthRecorder? accessHealth = null,
|
||||
Microsoft.Extensions.Logging.ILogger<GraphTokenService>? logger = null) =>
|
||||
new GraphTokenService(
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
logger ?? NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
new EntraAppRegistry(config, NullLogger<EntraAppRegistry>.Instance),
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
accessHealth ?? Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
|
||||
private static int TokenPosts(StubHttpMessageHandler handler, string tenantId)
|
||||
=> handler.Requests.Count(u => u.Contains($"/{tenantId}/oauth2/v2.0/token"));
|
||||
@@ -240,7 +245,8 @@ public class GraphTokenServiceTests
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
NullLogger<AutopilotMonitor.Functions.Services.TenantConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())));
|
||||
new MemoryCache(new MemoryCacheOptions())),
|
||||
Moq.Mock.Of<IGraphAccessHealthRecorder>());
|
||||
|
||||
var probe1 = await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
var probe2 = await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
@@ -260,6 +266,195 @@ public class GraphTokenServiceTests
|
||||
Assert.Equal(3, TokenPosts(handler, TenantA));
|
||||
}
|
||||
|
||||
// ── Entra's refusal ─────────────────────────────────────────────────────
|
||||
|
||||
private const string MissingAppBody = "{\"error\":\"invalid_client\",\"error_description\":\"AADSTS7000229: missing service principal\",\"error_codes\":[7000229]}";
|
||||
private const string ConditionalAccessBody = "{\"error\":\"invalid_request\",\"error_description\":\"AADSTS53003: blocked\",\"error_codes\":[53003]}";
|
||||
|
||||
private static GraphTokenService ServiceFor(
|
||||
StubHttpMessageHandler handler,
|
||||
Moq.Mock<IGraphAccessHealthRecorder> accessHealth,
|
||||
Microsoft.Extensions.Logging.ILogger<GraphTokenService>? logger = null)
|
||||
{
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "test-client",
|
||||
["EntraId:ClientSecret"] = "test-secret",
|
||||
})
|
||||
.Build();
|
||||
return NewService(config, new StubHttpClientFactory(handler), accessHealth.Object, logger);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Rejection_survives_a_budget_that_ends_the_retries()
|
||||
{
|
||||
// The validators run on 8 s and the recheck on 3 s; the chain's first retry waits 5 s.
|
||||
var handler = new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.Unauthorized, MissingAppBody);
|
||||
var (svc, _) = Create(handler);
|
||||
using var budget = new System.Threading.CancellationTokenSource(TimeSpan.FromMilliseconds(500));
|
||||
|
||||
var result = await svc.GetAccessTokenAsync(TenantA, budget.Token);
|
||||
|
||||
Assert.Null(result.AccessToken);
|
||||
Assert.True(result.IsTransient);
|
||||
Assert.NotNull(result.Rejection);
|
||||
Assert.Equal(401, result.Rejection!.StatusCode);
|
||||
Assert.True(result.Rejection.IsServicePrincipalMissing);
|
||||
Assert.Equal(1, TokenPosts(handler, TenantA));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.ServiceUnavailable)]
|
||||
[InlineData(HttpStatusCode.TooManyRequests)]
|
||||
[InlineData(HttpStatusCode.RequestTimeout)]
|
||||
public async Task A_server_error_or_throttle_is_no_rejection(HttpStatusCode status)
|
||||
{
|
||||
var handler = new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", status, MissingAppBody);
|
||||
var (svc, _) = Create(handler);
|
||||
using var budget = new System.Threading.CancellationTokenSource(TimeSpan.FromMilliseconds(200));
|
||||
|
||||
var result = await svc.GetAccessTokenAsync(TenantA, budget.Token);
|
||||
|
||||
Assert.True(result.IsTransient);
|
||||
Assert.Null(result.Rejection);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_reports_a_rejection()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody), health);
|
||||
|
||||
var result = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.False(result.IsTransient);
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, Moq.It.Is<EntraTokenRejection>(r => r.ErrorCode == "AADSTS53003")), Moq.Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_portal_path_does_not_report_a_rejection()
|
||||
{
|
||||
// The consent flows see refusals as a matter of course while a consent propagates.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody), health);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Only_a_fresh_token_counts_as_proof_of_access()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body(TestGraphToken.ValidationRole)), health);
|
||||
|
||||
var minted = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
var cached = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.True(minted.IsFreshlyMinted);
|
||||
Assert.False(cached.IsFreshlyMinted);
|
||||
health.Verify(h => h.RecordMintedAsync(TenantA), Moq.Times.Once);
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_token_without_a_role_is_no_proof_of_access()
|
||||
{
|
||||
// A sign-in consent re-created the removed enterprise application without any permission.
|
||||
// The notice must survive it: this is the moment an admin signs in to read it.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordMintedAsync(Moq.It.IsAny<string>()), Moq.Times.Never);
|
||||
health.Verify(h => h.RecordRejectedAsync(Moq.It.IsAny<string>(), Moq.It.IsAny<EntraTokenRejection>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_counts_a_token_without_a_role_as_a_refusal()
|
||||
{
|
||||
// Graph refuses every call made with it, so the agent is locked out all the same.
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
var result = await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
Assert.NotNull(result.AccessToken);
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, EntraTokenRejection.NoApplicationRole), Moq.Times.Once);
|
||||
health.Verify(h => h.RecordMintedAsync(Moq.It.IsAny<string>()), Moq.Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Agent_validation_does_not_judge_a_token_from_the_cache()
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.OK, TestGraphToken.Body()), health);
|
||||
|
||||
await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
await svc.GetAccessTokenForAgentValidationAsync(TenantA);
|
||||
|
||||
health.Verify(h => h.RecordRejectedAsync(TenantA, EntraTokenRejection.NoApplicationRole), Moq.Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_probe_under_explicit_credentials_reports_nothing()
|
||||
{
|
||||
// A token of the app the tenant is not homed on says nothing about the tenant's access.
|
||||
var config = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["EntraId:ClientId"] = "test-client",
|
||||
["EntraId:ClientSecret"] = "test-secret",
|
||||
})
|
||||
.Build();
|
||||
var registry = new EntraAppRegistry(config, NullLogger<EntraAppRegistry>.Instance);
|
||||
|
||||
foreach (var (status, body) in new[]
|
||||
{
|
||||
(HttpStatusCode.OK, TestGraphToken.Body(TestGraphToken.ValidationRole)),
|
||||
(HttpStatusCode.OK, TestGraphToken.Body()),
|
||||
(HttpStatusCode.BadRequest, ConditionalAccessBody),
|
||||
})
|
||||
{
|
||||
var health = new Moq.Mock<IGraphAccessHealthRecorder>();
|
||||
var svc = NewService(config, new StubHttpClientFactory(new StubHttpMessageHandler().When($"/{TenantA}/oauth2", status, body)), health.Object);
|
||||
|
||||
await svc.GetAccessTokenForAppAsync(TenantA, registry.Primary);
|
||||
|
||||
health.VerifyNoOtherCalls();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Entra_refusal_is_logged_once_per_window()
|
||||
{
|
||||
var logger = new Moq.Mock<Microsoft.Extensions.Logging.ILogger<GraphTokenService>>();
|
||||
var svc = ServiceFor(new StubHttpMessageHandler()
|
||||
.When($"/{TenantA}/oauth2", HttpStatusCode.BadRequest, ConditionalAccessBody),
|
||||
new Moq.Mock<IGraphAccessHealthRecorder>(), logger.Object);
|
||||
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
await svc.GetAccessTokenAsync(TenantA);
|
||||
|
||||
logger.Verify(l => l.Log(
|
||||
Microsoft.Extensions.Logging.LogLevel.Warning,
|
||||
Moq.It.IsAny<Microsoft.Extensions.Logging.EventId>(),
|
||||
Moq.It.Is<Moq.It.IsAnyType>((state, _) => state.ToString()!.Contains("rejected by Entra") && state.ToString()!.Contains("AADSTS53003")),
|
||||
Moq.It.IsAny<Exception?>(),
|
||||
Moq.It.IsAny<Func<Moq.It.IsAnyType, Exception?, string>>()), Moq.Times.Once);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
||||
{
|
||||
var deadline = DateTime.UtcNow + timeout;
|
||||
|
||||
@@ -79,7 +79,7 @@ public class IntuneEnrollmentValidationTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokens = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object, Mock.Of<IGraphAccessHealthRecorder>())
|
||||
{ CallBase = false };
|
||||
tokens.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using Xunit;
|
||||
@@ -7,6 +8,18 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class MaintenanceStepTimerTests
|
||||
{
|
||||
private const long Mb = 1024 * 1024;
|
||||
|
||||
private static MaintenanceStepTimer TimerOver(params (long PeakMb, long AllocatedMb)[] readings)
|
||||
{
|
||||
var queue = new Queue<(long PeakMb, long AllocatedMb)>(readings);
|
||||
return new MaintenanceStepTimer(() =>
|
||||
{
|
||||
var (peakMb, allocatedMb) = queue.Dequeue();
|
||||
return new MaintenanceMemoryReading(peakMb * Mb, allocatedMb * Mb, 1740 * Mb, 2088 * Mb);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Records_every_step_in_order_and_returns_the_body_result()
|
||||
{
|
||||
@@ -30,4 +43,49 @@ public class MaintenanceStepTimerTests
|
||||
|
||||
Assert.True(timer.Steps.ContainsKey("boom"));
|
||||
}
|
||||
|
||||
// The memory columns have to name the step that set the peak: each step records the process
|
||||
// peak after it ran and only the allocation of its own span.
|
||||
[Fact]
|
||||
public async Task Each_step_reports_the_process_peak_after_it_and_its_own_allocation()
|
||||
{
|
||||
var timer = TimerOver((400, 100), (400, 150), (400, 150), (1900, 1450), (1900, 1450));
|
||||
|
||||
await timer.RunAsync("light", () => Task.CompletedTask);
|
||||
await timer.RunAsync<int>("heavy", () => Task.FromResult(7));
|
||||
|
||||
var report = timer.MemoryReport();
|
||||
Assert.Equal(400, report.StepsPeakMb["light"]);
|
||||
Assert.Equal(50, report.StepsAllocMb["light"]);
|
||||
Assert.Equal(1900, report.StepsPeakMb["heavy"]);
|
||||
Assert.Equal(1300, report.StepsAllocMb["heavy"]);
|
||||
Assert.Equal(1740, report.GcHeapLimitMb);
|
||||
Assert.Equal(2088, report.GcHighLoadMb);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_throwing_step_still_reports_its_memory()
|
||||
{
|
||||
var timer = TimerOver((300, 10), (800, 60), (800, 60));
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => timer.RunAsync("boom", () => throw new InvalidOperationException("x")));
|
||||
|
||||
var report = timer.MemoryReport();
|
||||
Assert.Equal(800, report.StepsPeakMb["boom"]);
|
||||
Assert.Equal(50, report.StepsAllocMb["boom"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_real_reading_sees_the_process_peak_and_the_gc_limits()
|
||||
{
|
||||
GC.Collect();
|
||||
|
||||
var reading = MaintenanceMemoryReading.Capture();
|
||||
|
||||
Assert.True(reading.PeakWorkingSetBytes > 0);
|
||||
Assert.True(reading.AllocatedBytes > 0);
|
||||
Assert.True(reading.GcHeapLimitBytes > 0);
|
||||
Assert.True(reading.GcHighLoadBytes > 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -435,10 +435,10 @@ public class NotificationChannelHealthTests
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(webhookResult);
|
||||
}
|
||||
var telegram = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance) { CallBase = false };
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>())).ReturnsAsync(Ok());
|
||||
return new NotificationChannelDispatcher(webhook.Object, telegram.Object, (push ?? new Mock<IPushChannelSender>()).Object,
|
||||
Mock.Of<IEmailChannelSender>(), recorder.Object, NullLogger<NotificationChannelDispatcher>.Instance);
|
||||
var email = new Mock<IEmailChannelSender>();
|
||||
email.Setup(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>())).ReturnsAsync(Ok());
|
||||
return new NotificationChannelDispatcher(webhook.Object, (push ?? new Mock<IPushChannelSender>()).Object,
|
||||
email.Object, recorder.Object, NullLogger<NotificationChannelDispatcher>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -447,13 +447,13 @@ public class NotificationChannelHealthTests
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = Dispatcher(recorder, push: push);
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
var mail = new NotificationChannel { Id = "mail", Name = "mail", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
var phones = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Push, Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram, phones }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), mail, phones }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"), It.Is<NotificationSendResult>(x => !x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.Is<NotificationSendResult>(x => x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "mail"), It.Is<NotificationSendResult>(x => x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(It.IsAny<NotificationScope>(), It.Is<NotificationChannel>(c => c.Id == "push"), It.IsAny<NotificationSendResult>(), It.IsAny<ChannelSendKind>()), Times.Never);
|
||||
push.Verify(p => p.SendAsync(Tenant, It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
@@ -474,13 +474,13 @@ public class NotificationChannelHealthTests
|
||||
{
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var dispatcher = Dispatcher(recorder, webhookResult: () => throw new InvalidOperationException("boom"));
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
var mail = new NotificationChannel { Id = "mail", Name = "mail", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), mail }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"),
|
||||
It.Is<NotificationSendResult>(x => !x.Success && x.Message == "Unexpected error: boom"), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.IsAny<NotificationSendResult>(), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "mail"), It.IsAny<NotificationSendResult>(), ChannelSendKind.Delivery), Times.Once);
|
||||
}
|
||||
|
||||
// ── Read endpoint projection ──────────────────────────────────────────
|
||||
|
||||
@@ -36,6 +36,7 @@ public class NotificationChannelsTests
|
||||
"{\"name\":\"no-id\",\"providerType\":20,\"url\":\"https://x.example\"}",
|
||||
ChannelJson("none-provider", providerType: 0),
|
||||
ChannelJson("retired-teams-connector", providerType: 1),
|
||||
ChannelJson("retired-provider", providerType: 40),
|
||||
ChannelJson("unknown-provider", providerType: 99)) + "]";
|
||||
|
||||
var parsed = NotificationChannel.ParseList(json);
|
||||
@@ -153,6 +154,20 @@ public class NotificationChannelsTests
|
||||
Assert.Null(redacted[1].SigningSecret);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RedactedCopy_MasksTheRecipientsOfAnEmailChannel()
|
||||
{
|
||||
var cfg = new TenantConfiguration
|
||||
{
|
||||
NotificationChannelsJson = "[" + ChannelJson("mail", providerType: 60, url: "ops@example.invalid") + "]",
|
||||
};
|
||||
|
||||
var redacted = NotificationChannel.ParseList(cfg.RedactedCopyForReader().NotificationChannelsJson);
|
||||
|
||||
Assert.Equal("mail", Assert.Single(redacted).Id);
|
||||
Assert.Equal(Constants.RedactedSecretPlaceholder, redacted[0].Url);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void RedactedCopy_UnparseableChannelsJson_FallsBackToWholeStringRedaction()
|
||||
{
|
||||
@@ -218,6 +233,7 @@ public class NotificationChannelsTests
|
||||
[InlineData("[{\"name\":\"no-id\",\"providerType\":20}]", "needs an id")]
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":99}]", "invalid provider type")]
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":1}]", "invalid provider type")] // retired Teams Office 365 Connector
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":40}]", "invalid provider type")] // retired, never reused
|
||||
[InlineData("[{\"id\":\"a\",\"providerType\":20,\"url\":\"http://plain.example\"}]", "")] // non-https rejected by SsrfGuard format check
|
||||
public void ValidateNotificationChannels_RejectsInvalidEntries(string json, string expectedFragment)
|
||||
{
|
||||
@@ -238,6 +254,7 @@ public class NotificationChannelsTests
|
||||
|
||||
[Theory]
|
||||
[InlineData(1, false)] // retired Teams Office 365 Connector
|
||||
[InlineData(40, false)] // retired, never reused
|
||||
[InlineData(99, false)]
|
||||
[InlineData(0, true)]
|
||||
[InlineData(2, true)]
|
||||
@@ -257,8 +274,6 @@ public class NotificationChannelsTests
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(40, false)] // Telegram
|
||||
[InlineData(40, true)]
|
||||
[InlineData(50, false)] // Push
|
||||
[InlineData(50, true)]
|
||||
[InlineData(60, false)] // E-mail
|
||||
@@ -275,7 +290,6 @@ public class NotificationChannelsTests
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(40)]
|
||||
[InlineData(50)]
|
||||
[InlineData(60)]
|
||||
public void ValidateModel_LegacyWebhook_KeepsAnUnchangedStoredNonWebhookProviderSaveable(int providerType)
|
||||
|
||||
@@ -9,7 +9,7 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Ops-event payload delivery: the structured Details object reaches outbound channels, both as
|
||||
/// readable facts (card + Telegram formats) and verbatim under "data" (generic JSON consumers),
|
||||
/// readable facts (card + e-mail formats) and verbatim under "data" (generic JSON consumers),
|
||||
/// and only the channels whose rule opted in get it — on the platform scope, like every ops send.
|
||||
/// Before this, an alert carried only category/event/severity and a tenant GUID.
|
||||
/// </summary>
|
||||
@@ -117,10 +117,10 @@ public class OpsAlertPayloadTests
|
||||
=> Assert.Equal("1.0", JObject.Parse(new GenericJsonRenderer().RenderToJson(
|
||||
new NotificationAlert { Title = "t", Summary = "s", DataJson = """{"a":1}""" }))["schemaVersion"]!.Value<string>());
|
||||
|
||||
// ── Telegram text carries the same information ────────────────────────
|
||||
// ── E-mail text carries the same information ──────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TelegramRendering_IncludesFlattenedFacts()
|
||||
public void EmailRendering_IncludesFlattenedFacts()
|
||||
{
|
||||
var alert = new NotificationAlert
|
||||
{
|
||||
@@ -130,7 +130,7 @@ public class OpsAlertPayloadTests
|
||||
Facts = OpsAlertDispatchService.FlattenDetails("""{"domainName":"contoso.example"}"""),
|
||||
};
|
||||
|
||||
var text = TelegramNotificationService.RenderAlertText(alert);
|
||||
var text = EmailAlertRenderer.Text(alert);
|
||||
|
||||
Assert.Contains("Domain Name: contoso.example", text);
|
||||
Assert.Contains("Pro trial started", text);
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The ops events behind a customer submission are the only signal an alert channel gets. They
|
||||
/// name ids and the portal path, never the submitter or a comment; the submitter stays in the
|
||||
/// row's user id, which the dispatch never forwards.
|
||||
/// </summary>
|
||||
public class OpsEventSubmissionSignalsTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string Submitter = "someone@contoso.invalid";
|
||||
|
||||
private readonly List<OpsEventEntry> _events = new();
|
||||
private readonly OpsEventService _sut;
|
||||
|
||||
public OpsEventSubmissionSignalsTests()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(_events.Add)
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance, new MemoryCache(new MemoryCacheOptions()))
|
||||
{ CallBase = false };
|
||||
|
||||
_sut = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(adminConfig.Object));
|
||||
}
|
||||
|
||||
private OpsEventEntry AssertSingleSignal(string eventType, string portalPath)
|
||||
{
|
||||
var entry = Assert.Single(_events);
|
||||
Assert.Equal(eventType, entry.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, entry.Severity);
|
||||
Assert.Equal(TenantId, entry.TenantId);
|
||||
Assert.Contains(portalPath, entry.Details!);
|
||||
|
||||
Assert.Equal(Submitter, entry.UserId);
|
||||
Assert.DoesNotContain(Submitter, entry.Message);
|
||||
Assert.DoesNotContain(Submitter, entry.Details!);
|
||||
return entry;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TenantSignup_names_the_tenant_page()
|
||||
{
|
||||
await _sut.RecordTenantSignupAsync(TenantId, Submitter);
|
||||
|
||||
AssertSingleSignal(OpsEventTypes.TenantSignup, $"/admin/tenants/management?tenantId={TenantId}");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionReport_names_the_session_and_the_report_only()
|
||||
{
|
||||
await _sut.RecordSessionReportSubmittedAsync(TenantId, Submitter, "session-1", "report-1");
|
||||
|
||||
var entry = AssertSingleSignal(OpsEventTypes.SessionReportSubmitted, "/admin/reports/session-reports?reportId=report-1");
|
||||
Assert.Contains("session-1", entry.Details!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task DiagFilesReport_names_the_report_only()
|
||||
{
|
||||
await _sut.RecordDiagFilesReportSubmittedAsync(TenantId, Submitter, "report-2");
|
||||
|
||||
AssertSingleSignal(OpsEventTypes.DiagFilesReportSubmitted, "/admin/reports/session-reports?reportId=report-2");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RuleSubmission_names_the_submission_and_the_rule_only()
|
||||
{
|
||||
await _sut.RecordRuleSubmissionReceivedAsync(TenantId, Submitter, "abc123def456", "analyze", "ANALYZE-X-1");
|
||||
|
||||
var entry = AssertSingleSignal(OpsEventTypes.RuleSubmissionReceived, "/admin/reports/rule-submissions?submissionId=abc123def456");
|
||||
Assert.Contains("ANALYZE-X-1", entry.Details!);
|
||||
}
|
||||
}
|
||||
@@ -12,28 +12,12 @@ using Moq;
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Platform (ops) notification channels: legacy-slot synthesis so the pre-channels dispatch
|
||||
/// behavior survives the migration untouched, per-rule channel routing, the platform dispatch
|
||||
/// scope (an ops alert's tenant is its subject, never the channel owner), and redaction of the
|
||||
/// new secret-bearing field.
|
||||
/// Platform (ops) notification channels: the stored list, per-rule channel routing, the platform
|
||||
/// dispatch scope (an ops alert's tenant is its subject, never the channel owner), and redaction of
|
||||
/// the secret-bearing field.
|
||||
/// </summary>
|
||||
public class OpsNotificationChannelsTests
|
||||
{
|
||||
private static AdminConfiguration LegacyConfig(
|
||||
bool telegramEnabled = true, string telegramChat = "-1003785642894",
|
||||
bool teamsEnabled = true, string teamsUrl = "https://teams.example/hook",
|
||||
bool slackEnabled = false, string slackUrl = "")
|
||||
=> new()
|
||||
{
|
||||
UpdatedBy = "test",
|
||||
OpsAlertTelegramEnabled = telegramEnabled,
|
||||
OpsAlertTelegramChatId = telegramChat,
|
||||
OpsAlertTeamsEnabled = teamsEnabled,
|
||||
OpsAlertTeamsWebhookUrl = teamsUrl,
|
||||
OpsAlertSlackEnabled = slackEnabled,
|
||||
OpsAlertSlackWebhookUrl = slackUrl,
|
||||
};
|
||||
|
||||
private static NotificationChannel Channel(string id, bool enabled = true, int providerType = 20)
|
||||
=> new() { Id = id, Name = id, ProviderType = providerType, Url = "https://x.example/" + id, Enabled = enabled };
|
||||
|
||||
@@ -61,44 +45,20 @@ public class OpsNotificationChannelsTests
|
||||
private static IEnumerable<string> WithPayload(IReadOnlyList<OpsAlertRule> rules, IReadOnlyList<NotificationChannel> channels)
|
||||
=> OpsAlertDispatchService.ResolveTargets(rules, channels).WithPayload.Select(c => c.Id);
|
||||
|
||||
// ── Legacy synthesis ──────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_SynthesizesConfiguredLegacySlots()
|
||||
{
|
||||
var channels = LegacyConfig().GetOpsNotificationChannels();
|
||||
|
||||
Assert.Equal(2, channels.Count); // Slack slot is empty → not synthesized
|
||||
var telegram = channels[0];
|
||||
Assert.Equal(AdminConfiguration.LegacyTelegramChannelId, telegram.Id);
|
||||
Assert.Equal(AdminConfiguration.LegacyTelegramChannelName, telegram.Name);
|
||||
Assert.Equal((int)WebhookProviderType.Telegram, telegram.ProviderType);
|
||||
Assert.Equal("-1003785642894", telegram.Url);
|
||||
Assert.True(telegram.Enabled);
|
||||
|
||||
Assert.Equal(AdminConfiguration.LegacyTeamsChannelId, channels[1].Id);
|
||||
Assert.Equal((int)WebhookProviderType.TeamsWorkflowWebhook, channels[1].ProviderType);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_CarriesLegacyEnabledFlags()
|
||||
{
|
||||
// A configured-but-disabled provider must stay disabled, or the migration would start
|
||||
// sending to a destination the operator had deliberately switched off.
|
||||
var channels = LegacyConfig(telegramEnabled: false).GetOpsNotificationChannels();
|
||||
|
||||
Assert.False(channels.Single(c => c.Id == AdminConfiguration.LegacyTelegramChannelId).Enabled);
|
||||
}
|
||||
// ── Stored list ───────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_EmptyWhenNothingConfigured()
|
||||
=> Assert.Empty(new AdminConfiguration { UpdatedBy = "t" }.GetOpsNotificationChannels());
|
||||
|
||||
[Fact]
|
||||
public void GetOpsNotificationChannels_StoredListWinsOverLegacySlots()
|
||||
public void GetOpsNotificationChannels_ReturnsTheStoredList()
|
||||
{
|
||||
var config = LegacyConfig();
|
||||
config.OpsNotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("sales") });
|
||||
var config = new AdminConfiguration
|
||||
{
|
||||
UpdatedBy = "t",
|
||||
OpsNotificationChannelsJson = NotificationChannel.SerializeList(new[] { Channel("sales") }),
|
||||
};
|
||||
|
||||
var channels = config.GetOpsNotificationChannels();
|
||||
|
||||
|
||||
@@ -46,6 +46,25 @@ public class OrphanSweepOpsEventTests
|
||||
Assert.Contains("\"aggregateMetrics\":90000", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MaintenanceCompleted_carries_the_step_memory_and_the_gc_limits()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var memory = new MaintenanceMemoryReport(
|
||||
new Dictionary<string, long> { ["appVersionRadar"] = 1900 },
|
||||
new Dictionary<string, long> { ["appVersionRadar"] = 1300 },
|
||||
GcHeapLimitMb: 1740,
|
||||
GcHighLoadMb: 2088);
|
||||
|
||||
await service.RecordMaintenanceCompletedAsync(123456, "Timer", memory: memory);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Contains("\"stepsPeakMb\":{\"appVersionRadar\":1900}", evt.Details);
|
||||
Assert.Contains("\"stepsAllocMb\":{\"appVersionRadar\":1300}", evt.Details);
|
||||
Assert.Contains("\"gcHeapLimitMb\":1740", evt.Details);
|
||||
Assert.Contains("\"gcHighLoadMb\":2088", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
using System;
|
||||
using AutopilotMonitor.Functions.Telemetry;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The memory metric is a percentage of the limits the GC works against, so a reading keeps its
|
||||
/// meaning when the instance size changes, and an alert can name one number for every size.
|
||||
/// </summary>
|
||||
public class ProcessMemoryPressureTests
|
||||
{
|
||||
[Fact]
|
||||
public void Heap_is_committed_memory_against_the_heap_limit()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { CommittedBytes = 1305, TotalAvailableMemoryBytes = 1740 };
|
||||
|
||||
Assert.Equal(75.0, ProcessMemoryPressure.HeapPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Load_is_memory_load_against_the_high_load_threshold()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { MemoryLoadBytes = 2088, HighMemoryLoadThresholdBytes = 2088 };
|
||||
|
||||
Assert.Equal(100.0, ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_value_past_the_limit_is_not_capped()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { MemoryLoadBytes = 2320, HighMemoryLoadThresholdBytes = 2088 };
|
||||
|
||||
Assert.Equal(111.1, ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Before_the_first_gc_there_is_no_reading()
|
||||
{
|
||||
var snapshot = default(ProcessSnapshot) with { WorkingSetBytes = 500 };
|
||||
|
||||
Assert.Null(ProcessMemoryPressure.HeapPercent(snapshot));
|
||||
Assert.Null(ProcessMemoryPressure.LoadPercent(snapshot));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_captured_snapshot_after_a_gc_has_both_readings()
|
||||
{
|
||||
GC.Collect();
|
||||
|
||||
var snapshot = ProcessSnapshot.Capture();
|
||||
|
||||
Assert.True(ProcessMemoryPressure.HeapPercent(snapshot) > 0);
|
||||
Assert.True(ProcessMemoryPressure.LoadPercent(snapshot) > 0);
|
||||
}
|
||||
}
|
||||
@@ -20,6 +20,7 @@ public class ProcessStallClassifierTests
|
||||
PendingWorkItems: pending,
|
||||
CompletedWorkItems: completed,
|
||||
WorkingSetBytes: 0, HeapSizeBytes: 0, CommittedBytes: 0,
|
||||
TotalAvailableMemoryBytes: 0, MemoryLoadBytes: 0, HighMemoryLoadThresholdBytes: 0,
|
||||
LastGcGeneration: 0, LastGcPause: TimeSpan.Zero, LastGcCompacted: false);
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -17,7 +17,7 @@ namespace AutopilotMonitor.Functions.Tests.Push;
|
||||
/// <summary>
|
||||
/// Push as a channel provider: the dispatcher's Push branch (scope passed through, never the
|
||||
/// webhook path, one fan-out per send), the validation rules (no destination, open to tenant
|
||||
/// admins — only Telegram and E-mail stay behind the GA gate — and one Push channel per scope),
|
||||
/// admins — only E-mail stays behind the GA gate — and one Push channel per scope),
|
||||
/// the raw-table deny-list and the dependency redaction.
|
||||
/// </summary>
|
||||
public class PushChannelRoutingTests
|
||||
@@ -35,15 +35,6 @@ public class PushChannelRoutingTests
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel PushChannel(string id = "push", bool enabled = true) =>
|
||||
new() { Id = id, Name = id, ProviderType = (int)WebhookProviderType.Push, Url = null, Enabled = enabled };
|
||||
|
||||
@@ -56,9 +47,8 @@ public class PushChannelRoutingTests
|
||||
public async Task Push_channel_reaches_the_push_sender_with_the_callers_scope_and_never_the_webhook_path()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, push.Object);
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { PushChannel() }, alert, Tenant);
|
||||
@@ -66,7 +56,6 @@ public class PushChannelRoutingTests
|
||||
push.Verify(p => p.SendAsync(Tenant, alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -75,7 +64,7 @@ public class PushChannelRoutingTests
|
||||
// A pair stored before validation allowed only one: each would reach every paired device.
|
||||
var webhook = WebhookMock();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, TelegramMock().Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, push.Object);
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
var slack = new NotificationChannel { Id = "slack", Name = "slack", ProviderType = (int)WebhookProviderType.Slack, Url = "https://hooks.slack.example/x", Enabled = true };
|
||||
|
||||
@@ -94,7 +83,7 @@ public class PushChannelRoutingTests
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
push.Setup(p => p.SendWithResultAsync(It.IsAny<NotificationScope>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, push.Object);
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, push.Object);
|
||||
|
||||
var result = await dispatcher.SendTestAsync(PushChannel(), new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
@@ -109,7 +98,6 @@ public class PushChannelRoutingTests
|
||||
Assert.False(new NotificationChannel { Id = "s", ProviderType = (int)WebhookProviderType.Slack, Url = "" }.HasDestination());
|
||||
Assert.True(new NotificationChannel { Id = "s", ProviderType = (int)WebhookProviderType.Slack, Url = "https://hooks.slack.example/x" }.HasDestination());
|
||||
Assert.False(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Push));
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Telegram));
|
||||
Assert.True(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Email));
|
||||
Assert.False(NotificationChannel.IsGlobalAdminOnlyProvider((int)WebhookProviderType.Slack));
|
||||
}
|
||||
@@ -127,23 +115,23 @@ public class PushChannelRoutingTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Gate_leaves_a_push_channel_to_the_tenant_admin_but_not_its_change_into_telegram()
|
||||
public void Gate_leaves_a_push_channel_to_the_tenant_admin_but_not_its_change_into_email()
|
||||
{
|
||||
var json = Json(PushChannel());
|
||||
|
||||
// Add, disable, rename and toggle: all the tenant admin's own.
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(json, existingJson: null, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(PushChannel(enabled: false)), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(json, existingJson: null, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(PushChannel(enabled: false)), json, isGlobalAdmin: false));
|
||||
var renamed = PushChannel();
|
||||
renamed.Name = "Phones";
|
||||
renamed.NotifyOnFailure = true;
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(renamed), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(renamed), json, isGlobalAdmin: false));
|
||||
|
||||
// The same channel id turned into Telegram still needs a Global Administrator.
|
||||
var retyped = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
Assert.Equal("Telegram channels can only be configured by a Global Administrator.",
|
||||
TenantConfigValidation.ValidateTelegramChannelGate(Json(retyped), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(Json(retyped), json, isGlobalAdmin: true));
|
||||
// The same channel id turned into E-mail still needs a Global Administrator.
|
||||
var retyped = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Email, Url = "ops@example.invalid", Enabled = true };
|
||||
Assert.Equal("E-mail channels can only be configured by a Global Administrator.",
|
||||
TenantConfigValidation.ValidatePlatformProviderGate(Json(retyped), json, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidatePlatformProviderGate(Json(retyped), json, isGlobalAdmin: true));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -72,7 +72,7 @@ public class SessionDeletionWorkerTests
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()),
|
||||
Times.AtLeastOnce);
|
||||
// PR-B audit consolidation: deletion_poisoned audit moved to SessionDeletionPoisoned
|
||||
// OpsEvent (operator-bound, Telegram-routable). Tenant-audit row is no longer written.
|
||||
// OpsEvent (operator-bound, routable through alert rules). Tenant-audit row is no longer written.
|
||||
var poisoned = Assert.Single(harness.OpsEventCalls, e => e.EventType == "SessionDeletionPoisoned");
|
||||
Assert.Equal(TenantId, poisoned.TenantId);
|
||||
Assert.Equal(SessionId, poisoned.SessionId);
|
||||
|
||||
@@ -1013,7 +1013,7 @@ public class SlaTenantStatusLifecycleTests
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var channelDispatcher = new NotificationChannelDispatcher(webhook, new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance), Mock.Of<IPushChannelSender>());
|
||||
var channelDispatcher = new NotificationChannelDispatcher(webhook, Mock.Of<IPushChannelSender>());
|
||||
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
|
||||
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
|
||||
|
||||
@@ -66,7 +66,7 @@ public sealed class SubmitGeneralFeedbackFunctionTests
|
||||
[Fact]
|
||||
public async Task TheAlert_CarriesNeitherTextNorSenderNorContactAddress()
|
||||
{
|
||||
// Channels bound to FeedbackReceived (Telegram, Slack, webhooks) receive message + details;
|
||||
// Channels bound to FeedbackReceived (Slack, webhooks, e-mail) receive message + details;
|
||||
// the feedback itself is read in the portal. Personal data must never ride along.
|
||||
var h = new Harness();
|
||||
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ public sealed class SubmitOffboardingFeedbackFunctionTests
|
||||
Assert.Equal("Pricing was too high for our use case.", h.Repo.PersistedOffboardingFeedback.Comment);
|
||||
Assert.NotNull(h.Repo.PersistedOffboardingFeedback.InteractedAt);
|
||||
|
||||
// OpsEvent emitted with the canonical type catalog-registered for Telegram-routing.
|
||||
// OpsEvent emitted with the canonical type catalog-registered for alert routing.
|
||||
var ops = Assert.Single(h.OpsEvents);
|
||||
Assert.Equal("OffboardingFeedbackReceived", ops.EventType);
|
||||
Assert.Equal(TenantId, ops.TenantId);
|
||||
@@ -283,7 +283,7 @@ public sealed class SubmitOffboardingFeedbackFunctionTests
|
||||
|
||||
// Alert-dispatch dependencies are inert: empty AdminConfig service + bare HTTP
|
||||
// clients. The feedback path emits Info-severity so even if alerts ARE evaluated,
|
||||
// no Telegram/webhook fires for an unconfigured rule set.
|
||||
// no channel fires for an unconfigured rule set.
|
||||
var memCache = new MemoryCache(new MemoryCacheOptions());
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(),
|
||||
|
||||
@@ -45,7 +45,7 @@ public class TableLifecycleBucketTests
|
||||
[Constants.TableNames.GlobalAdmins] = "platform operator identities",
|
||||
[Constants.TableNames.McpUsers] = "platform MCP identities (PK='McpUsers', RK=UPN)",
|
||||
[Constants.TableNames.TenantGroupAssignments] = "per-UPN rows without tenant anchor; offboarding never touches it (see Constants doc)",
|
||||
[Constants.TableNames.PreviewConfig] = "global TelegramBot config row",
|
||||
[Constants.TableNames.PreviewConfig] = "global e-mail template overrides",
|
||||
[Constants.TableNames.Feedback] = "product feedback deliberately survives offboarding (TableFeedbackRepositoryTests contract)",
|
||||
[Constants.TableNames.BlockedVersions] = "platform-wide agent version blocks",
|
||||
[Constants.TableNames.GlobalNotifications] = "GA notifications; time-retention only",
|
||||
|
||||
@@ -1,236 +0,0 @@
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Telegram as a notification-channel provider: dispatch routing (it must never reach the
|
||||
/// SSRF-guarded webhook path), chat-ID validation, and the Global-Admin gate that keeps a
|
||||
/// tenant admin from sending through the platform-owned bot.
|
||||
/// </summary>
|
||||
public class TelegramChannelProviderTests
|
||||
{
|
||||
private static Mock<WebhookNotificationService> WebhookMock()
|
||||
{
|
||||
var mock = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
mock.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel Channel(int providerType, string? url, string id = "c1", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = providerType, Url = url, Enabled = enabled };
|
||||
|
||||
private static string ChannelJson(int providerType, string url, string id = "c1", bool enabled = true)
|
||||
=> $"[{{\"id\":\"{id}\",\"name\":\"{id}\",\"providerType\":{providerType},\"url\":\"{url}\",\"enabled\":{(enabled ? "true" : "false")}}}]";
|
||||
|
||||
// ── Dispatch routing ──────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_RoutesTelegramToBotAndNeverToWebhook()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Telegram, "-1003785642894") }, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync("-1003785642894", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_RoutesWebhookProvidersToWebhookService()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Slack, "https://hooks.slack.example/x") }, alert, NotificationScope.Platform);
|
||||
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://hooks.slack.example/x", WebhookProviderType.Slack, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_MixedList_EachChannelReachesItsOwnTransport()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
var alert = new NotificationAlert { Title = "t", Summary = "s" };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel((int)WebhookProviderType.Telegram, "-100123", "tg"),
|
||||
Channel((int)WebhookProviderType.GenericJson, "https://sales.example/hook", "sales"),
|
||||
}, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync("-100123", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://sales.example/hook", WebhookProviderType.GenericJson, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendToChannels_SkipsChannelsWithoutDestination()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[]
|
||||
{
|
||||
Channel((int)WebhookProviderType.Telegram, null, "tg"),
|
||||
Channel((int)WebhookProviderType.Slack, "", "slack"),
|
||||
}, new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendTest_RoutesTelegramToBot()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
var result = await dispatcher.SendTestAsync(
|
||||
Channel((int)WebhookProviderType.Telegram, "@salesdesk"),
|
||||
new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
Assert.True(result.Success);
|
||||
telegram.Verify(t => t.SendAlertAsync("@salesdesk", It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
|
||||
// ── Chat-ID validation ────────────────────────────────────────────────
|
||||
|
||||
[Theory]
|
||||
[InlineData("-1003785642894")]
|
||||
[InlineData("123456789")]
|
||||
[InlineData("@salesdesk")]
|
||||
[InlineData("@Sales_Desk_01")]
|
||||
public void ValidateTelegramChatId_AcceptsNumericAndUsernameForms(string chatId)
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChatId(chatId));
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("")]
|
||||
[InlineData(" ")]
|
||||
[InlineData("https://api.telegram.org/bot123/sendMessage")]
|
||||
[InlineData("-100abc")]
|
||||
[InlineData("@abc")] // too short
|
||||
[InlineData("@1sales")] // must start with a letter
|
||||
[InlineData("@sales-desk")] // hyphen not allowed
|
||||
[InlineData("123456789012345678901")] // 21 digits
|
||||
public void ValidateTelegramChatId_RejectsEverythingElse(string? chatId)
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateTelegramChatId(chatId));
|
||||
|
||||
[Fact]
|
||||
public void ValidateNotificationChannels_TelegramChatIdBypassesSsrfUrlGate()
|
||||
{
|
||||
// A chat ID is not a URL — the webhook SSRF gate would reject it. Proves the
|
||||
// provider-specific destination branch is wired, not just the format helper.
|
||||
Assert.Null(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-1003785642894")));
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Slack, "-1003785642894")));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ValidateNotificationChannels_RejectsTelegramChannelWithUrlDestination()
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateNotificationChannels(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "https://hooks.example/x")));
|
||||
|
||||
// ── Global-Admin gate ─────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_GlobalAdminMayAddChannel()
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-100123"), null, isGlobalAdmin: true));
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotAddChannel()
|
||||
=> Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.Telegram, "-100123"), null, isGlobalAdmin: false));
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotRetargetExistingChannel()
|
||||
{
|
||||
var existing = ChannelJson((int)WebhookProviderType.Telegram, "-100123");
|
||||
var retargeted = ChannelJson((int)WebhookProviderType.Telegram, "-100999");
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(retargeted, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMayNotFlipEnabledState()
|
||||
{
|
||||
var existing = ChannelJson((int)WebhookProviderType.Telegram, "-100123", enabled: true);
|
||||
var disabled = ChannelJson((int)WebhookProviderType.Telegram, "-100123", enabled: false);
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateTelegramChannelGate(disabled, existing, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_TenantAdminMaySaveWithUnchangedGaCreatedChannel()
|
||||
{
|
||||
// The gate must not turn a GA-created Telegram channel into a permanent save block
|
||||
// for the tenant admin's unrelated configuration changes.
|
||||
var json = ChannelJson((int)WebhookProviderType.Telegram, "-100123");
|
||||
|
||||
Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(json, json, isGlobalAdmin: false));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TelegramGate_IgnoresNonTelegramChannels()
|
||||
=> Assert.Null(TenantConfigValidation.ValidateTelegramChannelGate(
|
||||
ChannelJson((int)WebhookProviderType.GenericJson, "https://sales.example/hook"), null, isGlobalAdmin: false));
|
||||
|
||||
// ── Both write paths (PUT + field patch) go through ValidateModel ─────
|
||||
|
||||
[Fact]
|
||||
public void ValidateModel_EnforcesTelegramGateForNonGlobalAdmin()
|
||||
{
|
||||
var candidate = new TenantConfiguration
|
||||
{
|
||||
TenantId = "t1",
|
||||
NotificationChannelsJson = ChannelJson((int)WebhookProviderType.Telegram, "-100123"),
|
||||
};
|
||||
var existing = new TenantConfiguration { TenantId = "t1" };
|
||||
|
||||
Assert.NotNull(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: false));
|
||||
Assert.Null(TenantConfigValidation.ValidateModel(candidate, existing, isGlobalAdmin: true));
|
||||
}
|
||||
}
|
||||
@@ -310,6 +310,9 @@ public class TenantConfigPatchServiceTests
|
||||
[InlineData("domainName")]
|
||||
[InlineData("lastUpdated")]
|
||||
[InlineData("updatedBy")]
|
||||
[InlineData("graphAccessRejectedSince")]
|
||||
[InlineData("graphAccessRejectedCode")]
|
||||
[InlineData("graphAccessRejectionConfirmedAt")]
|
||||
public async Task Patch_DeniedField_Returns400WithFieldName(string field)
|
||||
{
|
||||
var harness = new Harness(Stored());
|
||||
|
||||
@@ -33,6 +33,7 @@ public class TenantConfigProjectionTests
|
||||
"onboardedAt", "onboardedBy", "dpaVersion", "dpaAcceptancePending", "dpaAcceptedBy", "dpaAcceptedAt",
|
||||
"lastUpdated", "dataRetentionDays",
|
||||
"homedAppClientId", "lastAuthClientId", "lastAuthClientIdSince",
|
||||
"graphAccessRejectedSince", "graphAccessRejectedCode", "graphAccessRejectionConfirmedAt",
|
||||
};
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -213,6 +213,40 @@ public class TenantConfigTableSerializationTests
|
||||
Assert.Null(mapped.LastAuthClientIdSince);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Roundtrip_GraphAccessFields_SurviveStoreAndMap()
|
||||
{
|
||||
var since = new DateTime(2026, 11, 2, 8, 0, 0, DateTimeKind.Utc);
|
||||
var config = new TenantConfiguration
|
||||
{
|
||||
TenantId = TenantId,
|
||||
DomainName = "contoso.com",
|
||||
UpdatedBy = "admin@contoso.com",
|
||||
GraphAccessRejectedSince = since,
|
||||
GraphAccessRejectedCode = "AADSTS7000229",
|
||||
GraphAccessRejectionConfirmedAt = since.AddMinutes(20),
|
||||
};
|
||||
|
||||
var mapped = TableConfigRepository.ConvertFromTenantTableEntity(
|
||||
TableConfigRepository.ConvertToTenantTableEntity(config));
|
||||
|
||||
Assert.Equal(since, mapped.GraphAccessRejectedSince);
|
||||
Assert.Equal("AADSTS7000229", mapped.GraphAccessRejectedCode);
|
||||
Assert.Equal(since.AddMinutes(20), mapped.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Map_LegacyRow_WithoutGraphAccessColumns_ReadsNull()
|
||||
{
|
||||
var entity = new TableEntity(TenantId, "config") { { "DomainName", "fabrikam.com" } };
|
||||
|
||||
var mapped = TableConfigRepository.ConvertFromTenantTableEntity(entity);
|
||||
|
||||
Assert.Null(mapped.GraphAccessRejectedSince);
|
||||
Assert.Null(mapped.GraphAccessRejectedCode);
|
||||
Assert.Null(mapped.GraphAccessRejectionConfirmedAt);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData(0)]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
using System.Net.Http;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
@@ -10,11 +9,10 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Inert notification plumbing for tests that need an <see cref="OpsEventService"/> but do not
|
||||
/// assert on anything being sent. Nothing leaves the process: the webhook transport gets a bare
|
||||
/// HttpClient it never uses (no channel is configured in these stubs), and the Telegram transport
|
||||
/// resolves its bot URL from a mocked repository that returns nothing.
|
||||
/// HttpClient it never uses (no channel is configured in these stubs).
|
||||
/// <para>
|
||||
/// Tests that DO assert on dispatch should mock the transports directly — see
|
||||
/// <c>TelegramChannelProviderTests</c> — rather than extending this helper.
|
||||
/// <c>EmailChannelProviderTests</c> — rather than extending this helper.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class TestNotifications
|
||||
@@ -22,8 +20,6 @@ internal static class TestNotifications
|
||||
internal static NotificationChannelDispatcher InertDispatcher()
|
||||
=> new(
|
||||
new WebhookNotificationService(new HttpClient()),
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
Mock.Of<IPushChannelSender>());
|
||||
|
||||
internal static OpsAlertDispatchService InertOpsAlertDispatch(AdminConfigurationService adminConfigService)
|
||||
|
||||
@@ -376,6 +376,8 @@ public class TypedContractCompletionWireParityTests
|
||||
bootstrapTokenEnabled = true,
|
||||
diagnosticsUploadConfigured = false,
|
||||
deviceValidationEnabled = true,
|
||||
graphAccessRejected = true,
|
||||
graphAccessAppMissing = false,
|
||||
appHomingFunnelActive = false,
|
||||
showScriptOutput = true,
|
||||
enableSoftwareInventoryAnalyzer = false,
|
||||
@@ -402,6 +404,8 @@ public class TypedContractCompletionWireParityTests
|
||||
BootstrapTokenEnabled = true,
|
||||
DiagnosticsUploadConfigured = false,
|
||||
DeviceValidationEnabled = true,
|
||||
GraphAccessRejected = true,
|
||||
GraphAccessAppMissing = false,
|
||||
AppHomingFunnelActive = false,
|
||||
ShowScriptOutput = true,
|
||||
EnableSoftwareInventoryAnalyzer = false,
|
||||
|
||||
@@ -88,8 +88,7 @@ public class WhatsNewNotificationServiceTests
|
||||
public Harness()
|
||||
{
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var telegram = new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance);
|
||||
Dispatcher = new Mock<NotificationChannelDispatcher>(webhook, telegram, Mock.Of<IPushChannelSender>());
|
||||
Dispatcher = new Mock<NotificationChannelDispatcher>(webhook, Mock.Of<IPushChannelSender>());
|
||||
Dispatcher
|
||||
.Setup(d => d.SendToChannelsAsync(It.IsAny<IEnumerable<NotificationChannel>>(), It.IsAny<NotificationAlert>(), It.IsAny<NotificationScope>()))
|
||||
.Returns<IEnumerable<NotificationChannel>, NotificationAlert, NotificationScope>((channels, alert, scope) =>
|
||||
@@ -434,14 +433,14 @@ public class WhatsNewNotificationServiceTests
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildWhatsNewAlert_TelegramText_CarriesSectionsAndLinks()
|
||||
public void BuildWhatsNewAlert_EmailText_CarriesSectionsAndLinks()
|
||||
{
|
||||
var alert = NotificationAlertBuilder.BuildWhatsNewAlert(
|
||||
new List<WhatsNewFeedEntry> { Entry("platform", "p", title: "**Lead**", body: "Body.") },
|
||||
null,
|
||||
WhatsNewNotificationService.PortalWhatsNewUrl("platform"));
|
||||
|
||||
var text = TelegramNotificationService.RenderAlertText(alert);
|
||||
var text = EmailAlertRenderer.Text(alert);
|
||||
|
||||
Assert.Contains("What's new in Autopilot Monitor", text);
|
||||
Assert.Contains("Platform \u00b7 Sep 8, 2026 \u2014 Lead", text);
|
||||
|
||||
+10
-70
@@ -32,12 +32,14 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
|
||||
/// <summary>
|
||||
/// Changes to ONLY these properties do not snapshot the tenant-config row: the
|
||||
/// LastAuthClientId pair flips as an auth-flow side effect and would flood the
|
||||
/// two backup slots with states nobody ever wants to revert to.
|
||||
/// LastAuthClientId pair flips as an auth-flow side effect and the Graph access state as
|
||||
/// a token side effect; both would flood the two backup slots with states nobody ever
|
||||
/// wants to revert to.
|
||||
/// </summary>
|
||||
private static readonly HashSet<string> TenantBackupNoiseProperties = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"LastUpdated", "UpdatedBy", "LastAuthClientId", "LastAuthClientIdSince",
|
||||
"GraphAccessRejectedSince", "GraphAccessRejectedCode", "GraphAccessRejectionConfirmedAt",
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> AdminBackupNoiseProperties = new(StringComparer.OrdinalIgnoreCase)
|
||||
@@ -541,62 +543,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
}
|
||||
}
|
||||
|
||||
// --- Preview Config ---
|
||||
|
||||
public async Task<Dictionary<string, string>> GetPreviewConfigAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var config = new Dictionary<string, string>();
|
||||
var entity = await _previewConfigTableClient.GetEntityAsync<TableEntity>("TelegramBot", "config");
|
||||
|
||||
foreach (var kvp in entity.Value)
|
||||
{
|
||||
if (kvp.Key == "odata.etag" || kvp.Key == "PartitionKey" || kvp.Key == "RowKey" || kvp.Key == "Timestamp")
|
||||
continue;
|
||||
config[kvp.Key] = kvp.Value?.ToString() ?? string.Empty;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
return new Dictionary<string, string>();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error loading preview config");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<bool> SavePreviewConfigAsync(string key, string value)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Get existing entity or create new one
|
||||
TableEntity entity;
|
||||
try
|
||||
{
|
||||
var existing = await _previewConfigTableClient.GetEntityAsync<TableEntity>("TelegramBot", "config");
|
||||
entity = existing.Value;
|
||||
}
|
||||
catch (RequestFailedException ex) when (ex.Status == 404)
|
||||
{
|
||||
entity = new TableEntity("TelegramBot", "config");
|
||||
}
|
||||
|
||||
entity[key] = value;
|
||||
await _previewConfigTableClient.UpsertEntityAsync(entity);
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error saving preview config key {Key}", key);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Email Template Overrides ---
|
||||
|
||||
private const string EmailTemplatesPartition = "EmailTemplates";
|
||||
@@ -871,6 +817,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "HomedAppClientId", config.HomedAppClientId },
|
||||
{ "LastAuthClientId", config.LastAuthClientId },
|
||||
{ "LastAuthClientIdSince", config.LastAuthClientIdSince },
|
||||
{ "GraphAccessRejectedSince", config.GraphAccessRejectedSince },
|
||||
{ "GraphAccessRejectedCode", config.GraphAccessRejectedCode },
|
||||
{ "GraphAccessRejectionConfirmedAt", config.GraphAccessRejectionConfirmedAt },
|
||||
{ "PlanTier", config.PlanTier },
|
||||
{ "TrialExpiresUtc", config.TrialExpiresUtc },
|
||||
{ "TrialStartedUtc", config.TrialStartedUtc },
|
||||
@@ -994,6 +943,9 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
HomedAppClientId = entity.GetString("HomedAppClientId"),
|
||||
LastAuthClientId = entity.GetString("LastAuthClientId"),
|
||||
LastAuthClientIdSince = entity.GetDateTime("LastAuthClientIdSince"),
|
||||
GraphAccessRejectedSince = entity.GetDateTime("GraphAccessRejectedSince"),
|
||||
GraphAccessRejectedCode = entity.GetString("GraphAccessRejectedCode"),
|
||||
GraphAccessRejectionConfirmedAt = entity.GetDateTime("GraphAccessRejectionConfirmedAt"),
|
||||
PlanTier = entity.GetString("PlanTier") ?? "free",
|
||||
TrialExpiresUtc = entity.GetDateTime("TrialExpiresUtc"),
|
||||
TrialStartedUtc = entity.GetDateTime("TrialStartedUtc"),
|
||||
@@ -1054,12 +1006,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
{ "SlaNotificationCooldownHours", config.SlaNotificationCooldownHours },
|
||||
// Ops Alert settings
|
||||
{ "OpsAlertRulesJson", config.OpsAlertRulesJson ?? string.Empty },
|
||||
{ "OpsAlertTelegramEnabled", config.OpsAlertTelegramEnabled },
|
||||
{ "OpsAlertTelegramChatId", config.OpsAlertTelegramChatId ?? string.Empty },
|
||||
{ "OpsAlertTeamsEnabled", config.OpsAlertTeamsEnabled },
|
||||
{ "OpsAlertTeamsWebhookUrl", config.OpsAlertTeamsWebhookUrl ?? string.Empty },
|
||||
{ "OpsAlertSlackEnabled", config.OpsAlertSlackEnabled },
|
||||
{ "OpsAlertSlackWebhookUrl", config.OpsAlertSlackWebhookUrl ?? string.Empty },
|
||||
{ "OpsNotificationChannelsJson", config.OpsNotificationChannelsJson ?? string.Empty },
|
||||
// Per-line agent binary integrity: written by the release pipeline via Merge.
|
||||
// The backend never changes them (the PATCH gate rejects them), so the
|
||||
@@ -1130,12 +1076,6 @@ namespace AutopilotMonitor.Functions.DataAccess.TableStorage
|
||||
SlaNotificationCooldownHours = entity.GetInt32("SlaNotificationCooldownHours") ?? 24,
|
||||
// Ops Alert settings
|
||||
OpsAlertRulesJson = entity.GetString("OpsAlertRulesJson"),
|
||||
OpsAlertTelegramEnabled = entity.GetBoolean("OpsAlertTelegramEnabled") ?? false,
|
||||
OpsAlertTelegramChatId = entity.GetString("OpsAlertTelegramChatId"),
|
||||
OpsAlertTeamsEnabled = entity.GetBoolean("OpsAlertTeamsEnabled") ?? false,
|
||||
OpsAlertTeamsWebhookUrl = entity.GetString("OpsAlertTeamsWebhookUrl"),
|
||||
OpsAlertSlackEnabled = entity.GetBoolean("OpsAlertSlackEnabled") ?? false,
|
||||
OpsAlertSlackWebhookUrl = entity.GetString("OpsAlertSlackWebhookUrl"),
|
||||
OpsNotificationChannelsJson = entity.GetString("OpsNotificationChannelsJson"),
|
||||
// Per-line agent binary integrity. V2 is the only wired line (V1 retired).
|
||||
AllowAgentDowngrade = entity.GetBoolean("AllowAgentDowngrade") ?? false,
|
||||
|
||||
+3
-1
@@ -302,13 +302,15 @@ public class AutopilotDeviceValidationConsentFunction
|
||||
/// <summary>
|
||||
/// Known consent triggers — frontend flows that initiate the AAD admin-consent dialog:
|
||||
/// one per serial-based validation (Autopilot, corporate identifier, and
|
||||
/// "device-preparation" for Device Association Validation).
|
||||
/// "device-preparation" for Device Association Validation), plus "restore" for repairing the
|
||||
/// Graph access of validations that are already on.
|
||||
/// </summary>
|
||||
internal static readonly HashSet<string> KnownConsentTriggers = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"autopilot",
|
||||
"corporate",
|
||||
"device-preparation",
|
||||
"restore",
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -14,17 +14,20 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
private readonly MaintenanceService _maintenanceService;
|
||||
private readonly Services.Push.PushMaintenanceService _pushMaintenance;
|
||||
private readonly Services.Notifications.NotificationChannelHealthMaintenance _channelHealthMaintenance;
|
||||
private readonly GraphAccessRecheckMaintenance _graphAccessRecheck;
|
||||
private readonly ILogger<MaintenanceFunction> _logger;
|
||||
|
||||
public MaintenanceFunction(
|
||||
MaintenanceService maintenanceService,
|
||||
Services.Push.PushMaintenanceService pushMaintenance,
|
||||
Services.Notifications.NotificationChannelHealthMaintenance channelHealthMaintenance,
|
||||
GraphAccessRecheckMaintenance graphAccessRecheck,
|
||||
ILogger<MaintenanceFunction> logger)
|
||||
{
|
||||
_maintenanceService = maintenanceService;
|
||||
_pushMaintenance = pushMaintenance;
|
||||
_channelHealthMaintenance = channelHealthMaintenance;
|
||||
_graphAccessRecheck = graphAccessRecheck;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
@@ -45,6 +48,8 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
await _pushMaintenance.RunAsync();
|
||||
// Health rows of deleted notification channels (D-342); fail-soft inside.
|
||||
await _channelHealthMaintenance.RunAsync();
|
||||
// Confirms or clears Graph access suspicions without waiting for device traffic; fail-soft inside.
|
||||
await _graphAccessRecheck.RunAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace AutopilotMonitor.Functions.Functions.Admin
|
||||
/// Timer-triggered watcher for the Azure SignalR plan limits (Standard S1
|
||||
/// x1 unit: 1,000 concurrent connections / 1M messages per UTC day). Runs
|
||||
/// every full hour and emits OpsEvents when the configured thresholds are
|
||||
/// crossed so operators get a Telegram ping before clients start getting
|
||||
/// crossed so operators get an alert before clients start getting
|
||||
/// 429'd or message overage is billed.
|
||||
///
|
||||
/// Separate from the 2h MaintenanceFunction: the SignalR limits can be
|
||||
|
||||
+1
-2
@@ -178,8 +178,7 @@ public class SubmitOffboardingFeedbackFunction
|
||||
}
|
||||
|
||||
// Best-effort OpsEvent so operators see "feedback received" on the Ops dashboard.
|
||||
// Telegram-wiring done by the admin via OpsAlertRules UI (memory:
|
||||
// feedback_ops_event_types_dual_register).
|
||||
// Alert channels are bound by the admin in the Ops Alert Rules UI.
|
||||
try
|
||||
{
|
||||
await _opsEvents.RecordOffboardingFeedbackReceivedAsync(
|
||||
|
||||
+2
-19
@@ -3,7 +3,6 @@ using System.Net;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Azure.Functions.Worker;
|
||||
using Microsoft.Azure.Functions.Worker.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -37,24 +36,8 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
|
||||
var config = await _adminConfigService.GetConfigurationAsync();
|
||||
|
||||
// Ops channels: a tenant that predates named channels stores an empty list, and the
|
||||
// dispatcher synthesizes one channel per configured legacy provider slot. Project the
|
||||
// SAME synthesis into the read model so the editor shows what actually receives
|
||||
// alerts today — the alternative is a second copy of the synthesis rules in the web
|
||||
// app, drifting from the one the dispatcher uses. The first save then persists
|
||||
// exactly the list the operator was shown. Never mutates the cached instance.
|
||||
if (string.IsNullOrWhiteSpace(config.OpsNotificationChannelsJson))
|
||||
{
|
||||
var synthesized = config.GetOpsNotificationChannels();
|
||||
if (synthesized.Count > 0)
|
||||
{
|
||||
config = config.ShallowCopy();
|
||||
config.OpsNotificationChannelsJson = NotificationChannel.SerializeList(synthesized);
|
||||
}
|
||||
}
|
||||
|
||||
// A read-only GlobalReader gets secrets (NvdApiKey, SAS, ops webhook URLs and the
|
||||
// per-channel destinations) redacted; a Global Admin gets the full config. Redaction
|
||||
// A read-only GlobalReader gets secrets (NvdApiKey, SAS and the per-channel
|
||||
// destinations) redacted; a Global Admin gets the full config. Redaction
|
||||
// returns a copy — the cached instance is never mutated.
|
||||
if (requestCtx.IsGlobalReader)
|
||||
config = config.RedactedCopyForReader();
|
||||
|
||||
+6
@@ -74,6 +74,7 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
var resolution = FeatureEntitlementCatalog.Resolve(config, nowUtc);
|
||||
var entitlements = FeatureEntitlementCatalog.Get(resolution);
|
||||
var isTrial = resolution.IsTrial;
|
||||
var graphAccessRejected = config.HasAnyDeviceValidation() && config.GraphAccessRejectionConfirmedAt != null;
|
||||
|
||||
return new TenantFeatureFlagsResponse
|
||||
{
|
||||
@@ -87,6 +88,11 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
DiagnosticsUploadConfigured = DiagnosticsUploadConfigChange.IsConfigured(config),
|
||||
// Drives the "no device validation enabled" dashboard banner (useTenantSecurityConfig).
|
||||
DeviceValidationEnabled = config.HasAnyDeviceValidation(),
|
||||
// Drives the "enrollment monitoring is interrupted" notice. A suspicion alone shows
|
||||
// nothing; the error code itself stays in the admin-gated full config.
|
||||
GraphAccessRejected = graphAccessRejected,
|
||||
GraphAccessAppMissing = graphAccessRejected
|
||||
&& string.Equals(config.GraphAccessRejectedCode, EntraTokenRejection.ServicePrincipalMissingCode, StringComparison.OrdinalIgnoreCase),
|
||||
// Dual app-reg self-service migration: when true, running the consent flow (or
|
||||
// "Detect existing access") targets the NEW app registration and auto-flips this
|
||||
// tenant's homing after verification — drives the explanatory banner in the
|
||||
|
||||
+2
-2
@@ -207,12 +207,12 @@ namespace AutopilotMonitor.Functions.Functions.Config
|
||||
nameof(AdminConfiguration.OpsNotificationChannelsJson), StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
// A GlobalReader is served destinations as the ***REDACTED*** sentinel; persisting it as a
|
||||
// chat ID or webhook URL would silently break every alert, so restore per channel id.
|
||||
// recipient or webhook URL would silently break every alert, so restore per channel id.
|
||||
draft.OpsNotificationChannelsJson = NotificationChannel.RestoreRedactedList(
|
||||
draft.OpsNotificationChannelsJson, storedChannels);
|
||||
|
||||
// Same structural gate and one-Push-channel rule the tenant channel list passes. No GA
|
||||
// gate on Telegram here: this endpoint is GlobalAdminOnly by policy.
|
||||
// gate on platform providers here: this endpoint is GlobalAdminOnly by policy.
|
||||
var channelsError = TenantConfigValidation.ValidateNotificationChannels(draft.OpsNotificationChannelsJson)
|
||||
?? TenantConfigValidation.ValidateSinglePushChannel(draft.OpsNotificationChannelsJson, storedChannels);
|
||||
if (channelsError != null)
|
||||
|
||||
@@ -21,7 +21,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
private readonly TenantConfigurationService _tenantConfigService;
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly TenantAdminsService _tenantAdminsService;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly ISessionRepository _sessionRepo;
|
||||
private readonly IFeedbackRepository _feedbackRepo;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
@@ -32,7 +31,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
TenantConfigurationService tenantConfigService,
|
||||
AdminConfigurationService adminConfigService,
|
||||
TenantAdminsService tenantAdminsService,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
ISessionRepository sessionRepo,
|
||||
IFeedbackRepository feedbackRepo,
|
||||
OpsEventService opsEvents,
|
||||
@@ -42,7 +40,6 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_adminConfigService = adminConfigService;
|
||||
_tenantAdminsService = tenantAdminsService;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_sessionRepo = sessionRepo;
|
||||
_feedbackRepo = feedbackRepo;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -185,7 +182,7 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
|
||||
// Only a rating that was actually recorded is announced — at most once per person.
|
||||
if (stored is { Submitted: true } && stored.Rating is int rating)
|
||||
await AnnounceRatingAsync(tenantId, upn, displayName, stored.FeedbackId!, rating, comment);
|
||||
await AnnounceRatingAsync(tenantId, upn, stored.FeedbackId!, rating, comment);
|
||||
|
||||
_logger.LogInformation("Feedback {Action} by {Upn} (tenant {TenantId})",
|
||||
stored == null ? "unchanged" : body.Dismissed ? "dismissed" : "submitted", upn, tenantId);
|
||||
@@ -196,13 +193,11 @@ namespace AutopilotMonitor.Functions.Functions.Feedback
|
||||
/// <summary>
|
||||
/// The signals of a recorded rating, all best effort: a stored rating never fails because of
|
||||
/// them. The ops event carries the stars and the comment length only, so an alert channel
|
||||
/// receives no personal data; the Telegram ping keeps sender and comment until it is retired.
|
||||
/// receives no personal data.
|
||||
/// </summary>
|
||||
private async Task AnnounceRatingAsync(
|
||||
string tenantId, string upn, string displayName, string feedbackId, int rating, string? comment)
|
||||
string tenantId, string upn, string feedbackId, int rating, string? comment)
|
||||
{
|
||||
_ = _telegramNotificationService.SendFeedbackAsync(tenantId, upn, displayName, rating, comment);
|
||||
|
||||
var normalizedTenantId = tenantId.ToLowerInvariant();
|
||||
// The domain makes the alert and the bell readable. The lookup never creates a
|
||||
// configuration row; an unknown tenant simply keeps its GUID label.
|
||||
|
||||
@@ -31,7 +31,6 @@ public class AuthFunction
|
||||
private readonly TenantAdminsService _tenantAdminsService;
|
||||
private readonly IMetricsRepository _metricsRepo;
|
||||
private readonly PreviewWhitelistService _previewWhitelistService;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly McpUserService _mcpUserService;
|
||||
private readonly Services.Activation.ITenantAutoApproveEnqueuer _tenantAutoApproveEnqueuer;
|
||||
@@ -41,7 +40,7 @@ public class AuthFunction
|
||||
private readonly AdminConfigurationService _adminConfigService;
|
||||
private readonly IOffboardingAuditRepository _offboardingRepo;
|
||||
private readonly Services.Push.PushOwnerSignInService? _pushSignIn;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public AuthFunction(
|
||||
ILogger<AuthFunction> logger,
|
||||
@@ -51,7 +50,6 @@ public class AuthFunction
|
||||
TenantAdminsService tenantAdminsService,
|
||||
IMetricsRepository metricsRepo,
|
||||
PreviewWhitelistService previewWhitelistService,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
McpUserService mcpUserService,
|
||||
Services.Activation.ITenantAutoApproveEnqueuer tenantAutoApproveEnqueuer,
|
||||
@@ -60,8 +58,8 @@ public class AuthFunction
|
||||
LiveStreamRevoker streams,
|
||||
AdminConfigurationService adminConfigService,
|
||||
IOffboardingAuditRepository offboardingRepo,
|
||||
Services.Push.PushOwnerSignInService? pushSignIn = null,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents,
|
||||
Services.Push.PushOwnerSignInService? pushSignIn = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_pushSignIn = pushSignIn;
|
||||
@@ -76,7 +74,6 @@ public class AuthFunction
|
||||
_tenantAdminsService = tenantAdminsService;
|
||||
_metricsRepo = metricsRepo;
|
||||
_previewWhitelistService = previewWhitelistService;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_mcpUserService = mcpUserService;
|
||||
_tenantAutoApproveEnqueuer = tenantAutoApproveEnqueuer;
|
||||
@@ -444,7 +441,7 @@ public class AuthFunction
|
||||
|
||||
/// <summary>
|
||||
/// If the tenant has no domain name yet, extracts it from the UPN, persists it,
|
||||
/// and fires best-effort notifications (Telegram + global notification). The write lands
|
||||
/// and fires the signup signals (ops event + global notification). The write lands
|
||||
/// only while the STORED row still has no domain (conditional create or update), and only
|
||||
/// the login that wrote it fires the signup side effects.
|
||||
/// </summary>
|
||||
@@ -559,15 +556,8 @@ public class AuthFunction
|
||||
Seed(tenantConfig);
|
||||
|
||||
// The ops event is the routable record (alert rules → any channel, Push included); it is
|
||||
// written before the response and never throws. The Telegram ping below stays until retired.
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordTenantSignupAsync(tenantId, upn);
|
||||
|
||||
// Fire-and-forget: Telegram
|
||||
_ = _telegramNotificationService.SendNewTenantSignupAsync(tenantId, upn)
|
||||
.ContinueWith(t => _logger.LogWarning(t.Exception?.InnerException,
|
||||
"Fire-and-forget Telegram notification failed for tenant {TenantId}", tenantId),
|
||||
TaskContinuationOptions.OnlyOnFaulted);
|
||||
// written before the response and never throws.
|
||||
await _opsEvents.RecordTenantSignupAsync(tenantId, upn);
|
||||
|
||||
// Fire-and-forget: Global notification
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@ namespace AutopilotMonitor.Functions.Functions.Infrastructure
|
||||
/// <summary>
|
||||
/// Receives Azure Monitor alert notifications (Common Alert Schema) from an Action Group
|
||||
/// webhook receiver and re-emits them as ops events, so platform alerts flow through the
|
||||
/// same OpsEventService → OpsAlertDispatchService fan-out (Telegram/Teams/Slack providers +
|
||||
/// same OpsEventService → OpsAlertDispatchService fan-out (the notification channels +
|
||||
/// the portal ops-events list) as every internally raised operational event.
|
||||
///
|
||||
/// The delivery path is deliberately independent of the portal: Azure Monitor → this
|
||||
|
||||
+2
-2
@@ -29,12 +29,12 @@ namespace AutopilotMonitor.Functions.Functions.Maintenance
|
||||
/// Independent cadence from the 2h <c>Maintenance</c> function so cascade-lifecycle
|
||||
/// work cannot starve rule-stats aggregation / cert-expiry checks, and operators get a
|
||||
/// dedicated OpsEvent stream (<see cref="OpsEventService.RecordSessionDeletionMaintenanceLongRunningAsync"/>
|
||||
/// + siblings) wired to Telegram via the standard alert-rules UI.
|
||||
/// + siblings) wired to channels via the standard alert-rules UI.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// 30-minute watchdog emits a Warning OpsEvent; 60-minute watchdog escalates to Error. The
|
||||
/// 60-minute mark matches the Flex Consumption hard <c>functionTimeout</c> (host.json) so the
|
||||
/// operator gets a Telegram alert before the Azure runtime abort. The retention fanout stops
|
||||
/// operator gets an alert before the Azure runtime abort. The retention fanout stops
|
||||
/// itself at the <see cref="RunBudget"/> (50min) deadline so a growing backlog degrades into
|
||||
/// a clean <c>BudgetExceeded</c> + partial sweep instead of a host abort. Cron
|
||||
/// <c>0 0 */12 * * *</c> (every 12h at 00:00 / 12:00 UTC) — const per repo convention;
|
||||
|
||||
+3
-11
@@ -22,24 +22,21 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
private readonly ILogger<SubmitDiagFilesReportFunction> _logger;
|
||||
private readonly SessionReportService _sessionReportService;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly ReportUploadGate _uploadGate;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitDiagFilesReportFunction(
|
||||
ILogger<SubmitDiagFilesReportFunction> logger,
|
||||
SessionReportService sessionReportService,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
ReportUploadGate uploadGate,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_sessionReportService = sessionReportService;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_uploadGate = uploadGate;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -122,12 +119,7 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
}
|
||||
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordDiagFilesReportSubmittedAsync(request.TenantId, userIdentifier, metadata.ReportId);
|
||||
|
||||
// Telegram notification — best effort
|
||||
_ = _telegramNotificationService.SendDiagFilesReportAsync(
|
||||
request.TenantId, userIdentifier, metadata.ReportId, request.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordDiagFilesReportSubmittedAsync(request.TenantId, userIdentifier, metadata.ReportId);
|
||||
|
||||
// Persistent in-app notification for Global Admins — best effort
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+3
-11
@@ -16,24 +16,21 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
private readonly ILogger<SubmitSessionReportFunction> _logger;
|
||||
private readonly SessionReportService _sessionReportService;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegramNotificationService;
|
||||
private readonly GlobalNotificationService _globalNotificationService;
|
||||
private readonly ReportUploadGate _uploadGate;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitSessionReportFunction(
|
||||
ILogger<SubmitSessionReportFunction> logger,
|
||||
SessionReportService sessionReportService,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegramNotificationService,
|
||||
GlobalNotificationService globalNotificationService,
|
||||
ReportUploadGate uploadGate,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_sessionReportService = sessionReportService;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegramNotificationService = telegramNotificationService;
|
||||
_globalNotificationService = globalNotificationService;
|
||||
_uploadGate = uploadGate;
|
||||
_opsEvents = opsEvents;
|
||||
@@ -131,12 +128,7 @@ namespace AutopilotMonitor.Functions.Functions.Reports
|
||||
}
|
||||
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordSessionReportSubmittedAsync(request.TenantId, userIdentifier, sessionId, metadata.ReportId);
|
||||
|
||||
// Telegram notification — best effort
|
||||
_ = _telegramNotificationService.SendSessionReportAsync(
|
||||
request.TenantId, userIdentifier, sessionId, metadata.ReportId, request.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordSessionReportSubmittedAsync(request.TenantId, userIdentifier, sessionId, metadata.ReportId);
|
||||
|
||||
// Persistent in-app notification for Global Admins — best effort
|
||||
_ = _globalNotificationService.CreateNotificationAsync(
|
||||
|
||||
+3
-8
@@ -18,22 +18,19 @@ namespace AutopilotMonitor.Functions.Functions.Rules.Submissions
|
||||
private readonly ILogger<SubmitRuleSubmissionsFunction> _logger;
|
||||
private readonly RuleSubmissionService _service;
|
||||
private readonly IMaintenanceRepository _maintenanceRepo;
|
||||
private readonly TelegramNotificationService _telegram;
|
||||
private readonly GlobalNotificationService _globalNotifications;
|
||||
private readonly OpsEventService? _opsEvents;
|
||||
private readonly OpsEventService _opsEvents;
|
||||
|
||||
public SubmitRuleSubmissionsFunction(
|
||||
ILogger<SubmitRuleSubmissionsFunction> logger,
|
||||
RuleSubmissionService service,
|
||||
IMaintenanceRepository maintenanceRepo,
|
||||
TelegramNotificationService telegram,
|
||||
GlobalNotificationService globalNotifications,
|
||||
OpsEventService? opsEvents = null)
|
||||
OpsEventService opsEvents)
|
||||
{
|
||||
_logger = logger;
|
||||
_service = service;
|
||||
_maintenanceRepo = maintenanceRepo;
|
||||
_telegram = telegram;
|
||||
_globalNotifications = globalNotifications;
|
||||
_opsEvents = opsEvents;
|
||||
}
|
||||
@@ -112,9 +109,7 @@ namespace AutopilotMonitor.Functions.Functions.Rules.Submissions
|
||||
foreach (var s in created)
|
||||
{
|
||||
// Ops event (routable through alert rules, Push included) — written before the response, never throws
|
||||
if (_opsEvents != null)
|
||||
await _opsEvents.RecordRuleSubmissionReceivedAsync(request.TenantId, userIdentifier, s.SubmissionId, s.RuleKind, s.SourceRuleId);
|
||||
_ = _telegram.SendRuleSubmissionAsync(s.SubmissionId, request.TenantId, userIdentifier, s.RuleKind, s.SourceRuleId, s.Title, s.Comment ?? string.Empty);
|
||||
await _opsEvents.RecordRuleSubmissionReceivedAsync(request.TenantId, userIdentifier, s.SubmissionId, s.RuleKind, s.SourceRuleId);
|
||||
_ = _globalNotifications.CreateNotificationAsync(
|
||||
"rule_submission",
|
||||
$"Rule submission {s.SubmissionId}",
|
||||
|
||||
@@ -29,7 +29,7 @@ public sealed class ResiliencePolicies
|
||||
public IAsyncPolicy<HttpResponseMessage> ExternalDataApi { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Policy for notification clients (webhook, Telegram):
|
||||
/// Policy for notification clients (webhook):
|
||||
/// 2 retries with 2s fixed delay. No circuit breaker — a single failing endpoint
|
||||
/// should not block all notifications across tenants.
|
||||
/// </summary>
|
||||
|
||||
@@ -52,6 +52,10 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
("homedAppClientId", c => c.HomedAppClientId),
|
||||
("lastAuthClientId", c => c.LastAuthClientId),
|
||||
("lastAuthClientIdSince", c => c.LastAuthClientIdSince),
|
||||
// Graph access state: two timestamps and a Microsoft error code, not secrets.
|
||||
("graphAccessRejectedSince", c => c.GraphAccessRejectedSince),
|
||||
("graphAccessRejectedCode", c => c.GraphAccessRejectedCode),
|
||||
("graphAccessRejectionConfirmedAt", c => c.GraphAccessRejectionConfirmedAt),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -187,8 +187,8 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
return "Invalid webhook provider type.";
|
||||
|
||||
// ...and only a webhook provider: its destination passed the webhook URL gate above, never
|
||||
// the channel gates of Telegram, Push or E-mail. Only a CHANGED value is checked, so a
|
||||
// stored one stays saveable.
|
||||
// the channel gates of Push or E-mail. Only a CHANGED value is checked, so a stored one
|
||||
// stays saveable.
|
||||
if (candidate.WebhookProviderType != existing.WebhookProviderType
|
||||
&& candidate.WebhookProviderType != 0
|
||||
&& !Shared.Models.Notifications.NotificationChannel.IsWebhookUrlProvider(candidate.WebhookProviderType))
|
||||
@@ -202,10 +202,10 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
if (channelsError != null)
|
||||
return $"Invalid notification channels: {channelsError}";
|
||||
|
||||
var telegramGateError = ValidateTelegramChannelGate(
|
||||
var platformProviderError = ValidatePlatformProviderGate(
|
||||
candidate.NotificationChannelsJson, existing.NotificationChannelsJson, isGlobalAdmin);
|
||||
if (telegramGateError != null)
|
||||
return telegramGateError;
|
||||
if (platformProviderError != null)
|
||||
return platformProviderError;
|
||||
|
||||
var pushCountError = ValidateSinglePushChannel(candidate.NotificationChannelsJson, existing.NotificationChannelsJson);
|
||||
if (pushCountError != null)
|
||||
@@ -365,15 +365,11 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
if (!Shared.Models.Notifications.NotificationChannel.IsSupportedProviderType(channel.ProviderType))
|
||||
return $"channel \"{label}\" has an invalid provider type.";
|
||||
|
||||
// Telegram's destination is a chat ID, not a URL — the SSRF gate does not apply
|
||||
// (there is no caller-supplied endpoint; the bot URL is platform-owned) and would
|
||||
// reject every valid value. Format-check the chat ID instead. A Push channel has
|
||||
// no destination at all: a URL on it is refused so it can never fall through to
|
||||
// the webhook path should a future branch forget the provider.
|
||||
// A Push channel has no destination at all: a URL on it is refused so it can never
|
||||
// fall through to the webhook path should a future branch forget the provider. An
|
||||
// e-mail channel names recipients, not a URL, so the SSRF gate does not apply.
|
||||
var destinationError =
|
||||
channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Telegram
|
||||
? ValidateTelegramChatId(channel.Url)
|
||||
: channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Push
|
||||
channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Push
|
||||
? ValidatePushDestination(channel.Url)
|
||||
: channel.ProviderType == (int)Shared.Models.Notifications.WebhookProviderType.Email
|
||||
? ValidateEmailDestination(channel.Url)
|
||||
@@ -393,45 +389,6 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a Telegram destination chat ID. Returns an error message, or null when valid.
|
||||
/// Accepts a numeric id (negative for groups/channels, e.g. -1003785642894) or an @username.
|
||||
/// Empty is rejected: unlike a webhook URL, a Telegram channel with no chat ID cannot be
|
||||
/// saved as "configured but idle" — it would silently drop every message.
|
||||
/// </summary>
|
||||
internal static string? ValidateTelegramChatId(string? chatId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(chatId))
|
||||
return "a Telegram chat ID is required.";
|
||||
|
||||
var trimmed = chatId!.Trim();
|
||||
|
||||
if (trimmed.StartsWith("@", StringComparison.Ordinal))
|
||||
{
|
||||
// Telegram public usernames: 5-32 chars, letters/digits/underscore, letter first.
|
||||
var name = trimmed.Substring(1);
|
||||
if (name.Length < 5 || name.Length > 32 || !char.IsLetter(name[0]))
|
||||
return "the @username must be 5-32 characters and start with a letter.";
|
||||
foreach (var ch in name)
|
||||
{
|
||||
if (!char.IsLetterOrDigit(ch) && ch != '_')
|
||||
return "the @username may only contain letters, digits and underscores.";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
var digits = trimmed.StartsWith("-", StringComparison.Ordinal) ? trimmed.Substring(1) : trimmed;
|
||||
if (digits.Length == 0 || digits.Length > 20)
|
||||
return "must be a numeric chat ID (up to 20 digits) or an @username.";
|
||||
foreach (var ch in digits)
|
||||
{
|
||||
if (!char.IsDigit(ch))
|
||||
return "must be a numeric chat ID (up to 20 digits) or an @username.";
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A Push channel carries no destination (its recipients are the scope's paired devices,
|
||||
/// resolved at send time); a URL on it is refused rather than ignored.
|
||||
@@ -467,9 +424,8 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
/// <summary>
|
||||
/// Global-Admin gate for the providers that send through PLATFORM-owned infrastructure
|
||||
/// (<see cref="Shared.Models.Notifications.NotificationChannel.IsGlobalAdminOnlyProvider"/>):
|
||||
/// Telegram through the platform bot (the token lives in PreviewConfig, not in tenant
|
||||
/// config), E-mail through the platform's sender identity and provider key. A tenant admin
|
||||
/// must not be able to create or retarget one — hiding the provider in the UI is not a control.
|
||||
/// E-mail through the platform's sender identity and provider key. A tenant admin must not be
|
||||
/// able to create or retarget one — hiding the provider in the UI is not a control.
|
||||
/// <para>
|
||||
/// Compares against the stored config so a non-GA caller can still save unrelated changes
|
||||
/// while a GA-created gated channel is present: only ADDING one, changing an existing
|
||||
@@ -477,7 +433,7 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
/// deletion stay with the tenant admin. Same shape as the retention-cap check.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static string? ValidateTelegramChannelGate(string? candidateJson, string? existingJson, bool isGlobalAdmin)
|
||||
internal static string? ValidatePlatformProviderGate(string? candidateJson, string? existingJson, bool isGlobalAdmin)
|
||||
{
|
||||
if (isGlobalAdmin)
|
||||
return null;
|
||||
@@ -501,12 +457,7 @@ namespace AutopilotMonitor.Functions.Helpers
|
||||
|| !string.Equals(stored.Url ?? string.Empty, channel.Url ?? string.Empty, StringComparison.Ordinal)
|
||||
|| stored.Enabled != channel.Enabled)
|
||||
{
|
||||
var provider = channel.ProviderType switch
|
||||
{
|
||||
(int)Shared.Models.Notifications.WebhookProviderType.Email => "E-mail",
|
||||
_ => "Telegram",
|
||||
};
|
||||
return $"{provider} channels can only be configured by a Global Administrator.";
|
||||
return "E-mail channels can only be configured by a Global Administrator.";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -458,6 +458,10 @@ builder.Services.AddSingleton<RegisterSessionFunction>();
|
||||
builder.Services.AddSingleton<GetAgentConfigFunction>();
|
||||
builder.Services.AddSingleton<ReportAgentErrorFunction>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.EntraAppRegistry>();
|
||||
builder.Services.AddSingleton<
|
||||
AutopilotMonitor.Functions.Security.IGraphAccessHealthRecorder,
|
||||
AutopilotMonitor.Functions.Security.GraphAccessHealthRecorder>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphAccessRecheckMaintenance>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.GraphTokenService>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.AutopilotDeviceValidator>();
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Security.CorporateIdentifierValidator>();
|
||||
@@ -473,10 +477,8 @@ builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphResolutio
|
||||
builder.Services.AddSingleton<AutopilotMonitor.Functions.Services.GraphResolution.IScriptDisplayNameResolver,
|
||||
AutopilotMonitor.Functions.Services.GraphResolution.ScriptDisplayNameResolver>();
|
||||
builder.Services.AddWebhookNotificationHttpClient();
|
||||
builder.Services.AddHttpClient<TelegramNotificationService>()
|
||||
.AddPolicyHandler((sp, _) => sp.GetRequiredService<ResiliencePolicies>().Notification);
|
||||
// Channel-level send API — routes each NotificationChannel to its transport (webhook renderer
|
||||
// vs. the platform Telegram bot). Transient: both transports are typed HttpClients.
|
||||
// Channel-level send API — routes each NotificationChannel to its transport. Transient: the
|
||||
// webhook transport is a typed HttpClient.
|
||||
builder.Services.AddTransient<AutopilotMonitor.Functions.Services.Notifications.NotificationChannelDispatcher>();
|
||||
// Every channel send lands in the channel's health row (status dot in the portal, failing bell).
|
||||
builder.Services.AddSingleton<
|
||||
|
||||
@@ -114,7 +114,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new AutopilotDeviceValidationResult
|
||||
|
||||
@@ -119,7 +119,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new CloudPcValidationResult
|
||||
|
||||
@@ -121,7 +121,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new CorporateIdentifierValidationResult
|
||||
|
||||
@@ -111,7 +111,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new DeviceAssociationResult
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
/// <summary>
|
||||
/// Microsoft Entra's answer when it refuses an app token: the HTTP status plus the AADSTS code
|
||||
/// from the response body. Only a client error is a rejection; a 5xx, a throttle or a network
|
||||
/// failure says nothing about the tenant and never becomes one.
|
||||
/// </summary>
|
||||
public sealed record EntraTokenRejection(int StatusCode, string? ErrorCode, string? Description)
|
||||
{
|
||||
/// <summary>The app has no service principal in the tenant: its enterprise application was removed.</summary>
|
||||
public const string ServicePrincipalMissingCode = "AADSTS7000229";
|
||||
|
||||
/// <summary>
|
||||
/// Not a refusal in Entra's words but the same lockout: a token without any application role,
|
||||
/// which Graph refuses on every call. Typically a removed enterprise application that a user's
|
||||
/// sign-in consent re-created. The repair is the same admin consent.
|
||||
/// </summary>
|
||||
public static readonly EntraTokenRejection NoApplicationRole =
|
||||
new(200, "NoApplicationRole", "The app token carries no application permission");
|
||||
|
||||
private const string CodePrefix = "AADSTS";
|
||||
private const int MaxDescriptionLength = 300;
|
||||
|
||||
// Wrong, expired or missing client secret: our configuration, never the customer's tenant.
|
||||
private static readonly string[] OurCredentialCodes = { "AADSTS7000215", "AADSTS7000222", "AADSTS7000218" };
|
||||
|
||||
public bool IsServicePrincipalMissing =>
|
||||
string.Equals(ErrorCode, ServicePrincipalMissingCode, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public bool IsOurCredential =>
|
||||
ErrorCode != null && OurCredentialCodes.Contains(ErrorCode, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>Reads the code and the description from a token error body; a body that is not JSON keeps only the status.</summary>
|
||||
public static EntraTokenRejection Parse(int statusCode, string? body)
|
||||
{
|
||||
var json = TryParseObject(body);
|
||||
var description = (json?["error_description"] as JValue)?.Value as string;
|
||||
return new EntraTokenRejection(statusCode, ReadCode(json, description), Shorten(description));
|
||||
}
|
||||
|
||||
private static JObject? TryParseObject(string? body)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(body)) return null;
|
||||
try
|
||||
{
|
||||
return JsonConvert.DeserializeObject<JToken>(body) as JObject;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ReadCode(JObject? json, string? description)
|
||||
{
|
||||
var first = (json?["error_codes"] as JArray)?.FirstOrDefault(t => t.Type == JTokenType.Integer);
|
||||
if (first != null) return CodePrefix + first.Value<long>();
|
||||
|
||||
// Fallback for a body without error_codes: the description starts with the code.
|
||||
var start = description?.IndexOf(CodePrefix, StringComparison.Ordinal) ?? -1;
|
||||
if (start < 0) return null;
|
||||
var digits = new string(description!.Skip(start + CodePrefix.Length).TakeWhile(char.IsDigit).ToArray());
|
||||
return digits.Length == 0 ? null : CodePrefix + digits;
|
||||
}
|
||||
|
||||
private static string? Shorten(string? description)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(description)) return null;
|
||||
var line = description.Split('\n')[0].Trim();
|
||||
return line.Length <= MaxDescriptionLength ? line : line.Substring(0, MaxDescriptionLength);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
/// <summary>Folds what Entra answers for a tenant's app token into the tenant's Graph access state.</summary>
|
||||
public interface IGraphAccessHealthRecorder
|
||||
{
|
||||
/// <summary>Entra refused the token on an agent request. Never throws.</summary>
|
||||
Task RecordRejectedAsync(string tenantId, EntraTokenRejection rejection);
|
||||
|
||||
/// <summary>A token with an application role was freshly minted for the app acting on this tenant. Never throws.</summary>
|
||||
Task RecordMintedAsync(string tenantId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Two stages, so one failed request never reaches a customer. The first rejection only stamps
|
||||
/// the suspicion. A rejection at least <see cref="ConfirmAfter"/> later confirms it: only then
|
||||
/// does the portal show its notice and the operator get an alert. A freshly minted token with an
|
||||
/// application role ends the episode. A token from the cache proves nothing and never reaches
|
||||
/// this class: it stays valid for up to 55 minutes after the enterprise application is gone.
|
||||
/// Neither does a token without a role, or the notice would vanish the moment an admin signs in
|
||||
/// to read it: that sign-in re-creates the application without any permission.
|
||||
/// <para>
|
||||
/// Every decision runs on the fresh row, so across instances only one write wins and the ops
|
||||
/// event fires once. Codes that blame our own credentials never touch the tenant: an expired
|
||||
/// secret would otherwise tell every customer to repair something on their side.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class GraphAccessHealthRecorder : IGraphAccessHealthRecorder
|
||||
{
|
||||
public static readonly TimeSpan ConfirmAfter = TimeSpan.FromMinutes(15);
|
||||
|
||||
private const string BackupSource = "graph-access";
|
||||
private static readonly TimeSpan CredentialAlertInterval = TimeSpan.FromHours(1);
|
||||
|
||||
private readonly TenantConfigurationService _tenantConfig;
|
||||
private readonly Lazy<OpsEventService> _opsEvents;
|
||||
private readonly ILogger<GraphAccessHealthRecorder> _logger;
|
||||
private readonly TimeProvider _time;
|
||||
private long _lastCredentialAlertTicks;
|
||||
|
||||
public GraphAccessHealthRecorder(
|
||||
TenantConfigurationService tenantConfig,
|
||||
Lazy<OpsEventService> opsEvents,
|
||||
ILogger<GraphAccessHealthRecorder> logger,
|
||||
TimeProvider? time = null)
|
||||
{
|
||||
_tenantConfig = tenantConfig;
|
||||
_opsEvents = opsEvents;
|
||||
_logger = logger;
|
||||
_time = time ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
public async Task RecordRejectedAsync(string tenantId, EntraTokenRejection rejection)
|
||||
{
|
||||
try
|
||||
{
|
||||
var now = _time.GetUtcNow().UtcDateTime;
|
||||
if (rejection.IsOurCredential)
|
||||
{
|
||||
await AlertOurCredentialAsync(tenantId, rejection, now).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// Cheap pre-filter on the cached view; the write decides again on the fresh row.
|
||||
var cached = await _tenantConfig.GetConfigurationIfExistsAsync(tenantId).ConfigureAwait(false);
|
||||
if (cached == null || !cached.HasAnyDeviceValidation() || cached.GraphAccessRejectionConfirmedAt != null)
|
||||
return;
|
||||
if (cached.GraphAccessRejectedSince is { } cachedSince && now - cachedSince < ConfirmAfter)
|
||||
return;
|
||||
|
||||
var confirmed = false;
|
||||
var update = await _tenantConfig.UpdateAsync(tenantId, row =>
|
||||
{
|
||||
confirmed = false;
|
||||
if (row.GraphAccessRejectionConfirmedAt != null)
|
||||
return false;
|
||||
if (row.GraphAccessRejectedSince is not { } since)
|
||||
{
|
||||
row.GraphAccessRejectedSince = now;
|
||||
row.GraphAccessRejectedCode = rejection.ErrorCode;
|
||||
return true;
|
||||
}
|
||||
if (now - since < ConfirmAfter)
|
||||
return false;
|
||||
|
||||
row.GraphAccessRejectionConfirmedAt = now;
|
||||
row.GraphAccessRejectedCode = rejection.ErrorCode;
|
||||
confirmed = true;
|
||||
return true;
|
||||
}, BackupSource, "Entra rejected the app token").ConfigureAwait(false);
|
||||
|
||||
if (update.Status != TenantConfigUpdateStatus.Updated)
|
||||
return;
|
||||
|
||||
if (!confirmed)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Graph access for tenant {TenantId} is suspect: Entra rejected the app token ({ErrorCode}, status {StatusCode})",
|
||||
tenantId, rejection.ErrorCode, rejection.StatusCode);
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogWarning(
|
||||
"Graph access for tenant {TenantId} is confirmed lost: Entra still rejects the app token ({ErrorCode}, status {StatusCode})",
|
||||
tenantId, rejection.ErrorCode, rejection.StatusCode);
|
||||
await _opsEvents.Value.RecordGraphAccessRejectedAsync(tenantId, rejection.ErrorCode, rejection.StatusCode).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Recording the Graph token rejection of tenant {TenantId} failed", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task RecordMintedAsync(string tenantId)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cached = await _tenantConfig.GetConfigurationIfExistsAsync(tenantId).ConfigureAwait(false);
|
||||
if (cached?.GraphAccessRejectedSince == null)
|
||||
return;
|
||||
|
||||
var wasConfirmed = false;
|
||||
string? code = null;
|
||||
var update = await _tenantConfig.UpdateAsync(tenantId, row =>
|
||||
{
|
||||
if (row.GraphAccessRejectedSince == null)
|
||||
return false;
|
||||
wasConfirmed = row.GraphAccessRejectionConfirmedAt != null;
|
||||
code = row.GraphAccessRejectedCode;
|
||||
row.GraphAccessRejectedSince = null;
|
||||
row.GraphAccessRejectedCode = null;
|
||||
row.GraphAccessRejectionConfirmedAt = null;
|
||||
return true;
|
||||
}, BackupSource, "Entra issued an app token again").ConfigureAwait(false);
|
||||
|
||||
if (update.Status != TenantConfigUpdateStatus.Updated || !wasConfirmed)
|
||||
return;
|
||||
|
||||
_logger.LogWarning("Graph access for tenant {TenantId} is back: Entra issued an app token again (was {ErrorCode})", tenantId, code);
|
||||
await _opsEvents.Value.RecordGraphAccessRecoveredAsync(tenantId, code).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Recording the recovered Graph access of tenant {TenantId} failed", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AlertOurCredentialAsync(string tenantId, EntraTokenRejection rejection, DateTime now)
|
||||
{
|
||||
// Hits every tenant at once: one alert per instance and hour says everything.
|
||||
var last = Interlocked.Read(ref _lastCredentialAlertTicks);
|
||||
if (now.Ticks - last < CredentialAlertInterval.Ticks)
|
||||
return;
|
||||
if (Interlocked.CompareExchange(ref _lastCredentialAlertTicks, now.Ticks, last) != last)
|
||||
return;
|
||||
|
||||
_logger.LogError(
|
||||
"Entra rejects our app credentials ({ErrorCode}, status {StatusCode}, seen on tenant {TenantId}); device validation fails for every tenant on that app",
|
||||
rejection.ErrorCode, rejection.StatusCode, tenantId);
|
||||
await _opsEvents.Value.RecordGraphAppCredentialRejectedAsync(tenantId, rejection.ErrorCode, rejection.StatusCode).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using AutopilotMonitor.Functions.Services.GraphResolution;
|
||||
using AutopilotMonitor.Shared;
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
@@ -30,6 +31,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
private static readonly TimeSpan MaxTokenTtl = TimeSpan.FromMinutes(55);
|
||||
private static readonly TimeSpan DefaultTokenLifetime = TimeSpan.FromMinutes(35); // when AAD omits expires_in
|
||||
private static readonly TimeSpan TokenGateWait = TimeSpan.FromSeconds(10);
|
||||
private static readonly TimeSpan RejectionLogWindow = TimeSpan.FromMinutes(15);
|
||||
|
||||
// One coalescing gate per tenant (see GetAccessTokenAsync). A SemaphoreSlim is tiny and
|
||||
// tenant cardinality is bounded + validated upstream, so these are never evicted.
|
||||
@@ -52,19 +54,22 @@ namespace AutopilotMonitor.Functions.Security
|
||||
private readonly IMemoryCache _cache;
|
||||
private readonly EntraAppRegistry _appRegistry;
|
||||
private readonly Services.TenantConfigurationService _tenantConfigService;
|
||||
private readonly IGraphAccessHealthRecorder _accessHealth;
|
||||
|
||||
public GraphTokenService(
|
||||
ILogger<GraphTokenService> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IMemoryCache cache,
|
||||
EntraAppRegistry appRegistry,
|
||||
Services.TenantConfigurationService tenantConfigService)
|
||||
Services.TenantConfigurationService tenantConfigService,
|
||||
IGraphAccessHealthRecorder accessHealth)
|
||||
{
|
||||
_logger = logger;
|
||||
_httpClientFactory = httpClientFactory;
|
||||
_cache = cache;
|
||||
_appRegistry = appRegistry;
|
||||
_tenantConfigService = tenantConfigService;
|
||||
_accessHealth = accessHealth;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -138,9 +143,37 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// the resolved client id, so a tenant flipped between apps can never be served a token
|
||||
// minted under the other app.
|
||||
var credentials = await ResolveCredentialsAsync(tenantId);
|
||||
return await GetAccessTokenWithCredentialsAsync(tenantId, credentials, ct);
|
||||
var result = await GetAccessTokenWithCredentialsAsync(tenantId, credentials, ct);
|
||||
if (result.IsFreshlyMinted && CarriesApplicationRole(result.AccessToken))
|
||||
await _accessHealth.RecordMintedAsync(tenantId);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="GetAccessTokenAsync"/> for the device validators of the agent routes and the
|
||||
/// maintenance recheck, the only callers whose rejection counts as evidence of lost access.
|
||||
/// The consent flows in the portal see 401s as a matter of course while a consent propagates.
|
||||
/// </summary>
|
||||
public async Task<GraphTokenResult> GetAccessTokenForAgentValidationAsync(string tenantId, CancellationToken ct = default)
|
||||
{
|
||||
var result = await GetAccessTokenAsync(tenantId, ct);
|
||||
var refusal = result.Rejection
|
||||
?? (result.IsFreshlyMinted && !CarriesApplicationRole(result.AccessToken) ? EntraTokenRejection.NoApplicationRole : null);
|
||||
if (refusal != null)
|
||||
await _accessHealth.RecordRejectedAsync(tenantId, refusal);
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a token can do anything at Graph. A user's sign-in consent re-creates a removed
|
||||
/// enterprise application without any application permission: Entra issues tokens again, and
|
||||
/// Graph refuses every call made with them. Only a role proves access.
|
||||
/// </summary>
|
||||
private static bool CarriesApplicationRole(string? accessToken) =>
|
||||
!string.IsNullOrEmpty(accessToken)
|
||||
&& GraphFeatureDetector.TryParseToken(accessToken, out var roles, out _)
|
||||
&& roles.Count > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Acquires an app-only token for this tenant under EXPLICIT app credentials, skipping the
|
||||
/// homing resolution. Used by the app-homing consent probe: proving that the PRIMARY app
|
||||
@@ -238,6 +271,8 @@ namespace AutopilotMonitor.Functions.Security
|
||||
string? responseBody = null;
|
||||
int attempt = 0;
|
||||
bool lastAttemptWasTransient = false;
|
||||
// Survives the budget: a caller cut off between retries still learns what Entra said.
|
||||
EntraTokenRejection? rejection = null;
|
||||
|
||||
while (true)
|
||||
{
|
||||
@@ -245,7 +280,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
{
|
||||
// Caller's budget exhausted — treat as transient so we never cache "no consent"
|
||||
// for a tenant just because the optional path timed out.
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
try
|
||||
{
|
||||
@@ -271,7 +306,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// the token's own expires_in, minus a skew margin, clamped to a safe band.
|
||||
CacheToken(tenantId, clientId, accessToken!, genAtStart, (long?)tokenJson?["expires_in"]);
|
||||
}
|
||||
return GraphTokenResult.Success(accessToken);
|
||||
return GraphTokenResult.Minted(accessToken);
|
||||
}
|
||||
|
||||
// Classify the error: consent-propagation errors are retryable but ultimately permanent (no consent).
|
||||
@@ -287,6 +322,9 @@ namespace AutopilotMonitor.Functions.Security
|
||||
|| isConsentError || isServerError;
|
||||
|
||||
lastAttemptWasTransient = isServerError;
|
||||
rejection = isServerError ? null : EntraTokenRejection.Parse(statusCode, responseBody);
|
||||
if (rejection != null)
|
||||
LogRejectionOncePerWindow(tenantId, clientId, rejection);
|
||||
|
||||
if (!isRetryable || attempt >= retryDelays.Length)
|
||||
{
|
||||
@@ -300,7 +338,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
// Server errors are transient (infrastructure issue), consent errors are permanent (no consent granted)
|
||||
return lastAttemptWasTransient
|
||||
? GraphTokenResult.TransientFailure()
|
||||
: GraphTokenResult.PermanentFailure();
|
||||
: GraphTokenResult.PermanentFailure(rejection);
|
||||
}
|
||||
|
||||
var delay = retryDelays[attempt];
|
||||
@@ -328,7 +366,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Caller budget exhausted between retries — transient by definition.
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
@@ -338,7 +376,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
_logger.LogWarning(ex,
|
||||
"Graph token acquisition for tenant {TenantId} failed with network error after {Attempts} attempt(s)",
|
||||
tenantId, attempt + 1);
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
|
||||
var delay = retryDelays[attempt];
|
||||
@@ -349,13 +387,29 @@ namespace AutopilotMonitor.Functions.Security
|
||||
try { await Task.Delay(delay, ct); }
|
||||
catch (OperationCanceledException) when (ct.IsCancellationRequested)
|
||||
{
|
||||
return GraphTokenResult.TransientFailure();
|
||||
return GraphTokenResult.TransientFailure(rejection);
|
||||
}
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The retry chain's own failure line only comes after its 50 s, which the validation budget
|
||||
/// never reaches, so without this line Entra's reason would not reach App Insights at all.
|
||||
/// </summary>
|
||||
private void LogRejectionOncePerWindow(string tenantId, string clientId, EntraTokenRejection rejection)
|
||||
{
|
||||
var key = $"graph-token-rejection-logged:{tenantId}:{clientId.ToLowerInvariant()}";
|
||||
if (_cache.TryGetValue(key, out _))
|
||||
return;
|
||||
_cache.Set(key, true, RejectionLogWindow);
|
||||
|
||||
_logger.LogWarning(
|
||||
"Graph token rejected by Entra for tenant {TenantId}, app {ClientId}: status {StatusCode}, {ErrorCode}. {Description}",
|
||||
tenantId, clientId, rejection.StatusCode, rejection.ErrorCode, rejection.Description);
|
||||
}
|
||||
|
||||
// Keys include the resolved client id: during the dual app-reg window a tenant flip
|
||||
// (HomedAppClientId change) must never be served a token/verdict minted under the other
|
||||
// app. Normalized to lowercase so casing differences in config can't split the cache.
|
||||
@@ -440,9 +494,16 @@ namespace AutopilotMonitor.Functions.Security
|
||||
/// </summary>
|
||||
public bool IsTransient { get; private set; }
|
||||
|
||||
/// <summary>True when Entra issued this token just now; false for one from the cache.</summary>
|
||||
public bool IsFreshlyMinted { get; private set; }
|
||||
|
||||
/// <summary>What Entra last refused with, also when the caller's budget ended the retries; null without a refusal.</summary>
|
||||
public EntraTokenRejection? Rejection { get; private set; }
|
||||
|
||||
public static GraphTokenResult Success(string? token) => new() { AccessToken = token };
|
||||
public static GraphTokenResult PermanentFailure() => new() { AccessToken = null, IsTransient = false };
|
||||
public static GraphTokenResult TransientFailure() => new() { AccessToken = null, IsTransient = true };
|
||||
public static GraphTokenResult Minted(string? token) => new() { AccessToken = token, IsFreshlyMinted = !string.IsNullOrEmpty(token) };
|
||||
public static GraphTokenResult PermanentFailure(EntraTokenRejection? rejection = null) => new() { AccessToken = null, IsTransient = false, Rejection = rejection };
|
||||
public static GraphTokenResult TransientFailure(EntraTokenRejection? rejection = null) => new() { AccessToken = null, IsTransient = true, Rejection = rejection };
|
||||
}
|
||||
|
||||
public class GraphConsentStatusResult
|
||||
|
||||
@@ -247,7 +247,7 @@ namespace AutopilotMonitor.Functions.Security
|
||||
var ct = attemptCts.Token;
|
||||
try
|
||||
{
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenAsync(tenantId, ct);
|
||||
var tokenResult = await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, ct);
|
||||
if (string.IsNullOrEmpty(tokenResult.AccessToken))
|
||||
{
|
||||
return new IntuneDeviceBindingResult
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ namespace AutopilotMonitor.Functions.Services.Activation
|
||||
/// <summary>
|
||||
/// Queue message contract for <c>tenant-auto-approve</c>. One envelope per new tenant
|
||||
/// signup, enqueued with <see cref="ActivationDelay"/> visibility delay so the operator's
|
||||
/// Telegram / portal notification lands before the activation happens.
|
||||
/// alert and portal notification land before the activation happens.
|
||||
/// </summary>
|
||||
public sealed class TenantAutoApproveEnvelope
|
||||
{
|
||||
|
||||
@@ -144,7 +144,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
|
||||
var legacy = await _graphTokenService.GetAccessTokenForAppAsync(tenantId, legacyApp, ct);
|
||||
if (string.IsNullOrWhiteSpace(legacy.AccessToken) && legacy.IsTransient)
|
||||
// A missing service principal is a fact even when the caller's budget cut the retries
|
||||
// short: the access check runs on 4 s, its first retry waits 5 s.
|
||||
if (string.IsNullOrWhiteSpace(legacy.AccessToken) && legacy.IsTransient
|
||||
&& legacy.Rejection?.IsServicePrincipalMissing != true)
|
||||
{
|
||||
return new AppHomingProbeResult(Succeeded: false, IsTransient: true);
|
||||
}
|
||||
@@ -238,6 +241,10 @@ namespace AutopilotMonitor.Functions.Services
|
||||
oldValue = row.HomedAppClientId;
|
||||
row.HomedAppClientId = normalizedTarget;
|
||||
row.UpdatedBy = actorUpn;
|
||||
// A Graph access rejection belongs to the app the tenant just left.
|
||||
row.GraphAccessRejectedSince = null;
|
||||
row.GraphAccessRejectedCode = null;
|
||||
row.GraphAccessRejectionConfirmedAt = null;
|
||||
return true;
|
||||
}, "app-homing", reason);
|
||||
|
||||
|
||||
+1
-1
@@ -336,7 +336,7 @@ namespace AutopilotMonitor.Functions.Services.Backup.Queue
|
||||
await _jobs.TryUpdateWithCasAsync(lostJob, lostETag.Value, ct).ConfigureAwait(false);
|
||||
}
|
||||
// Codex-Hotfix Wave2 #2: emit OpsEvent for the lease-loss terminal path so
|
||||
// Telegram alert rules fire on this manual-queue failure mode the same way
|
||||
// alert rules fire on this manual-queue failure mode the same way
|
||||
// they do for the timer path.
|
||||
await TryRecordOpsEventAsync(() => _opsEvents.RecordCriticalTableBackupFailedAsync(
|
||||
backupIdForEvent, leaseLossReason, runningJobRequestedBy)).ConfigureAwait(false);
|
||||
|
||||
@@ -136,7 +136,7 @@ namespace AutopilotMonitor.Functions.Services.Deletion
|
||||
}
|
||||
|
||||
// PR-B audit consolidation: poisoned cascades surface as a global-scope OpsEvent
|
||||
// (operator-bound, Telegram-routable) rather than a per-tenant audit. Tenant admins
|
||||
// (operator-bound, routable through alert rules) rather than a per-tenant audit. Tenant admins
|
||||
// observe deletion via the lifecycle audits (started / completed / restored).
|
||||
//
|
||||
// Codex F4 follow-up: pull the handler's last-failure breadcrumb out of the
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Services
|
||||
{
|
||||
/// <summary>
|
||||
/// Asks Entra again for every tenant with a Graph access suspicion, folded into the 2-hourly
|
||||
/// maintenance run. Without it the state would depend on device traffic: a suspicion from a
|
||||
/// Friday evening would stay unconfirmed until Monday, and a repair made directly in Entra would
|
||||
/// keep the notice up until the next device sends.
|
||||
/// </summary>
|
||||
public class GraphAccessRecheckMaintenance
|
||||
{
|
||||
/// <summary>One token request answers the question; a rejection then only waits for its retry delay.</summary>
|
||||
internal static readonly TimeSpan AttemptBudget = TimeSpan.FromSeconds(3);
|
||||
|
||||
private readonly IConfigRepository _configRepository;
|
||||
private readonly GraphTokenService _graphTokenService;
|
||||
private readonly ILogger<GraphAccessRecheckMaintenance> _logger;
|
||||
|
||||
public GraphAccessRecheckMaintenance(
|
||||
IConfigRepository configRepository,
|
||||
GraphTokenService graphTokenService,
|
||||
ILogger<GraphAccessRecheckMaintenance> logger)
|
||||
{
|
||||
_configRepository = configRepository;
|
||||
_graphTokenService = graphTokenService;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>Never throws; returns how many tenants it asked about.</summary>
|
||||
public async Task<int> RunAsync()
|
||||
{
|
||||
string[] tenantIds;
|
||||
try
|
||||
{
|
||||
var configs = await _configRepository.GetAllTenantConfigurationsAsync().ConfigureAwait(false);
|
||||
// Without device validation the state means nothing: it waits for the switch to come back.
|
||||
tenantIds = configs
|
||||
.Where(c => c.GraphAccessRejectedSince != null && c.HasAnyDeviceValidation())
|
||||
.Select(c => c.TenantId)
|
||||
.ToArray();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Graph access recheck: reading the tenant configurations failed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
foreach (var tenantId in tenantIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
// A cached token proves nothing either way: only a fresh answer may decide.
|
||||
_graphTokenService.InvalidateTenant(tenantId);
|
||||
using var budget = new CancellationTokenSource(AttemptBudget);
|
||||
await _graphTokenService.GetAccessTokenForAgentValidationAsync(tenantId, budget.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Graph access recheck failed for tenant {TenantId}", tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
return tenantIds.Length;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,7 +503,7 @@ public class HealthCheckService
|
||||
/// overall status to at least <c>warning</c> so a flaky Storage call cannot silently
|
||||
/// mask a real backlog; an enumeration failure does the same. Thresholds live in
|
||||
/// <see cref="MaintenanceService"/> so the live health card and the timer-driven
|
||||
/// Telegram alert classify identically.
|
||||
/// alert classify identically.
|
||||
/// </summary>
|
||||
internal async Task<HealthCheck> CheckPoisonQueuesAsync()
|
||||
{
|
||||
|
||||
+74
-3
@@ -6,7 +6,7 @@ using System.Threading.Tasks;
|
||||
namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
{
|
||||
/// <summary>
|
||||
/// Per-step wall-clock of one maintenance run, reported as <c>stepsMs</c> on the
|
||||
/// Per-step wall-clock and memory of one maintenance run, reported on the
|
||||
/// <c>MaintenanceCompleted</c> ops event. Worker-side timing logs never reach App Insights,
|
||||
/// and the run's composition is what decides which steps leave the 2h timer next.
|
||||
/// A step that throws is still recorded (its time counts), then the exception propagates.
|
||||
@@ -14,12 +14,31 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
public sealed class MaintenanceStepTimer
|
||||
{
|
||||
private readonly Dictionary<string, long> _steps = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _peakMb = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _allocMb = new Dictionary<string, long>(StringComparer.Ordinal);
|
||||
private readonly Func<MaintenanceMemoryReading> _readMemory;
|
||||
|
||||
public MaintenanceStepTimer() : this(MaintenanceMemoryReading.Capture)
|
||||
{
|
||||
}
|
||||
|
||||
internal MaintenanceStepTimer(Func<MaintenanceMemoryReading> readMemory)
|
||||
{
|
||||
_readMemory = readMemory;
|
||||
}
|
||||
|
||||
/// <summary>Durations in declaration order (ms, rounded).</summary>
|
||||
public IReadOnlyDictionary<string, long> Steps => _steps;
|
||||
|
||||
public MaintenanceMemoryReport MemoryReport()
|
||||
{
|
||||
var reading = _readMemory();
|
||||
return new MaintenanceMemoryReport(_peakMb, _allocMb, reading.GcHeapLimitBytes >> 20, reading.GcHighLoadBytes >> 20);
|
||||
}
|
||||
|
||||
public async Task RunAsync(string step, Func<Task> body)
|
||||
{
|
||||
var before = _readMemory();
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
@@ -27,12 +46,13 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
Record(step, sw, before);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<T> RunAsync<T>(string step, Func<Task<T>> body)
|
||||
{
|
||||
var before = _readMemory();
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
@@ -40,7 +60,58 @@ namespace AutopilotMonitor.Functions.Services.Maintenance
|
||||
}
|
||||
finally
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
Record(step, sw, before);
|
||||
}
|
||||
}
|
||||
|
||||
private void Record(string step, Stopwatch sw, MaintenanceMemoryReading before)
|
||||
{
|
||||
_steps[step] = sw.ElapsedMilliseconds;
|
||||
|
||||
var after = _readMemory();
|
||||
_peakMb[step] = after.PeakWorkingSetBytes >> 20;
|
||||
_allocMb[step] = (after.AllocatedBytes - before.AllocatedBytes) >> 20;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="StepsPeakMb"/> is the process's peak working set after each step: the step where
|
||||
/// it jumps set the peak, without depending on when a counter happened to sample.
|
||||
/// <see cref="StepsAllocMb"/> is what the process allocated during each step.
|
||||
/// </summary>
|
||||
public sealed record MaintenanceMemoryReport(
|
||||
IReadOnlyDictionary<string, long> StepsPeakMb,
|
||||
IReadOnlyDictionary<string, long> StepsAllocMb,
|
||||
long GcHeapLimitMb,
|
||||
long GcHighLoadMb);
|
||||
|
||||
internal readonly record struct MaintenanceMemoryReading(
|
||||
long PeakWorkingSetBytes,
|
||||
long AllocatedBytes,
|
||||
long GcHeapLimitBytes,
|
||||
long GcHighLoadBytes)
|
||||
{
|
||||
internal static MaintenanceMemoryReading Capture()
|
||||
{
|
||||
var info = GC.GetGCMemoryInfo();
|
||||
return new MaintenanceMemoryReading(
|
||||
ReadPeakWorkingSet(),
|
||||
GC.GetTotalAllocatedBytes(precise: false),
|
||||
info.TotalAvailableMemoryBytes,
|
||||
info.HighMemoryLoadThresholdBytes);
|
||||
}
|
||||
|
||||
// A maintenance run must not fail on its own telemetry.
|
||||
private static long ReadPeakWorkingSet()
|
||||
{
|
||||
try
|
||||
{
|
||||
using var process = Process.GetCurrentProcess();
|
||||
return process.PeakWorkingSet64;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+50
-25
@@ -1,4 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
@@ -26,9 +27,8 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// or median back under 1.5×) deletes the row so a future regression rings again.
|
||||
/// The tracker's 30d retention sweep re-arms long-burning episodes by design.
|
||||
/// <para>
|
||||
/// One cross-tenant install-summary read over the trailing
|
||||
/// <see cref="AppVersionRegressionRadar.HorizonDays"/> days (the same query the global
|
||||
/// apps dashboard runs), then grouped by tenant — mirrors the rule radar's enumeration.
|
||||
/// Reads the trailing <see cref="AppVersionRegressionRadar.HorizonDays"/> days of install
|
||||
/// summaries tenant by tenant (see <see cref="RunAppVersionRadarPassAsync"/>).
|
||||
/// Idempotent per pass via the tracker dedup. Fail-soft per tenant and overall.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
@@ -44,33 +44,18 @@ namespace AutopilotMonitor.Functions.Services
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var sinceUtc = DateTime.UtcNow.AddDays(-AppVersionRegressionRadar.HorizonDays);
|
||||
var summaries = await _metricsRepo.GetAppsDashboardSummariesAsync(sinceUtc, tenantId: null);
|
||||
var tenantIds = await _maintenanceRepo.GetAllTenantIdsAsync();
|
||||
|
||||
var byTenant = summaries
|
||||
.Where(s => !string.IsNullOrEmpty(s.TenantId))
|
||||
.GroupBy(s => s.TenantId, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
int fired = 0, refreshed = 0, rearmed = 0, tenants = 0;
|
||||
foreach (var tenantGroup in byTenant)
|
||||
{
|
||||
tenants++;
|
||||
try
|
||||
{
|
||||
var (f, r, a) = await EvaluateTenantAppVersionRegressionsAsync(tenantGroup.Key, tenantGroup.ToList());
|
||||
fired += f;
|
||||
refreshed += r;
|
||||
rearmed += a;
|
||||
}
|
||||
catch (Exception tenantEx)
|
||||
{
|
||||
_logger.LogWarning(tenantEx, "App-version regression radar failed for tenant {TenantId} (non-fatal)", tenantGroup.Key);
|
||||
}
|
||||
}
|
||||
var pass = await RunAppVersionRadarPassAsync(
|
||||
tenantIds,
|
||||
tenantId => _metricsRepo.GetAppsDashboardSummariesAsync(sinceUtc, tenantId),
|
||||
EvaluateTenantAppVersionRegressionsAsync,
|
||||
(tenantId, tenantEx) => _logger.LogWarning(tenantEx, "App-version regression radar failed for tenant {TenantId} (non-fatal)", tenantId));
|
||||
|
||||
sw.Stop();
|
||||
_logger.LogInformation(
|
||||
"App-version regression radar: {Tenants} tenants over {Rows} install summaries — {Fired} fired, {Refreshed} refreshed, {Rearmed} re-armed in {Ms}ms",
|
||||
tenants, summaries.Count, fired, refreshed, rearmed, sw.ElapsedMilliseconds);
|
||||
pass.Tenants, pass.Rows, pass.Fired, pass.Refreshed, pass.Rearmed, sw.ElapsedMilliseconds);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -78,6 +63,46 @@ namespace AutopilotMonitor.Functions.Services
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads and evaluates one tenant at a time, so a pass holds at most one tenant's rows and
|
||||
/// its memory follows the largest tenant, not the fleet.
|
||||
/// </summary>
|
||||
internal static async Task<(int Tenants, long Rows, int Fired, int Refreshed, int Rearmed)> RunAppVersionRadarPassAsync(
|
||||
IEnumerable<string> tenantIds,
|
||||
Func<string, Task<List<AppInstallSummary>>> loadTenant,
|
||||
Func<string, List<AppInstallSummary>, Task<(int Fired, int Refreshed, int Rearmed)>> evaluateTenant,
|
||||
Action<string, Exception> onTenantFailed)
|
||||
{
|
||||
int tenants = 0, fired = 0, refreshed = 0, rearmed = 0;
|
||||
long rows = 0;
|
||||
|
||||
foreach (var tenantId in tenantIds)
|
||||
{
|
||||
try
|
||||
{
|
||||
var summaries = await loadTenant(tenantId);
|
||||
|
||||
// Never evaluate an empty list: re-arm reads "no rows" as "version drained" and
|
||||
// would delete the tenant's open episodes, and a failed read also returns empty.
|
||||
if (summaries.Count == 0)
|
||||
continue;
|
||||
|
||||
tenants++;
|
||||
rows += summaries.Count;
|
||||
var (f, r, a) = await evaluateTenant(tenantId, summaries);
|
||||
fired += f;
|
||||
refreshed += r;
|
||||
rearmed += a;
|
||||
}
|
||||
catch (Exception tenantEx)
|
||||
{
|
||||
onTenantFailed(tenantId, tenantEx);
|
||||
}
|
||||
}
|
||||
|
||||
return (tenants, rows, fired, refreshed, rearmed);
|
||||
}
|
||||
|
||||
private async Task<(int Fired, int Refreshed, int Rearmed)> EvaluateTenantAppVersionRegressionsAsync(
|
||||
string tenantId, System.Collections.Generic.List<AppInstallSummary> tenantSummaries)
|
||||
{
|
||||
|
||||
@@ -13,7 +13,7 @@ namespace AutopilotMonitor.Functions.Services
|
||||
/// <summary>
|
||||
/// Cert-expiry monitoring leg of MaintenanceService. Runs on the existing 2h
|
||||
/// timer and emits tiered OpsEvents for embedded Intune CA certs nearing
|
||||
/// NotAfter, so the operator gets a Telegram ping months before agent mTLS
|
||||
/// NotAfter, so the operator gets an alert months before agent mTLS
|
||||
/// would break - bundle rotation is the only fix and needs lead time to
|
||||
/// source new PEMs from a corporate-enrolled device.
|
||||
/// </summary>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user