mirror of
https://github.com/okieselbach/AutopilotMonitor.git
synced 2026-10-25 13:10:13 +00:00
Compare commits
23 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 dependabot[bot]
|
51aaa2e80a | ||
|
dependabot[bot]
|
0a6779cd26 | ||
|
dependabot[bot]
|
1a1c2b413e | ||
|
Oliver Kieselbach
|
5e763375b4 | ||
|
Oliver Kieselbach
|
9f4f456dc5 | ||
|
Oliver Kieselbach
|
af2f2c7abe | ||
|
Oliver Kieselbach
|
98d87a8579 | ||
|
Oliver Kieselbach
|
0709ce2f76 | ||
|
Oliver Kieselbach
|
69d876b0a6 | ||
|
Oliver Kieselbach
|
224eb668a1 | ||
|
Oliver Kieselbach
|
a6980a93f6 | ||
|
Oliver Kieselbach
|
eb57980b53 | ||
|
Oliver Kieselbach
|
c4c2415de8 | ||
|
Oliver Kieselbach
|
9331366025 | ||
|
Oliver Kieselbach
|
85a0cb3c31 | ||
|
Oliver Kieselbach
|
5efa7d078c | ||
|
Oliver Kieselbach
|
35d1445ebb | ||
|
Oliver Kieselbach
|
23cd9050ea | ||
|
Oliver Kieselbach
|
8615fb458f | ||
|
Oliver Kieselbach
|
645f747191 | ||
|
Oliver Kieselbach
|
5934ae9fc4 | ||
|
Oliver Kieselbach
|
3d10011460 | ||
|
Oliver Kieselbach
|
edad842a0b |
@@ -38,6 +38,8 @@
|
||||
- Simplicity first: minimal, targeted changes. No temporary fixes — find the root cause.
|
||||
- Where architecture is flawed, state is duplicated, or patterns are inconsistent, propose and implement the structural fix. Ask what a perfectionist reviewer would reject, and fix that too.
|
||||
- On non-trivial changes, ask whether there is a more elegant way before presenting. Skip it for simple fixes — don't over-engineer.
|
||||
- Reuse before you write: search for an existing helper by what it does (its key call, literal or regex), not by a guessed name. Web helpers live in `lib/` (route-only ones in a `lib/` beside the route), backend helpers in `Functions/Helpers/`.
|
||||
- Dead and copied code is gated: the .NET build fails on unused private members and parameters (`.editorconfig`), `npm run knip` covers web and MCP, and `npm run dup` in `tools/quality` fails on new clones. CI only reports, so run them before calling a change done.
|
||||
- English for code comments and everything that carries knowledge.
|
||||
|
||||
Private half of these instructions (process, working files, technical docs, operations): @../internal/CLAUDE.md
|
||||
|
||||
+1
-1
Submodule .claude/commands updated: 1be456ea91...ce2fb06a44
@@ -0,0 +1,11 @@
|
||||
# Analyzer severities for the .NET build. Directory.Build.props runs the IDE rules in
|
||||
# `dotnet build` (EnforceCodeStyleInBuild), and the warning freeze makes every warning an error.
|
||||
root = true
|
||||
|
||||
[*.cs]
|
||||
# Unused private members, private members that are written but never read, and unused parameters
|
||||
# of non-public methods. An injected dependency nobody reads is a feature that silently never runs.
|
||||
dotnet_diagnostic.IDE0051.severity = warning
|
||||
dotnet_diagnostic.IDE0052.severity = warning
|
||||
dotnet_diagnostic.IDE0060.severity = warning
|
||||
dotnet_code_quality_unused_parameters = non_public
|
||||
@@ -81,6 +81,23 @@ updates:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
# Duplicate-check tooling (jscpd for tools/quality/check-duplicates.mjs), installed by the
|
||||
# CI duplicates job from this lockfile. A jscpd update can change the clone fingerprints:
|
||||
# if its PR fails the job, regenerate the baseline (npm run dup:baseline) in that PR.
|
||||
- package-ecosystem: npm
|
||||
directory: /tools/quality
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
groups:
|
||||
quality-tooling:
|
||||
patterns: ['*']
|
||||
quality-tooling-security:
|
||||
applies-to: security-updates
|
||||
patterns: ['*']
|
||||
|
||||
# Every action is referenced by commit SHA with the version in a trailing comment;
|
||||
# action-pins.yml fails any workflow change that reintroduces a mutable tag. Dependabot
|
||||
# is what keeps those pins current: it rewrites the SHA and the comment together, so the
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
name: CI
|
||||
|
||||
# Tests, typechecks and drift-checks every push to main and every pull request:
|
||||
# the .NET suites, the web and MCP suites, and a guard that the generated rule
|
||||
# files still match rules/guardrails.json.
|
||||
# the .NET suites, the web and MCP suites, a guard that the generated rule
|
||||
# files still match rules/guardrails.json, and the duplicate-code check.
|
||||
#
|
||||
# The deploy workflows build but never test, so this is the only place the suites
|
||||
# actually run. It does not gate anything — main is deliberately unprotected, so a
|
||||
@@ -160,6 +160,12 @@ jobs:
|
||||
- name: Lint
|
||||
run: npm run lint -- --max-warnings 0
|
||||
|
||||
# Unused files, exports and dependencies (knip.json). Its own script on
|
||||
# purpose: chained into `lint`, the `-- --max-warnings 0` above would be
|
||||
# handed to knip and ESLint would lose its cap.
|
||||
- name: Unused code
|
||||
run: npm run knip
|
||||
|
||||
- name: Tests
|
||||
run: npm test
|
||||
|
||||
@@ -182,9 +188,19 @@ jobs:
|
||||
|
||||
- run: npm ci
|
||||
|
||||
# tsconfig.json covers sources, tests and scripts; the build emits via
|
||||
# tsconfig.build.json.
|
||||
- name: Typecheck
|
||||
run: npx tsc --noEmit
|
||||
|
||||
# Same rule as the web job: any warning fails — fix new findings, never
|
||||
# raise the cap.
|
||||
- name: Lint
|
||||
run: npm run lint -- --max-warnings 0
|
||||
|
||||
- name: Unused code
|
||||
run: npm run knip
|
||||
|
||||
# The vector-search tests tokenize with the real all-MiniLM-L6-v2 model
|
||||
# (~23 MB tokenizer + ONNX weights). transformers.js caches under
|
||||
# node_modules by default, which npm ci recreates — so every CI run
|
||||
@@ -247,8 +263,33 @@ jobs:
|
||||
|
||||
- name: Fail if the generated files drifted
|
||||
run: |
|
||||
if ! git diff --quiet -- src/Web/autopilot-monitor-web/utils/shared-manifests.generated.ts src/Web/autopilot-monitor-web/utils/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts; then
|
||||
if ! git diff --quiet -- src/Web/autopilot-monitor-web/lib/generated/shared-manifests.generated.ts src/Web/autopilot-monitor-web/lib/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-types.generated.ts src/McpServer/autopilot-monitor-mcp/src/generated/wire-vocabularies.generated.ts; then
|
||||
echo "::error::Generated wire types are out of date. Run 'npm run generate:manifests' in src/Web/autopilot-monitor-web and commit the result."
|
||||
git diff --stat
|
||||
exit 1
|
||||
fi
|
||||
|
||||
duplicates:
|
||||
name: Duplicate code
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: tools/quality
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: npm
|
||||
cache-dependency-path: tools/quality/package-lock.json
|
||||
|
||||
- run: npm ci --ignore-scripts
|
||||
|
||||
# jscpd over the C# and TypeScript sources (.jscpd.json). A clone missing from
|
||||
# jscpd-baseline.json fails, and so does a baseline entry whose clone is gone,
|
||||
# so the baseline only shrinks. Locally: npm run dup in tools/quality.
|
||||
- name: Duplicate check
|
||||
run: npm run dup
|
||||
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -- rules/dist/ \
|
||||
src/Web/autopilot-monitor-web/utils/guardrails.generated.ts \
|
||||
src/Web/autopilot-monitor-web/lib/generated/guardrails.generated.ts \
|
||||
src/McpServer/autopilot-monitor-mcp/src/rule-authoring.generated.ts
|
||||
if git diff --cached --quiet; then
|
||||
echo "Generated outputs unchanged - nothing to commit"
|
||||
|
||||
@@ -8,5 +8,8 @@
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<MSBuildTreatWarningsAsErrors>true</MSBuildTreatWarningsAsErrors>
|
||||
<WarningsNotAsErrors>NU1901;NU1902;NU1903;NU1904</WarningsNotAsErrors>
|
||||
<!-- Run the IDE code-style analyzers in the build too, so the rules .editorconfig raises
|
||||
to warning fail it like any compiler warning. -->
|
||||
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
|
||||
</PropertyGroup>
|
||||
</Project>
|
||||
|
||||
+1
-1
Submodule internal updated: 54d7076308...50f2c079bc
@@ -86,12 +86,12 @@ for (const config of configs) {
|
||||
// ── Guardrails generation ───────────────────────────────────────────────────
|
||||
|
||||
const guardrailsPath = path.join(rulesRoot, 'guardrails.json');
|
||||
// NOTE: consumers import from "@/utils/guardrails.generated" (see guardValidation.ts
|
||||
// NOTE: consumers import from "@/lib/generated/guardrails.generated" (see guardValidation.ts
|
||||
// and SectionGatherRules.tsx). The web re-org (commit a3e8e044) moved this file to
|
||||
// utils/ but left this output path pointing at the now-orphaned lib/ copy, which
|
||||
// silently froze the live allowlist. Keep this in sync with the import path.
|
||||
const guardrailsOutput = path.resolve(
|
||||
rulesRoot, '..', 'src', 'Web', 'autopilot-monitor-web', 'utils', 'guardrails.generated.ts'
|
||||
rulesRoot, '..', 'src', 'Web', 'autopilot-monitor-web', 'lib', 'generated', 'guardrails.generated.ts'
|
||||
);
|
||||
|
||||
if (fs.existsSync(guardrailsPath)) {
|
||||
|
||||
+4
-5
@@ -64,7 +64,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
private void Buffer(string? rebootUri, long? recordId = null, bool isBackfill = false, DateTime? timeCreatedUtc = null, string? description = null)
|
||||
{
|
||||
_tracker.ProcessEvent(
|
||||
eventId: MdmRebootPolicyTracker.EventId_PolicyRebootRequired,
|
||||
recordId: recordId ?? _nextRecordId++,
|
||||
timeCreatedUtc: timeCreatedUtc ?? At,
|
||||
rebootUri: rebootUri,
|
||||
@@ -226,13 +225,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
// The coalesced reboot restarts the agent; the post-reboot backfill re-reads the same
|
||||
// records — a FLUSHED record must be skipped by the next instance.
|
||||
var first = MakeTracker(stateDirectory: _tmp.Path);
|
||||
first.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
first.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: false);
|
||||
first.FlushPending();
|
||||
|
||||
var second = MakeTracker(stateDirectory: _tmp.Path);
|
||||
second.LoadWatermark();
|
||||
second.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
second.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: true);
|
||||
second.FlushPending();
|
||||
|
||||
@@ -245,13 +244,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.SystemSignals
|
||||
// Process killed by the very reboot the records announce: nothing was flushed, so the
|
||||
// next instance must re-read and emit the records.
|
||||
var first = MakeTracker(stateDirectory: _tmp.Path);
|
||||
first.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
first.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: false);
|
||||
// no FlushPending — simulated reboot kill
|
||||
|
||||
var second = MakeTracker(stateDirectory: _tmp.Path);
|
||||
second.LoadWatermark();
|
||||
second.ProcessEvent(MdmRebootPolicyTracker.EventId_PolicyRebootRequired, recordId: 500,
|
||||
second.ProcessEvent(recordId: 500,
|
||||
timeCreatedUtc: At, rebootUri: UriSvchost, formattedDescription: null, isBackfill: true);
|
||||
second.FlushPending();
|
||||
|
||||
|
||||
+83
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
@@ -49,6 +50,74 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
private static EmergencyReporter Reporter(FakeApiClient client)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null);
|
||||
|
||||
private static EmergencyReporter Reporter(FakeApiClient client, VirtualClock clock)
|
||||
=> new EmergencyReporter(client, "session", "tenant", "0.0.0", logger: null, clock: clock);
|
||||
|
||||
private static VirtualClock Clock() => new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_is_not_suppressed_by_the_cooldown_of_an_earlier_report()
|
||||
{
|
||||
// Terminal report, sent at most once per process right before the termination — a
|
||||
// routine report one minute earlier must not cost it. Every other type keeps the cooldown.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: 503);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(2, client.Calls);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, client.Reports[1].ErrorType);
|
||||
|
||||
// Control: a third report of another type inside the window is still held back.
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.ConfigFetchFailed, "m");
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Second_session_unknown_report_within_the_window_is_still_suppressed()
|
||||
{
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
clock.Advance(TimeSpan.FromMinutes(1));
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(1, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_still_counts_against_the_per_session_maximum()
|
||||
{
|
||||
// Five delivered reports (ten minutes apart, distinct keys) exhaust the budget; the
|
||||
// exemption waives only the cooldown, not the maximum.
|
||||
var clock = Clock();
|
||||
var client = new FakeApiClient();
|
||||
for (var i = 0; i < 6; i++) client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client, clock);
|
||||
|
||||
foreach (var status in new[] { 500, 502, 503, 504, 429 })
|
||||
{
|
||||
await reporter.TrySendAsync(AgentErrorType.IngestFailed, "m", httpStatusCode: status);
|
||||
clock.Advance(TimeSpan.FromMinutes(10));
|
||||
}
|
||||
Assert.Equal(5, client.Calls);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
Assert.Equal(5, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Failed_send_rolls_back_reservation_so_a_later_call_retries()
|
||||
{
|
||||
@@ -142,6 +211,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
|
||||
Assert.Equal(2, client.Calls);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_unknown_report_carries_error_type_and_http_410()
|
||||
{
|
||||
var client = new FakeApiClient();
|
||||
client.Results.Enqueue(true);
|
||||
var reporter = Reporter(client);
|
||||
|
||||
await reporter.TrySendAsync(AgentErrorType.TelemetryRejectedSessionUnknown, "m", httpStatusCode: 410);
|
||||
|
||||
var report = Assert.Single(client.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Session_age_is_carried_in_the_report_and_absent_by_default()
|
||||
{
|
||||
|
||||
+9
-9
@@ -77,7 +77,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -98,7 +98,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -127,7 +127,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var progress = false;
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => Volatile.Read(ref progress));
|
||||
|
||||
ing.Start();
|
||||
@@ -156,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var progress = false;
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => Volatile.Read(ref progress));
|
||||
|
||||
ing.Start();
|
||||
@@ -186,7 +186,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => true);
|
||||
|
||||
ing.Start();
|
||||
@@ -212,7 +212,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller,
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller,
|
||||
espTimeoutProvider: () => null, accountSetupProgressProbe: () => true);
|
||||
|
||||
ing.Start();
|
||||
@@ -236,7 +236,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -259,7 +259,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
using var ing = BuildIngress(clock);
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost("S1", "T1", ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
var host = new UserEspKeepAwakeHost(ing, clock, NewLogger(), controller: controller, espTimeoutProvider: () => null);
|
||||
|
||||
ing.Start();
|
||||
host.Start();
|
||||
@@ -282,7 +282,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Orchestration
|
||||
var api = new FakeKeepAwakeApi();
|
||||
var controller = new KeepAwakeController(NewLogger(), api);
|
||||
var host = new UserEspKeepAwakeHost(
|
||||
"S1", "T1", ing, clock, NewLogger(),
|
||||
ing, clock, NewLogger(),
|
||||
controller: controller,
|
||||
safetyCapOverride: TimeSpan.FromMilliseconds(75),
|
||||
espTimeoutProvider: () => null);
|
||||
|
||||
@@ -44,7 +44,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
// from a real backend). We exercise the bundle-shape contract here.
|
||||
var rcs = new AutopilotMonitor.Agent.V2.Core.Configuration.RemoteConfigService(
|
||||
auth.BackendApiClient, "tenant-x", logger,
|
||||
auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
|
||||
auth.EmergencyReporter, auth.AuthFailureTracker);
|
||||
var rc = new AgentConfigResponse();
|
||||
var merge = new RemoteConfigMergeResult();
|
||||
|
||||
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
#nullable enable
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Harness;
|
||||
using AutopilotMonitor.Agent.V2.Core.Tests.Orchestration;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Tests.Program
|
||||
{
|
||||
/// <summary>
|
||||
/// Host wiring for <c>TelemetryUploadOrchestrator.SessionGone</c>
|
||||
/// (HTTP 410 on the current session's telemetry): exactly one
|
||||
/// <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> emergency report carrying the
|
||||
/// status, then a synthetic forced <c>terminate_session</c> tagged <c>origin=session_gone</c>
|
||||
/// — and nothing at all once a shutdown is already in progress.
|
||||
/// </summary>
|
||||
public sealed class ServerControlPlaneSessionGoneTests
|
||||
{
|
||||
private sealed class RecordingEmergencyReporter : EmergencyReporter
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<(AgentErrorType ErrorType, string Message, int? HttpStatusCode)> Reports { get; } =
|
||||
new List<(AgentErrorType, string, int?)>();
|
||||
|
||||
public RecordingEmergencyReporter(List<string> sequence)
|
||||
: base(apiClient: null!, sessionId: "S1", tenantId: "T1", agentVersion: "0.0.0", logger: null!)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task TrySendAsync(
|
||||
AgentErrorType errorType,
|
||||
string message,
|
||||
int? httpStatusCode = null,
|
||||
long? sequenceNumber = null,
|
||||
int attempts = 1,
|
||||
TimeSpan? perAttemptTimeout = null,
|
||||
TimeSpan? retryDelay = null,
|
||||
double? sessionAgeHours = null,
|
||||
RegistrationFailureSummary? priorRegistrationFailure = null)
|
||||
{
|
||||
Reports.Add((errorType, message, httpStatusCode));
|
||||
_sequence.Add("report");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class RecordingDispatcher : ServerActionDispatcher
|
||||
{
|
||||
private readonly List<string> _sequence;
|
||||
public List<ServerAction> Dispatched { get; } = new List<ServerAction>();
|
||||
|
||||
public RecordingDispatcher(AgentConfiguration config, AgentLogger logger, InformationalEventPost post, List<string> sequence)
|
||||
: base(
|
||||
configuration: config,
|
||||
logger: logger,
|
||||
rotateConfigAsync: () => Task.FromResult(true),
|
||||
uploadDiagnosticsAsync: _ => Task.FromResult<DiagnosticsUploadResult>(null!),
|
||||
onTerminateRequested: _ => Task.CompletedTask,
|
||||
post: post)
|
||||
{
|
||||
_sequence = sequence;
|
||||
}
|
||||
|
||||
public override Task DispatchAsync(List<ServerAction> actions)
|
||||
{
|
||||
Dispatched.AddRange(actions);
|
||||
_sequence.Add("dispatch");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Rig : IDisposable
|
||||
{
|
||||
public TempDirectory Tmp { get; } = new TempDirectory();
|
||||
public AgentLogger Logger { get; }
|
||||
public List<string> Sequence { get; } = new List<string>();
|
||||
public RecordingEmergencyReporter Reporter { get; }
|
||||
public RecordingDispatcher Dispatcher { get; }
|
||||
public ManualResetEventSlim Shutdown { get; } = new ManualResetEventSlim(false);
|
||||
public ManualResetEventSlim ShutdownComplete { get; } = new ManualResetEventSlim(false);
|
||||
|
||||
public Rig()
|
||||
{
|
||||
Logger = new AgentLogger(Path.Combine(Tmp.Path, "logs"), AgentLogLevel.Info);
|
||||
var config = new AgentConfiguration { SessionId = "S1", TenantId = "T1", ApiBaseUrl = "http://localhost" };
|
||||
var post = new InformationalEventPost(
|
||||
new FakeSignalIngressSink(),
|
||||
new VirtualClock(new DateTime(2026, 10, 9, 10, 0, 0, DateTimeKind.Utc)));
|
||||
Reporter = new RecordingEmergencyReporter(Sequence);
|
||||
Dispatcher = new RecordingDispatcher(config, Logger, post, Sequence);
|
||||
}
|
||||
|
||||
public void Handle(string reason = "session gone: http 410") =>
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.HandleSessionGone(
|
||||
reason, Reporter, Dispatcher, Shutdown, ShutdownComplete, Logger);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
Shutdown.Dispose();
|
||||
ShutdownComplete.Dispose();
|
||||
Tmp.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Reports_once_with_410_then_dispatches_forced_terminate_tagged_session_gone()
|
||||
{
|
||||
using var rig = new Rig();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Equal(new[] { "report", "dispatch" }, rig.Sequence);
|
||||
|
||||
var report = Assert.Single(rig.Reporter.Reports);
|
||||
Assert.Equal(AgentErrorType.TelemetryRejectedSessionUnknown, report.ErrorType);
|
||||
Assert.Equal(410, report.HttpStatusCode);
|
||||
Assert.Contains("410", report.Message);
|
||||
|
||||
var action = Assert.Single(rig.Dispatcher.Dispatched);
|
||||
Assert.Equal(ServerActionTypes.TerminateSession, action.Type);
|
||||
Assert.Contains("410", action.Reason);
|
||||
Assert.NotNull(action.Params);
|
||||
var parameters = action.Params!;
|
||||
Assert.Equal("true", parameters["forceSelfDestruct"]);
|
||||
Assert.Equal("0", parameters["gracePeriodSeconds"]);
|
||||
Assert.Equal(TerminationOrigins.SessionGone, parameters["origin"]);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(true, false)]
|
||||
[InlineData(false, true)]
|
||||
public void Does_nothing_when_a_shutdown_is_already_in_progress(bool shutdownSet, bool shutdownCompleteSet)
|
||||
{
|
||||
using var rig = new Rig();
|
||||
if (shutdownSet) rig.Shutdown.Set();
|
||||
if (shutdownCompleteSet) rig.ShutdownComplete.Set();
|
||||
|
||||
rig.Handle();
|
||||
|
||||
Assert.Empty(rig.Reporter.Reports);
|
||||
Assert.Empty(rig.Dispatcher.Dispatched);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReadOrigin_returns_the_origin_param_or_null()
|
||||
{
|
||||
Assert.Equal(TerminationOrigins.SessionGone,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.SessionGone }));
|
||||
Assert.Equal(TerminationOrigins.KillSignal,
|
||||
global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = TerminationOrigins.KillSignal, ["forceSelfDestruct"] = "true" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["adminOutcome"] = "Failed" }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(
|
||||
new Dictionary<string, string> { ["origin"] = string.Empty }));
|
||||
Assert.Null(global::AutopilotMonitor.Agent.V2.Runtime.ServerControlPlane.ReadOrigin(null!));
|
||||
}
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -479,14 +479,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
public HashSet<string> ExistingKeys { get; } =
|
||||
new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
public IReadOnlyList<string> Enumerate(RegistryHive hive, string packagesPath)
|
||||
public IReadOnlyList<string> Enumerate(RegistryHive _, string packagesPath)
|
||||
{
|
||||
return PackagesByPath.TryGetValue(packagesPath, out var ids)
|
||||
? (IReadOnlyList<string>)ids
|
||||
: Array.Empty<string>();
|
||||
}
|
||||
|
||||
public bool TryRead(RegistryHive hive, string packagesPath, string packageId, out RealmJoinPackageSnapshot snapshot)
|
||||
public bool TryRead(RegistryHive _, string _1, string packageId, out RealmJoinPackageSnapshot snapshot)
|
||||
{
|
||||
snapshot = new RealmJoinPackageSnapshot(
|
||||
packageId: packageId,
|
||||
@@ -497,7 +497,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.SignalAdapters
|
||||
return true;
|
||||
}
|
||||
|
||||
public bool KeyExists(RegistryHive hive, string subPath)
|
||||
public bool KeyExists(RegistryHive _, string subPath)
|
||||
=> ExistingKeys.Contains(subPath);
|
||||
}
|
||||
|
||||
|
||||
+59
@@ -310,6 +310,19 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
reason, outcome, stage.ToString(), at ?? EndUtc);
|
||||
|
||||
/// <summary>
|
||||
/// The args <c>ServerControlPlane.OnTerminateRequested</c> synthesises for a
|
||||
/// <c>terminate_session</c> action tagged with <paramref name="origin"/>.
|
||||
/// </summary>
|
||||
private static EnrollmentTerminatedEventArgs ServerArgs(string origin) =>
|
||||
new EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason.DecisionTerminalStage,
|
||||
EnrollmentTerminationOutcome.Failed,
|
||||
SessionStage.Failed.ToString(),
|
||||
EndUtc,
|
||||
details: "Server-requested termination",
|
||||
origin: origin);
|
||||
|
||||
[Fact]
|
||||
public void Handle_writes_final_status_json_in_state_directory()
|
||||
{
|
||||
@@ -458,6 +471,52 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Termination
|
||||
Assert.Equal(0, rig.CleanupService.Invocations);
|
||||
}
|
||||
|
||||
// ============================================================= origin=session_gone (HTTP 410)
|
||||
|
||||
[Fact]
|
||||
public void Handle_session_gone_skips_spool_drain_and_diagnostics_but_still_cleans_up()
|
||||
{
|
||||
// The backend no longer knows the session: every drain would only burn its budget on
|
||||
// 410s and a diagnostics package would describe a deleted session. Marker, cleanup
|
||||
// and the (forced) self-destruct still run.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 5; }; // never drains — a poll would burn the budget
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.SessionGone));
|
||||
|
||||
Assert.Equal(0, spoolPolls);
|
||||
Assert.Equal(0, rig.DiagnosticsUploads);
|
||||
Assert.DoesNotContain(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Contains(Constants.EventTypes.AgentShuttingDown, rig.EmittedEventTypes);
|
||||
Assert.True(File.Exists(Path.Combine(rig.StateDir, "enrollment-complete.marker")));
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_kill_signal_origin_still_drains_spool_and_uploads_diagnostics()
|
||||
{
|
||||
// The administrator kill signal keeps its full sequence — only session_gone skips.
|
||||
using var rig = new Rig();
|
||||
rig.State = new DecisionStateBuilder(DecisionState.CreateInitial("S1", "T1")) { Stage = SessionStage.Failed }.Build();
|
||||
rig.SpoolDrainPeriodOverride = TimeSpan.FromSeconds(5);
|
||||
var spoolPolls = 0;
|
||||
rig.PendingItemCountAccessor = () => { spoolPolls++; return 0; };
|
||||
var cfg = rig.BuildConfig(diagEnabled: true, diagMode: "Always", selfDestruct: true);
|
||||
|
||||
rig.Build(cfg).Handle(sender: null!, ServerArgs(TerminationOrigins.KillSignal));
|
||||
|
||||
Assert.True(spoolPolls > 0);
|
||||
Assert.Equal(1, rig.DiagnosticsUploads);
|
||||
Assert.Contains(Constants.EventTypes.DiagnosticsCollecting, rig.EmittedEventTypes);
|
||||
Assert.Equal(1, rig.CleanupService.Invocations);
|
||||
Assert.Equal(1, rig.ShutdownSignalled);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Handle_signals_shutdown_exactly_once_even_on_error()
|
||||
{
|
||||
|
||||
+35
@@ -275,9 +275,44 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsSessionGone);
|
||||
Assert.Contains(((int)status).ToString(), result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_maps_to_SessionGone()
|
||||
{
|
||||
// 410 = the backend no longer knows this session. Not transient (no retry), not auth,
|
||||
// not poison, not a size problem — a distinct terminal kind the drain reacts to.
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "session unknown");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsTransient);
|
||||
Assert.False(result.IsAuthFailure);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.False(result.RequiresSplit);
|
||||
Assert.Contains("410", result.ErrorReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Gone_410_with_poison_body_is_still_SessionGone_never_poison()
|
||||
{
|
||||
// A poison body on 410 must not be honoured: dropping items + emitting
|
||||
// telemetry_upload_poisoned would only produce the next 410 (emit/reject loop).
|
||||
using var rig = new Rig();
|
||||
rig.Handler.QueueStatus(HttpStatusCode.Gone, body: "{\"poison\":true,\"rejectedRowKeys\":[\"rk1\"],\"reason\":\"x\"}");
|
||||
|
||||
var result = await rig.Sut.UploadBatchAsync(new[] { NewEventItem(1) }, CancellationToken.None);
|
||||
|
||||
Assert.True(result.IsSessionGone);
|
||||
Assert.False(result.IsPoison);
|
||||
Assert.Null(result.PoisonRowKeys);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(HttpStatusCode.RequestTimeout)] // 408
|
||||
[InlineData((HttpStatusCode)429)] // TooManyRequests
|
||||
|
||||
+6
@@ -68,6 +68,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueSessionGone(string reason = "session gone: http 410")
|
||||
{
|
||||
_script.Enqueue(_ => UploadResult.SessionGone(reason));
|
||||
return this;
|
||||
}
|
||||
|
||||
public FakeBackendTelemetryUploader QueueThrow(Exception ex)
|
||||
{
|
||||
_script.Enqueue(_ => throw ex);
|
||||
|
||||
@@ -10,10 +10,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetrySpoolTests
|
||||
{
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
private const string ForeignKey = "tenant_old-session";
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey = "row-1") => EventDraft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft EventDraft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: "{\"EventType\":\"agent_started\"}",
|
||||
isSessionScoped: true);
|
||||
@@ -195,6 +200,86 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
// ============================================================ Startup partition-key filter
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_leading_pending_items_of_another_session_and_persists_cursor()
|
||||
{
|
||||
// Same spool directory, new session id: the previous session's unsent items would only
|
||||
// draw a 410 that must not end the new session. They are dropped on startup; the own
|
||||
// item behind them stays pending.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.LastAssignedItemId);
|
||||
Assert.Equal(1, s2.PendingItemCount);
|
||||
Assert.Equal("own", Assert.Single(s2.Peek(10)).RowKey);
|
||||
|
||||
// Cursor persisted: a later start without the filter sees the same tail with the same bytes.
|
||||
var s3 = new TelemetrySpool(tmp.Path, Clock());
|
||||
Assert.Equal(1, s3.LastUploadedItemId);
|
||||
Assert.Equal(1, s3.PendingItemCount);
|
||||
Assert.Equal(s3.PendingBytes, s2.PendingBytes);
|
||||
|
||||
var next = s2.Enqueue(EventDraft("next", OwnKey));
|
||||
Assert.Equal(3, next.TelemetryItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_drops_nothing_when_a_foreign_item_follows_an_own_item()
|
||||
{
|
||||
// Impossible by construction (one process = one session); if it happens the order is
|
||||
// not trustworthy, so the filter keeps everything instead of guessing.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("own", OwnKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_keeps_pending_items_that_carry_the_expected_key()
|
||||
{
|
||||
// WhiteGlove Part 2 reuses the Part-1 session id: its leftovers carry the expected key.
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("a", OwnKey));
|
||||
s1.Enqueue(EventDraft("b", OwnKey));
|
||||
s1.Enqueue(EventDraft("c", OwnKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock(), expectedPartitionKey: OwnKey);
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(3, s2.PendingItemCount);
|
||||
Assert.Equal(s1.PendingBytes, s2.PendingBytes);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Ctor_without_expected_key_keeps_foreign_items()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var s1 = new TelemetrySpool(tmp.Path, Clock());
|
||||
s1.Enqueue(EventDraft("old-a", ForeignKey));
|
||||
s1.Enqueue(EventDraft("old-b", ForeignKey));
|
||||
|
||||
var s2 = new TelemetrySpool(tmp.Path, Clock());
|
||||
|
||||
Assert.Equal(-1, s2.LastUploadedItemId);
|
||||
Assert.Equal(2, s2.PendingItemCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Enqueue_with_RequiresImmediateFlush_raises_ImmediateFlushRequested()
|
||||
{
|
||||
|
||||
+118
-2
@@ -10,10 +10,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
{
|
||||
public sealed class TelemetryUploadOrchestratorTests
|
||||
{
|
||||
private static TelemetryItemDraft Draft(string rowKey) =>
|
||||
private const string OwnKey = "tenant_session";
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey) => Draft(rowKey, OwnKey);
|
||||
|
||||
private static TelemetryItemDraft Draft(string rowKey, string partitionKey) =>
|
||||
new TelemetryItemDraft(
|
||||
kind: TelemetryItemKind.Event,
|
||||
partitionKey: "tenant_session",
|
||||
partitionKey: partitionKey,
|
||||
rowKey: rowKey,
|
||||
payloadJson: $"{{\"rk\":\"{rowKey}\"}}",
|
||||
isSessionScoped: true);
|
||||
@@ -157,6 +161,118 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Transport
|
||||
Assert.Equal(1, blockedCount);
|
||||
}
|
||||
|
||||
// ============================================================ HTTP 410 — session gone
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_current_session_stops_drain_retains_cursor_and_raises_once()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("a"));
|
||||
spool.Enqueue(Draft("b"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
var gone = new System.Collections.Generic.List<string>();
|
||||
int blockedCount = 0;
|
||||
PoisonReport? poison = null;
|
||||
sut.SessionGone += (_, reason) => gone.Add(reason);
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
sut.BatchPoisoned += (_, p) => poison = p;
|
||||
|
||||
var first = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(first.Success);
|
||||
Assert.Equal(1, first.FailedBatches);
|
||||
Assert.Equal(0, first.UploadedItems);
|
||||
Assert.Equal(1, uploader.CallCount); // no retries — nothing can make a 410 succeed
|
||||
Assert.Equal(-1, sut.LastUploadedItemId); // retained — items stay on disk for the diag ZIP
|
||||
Assert.Single(gone);
|
||||
Assert.Contains("410", gone[0]);
|
||||
Assert.Equal(0, blockedCount); // not a "blocked" pipeline
|
||||
Assert.Null(poison);
|
||||
|
||||
// One-shot: the next drain hits the 410 again but does not raise again.
|
||||
var second = await sut.DrainAllAsync();
|
||||
Assert.False(second.Success);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
Assert.Single(gone);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_for_leading_foreign_items_drops_only_them_and_continues()
|
||||
{
|
||||
// Leftovers of a previous session id at the head of the spool (defence in depth — the
|
||||
// spool filters these at startup). A 410 for them must not end the current session:
|
||||
// drop exactly the leading foreign run, then carry on with the current session's items.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("old-a", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("old-b", "tenant_old-session"));
|
||||
spool.Enqueue(Draft("new", OwnKey));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueOk();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
int goneCount = 0, blockedCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
sut.UploadBlocked += (_, __) => blockedCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal(0, result.FailedBatches);
|
||||
Assert.Equal(1, result.UploadedItems);
|
||||
Assert.Equal(2, uploader.CallCount);
|
||||
Assert.Equal(3, uploader.Received[0].Count); // first batch carried all three
|
||||
Assert.Equal("new", Assert.Single(uploader.Received[1]).RowKey); // retry carried only the own item
|
||||
Assert.Equal(2, sut.LastUploadedItemId);
|
||||
Assert.Equal(0, goneCount);
|
||||
Assert.Equal(0, blockedCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_is_not_latched_before_a_handler_is_subscribed()
|
||||
{
|
||||
// The host subscribes after EnrollmentOrchestrator.Start; a 410 on the very first drain
|
||||
// must still reach it on the next drain instead of being swallowed by the one-shot guard.
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("only"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock(), currentPartitionKey: OwnKey);
|
||||
|
||||
await sut.DrainAllAsync(); // nobody listening yet
|
||||
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
await sut.DrainAllAsync();
|
||||
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionGone_without_a_current_key_treats_every_batch_as_own()
|
||||
{
|
||||
using var tmp = new TempDirectory();
|
||||
var spool = new TelemetrySpool(tmp.Path, Clock());
|
||||
spool.Enqueue(Draft("x", "tenant_whatever"));
|
||||
|
||||
var uploader = new FakeBackendTelemetryUploader().QueueSessionGone();
|
||||
using var sut = new TelemetryUploadOrchestrator(spool, uploader, Clock());
|
||||
int goneCount = 0;
|
||||
sut.SessionGone += (_, __) => goneCount++;
|
||||
|
||||
var result = await sut.DrainAllAsync();
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.Equal(1, goneCount);
|
||||
Assert.Equal(-1, sut.LastUploadedItemId);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task TooLarge_splits_batch_and_uploads_everything()
|
||||
{
|
||||
|
||||
@@ -21,7 +21,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _cacheFilePath;
|
||||
private readonly EmergencyReporter _emergencyReporter;
|
||||
private readonly DistressReporter _distressReporter;
|
||||
private readonly AuthFailureTracker _authFailureTracker;
|
||||
|
||||
private AgentConfigResponse _currentConfig;
|
||||
@@ -92,13 +91,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Configuration
|
||||
/// </summary>
|
||||
public bool RecoveredAfterFetchFailure { get; private set; }
|
||||
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, DistressReporter distressReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
public RemoteConfigService(BackendApiClient apiClient, string tenantId, AgentLogger logger, EmergencyReporter emergencyReporter = null, AuthFailureTracker authFailureTracker = null)
|
||||
{
|
||||
_apiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_emergencyReporter = emergencyReporter;
|
||||
_distressReporter = distressReporter;
|
||||
_authFailureTracker = authFailureTracker;
|
||||
|
||||
var cacheDir = Environment.ExpandEnvironmentVariables(@"%ProgramData%\AutopilotMonitor\Config");
|
||||
|
||||
+4
-4
@@ -990,7 +990,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// for each detection. When HS-NEW-RESULT later fires with the full payload, the UI
|
||||
/// reducer's dataCompleteness scoring keeps the more complete entry.
|
||||
/// </summary>
|
||||
private void HandleHealthScriptDetectionResult(Match match, Dictionary<string, string> parameters)
|
||||
private void HandleHealthScriptDetectionResult(Match match)
|
||||
{
|
||||
var id = match.Groups["id"]?.Value;
|
||||
var compliance = match.Groups["compliance"]?.Value;
|
||||
@@ -1112,7 +1112,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
sample,
|
||||
@"\[HS\] the (?<part>pre|post)-remdiation detection script compliance result for (?<id>[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}) is (?<compliance>True|False)",
|
||||
System.Text.RegularExpressions.RegexOptions.IgnoreCase);
|
||||
HandleHealthScriptDetectionResult(match, new Dictionary<string, string>());
|
||||
HandleHealthScriptDetectionResult(match);
|
||||
}
|
||||
|
||||
/// <summary>Test seam: seed the per-policy health-script cycle start timestamp as if an
|
||||
@@ -1125,7 +1125,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
_healthScriptStartTimes[policyId] = (startedAtUtc, provenance);
|
||||
}
|
||||
|
||||
private void HandleHealthScriptResult(Match match, Dictionary<string, string> parameters)
|
||||
private void HandleHealthScriptResult(Match match)
|
||||
{
|
||||
var json = match.Groups["json"]?.Value;
|
||||
if (string.IsNullOrEmpty(json))
|
||||
@@ -1141,7 +1141,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
/// <summary>
|
||||
/// Test seam that bypasses the regex-match plumbing and feeds a raw <c>[HS] new result</c>
|
||||
/// JSON payload directly into the parser. Production code path runs through
|
||||
/// <see cref="HandleHealthScriptResult(Match, Dictionary{string, string})"/>; this method
|
||||
/// <see cref="HandleHealthScriptResult(Match)"/>; this method
|
||||
/// exists so unit tests can exercise the JSON parser + 1-3 phase emit logic without
|
||||
/// having to write CMTrace log files into a temp directory.
|
||||
/// </summary>
|
||||
|
||||
+2
-2
@@ -882,11 +882,11 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.Ime
|
||||
break;
|
||||
|
||||
case "healthscriptresult":
|
||||
HandleHealthScriptResult(match, pattern.Parameters);
|
||||
HandleHealthScriptResult(match);
|
||||
break;
|
||||
|
||||
case "healthscriptdetectionresult":
|
||||
HandleHealthScriptDetectionResult(match, pattern.Parameters);
|
||||
HandleHealthScriptDetectionResult(match);
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
-2
@@ -237,7 +237,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
var rebootUri = ExtractRebootUri(ParseEventData(xml), description);
|
||||
|
||||
ProcessEvent(
|
||||
eventId: record.Id,
|
||||
recordId: recordId,
|
||||
timeCreatedUtc: record.TimeCreated?.ToUniversalTime(),
|
||||
rebootUri: rebootUri,
|
||||
@@ -251,7 +250,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// — after the debounce quiet-period, at backfill completion, or at <see cref="Stop"/>.
|
||||
/// </summary>
|
||||
internal void ProcessEvent(
|
||||
int eventId,
|
||||
long recordId,
|
||||
DateTime? timeCreatedUtc,
|
||||
string rebootUri,
|
||||
|
||||
+2
-2
@@ -1382,7 +1382,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
// window (user clicks "Try again" / IME re-sync flips the subcategory back
|
||||
// to inProgress) — firing the stale args would terminate the agent on a
|
||||
// failure Windows no longer reports, and the retry outcome is lost.
|
||||
recovered = IsSettledFailureRecovered(categoryName, args);
|
||||
recovered = IsSettledFailureRecovered(categoryName);
|
||||
if (recovered)
|
||||
{
|
||||
if (args.FailedSubcategory != null
|
||||
@@ -1447,7 +1447,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
/// and must fire (with the latched args — statusText/HRESULT of the new subcategory
|
||||
/// were never captured). Caller holds <see cref="_stateLock"/>.
|
||||
/// </summary>
|
||||
private bool IsSettledFailureRecovered(string categoryName, EspFailureDetectedEventArgs args)
|
||||
private bool IsSettledFailureRecovered(string categoryName)
|
||||
{
|
||||
if (_lastCategorySucceeded != null
|
||||
&& _lastCategorySucceeded.TryGetValue(categoryName, out var succeeded)
|
||||
|
||||
+2
-2
@@ -241,7 +241,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
if (alreadyFired)
|
||||
continue;
|
||||
|
||||
RunProbe(probeIndex, thresholdMinutes, idleMinutes, genAtStart);
|
||||
RunProbe(probeIndex, idleMinutes, genAtStart);
|
||||
|
||||
lock (_stateLock)
|
||||
{
|
||||
@@ -253,7 +253,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Enrollment.SystemSignals
|
||||
}
|
||||
}
|
||||
|
||||
private void RunProbe(int probeIndex, int thresholdMinutes, double idleMinutes, int generation)
|
||||
private void RunProbe(int probeIndex, double idleMinutes, int generation)
|
||||
{
|
||||
var started = DateTime.UtcNow;
|
||||
var result = new ProbeResult { ProbeIndex = probeIndex, IdleMinutes = Math.Round(idleMinutes, 1) };
|
||||
|
||||
+3
-3
@@ -1052,7 +1052,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
if (!IsAllowedBlobUploadUrl(blobUploadUrl))
|
||||
{
|
||||
_logger.Warning(
|
||||
$"Blob upload rejected: URL is not an allowed Azure Blob Storage endpoint " +
|
||||
$"Blob upload of {blobName} rejected: URL is not an allowed Azure Blob Storage endpoint " +
|
||||
$"(expected https://*.blob.core.windows.net or sovereign-cloud equivalent). " +
|
||||
$"URL prefix: {BuildSasUrlPrefix(blobUploadUrl)}");
|
||||
return (false, "url_host_rejected");
|
||||
@@ -1086,7 +1086,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
// Auth errors (401/403) are permanent — SAS token invalid or expired, retrying won't help
|
||||
if (statusCode == 401 || statusCode == 403)
|
||||
{
|
||||
_logger.Warning($"Blob upload auth error (not retryable): {lastErrorCode} - {responseBody}");
|
||||
_logger.Warning($"Blob upload of {blobName}: auth error (not retryable): {lastErrorCode} - {responseBody}");
|
||||
return (false, lastErrorCode);
|
||||
}
|
||||
|
||||
@@ -1106,7 +1106,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
_logger.Warning($"Diagnostics package upload failed after {maxRetries} attempts");
|
||||
_logger.Warning($"Diagnostics package upload failed after {maxRetries} attempts: {blobName}");
|
||||
return (false, lastErrorCode);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,7 +145,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
/// On any failure, returns normally so the current version continues.
|
||||
/// </summary>
|
||||
/// <param name="forceUpdate">
|
||||
/// When true, bypasses the IsNewerVersion check — used by the runtime-hash-mismatch trigger
|
||||
/// When true, bypasses the version comparison — used by the runtime-hash-mismatch trigger
|
||||
/// to cover hotfixes that reuse the same version number.
|
||||
/// </param>
|
||||
/// <param name="triggerReason">
|
||||
@@ -517,16 +517,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two version strings. Returns true if latest is newer than current.
|
||||
/// Strips SemVer suffixes (+metadata, -prerelease) before parsing because
|
||||
/// System.Version cannot handle them (e.g. "1.0.386+b7f8d3c..." would fail).
|
||||
/// </summary>
|
||||
private static bool IsNewerVersion(string current, string latest)
|
||||
{
|
||||
return CompareVersions(current, latest) > 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Compares two version strings. Returns negative when latest is older than current,
|
||||
/// zero when equal, positive when latest is newer than current. Returns
|
||||
|
||||
-5
@@ -827,11 +827,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Gather
|
||||
sb.Append(value);
|
||||
}
|
||||
|
||||
private EventSeverity ParseSeverity(string severity)
|
||||
{
|
||||
return GatherRuleContext.ParseSeverity(severity);
|
||||
}
|
||||
|
||||
private void StopAllTimers()
|
||||
{
|
||||
foreach (var timer in _intervalTimers.Values)
|
||||
|
||||
+2
-5
@@ -104,7 +104,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
private bool _preinstalledReported; // office_preinstalled_detected already emitted (emit-once guard)
|
||||
private DateTime? _startedAtUtc;
|
||||
private string? _startedTrigger;
|
||||
private OfficeDoSample? _lastDo;
|
||||
private OfficeDoSample? _peakDo; // highest-bytes sample seen — basis for the completed doSummary
|
||||
|
||||
public OfficeInstallDetector(
|
||||
@@ -252,7 +251,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
// _peakDo). The seed is persisted directly (it is the lifecycle's first DO sample).
|
||||
if (_state == DetectorState.Active)
|
||||
{
|
||||
_lastDo = sample;
|
||||
_peakDo = sample;
|
||||
PersistState(OfficeInstallStateData.StateActive);
|
||||
}
|
||||
@@ -261,7 +259,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
|
||||
// Active: no progress event — the sample only updates the DO summary data. A grown
|
||||
// peak is re-persisted (throttled) so a mid-install agent restart keeps the doSummary.
|
||||
_lastDo = sample;
|
||||
var peakGrew = _peakDo == null || sample.TotalBytesDownloaded > _peakDo.TotalBytesDownloaded;
|
||||
if (peakGrew) _peakDo = sample;
|
||||
if (peakGrew && _statePersistence != null
|
||||
@@ -380,7 +377,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
// wrapper that re-runs C2R) the watcher's initial scan completes the lifecycle synchronously.
|
||||
// Emitting started first guarantees the correct order (started → completed); otherwise a
|
||||
// pre-installed Office produced completed-before-started (field session a7525e97).
|
||||
EmitLifecycle(Constants.EventTypes.OfficeInstallStarted, snap, EventSeverity.Info, PhaseOf(snap), isTerminal: false);
|
||||
EmitLifecycle(Constants.EventTypes.OfficeInstallStarted, snap, EventSeverity.Info, InFlightPhase, isTerminal: false);
|
||||
// Persist Active BEFORE ObserveInstallationPath: when Office is already on disk the
|
||||
// host's binary watcher completes the lifecycle synchronously from inside that callback
|
||||
// (and persists Completed) — persisting Active afterwards would overwrite the terminal.
|
||||
@@ -508,7 +505,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Office
|
||||
|
||||
// StreamingFinished does not exist in the C2R registry, so the in-flight phase is reported as a
|
||||
// single "Installing"; the terminal completion uses "Completed".
|
||||
private static string PhaseOf(OfficeC2RSnapshot snap) => "Installing";
|
||||
private const string InFlightPhase = "Installing";
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Payload
|
||||
|
||||
+3
-12
@@ -26,7 +26,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly string _apiBaseUrl;
|
||||
|
||||
private NetworkStateSnapshot _lastKnownState;
|
||||
private Timer _debounceTimer;
|
||||
@@ -54,14 +53,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
string apiBaseUrl = null)
|
||||
AgentLogger logger)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_apiBaseUrl = apiBaseUrl;
|
||||
}
|
||||
|
||||
public void Start()
|
||||
@@ -225,15 +222,9 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Periodic
|
||||
{
|
||||
if (_disposed || _connectivityCheckClient == null) return;
|
||||
|
||||
// Build endpoint list: static MDM endpoints + dynamic backend URL
|
||||
// Static MDM endpoints only. The backend API stays out on purpose: enrollment does not
|
||||
// need it, so its outage must not raise a connectivity alarm.
|
||||
var endpoints = new List<(string Name, string Url)>(MdmEndpoints);
|
||||
// Optionally add the Autopilot Monitor API endpoint, but we don't want to cause false alarms if it's down or unreachable,
|
||||
// since it's not critical for enrollment success and may be available only after certain network changes (e.g. if it's behind a captive portal).
|
||||
// For now, we can omit it from the critical checks.
|
||||
// if (!string.IsNullOrEmpty(_apiBaseUrl))
|
||||
// {
|
||||
// endpoints.Add(("Autopilot Monitor API", _apiBaseUrl));
|
||||
// }
|
||||
|
||||
// Run all checks in parallel
|
||||
var tasks = endpoints.Select(ep => CheckEndpointAsync(ep.Name, ep.Url)).ToArray();
|
||||
|
||||
+1
-4
@@ -69,20 +69,17 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Telemetry.Provisioning
|
||||
private readonly string _tenantId;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock? _clock;
|
||||
|
||||
public ProvisioningPackageCollector(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
InformationalEventPost post,
|
||||
AgentLogger logger,
|
||||
IClock? clock = null)
|
||||
AgentLogger logger)
|
||||
{
|
||||
_sessionId = sessionId ?? throw new ArgumentNullException(nameof(sessionId));
|
||||
_tenantId = tenantId ?? throw new ArgumentNullException(nameof(tenantId));
|
||||
_post = post ?? throw new ArgumentNullException(nameof(post));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_clock = clock;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -2,6 +2,7 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.DecisionCore.Engine;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
@@ -14,6 +15,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
/// - Same error key (ErrorType + HTTP status code) is only ever sent once per session
|
||||
/// - At most <see cref="MaxReportsPerSession"/> emergency reports total per session
|
||||
/// - At most one report every <see cref="MinIntervalMinutes"/> minutes regardless of error type
|
||||
/// (the terminal <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/> report is exempt)
|
||||
///
|
||||
/// A failure in this reporter is silently swallowed — it must never cascade into the main loop.
|
||||
/// </summary>
|
||||
@@ -32,6 +34,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
private readonly string _tenantId;
|
||||
private readonly string _agentVersion;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly IClock _clock;
|
||||
|
||||
private readonly HashSet<string> _reportedKeys = new HashSet<string>();
|
||||
private readonly object _antiFloodLock = new object();
|
||||
@@ -43,13 +46,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
string agentVersion,
|
||||
AgentLogger logger)
|
||||
AgentLogger logger,
|
||||
IClock clock = null)
|
||||
{
|
||||
_apiClient = apiClient;
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_agentVersion = agentVersion;
|
||||
_logger = logger;
|
||||
_clock = clock ?? SystemClock.Instance;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -100,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
// All anti-flood checks are protected by a lock to prevent race conditions when
|
||||
// TrySendAsync is called concurrently (fire-and-forget from multiple threads).
|
||||
var key = $"{errorType}:{httpStatusCode}";
|
||||
var now = _clock.UtcNow;
|
||||
int currentReportCount;
|
||||
DateTime? previousLastReportTime;
|
||||
|
||||
@@ -117,8 +123,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
return;
|
||||
}
|
||||
|
||||
if (_lastReportTime != null &&
|
||||
(DateTime.UtcNow - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
// The session-unknown report is terminal and sent at most once per process (the
|
||||
// termination follows immediately): a routine report minutes earlier must not cost
|
||||
// the last word about the session. Only the cooldown is waived; key dedup and the
|
||||
// per-session maximum still apply.
|
||||
var exemptFromCooldown = errorType == AgentErrorType.TelemetryRejectedSessionUnknown;
|
||||
if (!exemptFromCooldown && _lastReportTime != null &&
|
||||
(now - _lastReportTime.Value).TotalMinutes < MinIntervalMinutes)
|
||||
{
|
||||
_logger?.Debug($"[EmergencyChannel] Suppressed (cooldown active, last report at {_lastReportTime.Value:HH:mm:ss}): {key}");
|
||||
return;
|
||||
@@ -127,7 +138,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
previousLastReportTime = _lastReportTime;
|
||||
_reportedKeys.Add(key);
|
||||
_reportCount++;
|
||||
_lastReportTime = DateTime.UtcNow;
|
||||
_lastReportTime = now;
|
||||
currentReportCount = _reportCount;
|
||||
}
|
||||
|
||||
@@ -145,7 +156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
|
||||
SessionAgeHours = sessionAgeHours,
|
||||
PriorRegistrationFailure = priorRegistrationFailure,
|
||||
AgentVersion = _agentVersion,
|
||||
Timestamp = DateTime.UtcNow
|
||||
Timestamp = now
|
||||
};
|
||||
|
||||
var delivered = false;
|
||||
|
||||
@@ -377,8 +377,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
if (analyzers.KeepAwakeDuringUserEsp)
|
||||
{
|
||||
hosts.Add(new UserEspKeepAwakeHost(
|
||||
sessionId: sessionId,
|
||||
tenantId: tenantId,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger,
|
||||
@@ -648,8 +646,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
tenantId: tenantId,
|
||||
ingress: ingress,
|
||||
clock: clock,
|
||||
logger: logger,
|
||||
apiBaseUrl: _agentConfig.ApiBaseUrl));
|
||||
logger: logger));
|
||||
|
||||
// Live AC/battery watcher — WMI Win32_PowerManagementEvent push (no polling), emits
|
||||
// power_state_change on AC↔battery transitions and 50/30/15 % downward threshold
|
||||
|
||||
@@ -371,8 +371,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
// 3) Telemetry-Transport. Batch size flows from AgentConfiguration.MaxBatchSize
|
||||
// via Program.cs (P1 fix: previously the remote-config knob was merged but
|
||||
// never applied, so tenants saw drainInterval/MaxBatchSize have no effect).
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger);
|
||||
_transport = new TelemetryUploadOrchestrator(_spool, _uploader, _clock, batchSize: _uploadBatchSize);
|
||||
// The partition key names the registered session: the spool drops leftovers of a
|
||||
// previous session id on startup, and the orchestrator tells a 410 for this session
|
||||
// (SessionGone → host terminates) from a 410 for such leftovers (dropped).
|
||||
var partitionKey = TelemetryPartitionKey.ForSession(_tenantId, _sessionId);
|
||||
_spool = new TelemetrySpool(_transportDirectory, _clock, _logger, expectedPartitionKey: partitionKey);
|
||||
_transport = new TelemetryUploadOrchestrator(
|
||||
_spool, _uploader, _clock, batchSize: _uploadBatchSize, currentPartitionKey: partitionKey, logger: _logger);
|
||||
|
||||
// TRACE-H1 — surface skipped oversized items + blocked-pipeline conditions as timeline
|
||||
// events. Raised after the drain guard releases, so posting back through the ingress
|
||||
|
||||
+25
-1
@@ -73,13 +73,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
/// </summary>
|
||||
public string? StopReason { get; }
|
||||
|
||||
/// <summary>
|
||||
/// The <c>origin</c> param of a server-requested <c>terminate_session</c> action
|
||||
/// (<see cref="TerminationOrigins"/>); null for every termination the agent decided
|
||||
/// itself. <see cref="TerminationOrigins.SessionGone"/> makes the termination handler
|
||||
/// skip the spool drain and the diagnostics upload — the backend no longer knows the
|
||||
/// session, so both could only fail.
|
||||
/// </summary>
|
||||
public string? Origin { get; }
|
||||
|
||||
public EnrollmentTerminatedEventArgs(
|
||||
EnrollmentTerminationReason reason,
|
||||
EnrollmentTerminationOutcome outcome,
|
||||
string? stageName,
|
||||
DateTime terminatedAtUtc,
|
||||
string? details = null,
|
||||
string? stopReason = null)
|
||||
string? stopReason = null,
|
||||
string? origin = null)
|
||||
{
|
||||
Reason = reason;
|
||||
Outcome = outcome;
|
||||
@@ -87,6 +97,20 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
TerminatedAtUtc = terminatedAtUtc;
|
||||
Details = details;
|
||||
StopReason = stopReason;
|
||||
Origin = origin;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Values of the <c>origin</c> param on a synthesised <c>terminate_session</c>
|
||||
/// <c>ServerAction</c>, carried into <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
|
||||
/// </summary>
|
||||
public static class TerminationOrigins
|
||||
{
|
||||
/// <summary>Administrator kill signal (<c>DeviceKillSignal</c> on a 2xx telemetry response).</summary>
|
||||
public const string KillSignal = "kill_signal";
|
||||
|
||||
/// <summary>The backend answered HTTP 410 for the current session's telemetry.</summary>
|
||||
public const string SessionGone = "session_gone";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private readonly DesktopArrivalDetector _detector;
|
||||
private readonly DesktopArrivalDetectorAdapter _adapter;
|
||||
private readonly AgentLogger _logger;
|
||||
private int _disposed;
|
||||
|
||||
public DesktopArrivalHost(
|
||||
@@ -30,7 +29,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
string? tenantId = null,
|
||||
Action<string>? onRealUserOwnerObserved = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_detector = new DesktopArrivalDetector(logger, noCandidateTimeoutMinutes);
|
||||
_adapter = new DesktopArrivalDetectorAdapter(_detector, ingress, clock);
|
||||
|
||||
|
||||
@@ -26,14 +26,13 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
string tenantId,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger,
|
||||
string? apiBaseUrl)
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_logger = logger;
|
||||
var post = new InformationalEventPost(ingress, clock);
|
||||
_detector = new NetworkChangeDetector(sessionId, tenantId, post, logger, apiBaseUrl);
|
||||
_detector = new NetworkChangeDetector(sessionId, tenantId, post, logger);
|
||||
}
|
||||
|
||||
public void Start()
|
||||
|
||||
+1
-1
@@ -60,7 +60,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
|
||||
var post = new InformationalEventPost(ingress, clock, logger);
|
||||
_collector = new ProvisioningPackageCollector(sessionId, tenantId, post, logger, clock);
|
||||
_collector = new ProvisioningPackageCollector(sessionId, tenantId, post, logger);
|
||||
_observableIngress = ingress as SignalIngress;
|
||||
}
|
||||
|
||||
|
||||
@@ -91,8 +91,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
|
||||
private static readonly string AccountSetupPhaseName = EnrollmentPhase.AccountSetup.ToString();
|
||||
|
||||
private readonly string _sessionId;
|
||||
private readonly string _tenantId;
|
||||
private readonly AgentLogger _logger;
|
||||
private readonly InformationalEventPost _post;
|
||||
private readonly KeepAwakeController _controller;
|
||||
@@ -112,8 +110,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
private int _disposed;
|
||||
|
||||
public UserEspKeepAwakeHost(
|
||||
string sessionId,
|
||||
string tenantId,
|
||||
ISignalIngressSink ingress,
|
||||
IClock clock,
|
||||
AgentLogger logger,
|
||||
@@ -125,8 +121,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Orchestration
|
||||
{
|
||||
if (ingress == null) throw new ArgumentNullException(nameof(ingress));
|
||||
if (clock == null) throw new ArgumentNullException(nameof(clock));
|
||||
_sessionId = sessionId ?? string.Empty;
|
||||
_tenantId = tenantId ?? string.Empty;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_post = new InformationalEventPost(ingress, clock, logger);
|
||||
_controller = controller ?? new KeepAwakeController(logger);
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Events
|
||||
_sequenceCounter = sequenceCounter ?? throw new ArgumentNullException(nameof(sequenceCounter));
|
||||
_sessionId = sessionId;
|
||||
_tenantId = tenantId;
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
_traceEventsEnabled = traceEventsEnabled;
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Signals
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+1
-1
@@ -43,7 +43,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Telemetry.Transitions
|
||||
if (string.IsNullOrEmpty(sessionId)) throw new ArgumentException("SessionId is mandatory.", nameof(sessionId));
|
||||
if (string.IsNullOrEmpty(tenantId)) throw new ArgumentException("TenantId is mandatory.", nameof(tenantId));
|
||||
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
|
||||
_partitionKey = $"{tenantId}_{sessionId}";
|
||||
_partitionKey = TelemetryPartitionKey.ForSession(tenantId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+29
-6
@@ -306,7 +306,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
});
|
||||
|
||||
DelayLateEventGrace();
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
// Option 2 (WG Part 1 graceful-exit hardening, 2026-04-30): write the
|
||||
// clean-exit marker BEFORE _signalShutdown returns control to the main
|
||||
@@ -329,7 +329,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
|
||||
RunUploadDiagnosticsWithEvents(args);
|
||||
WriteEnrollmentCompleteMarker(args);
|
||||
RunStandaloneRebootIfRequested();
|
||||
RunStandaloneRebootIfRequested(args);
|
||||
RunSelfDestructIfAppropriate(args);
|
||||
// Option 2 (same hardening) — covers the standard Completed / Failed terminal
|
||||
// path too. Cleanup PowerShell does not touch this marker, so writing it before
|
||||
@@ -979,6 +979,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// </param>
|
||||
private void RunUploadDiagnosticsWithEvents(EnrollmentTerminatedEventArgs args, string suffixOverride = null)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
// The backend answered 410 for this session: the package would describe a session
|
||||
// that no longer exists, and its diagnostics_* events could never be accepted.
|
||||
_logger.Info("EnrollmentTerminationHandler: diagnostics upload skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
var mode = _configuration.DiagnosticsUploadMode ?? "Off";
|
||||
if (!_configuration.DiagnosticsUploadEnabled || string.Equals(mode, "Off", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
@@ -1124,7 +1132,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
/// but enables <c>RebootOnComplete</c>, the agent's final act is <c>shutdown.exe /r</c>
|
||||
/// with the configured delay, giving the user a visible countdown.
|
||||
/// </summary>
|
||||
private void RunStandaloneRebootIfRequested()
|
||||
private void RunStandaloneRebootIfRequested(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (_configuration.SelfDestructOnComplete) return;
|
||||
if (!_configuration.RebootOnComplete) return;
|
||||
@@ -1148,7 +1156,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
ImmediateUpload = true,
|
||||
});
|
||||
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -1185,7 +1193,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
// just above. The WG-Part-1 and standalone-reboot paths already drain first; this
|
||||
// closes the gap on the self-destruct path. Bounded + best-effort (never blocks
|
||||
// termination on a wedged backend).
|
||||
DrainSpool();
|
||||
DrainSpool(args);
|
||||
|
||||
var service = _cleanupServiceFactory();
|
||||
service.ExecuteSelfDestruct();
|
||||
@@ -1318,8 +1326,23 @@ namespace AutopilotMonitor.Agent.V2.Core.Termination
|
||||
}
|
||||
}
|
||||
|
||||
private void DrainSpool()
|
||||
/// <summary>
|
||||
/// <see cref="TerminationOrigins.SessionGone"/>: the backend no longer knows the session.
|
||||
/// Every drain would only collect more 410s against a bounded budget, and a diagnostics
|
||||
/// package would describe a deleted session — both are skipped; marker, cleanup and
|
||||
/// self-destruct run as usual.
|
||||
/// </summary>
|
||||
private static bool IsSessionGone(EnrollmentTerminatedEventArgs args) =>
|
||||
string.Equals(args.Origin, TerminationOrigins.SessionGone, StringComparison.Ordinal);
|
||||
|
||||
private void DrainSpool(EnrollmentTerminatedEventArgs args)
|
||||
{
|
||||
if (IsSessionGone(args))
|
||||
{
|
||||
_logger.Info("EnrollmentTerminationHandler: spool drain skipped — backend no longer knows this session.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Block briefly so pending events can land before the next destructive step
|
||||
// (shutdown.exe, self-destruct). Two phases share the same bounded budget:
|
||||
//
|
||||
|
||||
+10
@@ -165,6 +165,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
_logger?.Warning($"BackendTelemetryUploader: ingest TRANSIENT (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else if (result.IsSessionGone)
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest rejected — backend no longer knows this session (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger?.Error($"BackendTelemetryUploader: ingest PERMANENT fail (corr={correlationId}, items={items.Count}, durationMs={sw.ElapsedMilliseconds}, status={(int)response.StatusCode}): {result.ErrorReason}");
|
||||
@@ -208,6 +212,12 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
|
||||
case HttpStatusCode.RequestEntityTooLarge: // 413 — the data is fine, only the batch
|
||||
return UploadResult.TooLarge(shortReason); // size is wrong → split & retry (P1).
|
||||
|
||||
case HttpStatusCode.Gone: // 410 — the backend no longer knows this session.
|
||||
// Decided BEFORE the poison-body check: a poison body on 410 would make the
|
||||
// drain drop items and emit telemetry_upload_poisoned, which the backend
|
||||
// rejects with 410 again — an emit/reject loop for a session that is over.
|
||||
return UploadResult.SessionGone($"session gone: {shortReason}");
|
||||
}
|
||||
|
||||
if (statusCode >= 500 && statusCode <= 599) return UploadResult.Transient(shortReason);
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
#nullable enable
|
||||
namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
{
|
||||
/// <summary>
|
||||
/// The one composition of a session's Azure-Table partition key. The three emitters stamp
|
||||
/// it on every <see cref="TelemetryItem"/>; the spool and the upload orchestrator compare
|
||||
/// against it to tell the current session's items from leftovers of a previous session id.
|
||||
/// A drift between producer and comparer would silently drop or retain the wrong items.
|
||||
/// </summary>
|
||||
public static class TelemetryPartitionKey
|
||||
{
|
||||
public static string ForSession(string tenantId, string sessionId) => $"{tenantId}_{sessionId}";
|
||||
}
|
||||
}
|
||||
@@ -74,7 +74,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
public int ByteLength { get; }
|
||||
}
|
||||
|
||||
public TelemetrySpool(string directoryPath, IClock clock, Logging.AgentLogger? logger = null)
|
||||
/// <param name="expectedPartitionKey">
|
||||
/// Partition key of the session this process serves (<see cref="TelemetryPartitionKey.ForSession"/>).
|
||||
/// When set, pending items of another session id found at the head of the spool are dropped
|
||||
/// on startup — see <see cref="DropLeadingForeignPending"/>. Null disables the filter.
|
||||
/// </param>
|
||||
public TelemetrySpool(
|
||||
string directoryPath,
|
||||
IClock clock,
|
||||
Logging.AgentLogger? logger = null,
|
||||
string? expectedPartitionKey = null)
|
||||
{
|
||||
if (string.IsNullOrEmpty(directoryPath))
|
||||
{
|
||||
@@ -96,6 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// count as "pending" and need to be rehydrated into _pending.
|
||||
_lastUploadedItemId = _cursor.Load();
|
||||
_lastAssignedItemId = ScanSpoolAndRehydratePending();
|
||||
DropLeadingForeignPending(expectedPartitionKey);
|
||||
_peakPendingCount = _pending.Count;
|
||||
}
|
||||
|
||||
@@ -312,5 +322,68 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
}
|
||||
return highest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Startup filter: drops the LEADING run of pending items whose partition key is not
|
||||
/// <paramref name="expectedPartitionKey"/>, advancing and persisting the cursor past them.
|
||||
/// Such items are leftovers of a previous session id in the same spool directory; the
|
||||
/// backend answers 410 for them, and that 410 must never end the session this process
|
||||
/// serves. One process serves one session, so a foreign item AFTER an own item cannot
|
||||
/// occur — if it does, the order is not trustworthy and nothing is dropped. WhiteGlove
|
||||
/// Part 2 reuses the Part-1 session id, so its leftovers carry the expected key and stay.
|
||||
/// Called once from the ctor, after rehydration.
|
||||
/// </summary>
|
||||
private void DropLeadingForeignPending(string? expectedPartitionKey)
|
||||
{
|
||||
if (string.IsNullOrEmpty(expectedPartitionKey) || _pending.Count == 0) return;
|
||||
|
||||
var leadingForeign = 0;
|
||||
long lastForeignId = -1;
|
||||
var seenOwn = false;
|
||||
foreach (var entry in _pending)
|
||||
{
|
||||
var isForeign = !string.Equals(entry.Item.PartitionKey, expectedPartitionKey, StringComparison.Ordinal);
|
||||
if (!isForeign)
|
||||
{
|
||||
seenOwn = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seenOwn)
|
||||
{
|
||||
leadingForeign++;
|
||||
lastForeignId = entry.Item.TelemetryItemId;
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger?.Error(
|
||||
$"TelemetrySpool: pending itemId={entry.Item.TelemetryItemId} carries a foreign partition key after items of the current session — spool order is inconsistent; keeping all {_pending.Count} pending item(s).");
|
||||
return;
|
||||
}
|
||||
|
||||
if (leadingForeign == 0) return;
|
||||
|
||||
try
|
||||
{
|
||||
_cursor.Save(lastForeignId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Keep the items rather than risk a cursor that disagrees with disk; the drain
|
||||
// loop drops the same leading run on its first 410 (defence in depth).
|
||||
_logger?.Warning($"TelemetrySpool: could not persist the cursor past {leadingForeign} foreign pending item(s): {ex.Message}");
|
||||
return;
|
||||
}
|
||||
|
||||
_lastUploadedItemId = lastForeignId;
|
||||
for (var i = 0; i < leadingForeign; i++)
|
||||
{
|
||||
_pendingBytes -= _pending.First!.Value.ByteLength;
|
||||
_pending.RemoveFirst();
|
||||
}
|
||||
|
||||
_logger?.Warning(
|
||||
$"TelemetrySpool: dropped {leadingForeign} pending item(s) of a previous session id (cursor advanced to {lastForeignId}, {_pending.Count} pending item(s) of the current session kept).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+81
-3
@@ -28,7 +28,6 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
private readonly ITelemetrySpool _spool;
|
||||
private readonly IBackendTelemetryUploader _uploader;
|
||||
private readonly IClock _clock;
|
||||
private readonly int _batchSize;
|
||||
private readonly IReadOnlyList<TimeSpan> _retryBackoffs;
|
||||
private readonly SemaphoreSlim _drainGuard = new SemaphoreSlim(1, 1);
|
||||
private bool _disposed;
|
||||
@@ -45,22 +44,39 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
// telemetry_upload_blocked exactly once per process rather than on every 30 s drain.
|
||||
private bool _uploadBlockedReported;
|
||||
|
||||
// Partition key of the session this process serves; null = every batch counts as own.
|
||||
private readonly string? _currentPartitionKey;
|
||||
private readonly Logging.AgentLogger? _logger;
|
||||
|
||||
// One-shot guard for SessionGone. Latched only when a handler is subscribed: the host
|
||||
// wires the handler after EnrollmentOrchestrator.Start, and a 410 on the very first
|
||||
// drain must not be swallowed before anyone listens.
|
||||
private bool _sessionGoneReported;
|
||||
|
||||
/// <param name="currentPartitionKey">
|
||||
/// <see cref="TelemetryPartitionKey.ForSession"/> of the registered session. A 410 on a
|
||||
/// batch that starts with this key ends the session (<see cref="SessionGone"/>); a 410 on
|
||||
/// a batch that starts with any other key only drops those leftover items.
|
||||
/// </param>
|
||||
public TelemetryUploadOrchestrator(
|
||||
ITelemetrySpool spool,
|
||||
IBackendTelemetryUploader uploader,
|
||||
IClock clock,
|
||||
int batchSize = 100,
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null)
|
||||
IReadOnlyList<TimeSpan>? retryBackoffs = null,
|
||||
string? currentPartitionKey = null,
|
||||
Logging.AgentLogger? logger = null)
|
||||
{
|
||||
_spool = spool ?? throw new ArgumentNullException(nameof(spool));
|
||||
_uploader = uploader ?? throw new ArgumentNullException(nameof(uploader));
|
||||
_clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
|
||||
if (batchSize <= 0) throw new ArgumentOutOfRangeException(nameof(batchSize), "BatchSize must be positive.");
|
||||
_batchSize = batchSize;
|
||||
_effectiveBatchSize = batchSize;
|
||||
|
||||
_retryBackoffs = retryBackoffs ?? DefaultRetryBackoffs;
|
||||
_currentPartitionKey = currentPartitionKey;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -97,6 +113,15 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// </summary>
|
||||
public event EventHandler<string>? UploadBlocked;
|
||||
|
||||
/// <summary>
|
||||
/// Raised once per process (after the drain guard releases) when the backend answered
|
||||
/// HTTP 410 for a batch of the CURRENT session: the session is over server-side (never
|
||||
/// registered, deleted, or being deleted) and no retry can succeed. The batch is retained
|
||||
/// (cursor stays) and the drain stops; the host reports once over the emergency channel
|
||||
/// and terminates the agent with self-destruct. Carries the backend error reason.
|
||||
/// </summary>
|
||||
public event EventHandler<string>? SessionGone;
|
||||
|
||||
/// <summary>
|
||||
/// <c>true</c> when the last successful upload returned <see cref="UploadResult.DeviceBlocked"/>
|
||||
/// and the paused-until window has not expired. Drain is a no-op in this state.
|
||||
@@ -124,6 +149,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
List<UploadResult>? pendingSignals = null;
|
||||
List<PoisonReport>? pendingPoison = null;
|
||||
string? pendingBlockedReason = null;
|
||||
string? pendingSessionGoneReason = null;
|
||||
DrainResult result;
|
||||
|
||||
try
|
||||
@@ -247,6 +273,35 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new PoisonReport(1, oversizeId, "oversize: " + outcome.ErrorReason, PoisonKind.Oversize));
|
||||
break;
|
||||
}
|
||||
else if (outcome.IsSessionGone)
|
||||
{
|
||||
// HTTP 410 — the backend no longer knows the session these items belong to.
|
||||
var foreignRun = CountLeadingForeignItems(batch);
|
||||
if (foreignRun > 0)
|
||||
{
|
||||
// Leftovers of a previous session id (the spool drops these at startup;
|
||||
// this is defence in depth). Drop only that leading run and carry on: a
|
||||
// legitimate new session must never be ended because of old items.
|
||||
var throughId = batch[foreignRun - 1].TelemetryItemId;
|
||||
_spool.MarkUploaded(throughId);
|
||||
_logger?.Warning(
|
||||
$"TelemetryUploadOrchestrator: dropped {foreignRun} item(s) of a previous session id the backend rejected with 410 (cursor advanced to {throughId}).");
|
||||
continue;
|
||||
}
|
||||
|
||||
// The current session is over server-side. RETAIN (cursor stays, the items
|
||||
// remain on disk for the diagnostics ZIP), stop this drain, and surface it
|
||||
// once — not as telemetry_upload_blocked, which would only queue another
|
||||
// item behind a batch that can never be accepted.
|
||||
failedBatches++;
|
||||
lastError = outcome.ErrorReason;
|
||||
if (!_sessionGoneReported && SessionGone != null)
|
||||
{
|
||||
_sessionGoneReported = true;
|
||||
pendingSessionGoneReason = outcome.ErrorReason;
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
// P1 — RETAIN. Transient (retry next drain), auth-permanent (uploader already
|
||||
@@ -301,9 +356,32 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
if (pendingSessionGoneReason != null)
|
||||
{
|
||||
try { SessionGone?.Invoke(this, pendingSessionGoneReason); }
|
||||
catch { /* handler must not abort the drain result */ }
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Number of items at the head of <paramref name="batch"/> whose partition key is not the
|
||||
/// current session's. 0 when no current key is configured (every batch counts as own).
|
||||
/// </summary>
|
||||
private int CountLeadingForeignItems(IReadOnlyList<TelemetryItem> batch)
|
||||
{
|
||||
if (_currentPartitionKey == null) return 0;
|
||||
|
||||
var count = 0;
|
||||
while (count < batch.Count
|
||||
&& !string.Equals(batch[count].PartitionKey, _currentPartitionKey, StringComparison.Ordinal))
|
||||
{
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static bool CarriesSignal(UploadResult outcome) =>
|
||||
outcome.DeviceBlocked
|
||||
|| outcome.DeviceKillSignal
|
||||
|
||||
@@ -30,7 +30,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
bool isAuthFailure = false,
|
||||
bool requiresSplit = false,
|
||||
bool isPoison = false,
|
||||
IReadOnlyList<string>? poisonRowKeys = null)
|
||||
IReadOnlyList<string>? poisonRowKeys = null,
|
||||
bool isSessionGone = false)
|
||||
{
|
||||
Success = success;
|
||||
ErrorReason = errorReason;
|
||||
@@ -44,6 +45,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
RequiresSplit = requiresSplit;
|
||||
IsPoison = isPoison;
|
||||
PoisonRowKeys = poisonRowKeys;
|
||||
IsSessionGone = isSessionGone;
|
||||
}
|
||||
|
||||
public bool Success { get; }
|
||||
@@ -81,6 +83,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
/// <summary>The RowKeys the backend named as poison (item-level). Non-empty when <see cref="IsPoison"/>.</summary>
|
||||
public IReadOnlyList<string>? PoisonRowKeys { get; }
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410: the backend no longer knows the session the batch belongs to (never
|
||||
/// registered, deleted, or being deleted). Not transient, not auth, not poison — retrying
|
||||
/// can never succeed. The orchestrator stops the drain (cursor stays) and raises
|
||||
/// <see cref="TelemetryUploadOrchestrator.SessionGone"/> when the batch belongs to the
|
||||
/// current session, or drops the batch's leading items when they are leftovers of a
|
||||
/// previous session id.
|
||||
/// </summary>
|
||||
public bool IsSessionGone { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Non-terminal quarantine signalled by the backend. The agent should stop draining
|
||||
/// uploads until <see cref="UnblockAt"/> (if set) elapses — the session stays alive so
|
||||
@@ -165,5 +177,14 @@ namespace AutopilotMonitor.Agent.V2.Core.Transport.Telemetry
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isPoison: true, poisonRowKeys: rejectedRowKeys ?? throw new ArgumentNullException(nameof(rejectedRowKeys)));
|
||||
|
||||
/// <summary>
|
||||
/// HTTP 410 — the backend no longer knows the session. Flagged <see cref="IsSessionGone"/>;
|
||||
/// never retried, never split, never treated as poison.
|
||||
/// </summary>
|
||||
public static UploadResult SessionGone(string reason) =>
|
||||
new UploadResult(false, reason ?? throw new ArgumentNullException(nameof(reason)), isTransient: false,
|
||||
deviceBlocked: false, unblockAt: null, deviceKillSignal: false, adminAction: null, actions: null,
|
||||
isSessionGone: true);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
|
||||
var remoteConfigService = new RemoteConfigService(
|
||||
auth.BackendApiClient, agentConfig.TenantId, logger,
|
||||
auth.EmergencyReporter, auth.DistressReporter, auth.AuthFailureTracker);
|
||||
auth.EmergencyReporter, auth.AuthFailureTracker);
|
||||
// retryOnTransientErrors:true is essential for the initial fetch — a Function App
|
||||
// cold-start (typically 30-60 s after a deploy) will time out a single-shot call and
|
||||
// silently strand the agent on built-in defaults for the entire session. Auth
|
||||
|
||||
@@ -439,6 +439,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
lifecyclePost,
|
||||
() => terminationHandler,
|
||||
agentConfig,
|
||||
auth.EmergencyReporter,
|
||||
shutdown,
|
||||
shutdownComplete,
|
||||
logger);
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Agent.V2.Core.Configuration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Logging;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Runtime;
|
||||
using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
|
||||
using AutopilotMonitor.Agent.V2.Core.Orchestration;
|
||||
using AutopilotMonitor.Agent.V2.Core.Termination;
|
||||
using AutopilotMonitor.Shared;
|
||||
@@ -103,6 +105,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
/// dispatcher because these are real actions, not admin-button echoes.</item>
|
||||
/// <item><b>DeviceBlocked</b> — non-terminal upload pause. Transport already reacted in
|
||||
/// <c>TelemetryUploadOrchestrator.ApplyControlSignals</c>; here we only log.</item>
|
||||
/// <item><b>SessionGone</b> (<see cref="TelemetryUploadOrchestrator.SessionGone"/>, HTTP 410
|
||||
/// on the current session's telemetry) — one emergency report, then the kill-signal
|
||||
/// termination path tagged <c>origin=session_gone</c>. See <see cref="HandleSessionGone"/>.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Shutdown race guard</b>: if the agent has already finished its own termination
|
||||
@@ -117,9 +122,14 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
InformationalEventPost post,
|
||||
Func<EnrollmentTerminationHandler> terminationHandlerAccessor,
|
||||
AgentConfiguration agentConfig,
|
||||
EmergencyReporter emergencyReporter,
|
||||
ManualResetEventSlim shutdown,
|
||||
ManualResetEventSlim shutdownComplete,
|
||||
AgentLogger logger)
|
||||
{
|
||||
orchestrator.Transport.SessionGone += (sender, reason) =>
|
||||
HandleSessionGone(reason, emergencyReporter, dispatcher, shutdown, shutdownComplete, logger);
|
||||
|
||||
orchestrator.Transport.ServerResponseReceived += (sender, upload) =>
|
||||
{
|
||||
if (shutdownComplete.IsSet)
|
||||
@@ -140,7 +150,7 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
{
|
||||
{ "forceSelfDestruct", "true" },
|
||||
{ "gracePeriodSeconds", "0" },
|
||||
{ "origin", "kill_signal" },
|
||||
{ "origin", TerminationOrigins.KillSignal },
|
||||
},
|
||||
};
|
||||
try { dispatcher.DispatchAsync(new List<ServerAction> { killAction }).GetAwaiter().GetResult(); }
|
||||
@@ -181,6 +191,76 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
};
|
||||
}
|
||||
|
||||
// Bounded blocking send for the one session-gone report: the terminate that follows ends
|
||||
// the process and the cleanup script deletes the agent, so a fire-and-forget send would
|
||||
// be abandoned mid-flight. Retries stay inside one anti-flood reservation (ONE report).
|
||||
private const int SessionGoneReportAttempts = 2;
|
||||
private static readonly TimeSpan SessionGoneReportTimeout = TimeSpan.FromSeconds(15);
|
||||
private static readonly TimeSpan SessionGoneReportRetryDelay = TimeSpan.FromSeconds(3);
|
||||
private static readonly TimeSpan SessionGoneReportBudget = TimeSpan.FromSeconds(40);
|
||||
|
||||
/// <summary>
|
||||
/// The backend answered HTTP 410 for the current session's telemetry: the session is over
|
||||
/// server-side. (a) Send ONE <see cref="AgentErrorType.TelemetryRejectedSessionUnknown"/>
|
||||
/// report over the emergency channel — the telemetry channel is exactly what was refused,
|
||||
/// and the backend accepts this report for unknown sessions. (b) Terminate through the same
|
||||
/// synthesised <c>terminate_session</c> path as the administrator kill signal (forced
|
||||
/// self-destruct, no grace), tagged <see cref="TerminationOrigins.SessionGone"/> so the
|
||||
/// termination handler skips the spool drain and the diagnostics upload. A shutdown that is
|
||||
/// already in progress owns the outcome — then nothing happens here.
|
||||
/// </summary>
|
||||
internal static void HandleSessionGone(
|
||||
string reason,
|
||||
EmergencyReporter emergencyReporter,
|
||||
ServerActionDispatcher dispatcher,
|
||||
ManualResetEventSlim shutdown,
|
||||
ManualResetEventSlim shutdownComplete,
|
||||
AgentLogger logger)
|
||||
{
|
||||
if (shutdownComplete.IsSet || shutdown.IsSet)
|
||||
{
|
||||
logger.Debug("ServerControlPlane: session gone (HTTP 410) while shutdown is already in progress — ignoring.");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.Error($"Backend no longer knows this session (HTTP 410: {reason}) — reporting once over the emergency channel, then terminating with self-destruct.");
|
||||
|
||||
if (emergencyReporter != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
emergencyReporter
|
||||
.TrySendAsync(
|
||||
AgentErrorType.TelemetryRejectedSessionUnknown,
|
||||
$"Telemetry upload rejected with HTTP 410 — backend no longer knows this session ({reason}). Agent terminating with self-destruct.",
|
||||
httpStatusCode: (int)HttpStatusCode.Gone,
|
||||
attempts: SessionGoneReportAttempts,
|
||||
perAttemptTimeout: SessionGoneReportTimeout,
|
||||
retryDelay: SessionGoneReportRetryDelay)
|
||||
.Wait(SessionGoneReportBudget);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.Debug($"Session-gone report send failed (best-effort): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
var terminateAction = new ServerAction
|
||||
{
|
||||
Type = ServerActionTypes.TerminateSession,
|
||||
Reason = "Backend no longer knows this session (HTTP 410)",
|
||||
QueuedAt = DateTime.UtcNow,
|
||||
Params = new Dictionary<string, string>
|
||||
{
|
||||
{ "forceSelfDestruct", "true" },
|
||||
{ "gracePeriodSeconds", "0" },
|
||||
{ "origin", TerminationOrigins.SessionGone },
|
||||
},
|
||||
};
|
||||
try { dispatcher.DispatchAsync(new List<ServerAction> { terminateAction }).GetAwaiter().GetResult(); }
|
||||
catch (Exception ex) { logger.Error("ServerActionDispatcher.DispatchAsync (session gone) threw during server-response wiring.", ex); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Soft admin-override handling. Portal Mark-Succeeded / Mark-Failed is an echo of an
|
||||
/// admin button click, not a ServerAction: it bypasses the dispatcher (no
|
||||
@@ -281,6 +361,17 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
: EnrollmentTerminationOutcome.Failed;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>origin</c> param of a synthesised <c>terminate_session</c> action
|
||||
/// (<see cref="TerminationOrigins"/>), or null when absent — it travels into
|
||||
/// <see cref="EnrollmentTerminatedEventArgs.Origin"/>.
|
||||
/// </summary>
|
||||
internal static string ReadOrigin(IReadOnlyDictionary<string, string> parameters)
|
||||
{
|
||||
if (parameters == null) return null;
|
||||
return parameters.TryGetValue("origin", out var origin) && !string.IsNullOrEmpty(origin) ? origin : null;
|
||||
}
|
||||
|
||||
// ============================================================ Private terminate-callback
|
||||
|
||||
/// <summary>
|
||||
@@ -326,8 +417,9 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
// before, masquerading every admin override as an error in SummaryDialog +
|
||||
// firing spurious diagnostics uploads).
|
||||
var mappedOutcome = MapAdminOutcome(action?.Params);
|
||||
var origin = ReadOrigin(action?.Params);
|
||||
|
||||
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}) — invoking termination handler.");
|
||||
logger.Warning($"ServerAction terminate_session received (ruleId={action?.RuleId}, reason={action?.Reason}, outcome={mappedOutcome}, origin={origin ?? "none"}) — invoking termination handler.");
|
||||
// Synthesise a Terminated event as if the kernel fired it.
|
||||
terminationHandler.Handle(
|
||||
sender: null,
|
||||
@@ -336,7 +428,8 @@ namespace AutopilotMonitor.Agent.V2.Runtime
|
||||
outcome: mappedOutcome,
|
||||
stageName: orchestrator.CurrentState?.Stage.ToString(),
|
||||
terminatedAtUtc: DateTime.UtcNow,
|
||||
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}"));
|
||||
details: $"Server-requested termination: ruleId={action?.RuleId}, reason={action?.Reason}",
|
||||
origin: origin));
|
||||
|
||||
// Plan §6.2 synchronous-shutdown — block the ingest dispatcher thread until the
|
||||
// main-thread cleanup (orchestrator.Stop, client disposal) has fully run. This
|
||||
|
||||
+1
-1
@@ -93,7 +93,7 @@ public class AnalyzeRulesFunctionAuthorEndpointTests
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var service = new AnalyzeRuleService(repo.Object, NullLogger<AnalyzeRuleService>.Instance);
|
||||
return (new AnalyzeRulesFunction(NullLogger<AnalyzeRulesFunction>.Instance, service), stored);
|
||||
return (new AnalyzeRulesFunction(service), stored);
|
||||
}
|
||||
|
||||
private static AnalyzeRule SpoofedPayload() => new()
|
||||
|
||||
@@ -71,7 +71,6 @@ public class AppHomingServiceTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
Mock.Of<System.Net.Http.IHttpClientFactory>(),
|
||||
cache,
|
||||
configuration,
|
||||
_registry,
|
||||
_tenantConfigMock.Object)
|
||||
{ CallBase = false };
|
||||
|
||||
@@ -69,8 +69,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache,
|
||||
configuration, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
@@ -202,8 +201,7 @@ public class AutopilotDeviceValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache,
|
||||
configuration, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
{ CallBase = false };
|
||||
CancellationToken observed = default;
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
|
||||
@@ -49,8 +49,7 @@ public class ConfigBackupRepositoryTests
|
||||
var serviceClient = new Mock<TableServiceClient>();
|
||||
serviceClient.Setup(c => c.GetTableClient(It.IsAny<string>())).Returns(Table.Object);
|
||||
Sut = new TableConfigBackupRepository(
|
||||
new TableStorageService(serviceClient.Object, NullLogger<TableStorageService>.Instance),
|
||||
NullLogger<TableConfigBackupRepository>.Instance);
|
||||
new TableStorageService(serviceClient.Object, NullLogger<TableStorageService>.Instance));
|
||||
}
|
||||
|
||||
public void SetupQuery(params TableEntity[] rows)
|
||||
|
||||
@@ -684,6 +684,65 @@ public class ConfigWireParityTests
|
||||
});
|
||||
}
|
||||
|
||||
// ---- Notification channel health (new endpoint, shape pinned) ------------------------
|
||||
|
||||
[Fact]
|
||||
public void NotificationChannelHealthResponse_pins_the_status_shape_and_omits_unset_fields()
|
||||
{
|
||||
var at = new DateTime(2026, 10, 10, 8, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
AssertParity(
|
||||
new
|
||||
{
|
||||
channels = new object[]
|
||||
{
|
||||
new
|
||||
{
|
||||
channelId = "c1",
|
||||
status = "Failing",
|
||||
recentAttempts = 5,
|
||||
recentFailures = 2,
|
||||
consecutiveFailures = 2,
|
||||
lastAttemptUtc = at,
|
||||
lastSuccessUtc = at.AddDays(-1),
|
||||
lastFailureUtc = at,
|
||||
failingSinceUtc = at.AddHours(-2),
|
||||
lastStatusCode = 401,
|
||||
lastError = "HTTP 401 Unauthorized",
|
||||
},
|
||||
new
|
||||
{
|
||||
channelId = "c2",
|
||||
status = "Unknown",
|
||||
recentAttempts = 0,
|
||||
recentFailures = 0,
|
||||
consecutiveFailures = 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
new NotificationChannelHealthResponse
|
||||
{
|
||||
Channels =
|
||||
{
|
||||
new NotificationChannelHealthDto
|
||||
{
|
||||
ChannelId = "c1",
|
||||
Status = AutopilotMonitor.Shared.Models.Notifications.NotificationChannelHealthStatus.Failing,
|
||||
RecentAttempts = 5,
|
||||
RecentFailures = 2,
|
||||
ConsecutiveFailures = 2,
|
||||
LastAttemptUtc = at,
|
||||
LastSuccessUtc = at.AddDays(-1),
|
||||
LastFailureUtc = at,
|
||||
FailingSinceUtc = at.AddHours(-2),
|
||||
LastStatusCode = 401,
|
||||
LastError = "HTTP 401 Unauthorized",
|
||||
},
|
||||
new NotificationChannelHealthDto { ChannelId = "c2" },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ---- UpdateAdminConfiguration --------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -65,8 +65,7 @@ public class CorporateIdentifierValidatorTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokenService = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache,
|
||||
configuration, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
{ CallBase = false };
|
||||
tokenService.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -140,8 +140,7 @@ public class CriticalTableRestoreServiceTests
|
||||
["IsEnabled"] = ("Boolean", true),
|
||||
});
|
||||
var sha = ComputeSha(lineBytes);
|
||||
var manifest = MakeManifestForTable(Constants.TableNames.GlobalAdmins,
|
||||
MakeEntry(Constants.TableNames.GlobalAdmins, sha, lineBytes.Length));
|
||||
var manifest = MakeManifestForTable(MakeEntry(Constants.TableNames.GlobalAdmins, sha, lineBytes.Length));
|
||||
|
||||
var fakeStore = new FakeStore(manifest, lineBytes, new ETag("\"0xABC\""));
|
||||
var tables = BuildTableStorageWithLiveRow(liveRow: null);
|
||||
@@ -179,9 +178,9 @@ public class CriticalTableRestoreServiceTests
|
||||
};
|
||||
|
||||
private static CriticalTableBackupManifest MakeManifest(CriticalTableBackupTableEntry entry) =>
|
||||
MakeManifestForTable(Table, entry);
|
||||
MakeManifestForTable(entry);
|
||||
|
||||
private static CriticalTableBackupManifest MakeManifestForTable(string tableName, CriticalTableBackupTableEntry entry) => new()
|
||||
private static CriticalTableBackupManifest MakeManifestForTable(CriticalTableBackupTableEntry entry) => new()
|
||||
{
|
||||
SchemaVersion = 1,
|
||||
BackupId = BackupId,
|
||||
|
||||
@@ -26,6 +26,7 @@ public class CrossTenantFanOutTests
|
||||
{
|
||||
private const string TenantA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
private const string TenantB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
|
||||
private const string TenantC = "cccccccc-cccc-4ccc-8ccc-cccccccccccc";
|
||||
|
||||
[Fact]
|
||||
public async Task SessionStats_DrainsIndexPerTenant_WithRowKeyBound_AndIsCached()
|
||||
@@ -33,7 +34,7 @@ public class CrossTenantFanOutTests
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 120), IndexRow(TenantB, "s2", SessionStatus.Failed, null));
|
||||
var sessions = MockTableClientReturning();
|
||||
var config = MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB));
|
||||
var storage = BuildStorage(config, sessions, index, apps: new Mock<TableClient>());
|
||||
var storage = BuildStorage(config, sessions, index, apps: new Mock<TableClient>(), counters: Counters(TenantA, TenantB));
|
||||
|
||||
var stats = await storage.GetAllSessionStatsAsync(tenantIdFilter: null, days: 7, allowedTenantIds: null);
|
||||
var again = await storage.GetAllSessionStatsAsync(tenantIdFilter: null, days: 7, allowedTenantIds: null);
|
||||
@@ -56,7 +57,7 @@ public class CrossTenantFanOutTests
|
||||
{
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60));
|
||||
var config = MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB));
|
||||
var storage = BuildStorage(config, MockTableClientReturning(), index, apps: new Mock<TableClient>());
|
||||
var storage = BuildStorage(config, MockTableClientReturning(), index, apps: new Mock<TableClient>(), counters: Counters(TenantA, TenantB));
|
||||
|
||||
await storage.GetAllSessionStatsAsync(tenantIdFilter: null, days: 30, allowedTenantIds: new[] { TenantB.ToUpperInvariant() });
|
||||
|
||||
@@ -69,7 +70,7 @@ public class CrossTenantFanOutTests
|
||||
{
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60));
|
||||
var sessions = MockTableClientReturning();
|
||||
var storage = BuildStorage(MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)), sessions, index, apps: new Mock<TableClient>());
|
||||
var storage = BuildStorage(MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)), sessions, index, apps: new Mock<TableClient>(), counters: Counters(TenantA, TenantB));
|
||||
|
||||
var start = DateTime.UtcNow.AddDays(-30);
|
||||
var end = DateTime.UtcNow.AddDays(1);
|
||||
@@ -99,7 +100,7 @@ public class CrossTenantFanOutTests
|
||||
public async Task AppSummaries_CrossTenant_OnePartitionQueryPerTenant_SharedAcrossConcurrentCallers()
|
||||
{
|
||||
var apps = MockTableClientReturning(AppRow(TenantA, "s1", "Teams"), AppRow(TenantB, "s2", "Office"));
|
||||
var storage = BuildStorage(MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)), MockTableClientReturning(), MockTableClientReturning(), apps);
|
||||
var storage = BuildStorage(MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)), MockTableClientReturning(), MockTableClientReturning(), apps, counters: Counters(TenantA, TenantB));
|
||||
|
||||
var since = DateTime.UtcNow.AddDays(-30);
|
||||
// The Installs tab fires these two at once; the usage compute adds the refs projection.
|
||||
@@ -132,6 +133,114 @@ public class CrossTenantFanOutTests
|
||||
Assert.All(apps.Selects, s => Assert.Equal(TableStorageService.AppMetricsProjection, s));
|
||||
}
|
||||
|
||||
// ── enrolled-tenant snapshot: the fan-out width is the set of tenants with an enrollment on record ──
|
||||
|
||||
[Fact]
|
||||
public async Task SessionList_FansOutOnlyOverEnrolledTenants_AndScansCountersOncePerTtl()
|
||||
{
|
||||
// Config knows three tenants: A enrolled, B's counter row stands at 0, C has no counter row at all.
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60));
|
||||
var counters = MockTableClientReturning(CounterRow(TenantA, 5), CounterRow(TenantB, 0));
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB), ConfigRow(TenantC)),
|
||||
MockTableClientReturning(), index, apps: new Mock<TableClient>(), counters: counters);
|
||||
|
||||
var page = await storage.GetAllSessionsPageAsync(tenantIdFilter: null, days: null, pageSize: 10, continuation: null);
|
||||
await storage.GetAllSessionsPageAsync(tenantIdFilter: null, days: null, pageSize: 10, continuation: null);
|
||||
|
||||
Assert.Single(page.Items);
|
||||
// Two pages, one partition query each — only the enrolled tenant is ever asked.
|
||||
Assert.Equal(2, index.Filters.Count);
|
||||
Assert.All(index.Filters, f => Assert.StartsWith($"PartitionKey eq '{TenantA}'", f));
|
||||
// The marker is one projected property-filter read per TTL, shared by both pages.
|
||||
Assert.Equal("RowKey eq 'current'", Assert.Single(counters.Filters));
|
||||
Assert.Equal(new[] { "PartitionKey", "TotalEnrollments" }, Assert.Single(counters.Selects));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionList_CounterScanFailure_FailsOpenToEveryConfiguredTenant()
|
||||
{
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60));
|
||||
var counters = new RecordingTableClient();
|
||||
counters.Setup(c => c.QueryAsync<TableEntity>(
|
||||
It.IsAny<string>(), It.IsAny<int?>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.Throws(new RequestFailedException(503, "PlatformStats unavailable"));
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)),
|
||||
MockTableClientReturning(), index, apps: new Mock<TableClient>(), counters: counters);
|
||||
|
||||
var page = await storage.GetAllSessionsPageAsync(tenantIdFilter: null, days: null, pageSize: 10, continuation: null);
|
||||
|
||||
// A failed marker read must never narrow the fan-out: every configured tenant is queried.
|
||||
Assert.Single(page.Items);
|
||||
Assert.Equal(2, index.Filters.Count);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionList_CounterRowOfAnUnconfiguredTenant_IsNeverQueried()
|
||||
{
|
||||
// An offboarded tenant may keep its counter row and even index rows; without a config row it is out.
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60), IndexRow(TenantB, "s2", SessionStatus.Failed, null));
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA)),
|
||||
MockTableClientReturning(), index, apps: new Mock<TableClient>(), counters: Counters(TenantA, TenantB));
|
||||
|
||||
var page = await storage.GetAllSessionsPageAsync(tenantIdFilter: null, days: null, pageSize: 10, continuation: null);
|
||||
|
||||
Assert.Single(page.Items);
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantA}'", Assert.Single(index.Filters));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionList_ConfiguredButNobodyEnrolled_IsAnEmptyPage_NeverTheLegacyScan()
|
||||
{
|
||||
// The primary-table scan is the EMPTY-CONFIG safety net only; "configured, nobody enrolled" is simply empty.
|
||||
var sessions = MockTableClientReturning(new TableEntity(TenantA, "leak"));
|
||||
var index = MockTableClientReturning();
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)),
|
||||
sessions, index, apps: new Mock<TableClient>(), counters: MockTableClientReturning());
|
||||
|
||||
var page = await storage.GetAllSessionsPageAsync(tenantIdFilter: null, days: null, pageSize: 10, continuation: null);
|
||||
var stats = await storage.GetAllSessionStatsAsync(tenantIdFilter: null, days: 7, allowedTenantIds: null);
|
||||
|
||||
Assert.Empty(page.Items);
|
||||
Assert.Null(page.NextRawToken);
|
||||
Assert.Equal(0, stats.TotalLastNDays);
|
||||
AssertNeverQueried(sessions);
|
||||
AssertNeverQueried(index);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SessionStats_BoundedCaller_IsCutToEnrolledTenants_AfterTheBound()
|
||||
{
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60), IndexRow(TenantB, "s2", SessionStatus.Succeeded, 60));
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)),
|
||||
MockTableClientReturning(), index, apps: new Mock<TableClient>(), counters: Counters(TenantA));
|
||||
|
||||
// The delegated caller manages both tenants; only A ever enrolled.
|
||||
await storage.GetAllSessionStatsAsync(tenantIdFilter: null, days: 30, allowedTenantIds: new[] { TenantA, TenantB });
|
||||
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantA}'", Assert.Single(index.Filters));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GeoWindowAndAppSummaries_CrossTenant_SkipTenantsWithoutEnrollment()
|
||||
{
|
||||
var index = MockTableClientReturning(IndexRow(TenantA, "s1", SessionStatus.Succeeded, 60));
|
||||
var apps = MockTableClientReturning(AppRow(TenantA, "s1", "Teams"));
|
||||
var storage = BuildStorage(
|
||||
MockTableClientReturning(ConfigRow(TenantA), ConfigRow(TenantB)),
|
||||
MockTableClientReturning(), index, apps, counters: Counters(TenantA));
|
||||
|
||||
await storage.GetGeoWindowSessionsAsync(DateTime.UtcNow.AddDays(-30), DateTime.UtcNow.AddDays(1), tenantId: null);
|
||||
await storage.GetAppMetricsSummariesAsync(DateTime.UtcNow.AddDays(-30), tenantId: null);
|
||||
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantA}'", Assert.Single(index.Filters));
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantA}'", Assert.Single(apps.Filters));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GeoProjection_IsFullyMirroredOnSessionsIndex()
|
||||
{
|
||||
@@ -161,16 +270,26 @@ public class CrossTenantFanOutTests
|
||||
private static TableEntity AppRow(string tenantId, string sessionId, string appName)
|
||||
=> new(tenantId, $"{sessionId}_{appName}") { ["SessionId"] = sessionId, ["AppName"] = appName, ["StartedAt"] = DateTime.UtcNow.AddHours(-2) };
|
||||
|
||||
private static TableStorageService BuildStorage(RecordingTableClient config, RecordingTableClient sessions, RecordingTableClient index, Mock<TableClient> apps)
|
||||
/// <param name="counters">PlatformStats counter rows — the enrolled-tenant marker; null = nobody has enrolled.</param>
|
||||
private static TableStorageService BuildStorage(RecordingTableClient config, RecordingTableClient sessions, RecordingTableClient index, Mock<TableClient> apps, RecordingTableClient? counters = null)
|
||||
{
|
||||
var serviceClient = new Mock<TableServiceClient>();
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.TenantConfiguration)).Returns(config.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.Sessions)).Returns(sessions.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.SessionsIndex)).Returns(index.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.AppInstallSummaries)).Returns(apps.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.PlatformStats)).Returns((counters ?? MockTableClientReturning()).Object);
|
||||
return new TableStorageService(serviceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>A per-tenant counter row (PlatformStats, RowKey "current") — the enrolled-tenant marker.</summary>
|
||||
private static TableEntity CounterRow(string tenantId, long totalEnrollments)
|
||||
=> new(tenantId, "current") { ["TotalEnrollments"] = totalEnrollments };
|
||||
|
||||
/// <summary>Counter rows marking every given tenant as enrolled.</summary>
|
||||
private static RecordingTableClient Counters(params string[] tenantIds)
|
||||
=> MockTableClientReturning(tenantIds.Select(t => CounterRow(t, 1)).ToArray());
|
||||
|
||||
/// <summary>Mock TableClient that records the filter + select of every string-filter QueryAsync call.</summary>
|
||||
private sealed class RecordingTableClient : Mock<TableClient>
|
||||
{
|
||||
|
||||
@@ -114,9 +114,16 @@ public class DelegatedBoundedAggregateTests
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.TenantConfiguration)).Returns(config.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.Sessions)).Returns(sessions.Object);
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.SessionsIndex)).Returns(index.Object);
|
||||
// Every tenant of these tests counts as enrolled: the bounded guard under test must not lean on the
|
||||
// enrolled-tenant snapshot narrowing the set (CrossTenantFanOutTests covers that snapshot).
|
||||
serviceClient.Setup(s => s.GetTableClient(Constants.TableNames.PlatformStats)).Returns(
|
||||
MockTableClientReturning(CounterRow(ConfigTenant), CounterRow(ManagedTenant), CounterRow(UnmanagedTenant)).Object);
|
||||
return new TableStorageService(serviceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>A per-tenant PlatformStats counter row — the enrolled-tenant marker of the cross-tenant fan-outs.</summary>
|
||||
private static TableEntity CounterRow(string tenantId) => new(tenantId, "current") { ["TotalEnrollments"] = 1L };
|
||||
|
||||
/// <summary>Mocks both QueryAsync overloads (string + expression filter) to return the given rows.</summary>
|
||||
private static Mock<TableClient> MockTableClientReturning(params TableEntity[] rows)
|
||||
{
|
||||
|
||||
@@ -568,7 +568,7 @@ public class DeletionManifestBuilderTests
|
||||
// ---------------------------------------------------------------- Test-fixture plumbing ----
|
||||
|
||||
private static DeletionManifestBuilder NewBuilder(ISessionDeletionInventoryReader reader)
|
||||
=> new DeletionManifestBuilder(reader, NullLogger<DeletionManifestBuilder>.Instance);
|
||||
=> new DeletionManifestBuilder(reader);
|
||||
|
||||
private static ISessionDeletionInventoryReader NewReader(Action<ReaderSeed> seedAction)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Analyze;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Vulnerability;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The two queue handlers derive rows for a session (rule results, reports, inventory counters,
|
||||
/// CveIndex). For a session the backend does not know, or one a cascade is tearing down, they
|
||||
/// must stop before the first read of session data — nothing derived may ever outlive the
|
||||
/// session's cascade. Both handlers are driven up to that gate with every later dependency
|
||||
/// unavailable, so a regression that moves the gate shows up as a null dereference or an
|
||||
/// unexpected mock call.
|
||||
/// </summary>
|
||||
public class DerivedWriterLivenessTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
private static SessionDeletionGuard GuardFor(TableEntity? row)
|
||||
{
|
||||
var reader = new Mock<ISessionDeletionInventoryReader>();
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(row);
|
||||
return new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance);
|
||||
}
|
||||
|
||||
private static Mock<AdminConfigurationService> AdminConfigWithCorrelationEnabled()
|
||||
{
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
adminConfig.Setup(a => a.GetConfigurationAsync())
|
||||
.ReturnsAsync(new AdminConfiguration { VulnerabilityCorrelationEnabled = true });
|
||||
return adminConfig;
|
||||
}
|
||||
|
||||
public static IEnumerable<object?[]> NotLiveRows()
|
||||
{
|
||||
yield return new object?[] { null };
|
||||
yield return new object?[] { new TableEntity(TenantId, SessionId) { ["DeletionState"] = SessionDeletionState.Running } };
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(NotLiveRows))]
|
||||
public async Task VulnerabilityCorrelate_skips_a_missing_or_locked_session_before_loading_inventory(TableEntity? row)
|
||||
{
|
||||
var inventoryLoader = new Mock<IVulnerabilityInventoryLoader>(MockBehavior.Strict);
|
||||
var vulnRepo = new Mock<IVulnerabilityRepository>(MockBehavior.Strict);
|
||||
var handler = new VulnerabilityCorrelateHandler(
|
||||
AdminConfigWithCorrelationEnabled().Object,
|
||||
vulnerabilityCorrelation: null!,
|
||||
vulnRepo.Object,
|
||||
Mock.Of<ISessionRepository>(),
|
||||
inventoryLoader.Object,
|
||||
Mock.Of<IAnalyzeOnEnrollmentEndProducer>(),
|
||||
signalRNotification: null!,
|
||||
GuardFor(row),
|
||||
NullLogger<VulnerabilityCorrelateHandler>.Instance);
|
||||
|
||||
await handler.HandleAsync(new VulnerabilityCorrelateEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = VulnerabilityCorrelateHandler.ReasonShutdownInventory, EnqueuedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
inventoryLoader.VerifyNoOtherCalls();
|
||||
vulnRepo.VerifyNoOtherCalls();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(NotLiveRows))]
|
||||
public async Task Analyze_skips_a_missing_or_locked_session_before_the_rule_engine_runs(TableEntity? row)
|
||||
{
|
||||
var ruleRepo = new Mock<IRuleRepository>(MockBehavior.Strict);
|
||||
var metricsRepo = new Mock<IMetricsRepository>(MockBehavior.Strict);
|
||||
var handler = new AnalyzeOnEnrollmentEndHandler(
|
||||
ruleService: null!,
|
||||
ruleRepo.Object,
|
||||
Mock.Of<ISessionRepository>(),
|
||||
metricsRepo.Object,
|
||||
signalRNotification: null!,
|
||||
configService: null!,
|
||||
channelDispatcher: null!,
|
||||
GuardFor(row),
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
|
||||
});
|
||||
|
||||
ruleRepo.VerifyNoOtherCalls();
|
||||
metricsRepo.VerifyNoOtherCalls();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_guard_read_error_propagates_so_the_queue_retries_instead_of_skipping()
|
||||
{
|
||||
var reader = new Mock<ISessionDeletionInventoryReader>();
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
|
||||
var handler = new AnalyzeOnEnrollmentEndHandler(
|
||||
ruleService: null!, Mock.Of<IRuleRepository>(), Mock.Of<ISessionRepository>(), Mock.Of<IMetricsRepository>(),
|
||||
signalRNotification: null!, configService: null!, channelDispatcher: null!,
|
||||
new SessionDeletionGuard(reader.Object, NullLogger<SessionDeletionGuard>.Instance),
|
||||
NullLogger<AnalyzeOnEnrollmentEndHandler>.Instance);
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => handler.HandleAsync(new AnalyzeOnEnrollmentEndEnvelope
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionId, Reason = AnalyzeOnEnrollmentEndHandler.ReasonEnrollmentComplete, EnqueuedAt = DateTime.UtcNow,
|
||||
}));
|
||||
}
|
||||
}
|
||||
@@ -18,7 +18,6 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
public class DiagnosticsBlobCascadeDeleterTests
|
||||
{
|
||||
private const string TenantA = "11111111-1111-1111-1111-111111111111";
|
||||
private const string TenantB = "22222222-2222-2222-2222-222222222222";
|
||||
private const string Filename = "AgentDiagnostics-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-20260519T120000.zip";
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -28,12 +28,6 @@ public class DistressRateLimitServiceTests
|
||||
return new DistressRateLimitService(cache, logger);
|
||||
}
|
||||
|
||||
private static DistressRateLimitService CreateService(IMemoryCache cache)
|
||||
{
|
||||
var logger = NullLogger<DistressRateLimitService>.Instance;
|
||||
return new DistressRateLimitService(cache, logger);
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Layer 1: Per-IP Rate Limiting (5 requests / 15 minutes)
|
||||
// =========================================================================
|
||||
|
||||
@@ -22,10 +22,30 @@ public class EmailChannelProviderTests
|
||||
private static readonly NotificationScope Platform = NotificationScope.Platform;
|
||||
|
||||
private static Mock<WebhookNotificationService> WebhookMock()
|
||||
=> new(new HttpClient(), NullLogger<WebhookNotificationService>.Instance) { CallBase = false };
|
||||
{
|
||||
var mock = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
mock.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
=> new(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance) { CallBase = false };
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<IEmailChannelSender> EmailMock()
|
||||
{
|
||||
var mock = new Mock<IEmailChannelSender>();
|
||||
mock.Setup(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel Channel(string? url, string id = "mail", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = Email, Url = url, Enabled = enabled };
|
||||
@@ -48,30 +68,30 @@ public class EmailChannelProviderTests
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
var email = new Mock<IEmailChannelSender>();
|
||||
var email = EmailMock();
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
var alert = Alert();
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid; two@example.invalid") }, alert, Platform);
|
||||
|
||||
email.Verify(e => e.SendOpsAlertAsync("ops@example.invalid; two@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendNotificationAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
email.Verify(e => e.SendAlertAsync("ops@example.invalid; two@example.invalid", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendOpsAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Email_channel_without_recipients_is_skipped_and_the_test_path_reports_the_sender_result()
|
||||
{
|
||||
var email = new Mock<IEmailChannelSender>();
|
||||
email.Setup(e => e.SendAlertWithResultAsync("ops@example.invalid", It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new WebhookTestResult { Success = true, Message = "Sent to 1 recipient(s)." });
|
||||
var email = EmailMock();
|
||||
email.Setup(e => e.SendAlertAsync("ops@example.invalid", It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "Sent to 1 recipient(s)." });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>(), email.Object);
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel(""), Channel(null, "n") }, Alert(), Platform);
|
||||
email.Verify(e => e.SendOpsAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
email.Verify(e => e.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
|
||||
var result = await dispatcher.SendWithResultAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
var result = await dispatcher.SendTestAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal("Sent to 1 recipient(s).", result.Message);
|
||||
}
|
||||
@@ -82,7 +102,7 @@ public class EmailChannelProviderTests
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Channel("ops@example.invalid") }, Alert(), Platform); // no throw
|
||||
var result = await dispatcher.SendWithResultAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
var result = await dispatcher.SendTestAsync(Channel("ops@example.invalid"), Alert(), Platform);
|
||||
|
||||
Assert.False(result.Success);
|
||||
Assert.Contains("not registered", result.Message);
|
||||
@@ -200,7 +220,7 @@ public class EmailChannelProviderTests
|
||||
var handler = new StubHandler();
|
||||
var (service, _) = Sender(handler, apiKey: "key");
|
||||
|
||||
var result = await service.SendAlertWithResultAsync("ops@example.invalid; second@example.invalid", Alert());
|
||||
var result = await service.SendAlertAsync("ops@example.invalid; second@example.invalid", Alert());
|
||||
|
||||
Assert.True(result.Success);
|
||||
Assert.Equal("Sent to 2 recipient(s). Provider status: sent, id abc,abc.", result.Message);
|
||||
@@ -219,17 +239,16 @@ public class EmailChannelProviderTests
|
||||
{
|
||||
var handler = new StubHandler { Responder = _ => StubHandler.Json(HttpStatusCode.OK, "[{\"email\":\"ops@example.invalid\",\"status\":\"rejected\",\"reject_reason\":\"hard-bounce\"}]") };
|
||||
var (service, _) = Sender(handler, apiKey: "key");
|
||||
var rejected = await service.SendAlertWithResultAsync("ops@example.invalid", Alert());
|
||||
var rejected = await service.SendAlertAsync("ops@example.invalid", Alert());
|
||||
Assert.False(rejected.Success);
|
||||
Assert.Contains("accepted 0 of 1", rejected.Message);
|
||||
Assert.Contains("Provider status: rejected", rejected.Message);
|
||||
|
||||
var (unconfigured, _) = Sender(new StubHandler(), apiKey: "");
|
||||
var skipped = await unconfigured.SendAlertWithResultAsync("ops@example.invalid", Alert());
|
||||
var skipped = await unconfigured.SendAlertAsync("ops@example.invalid", Alert());
|
||||
Assert.False(skipped.Success);
|
||||
Assert.Contains("not configured", skipped.Message);
|
||||
|
||||
await unconfigured.SendOpsAlertAsync("ops@example.invalid", Alert()); // never throws
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
||||
@@ -43,7 +43,7 @@ public class EventDataNormalizationTests
|
||||
[Fact]
|
||||
public void ParseEvent_normalizes_nested_objects_to_native_types()
|
||||
{
|
||||
var evt = TelemetryPayloadParser.ParseEvent(EventDto(HardwareSpecPayload), TenantId, SessionId);
|
||||
var evt = TelemetryPayloadParser.ParseEvent(EventDto(HardwareSpecPayload));
|
||||
|
||||
Assert.NotNull(evt);
|
||||
// Nested array must be a native List<object> of Dictionary<string,object>, NOT a JArray/JObject.
|
||||
@@ -58,7 +58,7 @@ public class EventDataNormalizationTests
|
||||
// The actual corruption manifests when the downstream DeviceSnapshot writer re-serializes
|
||||
// Data with System.Text.Json. Before the fix this produced "disks":[[[[]],...]]; now the
|
||||
// real fields must survive a round-trip.
|
||||
var evt = TelemetryPayloadParser.ParseEvent(EventDto(HardwareSpecPayload), TenantId, SessionId);
|
||||
var evt = TelemetryPayloadParser.ParseEvent(EventDto(HardwareSpecPayload));
|
||||
Assert.NotNull(evt);
|
||||
|
||||
var json = Stj.Serialize(evt!.Data);
|
||||
|
||||
+1
-1
@@ -98,7 +98,7 @@ public class GatherRulesFunctionAuthorEndpointTests
|
||||
.ReturnsAsync(true);
|
||||
|
||||
var service = new GatherRuleService(repo.Object, NullLogger<GatherRuleService>.Instance);
|
||||
return (new GatherRulesFunction(NullLogger<GatherRulesFunction>.Instance, service), stored);
|
||||
return (new GatherRulesFunction(service), stored);
|
||||
}
|
||||
|
||||
private static GatherRule SpoofedPayload() => new()
|
||||
|
||||
@@ -32,8 +32,7 @@ public class GraphAuthFailureTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
return new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache,
|
||||
configuration, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
{ CallBase = false };
|
||||
}
|
||||
|
||||
|
||||
-4
@@ -338,7 +338,6 @@ public class GraphFeatureDetectorTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
new SingleHandlerFactory(handler),
|
||||
cache,
|
||||
tokenConfig,
|
||||
new EntraAppRegistry(tokenConfig, NullLogger<EntraAppRegistry>.Instance),
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
@@ -497,7 +496,6 @@ public class GraphFeatureDetectorTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubEntraConfig(),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
{
|
||||
@@ -530,7 +528,6 @@ public class GraphFeatureDetectorTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
new NoopHttpClientFactory(),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
StubEntraConfig(),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
{
|
||||
@@ -569,7 +566,6 @@ public class GraphFeatureDetectorTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
cache,
|
||||
StubEntraConfig(),
|
||||
StubAppRegistry(),
|
||||
StubTenantConfigService())
|
||||
{
|
||||
|
||||
@@ -48,7 +48,6 @@ public class GraphTokenServiceTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
factory,
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
config,
|
||||
new EntraAppRegistry(config, NullLogger<EntraAppRegistry>.Instance),
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
@@ -237,7 +236,6 @@ public class GraphTokenServiceTests
|
||||
NullLogger<GraphTokenService>.Instance,
|
||||
new StubHttpClientFactory(handler),
|
||||
new MemoryCache(new MemoryCacheOptions()),
|
||||
config,
|
||||
registry,
|
||||
new AutopilotMonitor.Functions.Services.TenantConfigurationService(
|
||||
Moq.Mock.Of<AutopilotMonitor.Shared.DataAccess.IConfigRepository>(),
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
using AutopilotMonitor.Functions.Functions.Ingest;
|
||||
using AutopilotMonitor.Functions.Helpers;
|
||||
using AutopilotMonitor.Functions.Security;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
@@ -155,6 +156,27 @@ public class IngestTelemetryAuthPathTests
|
||||
Assert.Equal(SessionStatus.Succeeded, SessionRowProjections.TryReadStatus(returned));
|
||||
}
|
||||
|
||||
// ============================================================ Session unknown → 410
|
||||
|
||||
[Fact]
|
||||
public async Task SessionUnknown_guard_returns_null_that_Run_maps_to_410_session_unknown()
|
||||
{
|
||||
// Run: a null guard row (no Sessions row, no active tombstone) is refused with 410 via
|
||||
// WriteSessionUnknownAsync and stamped IngestRefusal=session_unknown; the cascade lock keeps
|
||||
// its own value so the two 410 reasons stay apart on the request row.
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, SessionId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
|
||||
var row = await guard.EnsureWritableAndGetRowAsync(TenantId, SessionId, callerContext: "V2.IngestTelemetry");
|
||||
|
||||
Assert.Null(row);
|
||||
Assert.NotEqual(RequestRowMarkers.IngestRefusal.SessionUnknown, RequestRowMarkers.IngestRefusal.CascadeLocked);
|
||||
Assert.Equal("IngestRefusal", RequestRowMarkers.IngestRefusalKey);
|
||||
}
|
||||
|
||||
// ============================================================ TryReadStatus (unique helper)
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -17,9 +17,6 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// </summary>
|
||||
public class IngestTelemetryMalformedTests
|
||||
{
|
||||
private const string TenantId = "a1b2c3d4-e5f6-7890-abcd-ef1234567890";
|
||||
private const string SessionId = "b2c3d4e5-f6a7-8901-bcde-f12345678901";
|
||||
|
||||
// ============================================================ Deeply-nested JSON (JSON-bomb)
|
||||
|
||||
[Theory]
|
||||
|
||||
@@ -79,8 +79,7 @@ public class IntuneEnrollmentValidationTests
|
||||
Mock.Of<IConfigRepository>(), NullLogger<TenantConfigurationService>.Instance, cache)
|
||||
{ CallBase = false };
|
||||
var tokens = new Mock<GraphTokenService>(
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache,
|
||||
configuration, registry, tenantConfig.Object)
|
||||
NullLogger<GraphTokenService>.Instance, Mock.Of<IHttpClientFactory>(), cache, registry, tenantConfig.Object)
|
||||
{ CallBase = false };
|
||||
tokens.Setup(t => t.GetAccessTokenAsync(TenantId, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(GraphTokenResult.Success("token"));
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using System;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
public class MaintenanceStepTimerTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Records_every_step_in_order_and_returns_the_body_result()
|
||||
{
|
||||
var timer = new MaintenanceStepTimer();
|
||||
|
||||
await timer.RunAsync("first", () => Task.CompletedTask);
|
||||
var value = await timer.RunAsync<int>("second", () => Task.FromResult(42));
|
||||
|
||||
Assert.Equal(42, value);
|
||||
Assert.Equal(new[] { "first", "second" }, timer.Steps.Keys);
|
||||
Assert.All(timer.Steps.Values, ms => Assert.True(ms >= 0));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_throwing_step_is_still_recorded_and_the_exception_propagates()
|
||||
{
|
||||
var timer = new MaintenanceStepTimer();
|
||||
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(
|
||||
() => timer.RunAsync("boom", () => throw new InvalidOperationException("x")));
|
||||
|
||||
Assert.True(timer.Steps.ContainsKey("boom"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,678 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.DataAccess.TableStorage;
|
||||
using AutopilotMonitor.Functions.Functions.Config;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Notifications;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Notifications;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Channel delivery health: the status rule and the fold of each send outcome
|
||||
/// (<see cref="NotificationChannelHealthEvaluator"/>), the recorder's compare-and-swap write and
|
||||
/// its once-per-episode bell, the dispatcher's recording of every non-Push send, the read
|
||||
/// endpoint's projection onto the stored channels, and the table mapping.
|
||||
/// </summary>
|
||||
public class NotificationChannelHealthTests
|
||||
{
|
||||
private const string TenantKey = "11111111-1111-1111-1111-111111111111";
|
||||
private static readonly DateTime T0 = new(2026, 10, 10, 8, 0, 0, DateTimeKind.Utc);
|
||||
private static readonly NotificationScope Tenant = NotificationScope.Tenant(TenantKey);
|
||||
|
||||
private static NotificationChannel Webhook(string id = "c1", string url = "https://hooks.example.invalid/a", string name = "Service Desk")
|
||||
=> new() { Id = id, Name = name, ProviderType = (int)WebhookProviderType.GenericJson, Url = url, Enabled = true };
|
||||
|
||||
private static NotificationSendResult Ok() => new() { Success = true, StatusCode = 202, Message = "Test notification sent successfully." };
|
||||
|
||||
private static NotificationSendResult Fail(int? status = 401, string message = "Webhook returned HTTP 401: {\"error\":\"secret body\"}")
|
||||
=> new() { Success = false, StatusCode = status, Message = message };
|
||||
|
||||
private static ChannelHealthFold Fold(NotificationChannelHealth? stored, NotificationSendResult result,
|
||||
ChannelSendKind kind = ChannelSendKind.Delivery, DateTime? at = null, NotificationChannel? channel = null)
|
||||
{
|
||||
channel ??= Webhook();
|
||||
return NotificationChannelHealthEvaluator.Apply(stored, TenantKey, channel.Id,
|
||||
NotificationChannelHealthEvaluator.Fingerprint(channel), result, kind, at ?? T0);
|
||||
}
|
||||
|
||||
// ── Status rule and fold ──────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void A_delivered_send_on_a_new_channel_is_ok()
|
||||
{
|
||||
var fold = Fold(null, Ok());
|
||||
|
||||
Assert.Equal("1", fold.Next.Recent);
|
||||
Assert.Equal(T0, fold.Next.LastSuccessUtc);
|
||||
Assert.Equal(NotificationChannelHealthStatus.Ok, NotificationChannelHealthEvaluator.Evaluate(fold.Next, T0));
|
||||
Assert.Equal(ChannelHealthTransition.None, fold.Transition);
|
||||
Assert.False(fold.RaiseBell);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void One_failure_degrades_with_the_status_line_and_never_the_body()
|
||||
{
|
||||
var fold = Fold(Fold(null, Ok()).Next, Fail());
|
||||
|
||||
Assert.Equal(1, fold.Next.ConsecutiveFailures);
|
||||
Assert.Equal("HTTP 401 Unauthorized", fold.Next.LastError);
|
||||
Assert.Equal(401, fold.Next.LastStatusCode);
|
||||
Assert.Equal(T0, fold.Next.FailingSinceUtc);
|
||||
Assert.Equal(NotificationChannelHealthStatus.Degraded, NotificationChannelHealthEvaluator.Evaluate(fold.Next, T0));
|
||||
Assert.False(fold.RaiseBell);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Two_failed_deliveries_in_a_row_are_failing_and_ring_the_bell_once_per_episode()
|
||||
{
|
||||
var first = Fold(null, Fail(), at: T0);
|
||||
var second = Fold(first.Next, Fail(), at: T0.AddHours(1));
|
||||
var third = Fold(second.Next, Fail(), at: T0.AddHours(2));
|
||||
|
||||
Assert.Equal(NotificationChannelHealthStatus.Failing, NotificationChannelHealthEvaluator.Evaluate(second.Next, T0.AddHours(1)));
|
||||
Assert.Equal(ChannelHealthTransition.BecameFailing, second.Transition);
|
||||
Assert.True(second.RaiseBell);
|
||||
Assert.Equal(T0.AddHours(1), second.Next.FailingNotifiedUtc);
|
||||
Assert.Equal(T0, second.Next.FailingSinceUtc);
|
||||
|
||||
Assert.Equal(ChannelHealthTransition.None, third.Transition);
|
||||
Assert.False(third.RaiseBell);
|
||||
Assert.Equal(3, third.Next.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failing_test_never_rings_but_the_next_failed_delivery_does()
|
||||
{
|
||||
var first = Fold(null, Fail(), ChannelSendKind.Test);
|
||||
var second = Fold(first.Next, Fail(), ChannelSendKind.Test);
|
||||
Assert.Equal(ChannelHealthTransition.BecameFailing, second.Transition);
|
||||
Assert.False(second.RaiseBell);
|
||||
Assert.Null(second.Next.FailingNotifiedUtc);
|
||||
|
||||
var delivery = Fold(second.Next, Fail());
|
||||
Assert.True(delivery.RaiseBell);
|
||||
Assert.Equal(ChannelHealthTransition.None, delivery.Transition);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(ChannelSendKind.Delivery)]
|
||||
[InlineData(ChannelSendKind.Test)]
|
||||
public void A_success_ends_the_episode_and_the_next_one_rings_again(ChannelSendKind kind)
|
||||
{
|
||||
var failing = Fold(Fold(null, Fail()).Next, Fail());
|
||||
var recovered = Fold(failing.Next, Ok(), kind, T0.AddHours(3));
|
||||
|
||||
Assert.Equal(ChannelHealthTransition.Recovered, recovered.Transition);
|
||||
Assert.Equal(0, recovered.Next.ConsecutiveFailures);
|
||||
Assert.Null(recovered.Next.FailingSinceUtc);
|
||||
Assert.Null(recovered.Next.FailingNotifiedUtc);
|
||||
Assert.Equal("HTTP 401 Unauthorized", recovered.Next.LastError); // the last failure stays readable
|
||||
|
||||
var again = Fold(Fold(recovered.Next, Fail()).Next, Fail());
|
||||
Assert.True(again.RaiseBell);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void One_failure_keeps_a_delivering_channel_degraded_for_seven_days()
|
||||
{
|
||||
var degraded = Fold(Fold(null, Fail(), at: T0).Next, Ok(), at: T0.AddMinutes(5)).Next;
|
||||
|
||||
Assert.Equal(NotificationChannelHealthStatus.Degraded, NotificationChannelHealthEvaluator.Evaluate(degraded, T0.AddDays(6)));
|
||||
Assert.Equal(NotificationChannelHealthStatus.Ok, NotificationChannelHealthEvaluator.Evaluate(degraded, T0.AddDays(7)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_failure_pushed_out_of_the_kept_outcomes_no_longer_degrades()
|
||||
{
|
||||
var row = Fold(null, Fail(), at: T0).Next;
|
||||
for (var i = 0; i < NotificationChannelHealth.RecentCapacity - 1; i++)
|
||||
row = Fold(row, Ok(), at: T0.AddMinutes(i + 1)).Next;
|
||||
Assert.Equal(NotificationChannelHealthStatus.Degraded, NotificationChannelHealthEvaluator.Evaluate(row, T0.AddHours(1)));
|
||||
|
||||
row = Fold(row, Ok(), at: T0.AddMinutes(30)).Next;
|
||||
Assert.Equal(NotificationChannelHealthStatus.Ok, NotificationChannelHealthEvaluator.Evaluate(row, T0.AddHours(1)));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Recent_keeps_the_last_twenty_outcomes()
|
||||
{
|
||||
var row = Fold(null, Fail()).Next;
|
||||
for (var i = 0; i < NotificationChannelHealth.RecentCapacity; i++)
|
||||
row = Fold(row, Ok()).Next;
|
||||
|
||||
Assert.Equal(new string('1', NotificationChannelHealth.RecentCapacity), row.Recent);
|
||||
var dto = NotificationChannelHealthEvaluator.ToDto("c1", row, T0);
|
||||
Assert.Equal(20, dto.RecentAttempts);
|
||||
Assert.Equal(0, dto.RecentFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_new_destination_starts_fresh_and_an_old_row_reads_as_unknown()
|
||||
{
|
||||
var failing = Fold(Fold(null, Fail()).Next, Fail()).Next;
|
||||
var moved = Webhook(url: "https://hooks.example.invalid/b");
|
||||
var movedFingerprint = NotificationChannelHealthEvaluator.Fingerprint(moved);
|
||||
|
||||
Assert.Null(NotificationChannelHealthEvaluator.Current(failing, movedFingerprint));
|
||||
Assert.Equal(NotificationChannelHealthStatus.Unknown,
|
||||
NotificationChannelHealthEvaluator.Evaluate(NotificationChannelHealthEvaluator.Current(failing, movedFingerprint), T0));
|
||||
|
||||
var fresh = Fold(failing, Ok(), channel: moved);
|
||||
Assert.Equal("1", fresh.Next.Recent);
|
||||
Assert.Equal(movedFingerprint, fresh.Next.Fingerprint);
|
||||
Assert.Equal(ChannelHealthTransition.None, fresh.Transition);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_fingerprint_follows_every_field_that_decides_the_destination()
|
||||
{
|
||||
var baseline = NotificationChannelHealthEvaluator.Fingerprint(Webhook());
|
||||
|
||||
Assert.Equal(baseline, NotificationChannelHealthEvaluator.Fingerprint(Webhook(name: "Renamed")));
|
||||
Assert.NotEqual(baseline, NotificationChannelHealthEvaluator.Fingerprint(Webhook(url: "https://hooks.example.invalid/b")));
|
||||
var headers = Webhook(); headers.CustomHeadersJson = "{\"Authorization\":\"Bearer x\"}";
|
||||
Assert.NotEqual(baseline, NotificationChannelHealthEvaluator.Fingerprint(headers));
|
||||
var signed = Webhook(); signed.SigningSecret = "s3cret";
|
||||
Assert.NotEqual(baseline, NotificationChannelHealthEvaluator.Fingerprint(signed));
|
||||
var slack = Webhook(); slack.ProviderType = (int)WebhookProviderType.Slack;
|
||||
Assert.NotEqual(baseline, NotificationChannelHealthEvaluator.Fingerprint(slack));
|
||||
Assert.Equal(16, baseline.Length);
|
||||
Assert.DoesNotContain("hooks", baseline);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(502, "Webhook returned HTTP 502: <html>body</html>", "HTTP 502 Bad Gateway")]
|
||||
[InlineData(404, "", "HTTP 404 Not Found")]
|
||||
[InlineData(599, "", "HTTP 599")]
|
||||
[InlineData(null, "Connection error: No such host is known.", "Connection error: No such host is known.")]
|
||||
[InlineData(null, "", "Delivery failed.")]
|
||||
public void DescribeFailure_uses_the_status_line_or_the_transport_message(int? status, string message, string expected)
|
||||
=> Assert.Equal(expected, NotificationChannelHealthEvaluator.DescribeFailure(new NotificationSendResult { StatusCode = status, Message = message }));
|
||||
|
||||
[Fact]
|
||||
public void DescribeFailure_caps_a_long_transport_message()
|
||||
{
|
||||
var described = NotificationChannelHealthEvaluator.DescribeFailure(new NotificationSendResult { Message = new string('x', 500) });
|
||||
Assert.Equal(NotificationChannelHealthEvaluator.MaxErrorLength, described.Length);
|
||||
Assert.EndsWith("…", described);
|
||||
}
|
||||
|
||||
// ── Recorder ──────────────────────────────────────────────────────────
|
||||
|
||||
private sealed class Clock : TimeProvider
|
||||
{
|
||||
public DateTime Now { get; set; } = T0;
|
||||
public override DateTimeOffset GetUtcNow() => new(Now);
|
||||
}
|
||||
|
||||
/// <summary>In-memory table with real ETag semantics; <see cref="BeforeWrite"/> lets a test slip in a competing writer.</summary>
|
||||
private sealed class FakeRepository : INotificationChannelHealthRepository
|
||||
{
|
||||
private readonly object _gate = new();
|
||||
private readonly Dictionary<(string, string), (NotificationChannelHealth Row, int Version)> _rows = new();
|
||||
|
||||
public Action<FakeRepository>? BeforeWrite { get; set; }
|
||||
public bool Throw { get; set; }
|
||||
public int Writes { get; private set; }
|
||||
|
||||
public NotificationChannelHealth? Get(string scope, string id)
|
||||
{
|
||||
lock (_gate) return _rows.TryGetValue((scope, id), out var e) ? e.Row : null;
|
||||
}
|
||||
|
||||
public Task<(NotificationChannelHealth? Row, string? ETag)> GetWithETagAsync(string scopeKey, string channelId)
|
||||
{
|
||||
if (Throw) throw new InvalidOperationException("storage down");
|
||||
lock (_gate)
|
||||
{
|
||||
return Task.FromResult<(NotificationChannelHealth?, string?)>(
|
||||
_rows.TryGetValue((scopeKey, channelId), out var e) ? (e.Row, e.Version.ToString()) : (null, null));
|
||||
}
|
||||
}
|
||||
|
||||
public Task<bool> TryUpsertAsync(NotificationChannelHealth row, string? ifMatchETag)
|
||||
{
|
||||
var hook = BeforeWrite;
|
||||
BeforeWrite = null;
|
||||
hook?.Invoke(this);
|
||||
lock (_gate)
|
||||
{
|
||||
var key = (row.ScopeKey, row.ChannelId);
|
||||
var exists = _rows.TryGetValue(key, out var current);
|
||||
if (ifMatchETag == null ? exists : !exists || current.Version.ToString() != ifMatchETag)
|
||||
return Task.FromResult(false);
|
||||
_rows[key] = (row, exists ? current.Version + 1 : 1);
|
||||
Writes++;
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
public Task<List<NotificationChannelHealth>> ListAsync(string scopeKey)
|
||||
{
|
||||
lock (_gate) return Task.FromResult(_rows.Where(r => r.Key.Item1 == scopeKey).Select(r => r.Value.Row).ToList());
|
||||
}
|
||||
|
||||
public Task<List<(NotificationChannelHealth Row, string ETag)>> ListAllAsync()
|
||||
{
|
||||
if (Throw) throw new InvalidOperationException("storage down");
|
||||
lock (_gate) return Task.FromResult(_rows.Values.Select(v => (v.Row, v.Version.ToString())).ToList());
|
||||
}
|
||||
|
||||
public Task<bool> TryDeleteAsync(NotificationChannelHealth row, string ifMatchETag)
|
||||
{
|
||||
var hook = BeforeWrite;
|
||||
BeforeWrite = null;
|
||||
hook?.Invoke(this);
|
||||
lock (_gate)
|
||||
{
|
||||
var key = (row.ScopeKey, row.ChannelId);
|
||||
if (!_rows.TryGetValue(key, out var current) || current.Version.ToString() != ifMatchETag)
|
||||
return Task.FromResult(false);
|
||||
_rows.Remove(key);
|
||||
return Task.FromResult(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Seeds a row as if its channel last sent at <paramref name="lastAttempt"/>.</summary>
|
||||
public void Seed(string scope, string channelId, DateTime lastAttempt)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
var version = _rows.TryGetValue((scope, channelId), out var existing) ? existing.Version + 1 : 1;
|
||||
_rows[(scope, channelId)] = (new NotificationChannelHealth
|
||||
{
|
||||
ScopeKey = scope, ChannelId = channelId, Fingerprint = "f", Recent = "1", LastAttemptUtc = lastAttempt, LastSuccessUtc = lastAttempt,
|
||||
}, version);
|
||||
}
|
||||
}
|
||||
|
||||
public bool Has(string scope, string channelId)
|
||||
{
|
||||
lock (_gate) return _rows.ContainsKey((scope, channelId));
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public FakeRepository Repository { get; } = new();
|
||||
public Mock<TenantNotificationService> TenantBell { get; } = new(Mock.Of<ITenantNotificationRepository>(), new FakeSignalRNotificationService(), NullLogger<TenantNotificationService>.Instance);
|
||||
public Mock<GlobalNotificationService> GlobalBell { get; } = new(Mock.Of<INotificationRepository>(), new FakeSignalRNotificationService(), NullLogger<GlobalNotificationService>.Instance);
|
||||
public Clock Clock { get; } = new();
|
||||
|
||||
public NotificationChannelHealthRecorder Recorder()
|
||||
=> new(Repository, TenantBell.Object, GlobalBell.Object, NullLogger<NotificationChannelHealthRecorder>.Instance, telemetry: null, time: Clock);
|
||||
|
||||
public void VerifyTenantBells(int times)
|
||||
=> TenantBell.Verify(b => b.CreateNotificationAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()), Times.Exactly(times));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Recorder_rings_the_tenant_bell_once_for_a_failing_channel()
|
||||
{
|
||||
var h = new Harness();
|
||||
var recorder = h.Recorder();
|
||||
var channel = Webhook();
|
||||
|
||||
await recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery);
|
||||
h.VerifyTenantBells(0);
|
||||
await recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery);
|
||||
await recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
h.VerifyTenantBells(1);
|
||||
h.TenantBell.Verify(b => b.CreateNotificationAsync(
|
||||
TenantKey,
|
||||
NotificationChannelHealthRecorder.TenantBellType,
|
||||
"Notification channel failing",
|
||||
"\"Service Desk\" could not deliver the last 2 notifications (HTTP 401 Unauthorized). Check the destination, then send a test.",
|
||||
NotificationChannelHealthRecorder.TenantSettingsHref), Times.Once);
|
||||
h.GlobalBell.Verify(b => b.CreateNotificationAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string?>()), Times.Never);
|
||||
Assert.Equal(3, h.Repository.Get(TenantKey, "c1")!.ConsecutiveFailures);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_writer_that_loses_the_race_to_the_bell_stamp_raises_no_second_bell()
|
||||
{
|
||||
var h = new Harness();
|
||||
var recorder = h.Recorder();
|
||||
var channel = Webhook();
|
||||
await recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
// A concurrent send stamps the episode between this writer's read and its write.
|
||||
h.Repository.BeforeWrite = repo =>
|
||||
{
|
||||
var (stored, etag) = repo.GetWithETagAsync(TenantKey, "c1").Result;
|
||||
var competitor = NotificationChannelHealthEvaluator.Apply(stored, TenantKey, "c1",
|
||||
NotificationChannelHealthEvaluator.Fingerprint(channel), Fail(), ChannelSendKind.Delivery, T0);
|
||||
Assert.True(competitor.RaiseBell);
|
||||
Assert.True(repo.TryUpsertAsync(competitor.Next, etag).Result);
|
||||
};
|
||||
await recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
h.VerifyTenantBells(0); // the competitor owned the bell; this writer re-read and saw the stamp
|
||||
var row = h.Repository.Get(TenantKey, "c1")!;
|
||||
Assert.Equal(3, row.ConsecutiveFailures);
|
||||
Assert.NotNull(row.FailingNotifiedUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_failures_raise_exactly_one_bell()
|
||||
{
|
||||
var h = new Harness();
|
||||
var recorder = h.Recorder();
|
||||
var channel = Webhook();
|
||||
|
||||
await Task.WhenAll(Enumerable.Range(0, 6).Select(_ => Task.Run(() => recorder.RecordAsync(Tenant, channel, Fail(), ChannelSendKind.Delivery))));
|
||||
|
||||
h.VerifyTenantBells(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_ops_channel_rings_the_global_bell()
|
||||
{
|
||||
var h = new Harness();
|
||||
var recorder = h.Recorder();
|
||||
var channel = Webhook(id: "ops");
|
||||
|
||||
await recorder.RecordAsync(NotificationScope.Platform, channel, Fail(), ChannelSendKind.Delivery);
|
||||
await recorder.RecordAsync(NotificationScope.Platform, channel, Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
h.GlobalBell.Verify(b => b.CreateNotificationAsync(NotificationChannelHealthRecorder.OpsBellType,
|
||||
It.IsAny<string>(), It.IsAny<string>(), NotificationChannelHealthRecorder.OpsSettingsHref), Times.Once);
|
||||
h.VerifyTenantBells(0);
|
||||
Assert.NotNull(h.Repository.Get("platform", "ops"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tests_never_ring_and_push_channels_are_not_recorded()
|
||||
{
|
||||
var h = new Harness();
|
||||
var recorder = h.Recorder();
|
||||
|
||||
await recorder.RecordAsync(Tenant, Webhook(), Fail(), ChannelSendKind.Test);
|
||||
await recorder.RecordAsync(Tenant, Webhook(), Fail(), ChannelSendKind.Test);
|
||||
var push = new NotificationChannel { Id = "push", Name = "Phones", ProviderType = (int)WebhookProviderType.Push, Enabled = true };
|
||||
await recorder.RecordAsync(Tenant, push, Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
h.VerifyTenantBells(0);
|
||||
Assert.Equal(2, h.Repository.Get(TenantKey, "c1")!.ConsecutiveFailures);
|
||||
Assert.Null(h.Repository.Get(TenantKey, "push"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_storage_failure_never_reaches_the_caller()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.Repository.Throw = true;
|
||||
|
||||
await h.Recorder().RecordAsync(Tenant, Webhook(), Fail(), ChannelSendKind.Delivery);
|
||||
|
||||
h.VerifyTenantBells(0);
|
||||
}
|
||||
|
||||
// ── Dispatcher ────────────────────────────────────────────────────────
|
||||
|
||||
private static NotificationChannelDispatcher Dispatcher(
|
||||
Mock<INotificationChannelHealthRecorder> recorder, Func<NotificationSendResult>? webhookResult = null, Mock<IPushChannelSender>? push = null)
|
||||
{
|
||||
var webhook = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
if (webhookResult == null)
|
||||
{
|
||||
webhook.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(Fail());
|
||||
}
|
||||
else
|
||||
{
|
||||
webhook.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(webhookResult);
|
||||
}
|
||||
var telegram = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance) { CallBase = false };
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>())).ReturnsAsync(Ok());
|
||||
return new NotificationChannelDispatcher(webhook.Object, telegram.Object, (push ?? new Mock<IPushChannelSender>()).Object,
|
||||
Mock.Of<IEmailChannelSender>(), recorder.Object, NullLogger<NotificationChannelDispatcher>.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Every_non_push_delivery_is_recorded_and_push_is_not()
|
||||
{
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
var dispatcher = Dispatcher(recorder, push: push);
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
var phones = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Push, Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram, phones }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"), It.Is<NotificationSendResult>(x => !x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.Is<NotificationSendResult>(x => x.Success), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(It.IsAny<NotificationScope>(), It.Is<NotificationChannel>(c => c.Id == "push"), It.IsAny<NotificationSendResult>(), It.IsAny<ChannelSendKind>()), Times.Never);
|
||||
push.Verify(p => p.SendAsync(Tenant, It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_test_send_is_recorded_as_a_test()
|
||||
{
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
|
||||
var result = await Dispatcher(recorder).SendTestAsync(Webhook(), new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
Assert.False(result.Success);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.IsAny<NotificationChannel>(), result, ChannelSendKind.Test), Times.Once);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_throwing_transport_is_recorded_as_a_failure_and_the_next_channel_still_sends()
|
||||
{
|
||||
var recorder = new Mock<INotificationChannelHealthRecorder>();
|
||||
var dispatcher = Dispatcher(recorder, webhookResult: () => throw new InvalidOperationException("boom"));
|
||||
var telegram = new NotificationChannel { Id = "tg", Name = "tg", ProviderType = (int)WebhookProviderType.Telegram, Url = "-100123", Enabled = true };
|
||||
|
||||
await dispatcher.SendToChannelsAsync(new[] { Webhook(), telegram }, new NotificationAlert { Title = "t", Summary = "s" }, Tenant);
|
||||
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "c1"),
|
||||
It.Is<NotificationSendResult>(x => !x.Success && x.Message == "Unexpected error: boom"), ChannelSendKind.Delivery), Times.Once);
|
||||
recorder.Verify(r => r.RecordAsync(Tenant, It.Is<NotificationChannel>(c => c.Id == "tg"), It.IsAny<NotificationSendResult>(), ChannelSendKind.Delivery), Times.Once);
|
||||
}
|
||||
|
||||
// ── Read endpoint projection ──────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public async Task The_health_list_follows_the_stored_channels()
|
||||
{
|
||||
var h = new Harness();
|
||||
var kept = Webhook(id: "kept");
|
||||
var moved = Webhook(id: "moved");
|
||||
var recorder = h.Recorder();
|
||||
await recorder.RecordAsync(Tenant, kept, Fail(), ChannelSendKind.Delivery);
|
||||
await recorder.RecordAsync(Tenant, kept, Fail(), ChannelSendKind.Delivery);
|
||||
await recorder.RecordAsync(Tenant, moved, Ok(), ChannelSendKind.Delivery);
|
||||
await recorder.RecordAsync(Tenant, Webhook(id: "deleted"), Ok(), ChannelSendKind.Delivery);
|
||||
|
||||
var function = new NotificationChannelHealthFunction(NullLogger<NotificationChannelHealthFunction>.Instance,
|
||||
null!, null!, h.Repository, h.Clock);
|
||||
var phones = new NotificationChannel { Id = "push", Name = "push", ProviderType = (int)WebhookProviderType.Push, Enabled = true };
|
||||
var never = Webhook(id: "never");
|
||||
var response = await function.BuildAsync(
|
||||
new[] { never, kept, Webhook(id: "moved", url: "https://hooks.example.invalid/new"), phones }, Tenant);
|
||||
|
||||
Assert.Equal(new[] { "never", "kept", "moved" }, response.Channels.Select(c => c.ChannelId));
|
||||
Assert.Equal(NotificationChannelHealthStatus.Unknown, response.Channels[0].Status);
|
||||
Assert.Equal(NotificationChannelHealthStatus.Failing, response.Channels[1].Status);
|
||||
Assert.Equal(2, response.Channels[1].RecentFailures);
|
||||
Assert.Equal("HTTP 401 Unauthorized", response.Channels[1].LastError);
|
||||
Assert.Equal(NotificationChannelHealthStatus.Unknown, response.Channels[2].Status); // destination changed since
|
||||
Assert.Null(response.Channels[2].LastAttemptUtc);
|
||||
}
|
||||
|
||||
// ── Orphan sweep (maintenance run) ────────────────────────────────────
|
||||
|
||||
private const string OtherTenant = "22222222-2222-2222-2222-222222222222";
|
||||
private static readonly DateTime Old = T0.AddDays(-NotificationChannelHealthMaintenance.OrphanRetentionDays - 1);
|
||||
private static readonly DateTime Recent = T0.AddDays(-NotificationChannelHealthMaintenance.OrphanRetentionDays + 1);
|
||||
|
||||
private static TenantConfiguration TenantWith(string tenantId, params string[] channelIds)
|
||||
{
|
||||
var config = TenantConfiguration.CreateDefault(tenantId);
|
||||
config.NotificationChannelsJson = NotificationChannel.SerializeList(channelIds.Select(id => Webhook(id: id)));
|
||||
return config;
|
||||
}
|
||||
|
||||
private static NotificationChannelHealthMaintenance Sweep(FakeRepository repository, Mock<IConfigRepository> configs)
|
||||
=> new(repository, configs.Object, NullLogger<NotificationChannelHealthMaintenance>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task The_sweep_deletes_old_rows_of_deleted_channels_and_keeps_everything_else()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed(TenantKey, "live-old", Old); // channel still configured, silent for a month
|
||||
repository.Seed(TenantKey, "gone-old", Old); // channel deleted, last send older than the grace
|
||||
repository.Seed(TenantKey, "gone-recent", Recent); // channel deleted, still inside the grace
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(TenantKey)).ReturnsAsync(TenantWith(TenantKey, "live-old", "never-sent"));
|
||||
|
||||
var result = await Sweep(repository, configs).RunAsync(T0);
|
||||
|
||||
Assert.True(repository.Has(TenantKey, "live-old"));
|
||||
Assert.False(repository.Has(TenantKey, "gone-old"));
|
||||
Assert.True(repository.Has(TenantKey, "gone-recent"));
|
||||
Assert.Equal(3, result.RowsScanned);
|
||||
Assert.Equal(1, result.RowsDeleted);
|
||||
Assert.Equal(0, result.ScopesSkipped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_config_read_skips_the_scope_and_deletes_nothing_there()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed(TenantKey, "gone-old", Old);
|
||||
repository.Seed(OtherTenant, "gone-old", Old);
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(TenantKey)).ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(OtherTenant)).ReturnsAsync(TenantWith(OtherTenant, "live"));
|
||||
|
||||
var result = await Sweep(repository, configs).RunAsync(T0);
|
||||
|
||||
Assert.True(repository.Has(TenantKey, "gone-old")); // read failure never reads as "no channels"
|
||||
Assert.False(repository.Has(OtherTenant, "gone-old")); // the other scope is still swept
|
||||
Assert.Equal(1, result.ScopesSkipped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_tenant_without_a_configuration_has_no_channels_left()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed(TenantKey, "any", Old);
|
||||
repository.Seed(TenantKey, "fresh", Recent);
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(TenantKey)).ReturnsAsync((TenantConfiguration?)null);
|
||||
|
||||
await Sweep(repository, configs).RunAsync(T0);
|
||||
|
||||
Assert.False(repository.Has(TenantKey, "any"));
|
||||
Assert.True(repository.Has(TenantKey, "fresh"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_synthesized_legacy_channel_counts_as_live()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed(TenantKey, TenantConfiguration.LegacyChannelId, Old);
|
||||
var legacy = TenantConfiguration.CreateDefault(TenantKey);
|
||||
legacy.WebhookUrl = "https://hooks.example.invalid/legacy";
|
||||
legacy.WebhookProviderType = (int)WebhookProviderType.GenericJson;
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(TenantKey)).ReturnsAsync(legacy);
|
||||
|
||||
await Sweep(repository, configs).RunAsync(T0);
|
||||
|
||||
Assert.True(repository.Has(TenantKey, TenantConfiguration.LegacyChannelId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ops_rows_follow_the_ops_channel_list_and_a_missing_admin_configuration_touches_nothing()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed("platform", "ops-live", Old);
|
||||
repository.Seed("platform", "ops-gone", Old);
|
||||
var admin = AdminConfiguration.CreateDefault();
|
||||
admin.OpsNotificationChannelsJson = NotificationChannel.SerializeList(new[] { Webhook(id: "ops-live") });
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetAdminConfigurationAsync()).ReturnsAsync(admin);
|
||||
|
||||
await Sweep(repository, configs).RunAsync(T0);
|
||||
Assert.True(repository.Has("platform", "ops-live"));
|
||||
Assert.False(repository.Has("platform", "ops-gone"));
|
||||
|
||||
repository.Seed("platform", "ops-gone", Old);
|
||||
configs.Setup(c => c.GetAdminConfigurationAsync()).ReturnsAsync((AdminConfiguration?)null);
|
||||
var result = await Sweep(repository, configs).RunAsync(T0);
|
||||
Assert.True(repository.Has("platform", "ops-gone"));
|
||||
Assert.Equal(1, result.ScopesSkipped);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_row_a_send_rewrote_during_the_sweep_survives()
|
||||
{
|
||||
var repository = new FakeRepository();
|
||||
repository.Seed(TenantKey, "gone-old", Old);
|
||||
var configs = new Mock<IConfigRepository>();
|
||||
configs.Setup(c => c.GetTenantConfigurationAsync(TenantKey)).ReturnsAsync(TenantWith(TenantKey));
|
||||
// A revert brought the channel back and it sent between the sweep's read and its delete.
|
||||
repository.BeforeWrite = repo => repo.Seed(TenantKey, "gone-old", T0);
|
||||
|
||||
var result = await Sweep(repository, configs).RunAsync(T0);
|
||||
|
||||
Assert.True(repository.Has(TenantKey, "gone-old"));
|
||||
Assert.Equal(0, result.RowsDeleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_listing_never_reaches_the_maintenance_run()
|
||||
{
|
||||
var repository = new FakeRepository { Throw = true };
|
||||
|
||||
var result = await Sweep(repository, new Mock<IConfigRepository>()).RunAsync(T0);
|
||||
|
||||
Assert.Equal(0, result.RowsScanned);
|
||||
}
|
||||
|
||||
// ── Table mapping ─────────────────────────────────────────────────────
|
||||
|
||||
[Fact]
|
||||
public void Mapping_roundtrips_every_field_and_omits_unset_dates()
|
||||
{
|
||||
var original = new NotificationChannelHealth
|
||||
{
|
||||
ScopeKey = TenantKey,
|
||||
ChannelId = "c1",
|
||||
Fingerprint = "0123456789abcdef",
|
||||
Recent = "1101",
|
||||
ConsecutiveFailures = 2,
|
||||
LastAttemptUtc = T0,
|
||||
LastSuccessUtc = T0.AddDays(-1),
|
||||
LastFailureUtc = T0,
|
||||
FailingSinceUtc = T0.AddHours(-1),
|
||||
LastStatusCode = 401,
|
||||
LastError = "HTTP 401 Unauthorized",
|
||||
FailingNotifiedUtc = T0,
|
||||
};
|
||||
|
||||
var back = TableNotificationChannelHealthRepository.MapFromEntity(TableNotificationChannelHealthRepository.MapToEntity(original));
|
||||
Assert.Equivalent(original, back, strict: true);
|
||||
|
||||
var sparse = TableNotificationChannelHealthRepository.MapToEntity(new NotificationChannelHealth { ScopeKey = TenantKey, ChannelId = "c2", Fingerprint = "f", Recent = "1", LastAttemptUtc = T0 });
|
||||
Assert.Null(sparse["FailingNotifiedUtc"]);
|
||||
var sparseBack = TableNotificationChannelHealthRepository.MapFromEntity(sparse);
|
||||
Assert.Null(sparseBack.LastStatusCode);
|
||||
Assert.Null(sparseBack.FailingNotifiedUtc);
|
||||
}
|
||||
}
|
||||
+2
-3
@@ -364,7 +364,7 @@ public class TenantOffboardingHandlerOrderingTests
|
||||
|
||||
h.Enumerator
|
||||
.Setup(e => e.EnumerateAsync(It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns<string, CancellationToken>((_, ct) => EmptyEnumerateAsync(ct));
|
||||
.Returns<string, CancellationToken>((_, _) => EmptyEnumerateAsync());
|
||||
|
||||
h.DrainProbe.Setup(p => p.IsCascadeCompletedAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
@@ -419,8 +419,7 @@ public class TenantOffboardingHandlerOrderingTests
|
||||
DrainPollCount = 0,
|
||||
};
|
||||
|
||||
private static async IAsyncEnumerable<string> EmptyEnumerateAsync(
|
||||
[EnumeratorCancellation] CancellationToken ct)
|
||||
private static async IAsyncEnumerable<string> EmptyEnumerateAsync()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
|
||||
-1
@@ -247,7 +247,6 @@ public class TenantOffboardingWorkerPoisonTests
|
||||
|
||||
Worker = new TenantOffboardingWorker(
|
||||
MainQueue.Object, PoisonQueue.Object, handler,
|
||||
BuildOpsService(),
|
||||
NullLogger<TenantOffboardingWorker>.Instance);
|
||||
}
|
||||
|
||||
|
||||
@@ -121,13 +121,17 @@ public class OffboardingSessionEnumeratorTests
|
||||
public Task<List<SessionSummary>> GetSessionsLeanAsync(string tenantId) => throw new NotImplementedException();
|
||||
public Task<List<SessionSummary>> GetAgentSilentSessionsAsync(string tenantId, DateTime silenceCutoff, DateTime hardCutoff) => throw new NotImplementedException();
|
||||
public Task<List<string>> GetAllTenantIdsAsync() => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionEventsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> DeleteSessionRuleResultsAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<int> BackfillSessionIndexAsync() => throw new NotImplementedException();
|
||||
public Task<int> CleanupGhostSessionIndexEntriesAsync() => throw new NotImplementedException();
|
||||
public Task<bool> IsSessionIndexEmptyAsync() => throw new NotImplementedException();
|
||||
public Task<List<OrphanedEventSession>> GetOrphanedEventSessionsAsync(TimeSpan gracePeriod) => throw new NotImplementedException();
|
||||
public Task DeleteEventSessionIndexEntryAsync(string tenantId, string sessionId) => throw new NotImplementedException();
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<IReadOnlyList<OrphanSessionHandle>> GetEventSessionIndexHandlesAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<OrphanSessionHandle?> GetEventSessionIndexHandleAsync(string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<string?> StampHandleInventoryDecrementedAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<bool> DeleteEventSessionIndexHandleAsync(string tenantId, string sessionId, string etag, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<int> DeleteOrphanSessionRowsAsync(string table, string tenantId, string sessionId, CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<HashSet<(string TenantId, string SessionId)>> GetEventTypeIndexSessionKeysAsync(CancellationToken ct = default) => throw new NotImplementedException();
|
||||
public Task<List<(string TenantId, string SessionId)>> GetAllSessionTombstoneKeysAsync() => throw new NotImplementedException();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Pins the per-table filter of the orphan-session sweep to the writer's key shape. A filter
|
||||
/// that names the wrong side (tenant where the session belongs, or vice versa) would either
|
||||
/// miss the session's rows or touch another session's — both silently.
|
||||
/// </summary>
|
||||
public class OrphanSessionRowFiltersTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionId = "22222222-2222-2222-2222-222222222222";
|
||||
|
||||
[Fact]
|
||||
public void Every_filter_table_has_a_filter_naming_both_ids()
|
||||
{
|
||||
foreach (var table in OrphanSessionRowFilters.Tables)
|
||||
{
|
||||
var filter = OrphanSessionRowFilters.For(table, TenantId, SessionId);
|
||||
Assert.Contains(TenantId, filter);
|
||||
Assert.Contains(SessionId, filter);
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Constants.TableNames.Events)]
|
||||
[InlineData(Constants.TableNames.RuleResults)]
|
||||
[InlineData(Constants.TableNames.Signals)]
|
||||
[InlineData(Constants.TableNames.DecisionTransitions)]
|
||||
[InlineData(Constants.TableNames.VulnerabilityReports)]
|
||||
public void Session_partition_tables_filter_on_the_composite_partition_key(string table)
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}_{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(Constants.TableNames.DeviceSnapshot)]
|
||||
[InlineData(Constants.TableNames.SessionTimeBreakdowns)]
|
||||
public void One_row_per_session_tables_filter_on_tenant_partition_and_session_row_key(string table)
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}' and RowKey eq '{SessionId}'", OrphanSessionRowFilters.For(table, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void AppInstallSummaries_filter_on_the_SessionId_column()
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{TenantId}' and SessionId eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.AppInstallSummaries, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void EventTypeIndex_filter_scans_the_tenant_range_and_matches_the_SessionId_column()
|
||||
{
|
||||
// RowKey is {invertedTicks}_{sessionId}; without a Sessions row the ticks are not reconstructible.
|
||||
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and SessionId eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.EventTypeIndex, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CveIndex_filter_scans_the_tenant_range_and_matches_the_session_row_key()
|
||||
{
|
||||
Assert.Equal($"PartitionKey ge '{TenantId}_' and PartitionKey lt '{TenantId}_~' and RowKey eq '{SessionId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.CveIndex, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionAnnotations_filter_names_every_lane()
|
||||
{
|
||||
var filter = OrphanSessionRowFilters.For(Constants.TableNames.SessionAnnotations, TenantId, SessionId);
|
||||
Assert.StartsWith($"PartitionKey eq '{TenantId}' and (", filter);
|
||||
foreach (var lane in AnnotationLanes.All)
|
||||
Assert.Contains($"RowKey eq '{SessionId}_{lane}'", filter);
|
||||
Assert.Equal(AnnotationLanes.All.Length - 1, filter.Split(" or ").Length - 1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SessionTenantLookup_filter_pins_the_row_to_this_tenant()
|
||||
{
|
||||
Assert.Equal($"PartitionKey eq '{SessionId}' and RowKey eq 'tenant' and TenantId eq '{TenantId}'",
|
||||
OrphanSessionRowFilters.For(Constants.TableNames.SessionTenantLookup, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Unknown_table_throws_instead_of_guessing()
|
||||
{
|
||||
Assert.Throws<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Sessions, TenantId, SessionId));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("not-a-guid", SessionId)]
|
||||
[InlineData(TenantId, "'; drop")]
|
||||
public void Non_guid_ids_throw(string tenantId, string sessionId)
|
||||
{
|
||||
Assert.ThrowsAny<ArgumentException>(() => OrphanSessionRowFilters.For(Constants.TableNames.Events, tenantId, sessionId));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Swept_tables_are_the_filter_tables_plus_inventory_pair_plus_handle()
|
||||
{
|
||||
var expected = OrphanSessionRowFilters.Tables
|
||||
.Concat(new[]
|
||||
{
|
||||
Constants.TableNames.SessionInventoryContributions,
|
||||
Constants.TableNames.SoftwareInventory,
|
||||
Constants.TableNames.EventSessionIndex,
|
||||
})
|
||||
.ToHashSet(StringComparer.Ordinal);
|
||||
Assert.Equal(expected, OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Backup;
|
||||
using AutopilotMonitor.Functions.Services.Deletion;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using AutopilotMonitor.Shared.Models.Deletion;
|
||||
using Azure.Data.Tables;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Behaviour of the orphan-session sweep: which handles become candidates, the re-checks right
|
||||
/// before the deletes, the inventory claim-then-decrement order, the handle as the last delete,
|
||||
/// the cap/budget, and the weekly read-only reconciliation. Storage is mocked at the repository
|
||||
/// seams; the ordered action log is the evidence for every ordering invariant.
|
||||
/// </summary>
|
||||
public class OrphanSessionSweeperTests
|
||||
{
|
||||
private const string TenantId = "11111111-1111-1111-1111-111111111111";
|
||||
private const string SessionA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
private const string SessionB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb";
|
||||
private const string SessionC = "cccccccc-cccc-4ccc-8ccc-cccccccccccc";
|
||||
|
||||
// A Wednesday: the weekly reconciliation is not due unless forced or overdue.
|
||||
private static readonly DateTime Now = new DateTime(2026, 10, 14, 10, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private sealed class Harness
|
||||
{
|
||||
public readonly Mock<IMaintenanceRepository> Maint = new();
|
||||
public readonly Mock<IVulnerabilityRepository> Vuln = new();
|
||||
public readonly Mock<ISessionDeletionInventoryReader> Reader = new();
|
||||
public readonly Mock<AdminConfigurationService> AdminConfig;
|
||||
public readonly Mock<OrphanSweepLockStore> LockStore;
|
||||
public readonly List<OpsEventEntry> Ops = new();
|
||||
public readonly List<string> Log = new();
|
||||
|
||||
public readonly HashSet<(string TenantId, string SessionId)> SessionKeys = new();
|
||||
public readonly List<OrphanSessionHandle> Handles = new();
|
||||
public readonly Dictionary<string, OrphanSessionHandle?> LiveHandles = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<string> RowsPresent = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<string> Tombstoned = new(StringComparer.Ordinal);
|
||||
public readonly Dictionary<string, List<DeletionDecrementKey>> Inventory = new(StringComparer.Ordinal);
|
||||
public readonly HashSet<(string TenantId, string SessionId)> IndexKeys = new();
|
||||
public readonly Dictionary<string, int> RowsPerTable = new(StringComparer.Ordinal);
|
||||
public readonly AdminConfiguration Config = new AdminConfiguration();
|
||||
|
||||
public DateTime Clock = Now;
|
||||
public TimeSpan Budget = TimeSpan.FromMinutes(5);
|
||||
public bool StampFails;
|
||||
public bool HandleDeleteFails;
|
||||
public bool DecrementFails;
|
||||
public bool LeaseHeld;
|
||||
public string? FailingTable;
|
||||
|
||||
public Harness()
|
||||
{
|
||||
Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>()))
|
||||
.Callback(() => Log.Add("sessions"))
|
||||
.ReturnsAsync(() => new HashSet<(string, string)>(SessionKeys));
|
||||
Maint.Setup(m => m.GetEventSessionIndexHandlesAsync(It.IsAny<CancellationToken>()))
|
||||
.Callback(() => Log.Add("handles"))
|
||||
.ReturnsAsync(() => Handles.ToList());
|
||||
Maint.Setup(m => m.GetEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
LiveHandles.TryGetValue(s, out var live) ? live : Handles.FirstOrDefault(h => h.SessionId == s));
|
||||
Maint.Setup(m => m.StampHandleInventoryDecrementedAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"stamp:{s}"))
|
||||
.ReturnsAsync(() => StampFails ? null : "etag-stamped");
|
||||
Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(It.IsAny<string>(), TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((string table, string _, string s, CancellationToken _) =>
|
||||
{
|
||||
Log.Add($"delete:{table}:{s}");
|
||||
if (string.Equals(table, FailingTable, StringComparison.Ordinal))
|
||||
throw new InvalidOperationException("storage said no");
|
||||
return Task.FromResult(RowsPerTable.TryGetValue(table, out var n) ? n : 1);
|
||||
});
|
||||
Maint.Setup(m => m.DeleteEventSessionIndexHandleAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, string, CancellationToken>((_, s, _, _) => Log.Add($"handle:{s}"))
|
||||
.ReturnsAsync(() => !HandleDeleteFails);
|
||||
Maint.Setup(m => m.GetEventTypeIndexSessionKeysAsync(It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => new HashSet<(string, string)>(IndexKeys));
|
||||
|
||||
Vuln.Setup(v => v.GetSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
Inventory.TryGetValue(s, out var keys) ? (IReadOnlyList<DeletionDecrementKey>?)keys : null);
|
||||
Vuln.Setup(v => v.DecrementSoftwareInventoryEntryAsync(TenantId, It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns((string _, string vendor, string name, string version, CancellationToken _) =>
|
||||
{
|
||||
Log.Add($"decrement:{name}");
|
||||
if (DecrementFails) throw new InvalidOperationException("counter busy");
|
||||
return Task.CompletedTask;
|
||||
});
|
||||
Vuln.Setup(v => v.DeleteSessionInventoryContributionsAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Callback<string, string, CancellationToken>((_, s, _) => Log.Add($"sideRow:{s}"))
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
Reader.Setup(r => r.GetSessionRowAsync(TenantId, It.IsAny<string>(), It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, IEnumerable<string> _, CancellationToken _) =>
|
||||
RowsPresent.Contains(s) ? new TableEntity(TenantId, s) : null);
|
||||
Reader.Setup(r => r.GetActiveSessionTombstoneAsync(TenantId, It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((string _, string s, CancellationToken _) =>
|
||||
Tombstoned.Contains(s) ? new TableEntity(TenantId, s) : null);
|
||||
|
||||
AdminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions()));
|
||||
AdminConfig.Setup(a => a.GetConfigurationAsync()).ReturnsAsync(() => Config);
|
||||
AdminConfig.Setup(a => a.UpdateAsync(It.IsAny<Func<AdminConfiguration, string?>>(), It.IsAny<string>(), It.IsAny<string?>()))
|
||||
.Returns((Func<AdminConfiguration, string?> mutate, string _, string? _) =>
|
||||
{
|
||||
mutate(Config);
|
||||
return Task.FromResult(new AdminConfigurationUpdateResult { Before = Config, After = Config });
|
||||
});
|
||||
|
||||
LockStore = new Mock<OrphanSweepLockStore>(new SessionDeletionMaintenanceFunctionTests.MaintenanceBlobStub());
|
||||
LockStore.Setup(l => l.AcquireLeaseAsync(It.IsAny<TimeSpan?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(() => LeaseHeld
|
||||
? throw new LeaseHeldException("held", new InvalidOperationException("lease"))
|
||||
: Task.FromResult(new Mock<Azure.Storage.Blobs.Specialized.BlobLeaseClient>().Object));
|
||||
}
|
||||
|
||||
public OrphanSessionHandle AddHandle(string sessionId, TimeSpan ingestAge, TimeSpan? writtenAge = null, string etag = "etag-1", bool decremented = false)
|
||||
{
|
||||
var handle = new OrphanSessionHandle
|
||||
{
|
||||
TenantId = TenantId,
|
||||
SessionId = sessionId,
|
||||
LastIngestAt = Now - ingestAge,
|
||||
WrittenAt = Now - (writtenAge ?? TimeSpan.FromDays(1)),
|
||||
ETag = etag,
|
||||
InventoryDecremented = decremented,
|
||||
};
|
||||
Handles.Add(handle);
|
||||
return handle;
|
||||
}
|
||||
|
||||
public OrphanSessionSweeper Build()
|
||||
{
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => { lock (Ops) Ops.Add(e); })
|
||||
.Returns(Task.CompletedTask);
|
||||
var ops = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance,
|
||||
TestNotifications.InertOpsAlertDispatch(AdminConfig.Object));
|
||||
return new OrphanSessionSweeper(Maint.Object, Vuln.Object, Reader.Object, AdminConfig.Object, LockStore.Object, ops,
|
||||
NullLogger<OrphanSessionSweeper>.Instance, () => Clock, Budget);
|
||||
}
|
||||
|
||||
public Task<OrphanSweepRunResult> RunCoreAsync(bool forceDeepCheck = false) => Build().RunCoreAsync(forceDeepCheck, CancellationToken.None);
|
||||
}
|
||||
|
||||
private static DeletionDecrementKey Key(string name) => new DeletionDecrementKey { Vendor = "v", Name = name, Version = "1" };
|
||||
|
||||
// ============================================================ candidates
|
||||
|
||||
[Fact]
|
||||
public async Task Candidates_are_handles_without_a_session_key_outside_both_grace_windows()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // orphan, old
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3)); // has a row
|
||||
h.SessionKeys.Add((TenantId, SessionB));
|
||||
h.AddHandle(SessionC, TimeSpan.FromHours(1)); // ingest grace
|
||||
h.AddHandle("dddddddd-dddd-4ddd-8ddd-dddddddddddd", TimeSpan.FromDays(3), writtenAge: TimeSpan.FromMinutes(30)); // restore grace
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.Candidates);
|
||||
Assert.Equal(SessionA, Assert.Single(result.Cleaned).SessionId);
|
||||
Assert.Equal(4, result.HandlesScanned);
|
||||
Assert.Equal(1, result.SessionKeys);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Sessions_are_drained_before_handles()
|
||||
{
|
||||
var h = new Harness();
|
||||
await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(new[] { "sessions", "handles" }, h.Log.Take(2));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Legacy_handles_are_flagged_and_come_first()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.AddHandle(SessionB, Now - OrphanSessionSweeper.LegacyHandleLastIngestAt);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(new[] { SessionB, SessionA }, result.Cleaned.Select(c => c.SessionId));
|
||||
Assert.True(result.Cleaned[0].Legacy);
|
||||
Assert.False(result.Cleaned[1].Legacy);
|
||||
}
|
||||
|
||||
// ============================================================ re-checks
|
||||
|
||||
[Fact]
|
||||
public async Task Handle_gone_at_recheck_is_skipped_silently()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.LiveHandles[SessionA] = null;
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleGone);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Handle_changed_since_the_scan_is_skipped()
|
||||
{
|
||||
var h = new Harness();
|
||||
var scanned = h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.LiveHandles[SessionA] = new OrphanSessionHandle
|
||||
{
|
||||
TenantId = TenantId, SessionId = SessionA, LastIngestAt = scanned.LastIngestAt, WrittenAt = scanned.WrittenAt, ETag = "etag-later",
|
||||
};
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleChanged);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Row_present_at_recheck_skips_everything()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.RowsPresent.Add(SessionA);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedRowReturned);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("delete:", StringComparison.Ordinal) || l.StartsWith("stamp:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tombstone_counts_as_afterCascade_and_the_session_is_still_cleaned()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Tombstoned.Add(SessionA);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.AfterCascade);
|
||||
Assert.True(Assert.Single(result.Cleaned).AfterCascade);
|
||||
}
|
||||
|
||||
// ============================================================ inventory + ordering
|
||||
|
||||
[Fact]
|
||||
public async Task Inventory_is_claimed_then_decremented_then_the_side_row_goes_before_the_tables()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
var stamp = h.Log.IndexOf($"stamp:{SessionA}");
|
||||
var firstDecrement = h.Log.IndexOf("decrement:app-1");
|
||||
var sideRow = h.Log.IndexOf($"sideRow:{SessionA}");
|
||||
var firstTable = h.Log.FindIndex(l => l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
var handle = h.Log.IndexOf($"handle:{SessionA}");
|
||||
Assert.True(0 <= stamp && stamp < firstDecrement && firstDecrement < sideRow && sideRow < firstTable && firstTable < handle,
|
||||
string.Join(" | ", h.Log));
|
||||
|
||||
var cleaned = Assert.Single(result.Cleaned);
|
||||
Assert.Equal(2, cleaned.InventoryKeysDecremented);
|
||||
Assert.Equal(2, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
|
||||
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.SessionInventoryContributions]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_rerun_with_the_stamp_set_skips_the_decrements_but_still_removes_the_side_row()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3), decremented: true);
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("stamp:", StringComparison.Ordinal) || l.StartsWith("decrement:", StringComparison.Ordinal));
|
||||
Assert.Contains($"sideRow:{SessionA}", h.Log);
|
||||
Assert.Single(result.Cleaned);
|
||||
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.SoftwareInventory]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_stamp_aborts_the_candidate_before_any_decrement_or_delete()
|
||||
{
|
||||
var h = new Harness { StampFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.SkippedHandleChanged);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.DoesNotContain(h.Log, l => l.StartsWith("decrement:", StringComparison.Ordinal) || l.StartsWith("delete:", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failed_decrement_is_counted_and_the_session_is_still_cleaned()
|
||||
{
|
||||
var h = new Harness { DecrementFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1"), Key("app-2") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(2, result.InventoryDecrementsFailed);
|
||||
Assert.Single(result.Cleaned);
|
||||
Assert.Contains($"handle:{SessionA}", h.Log);
|
||||
}
|
||||
|
||||
// ============================================================ the handle last
|
||||
|
||||
[Fact]
|
||||
public async Task Every_filter_table_is_cleared_and_the_handle_is_deleted_last_with_the_stamped_etag()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.Inventory[SessionA] = new List<DeletionDecrementKey> { Key("app-1") };
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
foreach (var table in OrphanSessionRowFilters.Tables)
|
||||
Assert.Contains($"delete:{table}:{SessionA}", h.Log);
|
||||
Assert.Equal($"handle:{SessionA}", h.Log.Last());
|
||||
h.Maint.Verify(m => m.DeleteEventSessionIndexHandleAsync(TenantId, SessionA, "etag-stamped", It.IsAny<CancellationToken>()), Times.Once);
|
||||
Assert.Equal(1, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
|
||||
Assert.Equal(OrphanSessionRowFilters.Tables.Length, Assert.Single(result.Cleaned).TotalRows);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_handle_rewritten_before_the_final_delete_is_kept_and_reported()
|
||||
{
|
||||
var h = new Harness { HandleDeleteFails = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
var cleaned = Assert.Single(result.Cleaned);
|
||||
Assert.True(cleaned.HandleKept);
|
||||
Assert.Equal(0, result.TotalsByTable[Constants.TableNames.EventSessionIndex]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_table_leaves_the_handle_and_the_run_continues_with_the_next_candidate()
|
||||
{
|
||||
var h = new Harness { FailingTable = Constants.TableNames.Signals };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(4));
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3));
|
||||
// Only session A's Signals delete fails: the harness throws for the table regardless of
|
||||
// session, so flip the failure off once A has been attempted.
|
||||
h.Maint.Setup(m => m.DeleteOrphanSessionRowsAsync(Constants.TableNames.Signals, TenantId, SessionB, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(1);
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(1, result.FailedSessions);
|
||||
Assert.DoesNotContain($"handle:{SessionA}", h.Log);
|
||||
Assert.Contains($"handle:{SessionB}", h.Log);
|
||||
Assert.Equal(SessionB, Assert.Single(result.Cleaned).SessionId);
|
||||
}
|
||||
|
||||
// ============================================================ cap + budget
|
||||
|
||||
[Fact]
|
||||
public async Task The_cap_leaves_the_rest_for_the_next_run()
|
||||
{
|
||||
var h = new Harness();
|
||||
for (var i = 0; i < OrphanSessionSweeper.MaxCandidatesPerRun + 3; i++)
|
||||
h.AddHandle(Guid.NewGuid().ToString(), TimeSpan.FromDays(2) + TimeSpan.FromMinutes(i));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.Equal(OrphanSessionSweeper.MaxCandidatesPerRun, result.Cleaned.Count);
|
||||
Assert.Equal(3, result.Remaining);
|
||||
Assert.False(result.BudgetExhausted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_exhausted_budget_stops_before_the_next_candidate()
|
||||
{
|
||||
var h = new Harness { Budget = TimeSpan.Zero };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
h.AddHandle(SessionB, TimeSpan.FromDays(3));
|
||||
|
||||
var result = await h.RunCoreAsync();
|
||||
|
||||
Assert.True(result.BudgetExhausted);
|
||||
Assert.Empty(result.Cleaned);
|
||||
Assert.Equal(2, result.Remaining);
|
||||
}
|
||||
|
||||
// ============================================================ weekly reconciliation
|
||||
|
||||
public static IEnumerable<object?[]> DeepCheckCases()
|
||||
{
|
||||
var sunday0845 = new DateTime(2026, 10, 18, 8, 45, 0, DateTimeKind.Utc);
|
||||
var sunday0445 = new DateTime(2026, 10, 18, 4, 45, 0, DateTimeKind.Utc);
|
||||
var tuesday = new DateTime(2026, 10, 20, 12, 45, 0, DateTimeKind.Utc);
|
||||
yield return new object?[] { null, sunday0845, true }; // never ran: first Sunday window
|
||||
yield return new object?[] { null, tuesday, false }; // never ran: wait for Sunday
|
||||
yield return new object?[] { sunday0845.AddDays(-7), sunday0845, true }; // weekly
|
||||
yield return new object?[] { sunday0845.AddDays(-7), sunday0445, false }; // Sunday, but before 08:00
|
||||
yield return new object?[] { sunday0845.AddDays(-3), sunday0845, false }; // ran this week already
|
||||
yield return new object?[] { tuesday.AddDays(-9), tuesday, true }; // missed Sunday: overdue
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(DeepCheckCases))]
|
||||
public void Deep_check_is_due_on_Sunday_mornings_or_when_overdue(DateTime? lastRun, DateTime now, bool due)
|
||||
{
|
||||
Assert.Equal(due, OrphanSessionSweeper.IsDeepCheckDue(lastRun, now));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_reports_only_sessions_without_a_row_and_without_a_handle_and_stamps_the_marker()
|
||||
{
|
||||
var h = new Harness();
|
||||
const string noHandleNoRow = "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee";
|
||||
const string noHandleButRow = "ffffffff-ffff-4fff-8fff-ffffffffffff";
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3)); // in the sweep's own pipeline
|
||||
h.SessionKeys.Add((TenantId, SessionB)); // live
|
||||
h.RowsPresent.Add(noHandleButRow); // registered after the drain
|
||||
h.IndexKeys.UnionWith(new[] { (TenantId, SessionA), (TenantId, SessionB), (TenantId, noHandleNoRow), (TenantId, noHandleButRow) });
|
||||
|
||||
var result = await h.RunCoreAsync(forceDeepCheck: true);
|
||||
|
||||
var reconcile = Assert.IsType<OrphanReconcileResult>(result.Reconcile);
|
||||
Assert.Equal(4, reconcile.IndexSessions);
|
||||
Assert.Equal(1, reconcile.ResidueSessions);
|
||||
Assert.Equal((TenantId, noHandleNoRow), Assert.Single(reconcile.Sample));
|
||||
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
|
||||
Assert.Equal(Now.ToString("o"), h.Config.OrphanDeepCheckLastRunUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Reconcile_without_residue_is_Info_and_is_not_due_again_mid_week()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.SessionKeys.Add((TenantId, SessionB));
|
||||
h.IndexKeys.Add((TenantId, SessionB));
|
||||
|
||||
var first = await h.RunCoreAsync(forceDeepCheck: true);
|
||||
var evt = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
|
||||
Assert.Equal(0, first.Reconcile!.ResidueSessions);
|
||||
|
||||
var second = await h.RunCoreAsync();
|
||||
Assert.Null(second.Reconcile);
|
||||
}
|
||||
|
||||
// ============================================================ RunAsync: lease + ops events
|
||||
|
||||
[Fact]
|
||||
public async Task A_timer_run_writes_the_heartbeat_and_EventsCleaned_only_when_something_was_cleaned()
|
||||
{
|
||||
var quiet = new Harness();
|
||||
Assert.True(await quiet.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
Assert.Single(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted && e.Severity == OpsEventSeverity.Info);
|
||||
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
|
||||
Assert.DoesNotContain(quiet.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
|
||||
var busy = new Harness();
|
||||
busy.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
Assert.True(await busy.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
var cleaned = Assert.Single(busy.Ops, e => e.EventType == OpsEventTypes.OrphanEventsCleaned);
|
||||
Assert.Equal(OpsEventSeverity.Warning, cleaned.Severity);
|
||||
Assert.Contains(SessionA, cleaned.Details ?? string.Empty);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_manual_run_always_includes_the_reconciliation()
|
||||
{
|
||||
var h = new Harness();
|
||||
Assert.True(await h.Build().RunAsync("alice@contoso.com", CancellationToken.None));
|
||||
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanReconcileCompleted);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_held_lease_records_SkippedLocked_and_touches_nothing()
|
||||
{
|
||||
var h = new Harness { LeaseHeld = true };
|
||||
h.AddHandle(SessionA, TimeSpan.FromDays(3));
|
||||
|
||||
Assert.False(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
|
||||
Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepSkippedLocked);
|
||||
Assert.Empty(h.Log);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_failing_scan_records_OrphanSweepFailed()
|
||||
{
|
||||
var h = new Harness();
|
||||
h.Maint.Setup(m => m.GetSessionKeysAsync(It.IsAny<CancellationToken>())).ThrowsAsync(new InvalidOperationException("storage down"));
|
||||
|
||||
Assert.True(await h.Build().RunAsync(OrphanSessionSweeper.TimerTrigger, CancellationToken.None));
|
||||
|
||||
var failed = Assert.Single(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepFailed);
|
||||
Assert.Equal(OpsEventSeverity.Error, failed.Severity);
|
||||
Assert.DoesNotContain(h.Ops, e => e.EventType == OpsEventTypes.OrphanSweepCompleted);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using AutopilotMonitor.Functions.Services;
|
||||
using AutopilotMonitor.Functions.Services.Maintenance;
|
||||
using AutopilotMonitor.Shared;
|
||||
using AutopilotMonitor.Shared.DataAccess;
|
||||
using AutopilotMonitor.Shared.Models;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace AutopilotMonitor.Functions.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Shape of the ops events the orphan sweep and the maintenance run write: the dashboards and the
|
||||
/// health pass read these details by name.
|
||||
/// </summary>
|
||||
public class OrphanSweepOpsEventTests
|
||||
{
|
||||
private static (OpsEventService Service, List<OpsEventEntry> Saved) Rig()
|
||||
{
|
||||
var saved = new List<OpsEventEntry>();
|
||||
var opsRepo = new Mock<IOpsEventRepository>();
|
||||
opsRepo.Setup(r => r.SaveOpsEventAsync(It.IsAny<OpsEventEntry>()))
|
||||
.Callback<OpsEventEntry>(e => { lock (saved) saved.Add(e); })
|
||||
.Returns(Task.CompletedTask);
|
||||
var adminConfig = new Mock<AdminConfigurationService>(
|
||||
Mock.Of<IConfigRepository>(), NullLogger<AdminConfigurationService>.Instance,
|
||||
new MemoryCache(new MemoryCacheOptions())) { CallBase = false };
|
||||
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
|
||||
return (new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch), saved);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task MaintenanceCompleted_carries_the_step_timings()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var steps = new Dictionary<string, long> { ["stalledSessions"] = 1200, ["aggregateMetrics"] = 90000 };
|
||||
|
||||
await service.RecordMaintenanceCompletedAsync(123456, "Timer", stepsMs: steps);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.MaintenanceCompleted, evt.EventType);
|
||||
Assert.Contains("\"stepsMs\"", evt.Details);
|
||||
Assert.Contains("\"aggregateMetrics\":90000", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanEventsCleaned_is_a_Warning_with_per_table_totals_and_the_legacy_count()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var result = new OrphanSweepRunResult { Remaining = 7 };
|
||||
var legacy = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-legacy", Legacy = true, InventoryKeysDecremented = 3, HadInventorySideRow = true };
|
||||
legacy.RowsByTable[Constants.TableNames.Events] = 18;
|
||||
legacy.RowsByTable[Constants.TableNames.EventTypeIndex] = 13;
|
||||
result.Add(legacy);
|
||||
var fresh = new OrphanSweepCleanedSession { TenantId = "t", SessionId = "s-fresh" };
|
||||
fresh.RowsByTable[Constants.TableNames.Events] = 1;
|
||||
result.Add(fresh);
|
||||
|
||||
await service.RecordOrphanEventsCleanedAsync(result);
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.OrphanEventsCleaned, evt.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Warning, evt.Severity);
|
||||
Assert.Contains("\"orphanSessions\":2", evt.Details);
|
||||
Assert.Contains("\"totalEventsDeleted\":19", evt.Details);
|
||||
Assert.Contains("\"legacySessions\":1", evt.Details);
|
||||
Assert.Contains("\"remaining\":7", evt.Details);
|
||||
Assert.Contains($"\"{Constants.TableNames.SoftwareInventory}\":3", evt.Details);
|
||||
Assert.Contains($"\"{Constants.TableNames.EventSessionIndex}\":2", evt.Details);
|
||||
Assert.Contains("\"legacy\":true", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanSweepCompleted_is_the_Info_heartbeat_with_the_counters()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var result = new OrphanSweepRunResult
|
||||
{
|
||||
HandlesScanned = 16784, SessionKeys = 16700, Candidates = 3, Remaining = 1, SkippedHandleGone = 2, DurationMs = 4200,
|
||||
Reconcile = new OrphanReconcileResult { IndexSessions = 16000, ResidueSessions = 0, DurationMs = 180000 },
|
||||
};
|
||||
|
||||
await service.RecordOrphanSweepCompletedAsync(result, "Timer");
|
||||
|
||||
var evt = Assert.Single(saved);
|
||||
Assert.Equal(OpsEventTypes.OrphanSweepCompleted, evt.EventType);
|
||||
Assert.Equal(OpsEventSeverity.Info, evt.Severity);
|
||||
Assert.Contains("\"handlesScanned\":16784", evt.Details);
|
||||
Assert.Contains("\"skippedHandleGone\":2", evt.Details);
|
||||
Assert.Contains("\"residueSessions\":0", evt.Details);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task OrphanReconcileCompleted_flips_to_Warning_with_a_sample_when_residue_exists()
|
||||
{
|
||||
var (service, saved) = Rig();
|
||||
var clean = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 0 };
|
||||
var dirty = new OrphanReconcileResult { IndexSessions = 10, ResidueSessions = 2, Sample = { ("t1", "s1"), ("t1", "s2") } };
|
||||
|
||||
await service.RecordOrphanReconcileCompletedAsync(clean);
|
||||
await service.RecordOrphanReconcileCompletedAsync(dirty);
|
||||
|
||||
Assert.Equal(OpsEventSeverity.Info, saved[0].Severity);
|
||||
Assert.Equal(OpsEventSeverity.Warning, saved[1].Severity);
|
||||
Assert.Contains("\"sessionId\":\"s2\"", saved[1].Details);
|
||||
Assert.Contains("\"sampleTruncated\":false", saved[1].Details);
|
||||
}
|
||||
}
|
||||
@@ -23,10 +23,22 @@ public class PushChannelRoutingTests
|
||||
private static readonly NotificationScope Tenant = NotificationScope.Tenant("11111111-1111-1111-1111-111111111111");
|
||||
|
||||
private static Mock<WebhookNotificationService> WebhookMock()
|
||||
=> new(new HttpClient(), NullLogger<WebhookNotificationService>.Instance) { CallBase = false };
|
||||
{
|
||||
var mock = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
mock.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
=> new(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance) { CallBase = false };
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel PushChannel(string id = "push") =>
|
||||
new() { Id = id, Name = id, ProviderType = (int)WebhookProviderType.Push, Url = null, Enabled = true };
|
||||
@@ -43,9 +55,9 @@ public class PushChannelRoutingTests
|
||||
await dispatcher.SendToChannelsAsync(new[] { PushChannel() }, alert, Tenant);
|
||||
|
||||
push.Verify(p => p.SendAsync(Tenant, alert), Times.Once);
|
||||
webhook.Verify(w => w.SendNotificationAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
webhook.Verify(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
telegram.Verify(t => t.SendOpsAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -53,10 +65,10 @@ public class PushChannelRoutingTests
|
||||
{
|
||||
var push = new Mock<IPushChannelSender>();
|
||||
push.Setup(p => p.SendWithResultAsync(It.IsAny<NotificationScope>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new WebhookTestResult { Success = true, Message = "ok" });
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(WebhookMock().Object, TelegramMock().Object, push.Object);
|
||||
|
||||
var result = await dispatcher.SendWithResultAsync(PushChannel(), new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
var result = await dispatcher.SendTestAsync(PushChannel(), new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
Assert.True(result.Success);
|
||||
push.Verify(p => p.SendWithResultAsync(NotificationScope.Platform, It.IsAny<NotificationAlert>()), Times.Once);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Azure.Data.Tables;
|
||||
@@ -133,6 +134,60 @@ public class SessionDeletionGuardTests
|
||||
Assert.Equal(SessionId, ex.SessionId);
|
||||
}
|
||||
|
||||
// ============================================================ GetLivenessAsync (derived writers) ====
|
||||
|
||||
[Fact]
|
||||
public async Task GetLivenessAsync_reports_Missing_when_the_row_does_not_exist()
|
||||
{
|
||||
// Unlike the ingest contract, a derived writer (queue handler, rescan) stops here: nothing
|
||||
// may be derived for a session the backend does not know.
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync((TableEntity?)null);
|
||||
|
||||
Assert.Equal(SessionLiveness.Missing, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
|
||||
reader.Verify(r => r.GetActiveSessionTombstoneAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Never);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(SessionDeletionState.Preparing)]
|
||||
[InlineData(SessionDeletionState.Queued)]
|
||||
[InlineData(SessionDeletionState.Running)]
|
||||
[InlineData(SessionDeletionState.Poisoned)]
|
||||
public async Task GetLivenessAsync_reports_Locked_for_every_lock_state(string state)
|
||||
{
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new TableEntity(TenantId, SessionId) { ["DeletionState"] = state });
|
||||
|
||||
Assert.Equal(SessionLiveness.Locked, await guard.GetLivenessAsync(TenantId, SessionId, "AnalyzeOnEnrollmentEnd"));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("None")]
|
||||
public async Task GetLivenessAsync_reports_Live_for_an_unlocked_row(string? state)
|
||||
{
|
||||
var guard = NewGuard(out var reader);
|
||||
var row = new TableEntity(TenantId, SessionId);
|
||||
if (state != null) row["DeletionState"] = state;
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(row);
|
||||
|
||||
Assert.Equal(SessionLiveness.Live, await guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityRescan"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task GetLivenessAsync_propagates_a_read_error()
|
||||
{
|
||||
// A storage failure must surface to the queue worker (retry/poison), never read as "missing".
|
||||
var guard = NewGuard(out var reader);
|
||||
reader.Setup(r => r.GetSessionRowAsync(TenantId, SessionId, It.IsAny<IEnumerable<string>>(), It.IsAny<CancellationToken>()))
|
||||
.ThrowsAsync(new Azure.RequestFailedException(503, "busy"));
|
||||
|
||||
await Assert.ThrowsAsync<Azure.RequestFailedException>(() => guard.GetLivenessAsync(TenantId, SessionId, "VulnerabilityCorrelate"));
|
||||
}
|
||||
|
||||
// ============================================================ Codex F3: tombstone-marker disambiguation ====
|
||||
|
||||
[Fact]
|
||||
|
||||
+3
-3
@@ -118,7 +118,7 @@ public class SessionDeletionPr3CodexFixesRound2Tests
|
||||
var harness = new ProducerHarness();
|
||||
harness.SetWrongState(SessionDeletionState.Preparing, "RACE-PREPARING");
|
||||
harness.SetSnapshotExists(true);
|
||||
harness.SetCas2WrongState(SessionDeletionState.Queued, "OTHER-PRODUCER-WON");
|
||||
harness.SetCas2WrongState("OTHER-PRODUCER-WON");
|
||||
|
||||
var result = await harness.Sut.EnqueueAsync(
|
||||
TenantId, SessionId, "admin_delete",
|
||||
@@ -147,7 +147,7 @@ public class SessionDeletionPr3CodexFixesRound2Tests
|
||||
Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
|
||||
Builder = new Mock<DeletionManifestBuilder>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(), NullLogger<DeletionManifestBuilder>.Instance);
|
||||
Mock.Of<ISessionDeletionInventoryReader>());
|
||||
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
@@ -213,7 +213,7 @@ public class SessionDeletionPr3CodexFixesRound2Tests
|
||||
});
|
||||
}
|
||||
|
||||
public void SetCas2WrongState(string fromState, string actualCurrentState)
|
||||
public void SetCas2WrongState(string actualCurrentState)
|
||||
{
|
||||
Storage.Setup(s => s.CasSetSessionDeletionStateAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(),
|
||||
|
||||
@@ -46,7 +46,7 @@ public class SessionDeletionPr3CodexFixesTests
|
||||
reader.Setup(r => r.QueryAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(EmptyAsyncEnumerable());
|
||||
|
||||
var builder = new DeletionManifestBuilder(reader.Object, NullLogger<DeletionManifestBuilder>.Instance);
|
||||
var builder = new DeletionManifestBuilder(reader.Object);
|
||||
|
||||
const string preAllocated = "PR3-PREALLOCATED-MANIFEST-ID";
|
||||
var manifest = await builder.BuildAsync(
|
||||
@@ -92,7 +92,7 @@ public class SessionDeletionPr3CodexFixesTests
|
||||
reader.Setup(r => r.QueryAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(EmptyAsyncEnumerable());
|
||||
|
||||
var builder = new DeletionManifestBuilder(reader.Object, NullLogger<DeletionManifestBuilder>.Instance);
|
||||
var builder = new DeletionManifestBuilder(reader.Object);
|
||||
|
||||
var manifest = await builder.BuildAsync(
|
||||
TenantId, SessionId, "preview",
|
||||
@@ -190,7 +190,7 @@ public class SessionDeletionPr3CodexFixesTests
|
||||
Mock.Of<TableServiceClient>(), NullLogger<TableStorageService>.Instance);
|
||||
|
||||
Builder = new Mock<DeletionManifestBuilder>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(), NullLogger<DeletionManifestBuilder>.Instance);
|
||||
Mock.Of<ISessionDeletionInventoryReader>());
|
||||
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
new Azure.Storage.Blobs.BlobServiceClient("UseDevelopmentStorage=true"),
|
||||
|
||||
@@ -228,8 +228,7 @@ public class SessionDeletionProducerTests
|
||||
|
||||
// DeletionManifestBuilder — needs ISessionDeletionInventoryReader + ILogger.
|
||||
Builder = new Mock<DeletionManifestBuilder>(
|
||||
Mock.Of<ISessionDeletionInventoryReader>(),
|
||||
NullLogger<DeletionManifestBuilder>.Instance);
|
||||
Mock.Of<ISessionDeletionInventoryReader>());
|
||||
|
||||
// BlobStorageService — internal test ctor with managedIdentity flag.
|
||||
Blob = new Mock<BlobStorageService>(
|
||||
|
||||
@@ -15,12 +15,12 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// Server-side free-text search (<c>q=</c>) for the dashboard search box. The predicate is the
|
||||
/// search grammar of <see cref="SessionSearchQuery"/> over the web dashboard's client-side
|
||||
/// searchable field set (minus derived-only tokens). Parity with the web parser is pinned by
|
||||
/// the shared case file <c>src/Web/autopilot-monitor-web/utils/session-search-syntax.cases.json</c>,
|
||||
/// which vitest runs against <c>app/dashboard/utils/sessionSearchQuery.ts</c>.
|
||||
/// the shared case file <c>src/Web/autopilot-monitor-web/app/dashboard/lib/session-search-syntax.cases.json</c>,
|
||||
/// which vitest runs against <c>app/dashboard/lib/sessionSearchQuery.ts</c>.
|
||||
/// </summary>
|
||||
public class SessionFreeTextSearchTests
|
||||
{
|
||||
private const string CasesRepoPath = "src/Web/autopilot-monitor-web/utils/session-search-syntax.cases.json";
|
||||
private const string CasesRepoPath = "src/Web/autopilot-monitor-web/app/dashboard/lib/session-search-syntax.cases.json";
|
||||
|
||||
private static SessionSummary Sample() => new()
|
||||
{
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// <summary>
|
||||
/// Cross-language parity anchor: reflects the wire-relevant catalogs out of
|
||||
/// AutopilotMonitor.Shared (models, enums, string vocabularies) into a canonical JSON
|
||||
/// manifest committed at <c>src/Web/autopilot-monitor-web/utils/shared-manifests.json</c>.
|
||||
/// manifest committed at <c>src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json</c>.
|
||||
/// The web's vitest suite and compile-time checks compare their hand-written TS mirrors
|
||||
/// against that manifest — so a C# change that would silently drift a TS mirror first
|
||||
/// fails HERE, and the TS side fails until its mirror follows.
|
||||
@@ -23,7 +23,7 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
/// </summary>
|
||||
public sealed class SharedManifestParityTests
|
||||
{
|
||||
private const string ManifestRepoPath = "src/Web/autopilot-monitor-web/utils/shared-manifests.json";
|
||||
private const string ManifestRepoPath = "src/Web/autopilot-monitor-web/lib/generated/shared-manifests.json";
|
||||
private const string WriteEnvVar = "AM_WRITE_SHARED_MANIFESTS";
|
||||
|
||||
[Fact]
|
||||
@@ -155,7 +155,7 @@ public sealed class SharedManifestParityTests
|
||||
// the portal's counters and upload budget read the same numbers the backend enforces.
|
||||
["submissionLimits"] = ConstInts(typeof(Constants.SubmissionLimits)),
|
||||
// Every IApiResponse + IApiRequest implementer + [WireContract] type, transitively closed —
|
||||
// the source of utils/wire-types.generated.ts. See WireTypeManifestBuilder.
|
||||
// the source of lib/generated/wire-types.generated.ts. See WireTypeManifestBuilder.
|
||||
["types"] = WireTypeManifestBuilder.BuildTypesSection(),
|
||||
};
|
||||
|
||||
|
||||
@@ -1012,7 +1012,7 @@ public class SlaTenantStatusLifecycleTests
|
||||
})
|
||||
.Returns(Task.CompletedTask);
|
||||
|
||||
var webhook = new WebhookNotificationService(new HttpClient(), NullLogger<WebhookNotificationService>.Instance);
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var channelDispatcher = new NotificationChannelDispatcher(webhook, new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance), Mock.Of<IPushChannelSender>());
|
||||
var alertDispatch = TestNotifications.InertOpsAlertDispatch(adminConfig.Object);
|
||||
var opsService = new OpsEventService(opsRepo.Object, NullLogger<OpsEventService>.Instance, alertDispatch);
|
||||
|
||||
@@ -81,6 +81,32 @@ public class TableLifecycleBucketTests
|
||||
Constants.TableNames.DelegationInvitations,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The orphan-session sweep is the cascade for sessions without a Sessions row, so it must
|
||||
/// clear exactly the tables the cascade manifest clears — minus the tombstone pair, which does
|
||||
/// not exist for such a session. A new manifest step without a sweep entry (or the reverse)
|
||||
/// fails here until someone decides the table's orphan lifecycle.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task OrphanSweep_ClearsEveryPerSessionCascadeTable_ExceptTheTombstonePair()
|
||||
{
|
||||
var cascade = await PerSessionCascadeTablesAsync();
|
||||
cascade.Remove(Constants.TableNames.Sessions);
|
||||
cascade.Remove(Constants.TableNames.SessionsIndex);
|
||||
|
||||
var swept = AutopilotMonitor.Functions.Services.Maintenance.OrphanSessionSweeper.SweptTables.ToHashSet(StringComparer.Ordinal);
|
||||
|
||||
var missingFromSweep = cascade.Except(swept, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
|
||||
var sweptButNotCascaded = swept.Except(cascade, StringComparer.Ordinal).OrderBy(t => t, StringComparer.Ordinal).ToList();
|
||||
|
||||
Assert.True(missingFromSweep.Count == 0,
|
||||
"Per-session cascade tables the orphan sweep does not clear — add them to OrphanSessionRowFilters.Tables "
|
||||
+ "(with the writer's key shape) or handle them in OrphanSessionSweeper:\n - " + string.Join("\n - ", missingFromSweep));
|
||||
Assert.True(sweptButNotCascaded.Count == 0,
|
||||
"Orphan sweep clears tables the cascade manifest does not know — add a manifest step first:\n - "
|
||||
+ string.Join("\n - ", sweptButNotCascaded));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void TableNamesConstants_SetEquals_TableNamesAll()
|
||||
{
|
||||
@@ -209,7 +235,7 @@ public class TableLifecycleBucketTests
|
||||
/// </summary>
|
||||
private static async Task<HashSet<string>> PerSessionCascadeTablesAsync()
|
||||
{
|
||||
var builder = new DeletionManifestBuilder(new EmptyInventoryReader(), NullLogger<DeletionManifestBuilder>.Instance);
|
||||
var builder = new DeletionManifestBuilder(new EmptyInventoryReader());
|
||||
var manifest = await builder.BuildAsync(
|
||||
TenantId, SessionId, "lifecycle_bucket_test",
|
||||
new DeletionActor { Type = "admin", Actor = "alice@contoso.com" },
|
||||
|
||||
+1
-1
@@ -404,7 +404,7 @@ public class TableOffboardingAuditRepositoryTests
|
||||
mockServiceClient.Setup(s => s.GetTableClient(It.IsAny<string>())).Returns(mockTableClient.Object);
|
||||
|
||||
var storage = new TableStorageService(mockServiceClient.Object, NullLogger<TableStorageService>.Instance);
|
||||
Sut = new TableOffboardingAuditRepository(storage, NullLogger<TableOffboardingAuditRepository>.Instance);
|
||||
Sut = new TableOffboardingAuditRepository(storage);
|
||||
}
|
||||
|
||||
private ETag StampEtag() => new($"\"0xFAKE_{++_etagCounter}\"");
|
||||
|
||||
@@ -18,11 +18,22 @@ namespace AutopilotMonitor.Functions.Tests;
|
||||
public class TelegramChannelProviderTests
|
||||
{
|
||||
private static Mock<WebhookNotificationService> WebhookMock()
|
||||
=> new(new HttpClient(), NullLogger<WebhookNotificationService>.Instance) { CallBase = false };
|
||||
{
|
||||
var mock = new Mock<WebhookNotificationService>(new HttpClient()) { CallBase = false };
|
||||
mock.Setup(w => w.SendAsync(It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static Mock<TelegramNotificationService> TelegramMock()
|
||||
=> new(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{
|
||||
var mock = new Mock<TelegramNotificationService>(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance)
|
||||
{ CallBase = false };
|
||||
mock.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true });
|
||||
return mock;
|
||||
}
|
||||
|
||||
private static NotificationChannel Channel(int providerType, string? url, string id = "c1", bool enabled = true)
|
||||
=> new() { Id = id, Name = id, ProviderType = providerType, Url = url, Enabled = enabled };
|
||||
@@ -43,8 +54,8 @@ public class TelegramChannelProviderTests
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Telegram, "-1003785642894") }, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendOpsAlertAsync("-1003785642894", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendNotificationAsync(
|
||||
telegram.Verify(t => t.SendAlertAsync("-1003785642894", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
@@ -60,10 +71,10 @@ public class TelegramChannelProviderTests
|
||||
await dispatcher.SendToChannelsAsync(
|
||||
new[] { Channel((int)WebhookProviderType.Slack, "https://hooks.slack.example/x") }, alert, NotificationScope.Platform);
|
||||
|
||||
webhook.Verify(w => w.SendNotificationAsync(
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://hooks.slack.example/x", WebhookProviderType.Slack, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
telegram.Verify(t => t.SendOpsAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -80,8 +91,8 @@ public class TelegramChannelProviderTests
|
||||
Channel((int)WebhookProviderType.GenericJson, "https://sales.example/hook", "sales"),
|
||||
}, alert, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendOpsAlertAsync("-100123", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendNotificationAsync(
|
||||
telegram.Verify(t => t.SendAlertAsync("-100123", alert), Times.Once);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
"https://sales.example/hook", WebhookProviderType.GenericJson, alert,
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Once);
|
||||
}
|
||||
@@ -99,27 +110,27 @@ public class TelegramChannelProviderTests
|
||||
Channel((int)WebhookProviderType.Slack, "", "slack"),
|
||||
}, new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
telegram.Verify(t => t.SendOpsAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendNotificationAsync(
|
||||
telegram.Verify(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()), Times.Never);
|
||||
webhook.Verify(w => w.SendAsync(
|
||||
It.IsAny<string>(), It.IsAny<WebhookProviderType>(), It.IsAny<NotificationAlert>(),
|
||||
It.IsAny<IReadOnlyDictionary<string, string>>(), It.IsAny<string>()), Times.Never);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task SendWithResult_RoutesTelegramToBot()
|
||||
public async Task SendTest_RoutesTelegramToBot()
|
||||
{
|
||||
var webhook = WebhookMock();
|
||||
var telegram = TelegramMock();
|
||||
telegram.Setup(t => t.SendAlertWithResultAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new WebhookTestResult { Success = true, Message = "ok" });
|
||||
telegram.Setup(t => t.SendAlertAsync(It.IsAny<string>(), It.IsAny<NotificationAlert>()))
|
||||
.ReturnsAsync(new NotificationSendResult { Success = true, Message = "ok" });
|
||||
var dispatcher = new NotificationChannelDispatcher(webhook.Object, telegram.Object, Mock.Of<IPushChannelSender>());
|
||||
|
||||
var result = await dispatcher.SendWithResultAsync(
|
||||
var result = await dispatcher.SendTestAsync(
|
||||
Channel((int)WebhookProviderType.Telegram, "@salesdesk"),
|
||||
new NotificationAlert { Title = "t", Summary = "s" }, NotificationScope.Platform);
|
||||
|
||||
Assert.True(result.Success);
|
||||
telegram.Verify(t => t.SendAlertWithResultAsync("@salesdesk", It.IsAny<NotificationAlert>()), Times.Once);
|
||||
telegram.Verify(t => t.SendAlertAsync("@salesdesk", It.IsAny<NotificationAlert>()), Times.Once);
|
||||
}
|
||||
|
||||
// ── Chat-ID validation ────────────────────────────────────────────────
|
||||
|
||||
@@ -36,7 +36,7 @@ public class TelemetryPayloadParserTests
|
||||
[InlineData("null")]
|
||||
public void ParseEvent_returns_null_for_missing_or_malformed_payload(string payload)
|
||||
{
|
||||
var result = TelemetryPayloadParser.ParseEvent(Dto("Event", payload), TenantId, SessionId);
|
||||
var result = TelemetryPayloadParser.ParseEvent(Dto("Event", payload));
|
||||
Assert.Null(result);
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ public class TelemetryPayloadParserTests
|
||||
"\"ReceivedAt\":\"2026-04-21T10:00:01Z\"" +
|
||||
"}";
|
||||
|
||||
var evt = TelemetryPayloadParser.ParseEvent(Dto("Event", payload), TenantId, SessionId);
|
||||
var evt = TelemetryPayloadParser.ParseEvent(Dto("Event", payload));
|
||||
|
||||
Assert.NotNull(evt);
|
||||
Assert.Equal("evt-123", evt!.EventId);
|
||||
|
||||
@@ -19,6 +19,7 @@ public class TenantNotificationAudienceCatalogTests
|
||||
[InlineData("sla_resolved", NotificationAudience.Member)]
|
||||
[InlineData("rule_frequency_regression", NotificationAudience.Admin)]
|
||||
[InlineData("app_version_duration_regression", NotificationAudience.Admin)]
|
||||
[InlineData("notification_channel_failing", NotificationAudience.Admin)]
|
||||
public void Resolve_KnownTypes_ReturnsExpectedAudience(string type, NotificationAudience expected)
|
||||
{
|
||||
Assert.Equal(expected, TenantNotificationAudienceCatalog.Resolve(type));
|
||||
|
||||
@@ -21,7 +21,7 @@ internal static class TestNotifications
|
||||
{
|
||||
internal static NotificationChannelDispatcher InertDispatcher()
|
||||
=> new(
|
||||
new WebhookNotificationService(new HttpClient(), NullLogger<WebhookNotificationService>.Instance),
|
||||
new WebhookNotificationService(new HttpClient()),
|
||||
new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(),
|
||||
NullLogger<TelegramNotificationService>.Instance),
|
||||
Mock.Of<IPushChannelSender>());
|
||||
|
||||
-2
@@ -320,11 +320,9 @@ public class VulnerabilityCorrelationServiceContributionsTests
|
||||
{
|
||||
return new VulnerabilityCorrelationService(
|
||||
vulnRepo.Object,
|
||||
sessionRepo: Mock.Of<ISessionRepository>(),
|
||||
nvdApi: null!,
|
||||
kevService: null!,
|
||||
msrcCorrelation: null!,
|
||||
adminConfig: null!,
|
||||
telemetryClient: null,
|
||||
logger: NullLogger<VulnerabilityCorrelationService>.Instance,
|
||||
opsEvents: opsEvents);
|
||||
|
||||
+2
-2
@@ -103,7 +103,7 @@ public class VulnerabilityCorrelationServiceTests
|
||||
private VulnerabilityCorrelationService CreateServiceForFilterTests()
|
||||
{
|
||||
// FilterByVersion doesn't use any dependencies — we can pass nulls
|
||||
return new VulnerabilityCorrelationService(null!, null!, null!, null!, null!, null!, null, null!);
|
||||
return new VulnerabilityCorrelationService(null!, null!, null!, null!, null, null!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -431,7 +431,7 @@ public class VulnerabilityCorrelationServiceTests
|
||||
|
||||
private VulnerabilityCorrelationService CreateServiceWithMappings(List<CpeMappingEntry> mappings)
|
||||
{
|
||||
var svc = new VulnerabilityCorrelationService(null!, null!, null!, null!, null!, null!, null, null!);
|
||||
var svc = new VulnerabilityCorrelationService(null!, null!, null!, null!, null, null!);
|
||||
svc._cpeMappings = mappings;
|
||||
return svc;
|
||||
}
|
||||
|
||||
@@ -222,10 +222,9 @@ public class WebhookConnectGateTests
|
||||
public async Task TestNotification_ReportsTheRefusal()
|
||||
{
|
||||
var resolver = new CountingResolver("10.0.0.8");
|
||||
var webhook = new WebhookNotificationService(ClientThrough(SsrfGuard.CreateConnectCallback(resolver.Resolve)),
|
||||
NullLogger<WebhookNotificationService>.Instance);
|
||||
var webhook = new WebhookNotificationService(ClientThrough(SsrfGuard.CreateConnectCallback(resolver.Resolve)));
|
||||
|
||||
var result = await webhook.SendNotificationWithResultAsync($"https://{Host}/hook", WebhookProviderType.GenericJson,
|
||||
var result = await webhook.SendAsync($"https://{Host}/hook", WebhookProviderType.GenericJson,
|
||||
new NotificationAlert { Title = "T", Summary = "S" });
|
||||
|
||||
Assert.False(result.Success);
|
||||
|
||||
@@ -87,7 +87,7 @@ public class WhatsNewNotificationServiceTests
|
||||
|
||||
public Harness()
|
||||
{
|
||||
var webhook = new WebhookNotificationService(new HttpClient(), NullLogger<WebhookNotificationService>.Instance);
|
||||
var webhook = new WebhookNotificationService(new HttpClient());
|
||||
var telegram = new TelegramNotificationService(new HttpClient(), Mock.Of<IConfigRepository>(), NullLogger<TelegramNotificationService>.Instance);
|
||||
Dispatcher = new Mock<NotificationChannelDispatcher>(webhook, telegram, Mock.Of<IPushChannelSender>());
|
||||
Dispatcher
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user