Files
Alphaeus Mote 28b2a53c29 feat(logging): one log line format everywhere, repeated warnings bounded [skip ci]
Every text line ExecuteR writes now reads
[2026-11-16T17:53:40.653Z] - [Category] - [LEVEL] - message key=value:
UTC time to the millisecond, a short category (the logging type without its
namespace, or Agent/Server), TRACE..FATAL, the rendered message and the
properties it did not use as key=value (correlation fields kept, enrichment
noise left out). Serilog writes it through LogLineFormatter (console, file
and the live views' message text); LOG_JSON keeps compact JSON. The watchdog
and its MSI/removal runners write the same format, and the Rust screen host
too (a tracing layer), with the agent taking each host line apart and logging
it once at the host's level and category, without a second timestamp.

LogLineParser reads the format back and the old ones still in device files
(services' [yyyy-MM-dd HH:mm:ss.fff LVL] Source: message, the watchdog's
...Z [LVL] message, tracing's default); the watchdog's live view uses it.

A warning, error or fatal that repeats (same level, SourceContext and
template) is written once, then counted for 60 s; the window's end writes one
line with repeated=N window=60s, a quiet window forgets it, at most 1,000 keys
are tracked and disposing writes open counts. The screen host does the same.

The console's log viewer shows the new level names and categories and copies
lines in the file format.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 08:55:11 -05:00

394 lines
16 KiB
C#

using System.Globalization;
using Serilog;
using Serilog.Core;
using Serilog.Events;
using Shared.Contracts.Api;
using Shared.Infrastructure.Logging;
using Xunit;
namespace Shared.Tests;
/// <summary>Captures what a logger writes, for formatting it by hand.</summary>
internal sealed class CollectingSink : ILogEventSink
{
public List<LogEvent> Events { get; } = [];
public void Emit(LogEvent logEvent)
{
lock (Events)
{
Events.Add(logEvent);
}
}
}
/// <summary>A clock that moves only when told.</summary>
internal sealed class ManualClock(DateTimeOffset start) : TimeProvider
{
public DateTimeOffset Now { get; private set; } = start;
public override DateTimeOffset GetUtcNow() => Now;
public void Advance(TimeSpan by) => Now += by;
}
/// <summary>ExecuteR's one text log format.</summary>
public sealed class LogLineFormatterTests
{
private static LogEvent Capture(Action<ILogger> log)
{
var sink = new CollectingSink();
using (var logger = new LoggerConfiguration().MinimumLevel.Verbose().Enrich.WithProperty("Application", "Agent").WriteTo.Sink(sink).CreateLogger())
{
log(logger);
}
return Assert.Single(sink.Events);
}
private static string Format(LogEvent logEvent)
{
using var text = new StringWriter(CultureInfo.InvariantCulture);
new LogLineFormatter("Agent").Format(logEvent, text);
return text.ToString();
}
[Fact]
public void ALineIsTimeCategoryLevelMessage_ThenTheUnusedPropertiesAsKeyValue()
{
var logEvent = Capture(logger => logger
.ForContext("SourceContext", "Agent.Fabric.EventBus")
.ForContext("CorrelationId", "c-1")
.ForContext("MachineName", "PC1")
.ForContext("RequestId", "r-1")
.Warning(
"The site's leaf could not be reached; the bus goes through the server meanwhile {Leaf}",
"wss://172.18.16.104:9001/"));
var withExtras = new LogEvent(
new DateTimeOffset(2026, 11, 16, 18, 53, 40, 687, TimeSpan.FromHours(1)),
logEvent.Level,
null,
logEvent.MessageTemplate,
logEvent.Properties.Select(p => new LogEventProperty(p.Key, p.Value)));
withExtras.AddOrUpdateProperty(new LogEventProperty("reason", new ScalarValue("TLS: The remote certificate is invalid")));
withExtras.AddOrUpdateProperty(new LogEventProperty("signer", new ScalarValue(null)));
withExtras.AddOrUpdateProperty(new LogEventProperty("count", new ScalarValue(1)));
withExtras.AddOrUpdateProperty(new LogEventProperty("ok", new ScalarValue(true)));
Assert.Equal(
"[2026-11-16T17:53:40.687Z] - [EventBus] - [WARN] - The site's leaf could not be reached; the bus goes through the server meanwhile wss://172.18.16.104:9001/"
+ " CorrelationId=c-1 reason=\"TLS: The remote certificate is invalid\" signer=null count=1 ok=true" + Environment.NewLine,
Format(withExtras));
}
[Fact]
public void WithoutASourceTheCategoryIsTheApplication_AndAnExceptionFollows()
{
var logEvent = Capture(logger => logger.Error(new InvalidOperationException("broke"), "It failed for {JobId}", Guid.Empty));
var lines = Format(logEvent).Split(Environment.NewLine);
Assert.Matches(@"^\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\] - \[Agent\] - \[ERROR\] - It failed for 00000000-0000-0000-0000-000000000000$", lines[0]);
Assert.StartsWith("System.InvalidOperationException: broke", lines[1]);
}
[Fact]
public void StringsInTheMessageAreUnquoted_AndStructuresAreJson()
{
var logEvent = Capture(logger => logger
.ForContext("Extra", new { A = 1 }, destructureObjects: true)
.Information("Path {Path} took {Ms:0.0} ms for {@Item}", "C:\\x y", 1.234, new { B = "c" }));
Assert.EndsWith("- [INFO] - Path C:\\x y took 1.2 ms for {\"B\":\"c\"} Extra=\"{\\\"A\\\":1}\"" + Environment.NewLine, Format(logEvent));
}
[Theory]
[InlineData("Agent.Screen.ScreenHost", "ScreenHost")]
[InlineData("Agent.Screen.ScreenHost+Session", "ScreenHost.Session")]
[InlineData("Shared.Infrastructure.Cache`1[[System.String, System.Private.CoreLib]]", "Cache")]
[InlineData("Shared.Infrastructure.Cache`1+Entry", "Cache.Entry")]
[InlineData("Backend.Things.Repository<Backend.Device>", "Repository")]
[InlineData("Microsoft.Hosting.Lifetime", "Lifetime")]
[InlineData("ScreenHost", "ScreenHost")]
[InlineData("ironrdp_server", "ironrdp_server")]
[InlineData(null, "Server")]
[InlineData(" ", "Server")]
public void TheCategoryIsTheTypeNameAlone(string? source, string category)
=> Assert.Equal(category, LogLineFormatter.Category(source, "Server"));
[Theory]
[InlineData("plain", "plain")]
[InlineData("C:\\Windows", "C:\\Windows")]
[InlineData("two words", "\"two words\"")]
[InlineData("a=b", "\"a=b\"")]
[InlineData("say \"hi\"", "\"say \\\"hi\\\"\"")]
[InlineData("line\nnext", "\"line\\nnext\"")]
[InlineData("", "\"\"")]
public void AValueIsQuotedOnlyWhenItMustBe(string value, string written)
=> Assert.Equal(written, LogLineFormatter.Quote(value));
[Theory]
[InlineData(LogEventLevel.Verbose, "TRACE")]
[InlineData(LogEventLevel.Debug, "DEBUG")]
[InlineData(LogEventLevel.Information, "INFO")]
[InlineData(LogEventLevel.Warning, "WARN")]
[InlineData(LogEventLevel.Error, "ERROR")]
[InlineData(LogEventLevel.Fatal, "FATAL")]
public void LevelsAreSpelledOut(LogEventLevel level, string name)
=> Assert.Equal(name, LogLineFormatter.LevelName(level));
}
/// <summary>Reading log lines back, in this format and the ones before it.</summary>
public sealed class LogLineParserTests
{
[Fact]
public void ALineInTheFormatReadsBack()
{
var line = LogLineFormatter.Line(
new DateTimeOffset(2026, 11, 16, 17, 53, 40, 653, TimeSpan.Zero), "Fabric.Leaf", "INFO",
"The site's leaf answers again; moving back to it from the hub leaf=wss://172.18.16.104:9001");
Assert.True(LogLineParser.TryParse(line, out var parsed));
Assert.Equal(new DateTimeOffset(2026, 11, 16, 17, 53, 40, 653, TimeSpan.Zero), parsed.TimestampUtc);
Assert.Equal(LogLevels.Information, parsed.Level);
Assert.Equal("Fabric.Leaf", parsed.Category);
Assert.Equal("The site's leaf answers again; moving back to it from the hub leaf=wss://172.18.16.104:9001", parsed.Message);
}
[Fact]
public void AServiceLineFromBeforeReadsBack_InLocalTime()
{
Assert.True(LogLineParser.TryParse("[2026-11-06 12:00:00.250 WRN] Agent.Service.AgentSession: Lost it: twice", out var parsed));
Assert.Equal(new DateTimeOffset(new DateTime(2026, 11, 6, 12, 0, 0, 250, DateTimeKind.Local)), parsed.TimestampUtc);
Assert.Equal(TimeSpan.Zero, parsed.TimestampUtc.Offset);
Assert.Equal(LogLevels.Warning, parsed.Level);
Assert.Equal("Agent.Service.AgentSession", parsed.Category);
Assert.Equal("Lost it: twice", parsed.Message);
Assert.True(LogLineParser.TryParse("[2026-11-06 12:00:00.250 INF] : no source", out var bare));
Assert.Null(bare.Category);
Assert.Equal("no source", bare.Message);
}
[Fact]
public void AWatchdogLineFromBeforeReadsBack_InUtc()
{
Assert.True(LogLineParser.TryParse("2026-11-06 12:00:01.000Z [ERR] Restarted the agent", out var parsed));
Assert.Equal(new DateTimeOffset(2026, 11, 6, 12, 0, 1, TimeSpan.Zero), parsed.TimestampUtc);
Assert.Equal(LogLevels.Error, parsed.Level);
Assert.Null(parsed.Category);
Assert.Equal("Restarted the agent", parsed.Message);
}
[Fact]
public void AnOlderScreenHostLineReadsBack()
{
Assert.True(LogLineParser.TryParse("2026-11-06T12:00:01.123456789Z WARN executer_screen::video: hardware H.264 encoder failed reason=x", out var parsed));
Assert.Equal(new DateTimeOffset(2026, 11, 6, 12, 0, 1, 123, TimeSpan.Zero).AddTicks(4567), parsed.TimestampUtc);
Assert.Equal(LogLevels.Warning, parsed.Level);
Assert.Equal("executer_screen::video", parsed.Category);
Assert.Equal("hardware H.264 encoder failed reason=x", parsed.Message);
}
[Theory]
[InlineData(" at Agent.Thing.Do() in C:\\src\\Thing.cs:line 4")]
[InlineData("just words")]
[InlineData("")]
[InlineData(null)]
public void AnythingElseIsNotALine(string? line)
=> Assert.False(LogLineParser.TryParse(line, out _));
}
/// <summary>Repeated warnings and errors are written once, then counted.</summary>
public sealed class RepeatThrottleSinkTests
{
private readonly ManualClock clock = new(new DateTimeOffset(2026, 11, 16, 12, 0, 0, TimeSpan.Zero));
private readonly CollectingSink written = new();
private RepeatThrottleSink Throttle(int maxKeys = RepeatThrottleSink.DefaultMaxKeys)
=> new(written, clock, maxKeys: maxKeys, sweepEvery: Timeout.InfiniteTimeSpan);
private static LogEvent Event(LogEventLevel level, string template, params object[] values)
{
var sink = new CollectingSink();
using (var logger = new LoggerConfiguration().MinimumLevel.Verbose().WriteTo.Sink(sink).CreateLogger())
{
logger.ForContext("SourceContext", "Agent.Fabric.EventBus").Write(level, template, values);
}
return sink.Events[0];
}
private static string Text(LogEvent logEvent) => LogLineFormatter.RenderMessage(logEvent);
[Fact]
public void TheFirstIsWritten_RepeatsAreCounted_AndTheWindowEndsWithOneSummary()
{
using var throttle = Throttle();
for (var i = 1; i <= 5; i++)
{
throttle.Emit(Event(LogEventLevel.Warning, "Leaf unreachable {Attempt}", i));
clock.Advance(TimeSpan.FromSeconds(5));
}
Assert.Equal(["Leaf unreachable 1"], written.Events.Select(Text));
throttle.Sweep();
Assert.Single(written.Events);
clock.Advance(TimeSpan.FromSeconds(40));
throttle.Sweep();
Assert.Equal(2, written.Events.Count);
Assert.Equal("Leaf unreachable 5 repeated=4 window=60s", Text(written.Events[1]));
Assert.Equal(LogEventLevel.Warning, written.Events[1].Level);
Assert.Equal(clock.Now, written.Events[1].Timestamp);
}
[Fact]
public void ItKeepsSummarizingOncePerWindow_AndAQuietWindowStartsOver()
{
using var throttle = Throttle();
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 1));
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 2));
clock.Advance(TimeSpan.FromSeconds(60));
throttle.Sweep();
Assert.Equal(["Broke 1", "Broke 2 repeated=1 window=60s"], written.Events.Select(Text));
// Still repeating: counted in the next window too.
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 3));
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 4));
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 5));
clock.Advance(TimeSpan.FromSeconds(60));
throttle.Sweep();
Assert.Equal("Broke 5 repeated=3 window=60s", Text(written.Events[^1]));
Assert.Equal(1, throttle.Tracked);
// A whole window without it: forgotten, so the next is written at once.
clock.Advance(TimeSpan.FromSeconds(60));
throttle.Sweep();
Assert.Equal(0, throttle.Tracked);
throttle.Emit(Event(LogEventLevel.Error, "Broke {N}", 6));
Assert.Equal("Broke 6", Text(written.Events[^1]));
Assert.Equal(4, written.Events.Count);
}
[Fact]
public void ARepeatAfterTheWindowWithoutASweepStillGetsItsSummaryFirst()
{
using var throttle = Throttle();
throttle.Emit(Event(LogEventLevel.Warning, "Slow {Ms}", 1));
throttle.Emit(Event(LogEventLevel.Warning, "Slow {Ms}", 2));
clock.Advance(TimeSpan.FromSeconds(61));
throttle.Emit(Event(LogEventLevel.Warning, "Slow {Ms}", 3));
Assert.Equal(["Slow 1", "Slow 2 repeated=1 window=60s"], written.Events.Select(Text));
clock.Advance(TimeSpan.FromSeconds(61));
throttle.Emit(Event(LogEventLevel.Warning, "Slow {Ms}", 4));
Assert.Equal("Slow 3 repeated=1 window=60s", Text(written.Events[^1]));
}
[Fact]
public void InformationAndBelowAreNeverHeldBack()
{
using var throttle = Throttle();
for (var i = 0; i < 3; i++)
{
throttle.Emit(Event(LogEventLevel.Information, "Polled {N}", i));
throttle.Emit(Event(LogEventLevel.Debug, "Polled {N}", i));
}
Assert.Equal(6, written.Events.Count);
Assert.Equal(0, throttle.Tracked);
}
[Fact]
public void LevelSourceAndTemplateTellEventsApart_AndABarePlaceholderByItsText()
{
using var throttle = Throttle();
throttle.Emit(Event(LogEventLevel.Warning, "A {N}", 1));
throttle.Emit(Event(LogEventLevel.Error, "A {N}", 1));
throttle.Emit(Event(LogEventLevel.Warning, "B {N}", 1));
var other = Event(LogEventLevel.Warning, "A {N}", 2);
other.AddOrUpdateProperty(new LogEventProperty("SourceContext", new ScalarValue("Agent.Other")));
throttle.Emit(other);
throttle.Emit(Event(LogEventLevel.Warning, "{HostLine}", "one thing"));
throttle.Emit(Event(LogEventLevel.Warning, "{HostLine}", "another thing"));
throttle.Emit(Event(LogEventLevel.Warning, "{HostLine}", "one thing"));
Assert.Equal(6, written.Events.Count);
Assert.Equal(6, throttle.Tracked);
}
[Fact]
public void TheTableIsBounded_AndAnEvictedCountIsWritten()
{
using var throttle = Throttle(maxKeys: 2);
throttle.Emit(Event(LogEventLevel.Warning, "First {N}", 1));
throttle.Emit(Event(LogEventLevel.Warning, "First {N}", 2));
throttle.Emit(Event(LogEventLevel.Warning, "Second {N}", 1));
throttle.Emit(Event(LogEventLevel.Warning, "Third {N}", 1));
Assert.Equal(2, throttle.Tracked);
Assert.Equal(["First 1", "Second 1", "First 2 repeated=1 window=60s", "Third 1"], written.Events.Select(Text));
}
[Fact]
public void DisposingWritesEveryOpenCount()
{
var throttle = Throttle();
throttle.Emit(Event(LogEventLevel.Warning, "A {N}", 1));
throttle.Emit(Event(LogEventLevel.Warning, "A {N}", 2));
throttle.Emit(Event(LogEventLevel.Warning, "A {N}", 3));
throttle.Emit(Event(LogEventLevel.Error, "B {N}", 1));
throttle.Dispose();
Assert.Equal(["A 1", "B 1", "A 3 repeated=2 window=60s"], written.Events.Select(Text));
}
[Fact]
public void TheLoggerThrottles_AndClosingItWritesTheCount_InTheFileToo()
{
var folder = Directory.CreateTempSubdirectory("log-format-").FullName;
try
{
var live = new LiveLog();
using (var logger = LoggingSetup.CreateLogger(new LogSettings { Level = "Debug", Directory = folder }, "Agent", live, clock))
{
var bus = logger.ForContext("SourceContext", "Agent.Fabric.EventBus");
for (var i = 0; i < 3; i++)
{
bus.Warning("A key or trust mismatch was reported {Kind} with {Password}", "server-reply", "hunter2");
}
logger.Information("Started");
}
var lines = live.Read(null, null).Lines;
Assert.Equal(3, lines.Count);
Assert.Equal("A key or trust mismatch was reported server-reply with [redacted]", lines[0].Message);
Assert.Equal("Agent.Fabric.EventBus", lines[0].Source);
Assert.Equal("Agent", lines[1].Source);
Assert.Equal("A key or trust mismatch was reported server-reply with [redacted] repeated=2 window=60s", lines[2].Message);
var file = Assert.Single(Directory.GetFiles(folder, "*.log"));
var text = File.ReadAllLines(file);
Assert.Equal(3, text.Length);
Assert.Equal(
"[2026-11-16T12:00:00.000Z] - [EventBus] - [WARN] - A key or trust mismatch was reported server-reply with [redacted] repeated=2 window=60s",
text[2]);
Assert.Matches(@"^\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\] - \[Agent\] - \[INFO\] - Started$", text[1]);
Assert.DoesNotContain(text, l => l.Contains("hunter2", StringComparison.Ordinal));
Assert.All(text, l => Assert.True(LogLineParser.TryParse(l, out _), l));
}
finally
{
Directory.Delete(folder, recursive: true);
}
}
}