Files
ExecuteR/tests/Agent.Tests/AgentUpgradeHandlerTests.cs
Alphaeus Mote 74d998e0ab feat: keep agent files in version folders, optionally offloaded with short-lived links
Agent files live in one folder per version (<version>/<file>, the version
from the package's VERSION marker) on disk and, with agent offload on, in the
object store under executer/agent/<version>/. Offload now works with the
built-in store as well as an external one (the chosen store is used), and a
removed version's folder leaves the store too. Keys recorded against another
store, endpoint or bucket are dropped and re-uploaded, so a link never points
at a store without the file.

Downloads ask this server every time: GET /api/agent-packages/{sha256} and
the install package route answer with a 302 (Cache-Control: no-store) to a
presigned link made at that moment, lasting the new agent link lifetime
(s3.agentLinkSeconds, 30-3600 s, default 300). The built-in store's link is
on PUBLIC_SERVER_URL, else the request's forwarded scheme and host
(PublicUrl.Resolve); an external store's is its own. With offload off, or the
store unusable, the server streams the file as before. Upgrade commands always
carry the server path, so a short lifetime holds however long a command waits.

The agent follows the redirect itself (up to 5 hops, relative and https to
http included) and still checks size and SHA-256. The per-type install
package upload is replaced by the catalog's offload pass, which "Upload Now"
runs on demand.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 17:40:22 -05:00

486 lines
19 KiB
C#

using System.IO.Compression;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using Agent.Upgrade;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Shared.Infrastructure.Routing;
using Xunit;
namespace Agent.Tests;
public sealed class AgentUpgradeHandlerTests : IDisposable
{
private const string Current = "2026.1.1.1000";
private const string Target = "2026.2.1.1000";
private readonly string root = Path.Combine(Path.GetTempPath(), "executer-upgrade-tests", Guid.NewGuid().ToString("N"));
private readonly UpgradeFolders folders;
private readonly RecordingEvents events = new();
private readonly FakeWatchdog watchdog = new();
private readonly List<string> timeline = [];
private DateTimeOffset now = new(2026, 10, 23, 12, 0, 0, TimeSpan.Zero);
private byte[] package;
private Func<HttpRequestMessage, HttpResponseMessage>? respond;
/// <summary>Called on every wait the handler makes (the jitter, the pick-up polls).</summary>
private Action<TimeSpan>? onDelay;
public AgentUpgradeHandlerTests()
{
folders = new UpgradeFolders(Path.Combine(root, "data"), Path.Combine(root, "install"));
Directory.CreateDirectory(folders.InstallDirectory);
package = Zip(new()
{
["ExecuteR-Agent.exe"] = $"agent {Target}",
["ExecuteR-Watchdog.exe"] = "watchdog",
["Modules/Shared/Shared.psm1"] = "module",
});
// By default the watchdog picks the hand-over up at its first look.
onDelay = _ =>
{
if (File.Exists(folders.Handover))
{
File.Delete(folders.Handover);
}
};
}
public void Dispose()
{
try
{
Directory.Delete(root, recursive: true);
}
catch (IOException)
{
}
}
private AgentUpgradeHandler CreateHandler(bool canSelfUpgrade = true, string rid = "win-x64", string version = Current, UpgradeGate? gate = null)
=> new(new AgentUpgradeContext
{
Folders = folders,
ClientId = "client-1",
CurrentVersion = version,
RuntimeIdentifier = rid,
CanSelfUpgrade = canSelfUpgrade,
ServerUrl = ServerUrl.Parse("https://control.example.com/executer"),
CreateHttpHandler = () => new StubHandler(request =>
{
timeline.Add($"GET {request.RequestUri}");
return respond?.Invoke(request) ?? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(package) };
}),
Delay = (delay, cancellationToken) =>
{
cancellationToken.ThrowIfCancellationRequested();
timeline.Add($"wait {delay.TotalSeconds:0.###}");
now += delay;
onDelay?.Invoke(delay);
return Task.CompletedTask;
},
RandomMilliseconds = max => max,
UtcNow = () => now,
Gate = gate ?? new UpgradeGate(),
}, events, watchdog, NullLogger.Instance);
private AgentUpgradeCommandPayload Command(string? sha = null, long? size = null, string url = "/api/agent/packages/zip/download",
string packageType = AgentPackageTypes.WindowsZip, string version = Target, int jitter = 0)
=> new(Guid.NewGuid(), version, packageType, url,
sha ?? Convert.ToHexStringLower(SHA256.HashData(package)), size ?? package.Length, Revert: false, HealthDeadlineSeconds: 600,
StartJitterSeconds: jitter);
private static byte[] Zip(Dictionary<string, string> files)
{
using var buffer = new MemoryStream();
using (var zip = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true))
{
foreach (var (path, content) in files)
{
using var writer = new StreamWriter(zip.CreateEntry(path).Open(), new UTF8Encoding(false));
writer.Write(content);
}
}
return buffer.ToArray();
}
private AgentUpgradeEventPayload Last => events.Published[^1];
[Fact]
public async Task RefusesWhenNotRunningAsTheInstalledService()
{
await CreateHandler(canSelfUpgrade: false).RunAsync(Command(), CancellationToken.None);
Assert.Single(events.Published);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Unsupported, Last.Reason);
Assert.Empty(timeline);
}
[Theory]
[InlineData("win-x64", AgentPackageTypes.LinuxX64)]
[InlineData("win-arm64", AgentPackageTypes.WindowsZip)]
[InlineData("linux-x64", AgentPackageTypes.WindowsZip)]
public async Task RefusesAnotherPlatformsPackage(string rid, string packageType)
{
await CreateHandler(rid: rid).RunAsync(Command(packageType: packageType), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Unsupported, Last.Reason);
Assert.Empty(timeline);
}
[Fact]
public async Task AnUpgradeToTheRunningVersionSucceedsAsAlready()
{
await CreateHandler(version: Target).RunAsync(Command(), CancellationToken.None);
Assert.Single(events.Published);
Assert.Equal(AgentUpgradeStates.Succeeded, Last.State);
Assert.Equal(AgentUpgradeReasons.Already, Last.Reason);
Assert.Empty(timeline);
}
[Theory]
[InlineData(WatchdogFiles.Handover)]
[InlineData(WatchdogFiles.Journal)]
[InlineData(WatchdogFiles.Outcome)]
[InlineData(WatchdogFiles.Removal)]
public async Task RefusesWhileAnotherUpgradeIsPending(string file)
{
Directory.CreateDirectory(folders.Root);
File.WriteAllText(Path.Combine(folders.Root, file), "{}");
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Busy, Last.Reason);
Assert.Empty(timeline);
}
[Fact]
public async Task RefusesAVersionThatCouldNotRenewItsCredential()
{
await CreateHandler(version: "1.1.0").RunAsync(Command(version: "1.0.9"), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Unsupported, Last.Reason);
Assert.Empty(timeline);
}
[Fact]
public async Task RefusesWhileAnotherUpgradeRunsInThisProcess()
{
var gate = new UpgradeGate();
Assert.True(gate.TryEnter());
await CreateHandler(gate: gate).RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeReasons.Busy, Last.Reason);
}
[Fact]
public async Task DownloadsVerifiesStagesAndHandsOver()
{
WatchdogHandover? handedOver = null;
onDelay = _ =>
{
handedOver ??= WatchdogFileStore.Read(folders.Handover, WatchdogJsonContext.Default.WatchdogHandover);
File.Delete(folders.Handover);
};
var command = Command();
await CreateHandler().RunAsync(command, CancellationToken.None);
Assert.Equal("GET https://control.example.com/executer/api/agent/packages/zip/download", timeline[0]);
Assert.Equal(AgentUpgradeStates.Downloading, events.Published[0].State);
Assert.Equal(AgentUpgradeStates.Installing, Last.State);
Assert.All(events.Published, e =>
{
Assert.Equal(command.UpgradeId, e.UpgradeId);
Assert.Equal("client-1", e.ClientId);
Assert.Equal(Current, e.FromVersion);
Assert.Equal(Target, e.ToVersion);
});
Assert.NotNull(handedOver);
Assert.Equal(command.UpgradeId, handedOver.UpgradeId);
Assert.Equal(Target, handedOver.Version);
Assert.Equal(Current, handedOver.FromVersion);
Assert.Equal(600, handedOver.HealthDeadlineSeconds);
Assert.Equal(Path.Combine(folders.Staging, $"{Target}-{command.UpgradeId:N}"), handedOver.StagingPath);
Assert.Equal(["ExecuteR-Agent.exe", "ExecuteR-Watchdog.exe", "Modules/Shared/Shared.psm1"], handedOver.Files.Select(f => f.Path).Order());
var agent = handedOver.Files.Single(f => f.Path == "ExecuteR-Agent.exe");
var staged = Path.Combine(handedOver.StagingPath, "ExecuteR-Agent.exe");
Assert.Equal(new FileInfo(staged).Length, agent.Size);
Assert.Equal(Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(staged))), agent.Sha256);
Assert.Equal(handedOver.StagingPath, watchdog.EnsuredFor);
// The verified package is kept by its hash for a repeat.
Assert.True(File.Exists(Path.Combine(folders.Downloads, command.Sha256 + ".zip")));
}
[Fact]
public async Task APackageThatDoesNotMatchItsHashFailsAndHandsNothingOver()
{
await CreateHandler().RunAsync(Command(sha: new string('a', 64)), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Hash, Last.Reason);
Assert.False(File.Exists(folders.Handover));
Assert.Null(watchdog.EnsuredFor);
Assert.Empty(Directory.GetFiles(folders.Downloads));
}
[Fact]
public async Task APackageOfAnotherSizeFailsAsHash()
{
await CreateHandler().RunAsync(Command(size: package.Length + 1), CancellationToken.None);
Assert.Equal(AgentUpgradeReasons.Hash, Last.Reason);
Assert.False(File.Exists(folders.Handover));
}
/// <summary>With agent offload on, the server's route answers with a redirect to a short-lived presigned link.</summary>
[Fact]
public async Task FollowsTheServersRedirectToAPresignedLinkAndStillVerifies()
{
const string link = "http://objects.example.com/bucket/executer/agent/2026.2.1.1000/Agent.zip?X-Amz-Expires=300&X-Amz-Signature=abc";
respond = request => request.RequestUri!.Host == "control.example.com"
? new HttpResponseMessage(HttpStatusCode.Found) { Headers = { Location = new Uri(link) } }
: new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(package) };
var command = Command(url: "/api/agent-packages/" + Convert.ToHexStringLower(SHA256.HashData(package)));
await CreateHandler().RunAsync(command, CancellationToken.None);
Assert.Equal(
[$"GET https://control.example.com/executer{command.DownloadUrl}", $"GET {link}"],
timeline.Where(t => t.StartsWith("GET ", StringComparison.Ordinal)));
Assert.Equal(AgentUpgradeStates.Installing, Last.State);
Assert.True(File.Exists(Path.Combine(folders.Downloads, command.Sha256 + ".zip")));
}
[Fact]
public async Task APresignedLinkServingOtherBytesFailsAsHash()
{
// A relative Location resolves against the server's address.
respond = request => request.RequestUri!.AbsolutePath.StartsWith("/elsewhere/", StringComparison.Ordinal)
? new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent([1, 2, 3]) }
: new HttpResponseMessage(HttpStatusCode.Redirect) { Headers = { Location = new Uri("/elsewhere/Agent.zip", UriKind.Relative) } };
await CreateHandler().RunAsync(Command(size: 3), CancellationToken.None);
Assert.Contains("GET https://control.example.com/elsewhere/Agent.zip", timeline);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Hash, Last.Reason);
Assert.False(File.Exists(folders.Handover));
}
[Fact]
public async Task ARedirectLoopEndsAsADownloadFailure()
{
respond = _ => new HttpResponseMessage(HttpStatusCode.Found) { Headers = { Location = new Uri("https://control.example.com/again") } };
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(1 + Agent.Files.ServerDownloads.MaxRedirects, timeline.Count(t => t.StartsWith("GET ", StringComparison.Ordinal)));
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Download, Last.Reason);
}
[Fact]
public async Task ADownloadErrorFailsAsDownload()
{
respond = _ => new HttpResponseMessage(HttpStatusCode.NotFound);
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Download, Last.Reason);
}
[Fact]
public async Task APackageWithoutTheAgentBinaryFailsAsPackage()
{
package = Zip(new() { ["README.txt"] = "nothing here" });
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeReasons.Package, Last.Reason);
Assert.False(File.Exists(folders.Handover));
Assert.Empty(Directory.GetDirectories(folders.Staging));
}
[Fact]
public async Task AWatchdogThatCannotBeStartedFailsTheUpgrade()
{
watchdog.Problem = "The watchdog service could not be started.";
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Watchdog, Last.Reason);
Assert.False(File.Exists(folders.Handover));
}
[Fact]
public async Task AHandOverTheWatchdogNeverTakesFailsAfterThePickupDeadline()
{
onDelay = null;
var started = now;
await CreateHandler().RunAsync(Command(), CancellationToken.None);
Assert.Equal(AgentUpgradeStates.Failed, Last.State);
Assert.Equal(AgentUpgradeReasons.Watchdog, Last.Reason);
Assert.False(File.Exists(folders.Handover));
Assert.True(now - started >= WatchdogFiles.HandoverPickupDeadline);
}
[Fact]
public async Task WaitsTheStartJitterBeforeAnyOtherWork()
{
await CreateHandler().RunAsync(Command(jitter: 15), CancellationToken.None);
// RandomMilliseconds returns its maximum here: the whole 15 s, first.
Assert.Equal("wait 15", timeline[0]);
Assert.StartsWith("GET ", timeline[1], StringComparison.Ordinal);
Assert.Equal(AgentUpgradeStates.Installing, Last.State);
}
[Fact]
public async Task TheJitterIsDrawnBetweenZeroAndItsMaximum()
{
var drawn = new List<int>();
var handler = new AgentUpgradeHandler(new AgentUpgradeContext
{
Folders = folders,
ClientId = "client-1",
CurrentVersion = Current,
RuntimeIdentifier = "win-x64",
CanSelfUpgrade = true,
ServerUrl = ServerUrl.Parse("https://control.example.com"),
CreateHttpHandler = () => new StubHandler(_ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(package) }),
Delay = (delay, _) =>
{
timeline.Add($"wait {delay.TotalMilliseconds}");
onDelay?.Invoke(delay);
return Task.CompletedTask;
},
RandomMilliseconds = max =>
{
drawn.Add(max);
return 4321;
},
Gate = new UpgradeGate(),
}, events, watchdog, NullLogger.Instance);
await handler.RunAsync(Command(jitter: 15), CancellationToken.None);
Assert.Equal([15_000], drawn);
Assert.Equal("wait 4321", timeline[0]);
}
[Fact]
public async Task NoJitterMeansNoWaitBeforeTheDownload()
{
await CreateHandler().RunAsync(Command(jitter: 0), CancellationToken.None);
Assert.StartsWith("GET ", timeline[0], StringComparison.Ordinal);
}
[Fact]
public async Task CancellingDuringTheJitterDownloadsNothing()
{
using var cancel = new CancellationTokenSource();
onDelay = _ => cancel.Cancel();
var handler = new AgentUpgradeHandler(new AgentUpgradeContext
{
Folders = folders,
ClientId = "client-1",
CurrentVersion = Current,
RuntimeIdentifier = "win-x64",
CanSelfUpgrade = true,
ServerUrl = ServerUrl.Parse("https://control.example.com"),
CreateHttpHandler = () => new StubHandler(request =>
{
timeline.Add("GET");
return new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(package) };
}),
Delay = async (delay, cancellationToken) =>
{
onDelay?.Invoke(delay);
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
},
RandomMilliseconds = max => max,
Gate = new UpgradeGate(),
}, events, watchdog, NullLogger.Instance);
await handler.RunAsync(Command(jitter: 15), cancel.Token);
Assert.DoesNotContain("GET", timeline);
Assert.Empty(events.Published);
}
[Theory]
[InlineData("/api/agent/packages/zip/download", "https://control.example.com/executer/api/agent/packages/zip/download")]
[InlineData("/downloads/agent.zip?sig=abc", "https://control.example.com/executer/downloads/agent.zip?sig=abc")]
[InlineData("https://bucket.s3.example.com/agent.zip?X-Amz-Signature=1", "https://bucket.s3.example.com/agent.zip?X-Amz-Signature=1")]
[InlineData("ftp://example.com/agent.zip", null)]
[InlineData("relative/agent.zip", null)]
[InlineData("//evil.example.com/agent.zip", null)]
[InlineData("", null)]
public void ResolvesTheDownloadUrl(string downloadUrl, string? expected)
{
var resolved = AgentUpgradeHandler.ResolveDownloadUrl(downloadUrl, ServerUrl.Parse("https://control.example.com/executer"));
Assert.Equal(expected, resolved?.ToString());
}
[Fact]
public void ARelativeUrlNeedsAServerUrl()
=> Assert.Null(AgentUpgradeHandler.ResolveDownloadUrl("/api/agent/packages/zip/download", null));
private sealed class StubHandler(Func<HttpRequestMessage, HttpResponseMessage> respond) : HttpMessageHandler
{
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
=> Task.FromResult(respond(request));
}
}
internal sealed class RecordingEvents : IUpgradeEvents
{
public List<AgentUpgradeEventPayload> Published { get; } = [];
public Exception? Failure { get; set; }
public Task PublishAsync(AgentUpgradeEventPayload payload, CancellationToken cancellationToken)
{
if (Failure is not null)
{
throw Failure;
}
Published.Add(payload);
return Task.CompletedTask;
}
}
internal sealed class FakeWatchdog : IWatchdogInstaller
{
public string? Problem { get; set; }
public string? EnsuredFor { get; private set; }
public Task<string?> EnsureRunningAsync(string stagingDirectory, CancellationToken cancellationToken)
{
EnsuredFor = stagingDirectory;
return Task.FromResult(Problem);
}
}