Files
Alphaeus Mote 408a960f42 feat(sequences): server-orchestrated script sequences that continue across restarts [skip ci]
A sequence is an ordered list of steps, each an existing script with its own
arguments, extra Functions, own S3 files, condition (previous outcome,
platform, full OS / Windows PE, groups), Restart On Completion (one
RestartSettings value over the script's own), Continue On Error and
Success/Warning/Failure exit-code lists (codes, ranges, hex, *; defaults
0 / 3010, 1641 / *; validated server side, no overlaps).

A start records the sequence run, one ordinary run per step with that
step's sealed snapshot, and each device's run with its first step's job.
Every step is a normal job sent by the JobDispatcher, so history, output,
run pages, dashboards and audit are unchanged. A SequenceWorker (on the
SCHEDULE_WORKER node, every 2 s) classifies a finished step, waits while its
restart is under way (Restarted, or a newer inventory boot time, which also
covers Windows PE), checks the next step's condition and queues it, with one
update conditioned on the device run's version and the next job in the same
transaction: duplicate or concurrent advances move a device once. The
batched job-state writer is untouched.

Sequences run from Run Now (POST /api/jobs with sequenceId), schedules and
event triggers (sequenceId on /api/triggers), and On Demand with the device
CLI, which returns at once for a sequence. Runs can be cancelled
(POST /api/sequence-runs/{id}/cancel); steps not yet sent are cancelled.
Jobs, runs and summaries carry a sequence step reference. Migrations for
SQLite, PostgreSQL and SQL Server.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 09:54:26 -05:00

390 lines
17 KiB
C#

using System.IO.Pipes;
using System.Security.AccessControl;
using System.Security.Principal;
using Agent.Commands;
using Agent.LocalRun;
using Microsoft.Extensions.Logging.Abstractions;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Xunit;
namespace Agent.Tests;
/// <summary>Stands in for the backend: answers each request with what the test says.</summary>
internal sealed class FakeDeviceRunClient(Func<Guid, DeviceRunReplyPayload> answer) : IDeviceRunClient
{
public List<(Guid RequestId, string? Reference, IReadOnlyDictionary<string, string>? Parameters, string? LocalUser, bool List)> Requests { get; } = [];
public Exception? Failure { get; set; }
public Task<DeviceRunReplyPayload> RequestAsync(
Guid requestId, string? reference, IReadOnlyDictionary<string, string>? parameters, string? localUser, bool list, CancellationToken cancellationToken)
{
lock (Requests)
{
Requests.Add((requestId, reference, parameters, localUser, list));
}
return Failure is not null ? Task.FromException<DeviceRunReplyPayload>(Failure) : Task.FromResult(answer(requestId));
}
}
public sealed class RunCommandTests
{
private static readonly Guid JobId = Guid.Parse("11111111-2222-3333-4444-555555555555");
// ── Command line ────────────────────────────────────────────────────
[Fact]
public void ABareRunIsTheServiceAndAnythingAfterItIsTheCli()
{
Assert.False(AgentCli.IsDeviceRun(["run"]));
Assert.True(AgentCli.IsDeviceRun(["run", "Fix Printers"]));
Assert.True(AgentCli.IsDeviceRun(["RUN", "--list"]));
Assert.False(AgentCli.IsDeviceRun(["foreground", "x"]));
}
[Fact]
public void ParsesAReferenceParametersWaitAndTimeout()
{
Assert.True(RunCommand.TryParse(
["Fix Printers", "--param", "Name=a=b", "-p", "Force=", "-p", "name=c", "--wait", "--timeout", "90"], out var options, out var error), error);
Assert.Equal("Fix Printers", options.Reference);
Assert.Equal(2, options.Parameters.Count);
Assert.Equal("c", options.Parameters["Name"]); // the last value for a name wins
Assert.Equal(string.Empty, options.Parameters["Force"]);
Assert.True(options.Wait);
Assert.Equal(90, options.TimeoutSeconds);
Assert.False(options.List);
}
[Fact]
public void SplitsOnTheFirstEquals()
{
Assert.True(RunCommand.TryParse(["x", "-p", "Filter=a=b"], out var options, out _));
Assert.Equal("a=b", options.Parameters["Filter"]);
}
[Fact]
public void ParsesListAndHelp()
{
Assert.True(RunCommand.TryParse(["--list"], out var list, out _));
Assert.True(list.List);
Assert.True(RunCommand.TryParse(["--help"], out var help, out _));
Assert.True(help.Help);
}
[Theory]
[InlineData(new string[0], "Name or id required.")]
[InlineData(new[] { "x", "-p", "NoEquals" }, "Expected Name=Value")]
[InlineData(new[] { "x", "-p", "=v" }, "Expected Name=Value")]
[InlineData(new[] { "x", "-p" }, "needs Name=Value")]
[InlineData(new[] { "x", "--timeout", "0", "--wait" }, "--timeout")]
[InlineData(new[] { "x", "--timeout", "abc", "--wait" }, "--timeout")]
[InlineData(new[] { "x", "--timeout", "5" }, "--timeout needs --wait")]
[InlineData(new[] { "x", "y" }, "Only one script")]
[InlineData(new[] { "x", "--bogus" }, "Unknown option")]
[InlineData(new[] { "--list", "x" }, "--list takes no other arguments")]
[InlineData(new[] { "--list", "--wait" }, "--list takes no other arguments")]
public void RejectsBadInput(string[] args, string message)
{
Assert.False(RunCommand.TryParse(args, out _, out var error));
Assert.Contains(message, error);
}
[Theory]
[InlineData(JobState.Succeeded, null, 0)]
[InlineData(JobState.Succeeded, 0, 0)]
[InlineData(JobState.Failed, 7, 7)]
[InlineData(JobState.Failed, null, 1)]
[InlineData(JobState.Failed, 0, 1)]
[InlineData(JobState.TimedOut, -1, -1)]
public void TheExitCodeIsTheScripts(JobState state, int? exitCode, int expected)
=> Assert.Equal(expected, RunCommand.ExitCodeOf(state, exitCode));
// ── Reply and pipe security ─────────────────────────────────────────
[Fact]
public void TheReplyIsReadFromItsEnvelope()
{
var reply = new DeviceRunReplyPayload(DeviceRunStatuses.Started, null, JobId, Guid.NewGuid(), "Reset-Spooler");
var envelope = MessageEnvelope.Create(MessageTypes.AgentRunRequest, new EndpointRef(EndpointKinds.Backend, "backend"), reply);
Assert.Equal(reply with { }, NatsDeviceRunClient.ParseReply(envelope.ToUtf8Bytes()));
Assert.Equal(DeviceRunStatuses.Started, NatsDeviceRunClient.ParseReply(JsonDefaults.SerializeToUtf8Bytes(reply))?.Status);
Assert.Null(NatsDeviceRunClient.ParseReply([]));
Assert.Null(NatsDeviceRunClient.ParseReply("nope"u8.ToArray()));
}
[Fact]
public void TheProductionPipeAdmitsOnlyAdministratorsAndSystem()
{
var security = WindowsPipeListener.ProductionSecurity();
var rules = security.GetAccessRules(true, true, typeof(SecurityIdentifier)).Cast<PipeAccessRule>().ToList();
Assert.True(security.AreAccessRulesProtected);
Assert.Equal(2, rules.Count);
Assert.All(rules, r => Assert.Equal(AccessControlType.Allow, r.AccessControlType));
Assert.All(rules, r => Assert.False(r.IsInherited));
Assert.Equal(
new[] { WellKnownSidType.BuiltinAdministratorsSid, WellKnownSidType.LocalSystemSid }.Select(t => new SecurityIdentifier(t, null)).OrderBy(s => s.Value),
rules.Select(r => (SecurityIdentifier)r.IdentityReference).OrderBy(s => s.Value));
}
[Fact]
public void ThePipeMustBeOwnedBySystemOrAdministrators()
{
Assert.True(WindowsPipeClient.IsTrustedOwner(new SecurityIdentifier(WellKnownSidType.LocalSystemSid, null)));
Assert.True(WindowsPipeClient.IsTrustedOwner(new SecurityIdentifier(WellKnownSidType.BuiltinAdministratorsSid, null)));
Assert.False(WindowsPipeClient.IsTrustedOwner(new SecurityIdentifier(WellKnownSidType.BuiltinUsersSid, null)));
Assert.False(WindowsPipeClient.IsTrustedOwner(null));
// An ordinary account is not trusted; the CI runner runs these tests as SYSTEM, which is.
var me = WindowsIdentity.GetCurrent().User;
if (me is not null && !me.IsWellKnown(WellKnownSidType.LocalSystemSid))
{
Assert.False(WindowsPipeClient.IsTrustedOwner(me));
}
}
// ── Over a real named pipe ──────────────────────────────────────────
private sealed class PipeRig : IAsyncDisposable
{
private readonly CancellationTokenSource stop = new();
private readonly Task listening;
public PipeRig(FakeDeviceRunClient client, bool allowed = true)
{
Observers = new JobObservers();
Name = $"executer-test-{Guid.NewGuid():N}";
var server = new LocalRunServer(client, Observers, NullLogger<LocalRunServer>.Instance);
var me = WindowsIdentity.GetCurrent().User!;
listening = new WindowsPipeListener(
Name, server, NullLogger.Instance,
security: () =>
{
// The test runs unelevated: admit this user instead of Administrators/SYSTEM.
var security = new PipeSecurity();
security.SetAccessRuleProtection(true, false);
security.AddAccessRule(new PipeAccessRule(me, PipeAccessRights.FullControl, AccessControlType.Allow));
return security;
},
authorize: _ => allowed).RunAsync(stop.Token);
}
public string Name { get; }
public JobObservers Observers { get; }
public StringWriter Out { get; } = new();
public StringWriter Error { get; } = new();
public Task<int> RunAsync(params string[] args)
{
Assert.True(RunCommand.TryParse(args, out var options, out var error), error);
var me = WindowsIdentity.GetCurrent().User;
return RunCommand.ExecuteAsync(
options,
ct => WindowsPipeClient.ConnectAsync(Name, ct, owner => owner == me || WindowsPipeClient.IsTrustedOwner(owner)),
Out, Error, "claimed-user", CancellationToken.None)
.WaitAsync(TimeSpan.FromSeconds(60));
}
public async ValueTask DisposeAsync()
{
await stop.CancelAsync();
await listening;
stop.Dispose();
}
}
private static DeviceRunReplyPayload Started() => new(DeviceRunStatuses.Started, null, JobId, Guid.NewGuid(), "Reset-Spooler");
[Fact]
public async Task WaitStreamsTheOutputAndExitsWithTheScriptsCode()
{
PipeRig? rig = null;
var client = new FakeDeviceRunClient(requestId =>
{
// The job's output and end arrive before the server's answer does.
rig!.Observers.Output(requestId, OutputStreamKind.Output, "line one");
rig.Observers.Output(requestId, OutputStreamKind.Error, "bad thing");
rig.Observers.Finished(requestId, JobState.Failed, 7, "Process exited with 7.");
return Started();
});
await using (rig = new PipeRig(client))
{
Assert.Equal(7, await rig.RunAsync("Reset-Spooler", "-p", "Force=true", "--wait"));
}
Assert.Contains("line one", rig.Out.ToString());
Assert.Contains("bad thing", rig.Error.ToString());
Assert.DoesNotContain("bad thing", rig.Out.ToString());
Assert.Contains($"job {JobId:D}", rig.Error.ToString());
var request = Assert.Single(client.Requests);
Assert.Equal("Reset-Spooler", request.Reference);
Assert.Equal("true", request.Parameters!["Force"]);
Assert.False(request.List);
// The agent names the caller from the pipe, not from the client's claim.
Assert.Equal(WindowsIdentity.GetCurrent().Name, request.LocalUser);
}
[Fact]
public async Task WaitFollowsAJobThatEndsLater()
{
PipeRig? rig = null;
var client = new FakeDeviceRunClient(requestId =>
{
_ = Task.Run(async () =>
{
await Task.Delay(300);
rig!.Observers.Output(requestId, OutputStreamKind.Output, "done");
rig.Observers.Finished(requestId, JobState.Succeeded, 0, null);
});
return Started();
});
await using (rig = new PipeRig(client))
{
Assert.Equal(0, await rig.RunAsync("Reset-Spooler", "--wait"));
}
Assert.Contains("done", rig.Out.ToString());
}
[Fact]
public async Task WithoutWaitItPrintsTheJobAndExitsZero()
{
await using var rig = new PipeRig(new FakeDeviceRunClient(_ => Started()));
Assert.Equal(0, await rig.RunAsync("Reset-Spooler"));
Assert.Contains(JobId.ToString("D"), rig.Out.ToString());
}
[Fact]
public async Task ASequenceIsNotWaitedOnEvenWithWait()
{
var sequenceRun = Guid.NewGuid();
await using var rig = new PipeRig(new FakeDeviceRunClient(_ =>
new DeviceRunReplyPayload(DeviceRunStatuses.Started, null, JobId, Guid.NewGuid(), "Image-Device", SequenceRunId: sequenceRun)));
// It carries on across restarts: --wait returns at once, without a timeout.
Assert.Equal(RunCommand.Ok, await rig.RunAsync("Image-Device", "--wait"));
Assert.Contains($"Started sequence Image-Device: run {sequenceRun:D}", rig.Out.ToString());
Assert.Contains("does not follow a sequence", rig.Error.ToString());
}
[Fact]
public async Task ATimeoutExits124AndTheJobCarriesOn()
{
await using var rig = new PipeRig(new FakeDeviceRunClient(_ => Started()));
Assert.Equal(RunCommand.TimedOut, await rig.RunAsync("Reset-Spooler", "--wait", "--timeout", "1"));
Assert.Contains("Timed out", rig.Error.ToString());
}
[Theory]
[InlineData(DeviceRunStatuses.NotFound, "No such script.")]
[InlineData(DeviceRunStatuses.Invalid, "Count must be a number.")]
[InlineData(DeviceRunStatuses.Failed, "Refused.")]
public async Task ARefusalExits1WithTheServersMessage(string status, string message)
{
await using var rig = new PipeRig(new FakeDeviceRunClient(_ => new DeviceRunReplyPayload(status, message)));
Assert.Equal(RunCommand.Refused, await rig.RunAsync("Nope", "--wait"));
Assert.Contains(message, rig.Error.ToString());
}
[Fact]
public async Task AnAmbiguousNameListsTheCandidates()
{
var first = new DeviceRunCandidate(Guid.NewGuid(), "Printers East", Guid.NewGuid(), "Reset-Spooler");
var second = new DeviceRunCandidate(Guid.NewGuid(), "Printers West", Guid.NewGuid(), "Reset-Spooler");
await using var rig = new PipeRig(new FakeDeviceRunClient(_ =>
new DeviceRunReplyPayload(DeviceRunStatuses.Ambiguous, "More than one matches.", Candidates: [first, second])));
Assert.Equal(RunCommand.Refused, await rig.RunAsync("Reset-Spooler"));
var error = rig.Error.ToString();
Assert.Contains(first.ScheduleId.ToString("D"), error);
Assert.Contains(second.ScheduleId.ToString("D"), error);
}
[Fact]
public async Task ListShowsSchedulesScriptsIdsAndParameters()
{
var candidate = new DeviceRunCandidate(
Guid.NewGuid(), "Printers", Guid.NewGuid(), "Reset-Spooler",
[new ScriptParameter("Force", ScriptParameterType.Boolean), new ScriptParameter("Queue", ScriptParameterType.String, Required: true)]);
var client = new FakeDeviceRunClient(_ => new DeviceRunReplyPayload(DeviceRunStatuses.Listed, null, Candidates: [candidate]));
await using var rig = new PipeRig(client);
Assert.Equal(0, await rig.RunAsync("--list"));
var output = rig.Out.ToString();
Assert.Contains("Printers", output);
Assert.Contains("Reset-Spooler", output);
Assert.Contains(candidate.ScheduleId.ToString("D"), output);
Assert.Contains("Queue", output);
Assert.Contains("Required", output);
Assert.True(Assert.Single(client.Requests).List);
}
[Fact]
public async Task NotConnectedToTheServerExits2()
{
var client = new FakeDeviceRunClient(_ => Started()) { Failure = new DeviceRunUnavailableException("Not connected to the server.") };
await using var rig = new PipeRig(client);
Assert.Equal(RunCommand.Unreachable, await rig.RunAsync("Reset-Spooler"));
Assert.Contains("Not connected to the server.", rig.Error.ToString());
}
[Fact]
public async Task ACallerWhoIsNotAnElevatedAdministratorIsRefused()
{
var client = new FakeDeviceRunClient(_ => Started());
await using var rig = new PipeRig(client, allowed: false);
Assert.Equal(RunCommand.Refused, await rig.RunAsync("Reset-Spooler"));
Assert.Contains("elevated administrator", rig.Error.ToString());
Assert.Empty(client.Requests);
}
[Fact]
public async Task NoAgentExits2()
{
Assert.True(RunCommand.TryParse(["Reset-Spooler"], out var options, out _));
var error = new StringWriter();
var code = await RunCommand.ExecuteAsync(
options, ct => WindowsPipeClient.ConnectAsync($"executer-absent-{Guid.NewGuid():N}", ct), new StringWriter(), error, null, CancellationToken.None);
Assert.Equal(RunCommand.Unreachable, code);
Assert.Contains("not running", error.ToString());
}
[Fact]
public async Task AnUntrustedPipeOwnerIsNotTalkedTo()
{
// Elevated, the test's own pipe would be owned by Administrators.
Assert.SkipWhen(new WindowsPrincipal(WindowsIdentity.GetCurrent()).IsInRole(WindowsBuiltInRole.Administrator), "Runs unelevated only");
var client = new FakeDeviceRunClient(_ => Started());
await using var rig = new PipeRig(client);
Assert.True(RunCommand.TryParse(["Reset-Spooler"], out var options, out _));
var error = new StringWriter();
// The production check: this unelevated user's pipe is neither SYSTEM's nor Administrators'.
var code = await RunCommand.ExecuteAsync(
options, ct => WindowsPipeClient.ConnectAsync(rig.Name, ct), new StringWriter(), error, null, CancellationToken.None);
Assert.Equal(RunCommand.Unreachable, code);
Assert.Contains("owned by", error.ToString());
Assert.Empty(client.Requests);
}
}