ea60dc7848
- Script editor: Add From S3 opens the S3 browser in multi-select mode
(files and folders, search, breadcrumb); each pick gets a unique
identifier name. Download To (default $ExecuteRJobDir\Files) with the
variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
for one run (needs scripts.manage or files.send), sealed into the
run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
array form still reads (absolute destinations, nothing removed). No
schema change.
- Server: only the selected keys are listed and presigned; single files
carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
$ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
undefined variables and ".." fail before the script runs. Missing
folders are created (inheriting the parent's ACL). Four downloads at
once, streamed, retried, checked against size and plain-MD5 ETags,
timeout/cancel aware. Information lines and progress records while it
runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
Downloads removes only what the job wrote, on success, failure,
timeout or cancel.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
243 lines
9.7 KiB
C#
243 lines
9.7 KiB
C#
using Agent.PowerShell;
|
|
using Agent.Storage;
|
|
using Microsoft.Extensions.Logging.Abstractions;
|
|
using Shared.Contracts.Messages;
|
|
using Shared.Contracts.Models;
|
|
using Shared.Infrastructure.Configuration;
|
|
using Xunit;
|
|
|
|
namespace Agent.Tests;
|
|
|
|
/// <summary>
|
|
/// File deliveries through a whole job: $Files as FileInfo / DirectoryInfo and
|
|
/// $FilesRoot in every engine, Bash's variables, and Clean Up Downloads after a
|
|
/// success, a failure, a timeout and a cancellation, never touching what was
|
|
/// already in a chosen root.
|
|
/// </summary>
|
|
public sealed class ScriptFileJobTests : IDisposable
|
|
{
|
|
private readonly string work = Path.Combine(Path.GetTempPath(), "executer-file-jobs-" + Guid.NewGuid().ToString("N"));
|
|
private readonly TestFileServer server = new();
|
|
private readonly RecordingPublisher publisher = new(() => DateTimeOffset.UtcNow);
|
|
private readonly JobHistoryStore history;
|
|
|
|
public ScriptFileJobTests()
|
|
{
|
|
Directory.CreateDirectory(work);
|
|
history = new JobHistoryStore(Path.Combine(work, "jobs.db"));
|
|
server.Add("/config.json", "{\"a\":1}");
|
|
server.Add("/tools/run.cmd", "echo hi");
|
|
server.Add("/tools/lib/helper.dll", "binary");
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
history.Dispose();
|
|
server.Dispose();
|
|
try
|
|
{
|
|
Directory.Delete(work, recursive: true);
|
|
}
|
|
catch (IOException)
|
|
{
|
|
}
|
|
}
|
|
|
|
private string JobRootPath => Path.Combine(work, "jobs");
|
|
|
|
private PowerShellJobHandler Handler() => new(
|
|
new PowerShellExecutionService(), publisher, history, new Agent.Files.ServerDownloads(null, false), NullLogger.Instance,
|
|
jobRoot: new JobRootInfo(JobRootPath, ConfigurationSource.Default));
|
|
|
|
private IReadOnlyList<ScriptFileDelivery> Deliveries() =>
|
|
[
|
|
new("Config", "config.json", false, false, [new ScriptFileItem("", server.Url("/config.json"), 7)]),
|
|
new("Tools", "tools", true, false,
|
|
[
|
|
new ScriptFileItem("run.cmd", server.Url("/tools/run.cmd"), 7),
|
|
new ScriptFileItem("lib/helper.dll", server.Url("/tools/lib/helper.dll"), 6),
|
|
]),
|
|
];
|
|
|
|
private async Task<(ScriptCommandPayload Command, JobStatePayload State, List<string> Output)> RunAsync(
|
|
string content,
|
|
ScriptEngine engine = ScriptEngine.PowerShell,
|
|
ScriptFileOptions? options = null,
|
|
bool cleanUpAfterRun = false,
|
|
int timeoutSeconds = 120,
|
|
CancellationToken cancellationToken = default,
|
|
IReadOnlyList<ScriptFileDelivery>? deliveries = null)
|
|
{
|
|
var command = new ScriptCommandPayload(
|
|
Guid.NewGuid(), Guid.NewGuid(), "Files Test", content, timeoutSeconds, Engine: engine,
|
|
Files: deliveries ?? Deliveries(), FileOptions: options ?? new ScriptFileOptions(), CleanUpAfterRun: cleanUpAfterRun);
|
|
await Handler().RunJobAsync(command, Guid.NewGuid(), null, cancellationToken);
|
|
var state = publisher.States.Last();
|
|
var output = publisher.Output.Where(o => o.JobId == command.JobId).OrderBy(o => o.Sequence).Select(o => o.Text).ToList();
|
|
return (command, state, output);
|
|
}
|
|
|
|
private string JobDir(ScriptCommandPayload command) => Path.Combine(JobRootPath, command.JobId.ToString("D"));
|
|
|
|
private const string TypesScript = """
|
|
"config=$($Files.Config.GetType().FullName)"
|
|
"tools=$($Files['tools'].GetType().FullName)"
|
|
"root=$($FilesRoot.GetType().FullName)"
|
|
"rootpath=$($FilesRoot.FullName)"
|
|
"configpath=$($Files.Config.FullName)"
|
|
"content=$(Get-Content -Raw -LiteralPath $Files.Config.FullName)"
|
|
"helper=$(Test-Path -LiteralPath (Join-Path $Files.Tools.FullName 'lib\helper.dll'))"
|
|
"order=$($Files.Keys -join ',')"
|
|
""";
|
|
|
|
private void AssertTypes(ScriptCommandPayload command, List<string> output)
|
|
{
|
|
var root = Path.Combine(JobDir(command), "Files");
|
|
Assert.Contains("config=System.IO.FileInfo", output);
|
|
Assert.Contains("tools=System.IO.DirectoryInfo", output);
|
|
Assert.Contains("root=System.IO.DirectoryInfo", output);
|
|
Assert.Contains($"rootpath={root}", output);
|
|
Assert.Contains($"configpath={Path.Combine(root, "config.json")}", output);
|
|
Assert.Contains("content={\"a\":1}", output);
|
|
Assert.Contains("helper=True", output);
|
|
Assert.Contains("order=Config,Tools", output);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PowerShell7SeesFileInfoDirectoryInfoAndTheRoot()
|
|
{
|
|
var (command, state, output) = await RunAsync(TypesScript);
|
|
|
|
Assert.Equal(JobState.Succeeded, state.State);
|
|
AssertTypes(command, output);
|
|
Assert.Contains(output, l => l.StartsWith("Downloading 1 file, 1 folder (3 objects, 20 bytes) to "));
|
|
Assert.Contains(output, l => l.StartsWith("Downloaded 1 file, 1 folder, 20 bytes in "));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WindowsPowerShellSeesTheSame()
|
|
{
|
|
Assert.SkipUnless(OperatingSystem.IsWindows(), "powershell.exe is Windows-only");
|
|
|
|
var (command, state, output) = await RunAsync(TypesScript, ScriptEngine.WindowsPowerShell);
|
|
|
|
Assert.Equal(JobState.Succeeded, state.State);
|
|
AssertTypes(command, output);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task BashGetsThePathsAsVariables()
|
|
{
|
|
Assert.SkipUnless(ExternalScriptRunner.IsAvailable(ScriptEngine.Bash), "bash is not available on this machine");
|
|
|
|
var (command, state, output) = await RunAsync(
|
|
"""
|
|
echo "named=$Config"
|
|
echo "prefixed=$EXECUTER_FILE_CONFIG"
|
|
echo "tools=$EXECUTER_FILE_TOOLS"
|
|
echo "root=$EXECUTER_FILES_ROOT"
|
|
""",
|
|
ScriptEngine.Bash);
|
|
|
|
Assert.Equal(JobState.Succeeded, state.State);
|
|
var root = Path.Combine(JobDir(command), "Files");
|
|
Assert.Contains($"named={Path.Combine(root, "config.json")}", output);
|
|
Assert.Contains($"prefixed={Path.Combine(root, "config.json")}", output);
|
|
Assert.Contains($"tools={Path.Combine(root, "tools")}", output);
|
|
Assert.Contains($"root={root}", output);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AFailedDownloadFailsTheJobBeforeTheScriptRuns()
|
|
{
|
|
var deliveries = new List<ScriptFileDelivery>
|
|
{
|
|
new("Missing", "missing.bin", false, false, [new ScriptFileItem("", server.Url("/missing.bin"))]),
|
|
};
|
|
|
|
var (_, state, output) = await RunAsync("'the script ran'", deliveries: deliveries);
|
|
|
|
Assert.Equal(JobState.Failed, state.State);
|
|
Assert.Contains("“Missing” failed to download", state.Message);
|
|
Assert.DoesNotContain("the script ran", output);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DownloadsInsideTheJobFolderGoAfterASuccess()
|
|
{
|
|
var (command, state, _) = await RunAsync("'ok'");
|
|
|
|
Assert.Equal(JobState.Succeeded, state.State);
|
|
Assert.True(Directory.Exists(JobDir(command)));
|
|
Assert.False(Directory.Exists(Path.Combine(JobDir(command), "Files")));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CleanUpOffKeepsThem()
|
|
{
|
|
var (command, _, _) = await RunAsync("'ok'", options: new ScriptFileOptions(CleanUp: false));
|
|
|
|
Assert.True(File.Exists(Path.Combine(JobDir(command), "Files", "tools", "lib", "helper.dll")));
|
|
}
|
|
|
|
[Theory]
|
|
[InlineData("failure")]
|
|
[InlineData("timeout")]
|
|
[InlineData("cancel")]
|
|
[InlineData("success")]
|
|
public async Task AChosenRootLosesOnlyWhatTheJobDownloaded(string outcome)
|
|
{
|
|
var root = Path.Combine(work, "drop");
|
|
Directory.CreateDirectory(root);
|
|
File.WriteAllText(Path.Combine(root, "before.txt"), "was here");
|
|
Directory.CreateDirectory(Path.Combine(root, "tools"));
|
|
File.WriteAllText(Path.Combine(root, "tools", "mine.txt"), "was here too");
|
|
|
|
using var cancel = new CancellationTokenSource();
|
|
var script = outcome switch
|
|
{
|
|
"failure" => "throw 'broken'",
|
|
"timeout" or "cancel" => "Start-Sleep -Seconds 30",
|
|
_ => "'fine'",
|
|
};
|
|
if (outcome == "cancel")
|
|
{
|
|
cancel.CancelAfter(TimeSpan.FromSeconds(3));
|
|
}
|
|
|
|
var (command, state, _) = await RunAsync(
|
|
script, options: new ScriptFileOptions(root), timeoutSeconds: outcome == "timeout" ? 3 : 120, cancellationToken: cancel.Token);
|
|
|
|
Assert.Equal(
|
|
outcome switch { "failure" => JobState.Failed, "timeout" => JobState.TimedOut, "cancel" => JobState.Cancelled, _ => JobState.Succeeded },
|
|
state.State);
|
|
Assert.False(File.Exists(Path.Combine(root, "config.json")));
|
|
Assert.False(File.Exists(Path.Combine(root, "tools", "run.cmd")));
|
|
Assert.False(Directory.Exists(Path.Combine(root, "tools", "lib")));
|
|
Assert.True(File.Exists(Path.Combine(root, "before.txt")));
|
|
Assert.True(File.Exists(Path.Combine(root, "tools", "mine.txt")));
|
|
_ = command;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CancellingWhileDownloadingCancelsTheJobAndCleansUp()
|
|
{
|
|
var slow = server.Add("/slow.bin", new byte[200_000]);
|
|
slow.Chunks = 100;
|
|
slow.ChunkDelay = TimeSpan.FromMilliseconds(100);
|
|
var root = Path.Combine(work, "slow-root");
|
|
using var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(1));
|
|
|
|
var (_, state, output) = await RunAsync(
|
|
"'the script ran'",
|
|
options: new ScriptFileOptions(root),
|
|
cancellationToken: cancel.Token,
|
|
deliveries: [new("Slow", "slow.bin", false, false, [new ScriptFileItem("", server.Url("/slow.bin"), 200_000)])]);
|
|
|
|
Assert.Equal(JobState.Cancelled, state.State);
|
|
Assert.Contains("while downloading", state.Message);
|
|
Assert.DoesNotContain("the script ran", output);
|
|
Assert.False(Directory.Exists(root));
|
|
}
|
|
}
|