ea60dc7848
- Script editor: Add From S3 opens the S3 browser in multi-select mode
(files and folders, search, breadcrumb); each pick gets a unique
identifier name. Download To (default $ExecuteRJobDir\Files) with the
variables on hover, per-binding Path inside it, Clean Up Downloads (on).
- Run Script: Review > Files > Change for This Run overrides the selection
for one run (needs scripts.manage or files.send), sealed into the
run's snapshot.
- Storage: FileBindingsJson becomes {root, cleanUp, bindings}; the older
array form still reads (absolute destinations, nothing removed). No
schema change.
- Server: only the selected keys are listed and presigned; single files
carry their listed size; a folder over 2,000 objects is refused.
- Agent: paths expand on the device ($ExecuteRJobDir, $ExecuteRAgentDir,
$ExecuteRAgentPath, %VAR%, $env:VAR, $VAR/${VAR} on Unix, $(...));
undefined variables and ".." fail before the script runs. Missing
folders are created (inheriting the parent's ACL). Four downloads at
once, streamed, retried, checked against size and plain-MD5 ETags,
timeout/cancel aware. Information lines and progress records while it
runs, and a summary. $Files (FileInfo/DirectoryInfo, ordered,
case-insensitive) and $FilesRoot in PowerShell 7 and 5.1;
EXECUTER_FILE_<NAME> and EXECUTER_FILES_ROOT in Bash. Clean Up
Downloads removes only what the job wrote, on success, failure,
timeout or cancel.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
165 lines
4.9 KiB
C#
165 lines
4.9 KiB
C#
using System.Collections.Concurrent;
|
|
using System.Net;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
|
|
namespace Agent.Tests;
|
|
|
|
/// <summary>
|
|
/// A loopback HTTP server for real download round trips: objects with an MD5 ETag
|
|
/// (or a wrong one), requests that fail before they succeed, slow responses, and
|
|
/// a count of how many are served at once.
|
|
/// </summary>
|
|
internal sealed class TestFileServer : IDisposable
|
|
{
|
|
private readonly HttpListener listener = new();
|
|
private readonly string baseUrl;
|
|
private readonly ConcurrentDictionary<string, Entry> entries = new();
|
|
private int inFlight;
|
|
private int maxInFlight;
|
|
private int requests;
|
|
|
|
public sealed class Entry
|
|
{
|
|
public required byte[] Body { get; init; }
|
|
|
|
public string? ETag { get; set; }
|
|
|
|
/// <summary>Requests answered 500 before the object is served.</summary>
|
|
public int FailFirst { get; set; }
|
|
|
|
/// <summary>A wait before the headers.</summary>
|
|
public TimeSpan Delay { get; set; }
|
|
|
|
/// <summary>The body in this many pieces, with this wait between them.</summary>
|
|
public int Chunks { get; set; } = 1;
|
|
|
|
public TimeSpan ChunkDelay { get; set; }
|
|
}
|
|
|
|
public TestFileServer()
|
|
{
|
|
for (var attempt = 0; ; attempt++)
|
|
{
|
|
var probe = new System.Net.Sockets.TcpListener(IPAddress.Loopback, 0);
|
|
probe.Start();
|
|
var port = ((IPEndPoint)probe.LocalEndpoint).Port;
|
|
probe.Stop();
|
|
|
|
baseUrl = $"http://127.0.0.1:{port}";
|
|
listener.Prefixes.Clear();
|
|
listener.Prefixes.Add(baseUrl + "/");
|
|
try
|
|
{
|
|
listener.Start();
|
|
break;
|
|
}
|
|
catch (HttpListenerException) when (attempt < 5)
|
|
{
|
|
}
|
|
}
|
|
|
|
_ = Task.Run(async () =>
|
|
{
|
|
while (listener.IsListening)
|
|
{
|
|
HttpListenerContext context;
|
|
try
|
|
{
|
|
context = await listener.GetContextAsync();
|
|
}
|
|
catch (Exception)
|
|
{
|
|
return;
|
|
}
|
|
|
|
_ = Task.Run(() => ServeAsync(context));
|
|
}
|
|
});
|
|
}
|
|
|
|
public int MaxConcurrent => Volatile.Read(ref maxInFlight);
|
|
|
|
public int Requests => Volatile.Read(ref requests);
|
|
|
|
public Entry Add(string path, string content, bool etag = true) => Add(path, Encoding.UTF8.GetBytes(content), etag);
|
|
|
|
public Entry Add(string path, byte[] body, bool etag = true)
|
|
{
|
|
var entry = new Entry { Body = body, ETag = etag ? Convert.ToHexStringLower(MD5.HashData(body)) : null };
|
|
entries[path] = entry;
|
|
return entry;
|
|
}
|
|
|
|
public string Url(string path) => baseUrl + path;
|
|
|
|
private async Task ServeAsync(HttpListenerContext context)
|
|
{
|
|
Interlocked.Increment(ref requests);
|
|
var now = Interlocked.Increment(ref inFlight);
|
|
for (var seen = Volatile.Read(ref maxInFlight); now > seen; seen = Volatile.Read(ref maxInFlight))
|
|
{
|
|
Interlocked.CompareExchange(ref maxInFlight, now, seen);
|
|
}
|
|
|
|
try
|
|
{
|
|
if (!entries.TryGetValue(context.Request.Url!.AbsolutePath, out var entry))
|
|
{
|
|
context.Response.StatusCode = 404;
|
|
return;
|
|
}
|
|
|
|
if (entry.Delay > TimeSpan.Zero)
|
|
{
|
|
await Task.Delay(entry.Delay);
|
|
}
|
|
|
|
lock (entry)
|
|
{
|
|
if (entry.FailFirst > 0)
|
|
{
|
|
entry.FailFirst--;
|
|
context.Response.StatusCode = 500;
|
|
return;
|
|
}
|
|
}
|
|
|
|
context.Response.StatusCode = 200;
|
|
context.Response.ContentLength64 = entry.Body.Length;
|
|
if (entry.ETag is not null)
|
|
{
|
|
context.Response.AddHeader("ETag", $"\"{entry.ETag}\"");
|
|
}
|
|
|
|
var size = Math.Max(1, (entry.Body.Length + entry.Chunks - 1) / entry.Chunks);
|
|
for (var offset = 0; offset < entry.Body.Length; offset += size)
|
|
{
|
|
await context.Response.OutputStream.WriteAsync(entry.Body.AsMemory(offset, Math.Min(size, entry.Body.Length - offset)));
|
|
await context.Response.OutputStream.FlushAsync();
|
|
if (entry.ChunkDelay > TimeSpan.Zero)
|
|
{
|
|
await Task.Delay(entry.ChunkDelay);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception)
|
|
{
|
|
// The client went away (a cancelled download).
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref inFlight);
|
|
try
|
|
{
|
|
context.Response.Close();
|
|
}
|
|
catch (Exception)
|
|
{
|
|
}
|
|
}
|
|
}
|
|
|
|
public void Dispose() => listener.Stop();
|
|
}
|