diff --git a/README.md b/README.md index 22416dc..f8fa8c8 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,14 @@ updates with **no SignalR, SSE, or polling**. 25. [Database providers](#database-providers) 26. [Logging](#logging) 27. [Device lifecycle: reconnect or orphan](#device-lifecycle-reconnect-or-orphan) -28. [Agent upgrades from the server](#agent-upgrades-from-the-server) -29. [S3 file delivery](#s3-file-delivery) -30. [CI/CD and GITEA_* variables](#cicd-and-gitea_-variables) -31. [Versioning](#versioning) -32. [Troubleshooting](#troubleshooting) -33. [Licence](#licence) +28. [Removing devices](#removing-devices) +29. [Removing devices](#removing-devices) +30. [Agent upgrades from the server](#agent-upgrades-from-the-server) +31. [S3 file delivery](#s3-file-delivery) +32. [CI/CD and GITEA_* variables](#cicd-and-gitea_-variables) +33. [Versioning](#versioning) +34. [Troubleshooting](#troubleshooting) +35. [Licence](#licence) --- @@ -721,6 +723,7 @@ still holds it. | `upgrade/msi/` | the MSI being installed, the one to go back to (`rollback/`) and the runner's copy of the watchdog (`runner/`) | | `\Logs\agent-msi--.log`, `ExecuteR-Watchdog-msi.log` | Windows Installer's verbose log of each MSI run, and the runner's own log | | `%ProgramData%\ExecuteR\packages\\` | agent MSIs kept for rollback (the newest three) | +| `upgrade/removal.json` | a removal under way, or one that failed until the agent has reported it ([On the device](#on-the-device)) | If the watchdog itself stops part way (power loss, a kill), its next start finishes or undoes the run from the journal: `stopping` starts over (the agent @@ -1101,6 +1104,24 @@ an object's properties as columns) rather than a type name such as `System.Collections.Hashtable`, still streamed line by line and without `…` truncation. Strings pass through unchanged. +**Exit codes.** A PowerShell 7 script reports what `pwsh -File` would: `exit N` +is exit code N and Failed when N is not 0, `exit 0` succeeds even after errors, +and a terminating error is exit 1. A native command's non-zero `$LASTEXITCODE` +only counts if the script exits with it (`exit $LASTEXITCODE`), and errors the +script silenced do not fail it. One difference, kept on purpose: anything on the +error stream (`Write-Error`, a failing cmdlet) fails a script that never calls +`exit`, with exit code 1. + +> **Changed behaviour:** agents before this version reported `exit 5` as Succeeded +> with exit code 0, and failed any script whose native command ended non-zero or +> that silenced an error with `-ErrorAction SilentlyContinue`. A script relying on +> either now reports differently. + +A failed job also records its first error's exception type, error id and message +(for Windows PowerShell and Bash, the first stderr line). For jobs from older +agents the run's error groups use the summary or the first error line of the +output. + **Downloads.** A job's output drawer downloads its complete stored output as `COMPUTERNAME.JobID.yyyy.MM.dd.HHmmss.log` (optionally with timestamps), and a run (Jobs › run, or a script's Executions tab) downloads every target's @@ -1317,6 +1338,38 @@ of one run shares one execution, which the result links to. call, answering with the job), `clientIds`, `groupId`, `groupIds` and `onlineOnly`, deduplicated and scope-filtered into one `runId`. +**Stagger** spreads a run's starts so a large fleet does not hit one share or +service at once: **Spread Over** (`staggerSeconds`, up to 24 hours) and +**Devices at a Time** (`staggerBatchSize`, default 1), on the Review tab and on +schedules. Devices fall into `ceil(targets ÷ batch)` batches evenly spaced over +the spread: the first starts at once, the last when the spread ends, in an order +shuffled per run so the same device is not always last. With one batch or no +spread the run dispatches as it always has. What it guarantees: + +- It is durable. Each job keeps its fixed start time and a worker on the + `SCHEDULE_WORKER` node sends it when it is due, so a backend restart loses + nothing and starting a run returns at once for any number of devices. +- Every device runs the script **as it was when the run started**, with its + arguments and prelude, from a snapshot sealed with Data Protection and cleared + when nothing is left to send. Secrets and presigned URLs are resolved when each + batch is sent and never stored. +- **Only Devices Online Now** is checked again at a device's start: offline by + then, it becomes skipped. +- A deleted script fails the jobs still waiting. **Cancel Waiting** + (`POST /api/executions/{runId}/cancel-waiting`, Run Jobs, audited, within the + caller's device scope) cancels what has not been sent. +- A job whose publish fails is retried by the same worker, with or without a + stagger, for up to 24 hours before it fails; a vault or S3 failure at send time + is retried for 5 minutes. + +**Watching a run** (Jobs › a run): while devices wait, a strip counts down to the +next and the last start, corrected for the browser's clock, and each waiting +device counts down in its row. Above the devices, updating live: a tile per +state, an outcome ring, exit codes, errors grouped by exception type and first +line with sample devices, and completions over time; tiles, exit codes and errors +filter the table. `GET /api/executions/{runId}/breakdown` serves the charts, +limited to the caller's device scope. + **Schedules** (the Schedules page, plus a Schedules tab on scripts, devices and groups) are scheduled triggers: *when* (one-shot or recurring), *target* (a device, a group whose members resolve at fire time, or all devices), an @@ -1342,6 +1395,8 @@ Semantics worth knowing: - The worker sweeps regularly; recurrence advances from *now* after each firing, so server downtime causes **at most one late firing**, never a storm. - One-shot schedules disable themselves after firing. +- A schedule's **Stagger** applies to each firing; event triggers run on one + device at a time and have none. - Each schedule records its **last result** (jobs queued, or the error — e.g. a vault failure) and the execution it created; scheduled jobs are attributed to `schedule`, run-now firings to the operator. @@ -1460,6 +1515,158 @@ the device keeps its history, groups and tags. --- +## Removing devices + +**Remove Agent…** on a device (row menu, bulk Actions, device page) or **Remove +Agents…** on a group or folder (nested groups included) takes the device out of +the system at once and uninstalls its agent now, or whenever it next comes back. +It needs the `devices.remove` permission, which Admin has and Operator and Viewer +do not. Scoped roles reach only the devices in their scope. + +The device row goes in the same transaction as its group memberships and trigger +runs; schedules aimed at that device alone are disabled and jobs that never +started are cancelled. Finished jobs, their output and the device's history stay, +and old runs keep its name. What stands in for the row is a **removal**, kept +forever as the history. + +```mermaid +flowchart TD + R["Remove Agent: device row deleted, removal waiting"] --> S{"Seen online?"} + S -- "No, however long" --> S + S -- "Yes, agent 1.1.0 or later" --> C["Remove command sent (at most every 5 min)"] + S -- "Yes, older agent" --> O["Agent too old: stays open, stays blocked"] + C --> U["Agent reports uninstalling"] + C -- "It failed" --> F["Couldn't Uninstall: asked again"] + C -- "No answer for 30 min" --> S + F --> S + U -- "Silent for 15 min" --> D["Removed: consumer deleted, commands purged"] + U -- "Still reporting after 15 min" --> S + D -- "Reports again" --> S +``` + +**Online.** The next scheduler pass (every 15 seconds, on the node with +`SCHEDULE_WORKER` on) sends the command. Its enrollment id must match what the +agent holds, so a stale command can never uninstall a device enrolled again since. +After a successful uninstall nobody is left to report success, so 15 minutes of +silence after "uninstalling" counts as gone. + +**Offline.** A removal waits for as long as the device stays away. Presence, +inventory and ready notices from a client id with no device row only update the +removal; they never bring the row back. + +**Credentials.** Agents from 1.1.0 on hold a NATS credential that lasts 24 hours, +renewed over HTTPS at half life and at every start. A renewal for a removed device +answers `410` with problem type `urn:problem:device-removed`, and the agent +uninstalls itself, so a device that slips past the command is still cut off within +a day. Only the holder of the enrolled key learns this: a bad signature gets the +same plain `401` as an id nobody knows. Older agents never renew and could not +survive an expiring credential, so enrollment still gives them one that never +expires. To keep that true an agent on 1.1.0 or later is never upgraded or rolled +back to anything older (it is left out as `credential-downgrade`); exactly 1.1.0 +is fine. + +**Tokens.** A bootstrap token created at or before the removal no longer enrolls +that device, whatever its other limits say, so the install it has and a GPO still +pushing an old token cannot bring it back. The response is the plain `401` of any +refused token, with the reason in the audit trail. A token created after the +removal enrolls it on a fresh device row: an open removal closes as removed, and +its old command consumer and commands are cleared first, so nothing old replays. + +**In the Devices list.** A removal shows in the list for 7 days after its last +change: dimmed, with its state, not selectable and not counted in the totals. After +that the list stays uncluttered, and the **Removals** option of the Status filter +shows every removal whatever its age, with who asked, when, and what happened. +Clicking a removal opens the device's history. `GET /api/device-removals` takes +`state`, `clientId` and `limit` (default 200). Scoped roles see the removals of +devices that were in one of their scope groups when they were removed; a device +that was in none is visible to unscoped roles only. + +**Stop Trying.** `POST /api/device-removals/{id}/stop` on an open removal ends the +chase: the device stays blocked, its pending command is purged and its agent stays +installed, cut off when its credential expires (a renewal gets a plain `401`, not +`410`, because `410` would uninstall it after all). Enrolling it again needs a new +token. + +**Agents too old.** An agent below 1.1.0 cannot take the command or renew. Its +removal stays open marked *Uninstall It by Hand or Upgrade It First*, and since +its credential never expires the server cannot cut it off from NATS: it can keep +connecting, but the server ignores everything it reports. Once such an agent reports +1.1.0 or later the command goes out as usual. + +The audit trail keeps `device-removal-requested`, `device-removed` and +`device-removal-stopped`. The project wiki is updated separately. + +### On the device + +**Credentials.** The agent's NATS credential lives 24 hours. A small background +service renews it over HTTPS (`POST /api/enrollment/credential`, signed with the +enrolled NKey, TLS as for enrollment) at every start and at half life, whether +or not NATS is connected, so an expired credential recovers by itself as soon as +the server can be reached, and a credential from before 1.1 that never expires +is replaced at the first start. The old credential stays until the new one is +stored; the NATS connection takes the new one at its next reconnect, at the +latest when the server ends the connection as the old one expires. + +| The server answers | The agent | +|---|---| +| a new credential | stores it, atomically, and looks again at half its life | +| network error, 5xx, anything unexpected | keeps what it has and tries again after 30 s, doubling to 15 minutes | +| 401 (orphaned, or the clock is off) | keeps what it has and tries again hourly; a configured bootstrap token enrolls it again ([Device lifecycle](#device-lifecycle-reconnect-or-orphan)) | +| 410 `urn:problem:device-removed` | uninstalls itself, as below. Not with `TlsSkipVerify` on or over plain http: an unverified server must not be able to uninstall an agent, so that only logs a warning | + +While an upgrade could still roll back (`handover.json`, `journal.json`, +`msi.json`), the stored credential is left alone: a version before 1.1 cannot +renew and would strand on a 24-hour one. It renews when the upgrade is kept. The +one exception is a credential within an hour of expiring, which would strand +either way and would keep the new version from reporting healthy. For the same +reason an agent on 1.1 or later refuses an upgrade to a version before 1.1 +(**failed / unsupported**). + +**The remove command.** `command.agent.remove` may be old, since it waits in its +stream, so the agent acts only when its enrollment ID is the one the agent holds +(else **failed / stale-enrollment**). A foreground or development run answers +**failed / unsupported**, an upgrade in progress **failed / busy** (the server +asks again). Otherwise the agent reports **uninstalling**, writes +`upgrade/removal.json` and leaves the rest to the watchdog. A device where no +watchdog can take it (none installed, or it has not taken it within 2 minutes) +runs the same runner from a copy of the agent's own binary. The watchdog starts +no upgrade while `removal.json` exists and a removal waits for an upgrade under +way; it does not supervise the agent during a removal. + +**The runner.** The watchdog copies itself out of both folders it is about to +wipe, into `%SystemRoot%\Temp\ExecuteR--\` (closed to SYSTEM and +Administrators), or `/var/lib/executer-removal-/` on Linux and macOS, and +starts the copy apart from its service: detached on Windows, as a transient +`systemd-run` unit on Linux, as a one-shot launchd daemon on macOS, since the +service manager would otherwise end it with the service. The runner: + +1. stops the agent; +2. uninstalls: an MSI install in process through Windows Installer + (`MsiConfigureProductExW`, no UI, `REBOOT=ReallySuppress`, never msiexec), + which removes both services; 1618 is waited out for up to 10 minutes. A + portable install loses both services through the service manager (the runner + refuses to wipe a folder that holds no agent binary, or a shared one such as + `C:\Program Files`); +3. verifies and finishes the purge, whatever the MSI's own purge managed. + +Three attempts, 5 minutes apart, 30 minutes at most each. A failed attempt starts +both services again. After the third, `removal.json` says **failed** with the +reason, the agent reports **failed / msi** or **failed / uninstall**, deletes the +file and carries on; the server asks again. After a success nobody is left to +report it. + +| Wiped | Left | +|---|---| +| `credentials.dat` and the whole data folder (`%ProgramData%\ExecuteR`, `/var/lib/executer`) | the policy key `HKLM\SOFTWARE\Policies\ExecuteR` (and the policy section of `agent.json`): it is group policy's | +| `HKLM\SOFTWARE\ExecuteR` with `Runtime`, `ServerURL`, the token, proxy and TLS values (`/etc/executer/agent.json`, the `/usr/local/bin` link) | a server CA the agent did not itself add to the trust store. That includes every CA installed before 1.1, which kept no record of it | +| the install folder, `Logs` included | the runner's folder on Windows, until the next restart | +| both services; the server CA the agent added (`managed-ca.txt`) | | + +A file still locked after a minute is deleted at the next restart. A failed +removal keeps its logs as `\Logs\agent-removal--*.log`. + +--- + ## Agent installer delivery **The installer ships inside the backend image — no manual seeding.** The CI diff --git a/src/Agent.Watchdog/AgentSupervisor.cs b/src/Agent.Watchdog/AgentSupervisor.cs index 746ce87..7388875 100644 --- a/src/Agent.Watchdog/AgentSupervisor.cs +++ b/src/Agent.Watchdog/AgentSupervisor.cs @@ -25,6 +25,15 @@ public sealed class AgentSupervisor(WatchdogOptions options, IAgentService agent public async Task CheckAsync(CancellationToken cancellationToken) { + // A removal stops the agent on purpose; only one that failed hands it back. + if (WatchdogFileStore.Read(options.RemovalPath, WatchdogJsonContext.Default.WatchdogRemoval) is { Phase: not Shared.Contracts.Models.WatchdogRemovalPhases.Failed }) + { + runningSince = null; + failedSince = null; + Note("A removal is under way; the agent is left to it"); + return; + } + var status = await agent.QueryAsync(cancellationToken); var now = clock.UtcNow; diff --git a/src/Agent.Watchdog/Msi/WindowsInstaller.cs b/src/Agent.Watchdog/Msi/WindowsInstaller.cs index c0cf237..c1b5e30 100644 --- a/src/Agent.Watchdog/Msi/WindowsInstaller.cs +++ b/src/Agent.Watchdog/Msi/WindowsInstaller.cs @@ -17,6 +17,12 @@ public interface IWindowsInstaller /// MsiInstallProductW: installs the package and returns Windows Installer's own code (what msiexec would exit with). int InstallProduct(string package, string commandLine); + + /// MsiConfigureProductExW with INSTALLSTATE_ABSENT: uninstalls the product and returns Windows Installer's own code. + int UninstallProduct(string productCode, string commandLine); + + /// MsiEnumRelatedProductsW: the product code installed under the UpgradeCode, or null. + string? RelatedProduct(string upgradeCode); } /// Windows Installer's answers an MSI run acts on. @@ -37,6 +43,8 @@ public static class MsiExitCodes public const int PackageInvalid = 1620; /// ERROR_PRODUCT_VERSION: a newer version is installed (a downgrade without ALLOWDOWNGRADE=1). public const int ProductVersion = 1638; + /// ERROR_UNKNOWN_PRODUCT: the product is not installed (an uninstall that has nothing to do). + public const int UnknownProduct = 1605; /// Done, done with a restart pending, or done with a restart started. public static bool Succeeded(int code) => code is Success or SuccessRebootRequired or SuccessRebootInitiated; @@ -51,6 +59,7 @@ public static class MsiExitCodes InstallFailure => "a fatal error during installation", PackageInvalid => "the package could not be opened", ProductVersion => "another version of the product is already installed", + UnknownProduct => "the product is not installed", 1619 => "the package could not be opened", 1625 => "the installation is forbidden by system policy", 1633 => "the package is not supported on this platform", @@ -64,6 +73,12 @@ public sealed partial class WindowsInstallerApi : IWindowsInstaller /// INSTALLUILEVEL_NONE. private const int InstallUiLevelNone = 2; + /// INSTALLLEVEL_DEFAULT. + private const int InstallLevelDefault = 0; + + /// INSTALLSTATE_ABSENT: Windows Installer sets REMOVE=ALL, as for msiexec's /x. + private const int InstallStateAbsent = 2; + /// /// msiexec's /l*v (msi.h INSTALLLOGMODE_*): FATALEXIT 0x1, ERROR 0x2, /// WARNING 0x4, USER 0x8, INFO 0x10, RESOLVESOURCE 0x40, OUTOFDISKSPACE 0x80, @@ -89,6 +104,21 @@ public sealed partial class WindowsInstallerApi : IWindowsInstaller public int InstallProduct(string package, string commandLine) => (int)MsiInstallProductW(package, commandLine); + public int UninstallProduct(string productCode, string commandLine) + => (int)MsiConfigureProductExW(productCode, InstallLevelDefault, InstallStateAbsent, commandLine); + + public unsafe string? RelatedProduct(string upgradeCode) + { + var buffer = stackalloc char[39]; + return MsiEnumRelatedProductsW(upgradeCode, 0, 0, buffer) == 0 ? new string(buffer, 0, 38) : null; + } + + [LibraryImport("msi.dll", StringMarshalling = StringMarshalling.Utf16)] + private static partial uint MsiConfigureProductExW(string product, int installLevel, int installState, string commandLine); + + [LibraryImport("msi.dll", StringMarshalling = StringMarshalling.Utf16)] + private static unsafe partial uint MsiEnumRelatedProductsW(string upgradeCode, uint reserved, uint index, char* productCode); + [LibraryImport("msi.dll")] private static partial int MsiSetInternalUI(int level, nint window); diff --git a/src/Agent.Watchdog/Platform/ServiceInterop.cs b/src/Agent.Watchdog/Platform/ServiceInterop.cs index f5ae041..1f7e86d 100644 --- a/src/Agent.Watchdog/Platform/ServiceInterop.cs +++ b/src/Agent.Watchdog/Platform/ServiceInterop.cs @@ -12,7 +12,20 @@ internal static partial class ServiceInterop internal const uint SC_MANAGER_CONNECT = 0x1; internal const uint SC_MANAGER_ALL_ACCESS = 0xF003F; internal const uint SERVICE_QUERY_STATUS = 0x4; + internal const uint SERVICE_START = 0x10; + internal const uint SERVICE_STOP = 0x20; + internal const uint DELETE = 0x10000; internal const uint SERVICE_ALL_ACCESS = 0xF01FF; + internal const uint SERVICE_CONTROL_STOP = 0x1; + internal const uint SERVICE_STOPPED = 0x1; + internal const uint SERVICE_START_PENDING = 0x2; + internal const uint SERVICE_STOP_PENDING = 0x3; + internal const uint SERVICE_RUNNING = 0x4; + internal const int ERROR_SERVICE_ALREADY_RUNNING = 1056; + internal const int ERROR_SERVICE_DOES_NOT_EXIST = 1060; + internal const int ERROR_SERVICE_CANNOT_ACCEPT_CTRL = 1061; + internal const int ERROR_SERVICE_NOT_ACTIVE = 1062; + internal const int ERROR_SERVICE_MARKED_FOR_DELETE = 1072; internal const uint SERVICE_WIN32_OWN_PROCESS = 0x10; internal const uint SERVICE_AUTO_START = 0x2; internal const uint SERVICE_ERROR_NORMAL = 0x1; @@ -62,6 +75,18 @@ internal static partial class ServiceInterop public uint dwServiceFlags; } + [StructLayout(LayoutKind.Sequential)] + internal struct SERVICE_STATUS + { + public uint dwServiceType; + public uint dwCurrentState; + public uint dwControlsAccepted; + public uint dwWin32ExitCode; + public uint dwServiceSpecificExitCode; + public uint dwCheckPoint; + public uint dwWaitHint; + } + [LibraryImport("advapi32.dll", EntryPoint = "OpenSCManagerW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)] internal static partial IntPtr OpenSCManager(string? machineName, string? databaseName, uint access); @@ -119,6 +144,14 @@ internal static partial class ServiceInterop [return: MarshalAs(UnmanagedType.Bool)] internal static partial bool CloseServiceHandle(IntPtr handle); + [LibraryImport("advapi32.dll", EntryPoint = "StartServiceW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static partial bool StartService(IntPtr service, int argumentCount, IntPtr arguments); + + [LibraryImport("advapi32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + internal static partial bool ControlService(IntPtr service, uint control, out SERVICE_STATUS status); + internal static Win32Exception LastError(string operation) => new(Marshal.GetLastPInvokeError(), $"{operation} failed"); diff --git a/src/Agent.Watchdog/Removal/RemovalCoordinator.cs b/src/Agent.Watchdog/Removal/RemovalCoordinator.cs new file mode 100644 index 0000000..b7810ad --- /dev/null +++ b/src/Agent.Watchdog/Removal/RemovalCoordinator.cs @@ -0,0 +1,124 @@ +using Shared.Contracts.Models; +using Shared.Contracts.Serialization; + +namespace Agent.Watchdog.Removal; + +/// +/// The watchdog's side of a removal: on removal.json it starts the detached +/// runner and leaves it to it. The runner stops and removes this watchdog, so a +/// runner that died can only be noticed if the removal failed and the watchdog is +/// still (or again) here; it is then started again, at most +/// times. An upgrade under way finishes first; the agent refuses a removal +/// during one, so that takes a command and a hand-over crossing. +/// +public sealed class RemovalCoordinator(WatchdogOptions options, IRemovalRunnerLauncher? launcher, IAgentService agent, IWatchdogClock clock, WatchdogLog log) +{ + public const int MaxLaunches = 5; + + /// A runner this watchdog just started is given this long to take its lock. + public static readonly TimeSpan LaunchGrace = TimeSpan.FromMinutes(1); + + private DateTimeOffset launchedUtc = DateTimeOffset.MinValue; + private int launches; + private bool waitingNoted; + + /// True while the removal owns this pass: no upgrade step and no supervision. + public async Task PassAsync(CancellationToken cancellationToken) + { + if (!File.Exists(options.RemovalPath)) + { + launches = 0; + return false; + } + + var removal = WatchdogFileStore.Read(options.RemovalPath, WatchdogJsonContext.Default.WatchdogRemoval); + if (removal is null) + { + log.Error($"{options.RemovalPath} does not read; it is removed and nothing was changed"); + WatchdogFileStore.TryDelete(options.RemovalPath); + return false; + } + + if (removal.Phase == WatchdogRemovalPhases.Failed) + { + // Over; the agent reports it and deletes the file. + return false; + } + + if (File.Exists(options.JournalPath) || File.Exists(options.MsiJournalPath)) + { + if (!waitingNoted) + { + log.Warn($"Removal {removal.RemovalId} waits for the upgrade under way to end"); + waitingNoted = true; + } + + return false; + } + + waitingNoted = false; + if (removal.RunnerDirectory is { } directory + && (RemovalRunLock.IsHeld(Path.Combine(directory, WatchdogRemovalFiles.RunnerLock)) || clock.UtcNow - launchedUtc < LaunchGrace)) + { + return true; + } + + if (launcher is null) + { + await GiveUpAsync(removal, "This watchdog runs in its development mode and cannot remove the agent.", cancellationToken); + return false; + } + + if (removal.Attempts >= WatchdogRemovalFiles.MaxAttempts || launches >= MaxLaunches) + { + await GiveUpAsync(removal, removal.LastError ?? $"The removal runner stopped before the removal ended, {launches} times.", cancellationToken); + return false; + } + + if (removal.RunnerDirectory is { } previous) + { + SwapPaths.TryDeleteDirectory(previous); + } + + log.BeginRun(); + log.Info($"Removal {removal.RemovalId} handed over ({removal.InstallKind} install); starting the runner"); + var (runnerDirectory, problem) = launcher.Prepare(removal.RemovalId); + if (problem is null) + { + WatchdogFileStore.Write(options.RemovalPath, + removal with { Phase = WatchdogRemovalPhases.Uninstalling, RunnerDirectory = runnerDirectory }, + WatchdogJsonContext.Default.WatchdogRemoval); + problem = launcher.Start(removal.RemovalId, runnerDirectory!); + } + + launches++; + if (problem is not null) + { + await GiveUpAsync(removal, $"The removal runner did not start: {problem}.", cancellationToken); + return false; + } + + launchedUtc = clock.UtcNow; + return true; + } + + private async Task GiveUpAsync(WatchdogRemoval removal, string why, CancellationToken cancellationToken) + { + log.Error($"Removal {removal.RemovalId} gave up: {why}"); + WatchdogFileStore.Write(options.RemovalPath, + removal with { Phase = WatchdogRemovalPhases.Failed, LastError = why }, + WatchdogJsonContext.Default.WatchdogRemoval); + try + { + var status = await agent.QueryAsync(cancellationToken); + if (status.State != AgentRunState.NotInstalled && !status.IsRunning) + { + await agent.StartAsync(cancellationToken); + } + } + catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested) + { + log.Warn("The agent could not be started", exception); + } + } +} diff --git a/src/Agent.Watchdog/Removal/RemovalPlatform.cs b/src/Agent.Watchdog/Removal/RemovalPlatform.cs new file mode 100644 index 0000000..ec9414a --- /dev/null +++ b/src/Agent.Watchdog/Removal/RemovalPlatform.cs @@ -0,0 +1,484 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using System.Security.Cryptography.X509Certificates; +using System.Text.Json.Nodes; +using Agent.Watchdog.Msi; +using Agent.Watchdog.Platform; +using Microsoft.Win32; +using Shared.Contracts.Models; +using Shared.Infrastructure.Constants; +using static Agent.Watchdog.Platform.ServiceInterop; + +namespace Agent.Watchdog.Removal; + +/// The services a portable install's removal takes out (an MSI's are the MSI's to remove); replaced in tests. +public interface IRemovalServices +{ + Task StopWatchdogAsync(CancellationToken cancellationToken); + + Task StartWatchdogAsync(CancellationToken cancellationToken); + + /// Removes the agent's and the watchdog's registrations; nothing to do for one already gone. + Task RemoveAsync(CancellationToken cancellationToken); +} + +/// +/// Hidden entry point of the runner, for both binaries that have it: the runner +/// uses no agent code and no watchdog-only code, so the agent can fall back on it +/// when no watchdog can take a removal. Logs to its own folder, since the agent's +/// log folder goes with the install. +/// +public static class RemovalHost +{ + public static async Task RunAsync(Guid removalId, string[] args) + { + string? Value(string name) + { + for (var i = 0; i < args.Length - 1; i++) + { + if (string.Equals(args[i], name, StringComparison.OrdinalIgnoreCase)) + { + return args[i + 1]; + } + } + + return null; + } + + if (Value("--install-dir") is not { } install || Value("--data-dir") is not { } data || Value("--runner-dir") is not { } runnerDirectory) + { + Console.Error.WriteLine("Usage: --install-dir --data-dir --runner-dir "); + return 1; + } + + var options = new WatchdogOptions + { + InstallDirectory = Path.GetFullPath(install), + DataDirectory = Path.GetFullPath(data), + SelfPath = null, + }; + var log = new WatchdogLog(runnerDirectory, console: false, name: RemovalRunner.LogFileName); + var agentPath = Path.Combine(options.InstallDirectory, options.AgentExecutable); + + IAgentService agent; + IRemovalServices services; + IRemovalMachine machine; + IWindowsInstaller? installer = null; + if (OperatingSystem.IsWindows()) + { + agent = new ScmService(Branding.AgentServiceName, agentPath, log); + services = new WindowsRemovalServices( + (ScmService)agent, + new ScmService(Branding.WatchdogServiceName, Path.Combine(options.InstallDirectory, Branding.WatchdogExecutableName), log)); + machine = new WindowsRemovalMachine(log); + installer = new WindowsInstallerApi(); + } + else if (OperatingSystem.IsMacOS()) + { + agent = new LaunchdAgentService(Branding.AgentLaunchdLabel); + services = new UnixRemovalServices(); + machine = new UnixRemovalMachine(options, log); + } + else + { + agent = new SystemdAgentService(Branding.AgentSystemdUnit); + services = new UnixRemovalServices(); + machine = new UnixRemovalMachine(options, log); + } + + var clock = SystemClock.Instance; + var purge = new RemovalPurge(options, machine, clock, log, runnerDirectory); + var runner = new RemovalRunner(options, agent, services, installer, purge, clock, log, runnerDirectory); + return await runner.RunAsync(removalId, CancellationToken.None); + } +} + +/// A Windows service through the SCM's own APIs (never sc.exe, and none of ServiceController, which the agent's linux/macOS build lacks). +[SupportedOSPlatform("windows")] +public sealed class ScmService(string serviceName, string executablePath, WatchdogLog log) : IAgentService +{ + private static readonly TimeSpan StateChangeTimeout = TimeSpan.FromSeconds(60); + + public Task QueryAsync(CancellationToken cancellationToken) + { + if (QueryStatus(serviceName) is not { } status) + { + return Task.FromResult(AgentServiceStatus.NotInstalled); + } + + var state = status.dwCurrentState switch + { + SERVICE_STOPPED => AgentRunState.Stopped, + SERVICE_START_PENDING => AgentRunState.Starting, + SERVICE_STOP_PENDING => AgentRunState.Stopping, + _ => AgentRunState.Running, + }; + return Task.FromResult(new AgentServiceStatus(state, true, false)); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + WithService(SERVICE_START, service => + { + if (!StartService(service, 0, IntPtr.Zero) && Marshal.GetLastPInvokeError() != ERROR_SERVICE_ALREADY_RUNNING) + { + throw LastError($"StartService({serviceName})"); + } + }); + await WaitForAsync(SERVICE_RUNNING, cancellationToken); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + WithService(SERVICE_STOP, service => + { + if (!ControlService(service, SERVICE_CONTROL_STOP, out _) + && Marshal.GetLastPInvokeError() is not (ERROR_SERVICE_NOT_ACTIVE or ERROR_SERVICE_CANNOT_ACCEPT_CTRL)) + { + throw LastError($"ControlService({serviceName})"); + } + }); + await WaitForAsync(SERVICE_STOPPED, cancellationToken); + await StopLeftoversAsync(cancellationToken); + } + + /// Removes the registration: gone once the last handle on it closes. + public void Delete() => WithService(DELETE, service => + { + if (!DeleteService(service) && Marshal.GetLastPInvokeError() != ERROR_SERVICE_MARKED_FOR_DELETE) + { + throw LastError($"DeleteService({serviceName})"); + } + + log.Info($"Service {serviceName} removed"); + }); + + /// Runs on the open service; a service that is not there is nothing to do. + private void WithService(uint access, Action action) + { + var scm = OpenSCManager(null, null, SC_MANAGER_CONNECT); + if (scm == IntPtr.Zero) + { + throw LastError("OpenSCManager"); + } + + try + { + var service = OpenService(scm, serviceName, access); + if (service == IntPtr.Zero) + { + if (Marshal.GetLastPInvokeError() == ERROR_SERVICE_DOES_NOT_EXIST) + { + return; + } + + throw LastError($"OpenService({serviceName})"); + } + + try + { + action(service); + } + finally + { + CloseServiceHandle(service); + } + } + finally + { + CloseServiceHandle(scm); + } + } + + private async Task WaitForAsync(uint state, CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + StateChangeTimeout; + while (QueryStatus(serviceName) is { } status && status.dwCurrentState != state) + { + if (DateTime.UtcNow >= deadline) + { + throw new System.TimeoutException($"{serviceName} did not reach state {state} within {StateChangeTimeout.TotalSeconds:0} s"); + } + + await Task.Delay(250, cancellationToken); + } + } + + /// The service process can outlive its stop report, and the agent's own children hold its binary: wait for them, then end what is left. + private async Task StopLeftoversAsync(CancellationToken cancellationToken) + { + var name = Path.GetFileNameWithoutExtension(executablePath); + var deadline = DateTime.UtcNow.AddSeconds(10); + while (true) + { + var leftovers = Process.GetProcessesByName(name).Where(IsThisBinary).ToList(); + if (leftovers.Count == 0) + { + return; + } + + var late = DateTime.UtcNow >= deadline; + foreach (var process in leftovers) + { + try + { + if (late) + { + log.Warn($"Ending process {process.Id} of {name} that outlived its service"); + process.Kill(entireProcessTree: true); + } + } + catch (Exception exception) when (exception is InvalidOperationException or Win32Exception) + { + } + finally + { + process.Dispose(); + } + } + + if (late) + { + return; + } + + await Task.Delay(500, cancellationToken); + } + } + + private bool IsThisBinary(Process process) + { + try + { + return string.Equals(process.MainModule?.FileName, executablePath, StringComparison.OrdinalIgnoreCase); + } + catch (Exception exception) when (exception is InvalidOperationException or Win32Exception) + { + return false; + } + } +} + +[SupportedOSPlatform("windows")] +public sealed class WindowsRemovalServices(ScmService agent, ScmService watchdog) : IRemovalServices +{ + public Task StopWatchdogAsync(CancellationToken cancellationToken) => watchdog.StopAsync(cancellationToken); + + public async Task StartWatchdogAsync(CancellationToken cancellationToken) + { + if ((await watchdog.QueryAsync(cancellationToken)) is { IsRunning: false, State: not AgentRunState.NotInstalled }) + { + await watchdog.StartAsync(cancellationToken); + } + } + + public Task RemoveAsync(CancellationToken cancellationToken) + { + agent.Delete(); + watchdog.Delete(); + return Task.CompletedTask; + } +} + +/// The normal registry key, the Local Machine certificate stores, and restart-time deletes. +[SupportedOSPlatform("windows")] +public sealed partial class WindowsRemovalMachine(WatchdogLog log) : IRemovalMachine +{ + private const uint MoveFileDelayUntilReboot = 0x4; + + public void RemoveConfiguration() => Registry.LocalMachine.DeleteSubKeyTree(Branding.AgentRegistryKey, throwOnMissingSubKey: false); + + public bool ConfigurationRemains + { + get + { + using var key = Registry.LocalMachine.OpenSubKey(Branding.AgentRegistryKey); + return key is not null; + } + } + + public void RemoveManagedCa(IReadOnlyCollection thumbprints) + { + foreach (var name in new[] { StoreName.Root, StoreName.CertificateAuthority }) + { + using var store = new X509Store(name, StoreLocation.LocalMachine); + store.Open(OpenFlags.ReadWrite); + foreach (var thumbprint in thumbprints) + { + foreach (var certificate in store.Certificates.Find(X509FindType.FindByThumbprint, thumbprint, validOnly: false)) + { + store.Remove(certificate); + log.Info($"Removed the server CA {certificate.Subject} from LocalMachine\\{name}"); + } + } + } + } + + public void DeleteAtRestart(string path) + { + if (!MoveFileExW(path, null, MoveFileDelayUntilReboot)) + { + log.Warn($"{path} could not be scheduled for deletion at the next restart (error {Marshal.GetLastPInvokeError()})"); + } + } + + /// Held by the running binary until it exits; its log stays for support until the restart that deletes it. + public void RemoveRunnerDirectory(string directory) + { + try + { + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + DeleteAtRestart(file); + } + + DeleteAtRestart(directory); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + log.Warn($"{directory} could not be scheduled for deletion at the next restart", exception); + } + } + + [LibraryImport("kernel32.dll", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static partial bool MoveFileExW(string existingFileName, string? newFileName, uint flags); +} + +/// Both units or daemons, run from a runner that is outside either's cgroup or process group. +public sealed class UnixRemovalServices : IRemovalServices +{ + public Task StopWatchdogAsync(CancellationToken cancellationToken) => Task.Run(UnixWatchdogRegistration.Stop, cancellationToken); + + public Task StartWatchdogAsync(CancellationToken cancellationToken) => Task.Run(UnixWatchdogRegistration.Start, cancellationToken); + + public Task RemoveAsync(CancellationToken cancellationToken) => Task.Run(() => + { + if (OperatingSystem.IsMacOS()) + { + Command.Run("launchctl", "bootout", $"system/{Branding.AgentLaunchdLabel}"); + DeleteIfThere($"/Library/LaunchDaemons/{Branding.AgentLaunchdLabel}.plist"); + } + else + { + Command.Run("systemctl", "disable", "--now", Branding.AgentSystemdUnit); + DeleteIfThere($"/etc/systemd/system/{Branding.AgentSystemdUnit}.service"); + } + + UnixWatchdogRegistration.Remove(); + }, cancellationToken); + + private static void DeleteIfThere(string path) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } +} + +/// agent.json (its policy section stays), the PATH link, and the trust anchors ServerTrust wrote. +public sealed class UnixRemovalMachine(WatchdogOptions options, WatchdogLog log) : IRemovalMachine +{ + private const string PathLink = "/usr/local/bin/" + Branding.AgentUnixExecutableName; + private const string AnchorName = "executer-server-ca.crt"; + private const string DebianAnchors = "/usr/local/share/ca-certificates/"; + private const string RedHatAnchors = "/etc/pki/ca-trust/source/anchors/"; + + private static string PolicySection => Branding.AgentPolicyRegistryKey.Replace('\\', '/'); + + public void RemoveConfiguration() + { + var path = Branding.AgentUnixConfigFile; + if (new FileInfo(PathLink).LinkTarget is { } target + && (SwapPaths.IsInside(target, options.InstallDirectory) || target.Contains("/executer/", StringComparison.Ordinal))) + { + File.Delete(PathLink); + } + + if (File.Exists(path)) + { + var kept = new JsonObject(); + try + { + if (JsonNode.Parse(File.ReadAllText(path)) is JsonObject root + && root.FirstOrDefault(p => string.Equals(p.Key, PolicySection, StringComparison.OrdinalIgnoreCase)) is { Value: JsonObject policy } section) + { + kept[section.Key] = policy.DeepClone(); + } + } + catch (System.Text.Json.JsonException) + { + log.Warn($"{path} does not read; it goes whole"); + } + + if (kept.Count > 0) + { + File.WriteAllText(path, kept.ToJsonString()); + return; + } + + File.Delete(path); + } + + if (Path.GetDirectoryName(path) is { } folder) + { + SwapPaths.TryDeleteIfEmpty(folder); + } + } + + public bool ConfigurationRemains + { + get + { + if (!File.Exists(Branding.AgentUnixConfigFile)) + { + return false; + } + + try + { + // Only the policy section may stay. + return JsonNode.Parse(File.ReadAllText(Branding.AgentUnixConfigFile)) is not JsonObject root + || root.Any(p => !string.Equals(p.Key, PolicySection, StringComparison.OrdinalIgnoreCase)); + } + catch (System.Text.Json.JsonException) + { + return true; + } + } + } + + public void RemoveManagedCa(IReadOnlyCollection thumbprints) + { + if (OperatingSystem.IsMacOS()) + { + foreach (var thumbprint in thumbprints) + { + Command.Run("security", "delete-certificate", "-Z", thumbprint, "/Library/Keychains/System.keychain"); + } + + return; + } + + foreach (var (folder, refresh) in new[] { (DebianAnchors, "update-ca-certificates"), (RedHatAnchors, "update-ca-trust") }) + { + var anchor = folder + AnchorName; + if (File.Exists(anchor)) + { + File.Delete(anchor); + Command.Run(refresh); + log.Info($"Removed the server CA {anchor}"); + } + } + } + + public void DeleteAtRestart(string path) + { + // Unlinking works here even for a file in use; a sweep already tried. + } + + public void RemoveRunnerDirectory(string directory) => SwapPaths.TryDeleteDirectory(directory); +} diff --git a/src/Agent.Watchdog/Removal/RemovalPurge.cs b/src/Agent.Watchdog/Removal/RemovalPurge.cs new file mode 100644 index 0000000..66d86e1 --- /dev/null +++ b/src/Agent.Watchdog/Removal/RemovalPurge.cs @@ -0,0 +1,247 @@ +using Shared.Contracts.Models; +using Shared.Infrastructure.Constants; + +namespace Agent.Watchdog.Removal; + +/// What a removal wipes outside the install and data folders; replaced in tests. +public interface IRemovalMachine +{ + /// + /// The agent's configuration and runtime state: the normal registry key tree + /// (agent.json and the PATH link on linux/macOS). The policy key is + /// group policy's and stays. + /// + void RemoveConfiguration(); + + bool ConfigurationRemains { get; } + + /// The server CA certificates the agent installed itself (). + void RemoveManagedCa(IReadOnlyCollection thumbprints); + + /// Deletes the file or empty folder at the next restart (nothing to do on linux/macOS, where nothing is locked). + void DeleteAtRestart(string path); + + /// The runner's folder, which holds the running binary: now where that works, otherwise at the next restart. + void RemoveRunnerDirectory(string directory); +} + +/// +/// Verifies and finishes the wipe, whichever way the product went. The MSI's +/// purge-credentials custom action runs with Return="ignore", so +/// its success proves nothing: credentials.dat, the configuration, the +/// data and install folders (Logs with them) are each checked and removed +/// here; what is still locked after a minute goes at the next restart. +/// +public sealed class RemovalPurge(WatchdogOptions options, IRemovalMachine machine, IWatchdogClock clock, WatchdogLog log, string runnerDirectory) +{ + public const int Sweeps = 7; + public static readonly TimeSpan SweepDelay = TimeSpan.FromSeconds(10); + + private readonly List thumbprints = []; + + private string CredentialPath => Path.Combine(options.DataDirectory, Branding.CredentialFileName); + + private string ManagedCaPath => Path.Combine(options.DataDirectory, WatchdogFiles.ManagedCa); + + private string ManagedCaCopy => Path.Combine(runnerDirectory, WatchdogFiles.ManagedCa); + + /// Takes what the purge needs later from the data folder, into the runner's folder (a resumed runner finds it there). + public void Snapshot() + { + try + { + if (File.Exists(ManagedCaPath)) + { + File.Copy(ManagedCaPath, ManagedCaCopy, overwrite: true); + } + + thumbprints.Clear(); + if (File.Exists(ManagedCaCopy)) + { + thumbprints.AddRange(File.ReadAllLines(ManagedCaCopy).Select(l => l.Trim()).Where(l => l.Length > 0)); + } + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + log.Warn("The list of installed server CAs could not be read; they stay", exception); + } + } + + /// Why the install folder must not be wiped; null when it may. Checked before a portable install's services go. + public string? InstallFolderRefusal() + { + if (IsProtectedFolder(options.InstallDirectory)) + { + return $"The agent runs from {options.InstallDirectory}, which is not a folder of its own; it was left alone."; + } + + return File.Exists(Path.Combine(options.InstallDirectory, options.AgentExecutable)) + ? null + : $"{options.InstallDirectory} holds no {options.AgentExecutable}; it was left alone."; + } + + /// Removes what is left; what stays after every sweep is listed and goes at the next restart. + public async Task> FinishAsync(CancellationToken cancellationToken) + { + var left = new List(); + for (var sweep = 1; sweep <= Sweeps; sweep++) + { + left = Sweep(); + if (left.Count == 0) + { + return left; + } + + if (sweep < Sweeps) + { + log.Info($"Still there (sweep {sweep} of {Sweeps}): {string.Join("; ", left)}"); + await clock.DelayAsync(SweepDelay, cancellationToken); + } + } + + foreach (var folder in new[] { options.DataDirectory, options.InstallDirectory }) + { + if (!IsProtectedFolder(folder)) + { + ScheduleAtRestart(folder); + } + } + + return left; + } + + public void RemoveRunnerDirectory(string directory) => machine.RemoveRunnerDirectory(directory); + + private List Sweep() + { + var left = new List(); + + // First and on its own: nothing else on the device can authenticate as it. + Try("credentials", () => + { + if (File.Exists(CredentialPath)) + { + File.SetAttributes(CredentialPath, FileAttributes.Normal); + File.Delete(CredentialPath); + } + }, left); + Try("configuration", machine.RemoveConfiguration, left); + if (thumbprints.Count > 0) + { + Try("server CA", () => machine.RemoveManagedCa(thumbprints), left); + } + + Try(options.DataDirectory, () => DeleteFolder(options.DataDirectory), left); + Try(options.InstallDirectory, () => DeleteFolder(options.InstallDirectory), left); + if (Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(options.InstallDirectory)) is { } parent + && !IsProtectedFolder(parent) + && string.Equals(Path.GetFileName(parent), Branding.ProductKey, StringComparison.OrdinalIgnoreCase)) + { + // %ProgramFiles%\ExecuteR, /opt/executer: only ever ours, when nothing else is in it. + SwapPaths.TryDeleteIfEmpty(parent); + } + + if (File.Exists(CredentialPath) && !left.Contains("credentials")) + { + left.Add("credentials"); + } + + if (machine.ConfigurationRemains && !left.Contains("configuration")) + { + left.Add("configuration"); + } + + return left; + } + + private void Try(string what, Action action, List left) + { + try + { + action(); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + log.Warn($"{what} could not be removed", exception); + left.Add(what); + } + } + + private static void DeleteFolder(string folder) + { + if (!Directory.Exists(folder)) + { + return; + } + + if (IsProtectedFolder(folder)) + { + throw new InvalidOperationException($"{folder} is not a folder of the agent's own"); + } + + foreach (var file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories)) + { + // Directory.Delete stops at a read-only file on Windows. + if ((File.GetAttributes(file) & FileAttributes.ReadOnly) != 0) + { + File.SetAttributes(file, FileAttributes.Normal); + } + } + + Directory.Delete(folder, recursive: true); + } + + private void ScheduleAtRestart(string folder) + { + try + { + if (!Directory.Exists(folder)) + { + return; + } + + foreach (var file in Directory.EnumerateFiles(folder, "*", SearchOption.AllDirectories)) + { + machine.DeleteAtRestart(file); + } + + // Deepest first: a folder goes only once it is empty. + foreach (var directory in Directory.EnumerateDirectories(folder, "*", SearchOption.AllDirectories).OrderByDescending(d => d.Length)) + { + machine.DeleteAtRestart(directory); + } + + machine.DeleteAtRestart(folder); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + log.Warn($"{folder} could not be scheduled for the next restart", exception); + } + } + + /// A root, or a folder the system or other software shares: a portable agent unpacked straight into one never takes it along. + public static bool IsProtectedFolder(string folder) + { + var full = Path.TrimEndingDirectorySeparator(Path.GetFullPath(folder)); + if (string.IsNullOrEmpty(Path.GetFileName(full))) + { + return true; + } + + IEnumerable shared = OperatingSystem.IsWindows() + ? + [ + Environment.GetFolderPath(Environment.SpecialFolder.Windows), + Environment.GetFolderPath(Environment.SpecialFolder.System), + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), + Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), + Environment.GetFolderPath(Environment.SpecialFolder.CommonProgramFiles), + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + Path.Combine(Path.GetPathRoot(full) ?? "", "Users"), + ] + : ["/bin", "/boot", "/dev", "/etc", "/home", "/lib", "/lib64", "/opt", "/proc", "/root", "/run", "/sbin", "/sys", "/tmp", "/usr", "/usr/bin", + "/usr/lib", "/usr/local", "/usr/local/bin", "/usr/sbin", "/var", "/var/lib", "/var/tmp", "/Applications", "/Library", "/System", "/Users"]; + return shared.Any(path => path.Length > 0 && SwapPaths.Comparer.Equals(Path.TrimEndingDirectorySeparator(path), full)); + } +} diff --git a/src/Agent.Watchdog/Removal/RemovalRunner.cs b/src/Agent.Watchdog/Removal/RemovalRunner.cs new file mode 100644 index 0000000..e493071 --- /dev/null +++ b/src/Agent.Watchdog/Removal/RemovalRunner.cs @@ -0,0 +1,344 @@ +using Agent.Watchdog.Msi; +using Shared.Contracts.Models; +using Shared.Contracts.Serialization; +using Shared.Infrastructure.Constants; + +namespace Agent.Watchdog.Removal; + +/// Timings of a removal; shortened in tests. +public sealed record RemovalRunnerSettings +{ + /// Between tries while Windows Installer is busy with another installation (1618). + public TimeSpan BusyRetryDelay { get; init; } = TimeSpan.FromSeconds(30); + + /// How long a busy Windows Installer is waited out within one attempt. + public TimeSpan BusyRetryFor { get; init; } = TimeSpan.FromMinutes(10); + + /// The longest one attempt's uninstall may take (real time: Windows Installer's call cannot be cut short). + public TimeSpan AttemptTimeout { get; init; } = WatchdogFiles.MsiInstallTimeout; +} + +/// +/// The detached removal runner (ExecuteR-Watchdog.exe remove-run <id>, or +/// ExecuteR-Agent.exe remove-self <id> where no watchdog can take it), +/// started from a copy of its binary outside the install and data folders, since +/// it wipes both. It owns the removal in removal.json while it holds +/// removal.lock in its own folder: +/// +/// stop the agent; +/// MSI install: Windows Installer in process, by product code +/// (REBOOT=ReallySuppress, no UI, verbose log in the runner's folder); +/// 0, 3010, 1641 and 1605 (already gone) are success, 1618 is tried again every +/// 30 s for up to 10 minutes. Portable install: the watchdog and agent services +/// go through the service manager; +/// the purge is verified and finished (): whatever +/// the MSI's own purge left is removed, and what is locked goes at the next restart. +/// +/// A failed attempt starts the services again and the next one follows 5 minutes +/// later, three in all; then the phase says failed, with the reason, for the +/// agent to report, and the device carries on. Unlike this +/// runner writes no outcome and verifies no health: on success nobody is left. +/// +public sealed class RemovalRunner( + WatchdogOptions options, + IAgentService agent, + IRemovalServices services, + IWindowsInstaller? installer, + RemovalPurge purge, + IWatchdogClock clock, + WatchdogLog log, + string runnerDirectory, + RemovalRunnerSettings? settings = null) +{ + public const string UninstallProperties = "REBOOT=ReallySuppress"; + public const string LogFileName = "removal"; + + private readonly RemovalRunnerSettings timings = settings ?? new RemovalRunnerSettings(); + + private string LockPath => Path.Combine(runnerDirectory, WatchdogRemovalFiles.RunnerLock); + + /// Runs (or resumes) the removal; 0 unless removal.json does not name it. + public async Task RunAsync(Guid removalId, CancellationToken cancellationToken) + { + bool gone; + using (var held = RemovalRunLock.TryHold(LockPath)) + { + if (held is null) + { + log.Info($"Another runner owns removal {removalId}; this one exits"); + return 0; + } + + var removal = WatchdogFileStore.Read(options.RemovalPath, WatchdogJsonContext.Default.WatchdogRemoval); + if (removal is null || removal.RemovalId != removalId) + { + log.Warn($"Removal {removalId} is not what {options.RemovalPath} names; nothing to do"); + return removal is null ? 1 : 0; + } + + log.BeginRun(); + log.Info($"Runner {Environment.ProcessId} owns removal {removalId} ({removal.InstallKind} install, {removal.Attempts} attempts so far)"); + + // The MSI's own purge takes the data folder this reads. + purge.Snapshot(); + + gone = await AttemptAllAsync(removal, cancellationToken); + if (gone) + { + var left = await purge.FinishAsync(cancellationToken); + log.Info(left.Count == 0 + ? $"Removal {removalId} is complete; nothing is left" + : $"Removal {removalId} is complete, except what goes at the next restart: {string.Join("; ", left)}"); + } + } + + purge.RemoveRunnerDirectory(runnerDirectory); + return 0; + } + + /// True once the product and its services are gone; false when every attempt failed. + private async Task AttemptAllAsync(WatchdogRemoval removal, CancellationToken cancellationToken) + { + while (true) + { + if (removal.Attempts >= WatchdogRemovalFiles.MaxAttempts) + { + Finish(removal with { LastError = removal.LastError ?? "The removal runner stopped before it ended." }); + return false; + } + + removal = removal with { Phase = WatchdogRemovalPhases.Uninstalling, Attempts = removal.Attempts + 1 }; + Write(removal); + log.Info($"Removing the agent (attempt {removal.Attempts} of {WatchdogRemovalFiles.MaxAttempts})"); + + string? problem; + try + { + problem = await AttemptAsync(removal, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + problem = $"The removal failed ({exception.GetBaseException().Message})."; + } + + if (problem is null) + { + return true; + } + + log.Error($"Attempt {removal.Attempts} failed: {problem}"); + await RestoreAsync(cancellationToken); + + removal = removal with { LastError = problem }; + if (removal.Attempts >= WatchdogRemovalFiles.MaxAttempts) + { + Finish(removal); + return false; + } + + Write(removal); + log.Info($"The next attempt follows in {WatchdogRemovalFiles.AttemptDelay.TotalMinutes:0} minutes; the agent runs meanwhile"); + await clock.DelayAsync(WatchdogRemovalFiles.AttemptDelay, cancellationToken); + } + } + + /// One attempt; null when the agent is gone, otherwise why not. + private async Task AttemptAsync(WatchdogRemoval removal, CancellationToken cancellationToken) + { + try + { + await agent.StopAsync(cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + // Windows Installer and the service manager both stop it themselves. + log.Warn("The agent did not stop", exception); + } + + var product = removal.InstallKind == AgentInstallKinds.Msi && installer is not null + ? installer.RelatedProduct(Branding.AgentMsiUpgradeCode) + : null; + return product is not null + ? await UninstallMsiAsync(removal, product) + : await RemovePortableAsync(removal, cancellationToken); + } + + private async Task UninstallMsiAsync(WatchdogRemoval removal, string product) + { + var logPath = MsiLogPath(removal.RemovalId); + log.Info($"Uninstalling {product} through Windows Installer ({UninstallProperties}); its log is {logPath}"); + var giveUp = clock.UtcNow + timings.BusyRetryFor; + while (true) + { + var answer = await UninstallOnceAsync(product, logPath); + if (answer is not { } code) + { + return $"Windows Installer did not finish within {timings.AttemptTimeout.TotalMinutes:0} minutes; its log is {logPath}."; + } + + if (code == MsiExitCodes.InstallerBusy) + { + if (clock.UtcNow + timings.BusyRetryDelay > giveUp) + { + return $"Another installation kept Windows Installer busy (1618) for {timings.BusyRetryFor.TotalMinutes:0} minutes."; + } + + log.Warn($"Windows Installer is busy with another installation (1618); trying again in {timings.BusyRetryDelay.TotalSeconds:0} s"); + await clock.DelayAsync(timings.BusyRetryDelay, CancellationToken.None); + continue; + } + + if (!MsiExitCodes.Succeeded(code) && code != MsiExitCodes.UnknownProduct) + { + return $"Windows Installer ended with {code} ({MsiExitCodes.Describe(code)}); its log is {logPath}."; + } + + log.Info($"Windows Installer ended with {code} ({MsiExitCodes.Describe(code)})"); + if (installer!.RelatedProduct(Branding.AgentMsiUpgradeCode) is { } still) + { + return $"Windows Installer ended with {code}, but {still} is still installed."; + } + + return null; + } + } + + /// + /// Windows Installer's answer, or null when it gave none within the timeout. + /// The three calls run on one background thread of their own; one cut off by + /// the timeout ends with the process. + /// + private async Task UninstallOnceAsync(string product, string logPath) + { + var uninstall = Task.Factory.StartNew(() => + { + installer!.SetSilentUI(); + installer.EnableLog(logPath); + return installer.UninstallProduct(product, UninstallProperties); + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + try + { + return await uninstall.WaitAsync(timings.AttemptTimeout); + } + catch (TimeoutException) + { + return null; + } + } + + private async Task RemovePortableAsync(WatchdogRemoval removal, CancellationToken cancellationToken) + { + if (purge.InstallFolderRefusal() is { } refusal) + { + return refusal; + } + + log.Info($"Removing the services of the install in {options.InstallDirectory}"); + await services.StopWatchdogAsync(cancellationToken); + await services.RemoveAsync(cancellationToken); + log.Info(removal.InstallKind == AgentInstallKinds.Msi + ? "Windows Installer no longer knows the product; its services were removed directly" + : "The agent and watchdog services are removed"); + return null; + } + + /// Whatever the failed attempt stopped runs again, the watchdog first. + private async Task RestoreAsync(CancellationToken cancellationToken) + { + try + { + await services.StartWatchdogAsync(cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + log.Warn("The watchdog service could not be started", exception); + } + + try + { + var status = await agent.QueryAsync(cancellationToken); + if (status.State != AgentRunState.NotInstalled && !status.IsRunning) + { + await agent.StartAsync(cancellationToken); + } + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + log.Warn("The agent could not be started", exception); + } + } + + /// Failed for good: the agent reports it from removal.json; the logs stay where an operator looks. + private void Finish(WatchdogRemoval removal) + { + log.Error($"Removal {removal.RemovalId} gave up after {removal.Attempts} attempts: {removal.LastError}"); + Write(removal with { Phase = WatchdogRemovalPhases.Failed }); + foreach (var file in new[] { Path.Combine(runnerDirectory, LogFileName + ".log"), MsiLogPath(removal.RemovalId) }) + { + try + { + if (File.Exists(file)) + { + Directory.CreateDirectory(options.LogDirectory); + File.Copy(file, Path.Combine(options.LogDirectory, $"agent-removal-{removal.RemovalId:N}-{Path.GetFileName(file)}"), overwrite: true); + } + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + log.Warn($"{file} could not be kept", exception); + } + } + } + + private string MsiLogPath(Guid removalId) => Path.Combine(runnerDirectory, $"agent-msi-uninstall-{removalId:N}.log"); + + private void Write(WatchdogRemoval removal) + => WatchdogFileStore.Write(options.RemovalPath, removal, WatchdogJsonContext.Default.WatchdogRemoval); +} + +/// +/// removal.lock in the runner's own folder, held open with no sharing for +/// as long as it lives: released when the process ends however it ends, so a lock +/// nobody holds is a runner that is gone. +/// +public static class RemovalRunLock +{ + public static FileStream? TryHold(string path) + { + try + { + return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + } + catch (IOException) + { + return null; + } + } + + public static bool IsHeld(string path) + { + if (!File.Exists(path)) + { + return false; + } + + try + { + using var probe = new FileStream(path, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + return false; + } + catch (FileNotFoundException) + { + return false; + } + catch (IOException) + { + return true; + } + catch (UnauthorizedAccessException) + { + return false; + } + } +} diff --git a/src/Agent.Watchdog/Removal/RemovalRunnerLauncher.cs b/src/Agent.Watchdog/Removal/RemovalRunnerLauncher.cs new file mode 100644 index 0000000..9742db3 --- /dev/null +++ b/src/Agent.Watchdog/Removal/RemovalRunnerLauncher.cs @@ -0,0 +1,211 @@ +using System.ComponentModel; +using System.Globalization; +using System.Runtime.Versioning; +using System.Security.AccessControl; +using System.Security.Principal; +using Agent.Watchdog.Msi; +using Agent.Watchdog.Platform; +using Shared.Contracts.Models; +using Shared.Infrastructure.Constants; + +namespace Agent.Watchdog.Removal; + +/// Starts the removal runner apart from the service that asks for it (the runner removes that service); replaced in tests. +public interface IRemovalRunnerLauncher +{ + /// A new folder outside the install and data folders, with a copy of this binary in it; or why not. + (string? RunnerDirectory, string? Problem) Prepare(Guid removalId); + + /// Starts the copy; null when it started, otherwise why not. + string? Start(Guid removalId, string runnerDirectory); +} + +/// The launcher for this platform; shared with the agent, which has it as its fallback when no watchdog can take a removal. +public static class RemovalRunnerLauncher +{ + public static IRemovalRunnerLauncher Create(WatchdogOptions options, WatchdogLog log, string verb) + { + if (OperatingSystem.IsWindows()) + { + return new WindowsRemovalRunnerLauncher(options, log, verb); + } + + return new UnixRemovalRunnerLauncher(options, log, verb); + } + + internal static string[] Arguments(string verb, Guid removalId, WatchdogOptions options, string runnerDirectory) => + [ + verb, removalId.ToString("D"), + "--install-dir", options.InstallDirectory, + "--data-dir", options.DataDirectory, + "--runner-dir", runnerDirectory, + ]; + + internal static string CopySelf(WatchdogOptions options, string runnerDirectory) + { + var self = options.SelfPath is { } path && File.Exists(path) + ? path + : throw new FileNotFoundException("this process has no binary of its own to run from"); + var copy = Path.Combine(runnerDirectory, Path.GetFileName(self)); + File.Copy(self, copy, overwrite: true); + SwapPaths.MakeExecutable(copy); + return copy; + } +} + +/// +/// %SystemRoot%\Temp\ExecuteR-<id>-<random>\: anyone may add to a folder +/// created under that Temp by default, and this one runs as SYSTEM and loads +/// system DLLs from its own folder, so it is created closed (SYSTEM and +/// Administrators only), under a name no one can have prepared, and its owner is +/// checked before the binary goes in. Started detached: stopping both services +/// never takes it down. +/// +[SupportedOSPlatform("windows")] +public sealed class WindowsRemovalRunnerLauncher(WatchdogOptions options, WatchdogLog log, string verb) : IRemovalRunnerLauncher +{ + public (string? RunnerDirectory, string? Problem) Prepare(Guid removalId) + { + var directory = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.Windows), "Temp", + $"{Branding.ProductKey}-{removalId:N}-{Random.Shared.Next():x8}"); + try + { + var folder = new DirectoryInfo(directory); + if (folder.Exists) + { + return (null, $"{directory} already exists"); + } + + folder.Create(ClosedFolder()); + var owner = folder.GetAccessControl().GetOwner(typeof(SecurityIdentifier)); + using var identity = WindowsIdentity.GetCurrent(); + if (owner is not SecurityIdentifier sid + || !(sid.IsWellKnown(WellKnownSidType.LocalSystemSid) || sid.IsWellKnown(WellKnownSidType.BuiltinAdministratorsSid) || sid == identity.User)) + { + return (null, $"{directory} belongs to someone else"); + } + + RemovalRunnerLauncher.CopySelf(options, directory); + return (directory, null); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + return (null, $"its binary could not be copied to {directory} ({exception.Message})"); + } + } + + public string? Start(Guid removalId, string runnerDirectory) + { + var copy = Path.Combine(runnerDirectory, Path.GetFileName(options.SelfPath!)); + try + { + var processId = DetachedProcess.Start(copy, RemovalRunnerLauncher.Arguments(verb, removalId, options, runnerDirectory), runnerDirectory); + log.Info($"Removal runner started (process {processId}) from {copy}"); + return null; + } + catch (Win32Exception exception) + { + return exception.Message; + } + } + + private static DirectorySecurity ClosedFolder() + { + var security = new DirectorySecurity(); + security.SetAccessRuleProtection(isProtected: true, preserveInheritance: false); + foreach (var account in new[] { WellKnownSidType.LocalSystemSid, WellKnownSidType.BuiltinAdministratorsSid }) + { + security.AddAccessRule(new FileSystemAccessRule( + new SecurityIdentifier(account, null), + FileSystemRights.FullControl, + InheritanceFlags.ContainerInherit | InheritanceFlags.ObjectInherit, + PropagationFlags.None, + AccessControlType.Allow)); + } + + return security; + } +} + +/// +/// <parent of data>/executer-removal-<random>/ (mode 700; /var/lib, which +/// unlike a temp folder is not mounted noexec), started through the service +/// manager, since a plain child dies with the service that started it: systemd +/// kills the service's whole cgroup, launchd its process group. Linux: a transient +/// unit (systemd-run). macOS: a one-shot daemon from a plist in that folder. +/// +public sealed class UnixRemovalRunnerLauncher(WatchdogOptions options, WatchdogLog log, string verb) : IRemovalRunnerLauncher +{ + public (string? RunnerDirectory, string? Problem) Prepare(Guid removalId) + { + if (OperatingSystem.IsWindows()) + { + return (null, "not on this platform"); + } + + var parent = Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(options.DataDirectory)); + var directory = Path.Combine(parent ?? "/var/lib", $"executer-removal-{Random.Shared.Next():x8}"); + try + { + Directory.CreateDirectory(directory, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + RemovalRunnerLauncher.CopySelf(options, directory); + return (directory, null); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + return (null, $"its binary could not be copied to {directory} ({exception.Message})"); + } + } + + public string? Start(Guid removalId, string runnerDirectory) + { + var copy = Path.Combine(runnerDirectory, Path.GetFileName(options.SelfPath!)); + var arguments = RemovalRunnerLauncher.Arguments(verb, removalId, options, runnerDirectory); + try + { + var (code, output) = OperatingSystem.IsMacOS() ? StartLaunchd(copy, arguments, removalId, runnerDirectory) : StartSystemd(copy, arguments, removalId); + if (code != 0) + { + return output.Trim(); + } + + log.Info($"Removal runner started from {copy}"); + return null; + } + catch (Exception exception) when (exception is Win32Exception or InvalidOperationException or IOException) + { + return exception.Message; + } + } + + private static (int Code, string Output) StartSystemd(string copy, string[] arguments, Guid removalId) + => Command.Run("systemd-run", [$"--unit={Branding.AgentSystemdUnit}-removal-{removalId:N}", "--collect", copy, .. arguments]); + + private static (int Code, string Output) StartLaunchd(string copy, string[] arguments, Guid removalId, string runnerDirectory) + { + // Booted from outside /Library/LaunchDaemons: gone at the next restart, with nothing to clean up. + var label = string.Create(CultureInfo.InvariantCulture, $"{Branding.AgentLaunchdLabel}.removal.{removalId:N}"); + var strings = string.Concat(new[] { copy }.Concat(arguments).Select(a => $"{System.Security.SecurityElement.Escape(a)}")); + var plist = Path.Combine(runnerDirectory, label + ".plist"); + File.WriteAllText(plist, $""" + + + + + Label{label} + ProgramArguments + {strings} + RunAtLoad + AbandonProcessGroup + + + """); + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode(plist, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead); + } + + return Command.Run("launchctl", "bootstrap", "system", plist); + } +} diff --git a/src/Agent.Watchdog/UpgradeApplier.cs b/src/Agent.Watchdog/UpgradeApplier.cs index c501f13..b04503b 100644 --- a/src/Agent.Watchdog/UpgradeApplier.cs +++ b/src/Agent.Watchdog/UpgradeApplier.cs @@ -309,6 +309,19 @@ public sealed class UpgradeApplier( } var handover = WatchdogFileStore.Read(options.HandoverPath, WatchdogJsonContext.Default.WatchdogHandover); + if (handover is not null && File.Exists(options.RemovalPath)) + { + // The device is being removed: an upgrade that crossed it never starts. + log.Warn($"Upgrade {handover.UpgradeId} is set aside: the agent is being removed"); + WatchdogFileStore.TryDelete(options.HandoverPath); + WatchdogFileStore.Write(options.OutcomePath, + new WatchdogOutcome(handover.UpgradeId, AgentUpgradeStates.Failed, AgentUpgradeReasons.Busy, handover.Version, handover.FromVersion, [], + clock.UtcNow, "The agent is being removed from this device; nothing was changed."), + WatchdogJsonContext.Default.WatchdogOutcome); + finished.Add(handover.UpgradeId); + return null; + } + if (handover is null) { var id = WatchdogFileStore.ReadUpgradeId(options.HandoverPath); diff --git a/src/Agent.Watchdog/WatchdogCli.cs b/src/Agent.Watchdog/WatchdogCli.cs index c57c829..0999449 100644 --- a/src/Agent.Watchdog/WatchdogCli.cs +++ b/src/Agent.Watchdog/WatchdogCli.cs @@ -60,6 +60,9 @@ public static class WatchdogCli && Guid.TryParse(args[1], out var upgradeId): return await RunMsiAsync(upgradeId, args[2..]); + case WatchdogRemovalFiles.RunVerb when args.Length > 1 && Guid.TryParse(args[1], out var removalId): + return await Removal.RemovalHost.RunAsync(removalId, args[2..]); + case null or "help" or "-h" or "--help" or "/?": PrintUsage(); return command is null ? 1 : 0; @@ -140,7 +143,9 @@ public static class WatchdogCli agent, clock, log); var applier = new UpgradeApplier(options, agent, clock, log, new ProcessSmokeTest(), restart, msi); var supervisor = new AgentSupervisor(options, agent, clock, log); - var service = new WatchdogService(options, applier, supervisor, clock, log); + var removal = new Removal.RemovalCoordinator(options, + processMode ? null : Removal.RemovalRunnerLauncher.Create(options, log, WatchdogRemovalFiles.RunVerb), agent, clock, log); + var service = new WatchdogService(options, applier, supervisor, clock, log, removal); if (processMode) { diff --git a/src/Agent.Watchdog/WatchdogOptions.cs b/src/Agent.Watchdog/WatchdogOptions.cs index 9a02f8e..893f543 100644 --- a/src/Agent.Watchdog/WatchdogOptions.cs +++ b/src/Agent.Watchdog/WatchdogOptions.cs @@ -63,6 +63,9 @@ public sealed record WatchdogOptions public string MsiRunnerLockPath => Path.Combine(UpgradeDirectory, WatchdogMsiFiles.RunnerLock); + /// A removal the agent handed over: while it is there no new upgrade starts, and until it failed the agent is not supervised. + public string RemovalPath => Path.Combine(UpgradeDirectory, WatchdogFiles.Removal); + /// Windows Installer's verbose log of one MSI run, beside the agent's own logs. public string MsiLogPath(string version, Guid upgradeId) => Path.Combine(LogDirectory, $"agent-msi-{version.Split('+')[0]}-{upgradeId:N}.log"); diff --git a/src/Agent.Watchdog/WatchdogService.cs b/src/Agent.Watchdog/WatchdogService.cs index 051d5ad..226d742 100644 --- a/src/Agent.Watchdog/WatchdogService.cs +++ b/src/Agent.Watchdog/WatchdogService.cs @@ -5,7 +5,13 @@ namespace Agent.Watchdog; /// left half done and tidies the install folder; then, every poll, it carries an /// upgrade on when one is waiting and otherwise supervises the agent. /// -public sealed class WatchdogService(WatchdogOptions options, UpgradeApplier applier, AgentSupervisor supervisor, IWatchdogClock clock, WatchdogLog log) +public sealed class WatchdogService( + WatchdogOptions options, + UpgradeApplier applier, + AgentSupervisor supervisor, + IWatchdogClock clock, + WatchdogLog log, + Removal.RemovalCoordinator? removal = null) { /// Runs until cancelled, or until the watchdog replaced itself (true: exit so the new binary takes over). public async Task RunAsync(CancellationToken cancellationToken) @@ -29,6 +35,12 @@ public sealed class WatchdogService(WatchdogOptions options, UpgradeApplier appl { try { + if (removal is not null && await removal.PassAsync(cancellationToken)) + { + await clock.DelayAsync(options.PollInterval, cancellationToken); + continue; + } + switch (await applier.PassAsync(cancellationToken)) { case PassResult.RestartRequested: diff --git a/src/Agent/Agent.csproj b/src/Agent/Agent.csproj index 5fa4593..40ea1f9 100644 --- a/src/Agent/Agent.csproj +++ b/src/Agent/Agent.csproj @@ -60,6 +60,22 @@ + + + + + + + + + + + + +