React console, StifleR sign-in, runs with bulk logs, Windows PE, shell choices and agent auto-upgrade #18

Merged
gsadmin merged 72 commits from dev into main 2026-10-23 23:31:56 +00:00
549 changed files with 76223 additions and 16787 deletions
+24 -21
View File
@@ -1,5 +1,5 @@
# End-to-End MVP Specification
## .NET Agent + NATS over WebSocket + ASP.NET Core Backend + SvelteKit Web UI
## .NET Agent + NATS over WebSocket + ASP.NET Core Backend + React (MUI, Mantis Pro) Web UI
**Document purpose:** Build-ready MVP specification for Claude or another coding agent.
@@ -23,7 +23,7 @@ Primary technologies:
- NATS JetStream
- System.Management.Automation SDK for PowerShell execution
- Windows CIM/WMI APIs for stable hardware identity and inventory
- SvelteKit
- React 19 + MUI 9 on the Mantis Pro template (Vite, TypeScript, SWR, React Router)
- TypeScript
- Monaco Editor for browser script editing
- xterm.js for browser terminal rendering
@@ -99,18 +99,21 @@ Use the following repository structure exactly unless a platform limitation requ
│ │ └── Security/
│ ├── WebUI/
│ │ ├── package.json
│ │ ├── svelte.config.js
│ │ ├── vite.config.ts
│ │ ├── vite.config.mts
│ │ ├── server.mjs
│ │ ├── src/
│ │ │ ├── lib/
│ │ │ │ ├── api/
│ │ │ │ ├── auth/
│ │ │ │ ├── components/
│ │ │ │ ├── messaging/
│ │ │ │ ├── stores/
│ │ │ │ └── theme/
│ │ │ └── routes/
│ │ └── static/
│ │ │ ├── api/
│ │ │ ├── components/
│ │ │ ├── contexts/
│ │ │ ├── hooks/
│ │ │ ├── layout/
│ │ │ ├── messaging/
│ │ │ ├── pages/
│ │ │ ├── routes/
│ │ │ ├── sections/
│ │ │ ├── themes/
│ │ │ └── utils/
│ │ └── public/
│ ├── Shared.Contracts/
│ │ ├── Shared.Contracts.csproj
│ │ ├── Api/
@@ -1174,7 +1177,7 @@ PowerShell ProgressRecord
-> NATS
-> Backend
-> browser-authorized NATS subject
-> Svelte store
-> console NATS client (React subscription)
-> live progress bar
```
@@ -1577,7 +1580,7 @@ Agent only acknowledges script command after it has durably recorded the job ID
## 32.1 General Design
SvelteKit UI should look polished rather than like an admin scaffold.
The React console (MUI 9 on the Mantis Pro shell, styled like the house consoles: slate surfaces, blue accent, page headers, side sheets, a group tree beside grids, menus as data with disabled-with-reason entries) should look polished rather than like an admin scaffold.
Required visual characteristics:
@@ -2029,9 +2032,9 @@ NATS_ACCOUNT_SIGNING_SEED=
# 45. WebUI Docker Image
Build SvelteKit using Node during build stage.
Build the console with Vite (`npm run build`, typecheck first) in a Node 24 build stage; the output is a static SPA in `dist/`.
Use SvelteKit Node adapter or a minimal production Node runtime.
Serve `dist/` from a minimal Node runtime with the dependency-free `server.mjs` (SPA fallback, immutable asset caching). The combined server binary embeds the same `dist/`.
Container listens on:
@@ -2238,7 +2241,7 @@ Behavior:
- build
- unit tests
- integration tests where feasible
- Svelte check/build
- WebUI check (typecheck + lint) and build
- no image push
- no release
@@ -2458,7 +2461,7 @@ The MVP is not complete until this sequence works reliably.
- JetStream durable script dispatch
## Browser
- SvelteKit
- React + MUI console (Mantis Pro shell)
- System/Light/Dark
- polished dashboard
- device list live status
@@ -2667,11 +2670,11 @@ Include Mermaid architecture diagrams.
flowchart LR
A[Windows Agent] -->|HTTPS bootstrap| P[Reverse Proxy :443]
A -->|WSS NATS| P
B[Browser / SvelteKit] -->|HTTPS REST| P
B[Browser / React console] -->|HTTPS REST| P
B -->|WSS NATS| P
P -->|/api| API[ASP.NET Core Backend]
P -->|/| UI[SvelteKit WebUI]
P -->|/| UI[React WebUI]
P -->|/messaging/agent| N[NATS WebSocket]
P -->|/messaging/browser| N
+1 -1
View File
@@ -28,7 +28,7 @@ csharp_style_prefer_primary_constructors = false:none
[*.{json,yml,yaml}]
indent_size = 2
[*.{js,ts,svelte,css,html}]
[*.{js,mjs,ts,tsx,css,html}]
indent_size = 2
[*.{ps1,psm1}]
+10 -5
View File
@@ -103,14 +103,19 @@ TLS_CERT_DIR=
# pruned). 0 disables pruning.
JOB_HISTORY_LIMIT=200
# ── Initial local admin (first startup only) ───────────────────────────────
# ── Break-glass account (first startup only) ───────────────────────────────
# The one recovery built-in account. Left empty, a one-time password is
# generated and written once to the server log. Recover with `auth-reset`.
INITIAL_ADMIN_USERNAME=admin
# Generate: openssl rand -base64 24
INITIAL_ADMIN_PASSWORD=
# ── OIDC (optional; also configurable from Settings → Authentication) ──────
# Register the redirect URI <public-url>/api/auth/oidc/callback at your provider.
# The path is fixed by design — the Settings page shows the exact value.
# ── Sign-in ────────────────────────────────────────────────────────────────
# Sign-in methods (Built-In, Local Windows Accounts, LDAP, OIDC) are set in
# Settings › Authentication. The OIDC_* and AUTH_LOGIN_MODE values below are
# read once, to carry an older install's settings over when no sign-in
# settings exist yet; after that the console is the only place to change them.
# The OIDC callback to register is <public-url>/api/auth/oidc/callback.
OIDC_ENABLED=0
OIDC_AUTHORITY=
OIDC_CLIENT_ID=
@@ -127,7 +132,7 @@ OIDC_DEFAULT_ROLE=Operator
OIDC_REQUIRE_MAPPED_ROLE=0
# LocalOnly | Both | OidcOnly | OidcAutoRedirect
AUTH_LOGIN_MODE=Both
# Break-glass: 1 bypasses OIDC entirely regardless of saved settings.
# Always offer the built-in form, whatever the saved settings say (a way back in).
AUTH_FORCE_LOCAL=0
# ── NATS decentralized auth (generated — do NOT hand-edit) ─────────────────
+1
View File
@@ -89,6 +89,7 @@ jobs:
run: |
dotnet test tests/Shared.Tests -c Release --no-build --nologo
dotnet test tests/Agent.Tests -c Release --no-build --nologo
dotnet test tests/Agent.Watchdog.Tests -c Release --no-build --nologo
dotnet test tests/Backend.Tests -c Release --no-build --nologo
- name: Integration tests
+2 -4
View File
@@ -12,10 +12,9 @@ TestResults/
*.user
*.suo
# Node / SvelteKit
# Node / WebUI (Vite)
node_modules/
.svelte-kit/
src/WebUI/build/
src/WebUI/dist/
npm-debug.log*
# Local state
@@ -43,4 +42,3 @@ Thumbs.db
src/Infra/backend/agent-package/*
!src/Infra/backend/agent-package/.gitkeep
src/WebUI/build-static/
+1
View File
@@ -17,6 +17,7 @@
<InformationalVersion Condition="'$(GitCommitShort)' == ''">$(UnifiedVersion)</InformationalVersion>
<IncludeSourceRevisionInInformationalVersion>false</IncludeSourceRevisionInInformationalVersion>
<Company>ExecuteR</Company>
<Copyright>Copyright (c) 2026 Grace Solutions. All rights reserved.</Copyright>
</PropertyGroup>
<PropertyGroup>
+4 -2
View File
@@ -6,7 +6,8 @@
<ItemGroup>
<!-- Transitive pins for known-vulnerability fixes (NU1903). -->
<PackageVersion Include="SQLitePCLRaw.lib.e_sqlite3" Version="2.1.12" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.10" />
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.13" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.13" />
</ItemGroup>
<ItemGroup>
@@ -38,7 +39,8 @@
<PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.10" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.10" />
<PackageVersion Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="10.0.10" />
<PackageVersion Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" Version="8.22.0" />
<PackageVersion Include="System.DirectoryServices.Protocols" Version="10.0.12" />
<!-- Agent -->
<PackageVersion Include="Microsoft.PowerShell.SDK" Version="7.6.4" />
+2
View File
@@ -1,12 +1,14 @@
<Solution>
<Folder Name="/src/">
<Project Path="src/Agent/Agent.csproj" />
<Project Path="src/Agent.Watchdog/Agent.Watchdog.csproj" />
<Project Path="src/Backend/Backend.csproj" />
<Project Path="src/Shared.Contracts/Shared.Contracts.csproj" />
<Project Path="src/Shared.Infrastructure/Shared.Infrastructure.csproj" />
</Folder>
<Folder Name="/tests/">
<Project Path="tests/Agent.Tests/Agent.Tests.csproj" />
<Project Path="tests/Agent.Watchdog.Tests/Agent.Watchdog.Tests.csproj" />
<Project Path="tests/Backend.Tests/Backend.Tests.csproj" />
<Project Path="tests/Integration.Tests/Integration.Tests.csproj" />
<Project Path="tests/Shared.Tests/Shared.Tests.csproj" />
+1 -1
View File
@@ -1,6 +1,6 @@
#=============================================================================
# ExecuteR - Windows endpoint management control plane
# Agent + NATS-over-WebSocket + ASP.NET Core backend + SvelteKit web UI
# Agent + NATS-over-WebSocket + ASP.NET Core backend + React (MUI) web console
#=============================================================================
#
# CREDENTIALS:
+12 -231
View File
@@ -1,235 +1,16 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (c) 2026 Grace Solutions. All rights reserved.
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
This software and its source code, documentation and associated files (the "Software") are proprietary to Grace
Solutions and are provided under licence only.
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
No part of the Software may be copied, modified, distributed, sublicensed, sold or made available to any third
party, in source or binary form, except as permitted by a written licence agreement with Grace Solutions.
Preamble
The Software includes third-party components that remain under their own licences; see THIRD-PARTY-NOTICES.md.
The web console is built on a commercial user-interface template licensed to Grace Solutions for use in this product
only; its source may not be extracted or redistributed on its own.
The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.
A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.
The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.
An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based on the Program.
To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.
A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
ExecuteR
Copyright (C) 2026 gsadmin
This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see <http://www.gnu.org/licenses/>.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL GRACE SOLUTIONS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+572 -114
View File
@@ -16,7 +16,7 @@ updates with **no SignalR, SSE, or polling**.
- **Backend** — ASP.NET Core control plane: REST API, SQLite, local + OIDC
auth, RBAC, NATS operator/JWT issuance, JetStream command dispatch, live
event relay.
- **WebUI** — SvelteKit dashboard: live device status, Monaco PowerShell editor
- **WebUI** — React + MUI console (Mantis Pro shell): live device status, a nested group tree, run now and schedules, Monaco PowerShell editor
with typed parameters and reusable Functions, live job output and
progress, xterm.js remote shell.
@@ -38,23 +38,27 @@ updates with **no SignalR, SSE, or polling**.
10. [Networking and TLS](#networking-and-tls)
11. [Reverse proxy routes](#reverse-proxy-routes)
12. [Agent foreground mode and service lifecycle](#agent-foreground-mode-and-service-lifecycle)
13. [MSI public properties](#msi-public-properties)
14. [Registry paths and GPO precedence](#registry-paths-and-gpo-precedence)
15. [Bootstrap and URL install](#bootstrap-and-url-install)
16. [Local login and OIDC](#local-login-and-oidc)
17. [Custom roles and permissions](#custom-roles-and-permissions)
18. [Typed script parameters and Functions](#typed-script-parameters-and-functions)
19. [Script engines](#script-engines)
20. [Secrets providers and $Secrets](#secrets-providers-and-secrets)
21. [Device groups and script triggers](#device-groups-and-script-triggers)
22. [Group runs and schedules](#group-runs-and-schedules)
23. [Database providers](#database-providers)
24. [Logging](#logging)
25. [Device lifecycle: reconnect or orphan](#device-lifecycle-reconnect-or-orphan)
26. [S3 file delivery](#s3-file-delivery)
27. [CI/CD and GITEA_* variables](#cicd-and-gitea_-variables)
28. [Versioning](#versioning)
29. [Troubleshooting](#troubleshooting)
13. [Agent upgrades and the watchdog](#agent-upgrades-and-the-watchdog)
14. [MSI public properties](#msi-public-properties)
15. [Registry paths and GPO precedence](#registry-paths-and-gpo-precedence)
16. [Bootstrap and URL install](#bootstrap-and-url-install)
17. [Windows PE](#windows-pe)
18. [Sign-in](#sign-in)
19. [Custom roles and permissions](#custom-roles-and-permissions)
20. [Typed script parameters and Functions](#typed-script-parameters-and-functions)
21. [Script engines](#script-engines)
22. [Secrets providers and $Secrets](#secrets-providers-and-secrets)
23. [Device groups and script triggers](#device-groups-and-script-triggers)
24. [Group runs and schedules](#group-runs-and-schedules)
25. [Database providers](#database-providers)
26. [Logging](#logging)
27. [Device lifecycle: reconnect or orphan](#device-lifecycle-reconnect-or-orphan)
28. [Agent upgrades from the server](#agent-upgrades-from-the-server)
29. [S3 file delivery](#s3-file-delivery)
30. [CI/CD and GITEA_* variables](#cicd-and-gitea_-variables)
31. [Versioning](#versioning)
32. [Troubleshooting](#troubleshooting)
33. [Licence](#licence)
---
@@ -67,7 +71,7 @@ WebSocket to the reverse proxy, which forwards to the NATS WebSocket listener.
```mermaid
flowchart LR
A["Windows Agent"]
B["Browser / SvelteKit"]
B["Browser / React console"]
P["Reverse Proxy :443"]
A -->|"HTTPS: bootstrap, install"| P
@@ -76,7 +80,7 @@ flowchart LR
B <-->|"WSS: NATS session"| P
subgraph Stack ["Docker stack (private network)"]
UI["SvelteKit WebUI"]
UI["React WebUI"]
API["ASP.NET Core Backend"]
N["NATS: WebSocket + TCP"]
DB[("SQLite / Postgres / SQL Server")]
@@ -127,7 +131,7 @@ How a request or event moves through the internal components.
```mermaid
flowchart TB
subgraph Browser
UI["SvelteKit pages"]
UI["React pages"]
BN["Browser NATS client (WSS)"]
end
@@ -204,7 +208,7 @@ Full script-execution, enrollment, and remote-shell sequence diagrams are in
## Prerequisites
- **.NET SDK 10** (`dotnet --version` gives 10.x)
- **Node.js LTS** (20+) and npm — for the WebUI
- **Node.js LTS** (22+) and npm — for the WebUI
- **Docker + Docker Compose** — to run the stack
- **Windows** — required to build/run the agent (CIM, ConPTY, DPAPI, Windows
service). The backend and WebUI are cross-platform.
@@ -224,7 +228,7 @@ Full script-execution, enrollment, and remote-shell sequence diagrams are in
# Run tests
./scripts/Test.ps1 # unit
./scripts/Test.ps1 -IncludeWebUI # + svelte-check
./scripts/Test.ps1 -IncludeWebUI # + WebUI typecheck and lint
```
Run the stack:
@@ -555,6 +559,149 @@ Remote shell works on every platform — ConPTY on Windows, a forkpty-backed
PTY (with a real controlling terminal, so job control and Ctrl-C behave) on
Linux and macOS.
On Windows devices the shell page's **Shell** picker chooses what starts:
**Automatic** (the default) takes the first of PowerShell 7 (`pwsh.exe`,
Program Files or `PATH`), Windows PowerShell, the agent's **Built-In
PowerShell** and Command Prompt (`%ComSpec%`); an explicit choice that is not
installed fails with a clear message instead of falling back. The last
choice is remembered per browser, and the shell page shows which shell
Automatic started. Built-In PowerShell is the agent's own embedded engine in
the SDK console host (the hidden `ExecuteR-Agent.exe shell-host` verb, with
the bundled modules on `PSModulePath`), so a PowerShell prompt is available
even where none is installed — Windows PE without the WinPE-PowerShell
component, for example. Linux and macOS agents ignore the choice and start
the login shell (`$SHELL`, else bash/zsh/sh).
In the console, **Pop Out** moves a shell into its own browser window
without interrupting it: the session, scrollback and anything running carry
over, and it keeps running while you use the rest of the console. **Return
to Console** (or **Bring Back** on the shell page) docks the same terminal
back. Closing the window brings it back when the console is showing that
device's shell and ends the session otherwise; reloading or closing the
console tab ends every popped-out shell, since they run on its page.
---
## Agent upgrades and the watchdog
Every agent package (zip and MSI) carries a second, small binary beside the
agent: the **watchdog** (`ExecuteR-Watchdog.exe`, `executer-watchdog` on
Linux/macOS; self-contained, trimmed, about 12 MB). It runs as its own service,
keeps the agent running and applies the upgrades the server sends (see
[Agent upgrades from the server](#agent-upgrades-from-the-server) for starting,
pacing and rolling them back).
| | Windows | Linux | macOS |
|---|---|---|---|
| Watchdog service | `ExecuteRWatchdog` (LocalSystem, Automatic, restart on failure) | systemd `executer-watchdog.service` | launchd `info.executer.watchdog` |
| Install folder | `%ProgramFiles%\ExecuteR\Agent` | `/opt/executer/agent` | `/opt/executer/agent` |
| Upgrade files | `%ProgramData%\ExecuteR\upgrade` | `/var/lib/executer/upgrade` | `/var/lib/executer/upgrade` |
| Watchdog log | `<install>\Logs\ExecuteR-Watchdog.log` | `<install>/Logs/executer-watchdog.log` | same |
The MSI installs and starts both services. `ExecuteR-Agent.exe install` /
`initialize` also register (and start) the watchdog when its binary sits next
to the agent, and `remove` takes the watchdog out first, so it cannot start the
agent again meanwhile. The watchdog has the same service verbs itself
(`install`, `remove`, `start`, `stop`, `status`, `version`, `foreground`). An
install from before the watchdog existed gets it with its first upgrade: the
agent copies it out of the downloaded package and registers it before handing
the package over.
### How an upgrade runs on the device
1. The agent takes the `command.agent.upgrade` command. It answers at once,
without downloading, when it cannot or need not upgrade: **unsupported**
(not running as the installed service, or the package is for another
platform), **busy** (another hand-over, swap or unreported outcome is
pending), or **succeeded / already** (it runs that version). A redelivered
command (same upgrade ID) is ignored.
2. It waits a random start jitter (0 to the seconds the server sends), then
downloads the package (a server path resolved against the configured server
URL, or a presigned URL as is; the configured proxy and TLS trust apply) to
`upgrade/downloads/<sha256>.zip`, reporting **downloading** with progress.
Size and SHA-256 must match the command (else **failed / hash**). It unpacks
to `upgrade/staging/<version>-<id>/` (no agent binary inside: **failed /
package**), makes sure the watchdog runs, writes `upgrade/handover.json` and
reports **installing**. A hand-over the watchdog has not taken within 10
minutes fails as **watchdog**.
3. The watchdog checks the staging again (location, safe relative paths, every
file's size and SHA-256), stops the agent service, copies the files into
`<install>/.next/<id>/` and checks them there, moves the files they replace
into `<install>/.previous/<id>/` and the new ones into place, and starts the
agent. A move is retried only while another process holds the file (10 × 500
ms). Failures before any file moved leave the agent running and are tried
again, three times in all, then end as **failed / watchdog**.
4. The new version is **kept** only when its health beacon reports the new
version, **connected** (command consumer bound and NATS open), written after
the start, within the health deadline the server sends (10 minutes by
default). Then `.next` and `.previous` go and the outcome is **succeeded**.
5. Otherwise it is **rolled back**: the new files go to `<install>/.failed/<id>/`
(and are then deleted), `.previous` comes back, the previous agent is started
and given 5 minutes to report healthy. The reason is **health-deadline**,
**crashed** (the new version exited three times, which ends the wait early)
or **swap** (a file could not be moved), with up to 40 log lines: the new
agent's own for a health or crash failure, the watchdog's for a swap.
6. The outcome is written to `upgrade/outcome.json` and reported by whichever
agent version runs afterwards as soon as it is connected; then the file is
deleted, and old downloads and staging folders are removed (the newest is
kept).
What a swap touches: the files the package brings, and every file inside a
top-level folder it ships (`Modules\`, so a module the new version dropped goes
too). `Logs\`, an operator's own files and the running watchdog binary are
never moved. The watchdog replaces **its own binary last**, after the new agent
is kept, and only when the package's watchdog is newer and runs (`version`
exits 0 within 30 s); it then restarts onto it through the service manager.
### What you see on disk
| Path | Meaning |
|---|---|
| `upgrade/health.json` | the agent's beacon (version, PID, connected, time), every 5 s; written only by the installed service |
| `upgrade/handover.json` | a package waiting for the watchdog |
| `upgrade/handover.taken.json` + `upgrade/journal.json` | a swap under way; the journal names the phase (`stopping`, `swapping`, `verifying`, `committed`, `rolling-back`, `rolled-back`) and the SHA-256 of every file on both sides |
| `upgrade/outcome.json` | how the last swap ended, until the agent has reported it |
| `upgrade/downloads/`, `upgrade/staging/` | the newest package, zipped and unpacked |
| `<install>/.next`, `.previous`, `.failed` | exist only during a swap |
If the watchdog itself stops part way (power loss, a kill), its next start
finishes or undoes the run from the journal: `stopping` starts over (the agent
runs meanwhile), `swapping` and `rolling-back` put the previous version back,
`verifying` goes on waiting for the new version (at least another minute when
the deadline has passed) and keeps or rolls it back. An upgrade that has a
journal or an outcome is never applied a second time.
### When the watchdog restarts the agent
The watchdog never fights an admin. It acts only while the agent service is set
to start **automatically**, and then only in two cases, at most once every 3
minutes:
- the agent runs, has run for over 3 minutes, and its beacon is older than 3
minutes: it hangs, so it is restarted (an agent that writes no beacon at all,
an older version, is left alone);
- the agent **stopped with a failure** and is still stopped 3 minutes later
(the service manager's own restarts gave up): it is started.
A clean stop is never undone. To stop the agent without the watchdog starting
it again:
```powershell
ExecuteR-Agent.exe stop # stays stopped until started or the next boot
Set-Service ExecuteRAgent -StartupType Manual # also across reboots: the watchdog leaves a non-automatic service alone
ExecuteR-Watchdog.exe stop # or stop the watchdog too
```
On Linux `systemctl stop executer-agent` (and `systemctl disable` for reboots).
On macOS the agent's launchd `KeepAlive` restarts a killed agent by itself; use
`launchctl bootout system/info.executer.agent` to keep it stopped.
For development, `ExecuteR-Watchdog.exe foreground --agent process
--install-dir <dir> --data-dir <dir>` runs the same loop with console logging
against an agent it starts as a child process instead of a service (an agent
in the foreground refuses upgrade commands itself, so drive it with a
hand-made `handover.json`).
---
## MSI public properties
@@ -643,74 +790,189 @@ connects over WSS. The device appears in the UI within seconds.
---
## Local login and OIDC
## Windows PE
Local auth uses ASP.NET Core `PasswordHasher` (no compiled default password).
The initial admin is created on first boot from `INITIAL_ADMIN_USERNAME` /
`INITIAL_ADMIN_PASSWORD`. Sessions are HTTP-only cookies with double-submit CSRF
protection on state-changing calls; login is rate limited.
The agent runs in Windows PE, so a device is reachable before it is imaged —
and it stays the **same device** afterwards. Its identity is derived from the
hardware (SMBIOS UUID, BIOS/baseboard serials, make and model), never from the
OS, so the installed OS enrolls into the same device row. In WinPE the OS reads
**Windows PE** (with the real build version, shown as a **WinPE** chip) and the
hostname is WinPE's random `MINWINPC-…`; when the installed OS enrolls, both
update and the transition is kept in the device's **Device History**.
Generic OIDC (any compliant provider — Authentik, Keycloak, Entra ID, Okta) is
configurable from **Settings → Authentication** (saved values override the
environment and apply without a restart) or bootstrapped via environment:
**Boot image prerequisites** (optional components added with `Dism /Add-Package`):
| Component | Why |
|-----------|-----|
| `WinPE-WMI` | **Required.** Hardware identity and inventory come from WMI; without it the agent logs `WMI is unavailable … add the WinPE-WMI optional component` and cannot enroll. |
| `WinPE-PowerShell` (brings `WinPE-NetFx` and `WinPE-Scripting`) | The PowerShell install one-liner, and the `powershell.exe` script engine. |
| `WinPE-NetFx` alone | Not needed — the agent is self-contained .NET. |
Scripts from the console run in the agent's embedded PowerShell either way. In
the remote shell, **Built-In PowerShell** (the agent's own PowerShell host)
gives a PowerShell prompt without `WinPE-PowerShell`, and **Command Prompt**
(`cmd.exe`) is always there; **Automatic** picks the first of PowerShell 7,
Windows PowerShell, Built-In PowerShell and Command Prompt that the device has.
WinPE runs from a RAM drive (`X:`) whose default scratch space (32 MB) is too
small for the agent — raise it on the image, and keep the default `wpeinit`
so networking is up:
```text
OIDC_ENABLED=1
OIDC_AUTHORITY=https://idp.example.com/application/o/executer/
OIDC_CLIENT_ID=executer
OIDC_CLIENT_SECRET=<secret>
OIDC_SCOPES=openid profile email
OIDC_ROLE_CLAIM=groups
OIDC_ADMIN_ROLE=executer-admins
OIDC_OPERATOR_ROLE=executer-operators
OIDC_VIEWER_ROLE=executer-viewers
AUTH_LOGIN_MODE=Both # LocalOnly | Both | OidcOnly | OidcAutoRedirect
Dism /Image:C:\WinPE_amd64\mount /Set-ScratchSpace:512
```
Register the redirect URI `<public-url>/api/auth/oidc/callback` at the
provider — the path is fixed by design and shown (copyable) in the Settings UI.
Displayed URLs and generated install commands use `PUBLIC_SERVER_URL` when
set; otherwise they derive from **trusted forwarded headers**, so they are
correct behind the reverse proxy without any configuration.
**One-liner.** The Windows command from **Add device** works as-is in WinPE
(paste it at the WinPE command prompt; it starts `powershell.exe`). The script
detects WinPE (`HKLM\SYSTEM\CurrentControlSet\Control\MiniNT`), takes the
portable ZIP instead of the MSI (Windows Installer does not run in WinPE),
unpacks to `X:\Program Files\ExecuteR\Agent` with .NET's ZipFile (no
`Expand-Archive`/CIM cmdlets needed) and registers and starts the Windows
service through `ExecuteR-Agent.exe initialize` — the Service Control Manager
APIs, not `sc.exe`. Everything lives on `X:` and is gone at the next reboot,
by design.
Role mapping: custom roles list their own IdP claim values (see
[Custom roles and permissions](#custom-roles-and-permissions)); the legacy
Admin → Operator → Viewer mapping above still resolves onto the seeded system
roles. Unmatched users get the configured default role, or are refused when
*Require mapped role* is on.
**`startnet.cmd` alternative** — no PowerShell needed: copy the portable ZIP's
contents into the mounted image (e.g. `<mount>\ExecuteR`) and run the agent in
the foreground from `Windows\System32\startnet.cmd`. Environment variables are
honored in `foreground` only (a service reads the registry):
**Lockout recovery** (an IdP outage is never terminal):
```bat
wpeinit
set AGENT_SERVER_URL=https://control.example.com
set AGENT_BOOTSTRAP_TOKEN=<token>
start "ExecuteR Agent" X:\ExecuteR\ExecuteR-Agent.exe foreground
```
- `/login?local=1` always reveals the local sign-in form, even in SSO-only modes.
- `docker exec EXECUTER-API-00001 dotnet Backend.dll auth-reset --admin admin --password '<new>'`
disables OIDC in the database and restores/creates a local admin.
- `AUTH_FORCE_LOCAL=1` (environment only) bypasses stored OIDC settings at startup.
To run it as a service there instead, use
`X:\ExecuteR\ExecuteR-Agent.exe configure --url <server> --token <token>` followed
by `X:\ExecuteR\ExecuteR-Agent.exe initialize`.
![Login](docs/screenshots/login.png)
**The token after imaging.** The installed OS enrolls through its own install
(the MSI or one-liner in the task sequence / `SetupComplete.cmd`). Re-enrolling
the same device with the token that enrolled it spends no further use, so one
token can cover both phases — but it must still be valid (not expired or
revoked) when the installed OS enrolls. Give it an expiry longer than imaging
takes, and use a multi-use token for a batch of devices.
**History.** The device page's **Device History** tab lists **Enrolled**
(with the name and OS it enrolled as), **Renamed** (`MINWINPC-ABC → LAB-PC-01`)
and **OS Changed** (`Windows PE 10.0.26100 → Windows 11 10.0.26100.4351`),
newest first; the same trail is `GET /api/devices/{clientId}/history`
(`devices.view`, device scope applies). Events are written only when a value
actually changes; each enrollment's audit entry also records hostname and OS.
**On-enrollment triggers** fire once per device per trigger — so one aimed at
all devices fires at the WinPE enrollment and not again after imaging. Aim it
at a dynamic group with **OS name** *not equals* `Windows PE` to run it in the
installed OS (enrolling a new OS drops the WinPE inventory, so the group sees
the new OS straight away), or *equals* `Windows PE` for a pre-imaging step.
---
## Sign-in
Sign-in works as it does in StifleR. There are four methods, each a tab under
**Settings › Authentication**, and several can be on at once:
| Method | What it checks | Groups that bind to roles |
|---|---|---|
| **Built-In** | ExecuteR's own accounts (PBKDF2-SHA512, at least 12 characters, changed at first sign-in) | Built-in groups |
| **Local Windows Accounts** | This server's own Windows accounts, checked by Windows (`LogonUser`); Windows servers only, on by default on Windows Server | The account's SID and its local groups' SIDs (for example `BUILTIN\Administrators`) |
| **LDAP** | A directory or Active Directory: LDAPS, StartTLS, or Kerberos (signed and sealed); user and group filters, nested groups | Group DNs |
| **OIDC** | Any OpenID Connect provider (authorization code with PKCE); ID token validated; sign-out at the provider | `claim=value`, for example `groups=executer-admins` |
The sign-in page offers the built-in form (one form for built-in and local
Windows accounts, with an **Account Type** toggle when the directory is on
too) and a **Sign In with {provider}** button. With **Redirect Mode** on and
OIDC the only way in, it goes straight to the provider, except after an error
or a sign-out.
**Access comes from groups, never from the person.** A role binding names a
group (built-in group, Windows SID, directory DN or OIDC claim) and one of
ExecuteR's roles. Someone in several bound groups gets all their roles:
permissions add up, and so do device scopes (a role with no scope reaches
every device). A person whose groups are bound to nothing is refused at
sign-in. Each method's tab has its own bindings card.
**Sessions** are database rows behind an HTTP-only `__Host-session` cookie.
They end on sign-out at once, on every node. They slide with use up to an
absolute limit (Built-In tab: idle 120 minutes, at most 12 hours by default),
and carry a CSRF token checked on every write.
**Safety rails:**
- **Failed sign-ins** back off per account and per address (from 30 seconds,
doubling to 15 minutes). Nothing is locked for good.
- **Turning a method on or off** asks you to confirm it is you (step-up: your
password, or your provider again).
- **The last method on** cannot be turned off.
- **Leaving only LDAP, OIDC or Windows on** needs a passed **Test** sign-in
through it in this session, with its settings as they are.
- **Secrets** (the LDAP service password, the OIDC client secret) are
write-only and encrypted at rest.
- **On a domain-joined Windows server**, LDAP sets itself up from the join
(Kerberos, the computer account) until you change its settings. A breaker
turns it off after repeated directory failures.
**Break-glass account.** One built-in account always has full access through
no binding, and cannot be turned off or removed. On an empty database it is
created from `INITIAL_ADMIN_USERNAME` / `INITIAL_ADMIN_PASSWORD`. Without
those, it is `admin` with a one-time password written once to the server log.
**Lockout recovery** (an identity-provider outage is never terminal):
- `AUTH_FORCE_LOCAL=1` (environment only) always offers the built-in form.
- `dotnet Backend.dll auth-reset` turns Built-In sign-in back on, turns
redirect off, and prints a new one-time password for the break-glass
account. Add `--user <name>` for another built-in account.
**Upgrading from an earlier ExecuteR** happens once, at the first start:
- Each user's role becomes membership of a built-in group named after the
role, bound to it.
- The old OIDC settings (saved, or `OIDC_*` / `AUTH_LOGIN_MODE` in the
environment) become the OIDC tab.
- Roles' claim values and the Admin/Operator/Viewer mappings become
`claim=value` bindings.
- The old *default role* has no equivalent. Bind the claim your people carry
instead.
Displayed URLs (the OIDC callback `<public-url>/api/auth/oidc/callback`, its
post-logout `<public-url>/sign-in`, install commands) use `PUBLIC_SERVER_URL`
when set, otherwise trusted forwarded headers.
![Sign in](docs/screenshots/login.png)
---
## Custom roles and permissions
**Access** (nav) manages users and roles. A role is a named permission set —
devices, groups, scripts, functions, jobs, automation, secrets, remote shell,
file delivery, settings, access — optionally limited to a **device scope**:
pick device groups and members only see and target those devices (device
listing, job dispatch including group fan-out, shell, file delivery, inventory
refresh and tags all honor it).
**Access** (nav) manages roles and shows people. A role is a named permission
set (devices, groups, scripts, functions, jobs, automation, secrets, remote
shell, file delivery, settings, access), optionally limited to a **device
scope**. Pick device groups, and holders only see and target those devices:
device listing, job dispatch including group fan-out, shell, file delivery,
inventory refresh and tags all honor it.
Three system roles are seeded and immutable — **Admin** (`*`, includes future
permissions), **Operator** (day-to-day operations), **Viewer** (read-only) —
matching the pre-RBAC tiers, so existing users and sessions carry over
unchanged. Custom roles apply to local users (assign in the Users table) and
OIDC sign-ins alike: give a role *OIDC claim values* and anyone presenting one
of them in the configured role claim lands on it (custom matches beat the
legacy mapping; the most privileged wins on ties).
Three system roles are seeded and immutable:
Permission edits apply within seconds without re-login (roles resolve through
a short-TTL cache at request time). Safety rails: the last full-access user
cannot be deleted or demoted, system roles cannot be edited, and a role with
assigned users cannot be dropped.
- **Admin**: `*`, which includes future permissions.
- **Operator**: day-to-day operations.
- **Viewer**: read-only.
People get roles only through role bindings on their groups (see
[Sign-in](#sign-in)). **People** lists everyone who has signed in, of every
kind, and what they hold and where each grant came from.
Permission and binding edits apply within seconds without signing in again:
roles and bindings resolve through short caches on every request.
Safety rails:
- System roles cannot be edited.
- A role still bound cannot be deleted.
- Removing the last binding that manages access is refused while there is no
break-glass account to recover with.
![Access control](docs/screenshots/access.png)
@@ -774,12 +1036,80 @@ An engine unavailable on a device (PowerShell 5 on Linux, bash without Git
Bash on Windows) fails the job immediately with a clear message instead of
running.
The editor itself pops out to fill the window (Esc restores) and offers
optional **IntelliSense** — a curated offline provider with common cmdlets,
keywords, snippets, plus the script's own parameters, `$Secrets` bindings and
`$Files` entries (typed as FileInfo or DirectoryInfo); no language server
involved. Execution flows vertically beneath the editor: target and
**Execution values**, Schedules, Triggers, then full-width output.
**Output reads the way PowerShell shows it.** Both PowerShell engines pipe
the script's output through `Out-String -Stream` at a 4096-column width, so
objects arrive as their table or list view (a hashtable's Name/Value rows,
an object's properties as columns) rather than a type name such as
`System.Collections.Hashtable`, still streamed line by line and without
`…` truncation. Strings pass through unchanged.
**Downloads.** A job's output drawer downloads its complete stored output as
`COMPUTERNAME.JobID.yyyy.MM.dd.HHmmss.log` (optionally with timestamps), and
a run (Jobs › run, or a script's Executions tab) downloads every target's
output in one `ScriptName.RunID.yyyy.MM.dd.HHmmss.log` with a section per
device and its state and exit code. The time is when the job or run was
created, in UTC on a 24-hour clock. Non-output streams are tagged
(`[ERROR]`, `[WARNING]`, …). Endpoints: `GET /api/jobs/{jobId}/output/download`
and `GET /api/executions/{runId}/output/download` (`?timestamps=true`);
device-scoped roles only reach their own devices' output.
**Bulk log downloads.** On a run, check devices and **Download Logs** (or
**Download All Logs** with none checked) saves `ScriptName.RunID.yyyy.MM.dd.HHmmss.zip`
holding one `COMPUTERNAME.JobID.yyyy.MM.dd.HHmmss.log` per device, the same
file its drawer downloads. Skipped targets have no log. Endpoint:
`POST /api/executions/{runId}/output/archive` with `{ "jobIds": [...], "timestamps": false }`
(no `jobIds` = every job); the zip streams as it is written.
The code editor is meant to feel like VS Code (offline, no language server):
- **Highlighting** after VS Code's Dark/Light Modern themes: cmdlets,
`-Parameters`, word operators, types and attributes, and variables and
`$( … )` inside double-quoted and here-strings.
- **IntelliSense** (toggle per browser) that follows the context. After `-`
it offers the command's parameters (cmdlets, functions in the script and
functions in attached modules) or operators after a value. After `$` it
offers variables the script declares, its parameters, `$Secrets` and
`$Files`. After `$Secrets.`, `$Files.`, `$env:` and `[` it offers bound
names, environment variables and types. It also includes snippets.
- **Hover cards** with syntax and synopsis for cmdlets and functions
(comment-based `.SYNOPSIS`), and where a variable is declared.
- **Navigation:** go to symbol (Ctrl+Shift+O), sticky scroll, go to
definition (F12), find references (Shift+F12), rename (F2), matching
highlights and bracket pair colours.
- **Problems:** squiggles for unbalanced brackets and for `$Secrets.X` /
`$Files.X` names not bound to the script (F8 steps through them).
- **Against the saved version:** change bars in the gutter, and **Compare
with Saved** (Ctrl+K D) as a side-by-side diff with per-change revert.
- **Status bar:** position and selection, problems, the function you're in,
indentation (spaces or tabs, convert), LF/CRLF, the command palette (F1),
and toggles for word wrap (Alt+Z), sticky scroll and the minimap
(remembered per browser).
It also fills the window (Esc restores). The editor is organized in tabs — Code, Parameters, Functions,
Secrets, Files, Executions (runs → targets → output) and Schedules — and
**Run…** / **Schedule…** open the shared run-now and schedule dialogs.
**Running a script** works the same way from a script and from devices. The
Run Script dialog has tabs:
- **Script:** searchable. It's fixed when you open the dialog from a script.
- **Devices:** search, a status filter and multi-select, with **Choose
Results** to pick everything the search found. Devices you chose on the
Devices page come preselected.
- **Groups:** the group tree with checkboxes.
- **Parameters**
- **Review:** reach (online, offline, orphaned) and **Only Devices Online Now**.
Starting a run opens its page (`/runs/{runId}`):
- one progress bar over every target;
- counters for Targets, Online, Pending, Running, Succeeded, Failed and
Skipped, each of which filters the target list;
- every target, skipped devices included; a row opens that device's output.
**Jobs** opens on **Runs**: recent runs across scripts, each with its
progress and source (Run Now, Schedule or Event Trigger). The **Jobs** tab
keeps the per-device job history.
![Script execution flow](docs/screenshots/script-execution.png)
@@ -831,19 +1161,40 @@ way as manual runs.
## Device groups and script triggers
Groups are either **static** (an explicit member list) or **dynamic** (rules
evaluated against live device data).
Groups form a **tree**. Each group is one of three kinds:
Rules are one or more **condition groups**. Conditions inside a group combine
with AND or OR, the groups themselves combine with AND or OR, groups can nest,
and both conditions and groups have an independent **NOT**.
| Kind | Members |
|------|---------|
| **Folder** | None of its own — it organizes the groups nested under it |
| **Static** | An explicit list of devices |
| **Dynamic** | The devices matching its rules, evaluated against live device data |
A group's membership **includes every group nested under it**, so running,
scheduling or scoping a role on `Sites` reaches `Sites › Office › Web` too.
Groups nest up to eight levels; a group can never be moved inside its own
subtree.
In the console the tree sits beside the device grid (Devices and Groups
pages): counts per node include nested groups, the selection and filters live
in the URL, groups are dragged onto other groups to nest them (with a
confirmation), and right-click / Shift+F10 / long-press opens a group's menu —
Run Script, Schedule Script, View Devices, New Group Here, Edit, Duplicate,
Move to Top Level, Delete. Deleting a group moves its nested groups up to its
parent and disables schedules that targeted it (with the reason recorded); a
group that limits a role's device scope cannot be deleted until it is removed
from that role, because dropping it would widen the role to every device.
Rules are one or more **condition blocks**. Conditions inside a block combine
with AND or OR, the blocks themselves combine with AND or OR, and both
conditions and blocks have an independent **NOT**.
Operators: `equals`, `notEquals`, `contains`, `notContains`, `startsWith`,
`endsWith`, `regex`, `in`, `notIn`, `>`, `>=`, `<`, `<=`, `exists`,
`notExists`, `inCidr`, `notInCidr`. The CIDR operators evaluate IP fields
against comma-separated ranges (`10.20.0.0/16, 192.168.1.0/24`) using the
BCL `System.Net.IPNetwork` — IPv4 and IPv6, bare IPs mean equality, and host
bits in the base address are normalized.
bits in the base address are normalized. The rule builder offers only the
operators that fit the chosen field.
Fields come from hardware inventory, tags, hostname/identity, and live status —
the full catalogue is served by `GET /api/groups/fields`, so the rule builder
@@ -869,14 +1220,15 @@ positive operator, and require **all** values for the negative ones. A missing
field satisfies only the negative operators. Invalid or runaway regex never
matches and never throws.
**Preview** evaluates candidate rules against live devices before you save.
The group editor **previews** candidate rules against live devices as you
type. Membership is evaluated once per request for every group, so the group
list, device list and fan-out stay cheap at fleet scale.
![Group rule builder](docs/screenshots/groups-rule-builder.png)
**Triggers** belong to scripts (the Triggers card on the script's page). A
trigger fires the script automatically — on a device lifecycle event, or on a
schedule — and each trigger carries **its own target**: all devices, a group
(members resolved at fire time), or a single device.
**Triggers** fire a script automatically — on a device lifecycle event, or on
a schedule — and each carries **its own target**: all devices, a group
(members, nested groups included, resolved at fire time), or a single device.
| Trigger | Fires when |
|---------|-----------|
@@ -885,42 +1237,61 @@ schedule — and each trigger carries **its own target**: all devices, a group
| `scheduled` | A cadence comes due — one-shot or recurring |
Lifecycle triggers fire **once per device** (recorded durably). Triggered runs
use the normal job pipeline, so they carry the script's Functions prelude and
typed parameters and appear alongside manual runs.
use the normal job pipeline, so they carry the script's Functions prelude,
typed parameters and the trigger's captured arguments, and appear alongside
manual runs.
---
## Group runs and scheduled triggers
## Group runs and schedules
The script's target selector offers **devices and groups**. A group run
creates one job per non-orphaned member — offline members receive theirs
durably when they reconnect — and the side panel tracks every job live, with
click-to-focus output per device. Orphaned members are reported as skipped.
**Run now** works from anywhere a target is in view: a device row's **…**
menu, the bulk **Actions** of chosen devices, a group in the tree, the device
and group pages, and the script editor. One dialog picks the script, any mix
of devices and groups, and typed arguments (starting from each parameter's
default). Before you confirm it shows the reach — how many devices, how many
online, and orphaned ones that will be skipped — and **Only Devices Online
Now** skips offline devices instead of queueing for their return (by default
offline members receive their job durably when they reconnect). Every target
of one run shares one execution, which the result links to.
**Scheduled triggers carry their cadence inline.** A `scheduled` trigger stores
*when* (one-shot or recurring), its *target* (a device, a group whose members
resolve at fire time, or all devices), and the execution values to run with —
no separate schedule object to manage.
`POST /api/jobs` accepts the same mix: `clientId` (the original single-device
call, answering with the job), `clientIds`, `groupId`, `groupIds` and
`onlineOnly`, deduplicated and scope-filtered into one `runId`.
**Schedules** (the Schedules page, plus a Schedules tab on scripts, devices
and groups) are scheduled triggers: *when* (one-shot or recurring), *target*
(a device, a group whose members resolve at fire time, or all devices), an
optional name, and the arguments to run with — no separate schedule object to
manage. Each can be edited in place, duplicated, paused, or **run now**
without disturbing its cadence.
The cadence builder leads with friendly selectors — *every N minutes/hours*,
*daily/weekly/monthly at a time* with weekday chips — and compiles them to
cron internally; a raw cron field exists only under **Advanced**. Whatever is
chosen is auto-described in plain English.
*daily/weekly/monthly at a time* with weekday chips, *once* — and compiles them
to cron; a raw cron field exists only under **Advanced**, and a saved cron
reopens in its friendly form when it has one. The next runs are computed by
the server (`POST /api/triggers/preview`) in the schedule's time zone, so what
you see is what will fire.
![Scheduled trigger builder](docs/screenshots/script-schedule.png)
Semantics worth knowing:
- Expressions evaluate in the trigger's **time zone** (default UTC), so
"daily at 01:00" tracks DST; storage and comparison are UTC.
- Expressions evaluate in the schedule's **time zone** (the browser's by
default), so "daily at 01:00" tracks DST; storage and comparison are UTC.
- An unknown time zone, a never-firing cron, or an enabled one-shot in the
past is refused when saving.
- The worker sweeps regularly; recurrence advances from *now* after each
firing, so server downtime causes **at most one late firing**, never a storm.
- One-shot triggers disable themselves after firing.
- Each trigger records its own **last result** (jobs queued, or the error —
e.g. a vault failure) without stalling, and triggered jobs are attributed to
`trigger:<type>`.
- Deleting a script removes its triggers with it.
- Execution values captured on the trigger are used for every firing; secret
- One-shot schedules disable themselves after firing.
- Each schedule records its **last result** (jobs queued, or the error — e.g.
a vault failure) and the execution it created; scheduled jobs are attributed
to `schedule`, run-now firings to the operator.
- A device-scoped role can only aim schedules inside its scope: a device it
can see, or a group whose members it can all see — never all devices.
- Deleting a script removes its schedules with it; schedule changes are
audited under the `automation` category.
- Arguments captured on the schedule are used for every firing; secret
bindings resolve fresh server-side at each fire time.
---
@@ -1067,6 +1438,84 @@ For a local build without CI:
---
## Agent upgrades from the server
How upgrades are started, paced and rolled back. What happens on the device is in
[Agent upgrades and the watchdog](#agent-upgrades-and-the-watchdog).
### On the server
**Versions arrive with each server release.** The agent packages that ship in
the backend image (or are dropped into `AGENT_PACKAGE_PATH`, with their
`VERSION` marker) are imported into the server's **version catalog** at
startup and every five minutes: the portable Windows zip and the Linux/macOS
builds (the MSI is for installs only — Windows upgrades swap files from the
zip). Each package is copied to `<DATA_PATH>/agent-packages/<version>/`, so a
device can still be rolled back after the image moves on. A development build
(`0.0.0.0`) or a package without a version is not catalogued.
The catalog keeps the newest **KeepVersions** versions (default 5). It never
drops the newest version, a version an open upgrade goes to, or a version some
device's last upgrade came from (its rollback target); removing a version by
hand is refused with the same reason (409).
Agents download from `GET /api/agent-packages/{sha256}` — anonymous like the
install package, with range support, and **not** rate-limited per address (a
fleet behind one NAT upgrades through it). With S3 agent offload on, catalog
packages are uploaded too and upgrade commands carry a presigned URL instead.
| Route | Permission | |
|---|---|---|
| `GET /api/agent-versions` | devices.view | Versions newest first: packages, devices on it, why it cannot be removed |
| `DELETE /api/agent-versions/{version}` | settings.manage | Remove a version (409 while needed) |
| `POST /api/agent-upgrades` | devices.manage | Upgrade devices and groups (nested included) to `version`, or the newest per platform |
| `POST /api/agent-upgrades/rollback` | devices.manage | Back to the version each device's last upgrade came from |
| `POST /api/agent-upgrades/{id}/cancel` | devices.manage | Cancel an upgrade not yet sent |
| `GET /api/agent-upgrades?clientId=&open=true&limit=` | devices.view | Upgrade attempts, newest first |
| `GET`/`PUT /api/settings/agent-upgrades` | devices.view / settings.manage | Settings › Agent Upgrades |
A start or roll back returns what was queued and who was left out: `already`
(on the target), `busy` (an upgrade is open), `no-package`, `unsupported` (no
build for its platform), `orphaned`, or `no-previous` for a roll back. The
package comes from the runtime the agent reports; older agents are matched by
their inventory's OS and architecture. Offline devices are queued and start
when they come back (a request waits up to 7 days, then expires).
**Upgrades start in paced batches** so a large fleet never downloads at once:
every 15 seconds the next batch of queued, online devices is sent, sized so
each starts within **StartWithinMinutes** (default 3) of being ready — at least
10 a batch — and each agent waits a random 0–15 seconds before downloading.
Only downloads are paced; a device installing has finished its download. A
device that goes quiet is failed `no-report` (30 minutes without an event
while downloading, or the health deadline plus 15 minutes while installing);
a late outcome still counts. When a device's presence already shows the target
version, its upgrade is closed as succeeded.
**Automatic Upgrades** (off by default) keep online devices outside the
excluded groups on the newest version held for their platform, through the
same paced batches. They never go to an older version and never retry a
version that already failed or rolled back on that device.
| Setting | Default | Range |
|---|---|---|
| AutomaticUpgrades | off | |
| StartWithinMinutes | 3 | 1–30 |
| HealthDeadlineMinutes | 10 | 2–60 (the watchdog keeps the new version only when it reports healthy in time) |
| ExcludedGroupIds | none | existing groups; nested members are excluded too |
| KeepVersions | 5 | 2–50 |
**Device History** shows `agent-upgraded` (`2026.10.20.1200 → 2026.10.23.1400`),
`agent-rolled-back` (an operator's roll back, or the watchdog undoing an
upgrade, with its reason) and `agent-upgrade-failed` (with the reason).
With several replicas, the upgrade scheduler runs where the schedules do
(`SCHEDULE_WORKER=1`), and every replica keeps its own catalog copies. Without
a shared data volume, a replica only serves versions it has itself carried, so
turn on S3 agent offload (or share `DATA_PATH`) to keep older versions
reachable. `AGENT_CATALOG_SYNC=0` turns the catalog import off.
---
## S3 file delivery
Configure an S3-compatible store (AWS S3, MinIO, Backblaze B2) from
@@ -1190,7 +1639,7 @@ is private — set `REGISTRY_USERNAME` and `REGISTRY_TOKEN` in `.env` and
Gitea Actions workflows live in `.gitea/workflows`:
- **Pull request to `main`** — restore, build, unit + integration tests, Svelte
- **Pull request to `main`** — restore, build, unit + integration tests, WebUI
check. No image push, no release.
- **Push to `dev`** — build + test, agent binaries for every platform
(win-x64, win-arm64, linux-x64, linux-arm64, osx-x64, osx-arm64) + MSI, the
@@ -1261,3 +1710,12 @@ Built to prove one idea cleanly: secure HTTPS enrollment, scoped NATS over
WebSocket through one proxy edge, durable work, in-process PowerShell, and a
secured terminal — with a browser that updates in real time over NATS, no
SignalR or SSE.
---
## Licence
ExecuteR is proprietary software of Grace Solutions, provided under licence
only; see [LICENSE](LICENSE). It includes third-party components under their
own licences, and its web console is built on the commercially licensed Mantis
Pro template — see [THIRD-PARTY-NOTICES.md](THIRD-PARTY-NOTICES.md).
+56
View File
@@ -0,0 +1,56 @@
# Third-party notices
ExecuteR includes the components below. Each remains under its own licence.
## Commercial
| Component | Licence | Notes |
| --- | --- | --- |
| Mantis React Material UI Pro template (CodedThemes) | Commercial licence purchased from the MUI Store | Basis of the web console's layout and theme. Licensed for this product only; the template source may not be redistributed on its own. |
## Shipped beside the server
| Component | Licence | Notes |
| --- | --- | --- |
| NATS server (`nats-server`, Synadia / NATS authors) | Apache-2.0 | Bundled unmodified in the standalone server packages and run as a separate process (the sidecar); the container deployments use the official `nats` image. |
## Server and agent
| Component | Licence |
| --- | --- |
| .NET runtime, ASP.NET Core, Entity Framework Core, Microsoft.Extensions.*, System.* | MIT |
| Microsoft.AspNetCore.Authentication.OpenIdConnect | MIT |
| Microsoft.EntityFrameworkCore.Sqlite, Microsoft.Data.Sqlite, Microsoft.EntityFrameworkCore.SqlServer | MIT |
| SQLitePCLRaw | Apache-2.0 |
| SQLite (e_sqlite3) | Public domain |
| Npgsql.EntityFrameworkCore.PostgreSQL | PostgreSQL |
| NATS.Net, NATS.Jwt, NATS.NKeys | Apache-2.0 |
| AWSSDK.S3 | Apache-2.0 |
| Cronos | MIT |
| Serilog and sinks | Apache-2.0 |
| Microsoft.PowerShell.SDK (PowerShell 7) | MIT |
| Microsoft.PowerShell.PSResourceGet, Microsoft.PowerShell.Archive, Microsoft.PowerShell.ThreadJob (bundled with the agent) | MIT |
## Web console
| Component | Licence |
| --- | --- |
| React, React DOM, React Router | MIT |
| MUI (Material UI) | MIT |
| Emotion | MIT |
| Ant Design colors and icons | MIT |
| SWR, notistack, lodash-es | MIT |
| Monaco Editor | MIT |
| xterm.js and its fit addon | MIT |
| NATS JavaScript client (@nats-io/nats-core, @nats-io/nkeys) | Apache-2.0 |
| Public Sans font (@fontsource/public-sans) | SIL Open Font License 1.1 |
## Build and packaging
| Component | Licence |
| --- | --- |
| WiX Toolset 5 and extensions | MS-RL |
| xUnit.net | Apache-2.0 |
| Vite, TypeScript, ESLint, Prettier | MIT / Apache-2.0 |
Full licence texts ship with each package in the NuGet and npm caches and in `src/WebUI/node_modules/*/LICENSE`.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 234 KiB

After

Width:  |  Height:  |  Size: 282 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 195 KiB

After

Width:  |  Height:  |  Size: 155 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 253 KiB

After

Width:  |  Height:  |  Size: 260 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 203 KiB

After

Width:  |  Height:  |  Size: 256 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 208 KiB

After

Width:  |  Height:  |  Size: 269 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 158 KiB

After

Width:  |  Height:  |  Size: 224 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 244 KiB

After

Width:  |  Height:  |  Size: 271 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 188 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 208 KiB

After

Width:  |  Height:  |  Size: 243 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 139 KiB

After

Width:  |  Height:  |  Size: 172 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 200 KiB

After

Width:  |  Height:  |  Size: 357 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 111 KiB

After

Width:  |  Height:  |  Size: 141 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 157 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 216 KiB

After

Width:  |  Height:  |  Size: 334 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 201 KiB

After

Width:  |  Height:  |  Size: 274 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 164 KiB

After

Width:  |  Height:  |  Size: 269 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 264 KiB

After

Width:  |  Height:  |  Size: 342 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 60 KiB

After

Width:  |  Height:  |  Size: 238 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 208 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 164 KiB

After

Width:  |  Height:  |  Size: 178 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 211 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 208 KiB

After

Width:  |  Height:  |  Size: 184 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 77 KiB

After

Width:  |  Height:  |  Size: 252 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 88 KiB

After

Width:  |  Height:  |  Size: 222 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 237 KiB

After

Width:  |  Height:  |  Size: 191 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 106 KiB

After

Width:  |  Height:  |  Size: 190 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 206 KiB

+4 -1
View File
@@ -15,7 +15,10 @@ if (-not $dotnet) { throw '.NET SDK 10.x is required (https://dotnet.microsoft.c
Write-Host "dotnet $(dotnet --version)"
$node = Get-Command node -ErrorAction SilentlyContinue
if (-not $node) { throw 'Node.js LTS is required (https://nodejs.org).' }
if (-not $node) { throw 'Node.js 22 or later is required (https://nodejs.org).' }
# The NATS browser client (@nats-io/*) needs Node 22+; npm enforces it too (engines).
$nodeMajor = [int]((node --version).TrimStart('v').Split('.')[0])
if ($nodeMajor -lt 22) { throw "Node.js 22 or later is required; found $(node --version)." }
Write-Host "node $(node --version)"
Write-Host '── Restoring .NET packages ──'
+23 -2
View File
@@ -1,7 +1,8 @@
<#
.SYNOPSIS
Publishes the self-contained single-file Agent executable per platform, zips
each, and builds the MSI (win-x64) when the WiX project is present.
Publishes the self-contained single-file Agent executable and its watchdog
per platform, zips each, and builds the MSI (win-x64) when the WiX project is
present.
Windows RIDs publish the net10.0-windows flavor (WMI, ConPTY, DPAPI, service);
everything else publishes the portable net10.0 flavor (file config,
@@ -62,7 +63,27 @@ foreach ($rid in $Runtimes) {
Rename-Item (Join-Path $ridPublishDir 'ExecuteR-Agent') 'executer-agent'
}
# The watchdog (keeps the agent running, applies upgrades) travels in every
# package beside the agent: one small trimmed single-file binary per RID.
$watchdogPublishDir = Join-Path $artifacts "watchdog/$Version/$rid"
if (Test-Path $watchdogPublishDir) { Remove-Item $watchdogPublishDir -Recurse -Force }
dotnet publish (Join-Path $repoRoot 'src/Agent.Watchdog/Agent.Watchdog.csproj') `
-c Release `
-r $rid `
-p:DebugType=None `
"-p:UnifiedVersion=$Version" `
"-p:GitCommitShort=$(git -C $repoRoot rev-parse --short HEAD 2>$null)" `
-o $watchdogPublishDir
if ($LASTEXITCODE -ne 0) { throw "Watchdog publish for $rid failed ($LASTEXITCODE)." }
if ($rid -like 'win-*') {
Copy-Item (Join-Path $watchdogPublishDir 'ExecuteR-Watchdog.exe') $ridPublishDir
} else {
Copy-Item (Join-Path $watchdogPublishDir 'ExecuteR-Watchdog') (Join-Path $ridPublishDir 'executer-watchdog')
}
Copy-Item $modulesStage (Join-Path $ridPublishDir 'Modules') -Recurse
# Licence terms and third-party notices travel with every package.
Copy-Item (Join-Path $repoRoot 'LICENSE'), (Join-Path $repoRoot 'THIRD-PARTY-NOTICES.md') $ridPublishDir
$zipPath = Join-Path $artifacts "ExecuteR-Agent-$Version-$rid.zip"
if (Test-Path $zipPath) { Remove-Item $zipPath -Force }
+5 -3
View File
@@ -37,16 +37,15 @@ if (-not $SkipIcon) {
Push-Location (Join-Path $repoRoot 'src/WebUI')
try {
if (-not (Test-Path 'node_modules')) { npm ci }
$env:ADAPTER = 'static'
$env:PUBLIC_BUILD_VERSION = $Version
npm run build
if ($LASTEXITCODE -ne 0) { throw "WebUI static build failed ($LASTEXITCODE)." }
}
finally {
Remove-Item Env:\ADAPTER -ErrorAction SilentlyContinue
Remove-Item Env:\PUBLIC_BUILD_VERSION -ErrorAction SilentlyContinue
Pop-Location
}
$staticBuild = Join-Path $repoRoot 'src/WebUI/build-static'
$staticBuild = Join-Path $repoRoot 'src/WebUI/dist'
if (-not (Test-Path (Join-Path $staticBuild 'index.html'))) {
throw "Static WebUI build missing at $staticBuild"
}
@@ -172,6 +171,9 @@ TRUSTED_NETWORKS (replace the default trust with an allow-list),
TRUSTED_HOSTS, TRUST_PROXY=0 to disable.
"@
# Licence terms and third-party notices travel with every package.
Copy-Item (Join-Path $repoRoot 'LICENSE'), (Join-Path $repoRoot 'THIRD-PARTY-NOTICES.md') $publishDir
# ── Zip it ───────────────────────────────────────────────────────────────
$zipPath = Join-Path $artifacts "ExecuteR-Server-$Version-$rid.zip"
if (Test-Path $zipPath) { Remove-Item $zipPath -Force }
+2 -2
View File
@@ -1,6 +1,6 @@
<#
.SYNOPSIS
Runs unit tests (and optionally the Svelte check + integration tests).
Runs unit tests (and optionally the WebUI typecheck + lint and the integration tests).
.EXAMPLE
./scripts/Test.ps1
@@ -43,7 +43,7 @@ if ($IncludeWebUI) {
if ($LASTEXITCODE -ne 0) { throw "npm ci failed ($LASTEXITCODE)." }
}
npm run check
if ($LASTEXITCODE -ne 0) { throw "svelte-check failed ($LASTEXITCODE)." }
if ($LASTEXITCODE -ne 0) { throw "WebUI check (typecheck + lint) failed ($LASTEXITCODE)." }
}
finally {
Pop-Location
+106
View File
@@ -0,0 +1,106 @@
using System.Diagnostics;
namespace Agent.Watchdog;
/// <summary>The watchdog's clock; replaced in tests so a ten-minute wait takes no time.</summary>
public interface IWatchdogClock
{
DateTimeOffset UtcNow { get; }
Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken);
}
public sealed class SystemClock : IWatchdogClock
{
public static readonly SystemClock Instance = new();
public DateTimeOffset UtcNow => DateTimeOffset.UtcNow;
public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) => Task.Delay(delay, cancellationToken);
}
public enum AgentRunState
{
NotInstalled,
Stopped,
Starting,
Running,
Stopping,
}
/// <summary>
/// What the service manager says about the agent. <c>AutoStart</c>: set to start
/// automatically (an admin's Manual or Disabled is left alone).
/// <c>StoppedWithFailure</c>: it stopped by exiting with an error rather than
/// by a stop request.
/// </summary>
public sealed record AgentServiceStatus(AgentRunState State, bool AutoStart, bool StoppedWithFailure)
{
public static readonly AgentServiceStatus NotInstalled = new(AgentRunState.NotInstalled, false, false);
public bool IsRunning => State is AgentRunState.Running or AgentRunState.Starting or AgentRunState.Stopping;
}
/// <summary>The agent, through its service manager (or as a child process in the development mode).</summary>
public interface IAgentService
{
Task<AgentServiceStatus> QueryAsync(CancellationToken cancellationToken);
/// <summary>Starts the agent; throws when the service manager refuses.</summary>
Task StartAsync(CancellationToken cancellationToken);
/// <summary>Stops the agent and waits until it and its processes are gone, so its files can be moved.</summary>
Task StopAsync(CancellationToken cancellationToken);
}
/// <summary>Runs a candidate watchdog binary once (<c>&lt;binary&gt; version</c>) to see that it starts.</summary>
public interface ISmokeTest
{
/// <summary>The version it printed when it exited 0 within the time allowed; null otherwise.</summary>
string? Run(string binary);
}
public sealed class ProcessSmokeTest : ISmokeTest
{
public static readonly TimeSpan Timeout = TimeSpan.FromSeconds(30);
public string? Run(string binary)
{
try
{
using var process = Process.Start(new ProcessStartInfo(binary, "version")
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
WorkingDirectory = Path.GetDirectoryName(binary)!,
});
if (process is null)
{
return null;
}
var output = process.StandardOutput.ReadToEndAsync();
_ = process.StandardError.ReadToEndAsync();
if (!process.WaitForExit(Timeout))
{
process.Kill(entireProcessTree: true);
return null;
}
return process.ExitCode == 0 ? output.GetAwaiter().GetResult().Trim() : null;
}
catch (Exception exception) when (exception is System.ComponentModel.Win32Exception or InvalidOperationException or IOException)
{
return null;
}
}
}
/// <summary>Restarts the watchdog onto its new binary after it replaced itself.</summary>
public interface IWatchdogRestart
{
/// <summary>True when a restart is under way and this process should now exit.</summary>
bool Restart(string newBinary);
}
+53
View File
@@ -0,0 +1,53 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<!-- One portable flavor: Windows-only code is guarded at run time, so the
same source publishes for every RID. -->
<TargetFramework>net10.0</TargetFramework>
<RootNamespace>Agent.Watchdog</RootNamespace>
<AssemblyName>ExecuteR-Watchdog</AssemblyName>
<Product>ExecuteR Watchdog</Product>
<AssemblyTitle>ExecuteR Watchdog</AssemblyTitle>
<RuntimeIdentifiers>win-x64;win-arm64;linux-x64;linux-arm64;osx-x64;osx-arm64</RuntimeIdentifiers>
<!-- Required by the LibraryImport interop source generator. -->
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<InvariantGlobalization>true</InvariantGlobalization>
<SatelliteResourceLanguages>en</SatelliteResourceLanguages>
<!-- Small on purpose: no hosting stack, JSON through a source-generated
context, trimmed whole (trim warnings are errors). -->
<EnableTrimAnalyzer>true</EnableTrimAnalyzer>
<ApplicationIcon Condition="Exists('$(MSBuildThisFileDirectory)..\..\assets\icons\app.ico')">$(MSBuildThisFileDirectory)..\..\assets\icons\app.ico</ApplicationIcon>
</PropertyGroup>
<!-- `dotnet publish -r <rid>` gives the shipping binary: one self-contained,
trimmed, compressed file. -->
<PropertyGroup Condition="'$(RuntimeIdentifier)' != ''">
<SelfContained>true</SelfContained>
<PublishSingleFile>true</PublishSingleFile>
<PublishTrimmed>true</PublishTrimmed>
<TrimMode>full</TrimMode>
<EnableCompressionInSingleFile>true</EnableCompressionInSingleFile>
<DebugType>none</DebugType>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="Agent.Watchdog.Tests" />
</ItemGroup>
<ItemGroup>
<!-- The one branding file, linked rather than referenced: Shared.Infrastructure
would bring Serilog and configuration into the watchdog. -->
<Compile Include="..\Shared.Infrastructure\Constants\Branding.cs" Link="Branding.cs" />
</ItemGroup>
<ItemGroup>
<!-- ServiceBase (the Windows service host) and ServiceController. -->
<PackageReference Include="System.ServiceProcess.ServiceController" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Shared.Contracts\Shared.Contracts.csproj" />
</ItemGroup>
</Project>
+103
View File
@@ -0,0 +1,103 @@
using Shared.Contracts.Serialization;
namespace Agent.Watchdog;
/// <summary>
/// Keeps the agent running without fighting an admin. It acts only while the
/// agent service is installed and set to start automatically (a Manual or
/// Disabled service is the admin's choice and left alone), and then only in two
/// cases, at most once per <see cref="WatchdogOptions.StaleAfter"/>:
/// <list type="bullet">
/// <item>the agent runs, has run for longer than the stale period, and its
/// health beacon is older than that: it hangs, so it is restarted (no beacon at
/// all means a version that writes none, and is left alone);</item>
/// <item>the agent stopped with a failure and is still stopped a stale period
/// later (the service manager's own restarts gave up): it is started.</item>
/// </list>
/// A clean stop (<c>ExecuteR-Agent.exe stop</c>, <c>systemctl stop</c>) is never undone.
/// </summary>
public sealed class AgentSupervisor(WatchdogOptions options, IAgentService agent, IWatchdogClock clock, WatchdogLog log)
{
private DateTimeOffset? runningSince;
private DateTimeOffset? failedSince;
private DateTimeOffset lastRestart = DateTimeOffset.MinValue;
private string? lastNote;
public async Task CheckAsync(CancellationToken cancellationToken)
{
var status = await agent.QueryAsync(cancellationToken);
var now = clock.UtcNow;
if (status.State == AgentRunState.NotInstalled || !status.AutoStart)
{
runningSince = null;
failedSince = null;
Note(status.State == AgentRunState.NotInstalled
? "The agent service is not installed; nothing to watch"
: "The agent service is not set to start automatically; it is left as the admin set it");
return;
}
Note(null);
if (status.IsRunning)
{
failedSince = null;
runningSince ??= now;
var beacon = WatchdogFileStore.Read(options.HealthPath, WatchdogJsonContext.Default.WatchdogHealthBeacon);
if (beacon is not null
&& now - beacon.AtUtc > options.StaleAfter
&& now - runningSince > options.StaleAfter
&& now - lastRestart > options.StaleAfter)
{
log.Warn($"The agent stopped answering (last health beacon {beacon.AtUtc:O}); restarting it");
lastRestart = now;
runningSince = now;
try
{
await agent.StopAsync(cancellationToken);
await agent.StartAsync(cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Warn("Restarting the agent failed", exception);
}
}
return;
}
runningSince = null;
if (!status.StoppedWithFailure || status.State != AgentRunState.Stopped)
{
failedSince = null;
return;
}
failedSince ??= now;
if (now - failedSince > options.StaleAfter && now - lastRestart > options.StaleAfter)
{
log.Warn("The agent stopped with a failure and was not started again; starting it");
lastRestart = now;
failedSince = null;
try
{
await agent.StartAsync(cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Warn("Starting the agent failed", exception);
}
}
}
/// <summary>Logs a standing condition once, not every poll.</summary>
private void Note(string? note)
{
if (note is not null && note != lastNote)
{
log.Info(note);
}
lastNote = note;
}
}
@@ -0,0 +1,105 @@
using System.Diagnostics;
namespace Agent.Watchdog.Platform;
/// <summary>
/// Development mode (<c>--agent process</c>): the agent runs as the watchdog's
/// child (<c>&lt;agent&gt; foreground</c>) instead of a service, so the swap and
/// rollback can be driven on a machine without installing services. An agent in
/// the foreground refuses upgrade commands itself; this mode is for driving the
/// watchdog with a stub agent and a hand-made hand-over.
/// </summary>
public sealed class ProcessAgentService(WatchdogOptions options, WatchdogLog log) : IAgentService, IDisposable
{
private readonly Lock gate = new();
private Process? process;
public Task<AgentServiceStatus> QueryAsync(CancellationToken cancellationToken)
{
lock (gate)
{
if (process is null)
{
return Task.FromResult(new AgentServiceStatus(AgentRunState.Stopped, true, false));
}
return Task.FromResult(process.HasExited
? new AgentServiceStatus(AgentRunState.Stopped, true, process.ExitCode != 0)
: new AgentServiceStatus(AgentRunState.Running, true, false));
}
}
public Task StartAsync(CancellationToken cancellationToken)
{
lock (gate)
{
if (process is { HasExited: false })
{
return Task.CompletedTask;
}
var executable = Path.Combine(options.InstallDirectory, options.AgentExecutable);
var start = new ProcessStartInfo(executable, "foreground")
{
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = options.InstallDirectory,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
process?.Dispose();
process = Process.Start(start) ?? throw new InvalidOperationException($"{executable} did not start");
process.OutputDataReceived += (_, _) => { };
process.ErrorDataReceived += (_, _) => { };
process.BeginOutputReadLine();
process.BeginErrorReadLine();
log.Info($"Agent process {process.Id} started ({executable})");
}
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken cancellationToken)
{
Process? running;
lock (gate)
{
running = process;
}
if (running is null || running.HasExited)
{
return;
}
try
{
running.Kill(entireProcessTree: true);
}
catch (InvalidOperationException)
{
}
await running.WaitForExitAsync(cancellationToken);
log.Info($"Agent process {running.Id} stopped");
}
public void Dispose()
{
lock (gate)
{
if (process is { HasExited: false } running)
{
try
{
running.Kill(entireProcessTree: true);
}
catch (InvalidOperationException)
{
}
}
process?.Dispose();
}
}
}
@@ -0,0 +1,158 @@
using System.ComponentModel;
using System.Runtime.InteropServices;
namespace Agent.Watchdog.Platform;
/// <summary>
/// Minimal advapi32 service-control interop (never sc.exe): registration and
/// the exit code of a stopped service, which ServiceController does not show.
/// </summary>
internal static partial class ServiceInterop
{
internal const uint SC_MANAGER_CONNECT = 0x1;
internal const uint SC_MANAGER_ALL_ACCESS = 0xF003F;
internal const uint SERVICE_QUERY_STATUS = 0x4;
internal const uint SERVICE_ALL_ACCESS = 0xF01FF;
internal const uint SERVICE_WIN32_OWN_PROCESS = 0x10;
internal const uint SERVICE_AUTO_START = 0x2;
internal const uint SERVICE_ERROR_NORMAL = 0x1;
internal const int SERVICE_CONFIG_DESCRIPTION = 1;
internal const int SERVICE_CONFIG_FAILURE_ACTIONS = 2;
internal const int SC_ACTION_RESTART = 1;
internal const int SC_STATUS_PROCESS_INFO = 0;
/// <summary>The service has not been started since the machine booted: not a failure.</summary>
internal const uint ERROR_SERVICE_NEVER_STARTED = 1077;
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
internal struct SERVICE_DESCRIPTION
{
public string lpDescription;
}
[StructLayout(LayoutKind.Sequential)]
internal struct SC_ACTION
{
public int Type;
public uint Delay;
}
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
internal struct SERVICE_FAILURE_ACTIONS
{
public uint dwResetPeriod;
public string? lpRebootMsg;
public string? lpCommand;
public uint cActions;
public IntPtr lpsaActions;
}
[StructLayout(LayoutKind.Sequential)]
internal struct SERVICE_STATUS_PROCESS
{
public uint dwServiceType;
public uint dwCurrentState;
public uint dwControlsAccepted;
public uint dwWin32ExitCode;
public uint dwServiceSpecificExitCode;
public uint dwCheckPoint;
public uint dwWaitHint;
public uint dwProcessId;
public uint dwServiceFlags;
}
[LibraryImport("advapi32.dll", EntryPoint = "OpenSCManagerW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)]
internal static partial IntPtr OpenSCManager(string? machineName, string? databaseName, uint access);
[LibraryImport("advapi32.dll", EntryPoint = "OpenServiceW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)]
internal static partial IntPtr OpenService(IntPtr scManager, string serviceName, uint access);
[LibraryImport("advapi32.dll", EntryPoint = "CreateServiceW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)]
internal static partial IntPtr CreateService(
IntPtr scManager,
string serviceName,
string displayName,
uint desiredAccess,
uint serviceType,
uint startType,
uint errorControl,
string binaryPathName,
string? loadOrderGroup,
IntPtr tagId,
string? dependencies,
string? serviceStartName,
string? password);
[LibraryImport("advapi32.dll", EntryPoint = "ChangeServiceConfigW", StringMarshalling = StringMarshalling.Utf16, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static partial bool ChangeServiceConfig(
IntPtr service,
uint serviceType,
uint startType,
uint errorControl,
string? binaryPathName,
string? loadOrderGroup,
IntPtr tagId,
string? dependencies,
string? serviceStartName,
string? password,
string? displayName);
[DllImport("advapi32.dll", EntryPoint = "ChangeServiceConfig2W", CharSet = CharSet.Unicode, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool ChangeServiceConfig2(IntPtr service, int infoLevel, ref SERVICE_DESCRIPTION info);
[DllImport("advapi32.dll", EntryPoint = "ChangeServiceConfig2W", CharSet = CharSet.Unicode, SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static extern bool ChangeServiceConfig2(IntPtr service, int infoLevel, ref SERVICE_FAILURE_ACTIONS info);
[LibraryImport("advapi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static partial bool QueryServiceStatusEx(IntPtr service, int infoLevel, out SERVICE_STATUS_PROCESS buffer, int bufferSize, out int bytesNeeded);
[LibraryImport("advapi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static partial bool DeleteService(IntPtr service);
[LibraryImport("advapi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
internal static partial bool CloseServiceHandle(IntPtr handle);
internal static Win32Exception LastError(string operation)
=> new(Marshal.GetLastPInvokeError(), $"{operation} failed");
/// <summary>The service's status block, or null when it cannot be read.</summary>
internal static SERVICE_STATUS_PROCESS? QueryStatus(string serviceName)
{
var scm = OpenSCManager(null, null, SC_MANAGER_CONNECT);
if (scm == IntPtr.Zero)
{
return null;
}
try
{
var service = OpenService(scm, serviceName, SERVICE_QUERY_STATUS);
if (service == IntPtr.Zero)
{
return null;
}
try
{
return QueryServiceStatusEx(service, SC_STATUS_PROCESS_INFO, out var status, Marshal.SizeOf<SERVICE_STATUS_PROCESS>(), out _)
? status
: null;
}
finally
{
CloseServiceHandle(service);
}
}
finally
{
CloseServiceHandle(scm);
}
}
}
+282
View File
@@ -0,0 +1,282 @@
using System.Diagnostics;
using Shared.Infrastructure.Constants;
namespace Agent.Watchdog.Platform;
/// <summary>Runs a service-manager command (systemctl, launchctl) and captures its output.</summary>
internal static class Command
{
public static (int ExitCode, string Output) Run(string fileName, params string[] arguments)
{
var start = new ProcessStartInfo(fileName)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
foreach (var argument in arguments)
{
start.ArgumentList.Add(argument);
}
using var process = Process.Start(start) ?? throw new InvalidOperationException($"{fileName} did not start");
var output = process.StandardOutput.ReadToEndAsync();
var error = process.StandardError.ReadToEndAsync();
process.WaitForExit();
return (process.ExitCode, output.GetAwaiter().GetResult() + error.GetAwaiter().GetResult());
}
public static void Require(string fileName, params string[] arguments)
{
var (code, output) = Run(fileName, arguments);
if (code != 0)
{
throw new InvalidOperationException($"{fileName} {string.Join(' ', arguments)} exited with {code}: {output.Trim()}");
}
}
}
/// <summary>The agent's systemd unit.</summary>
public sealed class SystemdAgentService(string unit) : IAgentService
{
public Task<AgentServiceStatus> QueryAsync(CancellationToken cancellationToken)
{
var (_, output) = Command.Run("systemctl", "show", unit, "--property=LoadState,ActiveState,Result,UnitFileState");
var values = output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Select(line => line.Split('=', 2))
.Where(pair => pair.Length == 2)
.ToDictionary(pair => pair[0], pair => pair[1], StringComparer.Ordinal);
if (values.GetValueOrDefault("LoadState") is null or "not-found")
{
return Task.FromResult(AgentServiceStatus.NotInstalled);
}
var active = values.GetValueOrDefault("ActiveState");
var state = active switch
{
"active" or "reloading" => AgentRunState.Running,
"activating" => AgentRunState.Starting,
"deactivating" => AgentRunState.Stopping,
_ => AgentRunState.Stopped,
};
var failed = state == AgentRunState.Stopped
&& (active == "failed" || values.GetValueOrDefault("Result") is { } result && result != "success");
var enabled = values.GetValueOrDefault("UnitFileState") is "enabled" or "enabled-runtime";
return Task.FromResult(new AgentServiceStatus(state, enabled, failed));
}
public Task StartAsync(CancellationToken cancellationToken) => Task.Run(() =>
{
// A unit systemd gave up on stays failed until reset.
Command.Run("systemctl", "reset-failed", unit);
Command.Require("systemctl", "start", unit);
}, cancellationToken);
// systemctl stop returns once the unit's processes (its whole cgroup) are gone.
public Task StopAsync(CancellationToken cancellationToken) => Task.Run(() => Command.Require("systemctl", "stop", unit), cancellationToken);
}
/// <summary>
/// The agent's launchd daemon. Its plist keeps it alive, so a stop unloads it
/// (<c>bootout</c>) and a start loads it again (<c>bootstrap</c>).
/// </summary>
public sealed class LaunchdAgentService(string label) : IAgentService
{
private string Plist => $"/Library/LaunchDaemons/{label}.plist";
public Task<AgentServiceStatus> QueryAsync(CancellationToken cancellationToken)
{
if (!File.Exists(Plist))
{
return Task.FromResult(AgentServiceStatus.NotInstalled);
}
var (code, output) = Command.Run("launchctl", "print", $"system/{label}");
if (code != 0)
{
// Not loaded: booted out by a stop.
return Task.FromResult(new AgentServiceStatus(AgentRunState.Stopped, true, false));
}
var running = output.Contains("state = running", StringComparison.Ordinal);
var lastExit = output.Split('\n').Select(l => l.Trim()).FirstOrDefault(l => l.StartsWith("last exit code = ", StringComparison.Ordinal));
var failed = !running && lastExit is not null && lastExit != "last exit code = 0";
return Task.FromResult(new AgentServiceStatus(running ? AgentRunState.Running : AgentRunState.Stopped, true, failed));
}
public Task StartAsync(CancellationToken cancellationToken) => Task.Run(() =>
{
if (Command.Run("launchctl", "print", $"system/{label}").ExitCode != 0)
{
Command.Require("launchctl", "bootstrap", "system", Plist);
}
else
{
Command.Require("launchctl", "kickstart", $"system/{label}");
}
}, cancellationToken);
public async Task StopAsync(CancellationToken cancellationToken)
{
Command.Run("launchctl", "bootout", $"system/{label}");
var deadline = DateTime.UtcNow.AddSeconds(60);
while (Command.Run("launchctl", "print", $"system/{label}").ExitCode == 0 && DateTime.UtcNow < deadline)
{
await Task.Delay(500, cancellationToken);
}
}
}
/// <summary>The watchdog's own systemd unit / launchd daemon: install, remove, start, stop, status, restart.</summary>
public static class UnixWatchdogRegistration
{
private static string Unit => $"/etc/systemd/system/{Branding.WatchdogSystemdUnit}.service";
private static string Plist => $"/Library/LaunchDaemons/{Branding.WatchdogLaunchdLabel}.plist";
public static int Install(string executablePath)
{
SwapPaths.MakeExecutable(executablePath);
if (OperatingSystem.IsMacOS())
{
File.WriteAllText(Plist, $"""
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key><string>{Branding.WatchdogLaunchdLabel}</string>
<key>ProgramArguments</key>
<array><string>{executablePath}</string><string>run</string></array>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
</dict>
</plist>
""");
File.SetUnixFileMode(Plist, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead);
Console.WriteLine($"launchd daemon {Branding.WatchdogLaunchdLabel} installed.");
return 0;
}
File.WriteAllText(Unit, $"""
[Unit]
Description={Branding.WatchdogServiceDisplayName}
After=network-online.target
[Service]
Type=simple
ExecStart={executablePath} run
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
""");
Command.Run("systemctl", "daemon-reload");
Command.Require("systemctl", "enable", Branding.WatchdogSystemdUnit);
Console.WriteLine($"systemd unit {Branding.WatchdogSystemdUnit} installed.");
return 0;
}
public static int Remove()
{
if (OperatingSystem.IsMacOS())
{
Command.Run("launchctl", "bootout", $"system/{Branding.WatchdogLaunchdLabel}");
if (File.Exists(Plist))
{
File.Delete(Plist);
}
Console.WriteLine($"launchd daemon {Branding.WatchdogLaunchdLabel} removed.");
return 0;
}
Command.Run("systemctl", "disable", "--now", Branding.WatchdogSystemdUnit);
if (File.Exists(Unit))
{
File.Delete(Unit);
}
Command.Run("systemctl", "daemon-reload");
Console.WriteLine($"systemd unit {Branding.WatchdogSystemdUnit} removed.");
return 0;
}
public static int Start()
{
if (OperatingSystem.IsMacOS())
{
var loaded = Command.Run("launchctl", "print", $"system/{Branding.WatchdogLaunchdLabel}").ExitCode == 0;
return loaded
? Command.Run("launchctl", "kickstart", $"system/{Branding.WatchdogLaunchdLabel}").ExitCode
: Command.Run("launchctl", "bootstrap", "system", Plist).ExitCode;
}
return Command.Run("systemctl", "start", Branding.WatchdogSystemdUnit).ExitCode;
}
public static int Stop()
{
if (OperatingSystem.IsMacOS())
{
Command.Run("launchctl", "bootout", $"system/{Branding.WatchdogLaunchdLabel}");
return 0;
}
Command.Run("systemctl", "stop", Branding.WatchdogSystemdUnit);
return 0;
}
/// <summary>One machine-readable line (the agent reads it), then a human one.</summary>
public static int Status()
{
bool installed;
string state;
if (OperatingSystem.IsMacOS())
{
installed = File.Exists(Plist);
var (code, output) = Command.Run("launchctl", "print", $"system/{Branding.WatchdogLaunchdLabel}");
state = !installed ? "NotInstalled" : code == 0 && output.Contains("state = running", StringComparison.Ordinal) ? "Running" : "Stopped";
}
else
{
installed = File.Exists(Unit);
var active = Command.Run("systemctl", "is-active", Branding.WatchdogSystemdUnit).Output.Trim();
state = !installed ? "NotInstalled" : active == "active" ? "Running" : "Stopped";
}
Console.WriteLine($"{{\"service\":\"{(OperatingSystem.IsMacOS() ? Branding.WatchdogLaunchdLabel : Branding.WatchdogSystemdUnit)}\",\"installed\":{(installed ? "true" : "false")},\"state\":\"{state}\"}}");
Console.WriteLine($"{Branding.WatchdogServiceDisplayName}: {state}");
return 0;
}
}
/// <summary>
/// Restarts the watchdog onto its new binary through the service manager:
/// systemd queues a restart job (<c>--no-block</c>) and stops this instance
/// itself; launchd's <c>kickstart -k</c> does the same.
/// </summary>
public sealed class UnixWatchdogRestart(WatchdogLog log) : IWatchdogRestart
{
public bool Restart(string newBinary)
{
try
{
var (code, output) = OperatingSystem.IsMacOS()
? Command.Run("launchctl", "kickstart", "-k", $"system/{Branding.WatchdogLaunchdLabel}")
: Command.Run("systemctl", "restart", "--no-block", Branding.WatchdogSystemdUnit);
if (code == 0)
{
return true;
}
log.Warn($"The service manager would not restart the watchdog ({output.Trim()}); the new binary runs at the next start");
return false;
}
catch (Exception exception) when (exception is InvalidOperationException or System.ComponentModel.Win32Exception)
{
log.Warn("The service manager could not restart the watchdog; the new binary runs at the next start", exception);
return false;
}
}
}
@@ -0,0 +1,487 @@
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Runtime.Versioning;
using System.ServiceProcess;
using Shared.Infrastructure.Constants;
using static Agent.Watchdog.Platform.ServiceInterop;
namespace Agent.Watchdog.Platform;
/// <summary>The agent's Windows service, through the Service Control Manager.</summary>
[SupportedOSPlatform("windows")]
public sealed class WindowsAgentService(string serviceName, string agentExecutablePath, WatchdogLog log) : IAgentService
{
private static readonly TimeSpan StateChangeTimeout = TimeSpan.FromSeconds(60);
public Task<AgentServiceStatus> QueryAsync(CancellationToken cancellationToken)
{
using var controller = new ServiceController(serviceName);
ServiceControllerStatus status;
ServiceStartMode startMode;
try
{
status = controller.Status;
startMode = controller.StartType;
}
catch (InvalidOperationException)
{
return Task.FromResult(AgentServiceStatus.NotInstalled);
}
var state = status switch
{
ServiceControllerStatus.Running or ServiceControllerStatus.Paused or ServiceControllerStatus.PausePending
or ServiceControllerStatus.ContinuePending => AgentRunState.Running,
ServiceControllerStatus.StartPending => AgentRunState.Starting,
ServiceControllerStatus.StopPending => AgentRunState.Stopping,
_ => AgentRunState.Stopped,
};
// A stop request leaves exit code 0; a crash or a failed start leaves an error.
var failed = state == AgentRunState.Stopped
&& QueryStatus(serviceName) is { } block
&& block.dwWin32ExitCode is not (0 or ERROR_SERVICE_NEVER_STARTED);
return Task.FromResult(new AgentServiceStatus(state, startMode == ServiceStartMode.Automatic, failed));
}
public Task StartAsync(CancellationToken cancellationToken) => Task.Run(() =>
{
using var controller = new ServiceController(serviceName);
if (controller.Status is ServiceControllerStatus.Running)
{
return;
}
if (controller.Status is ServiceControllerStatus.StopPending)
{
controller.WaitForStatus(ServiceControllerStatus.Stopped, StateChangeTimeout);
}
if (controller.Status is not ServiceControllerStatus.StartPending)
{
controller.Start();
}
controller.WaitForStatus(ServiceControllerStatus.Running, StateChangeTimeout);
log.Info($"Agent service {serviceName} started");
}, cancellationToken);
public async Task StopAsync(CancellationToken cancellationToken)
{
await Task.Run(() =>
{
using var controller = new ServiceController(serviceName);
if (controller.Status is ServiceControllerStatus.Stopped)
{
return;
}
if (controller.Status is ServiceControllerStatus.StartPending)
{
controller.WaitForStatus(ServiceControllerStatus.Running, StateChangeTimeout);
}
if (controller.Status is not ServiceControllerStatus.StopPending)
{
controller.Stop();
}
controller.WaitForStatus(ServiceControllerStatus.Stopped, StateChangeTimeout);
log.Info($"Agent service {serviceName} stopped");
}, cancellationToken);
// The service process can outlive its STOPPED report by a moment, and the
// agent's own children (its remote-shell host is the agent binary) hold
// the binary: wait for them, then end what is left.
await StopLeftoversAsync(cancellationToken);
}
private async Task StopLeftoversAsync(CancellationToken cancellationToken)
{
var name = Path.GetFileNameWithoutExtension(agentExecutablePath);
var deadline = DateTime.UtcNow.AddSeconds(10);
while (true)
{
var leftovers = Process.GetProcessesByName(name).Where(IsAgentBinary).ToList();
if (leftovers.Count == 0)
{
return;
}
if (DateTime.UtcNow >= deadline)
{
foreach (var process in leftovers)
{
try
{
log.Warn($"Ending agent process {process.Id} that outlived the service");
process.Kill(entireProcessTree: true);
}
catch (Exception exception) when (exception is InvalidOperationException or System.ComponentModel.Win32Exception)
{
}
}
return;
}
foreach (var process in leftovers)
{
process.Dispose();
}
await Task.Delay(500, cancellationToken);
}
}
private bool IsAgentBinary(Process process)
{
try
{
return string.Equals(process.MainModule?.FileName, agentExecutablePath, StringComparison.OrdinalIgnoreCase);
}
catch (Exception exception) when (exception is InvalidOperationException or System.ComponentModel.Win32Exception)
{
return false;
}
}
}
/// <summary>
/// The watchdog's own Windows service: LocalSystem, automatic start, restarted
/// by the Service Control Manager when it fails. Registered through the SCM
/// APIs, never sc.exe; idempotent.
/// </summary>
[SupportedOSPlatform("windows")]
public static class WindowsWatchdogRegistration
{
private static readonly TimeSpan StateChangeTimeout = TimeSpan.FromSeconds(60);
private static readonly TimeSpan RecoveryRestartDelay = TimeSpan.FromSeconds(10);
public static int Install(string executablePath)
{
var binaryPath = $"\"{executablePath}\" run";
var scm = OpenSCManager(null, null, SC_MANAGER_ALL_ACCESS);
if (scm == IntPtr.Zero)
{
throw LastError("OpenSCManager (administrator rights required)");
}
try
{
var service = OpenService(scm, Branding.WatchdogServiceName, SERVICE_ALL_ACCESS);
if (service == IntPtr.Zero)
{
service = CreateService(scm, Branding.WatchdogServiceName, Branding.WatchdogServiceDisplayName, SERVICE_ALL_ACCESS,
SERVICE_WIN32_OWN_PROCESS, SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, binaryPath,
loadOrderGroup: null, tagId: IntPtr.Zero, dependencies: null, serviceStartName: null, password: null);
if (service == IntPtr.Zero)
{
throw LastError("CreateService");
}
Console.WriteLine($"Service '{Branding.WatchdogServiceName}' created.");
}
else
{
if (!ChangeServiceConfig(service, SERVICE_WIN32_OWN_PROCESS, SERVICE_AUTO_START, SERVICE_ERROR_NORMAL, binaryPath,
loadOrderGroup: null, tagId: IntPtr.Zero, dependencies: null, serviceStartName: null, password: null,
Branding.WatchdogServiceDisplayName))
{
throw LastError("ChangeServiceConfig");
}
Console.WriteLine($"Service '{Branding.WatchdogServiceName}' updated.");
}
try
{
ApplyMetadata(service);
}
finally
{
CloseServiceHandle(service);
}
return 0;
}
finally
{
CloseServiceHandle(scm);
}
}
private static void ApplyMetadata(IntPtr service)
{
var description = new SERVICE_DESCRIPTION { lpDescription = Branding.WatchdogServiceDescription };
if (!ChangeServiceConfig2(service, SERVICE_CONFIG_DESCRIPTION, ref description))
{
throw LastError("ChangeServiceConfig2(description)");
}
var size = Marshal.SizeOf<SC_ACTION>();
var actionsPtr = Marshal.AllocHGlobal(size * 3);
try
{
for (var i = 0; i < 3; i++)
{
Marshal.StructureToPtr(new SC_ACTION { Type = SC_ACTION_RESTART, Delay = (uint)RecoveryRestartDelay.TotalMilliseconds },
actionsPtr + i * size, false);
}
var failureActions = new SERVICE_FAILURE_ACTIONS
{
dwResetPeriod = (uint)TimeSpan.FromDays(1).TotalSeconds,
cActions = 3,
lpsaActions = actionsPtr,
};
if (!ChangeServiceConfig2(service, SERVICE_CONFIG_FAILURE_ACTIONS, ref failureActions))
{
throw LastError("ChangeServiceConfig2(failure actions)");
}
}
finally
{
Marshal.FreeHGlobal(actionsPtr);
}
}
public static int Remove()
{
if (!Exists())
{
Console.WriteLine($"Service '{Branding.WatchdogServiceName}' is not installed.");
return 0;
}
Stop();
var scm = OpenSCManager(null, null, SC_MANAGER_ALL_ACCESS);
if (scm == IntPtr.Zero)
{
throw LastError("OpenSCManager (administrator rights required)");
}
try
{
var service = OpenService(scm, Branding.WatchdogServiceName, SERVICE_ALL_ACCESS);
if (service == IntPtr.Zero)
{
return 0;
}
try
{
if (!DeleteService(service))
{
throw LastError("DeleteService");
}
}
finally
{
CloseServiceHandle(service);
}
Console.WriteLine($"Service '{Branding.WatchdogServiceName}' removed.");
return 0;
}
finally
{
CloseServiceHandle(scm);
}
}
public static int Start()
{
if (!Exists())
{
Console.Error.WriteLine($"Service '{Branding.WatchdogServiceName}' is not installed.");
return 1;
}
using var controller = new ServiceController(Branding.WatchdogServiceName);
if (controller.Status is ServiceControllerStatus.Running)
{
Console.WriteLine("Service is already running.");
return 0;
}
if (controller.Status is not ServiceControllerStatus.StartPending)
{
controller.Start();
}
controller.WaitForStatus(ServiceControllerStatus.Running, StateChangeTimeout);
Console.WriteLine("Service started.");
return 0;
}
public static int Stop()
{
if (!Exists())
{
Console.WriteLine($"Service '{Branding.WatchdogServiceName}' is not installed.");
return 0;
}
using var controller = new ServiceController(Branding.WatchdogServiceName);
if (controller.Status is ServiceControllerStatus.Stopped)
{
Console.WriteLine("Service is already stopped.");
return 0;
}
if (controller.CanStop)
{
controller.Stop();
}
controller.WaitForStatus(ServiceControllerStatus.Stopped, StateChangeTimeout);
Console.WriteLine("Service stopped.");
return 0;
}
/// <summary>One machine-readable line (the agent reads it), then a human one.</summary>
public static int Status()
{
var installed = Exists();
string state;
if (installed)
{
using var controller = new ServiceController(Branding.WatchdogServiceName);
state = controller.Status.ToString();
}
else
{
state = "NotInstalled";
}
Console.WriteLine($"{{\"service\":\"{Branding.WatchdogServiceName}\",\"installed\":{(installed ? "true" : "false")},\"state\":\"{state}\"}}");
Console.WriteLine($"Service '{Branding.WatchdogServiceDisplayName}': {state}");
return 0;
}
/// <summary>
/// Hidden verb run by a watchdog that replaced its own binary: started from
/// the new binary, it outlives the service process (the SCM kills no
/// children), waits for it to exit and starts the service again.
/// </summary>
public static int RestartAfterExit(int processId)
{
try
{
using var previous = Process.GetProcessById(processId);
previous.WaitForExit(TimeSpan.FromMinutes(2));
}
catch (ArgumentException)
{
// Already gone.
}
for (var attempt = 1; ; attempt++)
{
try
{
using var controller = new ServiceController(Branding.WatchdogServiceName);
controller.WaitForStatus(ServiceControllerStatus.Stopped, TimeSpan.FromSeconds(30));
controller.Start();
controller.WaitForStatus(ServiceControllerStatus.Running, StateChangeTimeout);
return 0;
}
catch (Exception exception) when (exception is InvalidOperationException or System.ServiceProcess.TimeoutException && attempt < 3)
{
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
}
private static bool Exists()
=> ServiceController.GetServices().Any(s =>
string.Equals(s.ServiceName, Branding.WatchdogServiceName, StringComparison.OrdinalIgnoreCase));
}
/// <summary>Restarts the watchdog service onto its new binary (see <see cref="WindowsWatchdogRegistration.RestartAfterExit"/>).</summary>
[SupportedOSPlatform("windows")]
public sealed class WindowsWatchdogRestart(WatchdogLog log) : IWatchdogRestart
{
public bool Restart(string newBinary)
{
try
{
var start = new ProcessStartInfo(newBinary)
{
UseShellExecute = false,
CreateNoWindow = true,
WorkingDirectory = Path.GetDirectoryName(newBinary)!,
};
start.ArgumentList.Add(WatchdogCli.RestartVerb);
start.ArgumentList.Add(Environment.ProcessId.ToString(System.Globalization.CultureInfo.InvariantCulture));
Process.Start(start)?.Dispose();
return true;
}
catch (Exception exception) when (exception is System.ComponentModel.Win32Exception or InvalidOperationException)
{
log.Warn("The watchdog could not arrange its restart; the new binary runs at the next start", exception);
return false;
}
}
}
/// <summary>The watchdog as a Windows service: the loop runs until the SCM stops it, or it replaced itself.</summary>
[SupportedOSPlatform("windows")]
public sealed class WindowsWatchdogHost : ServiceBase
{
private readonly Func<CancellationToken, Task<bool>> run;
private readonly CancellationTokenSource stopping = new();
private Task? loop;
public WindowsWatchdogHost(Func<CancellationToken, Task<bool>> run)
{
this.run = run;
ServiceName = Branding.WatchdogServiceName;
CanStop = true;
CanShutdown = true;
}
protected override void OnShutdown() => OnStop();
protected override void OnStart(string[] args)
{
loop = Task.Run(async () =>
{
bool restart;
try
{
restart = await run(stopping.Token);
}
catch (OperationCanceledException)
{
restart = false;
}
if (!stopping.IsCancellationRequested)
{
// Ended by itself (a restart onto the new binary): stop the
// service from another thread, since OnStop waits for this one.
_ = Task.Run(Stop);
}
return restart;
});
}
protected override void OnStop()
{
stopping.Cancel();
loop?.Wait(TimeSpan.FromSeconds(30));
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
stopping.Dispose();
}
base.Dispose(disposing);
}
}
+3
View File
@@ -0,0 +1,3 @@
using Agent.Watchdog;
return await WatchdogCli.RunAsync(args);
+98
View File
@@ -0,0 +1,98 @@
using System.Security.Cryptography;
using Shared.Contracts.Models;
namespace Agent.Watchdog;
/// <summary>Relative-path rules and file helpers of the swap.</summary>
public static class SwapPaths
{
/// <summary>File names compare as the platform's file system does.</summary>
public static StringComparer Comparer { get; } = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
public static StringComparison Comparison { get; } = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
/// <summary>The swap's own folders and the agent's log folder: never part of a package, never moved.</summary>
public static readonly string[] Reserved = [WatchdogFiles.NextFolder, WatchdogFiles.PreviousFolder, WatchdogFiles.FailedFolder, "Logs"];
/// <summary>
/// A package path the swap accepts: relative, '/'-separated, no empty,
/// "." or ".." segment, no drive or backslash, not in a reserved folder.
/// </summary>
public static bool IsSafe(string path)
=> path.Length is > 0 and < 400
&& !path.Contains('\\')
&& !path.Contains(':')
&& !path.StartsWith('/')
&& path.Split('/').All(segment => segment.Length > 0 && segment != "." && segment != "..")
&& !Reserved.Contains(FirstSegment(path), Comparer);
public static string FirstSegment(string relative)
{
var slash = relative.IndexOf('/');
return slash < 0 ? relative : relative[..slash];
}
public static string Relative(string root, string path) => Path.GetRelativePath(root, path).Replace('\\', '/');
public static string Combine(string root, string relative) => Path.Combine(root, relative.Replace('/', Path.DirectorySeparatorChar));
/// <summary>True when <paramref name="path"/> lies inside <paramref name="folder"/> (not the folder itself).</summary>
public static bool IsInside(string path, string folder)
{
var full = Path.GetFullPath(path);
var root = Path.TrimEndingDirectorySeparator(Path.GetFullPath(folder)) + Path.DirectorySeparatorChar;
return full.StartsWith(root, Comparison) && full.Length > root.Length;
}
public static string Sha256(string path)
{
using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete);
return Convert.ToHexStringLower(SHA256.HashData(stream));
}
/// <summary>A sharing or lock violation: another process has the file open for a moment.</summary>
public static bool IsInUse(IOException exception) => (exception.HResult & 0xFFFF) is 32 or 33;
public static void TryDeleteDirectory(string directory)
{
try
{
if (Directory.Exists(directory))
{
Directory.Delete(directory, recursive: true);
}
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
// Something still holds a file (a replaced watchdog that is still
// running, say); the next start's tidy takes it.
}
}
/// <summary>Removes the folder when nothing is left in it.</summary>
public static void TryDeleteIfEmpty(string directory)
{
try
{
if (Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any())
{
Directory.Delete(directory);
}
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
}
}
/// <summary>rwxr-xr-x for a binary on linux/macOS (a zip made on Windows carries no modes).</summary>
public static void MakeExecutable(string path)
{
if (!OperatingSystem.IsWindows())
{
File.SetUnixFileMode(path,
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute
| UnixFileMode.GroupRead | UnixFileMode.GroupExecute
| UnixFileMode.OtherRead | UnixFileMode.OtherExecute);
}
}
}
+848
View File
@@ -0,0 +1,848 @@
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
namespace Agent.Watchdog;
/// <summary>What one pass of the watchdog did.</summary>
public enum PassResult
{
/// <summary>No upgrade waiting: the agent is supervised this pass.</summary>
Idle,
/// <summary>An upgrade was worked on (or is waiting for its next attempt).</summary>
Worked,
/// <summary>The watchdog replaced its own binary and is restarting onto it: this process exits.</summary>
RestartRequested,
}
/// <summary>
/// Applies an upgrade the agent handed over. The run, journalled before each
/// step so a watchdog stopped half way finishes or undoes it at its next start:
/// <list type="number">
/// <item>take <c>handover.json</c> (it becomes <c>handover.taken.json</c>; the agent sees it picked up);</item>
/// <item>check the staging: in the data folder's staging folder, safe relative paths, the agent binary present,
/// every file there with the size and SHA-256 the hand-over names (otherwise failed: package / hash, nothing touched);</item>
/// <item><c>stopping</c>: stop the agent, copy the files into <c>&lt;install&gt;/.next/&lt;id&gt;/</c> and check them again;
/// a failure here moved nothing: the agent is started again and the run is tried again, three attempts in all,
/// then it ends failed / watchdog;</item>
/// <item><c>swapping</c>: the current files the package replaces (its files, and everything in a top-level folder it ships)
/// move to <c>.previous/&lt;id&gt;/</c>, the new ones in; a move is retried only while the file is in use (10 × 500 ms);</item>
/// <item><c>verifying</c>: start the agent and wait for a beacon of the new version, connected, written after the start,
/// within the hand-over's deadline; three exits of the agent end the wait early;</item>
/// <item><c>committed</c>: the swap folders go, outcome succeeded, then the watchdog replaces its own binary if the package
/// brings a newer one; or <c>rolling-back</c>: the new files go to <c>.failed/&lt;id&gt;/</c>, the previous ones come back,
/// the previous agent starts, and the outcome says rolled-back with the new version's last log lines.</item>
/// </list>
/// The watchdog's own binary is never part of the swap.
/// </summary>
public sealed class UpgradeApplier(
WatchdogOptions options,
IAgentService agent,
IWatchdogClock clock,
WatchdogLog log,
ISmokeTest? smoke = null,
IWatchdogRestart? restart = null)
{
public const int MaxAttempts = 3;
public const int MoveAttempts = 10;
public const int MaxExits = 3;
public const int LogLines = 40;
public static readonly TimeSpan MoveRetryDelay = TimeSpan.FromMilliseconds(500);
/// <summary>After a restart in <c>verifying</c> whose deadline has passed: how long the new version gets to be seen healthy.</summary>
public static readonly TimeSpan RecheckWindow = TimeSpan.FromMinutes(1);
private readonly HashSet<Guid> finished = [];
private DateTimeOffset notBefore = DateTimeOffset.MinValue;
/// <summary>The running watchdog's version: a packaged watchdog replaces it only when newer.</summary>
public string OwnVersion { get; init; } = WatchdogVersion.Current;
private string Install => options.InstallDirectory;
/// <summary>An upgrade waits or is under way: the agent is not supervised meanwhile.</summary>
public bool Pending => File.Exists(options.HandoverPath) || File.Exists(options.JournalPath);
/// <summary>The watchdog's own binary relative to the install folder, when it runs from there.</summary>
private string? SelfRelative => options.SelfPath is { } self && SwapPaths.IsInside(self, Install)
? SwapPaths.Relative(Install, Path.GetFullPath(self))
: null;
private bool IsSelf(string relative) => SelfRelative is { } self && SwapPaths.Comparer.Equals(relative, self);
private string RunPath(string folder, Guid upgradeId) => Path.Combine(Install, folder, upgradeId.ToString("N"));
/// <summary>The candidate a new watchdog binary is tried as, beside the running one.</summary>
private string? Candidate
{
get
{
if (SelfRelative is not { } self)
{
return null;
}
var current = SwapPaths.Combine(Install, self);
return Path.Combine(Path.GetDirectoryName(current)!,
Path.GetFileNameWithoutExtension(current) + ".next" + Path.GetExtension(current));
}
}
/// <summary>One pass: takes a hand-over, or carries a waiting run on. Idle when there is nothing to do.</summary>
public async Task<PassResult> PassAsync(CancellationToken cancellationToken)
{
TakeHandover();
var journal = ReadJournal();
if (journal is null)
{
return PassResult.Idle;
}
if (journal.Phase != WatchdogPhases.Stopping)
{
// Only a run cut short leaves a journal past "stopping" between passes.
return await RecoverAsync(cancellationToken);
}
if (clock.UtcNow < notBefore)
{
return PassResult.Worked;
}
var handover = WatchdogFileStore.Read(options.TakenHandoverPath, WatchdogJsonContext.Default.WatchdogHandover);
if (handover is null || handover.UpgradeId != journal.UpgradeId)
{
log.Error($"The upgrade to {journal.Version} lost its hand-over before any file moved; nothing was changed");
await EnsureAgentRunningAsync(cancellationToken);
Finish(journal, AgentUpgradeStates.Failed, AgentUpgradeReasons.Watchdog, log.RunTail(LogLines));
return PassResult.Worked;
}
return await AttemptAsync(handover, journal, cancellationToken);
}
/// <summary>
/// At start: a run the journal says was under way is finished or undone.
/// stopping → started over (the agent runs meanwhile); swapping or
/// rolling-back → the previous version is put back; verifying → the wait
/// goes on; committed / rolled-back → the tidy-up is finished.
/// </summary>
public async Task<PassResult> RecoverAsync(CancellationToken cancellationToken)
{
var journal = ReadJournal();
if (journal is null)
{
return PassResult.Idle;
}
log.BeginRun();
var handover = WatchdogFileStore.Read(options.TakenHandoverPath, WatchdogJsonContext.Default.WatchdogHandover) is { } taken
&& taken.UpgradeId == journal.UpgradeId
? taken
: null;
switch (journal.Phase)
{
case WatchdogPhases.Stopping:
await EnsureAgentRunningAsync(cancellationToken);
if (handover is null)
{
log.Error($"The upgrade to {journal.Version} was cut short before any file moved and its hand-over is gone; nothing was changed");
Finish(journal, AgentUpgradeStates.Failed, AgentUpgradeReasons.Watchdog, log.RunTail(LogLines));
}
else
{
log.Warn($"The upgrade to {journal.Version} was cut short before any file moved; it starts over (attempt {journal.Attempts + 1} of {MaxAttempts})");
}
return PassResult.Worked;
case WatchdogPhases.Swapping or WatchdogPhases.RollingBack:
log.Warn($"The watchdog stopped part way through the {(journal.Phase == WatchdogPhases.Swapping ? "swap" : "rollback")} to {journal.Version}; {journal.FromVersion} is put back");
await RollBackAsync(journal, AgentUpgradeReasons.Swap, journal.StartedUtc, cancellationToken);
return PassResult.Worked;
case WatchdogPhases.Verifying:
{
var now = clock.UtcNow;
var deadline = journal.DeadlineUtc is { } at && at > now + RecheckWindow ? at : now + RecheckWindow;
log.Warn($"The watchdog restarted while waiting for {journal.Version} to report healthy; it goes on waiting until {deadline:O}");
await EnsureAgentRunningAsync(cancellationToken);
var (healthy, exits) = await WaitHealthyAsync(journal.Version, journal.StartedUtc, deadline, 0, cancellationToken);
if (healthy)
{
return Commit(journal, handover);
}
await RollBackAsync(journal, exits >= MaxExits ? AgentUpgradeReasons.Crashed : AgentUpgradeReasons.HealthDeadline,
journal.StartedUtc, cancellationToken);
return PassResult.Worked;
}
case WatchdogPhases.Committed:
// Cut short after the new version was kept: the outcome is written
// unless it already was, and the tidy-up finished.
if (ReadOutcome()?.UpgradeId != journal.UpgradeId)
{
WriteOutcome(journal, AgentUpgradeStates.Succeeded, null, []);
}
Cleanup(journal);
return PassResult.Worked;
case WatchdogPhases.RolledBack:
if (ReadOutcome()?.UpgradeId != journal.UpgradeId)
{
WriteOutcome(journal, AgentUpgradeStates.RolledBack, AgentUpgradeReasons.Swap, []);
}
Cleanup(journal);
await EnsureAgentRunningAsync(cancellationToken);
return PassResult.Worked;
default:
log.Warn($"Unknown journal phase '{journal.Phase}'; the journal is dropped");
WatchdogFileStore.TryDelete(options.JournalPath);
return PassResult.Worked;
}
}
/// <summary>
/// Removes what runs cut short left in the install folder (<c>.next</c>,
/// <c>.previous</c>, <c>.failed</c>, a candidate watchdog binary) while no
/// upgrade is waiting or under way.
/// </summary>
public void Tidy()
{
if (Pending || File.Exists(options.TakenHandoverPath))
{
return;
}
foreach (var folder in new[] { WatchdogFiles.NextFolder, WatchdogFiles.PreviousFolder, WatchdogFiles.FailedFolder })
{
SwapPaths.TryDeleteDirectory(Path.Combine(Install, folder));
}
if (Candidate is { } candidate && File.Exists(candidate))
{
WatchdogFileStore.TryDelete(candidate);
}
}
/// <summary>handover.json → handover.taken.json and a new journal; a duplicate or unreadable one is set aside.</summary>
private void TakeHandover()
{
if (!File.Exists(options.HandoverPath))
{
return;
}
var handover = WatchdogFileStore.Read(options.HandoverPath, WatchdogJsonContext.Default.WatchdogHandover);
if (handover is null)
{
var id = WatchdogFileStore.ReadUpgradeId(options.HandoverPath);
log.Error($"The upgrade hand-over does not read; it is set aside and nothing was changed (upgrade {id?.ToString() ?? "unknown"})");
WatchdogFileStore.TryDelete(options.HandoverPath);
if (id is { } upgradeId)
{
WatchdogFileStore.Write(options.OutcomePath,
new WatchdogOutcome(upgradeId, AgentUpgradeStates.Failed, AgentUpgradeReasons.Watchdog, "", "", log.RunTail(LogLines), clock.UtcNow),
WatchdogJsonContext.Default.WatchdogOutcome);
}
return;
}
if (ReadJournal() is { } open)
{
// Never applied twice: a run with a journal is finished from the journal.
log.Warn(open.UpgradeId == handover.UpgradeId
? $"Upgrade {handover.UpgradeId} is already under way ({open.Phase}); the repeated hand-over is not applied again"
: $"Upgrade {open.UpgradeId} is under way; hand-over {handover.UpgradeId} is set aside");
WatchdogFileStore.TryDelete(options.HandoverPath);
return;
}
if (finished.Contains(handover.UpgradeId) || ReadOutcome()?.UpgradeId == handover.UpgradeId)
{
log.Warn($"Upgrade {handover.UpgradeId} has already ended; the repeated hand-over is not applied again");
WatchdogFileStore.TryDelete(options.HandoverPath);
return;
}
log.BeginRun();
log.Info($"Upgrade {handover.UpgradeId} handed over: {handover.FromVersion} -> {handover.Version}{(handover.Revert ? " (rollback)" : "")}, {handover.Files.Count} files");
WatchdogFileStore.Write(options.TakenHandoverPath, handover, WatchdogJsonContext.Default.WatchdogHandover);
WatchdogFileStore.TryDelete(options.HandoverPath);
WriteJournal(new WatchdogJournal(
handover.UpgradeId,
handover.Version,
handover.FromVersion,
WatchdogPhases.Stopping,
new Dictionary<string, string>(SwapPaths.Comparer),
handover.Files.Where(f => !IsSelf(f.Path))
.ToDictionary(f => f.Path, f => f.Sha256.ToLowerInvariant(), SwapPaths.Comparer),
clock.UtcNow,
DeadlineUtc: null,
Attempts: 0));
notBefore = DateTimeOffset.MinValue;
}
private async Task<PassResult> AttemptAsync(WatchdogHandover handover, WatchdogJournal journal, CancellationToken cancellationToken)
{
if (Validate(handover) is { } problem)
{
log.Error($"The staged package does not check out; nothing was changed ({problem.Detail})");
Finish(journal, AgentUpgradeStates.Failed, problem.Reason, log.RunTail(LogLines));
return PassResult.Worked;
}
var attempt = journal.Attempts + 1;
journal = journal with { Attempts = attempt, StartedUtc = clock.UtcNow };
WriteJournal(journal);
log.Info($"Applying {handover.Version} (attempt {attempt} of {MaxAttempts}): stopping the agent");
var next = RunPath(WatchdogFiles.NextFolder, handover.UpgradeId);
try
{
await agent.StopAsync(cancellationToken);
Prepare(handover, next);
journal = journal with { Phase = WatchdogPhases.Swapping, Previous = CurrentFiles(journal.Incoming) };
WriteJournal(journal);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
// Nothing has moved: the agent runs again and the run is tried again.
SwapPaths.TryDeleteDirectory(next);
await EnsureAgentRunningAsync(cancellationToken);
if (attempt >= MaxAttempts)
{
log.Error($"The upgrade to {handover.Version} could not start after {attempt} attempts; nothing was changed", exception);
Finish(journal, AgentUpgradeStates.Failed, AgentUpgradeReasons.Watchdog, log.RunTail(LogLines));
return PassResult.Worked;
}
log.Warn($"The upgrade to {handover.Version} could not start (attempt {attempt}); nothing was moved, it is tried again", exception);
notBefore = clock.UtcNow + options.AttemptRetryDelay;
return PassResult.Worked;
}
try
{
await SwapAsync(journal, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Error($"The swap to {handover.Version} failed part way; {handover.FromVersion} is put back", exception);
await RollBackAsync(journal, AgentUpgradeReasons.Swap, journal.StartedUtc, cancellationToken);
return PassResult.Worked;
}
var started = clock.UtcNow;
var deadline = started + (handover.HealthDeadlineSeconds > 0
? TimeSpan.FromSeconds(handover.HealthDeadlineSeconds)
: options.DefaultHealthDeadline);
journal = journal with { Phase = WatchdogPhases.Verifying, DeadlineUtc = deadline };
WriteJournal(journal);
var startFailures = 0;
try
{
await agent.StartAsync(cancellationToken);
log.Info($"Started {handover.Version}; waiting until {deadline:O} for it to report healthy");
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
// A start the service manager refuses counts as an exit; the wait goes on.
log.Warn($"The new agent did not start", exception);
startFailures = 1;
}
var (healthy, exits) = await WaitHealthyAsync(handover.Version, started, deadline, startFailures, cancellationToken);
if (healthy)
{
return Commit(journal, handover);
}
await RollBackAsync(journal, exits >= MaxExits ? AgentUpgradeReasons.Crashed : AgentUpgradeReasons.HealthDeadline, started, cancellationToken);
return PassResult.Worked;
}
/// <summary>The checks on the staging before anything is touched; null when it checks out.</summary>
private (string Reason, string Detail)? Validate(WatchdogHandover handover)
{
if (string.IsNullOrWhiteSpace(handover.StagingPath)
|| !SwapPaths.IsInside(handover.StagingPath, options.StagingDirectory)
|| !Directory.Exists(handover.StagingPath))
{
return (AgentUpgradeReasons.Package, "the staged package is not in the data folder's staging folder");
}
if (handover.Files.Count == 0 || handover.Files.Any(f => !SwapPaths.IsSafe(f.Path)))
{
return (AgentUpgradeReasons.Package, "the hand-over lists no files, or a file with an unsafe path");
}
if (!handover.Files.Any(f => SwapPaths.Comparer.Equals(f.Path, options.AgentExecutable)))
{
return (AgentUpgradeReasons.Package, $"the package has no {options.AgentExecutable}");
}
if (handover.Files.Select(f => f.Path).Distinct(SwapPaths.Comparer).Count() != handover.Files.Count)
{
return (AgentUpgradeReasons.Package, "the hand-over lists a file twice");
}
foreach (var file in handover.Files)
{
var staged = SwapPaths.Combine(handover.StagingPath, file.Path);
if (!File.Exists(staged))
{
return (AgentUpgradeReasons.Package, $"{file.Path} is not staged");
}
if (new FileInfo(staged).Length != file.Size
|| !string.Equals(SwapPaths.Sha256(staged), file.Sha256, StringComparison.OrdinalIgnoreCase))
{
return (AgentUpgradeReasons.Hash, $"{file.Path} does not match its size and SHA-256");
}
}
return null;
}
/// <summary>Copies the staged files into <c>.next/&lt;id&gt;</c> and checks them there, where they are moved from.</summary>
private void Prepare(WatchdogHandover handover, string next)
{
SwapPaths.TryDeleteDirectory(next);
Directory.CreateDirectory(next);
foreach (var file in handover.Files)
{
var target = SwapPaths.Combine(next, file.Path);
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
File.Copy(SwapPaths.Combine(handover.StagingPath, file.Path), target, overwrite: true);
if (new FileInfo(target).Length != file.Size
|| !string.Equals(SwapPaths.Sha256(target), file.Sha256, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidDataException($"{file.Path} changed while it was copied into {WatchdogFiles.NextFolder}");
}
if (SwapPaths.Comparer.Equals(file.Path, options.AgentExecutable) || IsSelf(file.Path))
{
SwapPaths.MakeExecutable(target);
}
}
}
/// <summary>
/// The installed files the package replaces, with their SHA-256: every file
/// it brings that is installed, and every file in a top-level folder it ships
/// (so a module it no longer has goes too). The log folder, the swap's
/// folders and the running watchdog are never part of it; a file the package
/// knows nothing of (an operator's) stays where it is.
/// </summary>
private Dictionary<string, string> CurrentFiles(IReadOnlyDictionary<string, string> incoming)
{
var keys = new HashSet<string>(incoming.Keys, SwapPaths.Comparer);
var folders = new HashSet<string>(incoming.Keys.Where(k => k.Contains('/')).Select(SwapPaths.FirstSegment), SwapPaths.Comparer);
var candidate = Candidate is { } c ? SwapPaths.Relative(Install, c) : null;
var current = new Dictionary<string, string>(SwapPaths.Comparer);
foreach (var path in Directory.EnumerateFiles(Install, "*", SearchOption.AllDirectories))
{
var relative = SwapPaths.Relative(Install, path);
var first = SwapPaths.FirstSegment(relative);
if (SwapPaths.Reserved.Contains(first, SwapPaths.Comparer) || IsSelf(relative) || SwapPaths.Comparer.Equals(relative, candidate))
{
continue;
}
if (keys.Contains(relative) || (relative.Contains('/') && folders.Contains(first)))
{
current[relative] = SwapPaths.Sha256(path);
}
}
return current;
}
/// <summary>The current files into this run's <c>.previous</c>, then every new one out of its <c>.next</c> into place.</summary>
private async Task SwapAsync(WatchdogJournal journal, CancellationToken cancellationToken)
{
var previous = RunPath(WatchdogFiles.PreviousFolder, journal.UpgradeId);
var next = RunPath(WatchdogFiles.NextFolder, journal.UpgradeId);
Directory.CreateDirectory(previous);
foreach (var relative in journal.Previous.Keys)
{
await MoveAsync(SwapPaths.Combine(Install, relative), SwapPaths.Combine(previous, relative), relative, overwrite: false, cancellationToken);
}
foreach (var relative in journal.Incoming.Keys)
{
await MoveAsync(SwapPaths.Combine(next, relative), SwapPaths.Combine(Install, relative), relative, overwrite: false, cancellationToken);
}
log.Info($"Swapped {journal.Incoming.Count} files in ({journal.Previous.Count} set aside); the previous version is kept until the new one is healthy");
}
/// <summary>
/// The new files into this run's <c>.failed</c>, and what <c>.previous</c>
/// holds back into place; safe to repeat after a stop half way. A new file is
/// one that has left <c>.next</c> and is not the previous version.
/// </summary>
private async Task RollBackFilesAsync(WatchdogJournal journal, CancellationToken cancellationToken)
{
var previous = RunPath(WatchdogFiles.PreviousFolder, journal.UpgradeId);
var next = RunPath(WatchdogFiles.NextFolder, journal.UpgradeId);
var failed = RunPath(WatchdogFiles.FailedFolder, journal.UpgradeId);
var previousHashes = new Dictionary<string, string>(journal.Previous, SwapPaths.Comparer);
foreach (var relative in journal.Incoming.Keys)
{
if (File.Exists(SwapPaths.Combine(next, relative)))
{
continue;
}
var path = SwapPaths.Combine(Install, relative);
if (File.Exists(path) && IsNewVersion(journal.Incoming[relative], previousHashes.GetValueOrDefault(relative), path))
{
await MoveAsync(path, SwapPaths.Combine(failed, relative), relative, overwrite: true, cancellationToken);
}
}
var kept = Directory.Exists(previous)
? Directory.EnumerateFiles(previous, "*", SearchOption.AllDirectories).Select(f => SwapPaths.Relative(previous, f)).ToList()
: [];
foreach (var relative in kept)
{
var path = SwapPaths.Combine(Install, relative);
if (File.Exists(path))
{
await MoveAsync(path, SwapPaths.Combine(failed, relative), relative, overwrite: true, cancellationToken);
}
await MoveAsync(SwapPaths.Combine(previous, relative), path, relative, overwrite: false, cancellationToken);
}
log.Info($"The previous version is back in place ({kept.Count} files)");
}
private static bool IsNewVersion(string incoming, string? previous, string path)
{
if (previous is null)
{
return true;
}
// A file both versions share is right either way and stays.
return !string.Equals(incoming, previous, StringComparison.OrdinalIgnoreCase)
&& !string.Equals(SwapPaths.Sha256(path), previous, StringComparison.OrdinalIgnoreCase);
}
/// <summary>One move of the swap, retried only while another process has the file open.</summary>
private async Task MoveAsync(string from, string to, string relative, bool overwrite, CancellationToken cancellationToken)
{
for (var attempt = 1; ; attempt++)
{
try
{
Directory.CreateDirectory(Path.GetDirectoryName(to)!);
File.Move(from, to, overwrite);
return;
}
catch (IOException exception) when (SwapPaths.IsInUse(exception) && attempt < MoveAttempts)
{
log.Info($"{relative} is in use; it is tried again shortly (attempt {attempt})");
await clock.DelayAsync(MoveRetryDelay, cancellationToken);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
throw new IOException($"{relative} could not be moved: {exception.Message}", exception);
}
}
}
/// <summary>
/// Healthy: a beacon of <paramref name="version"/>, connected, written after
/// <paramref name="since"/>, before <paramref name="deadline"/>. Each time the
/// agent is found not running counts as an exit (and it is started again);
/// the third ends the wait.
/// </summary>
private async Task<(bool Healthy, int Exits)> WaitHealthyAsync(
string version, DateTimeOffset since, DateTimeOffset deadline, int exits, CancellationToken cancellationToken)
{
var reported = clock.UtcNow;
while (clock.UtcNow < deadline && exits < MaxExits)
{
var beacon = WatchdogFileStore.Read(options.HealthPath, WatchdogJsonContext.Default.WatchdogHealthBeacon);
if (beacon is { Connected: true }
&& string.Equals(beacon.Version, version, StringComparison.OrdinalIgnoreCase)
&& beacon.AtUtc > since)
{
return (true, exits);
}
if (clock.UtcNow - reported >= TimeSpan.FromMinutes(1))
{
reported = clock.UtcNow;
log.Info($"Waiting for {version} to report healthy; beacon: {Describe(beacon)}");
}
if (!(await agent.QueryAsync(cancellationToken)).IsRunning)
{
exits++;
if (exits >= MaxExits)
{
log.Warn($"{version} exited {exits} times; the wait ends");
break;
}
log.Warn($"{version} is not running (exit {exits}); starting it again");
try
{
await agent.StartAsync(cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Warn("The agent did not start", exception);
exits++;
}
}
await clock.DelayAsync(options.PollInterval, cancellationToken);
}
return (false, exits);
}
private static string Describe(WatchdogHealthBeacon? beacon) => beacon is null
? "none"
: $"version {beacon.Version}, pid {beacon.ProcessId}, connected={beacon.Connected}, at {beacon.AtUtc:O}";
private PassResult Commit(WatchdogJournal journal, WatchdogHandover? handover)
{
journal = journal with { Phase = WatchdogPhases.Committed };
WriteJournal(journal);
log.Info($"{journal.Version} is healthy; the upgrade is kept");
// Last of all, and never part of the swap: the watchdog's own binary.
var replacedSelf = handover is not null ? ReplaceSelf(handover) : null;
Finish(journal, AgentUpgradeStates.Succeeded, null, []);
if (replacedSelf is not null && restart is not null && restart.Restart(replacedSelf))
{
log.Info("The watchdog restarts onto its new binary");
return PassResult.RestartRequested;
}
return PassResult.Worked;
}
private async Task RollBackAsync(WatchdogJournal journal, string reason, DateTimeOffset since, CancellationToken cancellationToken)
{
journal = journal with { Phase = WatchdogPhases.RollingBack };
WriteJournal(journal);
log.Warn($"Rolling back {journal.Version} ({reason}); {journal.FromVersion} is put back. Beacon: "
+ Describe(WatchdogFileStore.Read(options.HealthPath, WatchdogJsonContext.Default.WatchdogHealthBeacon)));
// A version that did not come up healthy is told by its own last log
// lines (taken before it is stopped); a swap that broke, by the watchdog's.
var lastLog = reason is AgentUpgradeReasons.HealthDeadline or AgentUpgradeReasons.Crashed ? TailAgentLog(since) : [];
try
{
await agent.StopAsync(cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Warn("The new agent could not be stopped", exception);
}
var result = AgentUpgradeStates.RolledBack;
try
{
await RollBackFilesAsync(journal, cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Error($"The previous version could not be put back whole", exception);
result = AgentUpgradeStates.Failed;
reason = AgentUpgradeReasons.Swap;
}
// An earlier version may write no beacon; the new one's must not outlive it.
WatchdogFileStore.TryDelete(options.HealthPath);
var restarted = clock.UtcNow;
await EnsureAgentRunningAsync(cancellationToken);
var (back, _) = await WaitHealthyAsync(journal.FromVersion, restarted, restarted + options.RollbackHealthDeadline, 0, cancellationToken);
log.Info(back
? $"{journal.FromVersion} is back and healthy"
: $"{journal.FromVersion} is back; it has not reported healthy within {options.RollbackHealthDeadline.TotalMinutes:0} minutes");
WriteOutcome(journal, result, reason, lastLog.Count > 0 ? lastLog : log.RunTail(LogLines));
WriteJournal(journal with { Phase = WatchdogPhases.RolledBack });
Cleanup(journal);
}
/// <summary>
/// The candidate binary, checked, tried once (<c>version</c>) and swapped in
/// by renaming the running one aside; only when the package's watchdog is
/// newer than this one. The path to restart onto, or null.
/// </summary>
private string? ReplaceSelf(WatchdogHandover handover)
{
if (SelfRelative is not { } self || Candidate is not { } candidate
|| handover.Files.FirstOrDefault(f => SwapPaths.Comparer.Equals(f.Path, self)) is not { } file)
{
return null;
}
var current = SwapPaths.Combine(Install, self);
var incoming = SwapPaths.Combine(RunPath(WatchdogFiles.NextFolder, handover.UpgradeId), self);
try
{
if (!File.Exists(incoming) || string.Equals(SwapPaths.Sha256(current), file.Sha256, StringComparison.OrdinalIgnoreCase))
{
return null;
}
File.Copy(incoming, candidate, overwrite: true);
SwapPaths.MakeExecutable(candidate);
if (!string.Equals(SwapPaths.Sha256(candidate), file.Sha256, StringComparison.OrdinalIgnoreCase))
{
log.Warn("The new watchdog binary changed while it was copied; this one keeps running");
WatchdogFileStore.TryDelete(candidate);
return null;
}
var version = (smoke ?? new ProcessSmokeTest()).Run(candidate);
if (version is null)
{
log.Warn("The new watchdog binary did not run; this one keeps running");
WatchdogFileStore.TryDelete(candidate);
return null;
}
if (!AgentVersions.IsNewer(version, OwnVersion))
{
log.Info($"The package's watchdog ({version}) is not newer than this one ({OwnVersion}); it is kept");
WatchdogFileStore.TryDelete(candidate);
return null;
}
var aside = SwapPaths.Combine(RunPath(WatchdogFiles.PreviousFolder, handover.UpgradeId), self);
Directory.CreateDirectory(Path.GetDirectoryName(aside)!);
File.Move(current, aside, overwrite: true);
File.Move(candidate, current);
log.Info($"The watchdog replaced itself: {OwnVersion} -> {version}");
return current;
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
log.Warn("The watchdog could not replace itself; it keeps its version", exception);
WatchdogFileStore.TryDelete(candidate);
return null;
}
}
private async Task EnsureAgentRunningAsync(CancellationToken cancellationToken)
{
try
{
if (!(await agent.QueryAsync(cancellationToken)).IsRunning)
{
await agent.StartAsync(cancellationToken);
}
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
log.Warn("The agent could not be started", exception);
}
}
/// <summary>Writes the outcome, then the run's files and folders go.</summary>
private void Finish(WatchdogJournal journal, string result, string? reason, IReadOnlyList<string> lastLog)
{
WriteOutcome(journal, result, reason, lastLog);
Cleanup(journal);
}
private void WriteOutcome(WatchdogJournal journal, string result, string? reason, IReadOnlyList<string> lastLog)
{
WatchdogFileStore.Write(options.OutcomePath,
new WatchdogOutcome(journal.UpgradeId, result, reason, journal.Version, journal.FromVersion,
[.. lastLog.Skip(Math.Max(0, lastLog.Count - LogLines))], clock.UtcNow),
WatchdogJsonContext.Default.WatchdogOutcome);
log.Info($"Upgrade {journal.UpgradeId} to {journal.Version} ended: {result}{(reason is null ? "" : $" ({reason})")}");
}
private void Cleanup(WatchdogJournal journal)
{
finished.Add(journal.UpgradeId);
notBefore = DateTimeOffset.MinValue;
foreach (var folder in new[] { WatchdogFiles.NextFolder, WatchdogFiles.PreviousFolder, WatchdogFiles.FailedFolder })
{
SwapPaths.TryDeleteDirectory(RunPath(folder, journal.UpgradeId));
SwapPaths.TryDeleteIfEmpty(Path.Combine(Install, folder));
}
WatchdogFileStore.TryDelete(options.TakenHandoverPath);
WatchdogFileStore.TryDelete(options.JournalPath);
}
/// <summary>The agent's last log lines, from its newest log file written since <paramref name="since"/>.</summary>
private List<string> TailAgentLog(DateTimeOffset since)
{
try
{
var prefix = Path.GetFileNameWithoutExtension(options.AgentExecutable);
var file = Directory.Exists(options.LogDirectory)
? new DirectoryInfo(options.LogDirectory).GetFiles("*.log")
.Where(f => f.Name.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)
&& f.LastWriteTimeUtc >= since.UtcDateTime.AddSeconds(-5))
.OrderByDescending(f => f.LastWriteTimeUtc)
.FirstOrDefault()
: null;
if (file is null)
{
return [];
}
using var stream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
using var reader = new StreamReader(stream);
var lines = new Queue<string>();
while (reader.ReadLine() is { } line)
{
lines.Enqueue(line.Length <= 2000 ? line : line[..2000]);
if (lines.Count > LogLines)
{
lines.Dequeue();
}
}
return [.. lines];
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
return [];
}
}
private WatchdogJournal? ReadJournal()
=> WatchdogFileStore.Read(options.JournalPath, WatchdogJsonContext.Default.WatchdogJournal) is { } journal
? journal with
{
Previous = new Dictionary<string, string>(journal.Previous ?? new Dictionary<string, string>(), SwapPaths.Comparer),
Incoming = new Dictionary<string, string>(journal.Incoming ?? new Dictionary<string, string>(), SwapPaths.Comparer),
}
: null;
private void WriteJournal(WatchdogJournal journal)
=> WatchdogFileStore.Write(options.JournalPath, journal, WatchdogJsonContext.Default.WatchdogJournal);
private WatchdogOutcome? ReadOutcome()
=> WatchdogFileStore.Read(options.OutcomePath, WatchdogJsonContext.Default.WatchdogOutcome);
}
+192
View File
@@ -0,0 +1,192 @@
using System.Globalization;
using System.Runtime.InteropServices;
using System.ServiceProcess;
using Agent.Watchdog.Platform;
using Shared.Infrastructure.Constants;
namespace Agent.Watchdog;
/// <summary>
/// The watchdog's verbs. <c>run</c> is what the service manager starts;
/// <c>foreground</c> runs the same loop with console logging. <c>install</c>,
/// <c>remove</c>, <c>start</c>, <c>stop</c> and <c>status</c> manage its own
/// service (the agent's install verbs call them).
/// </summary>
public static class WatchdogCli
{
/// <summary>Hidden: run by a watchdog that replaced itself (Windows), see <see cref="WindowsWatchdogRegistration.RestartAfterExit"/>.</summary>
public const string RestartVerb = "restart-after-exit";
public static async Task<int> RunAsync(string[] args)
{
var command = args.Length > 0 ? args[0].Trim().ToLowerInvariant() : null;
try
{
switch (command)
{
case "run":
return await RunLoopAsync(args[1..], asService: true);
case "foreground":
return await RunLoopAsync(args[1..], asService: false);
case "version":
Console.WriteLine(WatchdogVersion.Current);
return 0;
case "install":
return OperatingSystem.IsWindows() ? WindowsWatchdogRegistration.Install(SelfPath) : UnixWatchdogRegistration.Install(SelfPath);
case "remove":
return OperatingSystem.IsWindows() ? WindowsWatchdogRegistration.Remove() : UnixWatchdogRegistration.Remove();
case "start":
return OperatingSystem.IsWindows() ? WindowsWatchdogRegistration.Start() : UnixWatchdogRegistration.Start();
case "stop":
return OperatingSystem.IsWindows() ? WindowsWatchdogRegistration.Stop() : UnixWatchdogRegistration.Stop();
case "status":
return OperatingSystem.IsWindows() ? WindowsWatchdogRegistration.Status() : UnixWatchdogRegistration.Status();
case RestartVerb when OperatingSystem.IsWindows() && args.Length > 1
&& int.TryParse(args[1], NumberStyles.None, CultureInfo.InvariantCulture, out var processId):
return WindowsWatchdogRegistration.RestartAfterExit(processId);
case null or "help" or "-h" or "--help" or "/?":
PrintUsage();
return command is null ? 1 : 0;
default:
Console.Error.WriteLine($"Unknown command '{command}'.");
PrintUsage();
return 1;
}
}
catch (Exception exception)
{
Console.Error.WriteLine($"error: {exception.Message}");
return 2;
}
}
private static string SelfPath => Environment.ProcessPath ?? throw new InvalidOperationException("Cannot determine the executable path.");
/// <summary>
/// <c>--install-dir &lt;path&gt;</c> (default: the watchdog's own folder),
/// <c>--data-dir &lt;path&gt;</c> (default: the agent's data folder),
/// <c>--agent service|process</c> (default: service).
/// </summary>
internal static (WatchdogOptions Options, bool ProcessMode) ParseOptions(string[] args)
{
string? Value(string name)
{
for (var i = 0; i < args.Length - 1; i++)
{
if (string.Equals(args[i], name, StringComparison.OrdinalIgnoreCase))
{
return args[i + 1];
}
}
return null;
}
var options = new WatchdogOptions
{
InstallDirectory = Path.GetFullPath(Value("--install-dir") ?? AppContext.BaseDirectory),
DataDirectory = Path.GetFullPath(Value("--data-dir") ?? Branding.AgentProgramDataDirectory),
};
return (options, string.Equals(Value("--agent"), "process", StringComparison.OrdinalIgnoreCase));
}
private static async Task<int> RunLoopAsync(string[] args, bool asService)
{
var (options, processMode) = ParseOptions(args);
var log = new WatchdogLog(options.LogDirectory, console: !asService);
IAgentService agent;
if (processMode)
{
agent = new ProcessAgentService(options, log);
}
else if (OperatingSystem.IsWindows())
{
agent = new WindowsAgentService(Branding.AgentServiceName, Path.Combine(options.InstallDirectory, options.AgentExecutable), log);
}
else if (OperatingSystem.IsMacOS())
{
agent = new LaunchdAgentService(Branding.AgentLaunchdLabel);
}
else
{
agent = new SystemdAgentService(Branding.AgentSystemdUnit);
}
// In the foreground the replaced binary simply takes over at the next start.
IWatchdogRestart? restart = !asService ? null
: OperatingSystem.IsWindows() ? new WindowsWatchdogRestart(log)
: new UnixWatchdogRestart(log);
var clock = SystemClock.Instance;
var applier = new UpgradeApplier(options, agent, clock, log, new ProcessSmokeTest(), restart);
var supervisor = new AgentSupervisor(options, agent, clock, log);
var service = new WatchdogService(options, applier, supervisor, clock, log);
if (processMode)
{
await agent.StartAsync(CancellationToken.None);
}
try
{
if (asService && OperatingSystem.IsWindows())
{
using var host = new WindowsWatchdogHost(service.RunAsync);
ServiceBase.Run(host);
return 0;
}
using var stopping = new CancellationTokenSource();
Console.CancelKeyPress += (_, e) =>
{
e.Cancel = true;
stopping.Cancel();
};
using var sigterm = PosixSignalRegistration.Create(PosixSignal.SIGTERM, context =>
{
context.Cancel = true;
stopping.Cancel();
});
await service.RunAsync(stopping.Token);
return 0;
}
finally
{
(agent as IDisposable)?.Dispose();
}
}
private static void PrintUsage()
{
Console.WriteLine($"{Branding.WatchdogServiceDisplayName} {WatchdogVersion.Current}");
Console.WriteLine();
Console.WriteLine($"Usage: {Branding.WatchdogBinaryName} <command>");
Console.WriteLine();
Console.WriteLine("Commands:");
Console.WriteLine(" run Run as the service (started by the service manager)");
Console.WriteLine(" foreground Run interactively with console logging");
Console.WriteLine(" install Create/update the watchdog service registration");
Console.WriteLine(" remove Stop and remove the watchdog service");
Console.WriteLine(" start Start the watchdog service");
Console.WriteLine(" stop Stop the watchdog service");
Console.WriteLine(" status Show the watchdog service state");
Console.WriteLine(" version Print the build version");
Console.WriteLine();
Console.WriteLine("Options (run/foreground):");
Console.WriteLine(" --install-dir <path> The agent's install folder (default: this binary's folder)");
Console.WriteLine(" --data-dir <path> The agent's data folder (default: the installed one)");
Console.WriteLine(" --agent process Development: run the agent as a child process, not a service");
}
}
+120
View File
@@ -0,0 +1,120 @@
using System.Globalization;
namespace Agent.Watchdog;
/// <summary>
/// The watchdog's own log: one small rolling file in the agent's log folder
/// (<c>ExecuteR-Watchdog.log</c>, rolled at 1 MB, three files kept), the
/// console in the foreground, and the lines of the current upgrade run kept in
/// memory so a failure's outcome can carry them.
/// </summary>
public sealed class WatchdogLog
{
public const long RollBytes = 1024 * 1024;
public const int KeptFiles = 3;
private readonly Lock gate = new();
private readonly string? path;
private readonly bool console;
private readonly List<string> run = [];
private bool fileBroken;
public WatchdogLog(string? directory, bool console, string? name = null)
{
this.console = console;
if (!string.IsNullOrWhiteSpace(directory))
{
name ??= Path.GetFileNameWithoutExtension(Environment.ProcessPath) is { Length: > 0 } exe
&& !exe.Equals("dotnet", StringComparison.OrdinalIgnoreCase)
? exe
: "ExecuteR-Watchdog";
path = Path.Combine(directory, name + ".log");
}
}
/// <summary>Every line written, for tests and the foreground.</summary>
public event Action<string>? Written;
public void Info(string message) => Write("INF", message);
public void Warn(string message, Exception? exception = null) => Write("WRN", With(message, exception));
public void Error(string message, Exception? exception = null) => Write("ERR", With(message, exception));
/// <summary>Starts collecting the lines of an upgrade run.</summary>
public void BeginRun()
{
lock (gate)
{
run.Clear();
}
}
/// <summary>The last <paramref name="count"/> lines of the current run.</summary>
public IReadOnlyList<string> RunTail(int count)
{
lock (gate)
{
return [.. run.Skip(Math.Max(0, run.Count - count))];
}
}
private static string With(string message, Exception? exception)
=> exception is null ? message : $"{message}: {exception.GetBaseException().Message}";
private void Write(string level, string message)
{
var line = $"{DateTimeOffset.UtcNow.ToString("yyyy-MM-dd HH:mm:ss.fff'Z'", CultureInfo.InvariantCulture)} [{level}] {message}";
lock (gate)
{
run.Add(line);
if (run.Count > 400)
{
run.RemoveRange(0, run.Count - 400);
}
if (console)
{
Console.WriteLine(line);
}
if (path is not null && !fileBroken)
{
try
{
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
Roll();
File.AppendAllText(path, line + Environment.NewLine);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
// A log that cannot be written must never stop the watchdog.
fileBroken = true;
Console.Error.WriteLine($"warning: watchdog file logging disabled ({exception.Message})");
}
}
}
Written?.Invoke(line);
}
private void Roll()
{
var current = new FileInfo(path!);
if (!current.Exists || current.Length < RollBytes)
{
return;
}
var stem = Path.Combine(current.DirectoryName!, Path.GetFileNameWithoutExtension(current.Name));
for (var i = KeptFiles - 1; i >= 1; i--)
{
var from = i == 1 ? path! : $"{stem}.{i - 1}.log";
var to = $"{stem}.{i}.log";
if (File.Exists(from))
{
File.Move(from, to, overwrite: true);
}
}
}
}
+55
View File
@@ -0,0 +1,55 @@
using Shared.Contracts.Models;
using Shared.Infrastructure.Constants;
namespace Agent.Watchdog;
/// <summary>What the watchdog watches and where; defaults are the installed layout.</summary>
public sealed record WatchdogOptions
{
/// <summary>The agent's install folder: the watchdog's own folder unless <c>--install-dir</c> says otherwise.</summary>
public required string InstallDirectory { get; init; }
/// <summary>The data folder the agent and the watchdog share (<c>%ProgramData%\ExecuteR</c>, <c>/var/lib/executer</c>).</summary>
public required string DataDirectory { get; init; }
/// <summary>The agent binary's name inside the install folder.</summary>
public string AgentExecutable { get; init; } = Branding.AgentBinaryName;
/// <summary>The running watchdog's own binary: excluded from the agent swap and replaced last.</summary>
public string? SelfPath { get; init; } = Environment.ProcessPath;
public TimeSpan PollInterval { get; init; } = TimeSpan.FromSeconds(2);
/// <summary>A beacon this old while the agent should run gets it restarted (at most once per period).</summary>
public TimeSpan StaleAfter { get; init; } = WatchdogFiles.BeaconStaleAfter;
/// <summary>Used when a hand-over names no health deadline.</summary>
public TimeSpan DefaultHealthDeadline { get; init; } = TimeSpan.FromMinutes(10);
public TimeSpan RollbackHealthDeadline { get; init; } = WatchdogFiles.RollbackHealthDeadline;
/// <summary>How long a failed attempt (nothing moved yet) waits before the next one.</summary>
public TimeSpan AttemptRetryDelay { get; init; } = TimeSpan.FromSeconds(10);
/// <summary>Where the watchdog writes its own rolling log: the agent's log folder.</summary>
public string LogDirectory => Path.Combine(InstallDirectory, "Logs");
public string UpgradeDirectory => Path.Combine(DataDirectory, WatchdogFiles.Folder);
public string StagingDirectory => Path.Combine(UpgradeDirectory, WatchdogFiles.Staging);
public string HandoverPath => Path.Combine(UpgradeDirectory, WatchdogFiles.Handover);
/// <summary>
/// The hand-over once taken: handover.json is moved here as the run starts,
/// so the agent sees it picked up while a retry or a restarted watchdog can
/// still start over from it. Private to the watchdog; goes with the journal.
/// </summary>
public string TakenHandoverPath => Path.Combine(UpgradeDirectory, "handover.taken.json");
public string HealthPath => Path.Combine(UpgradeDirectory, WatchdogFiles.Health);
public string JournalPath => Path.Combine(UpgradeDirectory, WatchdogFiles.Journal);
public string OutcomePath => Path.Combine(UpgradeDirectory, WatchdogFiles.Outcome);
}
+63
View File
@@ -0,0 +1,63 @@
namespace Agent.Watchdog;
/// <summary>
/// The watchdog's loop: at start it finishes or undoes a swap a previous run
/// left half done and tidies the install folder; then, every poll, it carries an
/// upgrade on when one is waiting and otherwise supervises the agent.
/// </summary>
public sealed class WatchdogService(WatchdogOptions options, UpgradeApplier applier, AgentSupervisor supervisor, IWatchdogClock clock, WatchdogLog log)
{
/// <summary>Runs until cancelled, or until the watchdog replaced itself (true: exit so the new binary takes over).</summary>
public async Task<bool> RunAsync(CancellationToken cancellationToken)
{
log.Info($"Watchdog {WatchdogVersion.Current} watching {Path.Combine(options.InstallDirectory, options.AgentExecutable)} (data {options.DataDirectory})");
try
{
if (await applier.RecoverAsync(cancellationToken) == PassResult.RestartRequested)
{
return true;
}
}
catch (Exception exception) when (exception is not OperationCanceledException)
{
log.Error("Recovering an interrupted upgrade failed", exception);
}
applier.Tidy();
while (!cancellationToken.IsCancellationRequested)
{
try
{
switch (await applier.PassAsync(cancellationToken))
{
case PassResult.RestartRequested:
return true;
case PassResult.Idle:
await supervisor.CheckAsync(cancellationToken);
break;
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
break;
}
catch (Exception exception)
{
log.Warn("A watchdog pass failed; the next one tries again", exception);
}
try
{
await clock.DelayAsync(options.PollInterval, cancellationToken);
}
catch (OperationCanceledException)
{
break;
}
}
log.Info("Watchdog stopping");
return false;
}
}
+27
View File
@@ -0,0 +1,27 @@
using System.Reflection;
namespace Agent.Watchdog;
/// <summary>
/// The watchdog's unified version (<c>yyyy.MM.dd.HHmm</c>), read the way the
/// agent's BuildVersion reads its own: the informational version without the
/// short-commit suffix or build metadata.
/// </summary>
public static class WatchdogVersion
{
public static string Current { get; } = Resolve(typeof(WatchdogVersion).Assembly);
public static string Resolve(Assembly assembly)
{
var informational = assembly.GetCustomAttribute<AssemblyInformationalVersionAttribute>()?.InformationalVersion;
if (string.IsNullOrWhiteSpace(informational))
{
return "0.0.0.0";
}
var plus = informational.IndexOf('+');
var trimmed = plus >= 0 ? informational[..plus] : informational;
var dash = trimmed.IndexOf('-');
return dash >= 0 ? trimmed[..dash] : trimmed;
}
}
+61 -7
View File
@@ -46,11 +46,21 @@ public static class AgentCli
Console.WriteLine(BuildVersion.Resolve(typeof(AgentCli).Assembly));
return 0;
// install/initialize also register (and start) the watchdog when
// its binary sits next to the agent; remove takes it out first,
// so it cannot start the agent again meanwhile.
case "install":
return Lifecycle.Install();
{
var installResult = Lifecycle.Install();
return installResult != 0 ? installResult : Watchdog("install");
}
case "remove":
return Lifecycle.Remove();
{
var watchdogResult = Watchdog("remove");
var removeResult = Lifecycle.Remove();
return removeResult != 0 ? removeResult : watchdogResult;
}
case "start":
return Lifecycle.Start();
@@ -59,8 +69,21 @@ public static class AgentCli
return Lifecycle.Stop();
case "initialize":
{
var installResult = Lifecycle.Install();
return installResult != 0 ? installResult : Lifecycle.Start();
if (installResult == 0)
{
installResult = Watchdog("install");
}
if (installResult != 0)
{
return installResult;
}
var startResult = Lifecycle.Start();
return startResult != 0 ? startResult : Watchdog("start");
}
case "status":
return Lifecycle.Status();
@@ -77,6 +100,11 @@ public static class AgentCli
CancellationToken.None);
return 0;
// Hidden: the remote shell's built-in PowerShell console, started
// by the agent under a PTY. Touches no agent config or state.
case Agent.Shell.WindowsShellResolver.HostVerb:
return Agent.Shell.BuiltInShellHost.Run(args[1..]);
// Hidden: invoked by the MSI during explicit full uninstall
// (never during upgrade) to remove credentials and runtime state.
case "purge-credentials":
@@ -113,6 +141,32 @@ public static class AgentCli
#endif
}
/// <summary>
/// Runs one of the watchdog's own service verbs when its binary sits next to
/// the agent (a build without it, or a development run, has none: nothing to do).
/// </summary>
private static int Watchdog(string verb)
{
var binary = Path.Combine(AppContext.BaseDirectory, Branding.WatchdogBinaryName);
if (!File.Exists(binary))
{
return 0;
}
var (exitCode, output) = Agent.Upgrade.WatchdogInstaller.RunVerb(binary, verb);
if (!string.IsNullOrWhiteSpace(output))
{
Console.WriteLine(output.TrimEnd());
}
if (exitCode != 0)
{
Console.Error.WriteLine($"{Branding.WatchdogBinaryName} {verb} exited with {exitCode}");
}
return exitCode;
}
private static int Configure(string[] args)
{
string? url = null;
@@ -195,11 +249,11 @@ public static class AgentCli
Console.WriteLine();
Console.WriteLine("Commands:");
Console.WriteLine(" foreground Run interactively with console logging");
Console.WriteLine(" install Create/update the Windows service registration");
Console.WriteLine(" remove Stop and remove the Windows service");
Console.WriteLine(" install Create/update the agent and watchdog service registrations");
Console.WriteLine(" remove Stop and remove the watchdog and agent services");
Console.WriteLine(" start Start the installed service");
Console.WriteLine(" stop Stop the service");
Console.WriteLine(" initialize install + start");
Console.WriteLine(" stop Stop the service (the watchdog never undoes a stop)");
Console.WriteLine(" initialize install + start (agent and watchdog)");
Console.WriteLine(" status Show service and enrollment state");
Console.WriteLine(" version Print the build version");
}
+48
View File
@@ -4,6 +4,7 @@ using Agent.Inventory;
using Agent.Messaging;
using Agent.PowerShell;
using Agent.Storage;
using Agent.Upgrade;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
using NATS.Client.JetStream;
@@ -26,6 +27,8 @@ public sealed class CommandProcessor(
FileDownloadHandler fileHandler,
JobHistoryStore jobHistory,
AgentConfiguration configuration,
AgentUpgradeHandler upgrades,
AgentHealthState health,
ILogger logger)
{
public async Task RunAsync(CancellationToken cancellationToken)
@@ -52,17 +55,25 @@ public sealed class CommandProcessor(
publisher.ClientId, AgentEventKinds.Ready, "command consumer bound", DateTimeOffset.UtcNow),
cancellationToken: cancellationToken);
// Healthy for the watchdog from here: an upgrade is kept only once
// the new version gets this far.
health.CommandsBound = true;
await foreach (var message in consumer.ConsumeAsync<byte[]>(cancellationToken: cancellationToken))
{
await HandleMessageAsync(message, cancellationToken);
}
health.CommandsBound = false;
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
health.CommandsBound = false;
break;
}
catch (Exception ex)
{
health.CommandsBound = false;
logger.LogWarning(ex,
"Command consumer unavailable ({Message}); retrying in 15s", ex.Message);
try
@@ -110,6 +121,10 @@ public sealed class CommandProcessor(
await HandleFileCommandAsync(envelope, message, cancellationToken);
break;
case MessageTypes.CommandAgentUpgrade:
await HandleUpgradeCommandAsync(envelope, message, cancellationToken);
break;
default:
logger.LogWarning("Unknown command type {MessageType}; terminating delivery", envelope.MessageType);
await message.AckTerminateAsync(cancellationToken: cancellationToken);
@@ -161,6 +176,39 @@ public sealed class CommandProcessor(
cancellationToken);
}
private async Task HandleUpgradeCommandAsync(
MessageEnvelope envelope, INatsJSMsg<byte[]> message, CancellationToken cancellationToken)
{
var command = envelope.GetPayload<AgentUpgradeCommandPayload>();
if (command is null || command.UpgradeId == Guid.Empty || string.IsNullOrWhiteSpace(command.Version))
{
await message.AckTerminateAsync(cancellationToken: cancellationToken);
return;
}
// Idempotent per upgrade: recorded durably before the ack, so a
// redelivery (also to the version that runs after the swap) is skipped.
if (!jobHistory.TryRecordAccepted(command.UpgradeId))
{
logger.LogInformation("Duplicate delivery of upgrade {UpgradeId}; acknowledged without running it again", command.UpgradeId);
await message.AckAsync(cancellationToken: cancellationToken);
return;
}
await message.AckAsync(cancellationToken: cancellationToken);
_ = Task.Run(async () =>
{
try
{
await upgrades.RunAsync(command, cancellationToken);
}
finally
{
jobHistory.MarkCompleted(command.UpgradeId, "AgentUpgrade");
}
}, cancellationToken);
}
private async Task HandleFileCommandAsync(
MessageEnvelope envelope, INatsJSMsg<byte[]> message, CancellationToken cancellationToken)
{
+11 -1
View File
@@ -21,6 +21,16 @@ public sealed record AgentCredentials(
public const int CurrentVersion = 1;
}
/// <summary>Where the agent keeps its enrolled credentials (a seam for tests).</summary>
public interface ICredentialStore
{
AgentCredentials? Load();
void Save(AgentCredentials credentials);
void Delete();
}
/// <summary>
/// Credential persistence in the agent data directory. Windows: a
/// DPAPI(LocalMachine)-encrypted blob, file + directory ACL'd to SYSTEM and
@@ -28,7 +38,7 @@ public sealed record AgentCredentials(
/// a mode-700 directory (the platform's standard secret-at-rest posture).
/// Content is a versioned JSON payload either way.
/// </summary>
public sealed class CredentialStore(ILogger<CredentialStore> logger)
public sealed class CredentialStore(ILogger<CredentialStore> logger) : ICredentialStore
{
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web);
+103 -35
View File
@@ -3,6 +3,7 @@ using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using Agent.Configuration;
using Agent.Identity;
using Agent.Inventory;
using Agent.Storage;
using Microsoft.Extensions.Logging;
using NATS.NKeys;
@@ -19,9 +20,10 @@ namespace Agent.Enrollment;
/// the bootstrap token from the registry.
/// </summary>
public sealed class EnrollmentService(
CredentialStore credentialStore,
ICredentialStore credentialStore,
RuntimeStateStore runtimeState,
IHardwareIdentitySource hardwareIdentity,
IInventoryCollector inventory,
ILogger<EnrollmentService> logger)
{
/// <summary>
@@ -34,43 +36,91 @@ public sealed class EnrollmentService(
var clientId = ResolveClientId();
var stored = credentialStore.Load();
if (stored is not null)
if (stored is not null && stored.ClientId != clientId)
{
if (stored.ClientId != clientId)
{
// Hardware materially changed: treat as a new device (MVP rule).
logger.LogWarning(
"Stored credentials belong to client {StoredClientId} but recomputed identity is {ClientId}; re-enrolling as new device",
stored.ClientId, clientId);
credentialStore.Delete();
}
else if (configuration.HasBootstrapToken)
{
// A token is only present because an administrator just placed
// one (it is erased after every successful enrollment), so it
// is an explicit instruction to re-enroll — this is how an
// orphaned device is brought back.
logger.LogInformation(
"Bootstrap token present for already-enrolled client {ClientId}; re-enrolling", clientId);
credentialStore.Delete();
}
else
{
return stored;
}
// Hardware materially changed: treat as a new device (MVP rule).
logger.LogWarning(
"Stored credentials belong to client {StoredClientId} but recomputed identity is {ClientId}; re-enrolling as new device",
stored.ClientId, clientId);
credentialStore.Delete();
stored = null;
}
if (!configuration.HasBootstrapToken)
{
logger.LogError("Agent is not enrolled and no bootstrap token is configured");
return null;
if (stored is null)
{
logger.LogError("Agent is not enrolled and no bootstrap token is configured");
}
return stored;
}
if (stored is not null)
{
// A token is only present because an administrator just placed one
// (it is erased after every successful enrollment): a reinstall or
// redeploy, or bringing an orphaned device back. Re-enroll, but keep
// the current enrollment until the new one has succeeded.
logger.LogInformation(
"Bootstrap token present for already-enrolled client {ClientId}; re-enrolling", clientId);
}
BootstrapOutcome outcome;
try
{
outcome = await BootstrapAsync(configuration, clientId, cancellationToken);
}
catch (Exception ex) when (stored is not null
&& ex is HttpRequestException or TaskCanceledException
&& !cancellationToken.IsCancellationRequested)
{
// Server unreachable mid-redeploy: an enrolled agent keeps working.
logger.LogWarning(ex, "Re-enrollment could not reach the server");
outcome = BootstrapOutcome.Failed;
}
if (outcome.Credentials is not null || stored is null)
{
return outcome.Credentials;
}
if (outcome.Rejected)
{
// A spent, expired or revoked token must not strand a working agent:
// drop the token (retrying it can only fail) and carry on as before.
logger.LogWarning(
"Bootstrap token rejected; keeping the current enrollment of client {ClientId}", clientId);
runtimeState.ClearBootstrapToken();
}
else
{
logger.LogWarning(
"Re-enrollment did not complete; keeping the current enrollment of client {ClientId} and retrying at the next start",
clientId);
}
return stored;
}
/// <summary>What a bootstrap attempt produced; Rejected = the server refused the token.</summary>
private sealed record BootstrapOutcome(AgentCredentials? Credentials, bool Rejected)
{
public static readonly BootstrapOutcome Failed = new(null, false);
}
/// <summary>
/// One bootstrap against the server with a fresh NKey pair; on success the
/// new credentials replace the stored ones and the token is cleared.
/// </summary>
private async Task<BootstrapOutcome> BootstrapAsync(
AgentConfiguration configuration, string clientId, CancellationToken cancellationToken)
{
var serverUrl = configuration.ServerUrl;
if (serverUrl is null)
{
logger.LogError("Agent cannot enroll: ServerURL is missing or invalid");
return null;
return BootstrapOutcome.Failed;
}
using var userKey = KeyPair.CreatePair(PrefixByte.User);
@@ -114,7 +164,7 @@ public sealed class EnrollmentService(
{
logger.LogError(
"Server certificate is not trusted and the bootstrap token carries no CA fingerprint; refusing to enroll");
return null;
return BootstrapOutcome.Failed;
}
var (status, pinnedChain) = await ServerTrust.FetchAndInstallCaAsync(
@@ -122,7 +172,7 @@ public sealed class EnrollmentService(
if (status != ServerCaStatus.Installed)
{
logger.LogError("Could not anchor server trust ({Status}); refusing to enroll", status);
return null;
return BootstrapOutcome.Failed;
}
// Retry, validating the leaf against the CA we just verified so it
@@ -141,7 +191,9 @@ public sealed class EnrollmentService(
if (!response.IsSuccessStatusCode)
{
logger.LogError("Bootstrap rejected with HTTP {StatusCode}", (int)response.StatusCode);
return null;
// 401/403: the token itself was refused (spent, expired, revoked).
var refused = response.StatusCode is System.Net.HttpStatusCode.Unauthorized or System.Net.HttpStatusCode.Forbidden;
return new BootstrapOutcome(null, refused);
}
result = await response.Content.ReadFromJsonAsync<BootstrapResponse>(JsonDefaults.Options, cancellationToken);
@@ -150,7 +202,7 @@ public sealed class EnrollmentService(
if (result is null || string.IsNullOrWhiteSpace(result.UserJwt))
{
logger.LogError("Bootstrap response was empty or invalid");
return null;
return BootstrapOutcome.Failed;
}
var credentials = new AgentCredentials(
@@ -169,7 +221,7 @@ public sealed class EnrollmentService(
runtimeState.ClearBootstrapToken();
logger.LogInformation("Enrollment {EnrollmentId} complete; bootstrap token cleared", result.EnrollmentId);
return credentials;
return new BootstrapOutcome(credentials, false);
}
private static async Task<HttpResponseMessage> PostBootstrapAsync(
@@ -231,16 +283,32 @@ public sealed class EnrollmentService(
private EnrollDeviceInfo ReadDeviceInfo()
{
var identity = hardwareIdentity.Read();
// The same OS reading inventory reports ("Windows 11", or "Windows PE"
// before imaging), so the device row does not flip between two
// spellings of one OS at every enrollment.
string? osCaption = null;
string? osVersion = null;
try
{
osCaption = Environment.OSVersion.VersionString;
osVersion = Environment.OSVersion.Version.ToString();
var os = inventory.CollectOs();
osCaption = os.Caption;
osVersion = os.Version;
}
catch (InvalidOperationException)
catch (Exception ex)
{
// Non-fatal; inventory supplies full OS details later.
logger.LogDebug(ex, "OS details unavailable for enrollment");
}
if (string.IsNullOrWhiteSpace(osCaption))
{
osCaption = Environment.OSVersion.VersionString;
}
if (string.IsNullOrWhiteSpace(osVersion))
{
osVersion = Environment.OSVersion.Version.ToString();
}
return new EnrollDeviceInfo(
@@ -1,11 +1,15 @@
#if WINDOWS_AGENT
using System.Management;
using System.Runtime.InteropServices;
using Microsoft.Extensions.Logging;
namespace Agent.Identity;
/// <summary>Reads stable hardware identifiers from CIM/WMI via managed APIs.</summary>
public sealed class CimHardwareIdentitySource : IHardwareIdentitySource
public sealed class CimHardwareIdentitySource(ILogger? logger = null) : IHardwareIdentitySource
{
private int wmiUnavailableLogged;
public HardwareIdentity Read()
{
string? uuid = null;
@@ -33,7 +37,7 @@ public sealed class CimHardwareIdentitySource : IHardwareIdentitySource
return new HardwareIdentity(uuid, biosSerial, boardSerial, identifyingNumber, manufacturer, model);
}
private static void Query(string wql, Action<ManagementBaseObject> onFirst)
private void Query(string wql, Action<ManagementBaseObject> onFirst)
{
try
{
@@ -52,6 +56,18 @@ public sealed class CimHardwareIdentitySource : IHardwareIdentitySource
{
// Missing WMI class/values — treated as absent identifiers.
}
catch (Exception ex) when (ex is COMException or PlatformNotSupportedException or TypeInitializationException)
{
// No WMI service at all — typically Windows PE built without the
// WinPE-WMI optional component. Identity cannot be computed, so
// say why once instead of failing with a bare COM error.
if (Interlocked.Exchange(ref wmiUnavailableLogged, 1) == 0)
{
logger?.LogError(
"WMI is unavailable ({Error}); hardware identity cannot be read. In Windows PE, add the WinPE-WMI optional component",
ex.Message);
}
}
}
}
#endif
@@ -6,4 +6,10 @@ namespace Agent.Inventory;
public interface IInventoryCollector
{
InventoryReport Collect(string clientId);
/// <summary>
/// The operating system section alone — also sent at enrollment, so the
/// device reads the same before and after its first inventory.
/// </summary>
InventoryOs CollectOs();
}
+13 -6
View File
@@ -1,13 +1,17 @@
#if WINDOWS_AGENT
using System.Management;
using System.Runtime.InteropServices;
using Agent.Identity;
using Agent.Platform;
using Shared.Contracts.Models;
namespace Agent.Inventory;
/// <summary>Collects the MVP inventory snapshot from CIM/WMI and managed APIs.</summary>
public sealed class InventoryCollector(IHardwareIdentitySource identitySource) : IInventoryCollector
public sealed class InventoryCollector(IHardwareIdentitySource identitySource, WindowsPhase? phase = null) : IInventoryCollector
{
private readonly WindowsPhase phase = phase ?? WindowsPhase.Current;
public InventoryReport Collect(string clientId)
{
var hardware = identitySource.Read();
@@ -72,7 +76,7 @@ public sealed class InventoryCollector(IHardwareIdentitySource identitySource) :
CollectedUtc: DateTimeOffset.UtcNow);
}
private static InventoryOs CollectOs()
public InventoryOs CollectOs()
{
string? rawCaption = null;
string? version = null;
@@ -104,8 +108,10 @@ public sealed class InventoryCollector(IHardwareIdentitySource identitySource) :
version = $"{version}.{ubr}";
}
// "Microsoft Windows 11 Pro" -> caption "Windows 11", edition "Pro".
var (caption, edition) = ParseWindowsCaption(rawCaption);
// "Microsoft Windows 11 Pro" -> caption "Windows 11", edition "Pro";
// Windows PE reads "Windows PE" with its real build version.
var parsed = ParseWindowsCaption(rawCaption);
var (caption, edition) = phase.Describe(parsed.Caption, parsed.Edition);
var productType = productTypeCode switch
{
1 => "Workstation",
@@ -335,9 +341,10 @@ public sealed class InventoryCollector(IHardwareIdentitySource identitySource) :
}
}
}
catch (ManagementException)
catch (Exception ex) when (ex is ManagementException or COMException or PlatformNotSupportedException)
{
// Missing class/namespace — section stays empty.
// Missing class/namespace, or no WMI at all (Windows PE without
// WinPE-WMI) — section stays empty.
}
}
+16 -13
View File
@@ -15,7 +15,6 @@ public sealed class UnixInventoryCollector(IHardwareIdentitySource identitySourc
public InventoryReport Collect(string clientId)
{
var hardware = identitySource.Read();
var (osName, osVersion) = ReadOsRelease();
var identity = new InventoryIdentity(
ClientId: clientId,
@@ -32,8 +31,23 @@ public sealed class UnixInventoryCollector(IHardwareIdentitySource identitySourc
BaseboardManufacturer: ReadTrimmed("/sys/class/dmi/id/board_vendor"),
BaseboardProduct: ReadTrimmed("/sys/class/dmi/id/board_name"));
return new InventoryReport(
Identity: identity,
Os: CollectOs(),
Cpu: new InventoryCpu(ReadCpuName(), Environment.ProcessorCount, Environment.ProcessorCount),
Memory: [],
TotalPhysicalMemoryBytes: ReadTotalMemoryBytes(),
Disks: CollectDisks(),
Volumes: CollectVolumes(),
Network: CollectNetwork(),
CollectedUtc: DateTimeOffset.UtcNow);
}
public InventoryOs CollectOs()
{
var (osName, osVersion) = ReadOsRelease();
var role = DetectEdition(); // Server / Workstation
var os = new InventoryOs(
return new InventoryOs(
Caption: osName,
Version: osVersion,
Build: Environment.OSVersion.Version.ToString(),
@@ -42,17 +56,6 @@ public sealed class UnixInventoryCollector(IHardwareIdentitySource identitySourc
LastBootUtc: ReadLastBoot(),
Edition: role,
ProductType: role);
return new InventoryReport(
Identity: identity,
Os: os,
Cpu: new InventoryCpu(ReadCpuName(), Environment.ProcessorCount, Environment.ProcessorCount),
Memory: [],
TotalPhysicalMemoryBytes: ReadTotalMemoryBytes(),
Disks: CollectDisks(),
Volumes: CollectVolumes(),
Network: CollectNetwork(),
CollectedUtc: DateTimeOffset.UtcNow);
}
/// <summary>
+4 -3
View File
@@ -1,6 +1,7 @@
using Agent.Configuration;
using Agent.Identity;
using Agent.Inventory;
using Microsoft.Extensions.Logging;
namespace Agent.Platform;
@@ -18,16 +19,16 @@ public static class PlatformServices
=> new FileRegistryStore();
#endif
public static IHardwareIdentitySource CreateIdentitySource()
public static IHardwareIdentitySource CreateIdentitySource(ILogger? logger = null)
#if WINDOWS_AGENT
=> new CimHardwareIdentitySource();
=> new CimHardwareIdentitySource(logger);
#else
=> new UnixHardwareIdentitySource();
#endif
public static IInventoryCollector CreateInventoryCollector(IHardwareIdentitySource identitySource)
#if WINDOWS_AGENT
=> new InventoryCollector(identitySource);
=> new InventoryCollector(identitySource, WindowsPhase.Current);
#else
=> new UnixInventoryCollector(identitySource);
#endif
+47
View File
@@ -0,0 +1,47 @@
namespace Agent.Platform;
/// <summary>
/// Which Windows phase the agent runs in. Windows PE (the pre-imaging
/// environment) marks itself with the <c>HKLM\SYSTEM\CurrentControlSet\Control\MiniNT</c>
/// key; there the OS is reported as "Windows PE" with the real build version,
/// so the same device later reports its installed OS normally after imaging.
/// The probe is injectable for tests; linux/macOS are never Windows PE.
/// </summary>
public sealed class WindowsPhase(Func<bool> miniNtPresent)
{
public const string MiniNtKeyPath = @"SYSTEM\CurrentControlSet\Control\MiniNT";
public const string WinPeCaption = "Windows PE";
private readonly Lazy<bool> isWinPe = new(miniNtPresent);
/// <summary>The phase of the running system (probed once).</summary>
public static WindowsPhase Current { get; } = new(ProbeMiniNt);
public bool IsWinPe => isWinPe.Value;
/// <summary>
/// The reported OS caption and edition: "Windows PE" (no edition) in
/// Windows PE — whatever WMI calls it there — otherwise unchanged.
/// </summary>
public (string? Caption, string? Edition) Describe(string? caption, string? edition)
=> IsWinPe ? (WinPeCaption, null) : (caption, edition);
private static bool ProbeMiniNt()
{
#if WINDOWS_AGENT
try
{
using var key = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(MiniNtKeyPath);
return key is not null;
}
catch (Exception)
{
// Unreadable registry: assume a full OS (the normal case).
return false;
}
#else
return false;
#endif
}
}
+10 -2
View File
@@ -114,7 +114,9 @@ public static class ExternalScriptRunner
wrapper.AppendLine($"$__splat[{PsLiteral(name)}] = {PsLiteral(value)}");
}
wrapper.AppendLine($"& '{payloadPath}' @__splat");
// Same rendering as the in-process engine: objects as PowerShell shows
// them, at a width that does not truncate tables.
wrapper.AppendLine($"& '{payloadPath}' @__splat | Out-String -Stream -Width {PowerShellExecutionService.OutputWidth}");
wrapper.AppendLine("exit $LASTEXITCODE");
var wrapperPath = Path.Combine(workDir, "wrapper.ps1");
@@ -207,6 +209,11 @@ public static class ExternalScriptRunner
{
foreach (var (key, value) in environment)
{
// Windows environment names are case-insensitive: assigning
// "ApiKey" over an inherited "APIKEY" keeps the old spelling,
// and bash (case-sensitive) then sees $ApiKey as empty. Drop
// the inherited entry so the script's exact name is used.
process.StartInfo.Environment.Remove(key);
process.StartInfo.Environment[key] = value;
}
}
@@ -215,7 +222,8 @@ public static class ExternalScriptRunner
{
if (e.Data is not null)
{
onRecord(new NormalizedOutputRecord(OutputStreamKind.Output, e.Data));
// Formatted table/list lines are padded to the column width.
onRecord(new NormalizedOutputRecord(OutputStreamKind.Output, e.Data.TrimEnd()));
}
};
process.ErrorDataReceived += (_, e) =>
@@ -60,6 +60,7 @@ public sealed class PowerShellExecutionService
if (!string.IsNullOrWhiteSpace(prelude))
{
powerShell.AddScript(prelude);
AddFormatter(powerShell);
powerShell.AddStatement();
}
@@ -72,6 +73,8 @@ public sealed class PowerShellExecutionService
}
}
AddFormatter(powerShell);
var output = new PSDataCollection<PSObject>();
output.DataAdded += (_, e) => onRecord(PowerShellOutputNormalizer.FromOutput(output[e.Index]));
@@ -123,4 +126,20 @@ public sealed class PowerShellExecutionService
WasCancelled: wasCancelled,
FailureMessage: failureMessage);
}
/// <summary>
/// Width handed to the formatter: wide enough that tables and lists are not
/// cut short with "…" the way an 80/120-column console would.
/// </summary>
internal const int OutputWidth = 4096;
/// <summary>
/// Pipes the statement's output through <c>Out-String -Stream</c>, so pipeline
/// objects arrive as PowerShell would display them (their table or list view,
/// a hashtable's keys and values) instead of a bare ToString() such as
/// "System.Collections.Hashtable". Strings pass through unchanged, one record
/// per line, still streamed as they are produced.
/// </summary>
private static void AddFormatter(System.Management.Automation.PowerShell powerShell)
=> powerShell.AddCommand("Out-String").AddParameter("Stream").AddParameter("Width", OutputWidth);
}
@@ -49,6 +49,11 @@ public static class PowerShellOutputNormalizer
PercentComplete: record.PercentComplete >= 0 ? record.PercentComplete : null,
SecondsRemaining: record.SecondsRemaining >= 0 ? record.SecondsRemaining : null);
/// <summary>
/// Output arrives as formatted lines (see PowerShellExecutionService);
/// table/list lines carry padding out to the column width, so trailing
/// spaces are dropped.
/// </summary>
private static string ToText(PSObject? value)
{
if (value?.BaseObject is null)
@@ -56,6 +61,6 @@ public static class PowerShellOutputNormalizer
return string.Empty;
}
return value.ToString() ?? string.Empty;
return (value.ToString() ?? string.Empty).TrimEnd();
}
}
+4 -1
View File
@@ -1,3 +1,4 @@
using System.Runtime.InteropServices;
using Agent.Messaging;
using Microsoft.Extensions.Logging;
using Shared.Contracts.Messages;
@@ -30,7 +31,9 @@ public sealed class PresencePublisher(
Status: "online",
TimestampUtc: DateTimeOffset.UtcNow,
UptimeSeconds: Environment.TickCount64 / 1000,
ActiveJobs: activeJobCounter());
ActiveJobs: activeJobCounter(),
// The build's RID ("win-x64"): the server picks the upgrade package by it.
Runtime: RuntimeInformation.RuntimeIdentifier);
await publisher.PublishAsync(subject, MessageTypes.AgentPresence, payload, cancellationToken: cancellationToken);
}
+15 -2
View File
@@ -58,11 +58,17 @@ public static class AgentHost
builder.Services.AddSingleton(registry);
builder.Services.AddSingleton<RuntimeStateStore>();
builder.Services.AddSingleton<CredentialStore>();
builder.Services.AddSingleton(Platform.PlatformServices.CreateIdentitySource());
builder.Services.AddSingleton<ICredentialStore, CredentialStore>();
builder.Services.AddSingleton(sp => Platform.PlatformServices.CreateIdentitySource(
sp.GetRequiredService<ILoggerFactory>().CreateLogger("Agent.Identity")));
builder.Services.AddSingleton(sp => Platform.PlatformServices.CreateInventoryCollector(
sp.GetRequiredService<IHardwareIdentitySource>()));
builder.Services.AddSingleton<EnrollmentService>();
builder.Services.AddSingleton<Storage.JobHistoryStore>();
builder.Services.AddSingleton<IAgentSessionFactory, AgentSessionFactory>();
builder.Services.AddSingleton(new Upgrade.AgentRuntimeInfo(runAsService));
builder.Services.AddSingleton<Upgrade.AgentHealthState>();
builder.Services.AddSingleton(Upgrade.UpgradeFolders.Installed);
// Environment variables are development-only and never apply in
// service mode (and never override GPO in any mode).
@@ -73,6 +79,13 @@ public static class AgentHost
// renewals (no-op for a real/edge certificate).
builder.Services.AddHostedService<CaRefreshService>();
// The watchdog's health beacon, on its own timer; only the installed
// service writes it (a foreground run must not speak for the service).
if (runAsService)
{
builder.Services.AddHostedService<Upgrade.HealthBeaconService>();
}
return builder.Build();
}
}
+36 -4
View File
@@ -9,6 +9,7 @@ using Agent.PowerShell;
using Agent.Progress;
using Agent.Shell;
using Agent.Storage;
using Agent.Upgrade;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
@@ -29,15 +30,18 @@ public interface IAgentSession
public sealed class AgentSessionFactory(
IHardwareIdentitySource hardwareIdentity,
JobHistoryStore jobHistory,
AgentRuntimeInfo runtime,
AgentHealthState health,
ILoggerFactory loggerFactory) : IAgentSessionFactory
{
public IAgentSession Create(NatsConnection connection, AgentCredentials credentials, AgentConfiguration configuration)
=> new AgentSession(connection, credentials, configuration, hardwareIdentity, jobHistory, loggerFactory);
=> new AgentSession(connection, credentials, configuration, hardwareIdentity, jobHistory, runtime, health, loggerFactory);
}
/// <summary>
/// Runs the concurrent per-connection components: presence heartbeat,
/// inventory publishing and durable command processing.
/// inventory publishing, durable command processing and the report of an
/// upgrade's outcome.
/// </summary>
public sealed class AgentSession : IAgentSession
{
@@ -45,6 +49,9 @@ public sealed class AgentSession : IAgentSession
private readonly InventoryPublisher inventory;
private readonly CommandProcessor commands;
private readonly ShellCommandListener shell;
private readonly UpgradeOutcomeReporter upgradeOutcomes;
private readonly NatsConnection connection;
private readonly AgentHealthState health;
private readonly ILogger<AgentSession> logger;
private readonly AgentCredentials credentials;
private readonly AgentConfiguration configuration;
@@ -55,8 +62,12 @@ public sealed class AgentSession : IAgentSession
AgentConfiguration configuration,
IHardwareIdentitySource hardwareIdentity,
JobHistoryStore jobHistory,
AgentRuntimeInfo runtime,
AgentHealthState health,
ILoggerFactory loggerFactory)
{
this.connection = connection;
this.health = health;
this.credentials = credentials;
this.configuration = configuration;
logger = loggerFactory.CreateLogger<AgentSession>();
@@ -68,12 +79,29 @@ public sealed class AgentSession : IAgentSession
executionService, publisher, jobHistory, loggerFactory.CreateLogger<PowerShellJobHandler>());
var fileHandler = new FileDownloadHandler(publisher, jobHistory, loggerFactory.CreateLogger<FileDownloadHandler>());
var folders = UpgradeFolders.Installed;
var upgradeEvents = new NatsUpgradeEvents(publisher);
var upgrades = new AgentUpgradeHandler(
new AgentUpgradeContext
{
Folders = folders,
ClientId = credentials.ClientId,
CanSelfUpgrade = runtime.CanSelfUpgrade,
ServerUrl = configuration.ServerUrl,
TlsSkipVerify = configuration.TlsSkipVerify.Value,
},
upgradeEvents,
new WatchdogInstaller(folders.InstallDirectory, loggerFactory.CreateLogger<WatchdogInstaller>()),
loggerFactory.CreateLogger<AgentUpgradeHandler>());
upgradeOutcomes = new UpgradeOutcomeReporter(
folders, upgradeEvents, () => health.Connected, credentials.ClientId, loggerFactory.CreateLogger<UpgradeOutcomeReporter>());
presence = new PresencePublisher(
publisher, () => jobHandler.ActiveJobs, loggerFactory.CreateLogger<PresencePublisher>());
inventory = new InventoryPublisher(
collector, publisher, configuration.InventoryInterval.Value, loggerFactory.CreateLogger<InventoryPublisher>());
commands = new CommandProcessor(
connection, publisher, inventory, jobHandler, fileHandler, jobHistory, configuration,
connection, publisher, inventory, jobHandler, fileHandler, jobHistory, configuration, upgrades, health,
loggerFactory.CreateLogger<CommandProcessor>());
shell = new ShellCommandListener(
connection, publisher, configuration, loggerFactory.CreateLogger<ShellCommandListener>());
@@ -87,6 +115,8 @@ public sealed class AgentSession : IAgentSession
configuration.AllowRemoteShell.Value ? "enabled" : "disabled",
configuration.AllowScriptExecution.Value ? "enabled" : "disabled");
health.Attach(connection);
// Inventory publishes immediately (enrollment/reconnect), then on interval.
try
{
@@ -94,10 +124,12 @@ public sealed class AgentSession : IAgentSession
presence.RunAsync(cancellationToken),
inventory.RunAsync(cancellationToken),
commands.RunAsync(cancellationToken),
shell.RunAsync(cancellationToken));
shell.RunAsync(cancellationToken),
upgradeOutcomes.RunAsync(cancellationToken));
}
finally
{
health.Detach(connection);
shell.Dispose();
}
}
+36
View File
@@ -0,0 +1,36 @@
using System.Management.Automation.Runspaces;
using Agent.PowerShell;
using Microsoft.PowerShell;
using Shared.Infrastructure.Versioning;
namespace Agent.Shell;
/// <summary>
/// The hidden <c>shell-host</c> verb: an interactive PowerShell console on
/// the agent's embedded engine (the SDK's own console host), so a remote shell
/// is available where no PowerShell is installed — Windows PE without the
/// WinPE-PowerShell component, a stripped Server Core. Runs in its own process
/// under the session's PTY; it never starts the agent's workers or reads its
/// configuration and credentials, and exits with the console.
/// </summary>
public static class BuiltInShellHost
{
public static int Run(string[] args)
{
PowerShellRunspaceFactory.EnsureBundledModulesOnPath();
var initialSessionState = InitialSessionState.CreateDefault2();
initialSessionState.ExecutionPolicy = ExecutionPolicy.Bypass;
var banner = $"ExecuteR built-in PowerShell {System.Management.Automation.PSVersionInfo.PSVersion} (agent {BuildVersion.Resolve(typeof(BuiltInShellHost).Assembly)})";
// -NoLogo drops the stock PowerShell banner (and update notice); ours
// greets interactive sessions only, so `shell-host -Command ...` stays clean.
if (args.Length == 0)
{
Console.WriteLine(banner);
}
return ConsoleShell.Start(initialSessionState, banner, helpText: string.Empty, ["-NoLogo", .. args]);
}
}
+31 -24
View File
@@ -32,9 +32,20 @@ public sealed class ShellManager(AgentPublisher publisher, ILogger logger) : IDi
{
CloseCurrent("superseded by new session");
var (fileName, args) = ResolveShellCommand();
var pty = CreatePty();
Guid sessionId = command.SessionId;
ShellLaunch launch;
try
{
launch = ResolveShell(command.Shell);
}
catch (ShellUnavailableException ex)
{
logger.LogWarning("Shell session {SessionId} not opened: {Reason}", sessionId, ex.Message);
PublishState(sessionId, ShellSessionState.Failed, ex.Message);
return;
}
var pty = CreatePty();
var active = new ActiveShell
{
@@ -54,9 +65,10 @@ public sealed class ShellManager(AgentPublisher publisher, ILogger logger) : IDi
try
{
pty.Start(fileName, args, Math.Max(command.Cols, 1), Math.Max(command.Rows, 1));
PublishState(sessionId, ShellSessionState.Open, null);
logger.LogInformation("Shell session {SessionId} opened ({Shell})", sessionId, fileName);
pty.Start(launch.FileName, launch.Args, Math.Max(command.Cols, 1), Math.Max(command.Rows, 1));
PublishState(sessionId, ShellSessionState.Open, null, launch.Kind);
logger.LogInformation("Shell session {SessionId} opened: {Kind} ({Shell}, requested {Requested})",
sessionId, launch.Kind ?? "login shell", launch.FileName, command.Shell ?? ShellKinds.Auto);
}
catch (Exception ex)
{
@@ -152,12 +164,12 @@ public sealed class ShellManager(AgentPublisher publisher, ILogger logger) : IDi
payload);
}
private void PublishState(Guid sessionId, ShellSessionState state, string? message)
private void PublishState(Guid sessionId, ShellSessionState state, string? message, string? shell = null)
{
_ = publisher.PublishAsync(
NatsSubjects.AgentShellState(publisher.ClientId, sessionId),
MessageTypes.ShellState,
new ShellStatePayload(sessionId, state, message));
new ShellStatePayload(sessionId, state, message, shell));
}
private static void DisposeShell(ActiveShell? active, string reason)
@@ -181,24 +193,19 @@ public sealed class ShellManager(AgentPublisher publisher, ILogger logger) : IDi
#endif
#if WINDOWS_AGENT
/// <summary>Prefers PowerShell 7 (pwsh) when present, else Windows PowerShell.</summary>
private static (string FileName, string[] Args) ResolveShellCommand()
{
var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles);
var pwsh = Path.Combine(programFiles, "PowerShell", "7", "pwsh.exe");
if (File.Exists(pwsh))
{
return (pwsh, ["-NoLogo"]);
}
var windowsPowerShell = Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.System),
"WindowsPowerShell", "v1.0", "powershell.exe");
return (windowsPowerShell, ["-NoLogo"]);
}
/// <summary>The requested shell (see <see cref="WindowsShellResolver"/>).</summary>
private static ShellLaunch ResolveShell(string? requested) =>
WindowsShellResolver.Resolve(requested, WindowsShellLocations.FromEnvironment(), File.Exists);
#else
/// <summary>Honors $SHELL, then falls back to bash/zsh/sh.</summary>
private static (string FileName, string[] Args) ResolveShellCommand()
/// <summary>Honors $SHELL, then falls back to bash/zsh/sh; the Windows shell choice does not apply.</summary>
private static ShellLaunch ResolveShell(string? requested)
{
_ = requested;
var (fileName, args) = ResolveLoginShell();
return new ShellLaunch(null, fileName, args);
}
private static (string FileName, string[] Args) ResolveLoginShell()
{
var candidates = new[]
{
+120
View File
@@ -0,0 +1,120 @@
using Shared.Contracts.Models;
namespace Agent.Shell;
/// <summary>A resolved shell: the program the PTY starts, and which <see cref="ShellKinds"/> value it is (null on unix).</summary>
internal sealed record ShellLaunch(string? Kind, string FileName, string[] Args);
/// <summary>An explicitly requested shell is not on this device; the message goes to the operator.</summary>
internal sealed class ShellUnavailableException(string message) : Exception(message);
/// <summary>Where a Windows device keeps its shells (injectable for tests).</summary>
internal sealed record WindowsShellLocations(
string? ProgramFiles,
string? SystemDirectory,
string? ComSpec,
string? SearchPath,
string? AgentExecutable)
{
public static WindowsShellLocations FromEnvironment() => new(
Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles),
Environment.GetFolderPath(Environment.SpecialFolder.System),
Environment.GetEnvironmentVariable("ComSpec"),
Environment.GetEnvironmentVariable("PATH"),
Environment.ProcessPath);
}
/// <summary>
/// Picks the shell a Windows session starts. Automatic takes the first of
/// PowerShell 7, Windows PowerShell, the agent's built-in PowerShell host and
/// cmd.exe that exists; an explicit choice that is missing fails the open.
/// </summary>
internal static class WindowsShellResolver
{
/// <summary>Hidden agent verb that runs the built-in PowerShell console (see <see cref="BuiltInShellHost"/>).</summary>
public const string HostVerb = "shell-host";
public static ShellLaunch Resolve(string? requested, WindowsShellLocations locations, Func<string, bool> fileExists)
{
// Unknown values (a newer backend) fall back to automatic.
switch (ShellKinds.Normalize(requested) ?? ShellKinds.Auto)
{
case ShellKinds.Pwsh:
return FindPwsh(locations, fileExists)
?? throw new ShellUnavailableException("PowerShell 7 (pwsh.exe) is not installed on this device.");
case ShellKinds.WindowsPowerShell:
return FindWindowsPowerShell(locations, fileExists)
?? throw new ShellUnavailableException("Windows PowerShell (powershell.exe) is not installed on this device.");
case ShellKinds.Cmd:
return FindCmd(locations, fileExists)
?? throw new ShellUnavailableException("Command Prompt (cmd.exe) was not found on this device.");
case ShellKinds.BuiltIn:
return FindBuiltIn(locations, fileExists)
?? throw new ShellUnavailableException("The agent's built-in PowerShell is unavailable (agent executable not found).");
default:
return FindPwsh(locations, fileExists)
?? FindWindowsPowerShell(locations, fileExists)
?? FindBuiltIn(locations, fileExists)
?? FindCmd(locations, fileExists)
?? throw new ShellUnavailableException("No shell was found on this device.");
}
}
private static ShellLaunch? FindPwsh(WindowsShellLocations locations, Func<string, bool> fileExists)
{
var candidates = new List<string>();
if (!string.IsNullOrEmpty(locations.ProgramFiles))
{
candidates.Add(Path.Combine(locations.ProgramFiles, "PowerShell", "7", "pwsh.exe"));
}
foreach (var directory in (locations.SearchPath ?? string.Empty).Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries))
{
candidates.Add(Path.Combine(directory.Trim('"'), "pwsh.exe"));
}
var found = candidates.FirstOrDefault(fileExists);
return found is null ? null : new ShellLaunch(ShellKinds.Pwsh, found, ["-NoLogo"]);
}
private static ShellLaunch? FindWindowsPowerShell(WindowsShellLocations locations, Func<string, bool> fileExists)
{
if (string.IsNullOrEmpty(locations.SystemDirectory))
{
return null;
}
var path = Path.Combine(locations.SystemDirectory, "WindowsPowerShell", "v1.0", "powershell.exe");
return fileExists(path) ? new ShellLaunch(ShellKinds.WindowsPowerShell, path, ["-NoLogo"]) : null;
}
private static ShellLaunch? FindCmd(WindowsShellLocations locations, Func<string, bool> fileExists)
{
string?[] candidates =
[
locations.ComSpec,
string.IsNullOrEmpty(locations.SystemDirectory) ? null : Path.Combine(locations.SystemDirectory, "cmd.exe"),
];
var found = candidates.FirstOrDefault(path => !string.IsNullOrEmpty(path) && fileExists(path));
return found is null ? null : new ShellLaunch(ShellKinds.Cmd, found, []);
}
private static ShellLaunch? FindBuiltIn(WindowsShellLocations locations, Func<string, bool> fileExists)
{
var agent = locations.AgentExecutable;
// Only the agent's own apphost knows the verb (not dotnet.exe hosting a dll).
if (string.IsNullOrEmpty(agent)
|| string.Equals(Path.GetFileNameWithoutExtension(agent), "dotnet", StringComparison.OrdinalIgnoreCase)
|| !fileExists(agent))
{
return null;
}
return new ShellLaunch(ShellKinds.BuiltIn, agent, [HostVerb]);
}
}
+96
View File
@@ -0,0 +1,96 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NATS.Client.Core;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Shared.Infrastructure.Versioning;
namespace Agent.Upgrade;
/// <summary>How the agent runs: the installed service, or a foreground/development host.</summary>
public sealed record AgentRuntimeInfo(bool RunningAsService)
{
/// <summary>
/// Only the installed service upgrades itself: run by its service manager
/// from its own binary (not <c>foreground</c>, not <c>dotnet run</c>).
/// </summary>
public bool CanSelfUpgrade => RunningAsService
&& string.Equals(Path.GetFileName(Environment.ProcessPath), Shared.Infrastructure.Constants.Branding.AgentBinaryName,
OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
}
/// <summary>
/// Whether the agent is healthy for the watchdog: its command consumer is bound
/// (the moment it publishes "ready") and its NATS connection is open.
/// </summary>
public sealed class AgentHealthState
{
private volatile NatsConnection? connection;
private volatile bool commandsBound;
public void Attach(NatsConnection current) => connection = current;
public void Detach(NatsConnection current)
{
if (ReferenceEquals(connection, current))
{
connection = null;
commandsBound = false;
}
}
public bool CommandsBound
{
get => commandsBound;
set => commandsBound = value;
}
public bool Connected => commandsBound && connection?.ConnectionState == NatsConnectionState.Open;
}
/// <summary>
/// Writes <c>health.json</c> every <see cref="WatchdogFiles.BeaconInterval"/> on
/// its own timer, so long work (a download, a script) never holds it back. Only
/// the installed service writes it: a foreground run must not speak for it.
/// </summary>
public sealed class HealthBeaconService(UpgradeFolders folders, AgentHealthState health, ILogger<HealthBeaconService> logger) : BackgroundService
{
private static readonly string Version = BuildVersion.Resolve(typeof(HealthBeaconService).Assembly);
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
using var timer = new PeriodicTimer(WatchdogFiles.BeaconInterval);
var failing = false;
do
{
try
{
WatchdogFileStore.Write(folders.Health,
new WatchdogHealthBeacon(Version, Environment.ProcessId, health.Connected, DateTimeOffset.UtcNow),
WatchdogJsonContext.Default.WatchdogHealthBeacon);
failing = false;
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
if (!failing)
{
logger.LogWarning(exception, "Health beacon could not be written to {Path}", folders.Health);
failing = true;
}
}
}
while (await WaitAsync(timer, stoppingToken));
}
private static async Task<bool> WaitAsync(PeriodicTimer timer, CancellationToken cancellationToken)
{
try
{
return await timer.WaitForNextTickAsync(cancellationToken);
}
catch (OperationCanceledException)
{
return false;
}
}
}
+424
View File
@@ -0,0 +1,424 @@
using System.IO.Compression;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using Agent.Messaging;
using Microsoft.Extensions.Logging;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
using Shared.Infrastructure.Constants;
using Shared.Infrastructure.Nats;
using Shared.Infrastructure.Routing;
using Shared.Infrastructure.Versioning;
namespace Agent.Upgrade;
/// <summary>Where upgrade progress and outcomes go: the agent event subject in production, a list in tests.</summary>
public interface IUpgradeEvents
{
Task PublishAsync(AgentUpgradeEventPayload payload, CancellationToken cancellationToken);
}
public sealed class NatsUpgradeEvents(AgentPublisher publisher) : IUpgradeEvents
{
public async Task PublishAsync(AgentUpgradeEventPayload payload, CancellationToken cancellationToken)
=> await publisher.PublishAsync(
NatsSubjects.AgentEvent(publisher.ClientId), MessageTypes.AgentUpgrade, payload, cancellationToken: cancellationToken);
}
/// <summary>At most one upgrade per process at a time, across reconnects (a session is per connection).</summary>
public sealed class UpgradeGate
{
public static UpgradeGate Process { get; } = new();
private int busy;
public bool TryEnter() => Interlocked.CompareExchange(ref busy, 1, 0) == 0;
public void Exit() => Volatile.Write(ref busy, 0);
}
/// <summary>What the upgrade handler works with; every seam a test needs is here.</summary>
public sealed class AgentUpgradeContext
{
public required UpgradeFolders Folders { get; init; }
public required string ClientId { get; init; }
public string CurrentVersion { get; init; } = BuildVersion.Resolve(typeof(AgentUpgradeContext).Assembly);
public string? RuntimeIdentifier { get; init; } = RuntimeInformation.RuntimeIdentifier;
public bool CanSelfUpgrade { get; init; }
public ServerUrl? ServerUrl { get; init; }
public bool TlsSkipVerify { get; init; }
/// <summary>The download's HTTP handler: OS trust (the managed CA was installed there at enrollment), or accept-any when configured.</summary>
public Func<HttpMessageHandler>? CreateHttpHandler { get; init; }
public Func<TimeSpan, CancellationToken, Task> Delay { get; init; } = Task.Delay;
/// <summary>A random whole number of milliseconds in [0, max].</summary>
public Func<int, int> RandomMilliseconds { get; init; } = max => Random.Shared.Next(max + 1);
public Func<DateTimeOffset> UtcNow { get; init; } = () => DateTimeOffset.UtcNow;
public TimeSpan PickupPollInterval { get; init; } = TimeSpan.FromSeconds(2);
public TimeSpan PickupDeadline { get; init; } = WatchdogFiles.HandoverPickupDeadline;
public UpgradeGate Gate { get; init; } = UpgradeGate.Process;
}
/// <summary>
/// Handles <see cref="MessageTypes.CommandAgentUpgrade"/>: refuses what this
/// agent cannot do (not the installed service, another platform's package, an
/// upgrade already pending), reports an upgrade to the running version as done,
/// and otherwise waits its start jitter, downloads the package (size and SHA-256
/// checked), unpacks it into the staging folder, makes sure the watchdog runs,
/// and hands the package over to it. The swap's outcome is reported by whichever
/// version runs afterwards (<see cref="UpgradeOutcomeReporter"/>).
/// </summary>
public sealed class AgentUpgradeHandler(AgentUpgradeContext context, IUpgradeEvents events, IWatchdogInstaller watchdog, ILogger logger)
{
private static readonly TimeSpan DownloadTimeout = TimeSpan.FromMinutes(30);
private UpgradeFolders Folders => context.Folders;
/// <summary>The URL to download from: a server path against the configured server URL, an absolute http(s) URL as is.</summary>
public static Uri? ResolveDownloadUrl(string downloadUrl, ServerUrl? serverUrl)
{
if (string.IsNullOrWhiteSpace(downloadUrl))
{
return null;
}
if (downloadUrl.StartsWith('/') && !downloadUrl.StartsWith("//", StringComparison.Ordinal))
{
return serverUrl?.ApiEndpoint(downloadUrl);
}
return Uri.TryCreate(downloadUrl, UriKind.Absolute, out var absolute)
&& (absolute.Scheme == Uri.UriSchemeHttps || absolute.Scheme == Uri.UriSchemeHttp)
? absolute
: null;
}
public async Task RunAsync(AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
{
try
{
await RunCoreAsync(command, cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
logger.LogInformation("Upgrade {UpgradeId} cancelled by the agent stopping", command.UpgradeId);
}
catch (Exception exception)
{
logger.LogError(exception, "Upgrade {UpgradeId} failed", command.UpgradeId);
await PublishAsync(command, AgentUpgradeStates.Failed, AgentUpgradeReasons.Download, exception.Message, CancellationToken.None);
}
}
private async Task RunCoreAsync(AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
{
logger.LogInformation("Upgrade {UpgradeId} requested: {From} -> {To} ({PackageType})",
command.UpgradeId, context.CurrentVersion, command.Version, command.PackageType);
if (!context.CanSelfUpgrade)
{
await FailAsync(command, AgentUpgradeReasons.Unsupported,
"The agent runs in the foreground or a development host; only the installed service upgrades itself.", cancellationToken);
return;
}
var expected = AgentPackageTypes.ForRuntime(context.RuntimeIdentifier);
if (expected is null || !string.Equals(expected, command.PackageType, StringComparison.OrdinalIgnoreCase))
{
await FailAsync(command, AgentUpgradeReasons.Unsupported,
expected is null
? $"No upgrade package is built for this platform ({context.RuntimeIdentifier})."
: $"This agent ({context.RuntimeIdentifier}) upgrades from the {expected} package, not {command.PackageType}.",
cancellationToken);
return;
}
if (AgentVersions.Compare(context.CurrentVersion, command.Version) == 0)
{
await PublishAsync(command, AgentUpgradeStates.Succeeded, AgentUpgradeReasons.Already,
$"Already running {command.Version}.", cancellationToken);
return;
}
if (Folders.UpgradePending || !context.Gate.TryEnter())
{
await FailAsync(command, AgentUpgradeReasons.Busy, "Another upgrade is under way on this device.", cancellationToken);
return;
}
try
{
await UpgradeAsync(command, cancellationToken);
}
finally
{
context.Gate.Exit();
}
}
private async Task UpgradeAsync(AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
{
// A batch of devices must not start downloading in the same instant:
// wait the jitter before any other work.
if (command.StartJitterSeconds > 0)
{
var jitter = TimeSpan.FromMilliseconds(context.RandomMilliseconds(command.StartJitterSeconds * 1000));
logger.LogInformation("Upgrade {UpgradeId} starts in {Seconds:0.0} s", command.UpgradeId, jitter.TotalSeconds);
await context.Delay(jitter, cancellationToken);
}
var url = ResolveDownloadUrl(command.DownloadUrl, context.ServerUrl);
if (url is null)
{
await FailAsync(command, AgentUpgradeReasons.Download, $"The download URL '{command.DownloadUrl}' cannot be used.", cancellationToken);
return;
}
var package = await DownloadAsync(command, url, cancellationToken);
if (package is null)
{
return;
}
var staging = Path.Combine(Folders.Staging, $"{command.Version}-{command.UpgradeId:N}");
IReadOnlyList<WatchdogFile> files;
try
{
files = Unpack(package, staging);
}
catch (Exception exception) when (exception is IOException or InvalidDataException or UnauthorizedAccessException)
{
TryDeleteDirectory(staging);
await FailAsync(command, AgentUpgradeReasons.Package, $"The package could not be unpacked ({exception.Message}).", cancellationToken);
return;
}
if (!files.Any(f => string.Equals(f.Path, Branding.AgentBinaryName, StringComparison.OrdinalIgnoreCase)))
{
TryDeleteDirectory(staging);
await FailAsync(command, AgentUpgradeReasons.Package, $"The package has no {Branding.AgentBinaryName}.", cancellationToken);
return;
}
if (await watchdog.EnsureRunningAsync(staging, cancellationToken) is { } problem)
{
TryDeleteDirectory(staging);
await FailAsync(command, AgentUpgradeReasons.Watchdog, problem, cancellationToken);
return;
}
var handover = new WatchdogHandover(command.UpgradeId, command.Version, context.CurrentVersion, staging, files,
command.HealthDeadlineSeconds, command.Revert, context.UtcNow());
WatchdogFileStore.Write(Folders.Handover, handover, WatchdogJsonContext.Default.WatchdogHandover);
logger.LogInformation("Upgrade {UpgradeId} handed to the watchdog ({Files} files)", command.UpgradeId, files.Count);
await PublishAsync(command, AgentUpgradeStates.Installing, null, $"Handed to the watchdog ({files.Count} files).", cancellationToken);
await WaitForPickupAsync(command, cancellationToken);
}
/// <summary>The verified package in the downloads folder (reused when already there), or null after a failure was reported.</summary>
private async Task<string?> DownloadAsync(AgentUpgradeCommandPayload command, Uri url, CancellationToken cancellationToken)
{
var sha = command.Sha256.Trim().ToLowerInvariant();
if (sha.Length != 64 || !sha.All(Uri.IsHexDigit))
{
await FailAsync(command, AgentUpgradeReasons.Hash, "The command carries no valid SHA-256.", cancellationToken);
return null;
}
Directory.CreateDirectory(Folders.Downloads);
var path = Path.Combine(Folders.Downloads, sha + ".zip");
if (File.Exists(path) && new FileInfo(path).Length == command.SizeBytes && await HashAsync(path, cancellationToken) == sha)
{
logger.LogInformation("Upgrade {UpgradeId}: package already downloaded", command.UpgradeId);
File.SetLastWriteTimeUtc(path, DateTime.UtcNow);
await PublishAsync(command, AgentUpgradeStates.Downloading, null, "Package already downloaded.", cancellationToken);
return path;
}
await PublishAsync(command, AgentUpgradeStates.Downloading, null, $"Downloading 0% of {Megabytes(command.SizeBytes)}.", cancellationToken);
var partial = path + ".partial";
try
{
using var http = new HttpClient(CreateHandler(), disposeHandler: true) { Timeout = DownloadTimeout };
using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
response.EnsureSuccessStatusCode();
await using (var source = await response.Content.ReadAsStreamAsync(cancellationToken))
await using (var target = new FileStream(partial, FileMode.Create, FileAccess.Write, FileShare.None))
{
var buffer = new byte[81920];
long done = 0;
var reported = 0;
int read;
while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0)
{
await target.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
done += read;
if (done > command.SizeBytes)
{
break;
}
var percent = command.SizeBytes > 0 ? (int)(done * 100 / command.SizeBytes) : 0;
if (percent >= reported + 10 && percent < 100)
{
reported = percent / 10 * 10;
await PublishAsync(command, AgentUpgradeStates.Downloading, null,
$"Downloading {reported}% of {Megabytes(command.SizeBytes)}.", cancellationToken);
}
}
}
}
catch (Exception exception) when (exception is HttpRequestException or IOException
|| (exception is TaskCanceledException && !cancellationToken.IsCancellationRequested))
{
TryDelete(partial);
await FailAsync(command, AgentUpgradeReasons.Download, $"The package could not be downloaded ({exception.Message}).", cancellationToken);
return null;
}
var size = new FileInfo(partial).Length;
var actual = await HashAsync(partial, cancellationToken);
if (size != command.SizeBytes || actual != sha)
{
TryDelete(partial);
await FailAsync(command, AgentUpgradeReasons.Hash,
$"The package does not match: {size} bytes with SHA-256 {actual}, expected {command.SizeBytes} bytes with {sha}.", cancellationToken);
return null;
}
File.Move(partial, path, overwrite: true);
logger.LogInformation("Upgrade {UpgradeId}: package downloaded and verified", command.UpgradeId);
return path;
}
private HttpMessageHandler CreateHandler()
{
if (context.CreateHttpHandler is { } create)
{
return create();
}
// Same trust as enrollment: OS validation (a self-signed server's CA was
// installed into the OS store when the agent enrolled), or accept-any
// when TlsSkipVerify is configured. The system/override proxy applies.
var handler = new HttpClientHandler();
if (context.TlsSkipVerify)
{
handler.ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
return handler;
}
/// <summary>Unpacks the package and lists its files ('/'-separated, with size and SHA-256).</summary>
private static List<WatchdogFile> Unpack(string package, string staging)
{
TryDeleteDirectory(staging);
Directory.CreateDirectory(staging);
// ExtractToDirectory refuses an entry that would land outside the folder.
ZipFile.ExtractToDirectory(package, staging, overwriteFiles: true);
WatchdogInstaller.MakeExecutable(Path.Combine(staging, Branding.AgentBinaryName));
WatchdogInstaller.MakeExecutable(Path.Combine(staging, Branding.WatchdogBinaryName));
var files = new List<WatchdogFile>();
foreach (var path in Directory.EnumerateFiles(staging, "*", SearchOption.AllDirectories))
{
using var stream = File.OpenRead(path);
files.Add(new WatchdogFile(
Path.GetRelativePath(staging, path).Replace('\\', '/'),
stream.Length,
Convert.ToHexStringLower(SHA256.HashData(stream))));
}
return files;
}
/// <summary>Picked up once handover.json is gone; one the watchdog has not taken by the deadline fails the run.</summary>
private async Task WaitForPickupAsync(AgentUpgradeCommandPayload command, CancellationToken cancellationToken)
{
var deadline = context.UtcNow() + context.PickupDeadline;
while (File.Exists(Folders.Handover))
{
if (context.UtcNow() >= deadline)
{
if (WatchdogFileStore.Read(Folders.Journal, WatchdogJsonContext.Default.WatchdogJournal)?.UpgradeId == command.UpgradeId)
{
return;
}
WatchdogFileStore.TryDelete(Folders.Handover);
await FailAsync(command, AgentUpgradeReasons.Watchdog,
$"The watchdog did not pick the upgrade up within {context.PickupDeadline.TotalMinutes:0} minutes.", cancellationToken);
return;
}
await context.Delay(context.PickupPollInterval, cancellationToken);
}
logger.LogInformation("Upgrade {UpgradeId} picked up by the watchdog", command.UpgradeId);
}
private Task FailAsync(AgentUpgradeCommandPayload command, string reason, string detail, CancellationToken cancellationToken)
{
logger.LogWarning("Upgrade {UpgradeId} failed ({Reason}): {Detail}", command.UpgradeId, reason, detail);
return PublishAsync(command, AgentUpgradeStates.Failed, reason, detail, cancellationToken);
}
private async Task PublishAsync(AgentUpgradeCommandPayload command, string state, string? reason, string? detail, CancellationToken cancellationToken)
{
try
{
await events.PublishAsync(new AgentUpgradeEventPayload(
command.UpgradeId, context.ClientId, state, reason, detail, context.CurrentVersion, command.Version, null, context.UtcNow()),
cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
// A dropped connection must not end the upgrade; the outcome is
// reported again by the version that runs afterwards.
logger.LogWarning(exception, "Upgrade {UpgradeId} event {State} could not be published", command.UpgradeId, state);
}
}
private static async Task<string> HashAsync(string path, CancellationToken cancellationToken)
{
await using var stream = File.OpenRead(path);
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, cancellationToken));
}
private static string Megabytes(long bytes) => $"{bytes / 1024d / 1024d:0.0} MB";
private static void TryDelete(string path)
{
try
{
File.Delete(path);
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
}
}
private static void TryDeleteDirectory(string path)
{
try
{
if (Directory.Exists(path))
{
Directory.Delete(path, recursive: true);
}
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
}
}
}
+70
View File
@@ -0,0 +1,70 @@
using Shared.Contracts.Models;
using Shared.Infrastructure.Constants;
namespace Agent.Upgrade;
/// <summary>Where the agent and the watchdog meet (<c>&lt;data&gt;/upgrade/</c>) and where the agent is installed.</summary>
public sealed class UpgradeFolders(string dataDirectory, string installDirectory)
{
/// <summary>The installed layout: the data folder and the running binary's own folder.</summary>
public static UpgradeFolders Installed { get; } = new(Branding.AgentProgramDataDirectory, AppContext.BaseDirectory);
public string InstallDirectory { get; } = installDirectory;
public string Root { get; } = Path.Combine(dataDirectory, WatchdogFiles.Folder);
public string Downloads => Path.Combine(Root, WatchdogFiles.Downloads);
public string Staging => Path.Combine(Root, WatchdogFiles.Staging);
public string Handover => Path.Combine(Root, WatchdogFiles.Handover);
public string Health => Path.Combine(Root, WatchdogFiles.Health);
public string Journal => Path.Combine(Root, WatchdogFiles.Journal);
public string Outcome => Path.Combine(Root, WatchdogFiles.Outcome);
/// <summary>A hand-over, a swap or an outcome not yet reported: another upgrade must wait.</summary>
public bool UpgradePending => File.Exists(Handover) || File.Exists(Journal) || File.Exists(Outcome);
/// <summary>
/// Keeps the newest download and the newest staged package, removes the rest.
/// Never while a hand-over or a swap is under way (the watchdog reads the staging then).
/// </summary>
public void Tidy()
{
if (File.Exists(Handover) || File.Exists(Journal))
{
return;
}
if (Directory.Exists(Downloads))
{
foreach (var file in new DirectoryInfo(Downloads).GetFiles().OrderByDescending(f => f.LastWriteTimeUtc).Skip(1))
{
TryDelete(() => file.Delete());
}
}
if (Directory.Exists(Staging))
{
foreach (var folder in new DirectoryInfo(Staging).GetDirectories().OrderByDescending(d => d.LastWriteTimeUtc).Skip(1))
{
TryDelete(() => folder.Delete(recursive: true));
}
}
}
private static void TryDelete(Action delete)
{
try
{
delete();
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
{
// In use; the next tidy tries again.
}
}
}
+108
View File
@@ -0,0 +1,108 @@
using Microsoft.Extensions.Logging;
using Shared.Contracts.Messages;
using Shared.Contracts.Models;
using Shared.Contracts.Serialization;
namespace Agent.Upgrade;
/// <summary>
/// Reports the outcome the watchdog left (<c>outcome.json</c>) once the agent is
/// connected, then deletes it and tidies old downloads and staging. Whichever
/// version runs after a swap reports it: the new one after a commit (the outcome
/// appears after it connected, so this checks every few seconds), the previous
/// one after a rollback.
/// </summary>
public sealed class UpgradeOutcomeReporter(
UpgradeFolders folders, IUpgradeEvents events, Func<bool> connected, string clientId, ILogger logger)
{
public static readonly TimeSpan CheckInterval = TimeSpan.FromSeconds(5);
private bool tidied;
public async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
if (connected())
{
await ReportPendingAsync(cancellationToken);
}
try
{
await Task.Delay(CheckInterval, cancellationToken);
}
catch (OperationCanceledException)
{
break;
}
}
}
/// <summary>Publishes and deletes a waiting outcome; true when one was reported.</summary>
public async Task<bool> ReportPendingAsync(CancellationToken cancellationToken)
{
var reported = false;
var outcome = WatchdogFileStore.Read(folders.Outcome, WatchdogJsonContext.Default.WatchdogOutcome);
if (outcome is not null)
{
try
{
await events.PublishAsync(ToEvent(outcome, clientId), cancellationToken);
}
catch (Exception exception) when (exception is not OperationCanceledException || !cancellationToken.IsCancellationRequested)
{
// Kept for the next check.
logger.LogWarning(exception, "Upgrade outcome {UpgradeId} could not be published; retrying", outcome.UpgradeId);
return false;
}
logger.LogInformation("Upgrade {UpgradeId} to {Version} reported: {Result} {Reason}",
outcome.UpgradeId, outcome.Version, outcome.Result, outcome.Reason);
WatchdogFileStore.TryDelete(folders.Outcome);
reported = true;
tidied = false;
}
else if (File.Exists(folders.Outcome) && !File.Exists(folders.Journal))
{
// Unreadable, and no run that could still rewrite it: drop it.
logger.LogWarning("Upgrade outcome at {Path} does not read; it is removed", folders.Outcome);
WatchdogFileStore.TryDelete(folders.Outcome);
}
if (!tidied)
{
folders.Tidy();
tidied = true;
}
return reported;
}
public static AgentUpgradeEventPayload ToEvent(WatchdogOutcome outcome, string clientId)
{
var detail = outcome.Result switch
{
AgentUpgradeStates.Succeeded => $"Upgraded from {outcome.FromVersion} to {outcome.Version}.",
AgentUpgradeStates.RolledBack => outcome.Reason switch
{
AgentUpgradeReasons.HealthDeadline => $"{outcome.Version} did not report healthy in time; {outcome.FromVersion} was put back.",
AgentUpgradeReasons.Crashed => $"{outcome.Version} kept exiting; {outcome.FromVersion} was put back.",
AgentUpgradeReasons.Swap => $"The files of {outcome.Version} could not be swapped in; {outcome.FromVersion} was put back.",
_ => $"{outcome.FromVersion} was put back.",
},
_ => outcome.Reason switch
{
AgentUpgradeReasons.Hash => "The staged package did not match its SHA-256; nothing was changed.",
AgentUpgradeReasons.Package => "The staged package was not usable; nothing was changed.",
AgentUpgradeReasons.Swap => "The previous version could not be put back whole.",
_ => "The watchdog could not apply the package; nothing was changed.",
},
};
return new AgentUpgradeEventPayload(outcome.UpgradeId, clientId, outcome.Result, outcome.Reason, detail,
string.IsNullOrEmpty(outcome.FromVersion) ? null : outcome.FromVersion,
string.IsNullOrEmpty(outcome.Version) ? null : outcome.Version,
outcome.LastLog, outcome.FinishedUtc);
}
}
+140
View File
@@ -0,0 +1,140 @@
using System.Diagnostics;
using System.Text.Json;
using Microsoft.Extensions.Logging;
using Shared.Infrastructure.Constants;
namespace Agent.Upgrade;
/// <summary>Makes sure the watchdog is installed and running before an upgrade is handed to it.</summary>
public interface IWatchdogInstaller
{
/// <summary>Null when the watchdog runs; otherwise why it could not be made to.</summary>
Task<string?> EnsureRunningAsync(string stagingDirectory, CancellationToken cancellationToken);
}
/// <summary>
/// The watchdog binary beside the agent, driven through its own verbs
/// (<c>status</c>, <c>install</c>, <c>start</c>, <c>remove</c>): an install from
/// before the watchdog existed gets it from the staged package, so the first
/// upgrade it takes brings the watchdog along.
/// </summary>
public sealed class WatchdogInstaller(string installDirectory, ILogger logger) : IWatchdogInstaller
{
private static readonly TimeSpan VerbTimeout = TimeSpan.FromSeconds(90);
public string BinaryPath => Path.Combine(installDirectory, Branding.WatchdogBinaryName);
public Task<string?> EnsureRunningAsync(string stagingDirectory, CancellationToken cancellationToken) => Task.Run(() =>
{
try
{
if (!File.Exists(BinaryPath))
{
var staged = Path.Combine(stagingDirectory, Branding.WatchdogBinaryName);
if (!File.Exists(staged))
{
return "No watchdog is installed and the package brings none.";
}
File.Copy(staged, BinaryPath, overwrite: false);
MakeExecutable(BinaryPath);
logger.LogInformation("Watchdog installed from the staged package to {Path}", BinaryPath);
}
var (installed, state) = Status(BinaryPath);
if (!installed)
{
var (code, output) = RunVerb(BinaryPath, "install");
if (code != 0)
{
return $"The watchdog service could not be registered ({Trim(output)}).";
}
logger.LogInformation("Watchdog service registered");
}
if (!string.Equals(state, "Running", StringComparison.OrdinalIgnoreCase))
{
var (code, output) = RunVerb(BinaryPath, "start");
if (code != 0)
{
return $"The watchdog service could not be started ({Trim(output)}).";
}
}
(_, state) = Status(BinaryPath);
return string.Equals(state, "Running", StringComparison.OrdinalIgnoreCase)
? null
: $"The watchdog service is {state}, not running.";
}
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException
or System.ComponentModel.Win32Exception or InvalidOperationException)
{
return $"The watchdog could not be set up ({exception.Message}).";
}
}, cancellationToken);
/// <summary>The watchdog's <c>status</c> line: whether its service is registered, and its state.</summary>
public static (bool Installed, string State) Status(string binaryPath)
{
var (_, output) = RunVerb(binaryPath, "status");
var line = output.Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).FirstOrDefault(l => l.StartsWith('{'));
if (line is null)
{
return (false, "Unknown");
}
try
{
using var document = JsonDocument.Parse(line);
var root = document.RootElement;
return (root.TryGetProperty("installed", out var installed) && installed.ValueKind == JsonValueKind.True,
root.TryGetProperty("state", out var state) ? state.GetString() ?? "Unknown" : "Unknown");
}
catch (JsonException)
{
return (false, "Unknown");
}
}
/// <summary>Runs one of the watchdog's verbs and returns its exit code and output.</summary>
public static (int ExitCode, string Output) RunVerb(string binaryPath, string verb)
{
MakeExecutable(binaryPath);
using var process = Process.Start(new ProcessStartInfo(binaryPath, verb)
{
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
WorkingDirectory = Path.GetDirectoryName(binaryPath)!,
}) ?? throw new InvalidOperationException($"{binaryPath} did not start");
var output = process.StandardOutput.ReadToEndAsync();
var error = process.StandardError.ReadToEndAsync();
if (!process.WaitForExit(VerbTimeout))
{
process.Kill(entireProcessTree: true);
return (-1, $"{verb} timed out");
}
return (process.ExitCode, output.GetAwaiter().GetResult() + error.GetAwaiter().GetResult());
}
/// <summary>rwxr-xr-x on linux/macOS: a zip made on Windows carries no file modes.</summary>
public static void MakeExecutable(string path)
{
if (!OperatingSystem.IsWindows() && File.Exists(path))
{
File.SetUnixFileMode(path,
UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute
| UnixFileMode.GroupRead | UnixFileMode.GroupExecute
| UnixFileMode.OtherRead | UnixFileMode.OtherExecute);
}
}
private static string Trim(string output)
{
var text = output.ReplaceLineEndings(" ").Trim();
return text.Length <= 300 ? text : text[..300];
}
}
+57 -4
View File
@@ -17,11 +17,13 @@ public static class DeviceEndpoints
{
app.MapGet(ApiRoutes.Devices, ListAsync).RequirePermission(Permissions.DevicesView);
app.MapGet(ApiRoutes.Device, GetAsync).RequirePermission(Permissions.DevicesView);
app.MapGet(ApiRoutes.DeviceHistory, GetHistoryAsync).RequirePermission(Permissions.DevicesView);
app.MapPost(ApiRoutes.DeviceInventory, RefreshInventoryAsync).RequirePermission(Permissions.DevicesManage);
}
private static async Task<IResult> ListAsync(
AppDbContext db, DeviceScopeService scope, HttpContext context, CancellationToken cancellationToken)
AppDbContext db, Groups.DeviceGroupService groups, DeviceScopeService scope, HttpContext context,
CancellationToken cancellationToken)
{
var devices = await db.Devices.AsNoTracking()
.OrderBy(d => d.Hostname)
@@ -34,8 +36,31 @@ public static class DeviceEndpoints
devices = devices.Where(d => allowed.Contains(d.ClientId)).ToList();
}
// Direct group memberships ride along so the console can draw the
// group tree's counts and filter by node without a call per group.
Dictionary<string, List<Guid>>? groupIds = null;
if (await db.DeviceGroups.AnyAsync(cancellationToken))
{
var membership = await groups.GetMembershipAsync(cancellationToken);
groupIds = new Dictionary<string, List<Guid>>(StringComparer.Ordinal);
foreach (var group in membership.Groups)
{
foreach (var clientId in membership.Direct(group.Id))
{
if (!groupIds.TryGetValue(clientId, out var list))
{
groupIds[clientId] = list = [];
}
list.Add(group.Id);
}
}
}
var now = DateTimeOffset.UtcNow;
return Results.Ok(devices.Select(d => ToDto(d, now)).ToList());
return Results.Ok(devices
.Select(d => ToDto(d, now, groupIds is null ? null : groupIds.GetValueOrDefault(d.ClientId) ?? []))
.ToList());
}
private static async Task<IResult> GetAsync(
@@ -73,6 +98,33 @@ public static class DeviceEndpoints
await groups.GetGroupsForDeviceAsync(clientId, cancellationToken)));
}
/// <summary>The device's phase trail (enrolled, renamed, OS changed), newest first.</summary>
private static async Task<IResult> GetHistoryAsync(
string clientId, int? limit, AppDbContext db, DeviceScopeService scope,
HttpContext context, CancellationToken cancellationToken)
{
if (!SubjectIdentifier.IsValid(clientId))
{
return Results.BadRequest();
}
if (!await scope.IsAllowedAsync(context.User, clientId, cancellationToken)
|| !await db.Devices.AnyAsync(d => d.ClientId == clientId, cancellationToken))
{
return Results.NotFound();
}
var take = Math.Clamp(limit ?? 100, 1, 500);
var events = await db.AuditEvents.AsNoTracking()
.Where(e => e.ClientId == clientId && e.Category == Security.AuditCategories.Device)
.OrderByDescending(e => e.Id)
.Take(take)
.Select(e => new DeviceHistoryEventDto(e.Id, e.TimestampUtc, e.Action, e.Detail))
.ToListAsync(cancellationToken);
return Results.Ok(events);
}
private static async Task<IResult> RefreshInventoryAsync(
string clientId,
AppDbContext db,
@@ -103,7 +155,7 @@ public static class DeviceEndpoints
: Results.Problem(statusCode: StatusCodes.Status503ServiceUnavailable, title: "Messaging unavailable.");
}
internal static DeviceDto ToDto(DeviceEntity device, DateTimeOffset now)
internal static DeviceDto ToDto(DeviceEntity device, DateTimeOffset now, IReadOnlyList<Guid>? groupIds = null)
{
IReadOnlyList<string> addresses = [];
if (!string.IsNullOrEmpty(device.IpAddressesJson))
@@ -125,6 +177,7 @@ public static class DeviceEndpoints
EnrolledUtc: device.EnrolledUtc,
Tags: Groups.DeviceGroupService.ParseTags(device.TagsJson),
IsOrphaned: device.IsOrphaned,
PublicIp: device.PublicIp);
PublicIp: device.PublicIp,
GroupIds: groupIds);
}
}
+106
View File
@@ -0,0 +1,106 @@
using Backend.Data;
using Backend.Security;
namespace Backend.Api;
/// <summary>
/// A device's phase trail. One device (same hardware client id) passes
/// through phases — Windows PE before imaging, then its installed OS, often
/// renamed on the way — and each transition is kept as a "device" audit event
/// on that client id, written only when a reported value actually changes.
/// The rows are added to the caller's context; the caller saves.
/// </summary>
public static class DeviceHistory
{
public const string Enrolled = "device-enrolled";
public const string Renamed = "device-renamed";
public const string OsChanged = "device-os-changed";
/// <summary>The agent moved to a newer version ("2026.10.20.1200 → 2026.10.23.1400").</summary>
public const string AgentUpgraded = "agent-upgraded";
/// <summary>The agent went back to an earlier version: an operator's roll back, or the watchdog undoing an upgrade.</summary>
public const string AgentRolledBack = "agent-rolled-back";
/// <summary>An upgrade failed and the agent stayed where it was (detail carries the reason).</summary>
public const string AgentUpgradeFailed = "agent-upgrade-failed";
/// <summary>
/// Applies a reported hostname and (when <paramref name="reportsOs"/>) OS
/// to the device, recording a rename / OS change against the previous
/// values. A first sighting (nothing known before) records nothing.
/// True when the OS changed (a new phase).
/// </summary>
public static bool Apply(
AppDbContext db,
DeviceEntity device,
string hostname,
bool reportsOs = false,
string? osCaption = null,
string? osVersion = null)
{
var now = DateTimeOffset.UtcNow;
// A case-only difference is the same name, not a rename.
if (!string.IsNullOrWhiteSpace(device.Hostname)
&& !string.IsNullOrWhiteSpace(hostname)
&& !string.Equals(device.Hostname, hostname, StringComparison.OrdinalIgnoreCase))
{
Record(db, device.ClientId, Renamed, $"{device.Hostname} → {hostname}", now);
}
device.Hostname = hostname;
if (!reportsOs)
{
return false;
}
var osChanged = IsSpecific(device.OsCaption)
&& IsSpecific(osCaption)
&& !string.Equals(device.OsCaption, osCaption, StringComparison.OrdinalIgnoreCase);
if (osChanged)
{
Record(db, device.ClientId, OsChanged,
$"{Describe(device.OsCaption, device.OsVersion)} → {Describe(osCaption, osVersion)}", now);
}
device.OsCaption = osCaption;
device.OsVersion = osVersion;
return osChanged;
}
/// <summary>Records an enrollment (first or repeat) with the name and OS it enrolled as.</summary>
public static void RecordEnrolled(AppDbContext db, DeviceEntity device)
=> Record(db, device.ClientId, Enrolled,
$"{device.Hostname} · {Describe(device.OsCaption, device.OsVersion)}", DateTimeOffset.UtcNow);
/// <summary>Records an agent upgrade outcome (one of the Agent* actions).</summary>
public static void RecordAgentUpgrade(AppDbContext db, string clientId, string action, string detail)
=> Record(db, clientId, action, detail, DateTimeOffset.UtcNow);
/// <summary>"Windows PE 10.0.26100"; just the caption when the version is unknown.</summary>
public static string Describe(string? caption, string? version)
{
var text = $"{caption?.Trim()} {version?.Trim()}".Trim();
return text.Length > 0 ? text : "unknown OS";
}
/// <summary>
/// A real OS name rather than nothing, or the generic runtime string older
/// agents sent at enrollment ("Microsoft Windows NT 10.0…", "Unix 6.…") —
/// a transition to or from that is a change of spelling, not of OS.
/// </summary>
private static bool IsSpecific(string? caption)
=> !string.IsNullOrWhiteSpace(caption)
&& !caption.StartsWith("Microsoft Windows NT", StringComparison.OrdinalIgnoreCase)
&& !caption.StartsWith("Unix ", StringComparison.OrdinalIgnoreCase);
private static void Record(AppDbContext db, string clientId, string action, string detail, DateTimeOffset now)
=> db.AuditEvents.Add(new AuditEventEntity
{
TimestampUtc = now,
Category = AuditCategories.Device,
Action = action,
ClientId = clientId,
Detail = detail,
});
}
+10 -212
View File
@@ -8,9 +8,8 @@ using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>
/// Users and roles administration. System roles are immutable; the last user
/// holding the wildcard permission can never be deleted or demoted, so an
/// admin always exists.
/// Roles administration. System roles are immutable. People get roles through
/// role bindings (AccessEndpoints); a role still bound cannot be deleted.
/// </summary>
public static class AccessControlEndpoints
{
@@ -23,18 +22,12 @@ public static class AccessControlEndpoints
app.MapPost(ApiRoutes.AdminRoles, CreateRoleAsync).RequirePermission(Permissions.AccessManage);
app.MapPut(ApiRoutes.AdminRole, UpdateRoleAsync).RequirePermission(Permissions.AccessManage);
app.MapDelete(ApiRoutes.AdminRole, DeleteRoleAsync).RequirePermission(Permissions.AccessManage);
app.MapGet(ApiRoutes.AdminUsers, ListUsersAsync).RequirePermission(Permissions.AccessManage);
app.MapPost(ApiRoutes.AdminUsers, CreateUserAsync).RequirePermission(Permissions.AccessManage);
app.MapPut(ApiRoutes.AdminUser, UpdateUserAsync).RequirePermission(Permissions.AccessManage);
app.MapDelete(ApiRoutes.AdminUser, DeleteUserAsync).RequirePermission(Permissions.AccessManage);
}
private static async Task<IResult> ListRolesAsync(AppDbContext db, CancellationToken cancellationToken)
{
var counts = await db.Users.AsNoTracking()
.Where(u => u.RoleId != null)
.GroupBy(u => u.RoleId!.Value)
var counts = await db.RoleBindings.AsNoTracking()
.GroupBy(b => b.RoleId)
.Select(g => new { RoleId = g.Key, Count = g.Count() })
.ToDictionaryAsync(g => g.RoleId, g => g.Count, cancellationToken);
@@ -102,21 +95,14 @@ public static class AccessControlEndpoints
role.Name = request.Name.Trim();
Apply(role, request, context.User.Identity?.Name, DateTimeOffset.UtcNow);
// Users on this role keep a coherent coarse tier for display.
var tier = Permissions.Tier(request.Permissions.ToHashSet(StringComparer.OrdinalIgnoreCase));
foreach (var user in await db.Users.Where(u => u.RoleId == id).ToListAsync(cancellationToken))
{
user.Role = tier;
}
await db.SaveChangesAsync(cancellationToken);
cache.Invalidate();
await audit.LogAsync(AuditCategories.Admin, "role-updated", context.User.Identity?.Name,
detail: $"role={role.Name}", cancellationToken: cancellationToken);
var userCount = await db.Users.CountAsync(u => u.RoleId == id, cancellationToken);
return Results.Ok(ToDto(role, userCount));
var bindingCount = await db.RoleBindings.CountAsync(b => b.RoleId == id, cancellationToken);
return Results.Ok(ToDto(role, bindingCount));
}
private static async Task<IResult> DeleteRoleAsync(
@@ -134,9 +120,9 @@ public static class AccessControlEndpoints
return Results.Conflict(new { error = "System roles cannot be deleted." });
}
if (await db.Users.AnyAsync(u => u.RoleId == id, cancellationToken))
if (await db.RoleBindings.AnyAsync(b => b.RoleId == id, cancellationToken))
{
return Results.Conflict(new { error = "Reassign the users holding this role first." });
return Results.Conflict(new { error = "Remove the role bindings that grant this role first." });
}
db.Roles.Remove(role);
@@ -149,164 +135,6 @@ public static class AccessControlEndpoints
return Results.NoContent();
}
private static async Task<IResult> ListUsersAsync(AppDbContext db, CancellationToken cancellationToken)
{
var roleNames = await db.Roles.AsNoTracking()
.ToDictionaryAsync(r => r.Id, r => r.Name, cancellationToken);
var users = await db.Users.AsNoTracking().OrderBy(u => u.Username).ToListAsync(cancellationToken);
return Results.Ok(users.Select(u => new UserAccountDto(
u.Id, u.Username, u.DisplayName, u.AuthProvider, u.RoleId,
u.RoleId is { } rid ? roleNames.GetValueOrDefault(rid, u.Role.ToString()) : u.Role.ToString(),
u.CreatedUtc, u.LastLoginUtc)).ToList());
}
private static async Task<IResult> CreateUserAsync(
CreateUserRequest request, AppDbContext db, UserService users, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var username = request.Username.Trim();
if (username.Length is < 1 or > 64)
{
return Results.BadRequest(new { error = "Username must be 1-64 characters." });
}
if (request.Password.Length < 8)
{
return Results.BadRequest(new { error = "Password must be at least 8 characters." });
}
if (await db.Users.AnyAsync(u => u.Username == username, cancellationToken))
{
return Results.Conflict(new { error = "A user with that name already exists." });
}
var role = await db.Roles.AsNoTracking()
.SingleOrDefaultAsync(r => r.Id == request.RoleId, cancellationToken);
if (role is null)
{
return Results.BadRequest(new { error = "Role not found." });
}
var user = new UserEntity
{
Id = Guid.NewGuid(),
Username = username,
DisplayName = string.IsNullOrWhiteSpace(request.DisplayName) ? username : request.DisplayName.Trim(),
Role = Permissions.Tier(
RoleCache.ParseStrings(role.PermissionsJson).ToHashSet(StringComparer.OrdinalIgnoreCase)),
RoleId = role.Id,
AuthProvider = AuthConstants.ProviderLocal,
CreatedUtc = DateTimeOffset.UtcNow,
};
user.PasswordHash = users.HashPassword(user, request.Password);
db.Users.Add(user);
await db.SaveChangesAsync(cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "user-created", context.User.Identity?.Name,
detail: $"user={user.Username} role={role.Name}", cancellationToken: cancellationToken);
return Results.Created(ApiRoutes.For.AdminUser(user.Id), new UserAccountDto(
user.Id, user.Username, user.DisplayName, user.AuthProvider, user.RoleId, role.Name,
user.CreatedUtc, user.LastLoginUtc));
}
private static async Task<IResult> UpdateUserAsync(
Guid id, UpdateUserRequest request, AppDbContext db, UserService users, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var user = await db.Users.SingleOrDefaultAsync(u => u.Id == id, cancellationToken);
if (user is null)
{
return Results.NotFound();
}
RoleEntity? role = null;
if (request.RoleId is { } roleId && roleId != user.RoleId)
{
role = await db.Roles.AsNoTracking().SingleOrDefaultAsync(r => r.Id == roleId, cancellationToken);
if (role is null)
{
return Results.BadRequest(new { error = "Role not found." });
}
var demotingFromAdmin = await HoldsWildcardAsync(db, user, cancellationToken)
&& !RoleCache.ParseStrings(role.PermissionsJson).Contains(Permissions.All);
if (demotingFromAdmin && await WildcardHolderCountAsync(db, cancellationToken) <= 1)
{
return Results.Conflict(new { error = "At least one user must keep a full-access role." });
}
user.RoleId = role.Id;
user.Role = Permissions.Tier(
RoleCache.ParseStrings(role.PermissionsJson).ToHashSet(StringComparer.OrdinalIgnoreCase));
}
if (request.DisplayName is not null)
{
user.DisplayName = string.IsNullOrWhiteSpace(request.DisplayName)
? user.Username
: request.DisplayName.Trim();
}
if (!string.IsNullOrEmpty(request.Password))
{
if (user.AuthProvider != AuthConstants.ProviderLocal)
{
return Results.BadRequest(new { error = "Passwords apply to local accounts only." });
}
if (request.Password.Length < 8)
{
return Results.BadRequest(new { error = "Password must be at least 8 characters." });
}
user.PasswordHash = users.HashPassword(user, request.Password);
}
await db.SaveChangesAsync(cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "user-updated", context.User.Identity?.Name,
detail: $"user={user.Username}" + (role is not null ? $" role={role.Name}" : ""),
cancellationToken: cancellationToken);
var roleName = user.RoleId is { } rid
? await db.Roles.Where(r => r.Id == rid).Select(r => r.Name).SingleOrDefaultAsync(cancellationToken)
: null;
return Results.Ok(new UserAccountDto(
user.Id, user.Username, user.DisplayName, user.AuthProvider, user.RoleId,
roleName ?? user.Role.ToString(), user.CreatedUtc, user.LastLoginUtc));
}
private static async Task<IResult> DeleteUserAsync(
Guid id, AppDbContext db, AuditService audit, HttpContext context, CancellationToken cancellationToken)
{
var user = await db.Users.SingleOrDefaultAsync(u => u.Id == id, cancellationToken);
if (user is null)
{
return Results.NotFound();
}
if (string.Equals(user.Username, context.User.Identity?.Name, StringComparison.OrdinalIgnoreCase))
{
return Results.Conflict(new { error = "You cannot delete your own account." });
}
if (await HoldsWildcardAsync(db, user, cancellationToken)
&& await WildcardHolderCountAsync(db, cancellationToken) <= 1)
{
return Results.Conflict(new { error = "At least one user must keep a full-access role." });
}
db.Users.Remove(user);
await db.SaveChangesAsync(cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "user-deleted", context.User.Identity?.Name,
detail: $"user={user.Username}", cancellationToken: cancellationToken);
return Results.NoContent();
}
private static IResult? Validate(SaveRoleRequest request)
{
if (string.IsNullOrWhiteSpace(request.Name) || request.Name.Trim().Length > 64)
@@ -330,45 +158,15 @@ public static class AccessControlEndpoints
role.Description = string.IsNullOrWhiteSpace(request.Description) ? null : request.Description.Trim();
role.PermissionsJson = JsonSerializer.Serialize(
request.Permissions.Distinct(StringComparer.OrdinalIgnoreCase).ToList());
role.OidcClaimValuesJson = JsonSerializer.Serialize(
(request.OidcClaimValues ?? [])
.Select(v => v.Trim())
.Where(v => v.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList());
role.DeviceScopeGroupIdsJson = JsonSerializer.Serialize(
(request.DeviceScopeGroupIds ?? []).Distinct().ToList());
role.UpdatedUtc = now;
role.UpdatedBy = actor;
}
private static RoleDto ToDto(RoleEntity role, int userCount) => new(
private static RoleDto ToDto(RoleEntity role, int bindingCount) => new(
role.Id, role.Name, role.Description, role.IsSystem,
RoleCache.ParseStrings(role.PermissionsJson),
RoleCache.ParseStrings(role.OidcClaimValuesJson),
RoleCache.ParseGuids(role.DeviceScopeGroupIdsJson),
userCount);
private static async Task<bool> HoldsWildcardAsync(
AppDbContext db, UserEntity user, CancellationToken cancellationToken)
{
if (user.RoleId is not { } roleId)
{
return user.Role == UserRole.Admin;
}
var role = await db.Roles.AsNoTracking().SingleOrDefaultAsync(r => r.Id == roleId, cancellationToken);
return role is not null && RoleCache.ParseStrings(role.PermissionsJson).Contains(Permissions.All);
}
/// <summary>Users whose role grants everything (the pool of "last admins").</summary>
private static async Task<int> WildcardHolderCountAsync(AppDbContext db, CancellationToken cancellationToken)
{
var wildcardRoleIds = (await db.Roles.AsNoTracking().ToListAsync(cancellationToken))
.Where(r => RoleCache.ParseStrings(r.PermissionsJson).Contains(Permissions.All))
.Select(r => r.Id)
.ToList();
return await db.Users.CountAsync(
u => u.RoleId != null && wildcardRoleIds.Contains(u.RoleId.Value), cancellationToken);
}
bindingCount);
}
-40
View File
@@ -1,40 +0,0 @@
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>
/// Reads the environment bootstrap defaults for authentication. Anything saved
/// from the Settings UI layers on top of these.
/// </summary>
public static class AuthBootstrap
{
public static AuthSettings FromConfiguration(IConfiguration configuration)
{
var mode = configuration["AUTH_LOGIN_MODE"];
return new AuthSettings
{
OidcEnabled = configuration["OIDC_ENABLED"] == "1",
Authority = configuration["OIDC_AUTHORITY"],
ClientId = configuration["OIDC_CLIENT_ID"],
ClientSecret = configuration["OIDC_CLIENT_SECRET"],
ProviderName = configuration["OIDC_PROVIDER_NAME"],
Scopes = string.IsNullOrWhiteSpace(configuration["OIDC_SCOPES"])
? "openid profile email"
: configuration["OIDC_SCOPES"]!,
NameClaim = configuration["OIDC_NAME_CLAIM"],
RoleClaim = configuration["OIDC_ROLE_CLAIM"],
AdminRole = configuration["OIDC_ADMIN_ROLE"],
OperatorRole = configuration["OIDC_OPERATOR_ROLE"],
ViewerRole = configuration["OIDC_VIEWER_ROLE"],
DefaultRole = Enum.TryParse<UserRole>(configuration["OIDC_DEFAULT_ROLE"], true, out var role)
? role
: UserRole.Operator,
RequireMappedRole = configuration["OIDC_REQUIRE_MAPPED_ROLE"] == "1",
LoginMode = Enum.TryParse<LoginMode>(mode, true, out var parsed) ? parsed : LoginMode.Both,
// Break-glass: ignores stored OIDC settings entirely.
ForceLocal = configuration["AUTH_FORCE_LOCAL"] == "1",
};
}
}
+7 -12
View File
@@ -1,20 +1,15 @@
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>Auth scheme names, claim types and authorization policy names.</summary>
/// <summary>The session scheme and the claim types a signed-in request carries.</summary>
public static class AuthConstants
{
public const string CookieScheme = "ExecuteRCookie";
public const string OidcScheme = "oidc";
public const string SessionScheme = "ExecuteRSession";
public const string RoleClaim = "role";
/// <summary>Claim carrying the assigned RoleEntity id; the name claim is display/fallback.</summary>
/// <summary>One per role the caller's groups are bound to.</summary>
public const string RoleIdClaim = "role_id";
public const string ProviderClaim = "auth_provider";
public const string ProviderLocal = "local";
public const string ProviderOidc = "oidc";
public static string RoleName(UserRole role) => role.ToString();
public const string DisplayNameClaim = "display_name";
public const string SessionClaim = "session_id";
public const string BreakGlassClaim = "break_glass";
public const string PasswordChangeClaim = "password_change_required";
}
-183
View File
@@ -1,183 +0,0 @@
using System.Security.Claims;
using Backend.Proxy;
using Backend.Security;
using Microsoft.AspNetCore.Authentication;
using Microsoft.EntityFrameworkCore;
using Shared.Contracts.Api;
using Shared.Contracts.Models;
namespace Backend.Auth;
public static class AuthEndpoints
{
public static void Map(WebApplication app)
{
app.MapPost(ApiRoutes.AuthLogin, LoginAsync)
.RequireRateLimiting(RateLimitPolicies.Auth)
.AllowAnonymous();
app.MapPost(ApiRoutes.AuthChangePassword, ChangePasswordAsync)
.RequireRateLimiting(RateLimitPolicies.Auth)
.AllowAnonymous();
app.MapPost(ApiRoutes.AuthLogout, (Delegate)LogoutAsync);
app.MapGet(ApiRoutes.AuthSession, GetSessionAsync).AllowAnonymous();
// Challenge only when OIDC is actually usable, so a half-configured or
// disabled provider returns a clear error instead of a broken redirect.
app.MapGet(ApiRoutes.AuthOidcLogin, async (
HttpContext context, AuthSettingsService settings, CancellationToken cancellationToken) =>
{
var effective = await settings.GetEffectiveAsync(cancellationToken);
return effective.IsOidcUsable
? Results.Challenge(new AuthenticationProperties { RedirectUri = "/" }, [AuthConstants.OidcScheme])
: Results.NotFound(new { error = "Single sign-on is not configured." });
})
.AllowAnonymous();
// OIDC completes at CallbackPath (handler-owned); this route lets the
// SPA finalize the app cookie mapping if needed.
app.MapGet(ApiRoutes.AuthOidcCallback, () => Results.Redirect("/")).AllowAnonymous();
}
/// <summary>The built-in default; signing in with it forces a reset.</summary>
private const string DefaultAdminPassword = "admin";
private static async Task<IResult> LoginAsync(
LoginRequest request,
UserService users,
RoleCache roles,
AuditService audit,
Backend.Data.AppDbContext db,
HttpContext context,
CancellationToken cancellationToken)
{
var remoteIp = ClientIp.EffectiveText(context);
var user = await users.ValidateCredentialsAsync(request.Username, request.Password, cancellationToken);
if (user is null)
{
await audit.LogAsync(AuditCategories.Auth, "login-failed",
actor: request.Username, remoteIp: remoteIp, cancellationToken: cancellationToken);
return Results.Unauthorized();
}
// The reset presents only while the password is still the built-in
// default — a custom password chosen at install signs straight in,
// whatever the flag says (POST /api/auth/change-password completes
// the sign-in when the reset does apply).
if (user.MustChangePassword && request.Password == DefaultAdminPassword)
{
await audit.LogAsync(AuditCategories.Auth, "login-password-change-required",
actor: user.Username, remoteIp: remoteIp, cancellationToken: cancellationToken);
return Results.Ok(new SessionResponse(
false, user.Username, user.DisplayName, null, AuthConstants.ProviderLocal,
MustChangePassword: true));
}
if (user.MustChangePassword)
{
// Stale flag with a non-default password: clear it.
var tracked = await db.Users.SingleAsync(u => u.Id == user.Id, cancellationToken);
tracked.MustChangePassword = false;
await db.SaveChangesAsync(cancellationToken);
}
var session = await IssueSessionAsync(user, roles, context, cancellationToken);
await audit.LogAsync(AuditCategories.Auth, "login-succeeded",
actor: user.Username, remoteIp: remoteIp, cancellationToken: cancellationToken);
return Results.Ok(session);
}
private static async Task<IResult> ChangePasswordAsync(
ChangePasswordRequest request,
Backend.Data.AppDbContext db,
UserService users,
RoleCache roles,
AuditService audit,
HttpContext context,
CancellationToken cancellationToken)
{
var remoteIp = ClientIp.EffectiveText(context);
if (request.NewPassword.Length < 8)
{
return Results.BadRequest(new { error = "The new password must be at least 8 characters." });
}
var user = await users.ValidateCredentialsAsync(
request.Username, request.CurrentPassword, cancellationToken);
if (user is null)
{
await audit.LogAsync(AuditCategories.Auth, "password-change-failed",
actor: request.Username, remoteIp: remoteIp, cancellationToken: cancellationToken);
return Results.Unauthorized();
}
var tracked = await db.Users.SingleAsync(u => u.Id == user.Id, cancellationToken);
tracked.PasswordHash = users.HashPassword(tracked, request.NewPassword);
tracked.MustChangePassword = false;
await db.SaveChangesAsync(cancellationToken);
var session = await IssueSessionAsync(tracked, roles, context, cancellationToken);
await audit.LogAsync(AuditCategories.Auth, "password-changed",
actor: tracked.Username, remoteIp: remoteIp, cancellationToken: cancellationToken);
return Results.Ok(session);
}
/// <summary>Issues the cookie session + CSRF token and builds the response body.</summary>
private static async Task<SessionResponse> IssueSessionAsync(
Backend.Data.UserEntity user, RoleCache roles, HttpContext context, CancellationToken cancellationToken)
{
var role = user.RoleId is { } roleId ? await roles.FindByIdAsync(roleId, cancellationToken) : null;
role ??= await roles.FindByNameAsync(user.Role.ToString(), cancellationToken);
var principal = AuthExtensions.BuildPrincipal(
user.Username, user.DisplayName, role?.Name ?? user.Role.ToString(), role?.Id,
AuthConstants.ProviderLocal);
await context.SignInAsync(AuthConstants.CookieScheme, principal);
CsrfProtection.IssueToken(context);
var permissions = role?.Permissions.Order(StringComparer.Ordinal).ToList() ?? [];
return new SessionResponse(
true, user.Username, user.DisplayName,
role is null ? user.Role : Permissions.Tier(role.Permissions),
AuthConstants.ProviderLocal, role?.Name, permissions);
}
private static async Task<IResult> LogoutAsync(HttpContext context)
{
await context.SignOutAsync(AuthConstants.CookieScheme);
context.Response.Cookies.Delete(CsrfProtection.CookieName);
return Results.Ok();
}
private static async Task<IResult> GetSessionAsync(
HttpContext context, PermissionHandler permissions, CancellationToken cancellationToken)
{
var user = context.User;
if (user.Identity?.IsAuthenticated != true)
{
return Results.Ok(new SessionResponse(false, null, null, null, null));
}
// Ensure a CSRF token exists for this authenticated session.
if (!context.Request.Cookies.ContainsKey(CsrfProtection.CookieName))
{
CsrfProtection.IssueToken(context);
}
var role = await permissions.ResolveRoleAsync(user, cancellationToken);
_ = Enum.TryParse<UserRole>(user.FindFirstValue(ClaimTypes.Role), out var claimTier);
return Results.Ok(new SessionResponse(
true,
user.Identity.Name,
user.FindFirstValue("display_name") ?? user.Identity.Name,
role is null ? claimTier : Permissions.Tier(role.Permissions),
user.FindFirstValue(AuthConstants.ProviderClaim) ?? AuthConstants.ProviderLocal,
role?.Name ?? user.FindFirstValue(ClaimTypes.Role),
role?.Permissions.Order(StringComparer.Ordinal).ToList() ?? []));
}
}
-220
View File
@@ -1,220 +0,0 @@
using System.Security.Claims;
using Microsoft.AspNetCore.Authentication.Cookies;
using Microsoft.AspNetCore.Authentication.OpenIdConnect;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
using Microsoft.IdentityModel.Protocols.OpenIdConnect;
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>Wires cookie + OIDC authentication and role-based authorization.</summary>
public static class AuthExtensions
{
public static void AddControlPlaneAuth(this WebApplicationBuilder builder, AuthSettings bootstrap)
{
builder.Services.AddSingleton(bootstrap);
builder.Services.AddSingleton(new AuthSettingsCache(bootstrap));
builder.Services.AddScoped<AuthSettingsService>();
// OpenIdConnectOptions are rebuilt from the settings cache whenever the
// UI saves, so enabling or repointing OIDC needs no restart.
builder.Services.AddSingleton<IConfigureOptions<OpenIdConnectOptions>, ConfigureOidc>();
builder.Services.AddSingleton<IOptionsChangeTokenSource<OpenIdConnectOptions>, OidcChangeTokenSource>();
builder.Services
.AddAuthentication(options =>
{
options.DefaultScheme = AuthConstants.CookieScheme;
// Challenges are issued explicitly by the login endpoints, so the
// default stays local; that keeps API 401s from bouncing to an IdP.
options.DefaultChallengeScheme = AuthConstants.CookieScheme;
})
.AddCookie(AuthConstants.CookieScheme, options =>
{
options.Cookie.Name = "executer_session";
options.Cookie.HttpOnly = true;
options.Cookie.SecurePolicy = CookieSecurePolicy.SameAsRequest;
options.Cookie.SameSite = SameSiteMode.Lax;
options.SlidingExpiration = true;
options.ExpireTimeSpan = TimeSpan.FromHours(12);
options.Events.OnRedirectToLogin = context =>
{
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return Task.CompletedTask;
};
options.Events.OnRedirectToAccessDenied = context =>
{
context.Response.StatusCode = StatusCodes.Status403Forbidden;
return Task.CompletedTask;
};
})
// Registered unconditionally; ConfigureOidc fills it from current
// settings and the login endpoint refuses to challenge when unusable.
.AddOpenIdConnect(AuthConstants.OidcScheme, _ => { });
// Permission-based authorization: endpoints demand "perm:{key}" and the
// handler resolves the caller's role (custom or system) from the cache.
builder.Services.AddAuthorization();
builder.Services.AddSingleton<Microsoft.AspNetCore.Authorization.IAuthorizationPolicyProvider,
PermissionPolicyProvider>();
builder.Services.AddSingleton<RoleCache>();
builder.Services.AddSingleton<PermissionHandler>();
builder.Services.AddSingleton<Microsoft.AspNetCore.Authorization.IAuthorizationHandler>(sp =>
sp.GetRequiredService<PermissionHandler>());
builder.Services.AddScoped<RoleService>();
builder.Services.AddScoped<DeviceScopeService>();
}
public static ClaimsPrincipal BuildPrincipal(
string username, string? displayName, string roleName, Guid? roleId, string provider)
{
var identity = new ClaimsIdentity(AuthConstants.CookieScheme, ClaimTypes.Name, ClaimTypes.Role);
identity.AddClaim(new Claim(ClaimTypes.Name, username));
identity.AddClaim(new Claim(ClaimTypes.Role, roleName));
if (roleId is { } id)
{
identity.AddClaim(new Claim(AuthConstants.RoleIdClaim, id.ToString("D")));
}
identity.AddClaim(new Claim(AuthConstants.ProviderClaim, provider));
if (!string.IsNullOrWhiteSpace(displayName))
{
identity.AddClaim(new Claim("display_name", displayName));
}
return new ClaimsPrincipal(identity);
}
}
/// <summary>Builds OpenIdConnectOptions from the live settings cache.</summary>
internal sealed class ConfigureOidc(AuthSettingsCache cache, IServiceScopeFactory scopeFactory)
: IConfigureNamedOptions<OpenIdConnectOptions>
{
public void Configure(OpenIdConnectOptions options) => Configure(Options.DefaultName, options);
public void Configure(string? name, OpenIdConnectOptions options)
{
if (name != AuthConstants.OidcScheme)
{
return;
}
var settings = cache.Current;
// The OIDC handler is an IAuthenticationRequestHandler, so the
// authentication middleware initializes it on every request — and
// RemoteAuthenticationOptions.Validate() throws on a null Authority or
// ClientId. Placeholders keep validation happy while OIDC is off; the
// login endpoint refuses to challenge unless the real settings are
// usable, so these are never contacted.
options.Authority = string.IsNullOrWhiteSpace(settings.Authority)
? "https://oidc.invalid/not-configured"
: settings.Authority;
options.ClientId = string.IsNullOrWhiteSpace(settings.ClientId)
? "oidc-not-configured"
: settings.ClientId;
options.ClientSecret = settings.ClientSecret;
options.ResponseType = OpenIdConnectResponseType.Code;
// Fixed by design — must match the IdP registration exactly.
options.CallbackPath = AuthSettingsService.CallbackPath;
options.SaveTokens = false;
options.GetClaimsFromUserInfoEndpoint = true;
options.SignInScheme = AuthConstants.CookieScheme;
options.UsePkce = true;
options.Scope.Clear();
foreach (var scope in settings.Scopes.Split(' ', StringSplitOptions.RemoveEmptyEntries))
{
options.Scope.Add(scope);
}
if (!string.IsNullOrWhiteSpace(settings.NameClaim))
{
options.TokenValidationParameters.NameClaimType = settings.NameClaim;
}
if (!string.IsNullOrWhiteSpace(settings.RoleClaim))
{
options.TokenValidationParameters.RoleClaimType = settings.RoleClaim;
}
options.Events.OnTokenValidated = async context =>
{
var current = cache.Current;
if (context.Principal?.Identity is not ClaimsIdentity identity)
{
return;
}
var principal = context.Principal;
var subject = principal.FindFirstValue(ClaimTypes.NameIdentifier) ?? principal.FindFirstValue("sub");
if (string.IsNullOrEmpty(subject))
{
context.Fail("OIDC token has no subject claim.");
return;
}
var username = (current.NameClaim is not null ? principal.FindFirstValue(current.NameClaim) : null)
?? principal.FindFirstValue(ClaimTypes.Name)
?? principal.FindFirstValue("preferred_username")
?? principal.FindFirstValue(ClaimTypes.Email)
?? subject;
var displayName = principal.FindFirstValue("name") ?? username;
// Map identity-provider groups/roles onto an application role:
// custom roles' claim values first, legacy tier mapping second.
var claimValues = string.IsNullOrWhiteSpace(current.RoleClaim)
? []
: principal.FindAll(current.RoleClaim).Select(c => c.Value).ToList();
using var scope = scopeFactory.CreateScope();
var roleService = scope.ServiceProvider.GetRequiredService<RoleService>();
var role = await roleService.MapOidcRoleAsync(
claimValues, current, context.HttpContext.RequestAborted);
if (role is null)
{
context.Fail(
"Your account is not a member of any group mapped to an ExecuteR role.");
return;
}
var users = scope.ServiceProvider.GetRequiredService<UserService>();
await users.UpsertOidcUserAsync(
subject, username, displayName, role, context.HttpContext.RequestAborted);
identity.AddClaim(new Claim(ClaimTypes.Role, role.Name));
identity.AddClaim(new Claim(AuthConstants.RoleIdClaim, role.Id.ToString("D")));
identity.AddClaim(new Claim(AuthConstants.ProviderClaim, AuthConstants.ProviderOidc));
identity.AddClaim(new Claim("display_name", displayName));
if (identity.FindFirst(ClaimTypes.Name) is null)
{
identity.AddClaim(new Claim(ClaimTypes.Name, username));
}
};
// Never strand the operator on an IdP error page.
options.Events.OnRemoteFailure = context =>
{
context.Response.Redirect(
"/login?error=" + Uri.EscapeDataString(context.Failure?.Message ?? "Single sign-on failed."));
context.HandleResponse();
return Task.CompletedTask;
};
options.Events.OnAccessDenied = context =>
{
context.Response.Redirect("/login?error=" + Uri.EscapeDataString("Access denied by the identity provider."));
context.HandleResponse();
return Task.CompletedTask;
};
}
}
internal sealed class OidcChangeTokenSource(AuthSettingsCache cache) : IOptionsChangeTokenSource<OpenIdConnectOptions>
{
public string Name => AuthConstants.OidcScheme;
public IChangeToken GetChangeToken() => cache.GetChangeToken();
}
+46 -78
View File
@@ -1,111 +1,79 @@
using Backend.Data;
using Backend.Security;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>
/// Lockout recovery, runnable inside the container:
/// Lockout recovery, run on the server (or inside the container):
///
/// docker exec EXECUTER-API-00001 dotnet Backend.dll auth-reset
/// docker exec EXECUTER-API-00001 dotnet Backend.dll auth-reset --admin admin --password '&lt;new&gt;'
/// dotnet Backend.dll auth-reset Built-In sign-in on, OIDC redirect off, and a new one-time
/// password for the break-glass account (created if missing)
/// dotnet Backend.dll auth-reset --user jdoe a new one-time password for that built-in account instead
/// dotnet Backend.dll auth-reset --password 'secret' that password rather than a generated one (still changed at sign-in)
///
/// Disables OIDC and restores local sign-in, so an identity provider that is
/// down, misconfigured or has revoked the client can never permanently lock an
/// operator out. Optionally resets (or creates) a local admin.
///
/// Setting AUTH_FORCE_LOCAL=1 achieves the same at startup without a database
/// write, which is the faster option if the container will not stay up.
/// So an identity provider that is down, misconfigured or has revoked the
/// client never locks anyone out for good. AUTH_FORCE_LOCAL=1 offers the
/// built-in form at startup without a database write. Exit codes: 0 done,
/// 1 failed, 2 no such account.
/// </summary>
public static class AuthResetCli
{
public static async Task<int> RunAsync(string[] args, IConfiguration configuration)
{
var username = ArgValue(args, "--admin");
var userName = ArgValue(args, "--user") ?? ArgValue(args, "--admin");
var password = ArgValue(args, "--password");
var databaseOptions = DatabaseOptions.FromConfiguration(configuration);
var services = new ServiceCollection();
services.AddLogging(b => b.AddSimpleConsole());
services.AddDataProtection().PersistKeysToFileSystem(
new DirectoryInfo(DataProtectionPath(configuration)));
services.AddApplicationDatabase(databaseOptions);
services.AddScoped<Settings.SettingsStore>();
services.AddSingleton(configuration);
services.AddDataProtection().PersistKeysToFileSystem(new DirectoryInfo(DataProtectionPath(configuration)));
services.AddApplicationDatabase(DatabaseOptions.FromConfiguration(configuration));
services.AddScoped<AuditService>();
services.AddSingleton<BreakGlass>();
await using var provider = services.BuildServiceProvider();
using var scope = provider.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
// Clear the OIDC flags directly: this must work even when the rest of
// the application cannot start.
foreach (var (key, value) in new[]
{
(AuthSettingsService.Keys.OidcEnabled, "0"),
(AuthSettingsService.Keys.LoginMode, nameof(LoginMode.LocalOnly)),
})
try
{
var row = await db.AppSettings.SingleOrDefaultAsync(s => s.Key == key);
if (row is null)
using (var scope = provider.CreateScope())
{
db.AppSettings.Add(new AppSettingEntity
{
Key = key,
Value = value,
IsSecret = false,
UpdatedUtc = DateTimeOffset.UtcNow,
UpdatedBy = "auth-reset",
});
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var row = await SettingsDocuments.LoadAsync(db, AuthenticationMethods.Key, CancellationToken.None);
var before = SettingsDocuments.Parse(row?.Value, AuthenticationMethods.Defaults);
SettingsDocuments.Stage(db, row, AuthenticationMethods.Key, row is null ? null : before, before with { BuiltIn = true, OidcRedirect = false },
"auth-reset", always: true);
await db.SaveChangesAsync();
Console.WriteLine("Built-In sign-in is on and OIDC redirect is off.");
}
var (user, generated) = await provider.GetRequiredService<BreakGlass>().ResetAsync(userName, "auth-reset", CancellationToken.None);
if (!string.IsNullOrEmpty(password))
{
using var scope = provider.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var tracked = await db.Users.FindAsync(user.Id);
tracked!.PasswordHash = PasswordHashing.Hash(password);
await db.SaveChangesAsync();
Console.WriteLine($"{user.UserName}: password set; it must be changed at the next sign-in.");
}
else
{
row.Value = value;
row.UpdatedUtc = DateTimeOffset.UtcNow;
row.UpdatedBy = "auth-reset";
Console.WriteLine($"{user.UserName}: one-time password {generated} (change it at the next sign-in).");
}
Console.WriteLine("No restart needed; sign in at /sign-in.");
return 0;
}
await db.SaveChangesAsync();
Console.WriteLine("OIDC disabled; local sign-in restored.");
if (!string.IsNullOrWhiteSpace(username) && !string.IsNullOrWhiteSpace(password))
catch (InvalidOperationException exception)
{
var hasher = new PasswordHasher<UserEntity>();
var user = await db.Users.SingleOrDefaultAsync(
u => u.Username == username && u.AuthProvider == AuthConstants.ProviderLocal);
if (user is null)
{
user = new UserEntity
{
Id = Guid.NewGuid(),
Username = username!,
DisplayName = username,
Role = UserRole.Admin,
AuthProvider = AuthConstants.ProviderLocal,
CreatedUtc = DateTimeOffset.UtcNow,
};
db.Users.Add(user);
Console.WriteLine($"Local admin '{username}' created.");
}
else
{
user.Role = UserRole.Admin;
Console.WriteLine($"Local admin '{username}' password reset and role set to Admin.");
}
user.PasswordHash = hasher.HashPassword(user, password!);
await db.SaveChangesAsync();
Console.Error.WriteLine(exception.Message);
return 2;
}
else
catch (Exception exception)
{
Console.WriteLine("Tip: pass --admin <user> --password <pass> to also reset a local admin.");
Console.Error.WriteLine($"auth-reset failed: {exception.Message}");
return 1;
}
Console.WriteLine("Restart is not required; sign in at /login.");
return 0;
}
private static string DataProtectionPath(IConfiguration configuration)
-95
View File
@@ -1,95 +0,0 @@
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>How the sign-in page behaves.</summary>
public enum LoginMode
{
/// <summary>Local username/password only.</summary>
LocalOnly,
/// <summary>Both local fields and an SSO button.</summary>
Both,
/// <summary>SSO button only — local fields hidden (still reachable, see below).</summary>
OidcOnly,
/// <summary>Redirect straight to the identity provider, no sign-in page.</summary>
OidcAutoRedirect,
}
/// <summary>
/// Effective authentication configuration: values saved from the UI layered
/// over the environment bootstrap defaults.
///
/// The OIDC redirect path is deliberately NOT operator-tunable at runtime — see
/// <see cref="AuthSettingsService.CallbackPath"/>.
/// </summary>
public sealed record AuthSettings
{
public bool OidcEnabled { get; init; }
public string? Authority { get; init; }
public string? ClientId { get; init; }
public string? ClientSecret { get; init; }
/// <summary>Friendly IdP name shown on the sign-in button (e.g. "Authentik", "Entra ID").</summary>
public string? ProviderName { get; init; }
public string Scopes { get; init; } = "openid profile email";
public string? NameClaim { get; init; }
public string? RoleClaim { get; init; }
/// <summary>Claim values mapped to each application role.</summary>
public string? AdminRole { get; init; }
public string? OperatorRole { get; init; }
public string? ViewerRole { get; init; }
/// <summary>Role granted to an authenticated user matching no mapping.</summary>
public UserRole DefaultRole { get; init; } = UserRole.Operator;
/// <summary>Refuse sign-in entirely when no mapping matches.</summary>
public bool RequireMappedRole { get; init; }
public LoginMode LoginMode { get; init; } = LoginMode.LocalOnly;
/// <summary>
/// Emergency switch (env only, never settable from the UI): forces local
/// sign-in regardless of what is stored. See the auth-reset CLI.
/// </summary>
public bool ForceLocal { get; init; }
public bool IsOidcUsable => OidcEnabled
&& !ForceLocal
&& !string.IsNullOrWhiteSpace(Authority)
&& !string.IsNullOrWhiteSpace(ClientId);
/// <summary>Local sign-in is never truly switched off — see the lockout note.</summary>
public bool ShowLocalFields => !IsOidcUsable || LoginMode is LoginMode.LocalOnly or LoginMode.Both;
public bool AutoRedirect => IsOidcUsable && LoginMode == LoginMode.OidcAutoRedirect;
/// <summary>Maps identity-provider claim values to an application role.</summary>
public UserRole? MapRole(IEnumerable<string> claimValues)
{
var values = claimValues.ToList();
bool Matches(string? configured) =>
!string.IsNullOrWhiteSpace(configured)
&& values.Any(v => string.Equals(v, configured, StringComparison.OrdinalIgnoreCase));
if (Matches(AdminRole))
{
return UserRole.Admin;
}
if (Matches(OperatorRole))
{
return UserRole.Operator;
}
if (Matches(ViewerRole))
{
return UserRole.Viewer;
}
return RequireMappedRole ? null : DefaultRole;
}
}
-188
View File
@@ -1,188 +0,0 @@
using Backend.Security;
using Shared.Contracts.Api;
using Backend.Auth;
namespace Backend.Auth;
/// <summary>
/// Identity-provider configuration from the UI, plus the anonymous config the
/// sign-in page needs to decide what to render.
/// </summary>
public static class AuthSettingsEndpoints
{
public static void Map(WebApplication app)
{
app.MapGet(ApiRoutes.AuthConfig, GetPublicConfigAsync).AllowAnonymous();
app.MapGet(ApiRoutes.AuthSettings, GetAsync).RequirePermission(Permissions.SettingsManage);
app.MapPut(ApiRoutes.AuthSettings, SaveAsync).RequirePermission(Permissions.SettingsManage);
app.MapPost(ApiRoutes.AuthSettingsTest, TestAsync).RequirePermission(Permissions.SettingsManage);
}
/// <summary>Anonymous: only what the sign-in page needs to render itself.</summary>
private static async Task<IResult> GetPublicConfigAsync(
AuthSettingsService settings, CancellationToken cancellationToken)
{
var effective = await settings.GetEffectiveAsync(cancellationToken);
return Results.Ok(new AuthConfigDto(
OidcAvailable: effective.IsOidcUsable,
ShowLocalFields: effective.ShowLocalFields,
AutoRedirect: effective.AutoRedirect,
// Friendly name (Authentik, Entra ID, …) when set; authority host
// is only the fallback.
ProviderName: effective.IsOidcUsable
? (string.IsNullOrWhiteSpace(effective.ProviderName)
? DescribeAuthority(effective.Authority)
: effective.ProviderName)
: null));
}
private static string? DescribeAuthority(string? authority)
=> Uri.TryCreate(authority, UriKind.Absolute, out var uri) ? uri.Host : authority;
private static async Task<IResult> GetAsync(
AuthSettingsService settings,
HttpContext context,
IConfiguration configuration,
CancellationToken cancellationToken)
{
var effective = await settings.GetEffectiveAsync(cancellationToken);
return Results.Ok(ToDto(effective, context, configuration));
}
private static async Task<IResult> SaveAsync(
AuthSettingsDto request,
AuthSettingsService settings,
AuditService audit,
HttpContext context,
IConfiguration configuration,
CancellationToken cancellationToken)
{
var current = await settings.GetEffectiveAsync(cancellationToken);
if (!Enum.TryParse<LoginMode>(request.LoginMode, true, out var mode))
{
mode = LoginMode.LocalOnly;
}
// Refuse to hide local sign-in behind an IdP that is not usable —
// that is exactly how an instance locks itself out.
var wouldBeUsable = request.OidcEnabled
&& !string.IsNullOrWhiteSpace(request.Authority)
&& !string.IsNullOrWhiteSpace(request.ClientId);
if (!wouldBeUsable && mode is LoginMode.OidcOnly or LoginMode.OidcAutoRedirect)
{
return Results.BadRequest(new
{
error = "Configure and enable OIDC before choosing an SSO-only sign-in mode.",
});
}
var updated = current with
{
OidcEnabled = request.OidcEnabled,
Authority = request.Authority,
ClientId = request.ClientId,
ClientSecret = request.ClientSecret,
ProviderName = request.ProviderName,
Scopes = string.IsNullOrWhiteSpace(request.Scopes) ? "openid profile email" : request.Scopes,
NameClaim = request.NameClaim,
RoleClaim = request.RoleClaim,
AdminRole = request.AdminRole,
OperatorRole = request.OperatorRole,
ViewerRole = request.ViewerRole,
DefaultRole = request.DefaultRole,
RequireMappedRole = request.RequireMappedRole,
LoginMode = mode,
};
await settings.SaveAsync(updated, context.User.Identity?.Name, cancellationToken);
await audit.LogAsync(AuditCategories.Auth, "oidc-settings-updated", context.User.Identity?.Name,
detail: $"enabled={request.OidcEnabled} mode={mode} authority={request.Authority}",
cancellationToken: cancellationToken);
return Results.Ok(ToDto(await settings.GetEffectiveAsync(cancellationToken), context, configuration));
}
/// <summary>Fetches the discovery document so a bad authority is caught before saving.</summary>
private static async Task<IResult> TestAsync(
AuthSettingsDto request, AuthSettingsService settings, CancellationToken cancellationToken)
{
var steps = new List<S3TestStepDto>();
var authority = string.IsNullOrWhiteSpace(request.Authority)
? (await settings.GetEffectiveAsync(cancellationToken)).Authority
: request.Authority;
if (string.IsNullOrWhiteSpace(authority))
{
steps.Add(new S3TestStepDto("authority", false, "Authority is required."));
return Results.Ok(new AuthTestResultDto(false, steps));
}
var discovery = $"{authority.TrimEnd('/')}/.well-known/openid-configuration";
try
{
using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(15) };
using var response = await http.GetAsync(discovery, cancellationToken);
if (!response.IsSuccessStatusCode)
{
steps.Add(new S3TestStepDto("discovery", false,
$"HTTP {(int)response.StatusCode} from {discovery}"));
return Results.Ok(new AuthTestResultDto(false, steps));
}
steps.Add(new S3TestStepDto("discovery", true, $"{discovery} reachable"));
using var document = System.Text.Json.JsonDocument.Parse(
await response.Content.ReadAsStringAsync(cancellationToken));
var root = document.RootElement;
foreach (var required in new[] { "authorization_endpoint", "token_endpoint", "jwks_uri" })
{
var present = root.TryGetProperty(required, out _);
steps.Add(new S3TestStepDto(required, present, present ? "present" : "missing from discovery"));
}
var pkce = root.TryGetProperty("code_challenge_methods_supported", out var methods)
&& methods.EnumerateArray().Any(m => m.GetString() == "S256");
steps.Add(new S3TestStepDto("PKCE (S256)", pkce,
pkce ? "supported" : "not advertised — sign-in may fail"));
}
catch (Exception ex)
{
steps.Add(new S3TestStepDto("discovery", false, ex.Message));
}
return Results.Ok(new AuthTestResultDto(steps.All(s => s.Ok), steps));
}
private static AuthSettingsDto ToDto(AuthSettings s, HttpContext context, IConfiguration configuration)
{
// PUBLIC_SERVER_URL wins so the IdP registration matches what agents
// and browsers actually use, even when an admin browses via an
// internal hostname; trusted forwarded headers are the fallback.
var origin = Backend.Proxy.PublicUrl.Resolve(context, configuration);
return new AuthSettingsDto(
OidcEnabled: s.OidcEnabled,
Authority: s.Authority,
ClientId: s.ClientId,
ClientSecret: null,
ProviderName: s.ProviderName,
Scopes: s.Scopes,
NameClaim: s.NameClaim,
RoleClaim: s.RoleClaim,
AdminRole: s.AdminRole,
OperatorRole: s.OperatorRole,
ViewerRole: s.ViewerRole,
DefaultRole: s.DefaultRole,
RequireMappedRole: s.RequireMappedRole,
LoginMode: s.LoginMode.ToString(),
ClientSecretIsSet: !string.IsNullOrWhiteSpace(s.ClientSecret),
ForcedLocalByEnvironment: s.ForceLocal,
CallbackPath: AuthSettingsService.CallbackPath,
RedirectUri: origin + AuthSettingsService.CallbackPath);
}
}
-144
View File
@@ -1,144 +0,0 @@
using Backend.Settings;
using Microsoft.Extensions.Primitives;
using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>
/// Resolves effective auth settings (database over environment) and caches
/// them for the OIDC handler, which cannot take a scoped dependency.
///
/// Saving triggers a change token so the handler is rebuilt without a restart.
/// </summary>
public sealed class AuthSettingsService(SettingsStore store, AuthSettings environmentDefaults, AuthSettingsCache cache)
{
public const string Prefix = "auth.";
/// <summary>
/// The OIDC redirect path is fixed. It must match what is registered at the
/// identity provider exactly, and letting it drift at runtime is a reliable
/// way to break sign-in with no obvious cause. It lives under /api/ so the
/// reverse proxy's existing API location covers it, and it is shown in the
/// UI so it can be copied into the IdP verbatim.
/// </summary>
public const string CallbackPath = Shared.Contracts.Api.ApiRoutes.AuthOidcCallback;
public static class Keys
{
public const string OidcEnabled = Prefix + "oidcEnabled";
public const string Authority = Prefix + "authority";
public const string ClientId = Prefix + "clientId";
public const string ClientSecret = Prefix + "clientSecret";
public const string ProviderName = Prefix + "providerName";
public const string Scopes = Prefix + "scopes";
public const string NameClaim = Prefix + "nameClaim";
public const string RoleClaim = Prefix + "roleClaim";
public const string AdminRole = Prefix + "adminRole";
public const string OperatorRole = Prefix + "operatorRole";
public const string ViewerRole = Prefix + "viewerRole";
public const string DefaultRole = Prefix + "defaultRole";
public const string RequireMappedRole = Prefix + "requireMappedRole";
public const string LoginMode = Prefix + "loginMode";
}
public async Task<AuthSettings> GetEffectiveAsync(CancellationToken cancellationToken)
{
var saved = await store.GetAllAsync(Prefix, cancellationToken);
var effective = Merge(saved, environmentDefaults);
cache.Set(effective);
return effective;
}
internal static AuthSettings Merge(IReadOnlyDictionary<string, string> saved, AuthSettings defaults) => new()
{
OidcEnabled = Bool(saved, Keys.OidcEnabled, defaults.OidcEnabled),
Authority = Str(saved, Keys.Authority, defaults.Authority),
ClientId = Str(saved, Keys.ClientId, defaults.ClientId),
ClientSecret = Str(saved, Keys.ClientSecret, defaults.ClientSecret),
ProviderName = Str(saved, Keys.ProviderName, defaults.ProviderName),
Scopes = Str(saved, Keys.Scopes, defaults.Scopes) ?? "openid profile email",
NameClaim = Str(saved, Keys.NameClaim, defaults.NameClaim),
RoleClaim = Str(saved, Keys.RoleClaim, defaults.RoleClaim),
AdminRole = Str(saved, Keys.AdminRole, defaults.AdminRole),
OperatorRole = Str(saved, Keys.OperatorRole, defaults.OperatorRole),
ViewerRole = Str(saved, Keys.ViewerRole, defaults.ViewerRole),
DefaultRole = Enum.TryParse<UserRole>(Str(saved, Keys.DefaultRole, null), true, out var role)
? role
: defaults.DefaultRole,
RequireMappedRole = Bool(saved, Keys.RequireMappedRole, defaults.RequireMappedRole),
LoginMode = Enum.TryParse<LoginMode>(Str(saved, Keys.LoginMode, null), true, out var mode)
? mode
: defaults.LoginMode,
// Never persisted: environment-only escape hatch.
ForceLocal = defaults.ForceLocal,
};
public async Task SaveAsync(AuthSettings settings, string? actor, CancellationToken cancellationToken)
{
await store.SetAsync(Keys.OidcEnabled, settings.OidcEnabled ? "1" : "0", false, actor, cancellationToken);
await store.SetAsync(Keys.Authority, settings.Authority, false, actor, cancellationToken);
await store.SetAsync(Keys.ClientId, settings.ClientId, false, actor, cancellationToken);
await store.SetAsync(Keys.ProviderName, settings.ProviderName, false, actor, cancellationToken);
await store.SetAsync(Keys.Scopes, settings.Scopes, false, actor, cancellationToken);
await store.SetAsync(Keys.NameClaim, settings.NameClaim, false, actor, cancellationToken);
await store.SetAsync(Keys.RoleClaim, settings.RoleClaim, false, actor, cancellationToken);
await store.SetAsync(Keys.AdminRole, settings.AdminRole, false, actor, cancellationToken);
await store.SetAsync(Keys.OperatorRole, settings.OperatorRole, false, actor, cancellationToken);
await store.SetAsync(Keys.ViewerRole, settings.ViewerRole, false, actor, cancellationToken);
await store.SetAsync(Keys.DefaultRole, settings.DefaultRole.ToString(), false, actor, cancellationToken);
await store.SetAsync(
Keys.RequireMappedRole, settings.RequireMappedRole ? "1" : "0", false, actor, cancellationToken);
await store.SetAsync(Keys.LoginMode, settings.LoginMode.ToString(), false, actor, cancellationToken);
// Null secret means "keep the stored one".
if (!string.IsNullOrWhiteSpace(settings.ClientSecret))
{
await store.SetAsync(Keys.ClientSecret, settings.ClientSecret, true, actor, cancellationToken);
}
// Rebuild the OIDC handler with the new values.
cache.Set(await GetEffectiveAsync(cancellationToken));
cache.Reload();
}
/// <summary>Disables OIDC and restores local sign-in. Used by the auth-reset CLI.</summary>
public async Task ResetToLocalAsync(string? actor, CancellationToken cancellationToken)
{
await store.SetAsync(Keys.OidcEnabled, "0", false, actor, cancellationToken);
await store.SetAsync(Keys.LoginMode, nameof(LoginMode.LocalOnly), false, actor, cancellationToken);
cache.Set(await GetEffectiveAsync(cancellationToken));
cache.Reload();
}
public AuthSettings EnvironmentDefaults => environmentDefaults;
private static string? Str(IReadOnlyDictionary<string, string> saved, string key, string? fallback)
=> saved.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) ? value : fallback;
private static bool Bool(IReadOnlyDictionary<string, string> saved, string key, bool fallback)
=> saved.TryGetValue(key, out var value) ? value is "1" or "true" or "True" : fallback;
}
/// <summary>
/// Holds the current effective auth settings for the OIDC handler (a singleton
/// that cannot resolve scoped services), and signals options reload on change.
/// </summary>
public sealed class AuthSettingsCache(AuthSettings initial)
{
private AuthSettings current = initial;
private CancellationTokenSource changeSource = new();
public AuthSettings Current => Volatile.Read(ref current);
public void Set(AuthSettings settings) => Volatile.Write(ref current, settings);
public IChangeToken GetChangeToken() => new CancellationChangeToken(changeSource.Token);
/// <summary>Invalidates cached OpenIdConnectOptions so they are rebuilt.</summary>
public void Reload()
{
var previous = Interlocked.Exchange(ref changeSource, new CancellationTokenSource());
previous.Cancel();
previous.Dispose();
}
}
-80
View File
@@ -1,80 +0,0 @@
using System.Security.Cryptography;
namespace Backend.Auth;
/// <summary>
/// Double-submit-cookie CSRF protection for cookie-authenticated,
/// state-changing REST calls. The browser reads the non-HttpOnly
/// <c>csrf_token</c> cookie and echoes it in the <c>X-CSRF-Token</c> header;
/// the middleware requires the two to match. Bearer/enrollment endpoints and
/// safe methods are exempt.
/// </summary>
public static class CsrfProtection
{
public const string CookieName = "csrf_token";
public const string HeaderName = "X-CSRF-Token";
public static string IssueToken(HttpContext context)
{
var token = Convert.ToHexString(RandomNumberGenerator.GetBytes(32));
context.Response.Cookies.Append(CookieName, token, new CookieOptions
{
HttpOnly = false, // Must be readable by the SPA.
SameSite = SameSiteMode.Lax,
Secure = context.Request.IsHttps,
Path = "/",
});
return token;
}
public static RequestDelegate Middleware(RequestDelegate next) => async context =>
{
if (RequiresValidation(context))
{
var cookie = context.Request.Cookies[CookieName];
var header = context.Request.Headers[HeaderName].ToString();
if (string.IsNullOrEmpty(cookie)
|| string.IsNullOrEmpty(header)
|| !CryptographicOperations.FixedTimeEquals(
System.Text.Encoding.UTF8.GetBytes(cookie),
System.Text.Encoding.UTF8.GetBytes(header)))
{
context.Response.StatusCode = StatusCodes.Status403Forbidden;
await context.Response.WriteAsJsonAsync(new { error = "CSRF validation failed." });
return;
}
}
await next(context);
};
private static bool RequiresValidation(HttpContext context)
{
var method = context.Request.Method;
if (HttpMethods.IsGet(method) || HttpMethods.IsHead(method) || HttpMethods.IsOptions(method))
{
return false;
}
var path = context.Request.Path;
// Enrollment endpoints are agent-to-server (no cookie); exempt.
if (path.StartsWithSegments("/api/enrollment"))
{
return false;
}
// Login and the pre-session password change establish the session;
// both are protected by rate limiting + credentials.
if (path.StartsWithSegments("/api/auth/login")
|| path.StartsWithSegments("/api/auth/change-password"))
{
return false;
}
// Only guard cookie-authenticated API calls.
return path.StartsWithSegments("/api")
&& context.Request.Cookies.ContainsKey("executer_session");
}
}
+7 -6
View File
@@ -6,9 +6,10 @@ using Microsoft.EntityFrameworkCore;
namespace Backend.Auth;
/// <summary>
/// Resolves the device scope of the caller's role: the set of client ids a
/// scoped role may see and target. Null means unrestricted. Resolved once per
/// request (this service is scoped).
/// Resolves the device scope of the caller's roles: the set of client ids
/// they may see and target. Null means unrestricted (a role, or the
/// break-glass account, reaches every device). Resolved once per request
/// (this service is scoped).
/// </summary>
public sealed class DeviceScopeService(
AppDbContext db, PermissionHandler permissions, DeviceGroupService groups)
@@ -24,15 +25,15 @@ public sealed class DeviceScopeService(
return allowed;
}
var role = await permissions.ResolveRoleAsync(user, cancellationToken);
if (role is null || role.ScopeGroupIds.Count == 0)
var access = await permissions.ResolveAccessAsync(user, cancellationToken);
if (access.ScopeGroupIds is not { } scopeIds)
{
resolved = true;
return allowed = null;
}
var scopeGroups = await db.DeviceGroups.AsNoTracking()
.Where(g => role.ScopeGroupIds.Contains(g.Id))
.Where(g => scopeIds.Contains(g.Id))
.ToListAsync(cancellationToken);
var clientIds = new HashSet<string>(StringComparer.Ordinal);
+18 -25
View File
@@ -4,7 +4,7 @@ using Microsoft.Extensions.Options;
namespace Backend.Auth;
/// <summary>Demands one permission key from the caller's effective role.</summary>
/// <summary>Demands one permission key from the roles the caller's groups are bound to.</summary>
public sealed class PermissionRequirement(string permission) : IAuthorizationRequirement
{
public string Permission { get; } = permission;
@@ -20,41 +20,29 @@ public sealed class PermissionHandler(RoleCache roles) : AuthorizationHandler<Pe
return;
}
var role = await ResolveRoleAsync(context.User, CancellationToken.None);
if (role is null)
{
return;
}
if (role.Permissions.Contains(Permissions.All) || role.Permissions.Contains(requirement.Permission))
if ((await ResolveAccessAsync(context.User, CancellationToken.None)).Has(requirement.Permission))
{
context.Succeed(requirement);
}
}
/// <summary>
/// Prefers the role-id claim; falls back to the role-name claim, which also
/// resolves pre-upgrade cookies carrying the legacy Admin/Operator/Viewer
/// names onto the seeded system roles.
/// The caller's access from their claims: the bound roles (role-id claims,
/// re-read from the role cache so a role edit applies at once) and the
/// break-glass flag.
/// </summary>
public async ValueTask<RoleCache.RoleInfo?> ResolveRoleAsync(
ClaimsPrincipal user, CancellationToken cancellationToken)
public async ValueTask<EffectiveAccess> ResolveAccessAsync(ClaimsPrincipal user, CancellationToken cancellationToken)
{
if (Guid.TryParse(user.FindFirstValue(AuthConstants.RoleIdClaim), out var roleId)
&& await roles.FindByIdAsync(roleId, cancellationToken) is { } byId)
var bound = new List<RoleCache.RoleInfo>();
foreach (var claim in user.FindAll(AuthConstants.RoleIdClaim))
{
return byId;
}
foreach (var claim in user.FindAll(ClaimTypes.Role))
{
if (await roles.FindByNameAsync(claim.Value, cancellationToken) is { } byName)
if (Guid.TryParse(claim.Value, out var id) && await roles.FindByIdAsync(id, cancellationToken) is { } role)
{
return byName;
bound.Add(role);
}
}
return null;
return EffectiveAccess.Of(bound, user.HasClaim(AuthConstants.BreakGlassClaim, "true"));
}
}
@@ -79,7 +67,7 @@ public sealed class PermissionPolicyProvider(IOptions<AuthorizationOptions> opti
return fallback.GetPolicyAsync(policyName);
}
var policy = new AuthorizationPolicyBuilder(AuthConstants.CookieScheme)
var policy = new AuthorizationPolicyBuilder(AuthConstants.SessionScheme)
.RequireAuthenticatedUser()
.AddRequirements(new PermissionRequirement(policyName[Prefix.Length..]))
.Build();
@@ -89,8 +77,13 @@ public sealed class PermissionPolicyProvider(IOptions<AuthorizationOptions> opti
public static class PermissionEndpointExtensions
{
/// <summary>Requires an authenticated session whose role grants <paramref name="permission"/>.</summary>
/// <summary>Requires a signed-in session whose roles grant <paramref name="permission"/>.</summary>
public static TBuilder RequirePermission<TBuilder>(this TBuilder builder, string permission)
where TBuilder : IEndpointConventionBuilder
=> builder.RequireAuthorization(PermissionPolicyProvider.Prefix + permission);
/// <summary>Requires any signed-in session.</summary>
public static TBuilder RequireSignedIn<TBuilder>(this TBuilder builder)
where TBuilder : IEndpointConventionBuilder
=> builder.RequireAuthorization(new AuthorizationPolicyBuilder(AuthConstants.SessionScheme).RequireAuthenticatedUser().Build());
}
+3 -56
View File
@@ -6,13 +6,12 @@ using Shared.Contracts.Models;
namespace Backend.Auth;
/// <summary>
/// Role lifecycle: seeds the immutable system roles, backfills pre-RBAC users
/// onto them, and maps OIDC sign-ins to a role (custom-role claim values first,
/// then the legacy Admin/Operator/Viewer mapping from auth settings).
/// Seeds the immutable system roles (Admin, Operator, Viewer). People reach
/// roles through role bindings on their groups.
/// </summary>
public sealed class RoleService(AppDbContext db, RoleCache cache)
{
/// <summary>Idempotent: creates missing system roles and assigns roleless users.</summary>
/// <summary>Idempotent: creates the missing system roles.</summary>
public async Task EnsureSystemRolesAsync(CancellationToken cancellationToken)
{
var existing = await db.Roles
@@ -45,62 +44,10 @@ public sealed class RoleService(AppDbContext db, RoleCache cache)
if (changed)
{
await db.SaveChangesAsync(cancellationToken);
}
// Users created before custom roles carry only the legacy tier enum.
var roleless = await db.Users.Where(u => u.RoleId == null).ToListAsync(cancellationToken);
if (roleless.Count > 0)
{
var byName = await db.Roles
.Where(r => r.IsSystem)
.ToDictionaryAsync(r => r.Name, StringComparer.OrdinalIgnoreCase, cancellationToken);
foreach (var user in roleless)
{
user.RoleId = byName[user.Role.ToString()].Id;
}
await db.SaveChangesAsync(cancellationToken);
}
if (changed || roleless.Count > 0)
{
cache.Invalidate();
}
}
public Task<RoleEntity?> FindSystemRoleAsync(UserRole tier, CancellationToken cancellationToken)
{
var name = tier.ToString();
return db.Roles.SingleOrDefaultAsync(r => r.IsSystem && r.Name == name, cancellationToken);
}
/// <summary>
/// Maps identity-provider claim values to a role. Custom roles whose
/// configured claim values match win (ties broken toward the most
/// privileged); otherwise the legacy settings mapping picks a system role.
/// Null means sign-in must be refused (RequireMappedRole with no match).
/// </summary>
public async Task<RoleEntity?> MapOidcRoleAsync(
IReadOnlyCollection<string> claimValues, AuthSettings settings, CancellationToken cancellationToken)
{
var values = claimValues.ToHashSet(StringComparer.OrdinalIgnoreCase);
var candidates = (await db.Roles.AsNoTracking().ToListAsync(cancellationToken))
.Where(r => RoleCache.ParseStrings(r.OidcClaimValuesJson).Any(values.Contains))
.ToList();
if (candidates.Count > 0)
{
return candidates
.OrderByDescending(r => RoleCache.ParseStrings(r.PermissionsJson).Contains(Permissions.All))
.ThenByDescending(r => RoleCache.ParseStrings(r.PermissionsJson).Count)
.ThenBy(r => r.Name, StringComparer.OrdinalIgnoreCase)
.First();
}
var tier = settings.MapRole(claimValues);
return tier is null ? null : await FindSystemRoleAsync(tier.Value, cancellationToken);
}
private static IEnumerable<(string Name, string Description, IReadOnlyList<string> Permissions)> SystemRoles()
{
yield return (nameof(UserRole.Admin), "Full access to everything, including future permissions.",
+182
View File
@@ -0,0 +1,182 @@
using System.Text.Json.Serialization;
using Backend.Data;
using Microsoft.EntityFrameworkCore;
using Shared.Contracts.Serialization;
namespace Backend.Auth;
/// <summary>
/// The kinds of principal a role binding can name: groups from each sign-in
/// method. A <see cref="WindowsGroup"/> binding is by SID, so it may also name
/// one local Windows account by its own SID.
/// </summary>
public static class PrincipalKinds
{
public const string LocalGroup = "local-group";
public const string WindowsGroup = "windows-group";
public const string LdapGroup = "ldap-group";
public const string OidcGroup = "oidc-group";
public static IReadOnlyList<string> All { get; } = [LocalGroup, WindowsGroup, LdapGroup, OidcGroup];
/// <summary>
/// The form two ids of the same principal compare equal in: SIDs upper case,
/// distinguished names lower case without spaces around the separators,
/// GUIDs canonical. OIDC values are compared as the provider sends them.
/// </summary>
public static string Normalize(string kind, string id)
{
var value = id.Trim();
return kind switch
{
WindowsGroup => value.ToUpperInvariant(),
LdapGroup => string.Join(',', value.Split(',').Select(part => string.Join('=', part.Split('=').Select(p => p.Trim())))).ToLowerInvariant(),
LocalGroup => Guid.TryParse(value, out var guid) ? guid.ToString("D") : value.ToLowerInvariant(),
_ => value,
};
}
}
/// <summary>A group a person is in, as resolved at sign-in; stored with the session.</summary>
public sealed record GroupRef(string Kind, string Id, string? Name)
{
[JsonIgnore]
public string Key => $"{Kind}:{PrincipalKinds.Normalize(Kind, Id)}";
}
public static class GroupRefs
{
public static string Serialize(IReadOnlyList<GroupRef> groups) => JsonDefaults.Serialize(groups);
public static IReadOnlyList<GroupRef> Parse(string? json)
{
if (string.IsNullOrEmpty(json))
{
return [];
}
try
{
return JsonDefaults.Deserialize<List<GroupRef>>(json) ?? [];
}
catch (System.Text.Json.JsonException)
{
return [];
}
}
/// <summary>A built-in account's own groups, read from this server.</summary>
public static async Task<IReadOnlyList<GroupRef>> LocalAsync(AppDbContext db, Guid userId, CancellationToken cancellationToken)
{
var groups = await db.LocalGroupMembers.Where(m => m.UserId == userId)
.Join(db.LocalGroups, m => m.GroupId, g => g.Id, (_, g) => new { g.Id, g.Name })
.ToListAsync(cancellationToken);
return groups.ConvertAll(g => new GroupRef(PrincipalKinds.LocalGroup, g.Id.ToString("D"), g.Name));
}
}
/// <summary>One role and the binding that gave it.</summary>
public sealed record Grant(Guid RoleId, string RoleName, Guid BindingId, string PrincipalKind, string PrincipalId, string? PrincipalName);
/// <summary>
/// What a caller may do: the roles their groups are bound to, the union of
/// those roles' permissions, and the union of their device scopes (null: some
/// role reaches every device). The break-glass account holds everything.
/// </summary>
public sealed class EffectiveAccess
{
public static EffectiveAccess None { get; } = new([], [], new HashSet<string>(), []);
public EffectiveAccess(IReadOnlyList<RoleCache.RoleInfo> roles, IReadOnlyList<Grant> grants, IReadOnlySet<string> permissions,
IReadOnlyList<Guid>? scopeGroupIds, bool breakGlass = false)
{
Roles = roles;
Grants = grants;
Permissions = permissions;
ScopeGroupIds = scopeGroupIds;
BreakGlass = breakGlass;
}
public IReadOnlyList<RoleCache.RoleInfo> Roles { get; }
public IReadOnlyList<Grant> Grants { get; }
public IReadOnlySet<string> Permissions { get; }
/// <summary>Device groups the caller is limited to; null reaches every device.</summary>
public IReadOnlyList<Guid>? ScopeGroupIds { get; }
public bool BreakGlass { get; }
public bool Any => BreakGlass || Permissions.Count > 0;
public bool Has(string permission) => BreakGlass || Permissions.Contains(Auth.Permissions.All) || Permissions.Contains(permission);
public static EffectiveAccess FullAccess(IReadOnlyList<RoleCache.RoleInfo> roles) =>
new(roles, [], new HashSet<string>([Auth.Permissions.All], StringComparer.OrdinalIgnoreCase), null, breakGlass: true);
/// <summary>Access from roles alone (claims on a request), without the bindings that gave them.</summary>
public static EffectiveAccess Of(IReadOnlyList<RoleCache.RoleInfo> roles, bool breakGlass)
{
if (breakGlass)
{
return FullAccess(roles);
}
var permissions = roles.SelectMany(r => r.Permissions).ToHashSet(StringComparer.OrdinalIgnoreCase);
// One role that reaches every device lifts the limit; otherwise the scopes add up.
var scoped = roles.Count > 0 && roles.All(r => r.ScopeGroupIds.Count > 0);
return new EffectiveAccess(roles, [], permissions, scoped ? [.. roles.SelectMany(r => r.ScopeGroupIds).Distinct()] : null);
}
}
/// <summary>
/// Turns a caller's groups into roles through the role bindings. Bindings are a
/// small table read through a short cache, so an edit on any node applies
/// everywhere within <see cref="CacheLifetime"/>, and at once on the node that
/// made it.
/// </summary>
public sealed class AccessResolver(IServiceScopeFactory scopes, RoleCache roles)
{
public static readonly TimeSpan CacheLifetime = TimeSpan.FromSeconds(5);
private volatile CachedBindings? cached;
public async ValueTask<EffectiveAccess> ResolveAsync(IEnumerable<GroupRef> principals, bool breakGlass, CancellationToken cancellationToken)
{
var keys = principals.Select(p => p.Key).ToHashSet(StringComparer.Ordinal);
var grants = new List<Grant>();
var bound = new Dictionary<Guid, RoleCache.RoleInfo>();
foreach (var binding in await BindingsAsync(cancellationToken))
{
if (!keys.Contains($"{binding.PrincipalKind}:{PrincipalKinds.Normalize(binding.PrincipalKind, binding.PrincipalId)}")
|| await roles.FindByIdAsync(binding.RoleId, cancellationToken) is not { } role)
{
continue;
}
bound[role.Id] = role;
grants.Add(new Grant(role.Id, role.Name, binding.Id, binding.PrincipalKind, binding.PrincipalId, binding.PrincipalName));
}
var access = EffectiveAccess.Of([.. bound.Values], breakGlass);
return new EffectiveAccess(access.Roles, grants, access.Permissions, access.ScopeGroupIds, access.BreakGlass);
}
public async ValueTask<IReadOnlyList<RoleBindingEntity>> BindingsAsync(CancellationToken cancellationToken)
{
var now = DateTimeOffset.UtcNow;
if (cached is { } current && now < current.Expires)
{
return current.Bindings;
}
using var scope = scopes.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var bindings = await db.RoleBindings.AsNoTracking().ToListAsync(cancellationToken);
cached = new CachedBindings(bindings, now + CacheLifetime);
return bindings;
}
public void Invalidate() => cached = null;
private sealed record CachedBindings(IReadOnlyList<RoleBindingEntity> Bindings, DateTimeOffset Expires);
}
+591
View File
@@ -0,0 +1,591 @@
using Backend.Data;
using Backend.Security;
using Microsoft.EntityFrameworkCore;
using Shared.Contracts.Api;
namespace Backend.Auth;
/// <summary>
/// A person or built-in account. <see cref="LocalGroups"/> are a built-in
/// account's own groups; <see cref="Description"/>, <see cref="Sid"/> and
/// <see cref="WindowsGroups"/> are a Windows account's, read at its latest sign-in.
/// </summary>
public sealed record UserView(string Id, string Kind, string UserName, string? DisplayName, string? Email, bool Enabled, bool BreakGlass,
bool PasswordChangeRequired, DateTimeOffset CreatedUtc, DateTimeOffset? LastSignInUtc, IReadOnlyList<GroupView> LocalGroups, string? Description,
string? Sid, IReadOnlyList<WindowsGroupView> WindowsGroups);
public sealed record UserCreate(string? UserName, string? DisplayName, string? Email, string? Password, bool? GeneratePassword, IReadOnlyList<string>? LocalGroupIds);
/// <summary>The created account, and its generated password when one was asked for: shown in this answer only.</summary>
public sealed record UserCreated(UserView User, string? Password);
public sealed record UserUpdate(string? DisplayName, string? Email, bool? Enabled, IReadOnlyList<string>? LocalGroupIds);
public sealed record PasswordReset(string? Password, bool? Generate);
public sealed record PasswordResetResult(string? Password, int SessionsEnded);
public sealed record GrantView(string RoleId, string RoleName, string BindingId, string PrincipalKind, string PrincipalId, string? PrincipalName);
/// <summary>Effective access with its provenance. For a directory person, the groups are those of their latest session.</summary>
public sealed record AccessView(bool BreakGlass, IReadOnlyList<GroupView> Groups, IReadOnlyList<GrantView> Grants, IReadOnlyList<string> Permissions,
IReadOnlyList<Guid>? DeviceScopeGroupIds);
public sealed record LocalGroupView(string Id, string Name, string? Description, int MemberCount, IReadOnlyList<GroupMemberView> Members,
DateTimeOffset CreatedUtc, IReadOnlyList<string> Roles);
public sealed record GroupMemberView(string Id, string UserName, string? DisplayName);
public sealed record LocalGroupInput(string? Name, string? Description, IReadOnlyList<string>? MemberIds);
public sealed record RoleBindingView(string Id, string RoleId, string RoleName, string PrincipalKind, string PrincipalId, string? PrincipalName,
DateTimeOffset CreatedUtc, string? CreatedBy);
public sealed record RoleBindingInput(string? RoleId, string? PrincipalKind, string? PrincipalId, string? PrincipalName);
/// <summary>Users, built-in groups and role bindings (Access, and each sign-in method's bindings card).</summary>
public static class AccessEndpoints
{
public static void Map(WebApplication app)
{
var manage = Permissions.AccessManage;
app.MapGet(ApiRoutes.AdminUsers, ListUsersAsync).RequirePermission(manage);
app.MapPost(ApiRoutes.AdminUsers, CreateUserAsync).RequirePermission(manage);
app.MapGet(ApiRoutes.AdminUser, async (string id, AppDbContext db, HttpContext context) =>
{
var user = await FindUserAsync(db, id, context.RequestAborted);
return Results.Ok(View(user, await GroupsOfAsync(db, [user.Id], context.RequestAborted)));
}).RequirePermission(manage);
app.MapPut(ApiRoutes.AdminUser, UpdateUserAsync).RequirePermission(manage);
app.MapDelete(ApiRoutes.AdminUser, DeleteUserAsync).RequirePermission(manage);
app.MapPost(ApiRoutes.AdminUserPassword, ResetPasswordAsync).RequirePermission(manage);
app.MapGet(ApiRoutes.AdminUserAccess, UserAccessAsync).RequirePermission(manage);
app.MapGet(ApiRoutes.AdminLocalGroups, ListGroupsAsync).RequirePermission(manage);
app.MapPost(ApiRoutes.AdminLocalGroups, (LocalGroupInput input, AppDbContext db, AccessResolver access, AuditService audit, HttpContext context) =>
SaveGroupAsync(null, input, db, access, audit, context)).RequirePermission(manage);
app.MapPut(ApiRoutes.AdminLocalGroup, (string id, LocalGroupInput input, AppDbContext db, AccessResolver access, AuditService audit, HttpContext context) =>
SaveGroupAsync(id, input, db, access, audit, context)).RequirePermission(manage);
app.MapDelete(ApiRoutes.AdminLocalGroup, DeleteGroupAsync).RequirePermission(manage);
app.MapGet(ApiRoutes.AdminRoleBindings, ListBindingsAsync).RequirePermission(manage);
app.MapPost(ApiRoutes.AdminRoleBindings, CreateBindingAsync).RequirePermission(manage);
app.MapDelete(ApiRoutes.AdminRoleBinding, DeleteBindingAsync).RequirePermission(manage);
}
// ── Users ─────────────────────────────────────────────────────────────
private static async Task<IResult> ListUsersAsync(AppDbContext db, string? kind, CancellationToken cancellationToken)
{
var source = db.Users.AsNoTracking();
if (!string.IsNullOrWhiteSpace(kind))
{
source = source.Where(u => u.Kind == kind);
}
var users = await source.OrderBy(u => u.UserName).ToListAsync(cancellationToken);
var groups = await GroupsOfAsync(db, [.. users.Select(u => u.Id)], cancellationToken);
return Results.Ok(users.Select(u => View(u, groups)).ToList());
}
private static async Task<IResult> CreateUserAsync(UserCreate input, AppDbContext db, AuditService audit, HttpContext context, CancellationToken cancellationToken)
{
var errors = new Dictionary<string, string[]>();
var userName = input.UserName?.Trim() ?? string.Empty;
if (userName.Length is 0 or > 128 || userName.Any(c => char.IsWhiteSpace(c) || char.IsControl(c) || c is '\\' or '/'))
{
errors["userName"] = ["Enter a user name of up to 128 characters, without spaces or slashes."];
}
var generate = input.GeneratePassword == true || string.IsNullOrEmpty(input.Password);
var password = generate ? PasswordHashing.Generate() : input.Password!;
if (!generate)
{
foreach (var (field, messages) in SignInService.PasswordProblems(password, userName, "password"))
{
errors[field] = messages;
}
}
var groups = await ResolveGroupsAsync(db, input.LocalGroupIds, "localGroupIds", errors, cancellationToken);
if (errors.Count > 0)
{
throw AuthProblemException.InvalidFields(errors);
}
var lowered = userName.ToLowerInvariant();
if (await db.Users.AnyAsync(u => u.Kind == SignInProviders.Local && u.UserName.ToLower() == lowered, cancellationToken))
{
throw new AuthProblemException(AuthProblems.Conflict, $"There is already a built-in account named {userName}.");
}
var now = DateTimeOffset.UtcNow;
var user = new UserEntity
{
Id = Guid.NewGuid(),
Kind = SignInProviders.Local,
UserName = userName,
DisplayName = Clean(input.DisplayName),
Email = Clean(input.Email),
PasswordHash = PasswordHashing.Hash(password),
Enabled = true,
MustChangePassword = true,
CreatedUtc = now,
UpdatedUtc = now,
};
db.Users.Add(user);
db.LocalGroupMembers.AddRange(groups.Select(g => new LocalGroupMemberEntity { GroupId = g.Id, UserId = user.Id }));
await db.SaveChangesAsync(cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "auth.user.create", context.User.Identity?.Name,
detail: $"user={user.UserName} groups={string.Join(",", groups.Select(g => g.Name))}", cancellationToken: cancellationToken);
var view = View(user, [.. groups.Select(g => new GroupView(PrincipalKinds.LocalGroup, g.Id.ToString("D"), g.Name))]);
return Results.Created(ApiRoutes.AdminUser.Replace("{id}", user.Id.ToString("D"), StringComparison.Ordinal), new UserCreated(view, generate ? password : null));
}
private static async Task<IResult> UpdateUserAsync(string id, UserUpdate input, AppDbContext db, SessionStore sessions, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var user = await FindUserAsync(db, id, cancellationToken, tracking: true);
var errors = new Dictionary<string, string[]>();
var groups = input.LocalGroupIds is null ? null : await ResolveGroupsAsync(db, input.LocalGroupIds, "localGroupIds", errors, cancellationToken);
if (groups is not null && user.Kind != SignInProviders.Local)
{
errors["localGroupIds"] = ["Only built-in accounts join built-in groups; directory people are in their directory's groups."];
}
if (input.Enabled == false && user.IsBreakGlass)
{
errors["enabled"] = ["The break-glass account cannot be turned off."];
}
if (input.Enabled == false && context.Caller()?.UserId == user.Id)
{
errors["enabled"] = ["You cannot turn off the account you are signed in with."];
}
if (errors.Count > 0)
{
throw AuthProblemException.InvalidFields(errors);
}
user.DisplayName = input.DisplayName is null ? user.DisplayName : Clean(input.DisplayName);
user.Email = input.Email is null ? user.Email : Clean(input.Email);
user.Enabled = input.Enabled ?? user.Enabled;
user.UpdatedUtc = DateTimeOffset.UtcNow;
if (groups is not null)
{
await db.LocalGroupMembers.Where(m => m.UserId == user.Id).ExecuteDeleteAsync(cancellationToken);
db.LocalGroupMembers.AddRange(groups.Select(g => new LocalGroupMemberEntity { GroupId = g.Id, UserId = user.Id }));
}
await db.SaveChangesAsync(cancellationToken);
if (!user.Enabled)
{
await sessions.RevokeAllAsync(user.Id, null, cancellationToken);
}
await audit.LogAsync(AuditCategories.Admin, "auth.user.update", context.User.Identity?.Name,
detail: $"user={user.UserName} enabled={user.Enabled}{(groups is null ? string.Empty : $" groups={string.Join(",", groups.Select(g => g.Name))}")}",
cancellationToken: cancellationToken);
return Results.Ok(View(user, await GroupsOfAsync(db, [user.Id], cancellationToken)));
}
private static async Task<IResult> ResetPasswordAsync(string id, PasswordReset input, AppDbContext db, SessionStore sessions, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var user = await FindUserAsync(db, id, cancellationToken, tracking: true);
if (user.Kind != SignInProviders.Local)
{
throw new AuthProblemException(AuthProblems.Conflict, "Only a built-in account has a password here.");
}
var generate = input.Generate == true || string.IsNullOrEmpty(input.Password);
var password = generate ? PasswordHashing.Generate() : input.Password!;
if (!generate && SignInService.PasswordProblems(password, user.UserName, "password") is { Count: > 0 } errors)
{
throw AuthProblemException.InvalidFields(errors);
}
var now = DateTimeOffset.UtcNow;
user.PasswordHash = PasswordHashing.Hash(password);
user.MustChangePassword = true;
user.PasswordChangedUtc = now;
user.UpdatedUtc = now;
await db.SaveChangesAsync(cancellationToken);
var ended = await sessions.RevokeAllAsync(user.Id, context.Caller()?.SessionId, cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "auth.user.password.reset", context.User.Identity?.Name, detail: $"user={user.UserName}",
cancellationToken: cancellationToken);
return Results.Ok(new PasswordResetResult(generate ? password : null, ended));
}
private static async Task<IResult> DeleteUserAsync(string id, AppDbContext db, AuditService audit, HttpContext context, CancellationToken cancellationToken)
{
var user = await FindUserAsync(db, id, cancellationToken, tracking: true);
if (user.IsBreakGlass)
{
throw new AuthProblemException(AuthProblems.Conflict, "The break-glass account is kept for recovery and cannot be removed.");
}
if (context.Caller()?.UserId == user.Id)
{
throw new AuthProblemException(AuthProblems.Conflict, "You cannot remove the account you are signed in with.");
}
db.Users.Remove(user);
await db.SaveChangesAsync(cancellationToken);
await audit.LogAsync(AuditCategories.Admin, "auth.user.delete", context.User.Identity?.Name, detail: $"user={user.UserName} kind={user.Kind}",
cancellationToken: cancellationToken);
return Results.NoContent();
}
private static async Task<IResult> UserAccessAsync(string id, AppDbContext db, AccessResolver access, CancellationToken cancellationToken)
{
var user = await FindUserAsync(db, id, cancellationToken);
IReadOnlyList<GroupRef> groups = user.Kind == SignInProviders.Local
? await GroupRefs.LocalAsync(db, user.Id, cancellationToken)
: GroupRefs.Parse(await db.Sessions.AsNoTracking().Where(s => s.UserId == user.Id).OrderByDescending(s => s.CreatedUtc)
.Select(s => s.GroupsJson).FirstOrDefaultAsync(cancellationToken));
var granted = await access.ResolveAsync(groups, user.IsBreakGlass, cancellationToken);
return Results.Ok(new AccessView(
user.IsBreakGlass,
[.. groups.Select(g => new GroupView(g.Kind, g.Id, g.Name))],
[.. granted.Grants.Select(g => new GrantView(g.RoleId.ToString("D"), g.RoleName, g.BindingId.ToString("D"), g.PrincipalKind, g.PrincipalId, g.PrincipalName))],
granted.BreakGlass || granted.Permissions.Contains(Permissions.All)
? [.. Permissions.KnownKeys.Order(StringComparer.Ordinal)]
: [.. granted.Permissions.Order(StringComparer.Ordinal)],
granted.ScopeGroupIds));
}
// ── Built-in groups ───────────────────────────────────────────────────
private static async Task<IResult> ListGroupsAsync(AppDbContext db, RoleCache roles, CancellationToken cancellationToken)
{
var groups = await db.LocalGroups.AsNoTracking().OrderBy(g => g.Name).ToListAsync(cancellationToken);
var members = await MembersAsync(db, [.. groups.Select(g => g.Id)], cancellationToken);
var bound = await BoundRolesAsync(db, roles, cancellationToken);
return Results.Ok(groups.Select(g => GroupView(g, members, bound)).ToList());
}
private static async Task<IResult> SaveGroupAsync(string? id, LocalGroupInput input, AppDbContext db, AccessResolver access, AuditService audit,
HttpContext context)
{
var ct = context.RequestAborted;
var group = id is null ? null : await FindGroupAsync(db, id, ct, tracking: true);
var errors = new Dictionary<string, string[]>();
var name = input.Name?.Trim() ?? group?.Name ?? string.Empty;
if (name.Length is 0 or > 128)
{
errors["name"] = ["Enter a name of up to 128 characters."];
}
else if (await db.LocalGroups.AnyAsync(g => g.Name.ToLower() == name.ToLower() && (group == null || g.Id != group.Id), ct))
{
errors["name"] = [$"There is already a built-in group named {name}."];
}
List<UserEntity>? members = null;
if (input.MemberIds is not null)
{
var wanted = input.MemberIds.Select(i => Guid.TryParse(i, out var g) ? g : Guid.Empty).ToList();
members = await db.Users.Where(u => wanted.Contains(u.Id)).ToListAsync(ct);
if (members.Count != wanted.Distinct().Count())
{
errors["memberIds"] = ["Some of those users no longer exist."];
}
else if (members.Any(m => m.Kind != SignInProviders.Local))
{
errors["memberIds"] = ["Only built-in accounts join built-in groups."];
}
}
if (errors.Count > 0)
{
throw AuthProblemException.InvalidFields(errors);
}
var now = DateTimeOffset.UtcNow;
var created = group is null;
if (group is null)
{
group = new LocalGroupEntity { Id = Guid.NewGuid(), Name = name, CreatedUtc = now };
db.LocalGroups.Add(group);
}
group.Name = name;
group.Description = input.Description is null ? group.Description : Clean(input.Description);
group.UpdatedUtc = now;
if (members is not null)
{
await db.LocalGroupMembers.Where(m => m.GroupId == group.Id).ExecuteDeleteAsync(ct);
db.LocalGroupMembers.AddRange(members.Select(m => new LocalGroupMemberEntity { GroupId = group.Id, UserId = m.Id }));
}
// A renamed group keeps its bindings; their display name follows.
var principal = group.Id.ToString("D");
await db.RoleBindings.Where(b => b.PrincipalKind == PrincipalKinds.LocalGroup && b.PrincipalId == principal)
.ExecuteUpdateAsync(set => set.SetProperty(b => b.PrincipalName, name), ct);
await db.SaveChangesAsync(ct);
access.Invalidate();
await audit.LogAsync(AuditCategories.Admin, created ? "auth.local-group.create" : "auth.local-group.update", context.User.Identity?.Name,
detail: $"group={group.Name}{(members is null ? string.Empty : $" members={string.Join(",", members.Select(m => m.UserName))}")}", cancellationToken: ct);
var view = GroupView(group, await MembersAsync(db, [group.Id], ct), await BoundRolesAsync(db, context.RequestServices.GetRequiredService<RoleCache>(), ct));
return created ? Results.Created(ApiRoutes.AdminLocalGroup.Replace("{id}", principal, StringComparison.Ordinal), view) : Results.Ok(view);
}
private static async Task<IResult> DeleteGroupAsync(string id, AppDbContext db, AccessResolver access, RoleCache roles, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var group = await FindGroupAsync(db, id, cancellationToken, tracking: true);
var key = group.Id.ToString("D");
var bindings = await db.RoleBindings.Where(b => b.PrincipalKind == PrincipalKinds.LocalGroup && b.PrincipalId == key).ToListAsync(cancellationToken);
await RequireFullAccessRemainsAsync(db, roles, [.. bindings.Select(b => b.Id)], cancellationToken);
db.RoleBindings.RemoveRange(bindings);
db.LocalGroups.Remove(group);
await db.SaveChangesAsync(cancellationToken);
access.Invalidate();
await audit.LogAsync(AuditCategories.Admin, "auth.local-group.delete", context.User.Identity?.Name, detail: $"group={group.Name}",
cancellationToken: cancellationToken);
return Results.NoContent();
}
// ── Role bindings ─────────────────────────────────────────────────────
private static async Task<IResult> ListBindingsAsync(AppDbContext db, RoleCache roles, string? principalKind, string? roleId,
CancellationToken cancellationToken)
{
var source = db.RoleBindings.AsNoTracking();
if (!string.IsNullOrWhiteSpace(principalKind))
{
source = source.Where(b => b.PrincipalKind == principalKind);
}
if (Guid.TryParse(roleId, out var role))
{
source = source.Where(b => b.RoleId == role);
}
var bindings = await source.OrderBy(b => b.PrincipalName).ToListAsync(cancellationToken);
var views = new List<RoleBindingView>();
foreach (var binding in bindings)
{
views.Add(await BindingViewAsync(binding, roles, cancellationToken));
}
return Results.Ok(views);
}
private static async Task<IResult> CreateBindingAsync(RoleBindingInput input, AppDbContext db, AccessResolver access, RoleCache roles, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var errors = new Dictionary<string, string[]>();
var role = Guid.TryParse(input.RoleId, out var roleId) ? await roles.FindByIdAsync(roleId, cancellationToken) : null;
if (role is null)
{
errors["roleId"] = ["Choose a role."];
}
var kind = input.PrincipalKind ?? string.Empty;
var (principalId, principalName) = await ResolvePrincipalAsync(db, kind, input.PrincipalId, input.PrincipalName, errors, cancellationToken);
if (errors.Count > 0)
{
throw AuthProblemException.InvalidFields(errors);
}
var normalized = PrincipalKinds.Normalize(kind, principalId!);
if (await db.RoleBindings.AnyAsync(b => b.PrincipalKind == kind && b.PrincipalId == normalized && b.RoleId == role!.Id, cancellationToken))
{
throw new AuthProblemException(AuthProblems.Conflict, $"{principalName ?? principalId} already has the {role!.Name} role.");
}
var binding = new RoleBindingEntity
{
Id = Guid.NewGuid(),
RoleId = role!.Id,
PrincipalKind = kind,
PrincipalId = normalized,
PrincipalName = principalName,
CreatedUtc = DateTimeOffset.UtcNow,
CreatedBy = context.User.Identity?.Name,
};
db.RoleBindings.Add(binding);
await db.SaveChangesAsync(cancellationToken);
access.Invalidate();
await audit.LogAsync(AuditCategories.Admin, "auth.role-binding.create", context.User.Identity?.Name,
detail: $"role={role.Name} principal={kind}:{principalName ?? normalized}", cancellationToken: cancellationToken);
return Results.Created(ApiRoutes.AdminRoleBinding.Replace("{id}", binding.Id.ToString("D"), StringComparison.Ordinal),
await BindingViewAsync(binding, roles, cancellationToken));
}
private static async Task<IResult> DeleteBindingAsync(string id, AppDbContext db, AccessResolver access, RoleCache roles, AuditService audit,
HttpContext context, CancellationToken cancellationToken)
{
var binding = Guid.TryParse(id, out var guid) ? await db.RoleBindings.SingleOrDefaultAsync(b => b.Id == guid, cancellationToken) : null;
if (binding is null)
{
throw new AuthProblemException(AuthProblems.NotFound, "There is no such role binding.");
}
await RequireFullAccessRemainsAsync(db, roles, [binding.Id], cancellationToken);
db.RoleBindings.Remove(binding);
await db.SaveChangesAsync(cancellationToken);
access.Invalidate();
await audit.LogAsync(AuditCategories.Admin, "auth.role-binding.delete", context.User.Identity?.Name,
detail: $"role={(await roles.FindByIdAsync(binding.RoleId, cancellationToken))?.Name} principal={binding.PrincipalKind}:{binding.PrincipalName ?? binding.PrincipalId}",
cancellationToken: cancellationToken);
return Results.NoContent();
}
/// <summary>Refuses a change that would leave nobody able to manage access: no break-glass account and no other full-access binding.</summary>
private static async Task RequireFullAccessRemainsAsync(AppDbContext db, RoleCache roles, IReadOnlyList<Guid> removing, CancellationToken cancellationToken)
{
if (removing.Count == 0 || await db.Users.AnyAsync(u => u.IsBreakGlass && u.Enabled, cancellationToken))
{
return;
}
foreach (var binding in await db.RoleBindings.AsNoTracking().Where(b => !removing.Contains(b.Id)).ToListAsync(cancellationToken))
{
if (await roles.FindByIdAsync(binding.RoleId, cancellationToken) is { } role
&& (role.Permissions.Contains(Permissions.All) || role.Permissions.Contains(Permissions.AccessManage)))
{
return;
}
}
throw new AuthProblemException(AuthProblems.Conflict, "This would remove the last way to manage access, and there is no break-glass account to recover with.");
}
private static async Task<(string? Id, string? Name)> ResolvePrincipalAsync(AppDbContext db, string kind, string? id, string? name,
Dictionary<string, string[]> errors, CancellationToken cancellationToken)
{
var value = id?.Trim() ?? string.Empty;
switch (kind)
{
case PrincipalKinds.LocalGroup:
{
var group = Guid.TryParse(value, out var guid) ? await db.LocalGroups.AsNoTracking().SingleOrDefaultAsync(g => g.Id == guid, cancellationToken) : null;
if (group is null)
{
errors["principalId"] = ["Choose a built-in group."];
}
return (group?.Id.ToString("D"), group?.Name);
}
case PrincipalKinds.WindowsGroup:
{
var sid = value.Length == 0 ? null : WindowsSignIn.SidOf(value);
if (sid is null)
{
errors["principalId"] = ["Enter a Windows account or group this server can resolve (MACHINE\\Name or DOMAIN\\Group), or its SID (S-1-5-…)."];
}
return (sid, Clean(name) ?? (sid is null ? null : WindowsSignIn.NameOf(sid) ?? (value.StartsWith("S-1-", StringComparison.OrdinalIgnoreCase) ? null : value)));
}
case PrincipalKinds.LdapGroup:
if (value.Length is 0 or > 450 || !value.Contains('=', StringComparison.Ordinal))
{
errors["principalId"] = ["Enter the group's distinguished name, for example cn=helpdesk,ou=groups,dc=example,dc=com."];
}
return (value, Clean(name) ?? SignInService.FirstRdn(value));
case PrincipalKinds.OidcGroup:
if (value.Length is 0 or > 450 || !value.Contains('=', StringComparison.Ordinal))
{
errors["principalId"] = ["Enter the claim and its value as claim=value, for example groups=executer-admins."];
}
return (value, Clean(name) ?? value);
default:
errors["principalKind"] = [$"Choose one of: {string.Join(", ", PrincipalKinds.All)}."];
return (null, null);
}
}
private static async Task<List<LocalGroupEntity>> ResolveGroupsAsync(AppDbContext db, IReadOnlyList<string>? ids, string field,
Dictionary<string, string[]> errors, CancellationToken cancellationToken)
{
if (ids is null || ids.Count == 0)
{
return [];
}
var wanted = ids.Select(i => Guid.TryParse(i, out var g) ? g : Guid.Empty).Distinct().ToList();
var found = await db.LocalGroups.Where(g => wanted.Contains(g.Id)).ToListAsync(cancellationToken);
if (found.Count != wanted.Count)
{
errors[field] = ["Some of those built-in groups no longer exist."];
}
return found;
}
private static async Task<Dictionary<Guid, List<GroupView>>> GroupsOfAsync(AppDbContext db, IReadOnlyList<Guid> userIds, CancellationToken cancellationToken)
{
var rows = await db.LocalGroupMembers.AsNoTracking().Where(m => userIds.Contains(m.UserId))
.Join(db.LocalGroups, m => m.GroupId, g => g.Id, (m, g) => new { m.UserId, g.Id, g.Name }).ToListAsync(cancellationToken);
return rows.GroupBy(r => r.UserId).ToDictionary(g => g.Key, g => g.Select(r => new GroupView(PrincipalKinds.LocalGroup, r.Id.ToString("D"), r.Name))
.OrderBy(v => v.Name, StringComparer.OrdinalIgnoreCase).ToList());
}
private static async Task<Dictionary<Guid, List<GroupMemberView>>> MembersAsync(AppDbContext db, IReadOnlyList<Guid> groupIds, CancellationToken cancellationToken)
{
var rows = await db.LocalGroupMembers.AsNoTracking().Where(m => groupIds.Contains(m.GroupId))
.Join(db.Users, m => m.UserId, u => u.Id, (m, u) => new { m.GroupId, u.Id, u.UserName, u.DisplayName }).ToListAsync(cancellationToken);
return rows.GroupBy(r => r.GroupId).ToDictionary(g => g.Key, g => g.Select(r => new GroupMemberView(r.Id.ToString("D"), r.UserName, r.DisplayName))
.OrderBy(v => v.UserName, StringComparer.OrdinalIgnoreCase).ToList());
}
/// <summary>Role names bound to each built-in group, by group id.</summary>
private static async Task<Dictionary<string, List<string>>> BoundRolesAsync(AppDbContext db, RoleCache roles, CancellationToken cancellationToken)
{
var bindings = await db.RoleBindings.AsNoTracking().Where(b => b.PrincipalKind == PrincipalKinds.LocalGroup).ToListAsync(cancellationToken);
var result = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var binding in bindings)
{
if (await roles.FindByIdAsync(binding.RoleId, cancellationToken) is { } role)
{
(result.TryGetValue(binding.PrincipalId, out var list) ? list : result[binding.PrincipalId] = []).Add(role.Name);
}
}
return result;
}
private static async Task<UserEntity> FindUserAsync(AppDbContext db, string id, CancellationToken cancellationToken, bool tracking = false)
{
var source = tracking ? db.Users : db.Users.AsNoTracking();
return (Guid.TryParse(id, out var guid) ? await source.SingleOrDefaultAsync(u => u.Id == guid, cancellationToken) : null)
?? throw new AuthProblemException(AuthProblems.NotFound, "There is no such user.");
}
private static async Task<LocalGroupEntity> FindGroupAsync(AppDbContext db, string id, CancellationToken cancellationToken, bool tracking = false)
{
var source = tracking ? db.LocalGroups : db.LocalGroups.AsNoTracking();
return (Guid.TryParse(id, out var guid) ? await source.SingleOrDefaultAsync(g => g.Id == guid, cancellationToken) : null)
?? throw new AuthProblemException(AuthProblems.NotFound, "There is no such built-in group.");
}
private static UserView View(UserEntity user, Dictionary<Guid, List<GroupView>> groups) => View(user, groups.GetValueOrDefault(user.Id, []));
private static UserView View(UserEntity user, IReadOnlyList<GroupView> groups) => new(user.Id.ToString("D"), user.Kind, user.UserName, user.DisplayName,
user.Email, user.Enabled, user.IsBreakGlass, user.MustChangePassword && user.Kind == SignInProviders.Local, user.CreatedUtc, user.LastSignInUtc, groups,
user.Description, SignInService.SidOf(user), WindowsGroups.Parse(user.WindowsGroupsJson));
private static LocalGroupView GroupView(LocalGroupEntity group, Dictionary<Guid, List<GroupMemberView>> members, Dictionary<string, List<string>> roles)
{
var list = members.GetValueOrDefault(group.Id, []);
return new LocalGroupView(group.Id.ToString("D"), group.Name, group.Description, list.Count, list, group.CreatedUtc,
[.. roles.GetValueOrDefault(group.Id.ToString("D"), []).Order(StringComparer.OrdinalIgnoreCase)]);
}
private static async Task<RoleBindingView> BindingViewAsync(RoleBindingEntity b, RoleCache roles, CancellationToken cancellationToken) =>
new(b.Id.ToString("D"), b.RoleId.ToString("D"), (await roles.FindByIdAsync(b.RoleId, cancellationToken))?.Name ?? "(deleted role)", b.PrincipalKind,
b.PrincipalId, b.PrincipalName, b.CreatedUtc, b.CreatedBy);
private static string? Clean(string? value) => value?.Trim() is { Length: > 0 } text ? text : null;
}

Some files were not shown because too many files have changed in this diff Show More