diff --git a/cmd/pulse-agent-runner/main.go b/cmd/pulse-agent-runner/main.go index 6543005e2..69a3a22b6 100644 --- a/cmd/pulse-agent-runner/main.go +++ b/cmd/pulse-agent-runner/main.go @@ -25,6 +25,7 @@ type runtimeConfig struct { TokenFile string StateDir string HealthFile string + ActivationNonce string AgentIDFile string Hostname string ServerFingerprint string @@ -38,14 +39,15 @@ func loadConfig() (runtimeConfig, error) { TokenFile: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_TOKEN_FILE")), StateDir: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_STATE_DIR")), HealthFile: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_HEALTH_FILE")), + ActivationNonce: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_ACTIVATION_NONCE")), AgentIDFile: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE")), Hostname: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_HOSTNAME")), ServerFingerprint: strings.TrimSpace(os.Getenv("PULSE_SERVER_FINGERPRINT")), CAFile: strings.TrimSpace(os.Getenv("SSL_CERT_FILE")), Insecure: strings.EqualFold(strings.TrimSpace(os.Getenv("PULSE_INSECURE")), "true"), } - if config.PulseURL == "" || config.TokenFile == "" || config.StateDir == "" || config.HealthFile == "" || config.AgentIDFile == "" { - return runtimeConfig{}, errors.New("PULSE_URL and the action-runner token, state, health, and agent identity file settings are required") + if config.PulseURL == "" || config.TokenFile == "" || config.StateDir == "" || config.HealthFile == "" || config.AgentIDFile == "" || len(config.ActivationNonce) < 32 || len(config.ActivationNonce) > 128 { + return runtimeConfig{}, errors.New("PULSE_URL and the action-runner token, state, health, agent identity, and activation nonce settings are required") } if config.Hostname != "" { hostname, err := normalizeRunnerHostname(config.Hostname) @@ -105,7 +107,8 @@ func run() error { transportConfig := actionrunner.TransportConfig{ PulseURL: config.PulseURL, APIToken: token, StateDir: config.StateDir, HealthPath: config.HealthFile, InsecureSkipVerify: config.Insecure, - CACertPath: config.CAFile, ServerFingerprint: config.ServerFingerprint, + ActivationNonce: config.ActivationNonce, + CACertPath: config.CAFile, ServerFingerprint: config.ServerFingerprint, Logger: &logger, } containerRuntime, runtimeErr := dockeragent.NewActionRuntime(strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_CONTAINER_RUNTIME")), &logger) diff --git a/cmd/pulse-agent-runner/main_test.go b/cmd/pulse-agent-runner/main_test.go index 128c9acf5..7f0df22ae 100644 --- a/cmd/pulse-agent-runner/main_test.go +++ b/cmd/pulse-agent-runner/main_test.go @@ -22,6 +22,7 @@ func TestLoadConfigUsesDedicatedEnvironmentAndPrivateTokenFile(t *testing.T) { t.Setenv("PULSE_AGENT_RUNNER_STATE_DIR", filepath.Join(dir, "state")) t.Setenv("PULSE_AGENT_RUNNER_HEALTH_FILE", filepath.Join(dir, "state", "health.json")) t.Setenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE", agentID) + t.Setenv("PULSE_AGENT_RUNNER_ACTIVATION_NONCE", strings.Repeat("a", 32)) t.Setenv("PULSE_AGENT_RUNNER_HOSTNAME", " Node.Example. ") t.Setenv("PULSE_SERVER_FINGERPRINT", "sha256:test") config, err := loadConfig() @@ -44,6 +45,7 @@ func TestLoadConfigRejectsInvalidCanonicalHostnameOverride(t *testing.T) { t.Setenv("PULSE_AGENT_RUNNER_STATE_DIR", filepath.Join(dir, "state")) t.Setenv("PULSE_AGENT_RUNNER_HEALTH_FILE", filepath.Join(dir, "state", "health.json")) t.Setenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE", filepath.Join(dir, "agent-id")) + t.Setenv("PULSE_AGENT_RUNNER_ACTIVATION_NONCE", strings.Repeat("b", 32)) for _, hostname := range []string{"bad host", "-node.example", "node/example", strings.Repeat("a", 64) + ".example"} { t.Run(hostname, func(t *testing.T) { t.Setenv("PULSE_AGENT_RUNNER_HOSTNAME", hostname) @@ -67,6 +69,7 @@ func TestLoadConfigRejectsTokenInArgvEquivalentAndInsecureHTTPByDefault(t *testi t.Setenv("PULSE_AGENT_RUNNER_STATE_DIR", filepath.Join(dir, "state")) t.Setenv("PULSE_AGENT_RUNNER_HEALTH_FILE", filepath.Join(dir, "state", "health.json")) t.Setenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE", filepath.Join(dir, "agent-id")) + t.Setenv("PULSE_AGENT_RUNNER_ACTIVATION_NONCE", strings.Repeat("c", 32)) _, err := loadConfig() if err == nil || !strings.Contains(err.Error(), "HTTPS") { t.Fatalf("error = %v", err) diff --git a/docs/AGENT_SECURITY.md b/docs/AGENT_SECURITY.md index 45288f745..5b5ea5362 100644 --- a/docs/AGENT_SECURITY.md +++ b/docs/AGENT_SECURITY.md @@ -196,12 +196,12 @@ or generic command path. | Platform or capability | Preferred boundary | Safe-profile behavior | Qualification and default implication | Residual owner and removal condition | |---|---|---|---|---| | Proxmox VE/PBS/PMG inventory, status, storage, and ordinary metrics | API-only connection with a narrowly scoped token; no host agent | No collector, helper, or runner authority is required for this data | Supported independently of the safe host-agent profile; it does not prove host-local SMART, LXC filesystem, or action parity | `agent-lifecycle`: keep API permissions and returned telemetry covered by provider tests | -| Standard Linux systemd host telemetry and collector update | Unprivileged `pulse-agent` plus the root-owned typed helper | Core `/proc`, filesystem, network, RAID, and hwmon telemetry stays in the collector; signed update activation crosses the fixed helper transaction | **Qualified on disposable Ubuntu 24.04.4 arm64 at committed main** for install, migration, explicit/automatic rollback, update, helper health, reporting continuity, and process/credential separation. Remains opt-in pending exact-RC reproduction and external review | `deployment-installability` and `security-privacy`: reproduce the twelve-scenario receipt from the designated release candidate and accept the external boundary review | +| Standard Linux systemd host telemetry and collector update | Unprivileged `pulse-agent` plus the root-owned typed helper | Core `/proc`, filesystem, network, RAID, and hwmon telemetry stays in the collector; helper-backed signed update activation is implemented but its live activation/recovery transaction is not qualified | **Qualified on disposable Ubuntu 24.04.4 arm64 at committed main** for install, migration, explicit/automatic profile rollback, helper health, reporting continuity, and process/credential separation. The receipt's ordinary update ran under the downgraded root monitoring profile and does not prove `agent_update.activate.v1`, executable-digest commit, watchdog rollback, interrupted recovery, or last-known-good restoration. Remains opt-in pending those live scenarios, exact-RC reproduction, and external review | `deployment-installability` and `security-privacy`: qualify helper activation/failure/recovery from the designated release candidate and accept the external boundary review | | Linux SMART telemetry | `smart.snapshot` through the no-network helper; no caller-selected device or arguments | Implemented, unqualified on representative physical disks. Helper failure omits/degrades SMART only; the collector does not retry as root | Does not yet justify SMART parity or a default change | `agent-lifecycle`: record live SATA, SAS/controller, USB bridge, and NVMe evidence, including standby, permission failure, timeout, and partial-data cases | | Proxmox node-local LXC filesystem telemetry | `proxmox.lxc_filesystems` through the no-network helper using fixed bounded `pct` operations | Implemented, unqualified on a representative PVE node. Helper failure omits/degrades this snapshot only | Does not yet justify Proxmox host-agent parity or a default change | `agent-lifecycle`: record live running/stopped LXC, mount, timeout, output-bound, and helper-loss behavior on supported PVE versions | | Rootful Docker or Podman inventory | No direct collector access to a root-equivalent daemon socket | **Unavailable in the safe profile.** Migration disables the provider visibly. A closed helper `container.inventory` operation exists, but collector integration and live parity are not qualified | Rootful container parity is an explicit default blocker; the legacy/root profile is not safe-profile evidence | `agent-lifecycle`: either integrate and qualify bounded helper inventory or retain the explicit degradation permanently | | Collector-owned rootless Docker or Podman | Direct access only to one usable runtime socket owned by the `pulse-agent` UID | Implemented, unqualified live. Ambiguous, root-owned, unreadable, unwritable, or unavailable sockets disable container monitoring | Does not yet justify container-runtime parity or a default change | `deployment-installability`: record fresh install, migration, restart, socket-loss, ambiguity, and telemetry parity on both rootless Docker and rootless Podman | -| Separate runner package update and package-cache cleanup | Root-owned `pulse-agent-runner`, host-bound action credential, typed request, postcondition, and durable receipt | A real verified apt-cache mutation, stale-fingerprint refusal, replay, rotation, and self-revocation are qualified in the committed-main systemd receipt | Qualified only for the exercised apt-cache mutation; it does not qualify every runner operation or an exact release candidate | `agent-lifecycle` and `api-contracts`: reproduce from the RC and add representative package-update success/failure/cancellation evidence | +| Separate runner package update and package-cache cleanup | Root-owned `pulse-agent-runner`, host-bound action credential, typed request, postcondition, and durable receipt | A real verified apt-cache mutation, stale-fingerprint refusal, replay, and self-revocation are present in the committed-main systemd receipt. Two-phase credential activation and nonce-bound readiness are implemented but need fresh production-path qualification | Qualified only for the exercised apt-cache mutation; the receipt does not prove the current Router/TLS/durable-persistence credential lifecycle, every runner operation, or an exact release candidate | `agent-lifecycle` and `api-contracts`: reproduce through the production Router over HTTPS from the RC, including failed activation/rollback and representative package-update success/failure/cancellation evidence | | Separate runner Proxmox guest and container lifecycle/update actions | Root-owned runner with closed typed protocols; never the monitoring collector | Implemented, unqualified on representative PVE and container-runtime targets | No live-provider action-parity claim and no default change | `agent-lifecycle`: record target-bound success, stale-state refusal, cancellation, reconnect/replay, and independent postconditions on disposable real targets | | Appliance, non-systemd, Windows, and macOS host-agent profiles | Platform API where sufficient; otherwise an explicitly named legacy/full-trust profile | **Unavailable for safe-profile apply.** The installer fails closed instead of silently installing a root-equivalent profile | Excluded from the Linux safe-profile claim | `deployment-installability`: land a platform-specific service, filesystem, update, helper, migration, rollback, and live-proof contract before marking that platform supported | diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 640e7ffbe..6574bb5df 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -106,18 +106,28 @@ proving generic command denial. The runner service persists the same normalized canonical hostname used when its credential was issued; it must not substitute the machine's incidental OS hostname when the collector was enrolled under an override. Credential -rotation invalidates exactly the superseded organization/token/agent/hostname -session only after the replacement token inventory is durably stored. That -invalidation removes the session from dispatch, closes its transport, and -unblocks server-side waits; an already-started host mutation remains governed -by its typed receipt and best-effort cancellation semantics rather than being -described as rolled back. Runner uninstall attempts an authenticated +rotation is a prepare/commit transaction. Issuance stores a ten-minute pending +replacement beside the active predecessor; the pending transport may register +but is not dispatchable. The runner first durably replaces a private health +marker carrying the current installer-generated activation nonce, then calls +the authenticated activation method. Activation atomically removes the exact +predecessor set, removes the replacement expiry/pending state, and promotes the +exact registered session. Failure before that commit leaves the predecessor +valid, while persistence failure restores both records. The installer removes +the prior marker before restart, ignores wall-clock mtime as authority, and +accepts only an activated marker whose nonce and canonical agent ID match the +current attempt; rollback never restores a prior marker. An already-started +host mutation remains governed by its typed receipt and best-effort +cancellation semantics rather than being described as rolled back. Runner +uninstall attempts an authenticated credential self-revoke before deleting local state. The delete route may remove only the caller's exact host-bound action-runner record; an unreachable server cannot prevent local runner removal and leaves an explicit operator revocation residual. Runner readiness is exposed through a bounded, secret-free health marker that -is replaced atomically only after its contents reach stable storage. POSIX +is replaced atomically only after its contents reach stable storage. The +marker distinguishes registered/pending from activated and carries only the +per-attempt nonce, never the bearer credential. POSIX targets must sync the containing directory after rename; Windows targets must use a write-through replacement rather than attempting to flush the read-only directory handle returned by the standard library. The marker and helper-update diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 44bad58f0..7d7038b91 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -39,12 +39,22 @@ monitored `agentId` / `hostname` pair and returns a new, separately persisted runtime role `action-runner`, binding version 1, and capability `typed_actions.v1`. The route re-resolves the pair against exactly one live, non-conflicted, non-integration host in the request tenant. Issuing again -atomically replaces only the earlier action-runner record for that tenant and -agent ID; a persistence failure restores the complete token inventory and -returns no usable new secret. Only after that durable replacement may the -server close the exact superseded live action-runner session, remove it from -new dispatch, and unblock its outstanding server waits. The created response -is non-cacheable. +atomically prepares a ten-minute replacement beside the current active record; +older unactivated replacements are removed, while persistence failure restores +the complete token inventory and returns no usable new secret. The created +response is non-cacheable and exposes the activation-pending state and +deadline, not predecessor secrets. + +`PATCH /api/agents/action-runner/credential` is the runner-owned activation +commit. It requires the current `agent:exec` bearer, exact organization, +canonical agent/hostname binding, and an exact registered pending transport. +A pending transport is visible to this commit proof but unavailable to action +dispatch. The commit clears the pending expiry, durably removes only the +server-recorded predecessor IDs, restores the complete inventory on persistence +failure, promotes the exact replacement session, and invalidates exact stale +sessions without allowing a caller-selected token ID. Repeating activation for +an already active exact session is idempotent so a lost HTTP response can be +reconciled safely. `DELETE /api/agents/action-runner/credential` is the runner's narrowly scoped self-revoke operation. It requires the current `agent:exec` bearer credential, diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index e46065d9d..609b8013f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -146,9 +146,14 @@ The same supported Linux systemd profile can install `pulse-agent-runner` only through the separate `--enable-action-runner` choice, a private token file, and the already selected typed helper profile. The runner binary, unit, configuration, credential, health record, and receipt database are root-owned -and independent from collector state. Activation is transactional and restores -the previous runner-only files if health does not become current; disable and -uninstall remove only remediation and leave monitoring running. The action +and independent from collector state. Activation is transactional: the server +keeps the prior credential valid while a bounded replacement registers without +dispatch authority; the runner durably writes an installer-nonce-bound health +marker and commits activation before the installer removes backups. The +installer deletes the prior marker before restart, does not trust filesystem +timestamps, never restores a prior marker, and restores the previous runner-only +files if the current nonce never reaches activated state. Disable and uninstall +remove only remediation and leave monitoring running. The action credential is never placed in argv or reused as the collector token. The installer persists the canonical enrollment hostname for runner admission and uses a private curl configuration for best-effort exact self-revocation before diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index 28d0a6d89..f814afe21 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -89,16 +89,21 @@ set, so `ProtectSystem=strict` is not a valid runner sandbox claim. Its separate credential, no-listener transport, typed admission, target and digest binding, bounded capabilities, state privacy, and durable receipts are the compensating boundary; collector and helper filesystem hardening remains independent. -Action-runner issuance and rotation are one durable host-bound transition per -organization and canonical agent ID. Re-issuance replaces the prior runner -record even when the monitored hostname has changed, returns the new plaintext -secret only after persistence succeeds, and restores the complete prior token -inventory if persistence fails. A successful rotation invalidates the previous -secret immediately; it never widens the collector credential or turns a -monitoring session into an action session. +Action-runner issuance and rotation are a two-phase durable host-bound +transition per organization and canonical agent ID. Re-issuance prepares a +ten-minute replacement credential while retaining the prior active record, +returns the new plaintext secret only after persistence succeeds, and removes +older unactivated replacements. The pending credential may authenticate one +exact runner transport but cannot become dispatch authority. After the runner +durably records its current activation nonce, its authenticated activation +request atomically promotes the replacement and revokes the server-recorded +predecessor set. A persistence failure restores both sides of the transition; +an unactivated replacement expires without revoking the predecessor. That invalidation is exact and post-persistence: it matches organization, token, canonical agent, hostname, runtime role, and typed capability before -closing the session, so a stale rotation cannot evict a replacement. The +closing the session, so a stale rotation cannot evict a replacement. Activation +is idempotent for the exact live session so transport-level response loss is +recoverable. The runner may also delete only its own matching record using its bearer credential; browser sessions and a caller-selected token ID are rejected, and persistence failure restores the prior inventory. Installer teardown keeps the diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index f234e58a8..57943561f 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -34,10 +34,12 @@ preserves collector identity and installation files only; it must not rewrite storage/recovery evidence or reinterpret a restored legacy collector profile as recovery success. Typed `host.storage_cleanup` remains governed remediation, not storage-recovery authority inferred from the credential itself. -Runner credential rotation follows the shared token-inventory commit boundary: -failed persistence restores the complete prior inventory and returns no new -secret, while successful persistence replaces the previous organization/agent -binding and invalidates only that superseded live session. Exact bearer +Runner credential rotation follows the shared two-phase token-inventory commit +boundary: issuance durably prepares a bounded, non-dispatchable replacement +without removing the active predecessor; activation after durable runner health +proof promotes the exact registered replacement and removes only its recorded +predecessor set. Failed persistence restores the complete inventory, and an +unactivated replacement expires without changing the predecessor. Exact bearer self-revocation uses the same durable boundary and cannot select another token, organization, or host. That rollback protects restart-time credential truth only; it is not a customer backup, recovery point, restore diff --git a/frontend-modern/public/docs/AGENT_SECURITY.md b/frontend-modern/public/docs/AGENT_SECURITY.md index 45288f745..5b5ea5362 100644 --- a/frontend-modern/public/docs/AGENT_SECURITY.md +++ b/frontend-modern/public/docs/AGENT_SECURITY.md @@ -196,12 +196,12 @@ or generic command path. | Platform or capability | Preferred boundary | Safe-profile behavior | Qualification and default implication | Residual owner and removal condition | |---|---|---|---|---| | Proxmox VE/PBS/PMG inventory, status, storage, and ordinary metrics | API-only connection with a narrowly scoped token; no host agent | No collector, helper, or runner authority is required for this data | Supported independently of the safe host-agent profile; it does not prove host-local SMART, LXC filesystem, or action parity | `agent-lifecycle`: keep API permissions and returned telemetry covered by provider tests | -| Standard Linux systemd host telemetry and collector update | Unprivileged `pulse-agent` plus the root-owned typed helper | Core `/proc`, filesystem, network, RAID, and hwmon telemetry stays in the collector; signed update activation crosses the fixed helper transaction | **Qualified on disposable Ubuntu 24.04.4 arm64 at committed main** for install, migration, explicit/automatic rollback, update, helper health, reporting continuity, and process/credential separation. Remains opt-in pending exact-RC reproduction and external review | `deployment-installability` and `security-privacy`: reproduce the twelve-scenario receipt from the designated release candidate and accept the external boundary review | +| Standard Linux systemd host telemetry and collector update | Unprivileged `pulse-agent` plus the root-owned typed helper | Core `/proc`, filesystem, network, RAID, and hwmon telemetry stays in the collector; helper-backed signed update activation is implemented but its live activation/recovery transaction is not qualified | **Qualified on disposable Ubuntu 24.04.4 arm64 at committed main** for install, migration, explicit/automatic profile rollback, helper health, reporting continuity, and process/credential separation. The receipt's ordinary update ran under the downgraded root monitoring profile and does not prove `agent_update.activate.v1`, executable-digest commit, watchdog rollback, interrupted recovery, or last-known-good restoration. Remains opt-in pending those live scenarios, exact-RC reproduction, and external review | `deployment-installability` and `security-privacy`: qualify helper activation/failure/recovery from the designated release candidate and accept the external boundary review | | Linux SMART telemetry | `smart.snapshot` through the no-network helper; no caller-selected device or arguments | Implemented, unqualified on representative physical disks. Helper failure omits/degrades SMART only; the collector does not retry as root | Does not yet justify SMART parity or a default change | `agent-lifecycle`: record live SATA, SAS/controller, USB bridge, and NVMe evidence, including standby, permission failure, timeout, and partial-data cases | | Proxmox node-local LXC filesystem telemetry | `proxmox.lxc_filesystems` through the no-network helper using fixed bounded `pct` operations | Implemented, unqualified on a representative PVE node. Helper failure omits/degrades this snapshot only | Does not yet justify Proxmox host-agent parity or a default change | `agent-lifecycle`: record live running/stopped LXC, mount, timeout, output-bound, and helper-loss behavior on supported PVE versions | | Rootful Docker or Podman inventory | No direct collector access to a root-equivalent daemon socket | **Unavailable in the safe profile.** Migration disables the provider visibly. A closed helper `container.inventory` operation exists, but collector integration and live parity are not qualified | Rootful container parity is an explicit default blocker; the legacy/root profile is not safe-profile evidence | `agent-lifecycle`: either integrate and qualify bounded helper inventory or retain the explicit degradation permanently | | Collector-owned rootless Docker or Podman | Direct access only to one usable runtime socket owned by the `pulse-agent` UID | Implemented, unqualified live. Ambiguous, root-owned, unreadable, unwritable, or unavailable sockets disable container monitoring | Does not yet justify container-runtime parity or a default change | `deployment-installability`: record fresh install, migration, restart, socket-loss, ambiguity, and telemetry parity on both rootless Docker and rootless Podman | -| Separate runner package update and package-cache cleanup | Root-owned `pulse-agent-runner`, host-bound action credential, typed request, postcondition, and durable receipt | A real verified apt-cache mutation, stale-fingerprint refusal, replay, rotation, and self-revocation are qualified in the committed-main systemd receipt | Qualified only for the exercised apt-cache mutation; it does not qualify every runner operation or an exact release candidate | `agent-lifecycle` and `api-contracts`: reproduce from the RC and add representative package-update success/failure/cancellation evidence | +| Separate runner package update and package-cache cleanup | Root-owned `pulse-agent-runner`, host-bound action credential, typed request, postcondition, and durable receipt | A real verified apt-cache mutation, stale-fingerprint refusal, replay, and self-revocation are present in the committed-main systemd receipt. Two-phase credential activation and nonce-bound readiness are implemented but need fresh production-path qualification | Qualified only for the exercised apt-cache mutation; the receipt does not prove the current Router/TLS/durable-persistence credential lifecycle, every runner operation, or an exact release candidate | `agent-lifecycle` and `api-contracts`: reproduce through the production Router over HTTPS from the RC, including failed activation/rollback and representative package-update success/failure/cancellation evidence | | Separate runner Proxmox guest and container lifecycle/update actions | Root-owned runner with closed typed protocols; never the monitoring collector | Implemented, unqualified on representative PVE and container-runtime targets | No live-provider action-parity claim and no default change | `agent-lifecycle`: record target-bound success, stale-state refusal, cancellation, reconnect/replay, and independent postconditions on disposable real targets | | Appliance, non-systemd, Windows, and macOS host-agent profiles | Platform API where sufficient; otherwise an explicitly named legacy/full-trust profile | **Unavailable for safe-profile apply.** The installer fails closed instead of silently installing a root-equivalent profile | Excluded from the Linux safe-profile claim | `deployment-installability`: land a platform-specific service, filesystem, update, helper, migration, rollback, and live-proof contract before marking that platform supported | diff --git a/internal/agentexec/server.go b/internal/agentexec/server.go index 92e68bab9..ca51884c7 100644 --- a/internal/agentexec/server.go +++ b/internal/agentexec/server.go @@ -97,12 +97,13 @@ type organizationContextKey struct{} // session. The raw bearer token is deliberately not retained after // registration. type AgentAdmission struct { - OrganizationID string - TokenID string - AgentID string - Hostname string - RuntimeRole string - ActionCapability string + OrganizationID string + TokenID string + AgentID string + Hostname string + RuntimeRole string + ActionCapability string + ActivationPending bool } // AgentRegistrationValidator authenticates and binds a registration to one @@ -357,6 +358,13 @@ func (s *Server) connectionForOrganization(organizationID, agentID string) (*age if !ok { return nil, false } + // A prepared action-runner credential may establish the authenticated + // transport needed to commit activation, but it is not dispatch authority. + // Promotion clears this bit only after the server durably revokes the prior + // credential set. + if ac.admission.ActivationPending { + return nil, false + } if s.validateSession == nil || s.validateSession(ac.admission) { return ac, true } @@ -376,6 +384,38 @@ func (s *Server) connectionForOrganization(organizationID, agentID string) (*age return nil, false } +// HasActionRunnerSession reports whether the exact runner transport is +// currently registered. It intentionally bypasses dispatch readiness: the +// activation endpoint uses this proof to promote a pending session. +func (s *Server) HasActionRunnerSession(admission AgentAdmission) bool { + if s == nil { + return false + } + key := agentSessionKey(admission.OrganizationID, admission.AgentID) + s.mu.RLock() + current, ok := s.agents[key] + s.mu.RUnlock() + return ok && current != nil && sameActionRunnerAdmission(current.admission, admission) && + current.admission.ActivationPending == admission.ActivationPending +} + +// PromoteActionRunnerSession makes an exact prepared session dispatchable +// after its credential activation has been durably committed. +func (s *Server) PromoteActionRunnerSession(admission AgentAdmission) bool { + if s == nil { + return false + } + key := agentSessionKey(admission.OrganizationID, admission.AgentID) + s.mu.Lock() + defer s.mu.Unlock() + current, ok := s.agents[key] + if !ok || current == nil || !sameActionRunnerAdmission(current.admission, admission) { + return false + } + current.admission.ActivationPending = false + return true +} + // InvalidateActionRunnerSession closes exactly the currently admitted typed // action-runner session identified by admission. A stale rotation result must // never evict a replacement session that has already registered for the same @@ -1167,7 +1207,10 @@ func (s *Server) HandleWebSocket(w http.ResponseWriter, r *http.Request) { } // Close existing connection if any if existing, ok := s.agents[ac.sessionKey]; ok { - if !unifiedresources.HostnamesEquivalent(existing.agent.Hostname, ac.agent.Hostname) { + preparedActionRunnerRename := admission.ActivationPending && + admission.RuntimeRole == RuntimeRoleActionRunner && + existing.admission.RuntimeRole == RuntimeRoleActionRunner + if !unifiedresources.HostnamesEquivalent(existing.agent.Hostname, ac.agent.Hostname) && !preparedActionRunnerRename { s.mu.Unlock() log.Warn(). Str("organization_id", admission.OrganizationID). diff --git a/internal/agentexec/server_test.go b/internal/agentexec/server_test.go index 1b8382d0d..31b6a96b2 100644 --- a/internal/agentexec/server_test.go +++ b/internal/agentexec/server_test.go @@ -481,3 +481,28 @@ func TestGetAgentForHostKeepsDistinctFQDNsSeparate(t *testing.T) { t.Fatalf("GetAgentForHost(%q) = (%q, %v), want (a2, true)", "prox97.b.local", agentID, ok) } } + +func TestPreparedActionRunnerSessionIsNotDispatchableUntilPromoted(t *testing.T) { + s := NewServerWithAdmissionValidator(func(string, string, string) (AgentAdmission, bool) { + return AgentAdmission{}, false + }, func(AgentAdmission) bool { return true }) + admission := AgentAdmission{ + OrganizationID: "org-a", TokenID: "pending-token", AgentID: "agent-1", + Hostname: "host-1.local", RuntimeRole: RuntimeRoleActionRunner, + ActionCapability: ActionCapabilityTypedV1, ActivationPending: true, + } + key := agentSessionKey(admission.OrganizationID, admission.AgentID) + s.agents[key] = &agentConn{admission: admission, agent: ConnectedAgent{AgentID: admission.AgentID}, done: make(chan struct{})} + if _, ok := s.connectionForOrganization("org-a", "agent-1"); ok { + t.Fatal("prepared runner was dispatchable") + } + if !s.HasActionRunnerSession(admission) { + t.Fatal("exact prepared transport was not available to activation") + } + if !s.PromoteActionRunnerSession(admission) { + t.Fatal("exact prepared transport was not promoted") + } + if _, ok := s.connectionForOrganization("org-a", "agent-1"); !ok { + t.Fatal("promoted runner did not become dispatchable") + } +} diff --git a/internal/api/action_runner_credentials.go b/internal/api/action_runner_credentials.go index daf495e23..25d0bd764 100644 --- a/internal/api/action_runner_credentials.go +++ b/internal/api/action_runner_credentials.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strings" + "time" "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding" @@ -24,13 +25,15 @@ type actionRunnerCredentialRequest struct { } type actionRunnerCredentialResponse struct { - Token string `json:"token"` - TokenID string `json:"tokenId"` - OrganizationID string `json:"organizationId"` - AgentID string `json:"agentId"` - Hostname string `json:"hostname"` - RuntimeRole string `json:"runtimeRole"` - ActionCapability string `json:"actionCapability"` + Token string `json:"token"` + TokenID string `json:"tokenId"` + OrganizationID string `json:"organizationId"` + AgentID string `json:"agentId"` + Hostname string `json:"hostname"` + RuntimeRole string `json:"runtimeRole"` + ActionCapability string `json:"actionCapability"` + ActivationPending bool `json:"activationPending"` + ActivationDeadline *time.Time `json:"activationDeadline,omitempty"` } type actionRunnerCredentialSelfRevokeRequest struct { @@ -38,17 +41,20 @@ type actionRunnerCredentialSelfRevokeRequest struct { Hostname string `json:"hostname"` } -func actionRunnerCredentialRoute(cfg *config.Config, issue, selfRevoke http.HandlerFunc) http.HandlerFunc { +func actionRunnerCredentialRoute(cfg *config.Config, issue, activate, selfRevoke http.HandlerFunc) http.HandlerFunc { issue = RequireAdmin(cfg, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, issue))) + activate = RequireAuth(cfg, RequireScope(config.ScopeAgentExec, activate)) selfRevoke = RequireAuth(cfg, RequireScope(config.ScopeAgentExec, selfRevoke)) return func(w http.ResponseWriter, req *http.Request) { switch req.Method { case http.MethodPost: issue(w, req) + case http.MethodPatch: + activate(w, req) case http.MethodDelete: selfRevoke(w, req) default: - w.Header().Set("Allow", http.MethodPost+", "+http.MethodDelete) + w.Header().Set("Allow", http.MethodPost+", "+http.MethodPatch+", "+http.MethodDelete) http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) } } @@ -112,16 +118,90 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) _ = json.NewEncoder(w).Encode(actionRunnerCredentialResponse{ - Token: issued.Token, - TokenID: record.ID, - OrganizationID: record.OrgID, - AgentID: record.Metadata["bound_agent_id"], - Hostname: record.Metadata["bound_hostname"], - RuntimeRole: record.Metadata[agenttokens.RuntimeRoleMetadataKey], - ActionCapability: record.Metadata[agenttokens.ActionCapabilityMetadataKey], + Token: issued.Token, + TokenID: record.ID, + OrganizationID: record.OrgID, + AgentID: record.Metadata["bound_agent_id"], + Hostname: record.Metadata["bound_hostname"], + RuntimeRole: record.Metadata[agenttokens.RuntimeRoleMetadataKey], + ActionCapability: record.Metadata[agenttokens.ActionCapabilityMetadataKey], + ActivationPending: strings.TrimSpace(record.Metadata[agenttokens.ActionRunnerActivationPendingMetadataKey]) == "true", + ActivationDeadline: record.ExpiresAt, }) } +// handleActivateActionRunnerCredential commits a prepared rotation only after +// the exact replacement runner has registered and durably written its local +// activation proof. The runner calls this endpoint itself; no plaintext token +// or caller-selected predecessor identity crosses the boundary. +func (r *Router) handleActivateActionRunnerCredential(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPatch { + w.Header().Set("Allow", http.MethodPatch) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if r == nil || r.config == nil || r.agentExecServer == nil { + http.Error(w, "Action runner activation service unavailable", http.StatusServiceUnavailable) + return + } + caller := getAPITokenRecordFromRequest(req) + if caller == nil { + http.Error(w, "Action runner bearer credential required", http.StatusForbidden) + return + } + decoder := json.NewDecoder(io.LimitReader(req.Body, maxActionRunnerCredentialRequestBytes+1)) + decoder.DisallowUnknownFields() + var payload actionRunnerCredentialSelfRevokeRequest + if err := decoder.Decode(&payload); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + organizationID := strings.TrimSpace(GetOrgID(req.Context())) + if len(caller.GetBoundOrgs()) != 1 || strings.TrimSpace(caller.GetBoundOrgs()[0]) != organizationID || + !agentbinding.EvaluateActionRunner(caller, payload.AgentID, payload.Hostname).Admit { + http.Error(w, "Action runner credential binding mismatch", http.StatusForbidden) + return + } + pending := strings.TrimSpace(caller.Metadata[agenttokens.ActionRunnerActivationPendingMetadataKey]) == "true" + admission := agentexec.AgentAdmission{ + OrganizationID: organizationID, + TokenID: strings.TrimSpace(caller.ID), + AgentID: strings.TrimSpace(caller.Metadata["bound_agent_id"]), + Hostname: strings.TrimSpace(caller.Metadata["bound_hostname"]), + RuntimeRole: agentexec.RuntimeRoleActionRunner, + ActionCapability: agentexec.ActionCapabilityTypedV1, + ActivationPending: pending, + } + if !r.agentExecServer.HasActionRunnerSession(admission) { + http.Error(w, "Exact action runner session is not registered", http.StatusConflict) + return + } + _, revoked, changed, err := agenttokens.ActivateActionRunnerAndPersist( + r.config, r.persistence, caller.ID, payload.AgentID, payload.Hostname, + ) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, agenttokens.ErrRecord) { + status = http.StatusForbidden + } + http.Error(w, "Failed to activate action runner credential", status) + return + } + if changed { + for _, previous := range revoked { + r.invalidateActionRunnerRecord(previous) + } + admission.ActivationPending = false + r.agentExecServer.PromoteActionRunnerSession(admission) + LogAuditEventForTenant(organizationID, "action_runner_credential_activated", auth.GetUser(req.Context()), GetClientIP(req), req.URL.Path, true, "Activated host-bound typed action runner credential") + } + w.WriteHeader(http.StatusNoContent) +} + // handleSelfRevokeActionRunnerCredential lets the separately credentialed // runner revoke only its own exact tenant/host binding. It cannot select a // token ID or another host, and browser/session authentication is rejected. diff --git a/internal/api/action_runner_credentials_test.go b/internal/api/action_runner_credentials_test.go index 672302dcd..5dcaf641b 100644 --- a/internal/api/action_runner_credentials_test.go +++ b/internal/api/action_runner_credentials_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "os" @@ -15,7 +16,9 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens" "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" "github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt" + internalauth "github.com/rcourtman/pulse-go-rewrite/pkg/auth" ) type actionRunnerFailingPersistenceFS struct{} @@ -70,6 +73,24 @@ func issueActionRunnerCredentialForTest(t *testing.T, router *Router, hostID, ho return response } +func commitActionRunnerCredentialForTest(t *testing.T, router *Router, credential actionRunnerCredentialResponse) { + t.Helper() + if _, _, _, err := agenttokens.ActivateActionRunnerAndPersist(router.config, router.persistence, credential.TokenID, credential.AgentID, credential.Hostname); err != nil { + t.Fatalf("activate action runner credential: %v", err) + } +} + +func requestActionRunnerActivationForTest(t *testing.T, router *Router, credential actionRunnerCredentialResponse) *httptest.ResponseRecorder { + t.Helper() + body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: credential.AgentID, Hostname: credential.Hostname}) + req := httptest.NewRequest(http.MethodPatch, "/api/agents/action-runner/credential", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+credential.Token) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, credential.OrganizationID)) + rec := httptest.NewRecorder() + actionRunnerCredentialRoute(router.config, router.handleIssueActionRunnerCredential, router.handleActivateActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)(rec, req) + return rec +} + func connectActionRunnerCredentialForTest(t *testing.T, server *agentexec.Server, credential actionRunnerCredentialResponse) (*websocket.Conn, *httptest.Server) { t.Helper() ts := httptest.NewServer(http.HandlerFunc(server.HandleWebSocket)) @@ -152,7 +173,7 @@ func TestIssueActionRunnerCredentialRejectsUnknownOrMismatchedHost(t *testing.T) } } -func TestIssueActionRunnerCredentialRouteRotatesExistingHostBinding(t *testing.T) { +func TestIssueActionRunnerCredentialRoutePreparesExistingHostRotation(t *testing.T) { router, cfg, hostID := newActionRunnerCredentialTestRouter(t) issue := func() actionRunnerCredentialResponse { t.Helper() @@ -169,19 +190,24 @@ func TestIssueActionRunnerCredentialRouteRotatesExistingHostBinding(t *testing.T return response } first := issue() + commitActionRunnerCredentialForTest(t, router, first) second := issue() if first.TokenID == second.TokenID || first.Token == second.Token { t.Fatalf("re-enrollment did not rotate credential: first=%#v second=%#v", first, second) } - if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != second.TokenID { - t.Fatalf("route accumulated host-bound credentials: %#v", cfg.APITokens) + if !second.ActivationPending || second.ActivationDeadline == nil { + t.Fatalf("prepared response = %#v", second) + } + if len(cfg.APITokens) != 2 { + t.Fatalf("route did not retain active predecessor during prepare: %#v", cfg.APITokens) } } -func TestIssueActionRunnerCredentialRotationClosesReplacedLiveSessionAfterPersistence(t *testing.T) { - router, _, hostID := newActionRunnerCredentialTestRouter(t) +func TestIssueActionRunnerCredentialRotationCommitsOnlyAfterReplacementHealthHandshake(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + commitActionRunnerCredentialForTest(t, router, first) conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first) defer conn.Close() defer ts.Close() @@ -193,8 +219,29 @@ func TestIssueActionRunnerCredentialRotationClosesReplacedLiveSessionAfterPersis if second.TokenID == first.TokenID { t.Fatal("credential did not rotate") } + if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("prepare step revoked the prior runner before replacement activation") + } + if len(cfg.APITokens) != 2 { + t.Fatalf("prepared rotation inventory = %#v", cfg.APITokens) + } + pendingConn, pendingServer := connectActionRunnerCredentialForTest(t, router.agentExecServer, second) + defer pendingConn.Close() + defer pendingServer.Close() if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { - t.Fatal("replaced action-runner session remained connected") + t.Fatal("pending replacement became dispatchable before activation") + } + if rec := requestActionRunnerActivationForTest(t, router, second); rec.Code != http.StatusNoContent { + t.Fatalf("activation status = %d, body=%s", rec.Code, rec.Body.String()) + } + if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("activated replacement did not become dispatchable") + } + if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != second.TokenID || cfg.APITokens[0].ExpiresAt != nil { + t.Fatalf("committed rotation inventory = %#v", cfg.APITokens) + } + if _, ok := router.admitAgentExecToken(first.Token, hostID, first.Hostname); ok { + t.Fatal("committed rotation left prior secret valid") } } @@ -202,6 +249,7 @@ func TestIssueActionRunnerCredentialPersistenceFailureKeepsPriorLiveSession(t *t router, cfg, hostID := newActionRunnerCredentialTestRouter(t) router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + commitActionRunnerCredentialForTest(t, router, first) conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first) defer conn.Close() defer ts.Close() @@ -222,15 +270,59 @@ func TestIssueActionRunnerCredentialPersistenceFailureKeepsPriorLiveSession(t *t } } +func TestActivateActionRunnerCredentialPersistenceFailureKeepsBothCredentialsAndPendingSession(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) + router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) + first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + commitActionRunnerCredentialForTest(t, router, first) + second := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + pendingConn, pendingServer := connectActionRunnerCredentialForTest(t, router.agentExecServer, second) + defer pendingConn.Close() + defer pendingServer.Close() + router.persistence.SetFileSystem(actionRunnerFailingPersistenceFS{}) + + rec := requestActionRunnerActivationForTest(t, router, second) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("activation status = %d, body=%s", rec.Code, rec.Body.String()) + } + if len(cfg.APITokens) != 2 { + t.Fatalf("failed activation inventory = %#v", cfg.APITokens) + } + if _, ok := router.admitAgentExecToken(first.Token, hostID, first.Hostname); !ok { + t.Fatal("failed activation revoked the prior credential") + } + secondAdmission, ok := router.admitAgentExecToken(second.Token, hostID, second.Hostname) + if !ok || !secondAdmission.ActivationPending { + t.Fatalf("failed activation did not restore pending replacement: %#v, %v", secondAdmission, ok) + } + if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("failed activation made the pending runner dispatchable") + } +} + +func TestActivateActionRunnerCredentialRequiresExactRegisteredSession(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) + router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) + issued := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + rec := requestActionRunnerActivationForTest(t, router, issued) + if rec.Code != http.StatusConflict { + t.Fatalf("activation status = %d, body=%s", rec.Code, rec.Body.String()) + } + if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != issued.TokenID || cfg.APITokens[0].ExpiresAt == nil || cfg.APITokens[0].Metadata[agenttokens.ActionRunnerActivationPendingMetadataKey] != "true" { + t.Fatalf("unregistered activation changed inventory = %#v", cfg.APITokens) + } +} + func TestSelfRevokeActionRunnerCredentialRequiresExactBearerBindingAndClosesSession(t *testing.T) { router, cfg, hostID := newActionRunnerCredentialTestRouter(t) router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) issued := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + commitActionRunnerCredentialForTest(t, router, issued) conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, issued) defer conn.Close() defer ts.Close() - handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) + handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleActivateActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) request := func(organizationID, agentID, hostname, token string) *httptest.ResponseRecorder { t.Helper() body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: agentID, Hostname: hostname}) @@ -265,7 +357,7 @@ func TestActionRunnerCredentialRouteAuthSupportsAdminSessionAndScopedToken(t *te for _, mode := range []string{"api-token", "admin-session"} { t.Run(mode, func(t *testing.T) { router, cfg, hostID := newActionRunnerCredentialTestRouter(t) - handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) + handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleActivateActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "host-1.local")) switch mode { case "api-token": @@ -317,3 +409,156 @@ func TestActionRunnerCredentialCannotUseCollectorReportOrConfigScopes(t *testing } } } + +func TestActionRunnerRotationProductionRouterTLSPersistenceRestart(t *testing.T) { + dataPath := t.TempDir() + hashedPassword, err := internalauth.HashPassword("production-router-test-password") + if err != nil { + t.Fatal(err) + } + newRuntime := func(cfg *config.Config) (*Router, *monitoring.Monitor, *httptest.Server) { + t.Helper() + monitor, err := monitoring.New(cfg) + if err != nil { + t.Fatal(err) + } + router := NewRouter(cfg, monitor, nil, nil, func() error { return nil }, "test") + return router, monitor, httptest.NewTLSServer(router.Handler()) + } + shutdown := func(router *Router, monitor *monitoring.Monitor, server *httptest.Server) { + server.Close() + if router.agentExecServer != nil { + router.agentExecServer.Shutdown() + } + router.shutdownBackgroundWorkers() + router.ShutdownResourceStores() + router.ShutdownRBAC() + monitor.StopDiscoveryService() + monitor.Stop() + } + dial := func(server *httptest.Server, credential actionRunnerCredentialResponse) (*websocket.Conn, agentexec.RegisteredPayload) { + t.Helper() + transport, ok := server.Client().Transport.(*http.Transport) + if !ok || transport.TLSClientConfig == nil { + t.Fatal("TLS test server client has no TLS transport") + } + dialer := websocket.Dialer{TLSClientConfig: transport.TLSClientConfig.Clone()} + conn, _, err := dialer.Dial(wsURLForHTTP(server.URL)+"/api/agent/ws", wsHeadersForHTTP(t, server.URL)) + if err != nil { + t.Fatal(err) + } + message, err := agentexec.NewMessage(agentexec.MsgTypeAgentRegister, "", agentexec.AgentRegisterPayload{ + AgentID: credential.AgentID, Hostname: credential.Hostname, Token: credential.Token, + RuntimeRole: credential.RuntimeRole, ActionCapability: credential.ActionCapability, + OperationReceiptVersion: operationreceipt.ProtocolVersion, + }) + if err != nil { + conn.Close() + t.Fatal(err) + } + if err := conn.WriteJSON(message); err != nil { + conn.Close() + t.Fatal(err) + } + return conn, readRegisteredPayload(t, conn) + } + issue := func(server *httptest.Server, hostID string) actionRunnerCredentialResponse { + t.Helper() + body, _ := json.Marshal(actionRunnerCredentialRequest{AgentID: hostID, Hostname: "host-1.local"}) + req, _ := http.NewRequest(http.MethodPost, server.URL+"/api/agents/action-runner/credential", bytes.NewReader(body)) + req.SetBasicAuth("admin", "production-router-test-password") + req.Header.Set("Content-Type", "application/json") + response, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusCreated { + payload, _ := io.ReadAll(response.Body) + t.Fatalf("issue status = %d, body=%s", response.StatusCode, payload) + } + var credential actionRunnerCredentialResponse + if err := json.NewDecoder(response.Body).Decode(&credential); err != nil { + t.Fatal(err) + } + return credential + } + activate := func(server *httptest.Server, credential actionRunnerCredentialResponse) { + t.Helper() + body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: credential.AgentID, Hostname: credential.Hostname}) + req, _ := http.NewRequest(http.MethodPatch, server.URL+"/api/agents/action-runner/credential", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+credential.Token) + req.Header.Set("Content-Type", "application/json") + response, err := server.Client().Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusNoContent { + payload, _ := io.ReadAll(response.Body) + t.Fatalf("activation status = %d, body=%s", response.StatusCode, payload) + } + } + + cfg := &config.Config{DataPath: dataPath, ConfigPath: dataPath, AuthUser: "admin", AuthPass: hashedPassword, AllowedOrigins: "*", EnvOverrides: map[string]bool{}} + router, monitor, server := newRuntime(cfg) + hostID := seedUnifiedAgentHost(t, monitor) + first := issue(server, hostID) + firstConn, registered := dial(server, first) + if !registered.Success || router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + firstConn.Close() + shutdown(router, monitor, server) + t.Fatalf("prepared first registration = %#v", registered) + } + activate(server, first) + if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + firstConn.Close() + shutdown(router, monitor, server) + t.Fatal("first credential did not become active") + } + + second := issue(server, hostID) + if _, ok := cfg.ValidateAPIToken(first.Token); !ok { + firstConn.Close() + shutdown(router, monitor, server) + t.Fatal("rotation prepare rejected the first secret") + } + secondConn, registered := dial(server, second) + if !registered.Success || router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + secondConn.Close() + firstConn.Close() + shutdown(router, monitor, server) + t.Fatalf("prepared second registration = %#v", registered) + } + activate(server, second) + if _, ok := cfg.ValidateAPIToken(first.Token); ok { + secondConn.Close() + firstConn.Close() + shutdown(router, monitor, server) + t.Fatal("activation retained first secret") + } + secondConn.Close() + firstConn.Close() + shutdown(router, monitor, server) + + persisted, err := config.NewConfigPersistence(dataPath).LoadAPITokens() + if err != nil { + t.Fatal(err) + } + if len(persisted) != 1 || persisted[0].ID != second.TokenID || persisted[0].ExpiresAt != nil { + t.Fatalf("persisted activated inventory = %#v", persisted) + } + restartedConfig := &config.Config{DataPath: dataPath, ConfigPath: dataPath, AuthUser: "admin", AuthPass: hashedPassword, AllowedOrigins: "*", EnvOverrides: map[string]bool{}, APITokens: persisted} + restartedRouter, restartedMonitor, restartedServer := newRuntime(restartedConfig) + defer shutdown(restartedRouter, restartedMonitor, restartedServer) + oldConn, oldRegistration := dial(restartedServer, first) + oldConn.Close() + if oldRegistration.Success { + t.Fatal("durably revoked predecessor registered after server restart") + } + newConn, newRegistration := dial(restartedServer, second) + defer newConn.Close() + if !newRegistration.Success || !restartedRouter.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatalf("durably activated replacement after restart = %#v", newRegistration) + } +} diff --git a/internal/api/agent_exec_token_binding.go b/internal/api/agent_exec_token_binding.go index 765b67746..8e2a2605d 100644 --- a/internal/api/agent_exec_token_binding.go +++ b/internal/api/agent_exec_token_binding.go @@ -152,12 +152,13 @@ func (r *Router) admitAgentExecToken(token string, agentID string, hostname stri capability := strings.TrimSpace(record.Metadata[agenttokens.ActionCapabilityMetadataKey]) config.Mu.Unlock() return agentexec.AgentAdmission{ - OrganizationID: organizationID, - TokenID: tokenID, - AgentID: requestedID, - Hostname: requestedHost, - RuntimeRole: agentexec.RuntimeRoleActionRunner, - ActionCapability: capability, + OrganizationID: organizationID, + TokenID: tokenID, + AgentID: requestedID, + Hostname: requestedHost, + RuntimeRole: agentexec.RuntimeRoleActionRunner, + ActionCapability: capability, + ActivationPending: strings.TrimSpace(record.Metadata[agenttokens.ActionRunnerActivationPendingMetadataKey]) == "true", }, true } if runtimeRole != "" && runtimeRole != agenttokens.CredentialKindLegacyFullTrust { diff --git a/internal/api/agent_fleet_doctor.go b/internal/api/agent_fleet_doctor.go index 984b70660..6ae4f9377 100644 --- a/internal/api/agent_fleet_doctor.go +++ b/internal/api/agent_fleet_doctor.go @@ -137,6 +137,7 @@ func applyAgentFleetActionRunnerState( func actionRunnerCredentialActive(record config.APITokenRecord, now time.Time) bool { return record.HasScope(config.ScopeAgentExec) && + strings.TrimSpace(record.Metadata[agenttokens.ActionRunnerActivationPendingMetadataKey]) != "true" && strings.TrimSpace(record.Metadata[agenttokens.ActionCapabilityMetadataKey]) == agenttokens.ActionCapabilityTypedV1 && strings.TrimSpace(record.Metadata[agenttokens.ActionBindingVersionMetadataKey]) == agenttokens.ActionBindingVersion && (record.ExpiresAt == nil || record.ExpiresAt.After(now)) diff --git a/internal/api/agent_fleet_doctor_test.go b/internal/api/agent_fleet_doctor_test.go index 25c9f00a6..ff3b45a5c 100644 --- a/internal/api/agent_fleet_doctor_test.go +++ b/internal/api/agent_fleet_doctor_test.go @@ -75,6 +75,7 @@ func TestApplyAgentFleetActionRunnerStateMatchesOnlyBoundTypedRunner(t *testing. }, }, {AgentID: "host-b", Hostname: "host-b.example"}, + {AgentID: "host-c", Hostname: "host-c.example"}, }} applyAgentFleetActionRunnerState(&diagnostics, []config.APITokenRecord{ { @@ -95,11 +96,21 @@ func TestApplyAgentFleetActionRunnerStateMatchesOnlyBoundTypedRunner(t *testing. "bound_agent_id": "host-b", }, }, + { + OrgID: "default", Scopes: []string{config.ScopeAgentExec}, + Metadata: map[string]string{ + agenttokens.CredentialKindMetadataKey: agenttokens.CredentialKindActionRunner, + agenttokens.ActionCapabilityMetadataKey: agenttokens.ActionCapabilityTypedV1, + agenttokens.ActionBindingVersionMetadataKey: agenttokens.ActionBindingVersion, + agenttokens.ActionRunnerActivationPendingMetadataKey: "true", + "bound_agent_id": "host-c", + }, + }, {OrgID: "other-org", Metadata: map[string]string{agenttokens.CredentialKindMetadataKey: agenttokens.CredentialKindActionRunner, "bound_agent_id": "host-a"}}, }, []agentexec.ConnectedAgent{ {AgentID: "host-a", RuntimeRole: agentexec.RuntimeRoleActionRunner, ActionCapability: agentexec.ActionCapabilityTypedV1, Version: "6.3.0-linux-amd64", ConnectedAt: connectedAt, OperationReceiptVersion: 1, ActionPreflightVersion: 2, DockerObservationVersion: 2}, {AgentID: "host-b", RuntimeRole: agentexec.RuntimeRoleLegacyFullTrust, ActionCapability: agentexec.ActionCapabilityTypedV1, Version: "legacy"}, - {AgentID: "host-c", RuntimeRole: agentexec.RuntimeRoleActionRunner, ActionCapability: agentexec.ActionCapabilityTypedV1, Version: "other-host"}, + {AgentID: "host-d", RuntimeRole: agentexec.RuntimeRoleActionRunner, ActionCapability: agentexec.ActionCapabilityTypedV1, Version: "other-host"}, }, "default", connectedAt) hostA := diagnostics.Agents[0].Privilege @@ -115,4 +126,8 @@ func TestApplyAgentFleetActionRunnerStateMatchesOnlyBoundTypedRunner(t *testing. if hostB == nil || !hostB.ActionRunnerCredentialIssued || hostB.ActionRunnerCredentialActive || hostB.ActionRunnerConnected { t.Fatalf("host-b credential/session posture = %+v", hostB) } + hostC := diagnostics.Agents[2].Privilege + if hostC == nil || !hostC.ActionRunnerCredentialIssued || hostC.ActionRunnerCredentialActive || hostC.ActionRunnerConnected { + t.Fatalf("host-c pending credential/session posture = %+v", hostC) + } } diff --git a/internal/api/agenttokens/install.go b/internal/api/agenttokens/install.go index e3d27adcc..8bf78e37c 100644 --- a/internal/api/agenttokens/install.go +++ b/internal/api/agenttokens/install.go @@ -13,21 +13,24 @@ import ( ) const ( - IssuedAtMetadataKey = "install_issued_at" - OwnerUserIDMetadataKey = "owner_user_id" - CommandPolicyIntentMetadataKey = "command_policy_intent" - CommandPolicyAppliedAgentIDMetadataKey = "command_policy_applied_agent_id" - CommandPolicyIntentEnabled = "enabled" - CommandPolicyIntentDisabled = "disabled" - RuntimeRoleMetadataKey = internalauth.RuntimeRoleMetadataKey - CredentialKindMetadataKey = RuntimeRoleMetadataKey - CredentialKindMonitoringCollector = internalauth.RuntimeRoleMonitoringCollector - CredentialKindActionRunner = internalauth.RuntimeRoleActionRunner - CredentialKindLegacyFullTrust = internalauth.RuntimeRoleLegacyFullTrust - ActionCapabilityMetadataKey = "agent_action_capability" - ActionCapabilityTypedV1 = "typed_actions.v1" - ActionBindingVersionMetadataKey = "action_runner_binding_version" - ActionBindingVersion = "1" + IssuedAtMetadataKey = "install_issued_at" + OwnerUserIDMetadataKey = "owner_user_id" + CommandPolicyIntentMetadataKey = "command_policy_intent" + CommandPolicyAppliedAgentIDMetadataKey = "command_policy_applied_agent_id" + CommandPolicyIntentEnabled = "enabled" + CommandPolicyIntentDisabled = "disabled" + RuntimeRoleMetadataKey = internalauth.RuntimeRoleMetadataKey + CredentialKindMetadataKey = RuntimeRoleMetadataKey + CredentialKindMonitoringCollector = internalauth.RuntimeRoleMonitoringCollector + CredentialKindActionRunner = internalauth.RuntimeRoleActionRunner + CredentialKindLegacyFullTrust = internalauth.RuntimeRoleLegacyFullTrust + ActionCapabilityMetadataKey = "agent_action_capability" + ActionCapabilityTypedV1 = "typed_actions.v1" + ActionBindingVersionMetadataKey = "action_runner_binding_version" + ActionBindingVersion = "1" + ActionRunnerActivationPendingMetadataKey = "action_runner_activation_pending" + ActionRunnerReplacesTokenIDsMetadataKey = "action_runner_replaces_token_ids" + ActionRunnerActivationWindow = 10 * time.Minute ) var ( @@ -65,6 +68,14 @@ type ActionRunnerIssueResult struct { Replaced []config.APITokenRecord } +func cloneAPITokenRecords(records []config.APITokenRecord) []config.APITokenRecord { + cloned := make([]config.APITokenRecord, len(records)) + for index := range records { + cloned[index] = records[index].Clone() + } + return cloned +} + func ProxmoxScopes(enableCommands bool) []string { scopes := []string{ config.ScopeAgentReport, @@ -164,7 +175,7 @@ func issueAndPersistReplacing( config.Mu.Lock() defer config.Mu.Unlock() - previousTokens := append([]config.APITokenRecord(nil), cfg.APITokens...) + previousTokens := cloneAPITokenRecords(cfg.APITokens) replaced := make([]config.APITokenRecord, 0, 1) if replace == nil { cfg.APITokens = append(cfg.APITokens, *record) @@ -202,9 +213,11 @@ func IssueActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigP return result.Token, result.Record, err } -// IssueActionRunnerAndPersistDetailed is the rotation-aware form used by the -// API boundary. Replaced contains records only when the new credential was -// durably committed; persistence failure returns an empty result. +// IssueActionRunnerAndPersistDetailed prepares a bounded action-runner +// activation. The prior active credential remains valid until the replacement +// runner explicitly commits activation after writing its durable health proof. +// Replaced contains only older unactivated credentials removed by this prepare +// step; persistence failure returns an empty result. func IssueActionRunnerAndPersistDetailed(cfg *config.Config, persistence *config.ConfigPersistence, opts ActionRunnerIssueOptions) (ActionRunnerIssueResult, error) { agentID := strings.TrimSpace(opts.AgentID) hostname := unifiedresources.NormalizeFullHostname(opts.Hostname) @@ -226,30 +239,138 @@ func IssueActionRunnerAndPersistDetailed(cfg *config.Config, persistence *config if tokenName == "" { tokenName = "action-runner:" + hostname } - rawToken, record, replaced, err := issueAndPersistReplacing(cfg, persistence, IssueOptions{ - TokenName: tokenName, - OrgID: organizationID, - OwnerUserID: opts.OwnerUserID, - Scopes: ActionRunnerScopes(), - Metadata: map[string]string{ - CredentialKindMetadataKey: CredentialKindActionRunner, - ActionCapabilityMetadataKey: ActionCapabilityTypedV1, - ActionBindingVersionMetadataKey: ActionBindingVersion, - "bound_agent_id": agentID, - "bound_hostname": hostname, - "bound_at": time.Now().UTC().Format(time.RFC3339), - }, - }, func(record config.APITokenRecord) bool { - return strings.TrimSpace(record.OrgID) == organizationID && - strings.TrimSpace(record.Metadata[CredentialKindMetadataKey]) == CredentialKindActionRunner && - strings.TrimSpace(record.Metadata["bound_agent_id"]) == agentID - }) + rawToken, err := internalauth.GenerateAPIToken() if err != nil { - return ActionRunnerIssueResult{}, err + return ActionRunnerIssueResult{}, fmt.Errorf("%w: %w", ErrGeneration, err) } + record, err := config.NewAPITokenRecord(rawToken, tokenName, ActionRunnerScopes()) + if err != nil { + return ActionRunnerIssueResult{}, fmt.Errorf("%w: %w", ErrRecord, err) + } + record.OrgID = organizationID + setOwnerUserID(record, opts.OwnerUserID) + record.Metadata = map[string]string{ + CredentialKindMetadataKey: CredentialKindActionRunner, + ActionCapabilityMetadataKey: ActionCapabilityTypedV1, + ActionBindingVersionMetadataKey: ActionBindingVersion, + ActionRunnerActivationPendingMetadataKey: "true", + "bound_agent_id": agentID, + "bound_hostname": hostname, + "bound_at": time.Now().UTC().Format(time.RFC3339), + IssuedAtMetadataKey: record.CreatedAt.UTC().Format(time.RFC3339), + } + if err := normalizeCredentialKind(record); err != nil { + return ActionRunnerIssueResult{}, fmt.Errorf("%w: %w", ErrRecord, err) + } + deadline := time.Now().UTC().Add(ActionRunnerActivationWindow) + record.ExpiresAt = &deadline + + config.Mu.Lock() + defer config.Mu.Unlock() + previousTokens := cloneAPITokenRecords(cfg.APITokens) + nextTokens := make([]config.APITokenRecord, 0, len(cfg.APITokens)+1) + replaced := make([]config.APITokenRecord, 0, 1) + activeIDs := make([]string, 0, 1) + for _, existing := range cfg.APITokens { + matchingBinding := strings.TrimSpace(existing.OrgID) == organizationID && + strings.TrimSpace(existing.Metadata[CredentialKindMetadataKey]) == CredentialKindActionRunner && + strings.TrimSpace(existing.Metadata["bound_agent_id"]) == agentID + if !matchingBinding { + nextTokens = append(nextTokens, existing) + continue + } + if strings.TrimSpace(existing.Metadata[ActionRunnerActivationPendingMetadataKey]) == "true" { + replaced = append(replaced, existing.Clone()) + continue + } + activeIDs = append(activeIDs, strings.TrimSpace(existing.ID)) + nextTokens = append(nextTokens, existing) + } + if len(activeIDs) > 0 { + record.Metadata[ActionRunnerReplacesTokenIDsMetadataKey] = strings.Join(activeIDs, ",") + } + cfg.APITokens = append(nextTokens, *record) + cfg.SortAPITokens() + if persistence != nil { + if err := persistence.SaveAPITokens(cfg.APITokens); err != nil { + cfg.APITokens = previousTokens + cfg.SortAPITokens() + return ActionRunnerIssueResult{}, fmt.Errorf("%w: %w", ErrPersist, err) + } + } + return ActionRunnerIssueResult{Token: rawToken, Record: record, Replaced: replaced}, nil } +// ActivateActionRunnerAndPersist commits a prepared credential and revokes the +// exact predecessor set in the same durable token-inventory transaction. +func ActivateActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigPersistence, tokenID, agentID, hostname string) (*config.APITokenRecord, []config.APITokenRecord, bool, error) { + if cfg == nil { + return nil, nil, false, fmt.Errorf("%w: config is required", ErrRecord) + } + tokenID = strings.TrimSpace(tokenID) + agentID = strings.TrimSpace(agentID) + hostname = unifiedresources.NormalizeFullHostname(hostname) + if tokenID == "" || agentID == "" || hostname == "" { + return nil, nil, false, fmt.Errorf("%w: complete activation identity is required", ErrRecord) + } + + config.Mu.Lock() + defer config.Mu.Unlock() + index := -1 + for candidateIndex := range cfg.APITokens { + candidate := &cfg.APITokens[candidateIndex] + if candidate.ID == tokenID { + index = candidateIndex + break + } + } + if index < 0 { + return nil, nil, false, fmt.Errorf("%w: action runner credential not found", ErrRecord) + } + record := &cfg.APITokens[index] + if record.IsExpired() || strings.TrimSpace(record.Metadata[CredentialKindMetadataKey]) != CredentialKindActionRunner || + strings.TrimSpace(record.Metadata["bound_agent_id"]) != agentID || + !unifiedresources.HostnamesEquivalent(record.Metadata["bound_hostname"], hostname) { + return nil, nil, false, fmt.Errorf("%w: action runner activation binding mismatch", ErrRecord) + } + if strings.TrimSpace(record.Metadata[ActionRunnerActivationPendingMetadataKey]) != "true" { + clone := record.Clone() + return &clone, nil, false, nil + } + + previousTokens := cloneAPITokenRecords(cfg.APITokens) + replaceIDs := make(map[string]struct{}) + for _, replacedID := range strings.Split(record.Metadata[ActionRunnerReplacesTokenIDsMetadataKey], ",") { + if replacedID = strings.TrimSpace(replacedID); replacedID != "" && replacedID != tokenID { + replaceIDs[replacedID] = struct{}{} + } + } + delete(record.Metadata, ActionRunnerActivationPendingMetadataKey) + delete(record.Metadata, ActionRunnerReplacesTokenIDsMetadataKey) + record.ExpiresAt = nil + activated := record.Clone() + revoked := make([]config.APITokenRecord, 0, len(replaceIDs)) + nextTokens := make([]config.APITokenRecord, 0, len(cfg.APITokens)) + for _, existing := range cfg.APITokens { + if _, remove := replaceIDs[existing.ID]; remove { + revoked = append(revoked, existing.Clone()) + continue + } + nextTokens = append(nextTokens, existing) + } + cfg.APITokens = nextTokens + cfg.SortAPITokens() + if persistence != nil { + if err := persistence.SaveAPITokens(cfg.APITokens); err != nil { + cfg.APITokens = previousTokens + cfg.SortAPITokens() + return nil, nil, false, fmt.Errorf("%w: %w", ErrPersist, err) + } + } + return &activated, revoked, true, nil +} + func normalizeCredentialKind(record *config.APITokenRecord) error { if record == nil { return nil diff --git a/internal/api/agenttokens/install_test.go b/internal/api/agenttokens/install_test.go index 028ddbc17..9c2279f7b 100644 --- a/internal/api/agenttokens/install_test.go +++ b/internal/api/agenttokens/install_test.go @@ -74,14 +74,17 @@ func TestIssueActionRunnerAndPersistRejectsIncompleteBinding(t *testing.T) { } } -func TestIssueActionRunnerAndPersistReplacesMatchingBoundCredential(t *testing.T) { +func TestIssueActionRunnerAndPersistPreparesRotationWithoutRevokingActiveCredential(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} - _, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ + firstToken, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ OrgID: "org-a", AgentID: "machine-123", Hostname: "Node.EXAMPLE", }) if err != nil { t.Fatalf("first IssueActionRunnerAndPersist: %v", err) } + if _, _, changed, err := ActivateActionRunnerAndPersist(cfg, nil, first.ID, "machine-123", "node.example"); err != nil || !changed { + t.Fatalf("activate first credential = changed %v, error %v", changed, err) + } _, otherHost, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ OrgID: "org-a", AgentID: "machine-456", Hostname: "other.example", }) @@ -99,8 +102,8 @@ func TestIssueActionRunnerAndPersistReplacesMatchingBoundCredential(t *testing.T if replacement.ID == first.ID { t.Fatalf("replacement reused token id %q", replacement.ID) } - if len(cfg.APITokens) != 2 { - t.Fatalf("token inventory = %#v, want one replacement plus other host", cfg.APITokens) + if len(cfg.APITokens) != 3 { + t.Fatalf("token inventory = %#v, want pending replacement, active predecessor, and other host", cfg.APITokens) } foundReplacement, foundOther, foundFirst := false, false, false for _, record := range cfg.APITokens { @@ -108,9 +111,15 @@ func TestIssueActionRunnerAndPersistReplacesMatchingBoundCredential(t *testing.T foundOther = foundOther || record.ID == otherHost.ID foundFirst = foundFirst || record.ID == first.ID } - if !foundReplacement || !foundOther || foundFirst { + if !foundReplacement || !foundOther || !foundFirst { t.Fatalf("replacement inventory = %#v", cfg.APITokens) } + if _, ok := cfg.ValidateAPIToken(firstToken); !ok { + t.Fatal("prepare step revoked the active predecessor") + } + if replacement.ExpiresAt == nil || replacement.Metadata[ActionRunnerActivationPendingMetadataKey] != "true" || replacement.Metadata[ActionRunnerReplacesTokenIDsMetadataKey] != first.ID { + t.Fatalf("prepared replacement = %#v", replacement) + } } func TestIssueActionRunnerAndPersistDetailedReturnsOnlyDurablyReplacedRecords(t *testing.T) { @@ -135,6 +144,74 @@ func TestIssueActionRunnerAndPersistDetailedReturnsOnlyDurablyReplacedRecords(t } } +func TestActivateActionRunnerAndPersistAtomicallyPromotesAndRevokesPredecessor(t *testing.T) { + cfg := &config.Config{DataPath: t.TempDir()} + firstToken, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"}) + if err != nil { + t.Fatal(err) + } + if _, _, _, err := ActivateActionRunnerAndPersist(cfg, nil, first.ID, "machine-123", "node.example"); err != nil { + t.Fatal(err) + } + secondToken, second, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"}) + if err != nil { + t.Fatal(err) + } + activated, revoked, changed, err := ActivateActionRunnerAndPersist(cfg, config.NewConfigPersistence(cfg.DataPath), second.ID, "machine-123", "NODE") + if err != nil || !changed { + t.Fatalf("activation = (%#v, %#v, %v, %v)", activated, revoked, changed, err) + } + if activated.ExpiresAt != nil || activated.Metadata[ActionRunnerActivationPendingMetadataKey] != "" || len(revoked) != 1 || revoked[0].ID != first.ID { + t.Fatalf("activation result = activated %#v, revoked %#v", activated, revoked) + } + if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != second.ID { + t.Fatalf("activated inventory = %#v", cfg.APITokens) + } + if _, ok := cfg.ValidateAPIToken(firstToken); ok { + t.Fatal("activated rotation left predecessor valid") + } + if _, ok := cfg.ValidateAPIToken(secondToken); !ok { + t.Fatal("activated replacement is not valid") + } + if _, revoked, changed, err := ActivateActionRunnerAndPersist(cfg, nil, second.ID, "machine-123", "node.example"); err != nil || changed || len(revoked) != 0 { + t.Fatalf("idempotent activation = revoked %#v, changed %v, error %v", revoked, changed, err) + } +} + +func TestActivateActionRunnerAndPersistFailureRestoresPendingAndActiveInventory(t *testing.T) { + cfg := &config.Config{DataPath: t.TempDir()} + _, first, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"}) + if err != nil { + t.Fatal(err) + } + if _, _, _, err := ActivateActionRunnerAndPersist(cfg, nil, first.ID, "machine-123", "node.example"); err != nil { + t.Fatal(err) + } + _, second, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example"}) + if err != nil { + t.Fatal(err) + } + statePath := filepath.Join(t.TempDir(), "blocked-state") + persistence := config.NewConfigPersistence(statePath) + if err := os.RemoveAll(statePath); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(statePath, []byte("not a directory"), 0o600); err != nil { + t.Fatal(err) + } + if _, _, changed, err := ActivateActionRunnerAndPersist(cfg, persistence, second.ID, "machine-123", "node.example"); !errors.Is(err, ErrPersist) || changed { + t.Fatalf("activation = changed %v, error %v", changed, err) + } + if len(cfg.APITokens) != 2 { + t.Fatalf("restored inventory = %#v", cfg.APITokens) + } + for _, record := range cfg.APITokens { + if record.ID == second.ID && (record.ExpiresAt == nil || record.Metadata[ActionRunnerActivationPendingMetadataKey] != "true" || record.Metadata[ActionRunnerReplacesTokenIDsMetadataKey] != first.ID) { + t.Fatalf("pending replacement was not fully restored: %#v", record) + } + } +} + func TestIssueActionRunnerAndPersistRestoresReplacedCredentialOnPersistenceFailure(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} _, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ diff --git a/internal/api/router_routes_registration.go b/internal/api/router_routes_registration.go index e474a537a..8f10c4dec 100644 --- a/internal/api/router_routes_registration.go +++ b/internal/api/router_routes_registration.go @@ -55,7 +55,7 @@ func (r *Router) registerConfigSystemRoutes(updateHandlers *UpdateHandlers) { r.mux.HandleFunc("/api/agents/docker/report", RequireAuth(r.config, RequireScope(config.ScopeDockerReport, r.dockerAgentHandlers.HandleReport))) r.mux.HandleFunc("/api/agents/kubernetes/report", RequireAuth(r.config, RequireScope(config.ScopeKubernetesReport, r.kubernetesAgentHandlers.HandleReport))) r.mux.HandleFunc("/api/agents/agent/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport))) - r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAuth(r.config, actionRunnerCredentialRoute(r.config, r.handleIssueActionRunnerCredential, r.handleSelfRevokeActionRunnerCredential))) + r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAuth(r.config, actionRunnerCredentialRoute(r.config, r.handleIssueActionRunnerCredential, r.handleActivateActionRunnerCredential, r.handleSelfRevokeActionRunnerCredential))) r.mux.HandleFunc("/api/agents/collector/reduce-authority", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.handleReduceCollectorAuthority))) r.mux.HandleFunc("/api/agents/host/report", wrapLegacyHostAlias("/api/agents/host/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport)))) r.mux.HandleFunc("/api/agents/agent/lookup", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleLookup))) diff --git a/internal/api/security_regression_test.go b/internal/api/security_regression_test.go index 00561dead..ede62b604 100644 --- a/internal/api/security_regression_test.go +++ b/internal/api/security_regression_test.go @@ -716,7 +716,7 @@ func TestActionRunnerCredentialAdmissionRejectsWrongIdentityAndUnboundOrganizati } } -func TestActionRunnerCredentialRotationRevokesPreviousSession(t *testing.T) { +func TestActionRunnerCredentialRotationRevokesPreviousSecretOnlyAtActivation(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} firstToken, firstRecord, err := agenttokens.IssueActionRunnerAndPersist(cfg, nil, agenttokens.ActionRunnerIssueOptions{ OrgID: "org-a", AgentID: "machine-a", Hostname: "node.example", @@ -724,6 +724,9 @@ func TestActionRunnerCredentialRotationRevokesPreviousSession(t *testing.T) { if err != nil { t.Fatal(err) } + if _, _, _, err := agenttokens.ActivateActionRunnerAndPersist(cfg, nil, firstRecord.ID, "machine-a", "node.example"); err != nil { + t.Fatal(err) + } router := &Router{config: cfg} if _, ok := router.admitAgentExecToken(firstToken, "machine-a", "node.example"); !ok { t.Fatal("initial action runner credential was rejected") @@ -735,14 +738,20 @@ func TestActionRunnerCredentialRotationRevokesPreviousSession(t *testing.T) { if err != nil { t.Fatal(err) } - if firstRecord.ID == secondRecord.ID || firstToken == secondToken || len(cfg.APITokens) != 1 { - t.Fatalf("rotation did not replace the prior credential: %#v", cfg.APITokens) + if firstRecord.ID == secondRecord.ID || firstToken == secondToken || len(cfg.APITokens) != 2 { + t.Fatalf("rotation did not prepare beside the prior credential: %#v", cfg.APITokens) + } + if _, ok := router.admitAgentExecToken(firstToken, "machine-a", "node.example"); !ok { + t.Fatal("prepared rotation revoked the prior action runner credential") + } + if admission, ok := router.admitAgentExecToken(secondToken, "machine-a", "renamed.example"); !ok || !admission.ActivationPending { + t.Fatal("replacement action runner credential was rejected") + } + if _, revoked, changed, err := agenttokens.ActivateActionRunnerAndPersist(cfg, nil, secondRecord.ID, "machine-a", "renamed.example"); err != nil || !changed || len(revoked) != 1 || revoked[0].ID != firstRecord.ID { + t.Fatalf("rotation activation = revoked %#v, changed %v, error %v", revoked, changed, err) } if _, ok := router.admitAgentExecToken(firstToken, "machine-a", "node.example"); ok { - t.Fatal("replaced action runner credential remained admissible") - } - if _, ok := router.admitAgentExecToken(secondToken, "machine-a", "renamed.example"); !ok { - t.Fatal("replacement action runner credential was rejected") + t.Fatal("activated rotation left prior action runner credential valid") } } diff --git a/internal/api/security_tokens_lifecycle_test.go b/internal/api/security_tokens_lifecycle_test.go index 1c7ffeb0b..464e6df4d 100644 --- a/internal/api/security_tokens_lifecycle_test.go +++ b/internal/api/security_tokens_lifecycle_test.go @@ -35,7 +35,7 @@ func TestSelfRevokeActionRunnerCredentialRejectsNonRunnerExecBearer(t *testing.T req.Header.Set("Authorization", "Bearer "+raw) req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) rec := httptest.NewRecorder() - actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)(rec, req) + actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleActivateActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)(rec, req) if rec.Code != http.StatusForbidden || len(cfg.APITokens) != 1 { t.Fatalf("legacy self revoke = status %d tokens %#v body=%s", rec.Code, cfg.APITokens, rec.Body.String()) } diff --git a/internal/hostagent/action_runner_client.go b/internal/hostagent/action_runner_client.go index 5a0fca777..d2c8cfb66 100644 --- a/internal/hostagent/action_runner_client.go +++ b/internal/hostagent/action_runner_client.go @@ -1,8 +1,12 @@ package hostagent import ( + "bytes" + "context" "encoding/json" "fmt" + "io" + "net/http" "os" "path/filepath" "runtime" @@ -22,6 +26,7 @@ type ActionRunnerClientConfig struct { APIToken string StateDir string HealthPath string + ActivationNonce string InsecureSkipVerify bool CACertPath string ServerFingerprint string @@ -53,6 +58,7 @@ func NewActionRunnerClient(config ActionRunnerClientConfig, agentID, hostname, v client.runtimeRole = agentexec.RuntimeRoleActionRunner client.actionCapability = agentexec.ActionCapabilityTypedV1 client.healthPath = strings.TrimSpace(config.HealthPath) + client.actionActivationNonce = strings.TrimSpace(config.ActivationNonce) client.healthCapabilities = []string{ "host.storage_cleanup.v1", "host.update.v1", @@ -79,24 +85,30 @@ func allowedActionRunnerMessage(message messageType) bool { } type actionRunnerHealth struct { - Registered bool `json:"registered"` - RuntimeRole string `json:"runtime_role"` - Server string `json:"server"` - HostID string `json:"host_id"` - Hostname string `json:"hostname"` - Capabilities []string `json:"capabilities"` - RegisteredAt time.Time `json:"registered_at"` + Registered bool `json:"registered"` + Activated bool `json:"activated"` + ActivationNonce string `json:"activation_nonce"` + RuntimeRole string `json:"runtime_role"` + Server string `json:"server"` + HostID string `json:"host_id"` + Hostname string `json:"hostname"` + Capabilities []string `json:"capabilities"` + RegisteredAt time.Time `json:"registered_at"` } -func (c *CommandClient) writeActionRunnerHealth() error { +func (c *CommandClient) writeActionRunnerHealth(activated bool) error { if c == nil || !c.actionRunnerOnly || strings.TrimSpace(c.healthPath) == "" { return fmt.Errorf("action-runner health path is required") } + if nonce := strings.TrimSpace(c.actionActivationNonce); len(nonce) < 32 || len(nonce) > 128 { + return fmt.Errorf("action-runner activation nonce is invalid") + } capabilities := append([]string(nil), c.healthCapabilities...) sort.Strings(capabilities) health := actionRunnerHealth{ - Registered: true, RuntimeRole: agentexec.RuntimeRoleActionRunner, - Server: c.pulseURL, HostID: c.agentID, Hostname: c.hostname, + Registered: true, Activated: activated, ActivationNonce: c.actionActivationNonce, + RuntimeRole: agentexec.RuntimeRoleActionRunner, + Server: c.pulseURL, HostID: c.agentID, Hostname: c.hostname, Capabilities: capabilities, RegisteredAt: time.Now().UTC(), } encoded, err := json.Marshal(health) @@ -146,3 +158,33 @@ func (c *CommandClient) writeActionRunnerHealth() error { } return syncActionRunnerHealthDirectory(dir) } + +func (c *CommandClient) activateActionRunnerCredential(ctx context.Context) error { + if c == nil || !c.actionRunnerOnly { + return fmt.Errorf("action-runner activation requires the typed runner") + } + body, err := json.Marshal(map[string]string{"agentId": c.agentID, "hostname": c.hostname}) + if err != nil { + return err + } + request, err := http.NewRequestWithContext(ctx, http.MethodPatch, strings.TrimRight(c.pulseURL, "/")+"/api/agents/action-runner/credential", bytes.NewReader(body)) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+c.apiToken) + request.Header.Set("Content-Type", "application/json") + client, err := newAgentHTTPClient(c.caCertPath, c.insecureSkipVerify, c.serverFingerprint) + if err != nil { + return fmt.Errorf("build action-runner activation client: %w", err) + } + response, err := client.Do(request) + if err != nil { + return fmt.Errorf("activate action-runner credential: %w", err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusNoContent { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + return fmt.Errorf("activate action-runner credential: server returned %s", response.Status) + } + return nil +} diff --git a/internal/hostagent/action_runner_client_test.go b/internal/hostagent/action_runner_client_test.go index c329140d4..af7876b27 100644 --- a/internal/hostagent/action_runner_client_test.go +++ b/internal/hostagent/action_runner_client_test.go @@ -21,7 +21,8 @@ func TestNewActionRunnerClientIsTypedOnlyAndEmitsExplicitRole(t *testing.T) { logger := zerolog.Nop() client := NewActionRunnerClient(ActionRunnerClientConfig{ PulseURL: "https://pulse.example", APIToken: "separate-secret", - StateDir: t.TempDir(), HealthPath: filepath.Join(t.TempDir(), "health.json"), Logger: &logger, + StateDir: t.TempDir(), HealthPath: filepath.Join(t.TempDir(), "health.json"), + ActivationNonce: strings.Repeat("a", 32), Logger: &logger, }, "agent-1", "host-1", "v1") t.Cleanup(func() { _ = client.Close() }) if !client.actionRunnerOnly || client.runtimeRole != agentexec.RuntimeRoleActionRunner || client.actionCapability != agentexec.ActionCapabilityTypedV1 { @@ -43,6 +44,10 @@ func TestActionRunnerTransportRegistersRoleWritesHealthAndRejectsGenericExec(t * upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} registration := make(chan registerPayload, 1) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPatch { + w.WriteHeader(http.StatusNoContent) + return + } conn, err := upgrader.Upgrade(w, request, nil) if err != nil { return @@ -68,7 +73,7 @@ func TestActionRunnerTransportRegistersRoleWritesHealthAndRejectsGenericExec(t * logger := zerolog.Nop() client := NewActionRunnerClient(ActionRunnerClientConfig{ PulseURL: server.URL, APIToken: "runner-token", StateDir: filepath.Join(dir, "state"), - HealthPath: healthPath, InsecureSkipVerify: true, Logger: &logger, + HealthPath: healthPath, ActivationNonce: strings.Repeat("b", 32), InsecureSkipVerify: true, Logger: &logger, }, "agent-1", "host-1", "v1") defer client.Close() err := client.connectAndHandle(context.Background()) @@ -84,7 +89,7 @@ func TestActionRunnerTransportRegistersRoleWritesHealthAndRejectsGenericExec(t * t.Fatal(err) } var health actionRunnerHealth - if json.Unmarshal(data, &health) != nil || !health.Registered || health.HostID != "agent-1" { + if json.Unmarshal(data, &health) != nil || !health.Registered || !health.Activated || health.HostID != "agent-1" || health.ActivationNonce != strings.Repeat("b", 32) { t.Fatalf("health = %s", data) } } @@ -95,13 +100,14 @@ func TestActionRunnerHealthIsAtomicBoundedAndSecretFree(t *testing.T) { logger := zerolog.Nop() client := NewActionRunnerClient(ActionRunnerClientConfig{ PulseURL: "https://pulse.example", APIToken: "must-not-appear", - StateDir: filepath.Join(dir, "state"), HealthPath: healthPath, Logger: &logger, + StateDir: filepath.Join(dir, "state"), HealthPath: healthPath, + ActivationNonce: strings.Repeat("c", 32), Logger: &logger, }, "agent-1", "host-1", "v1") t.Cleanup(func() { _ = client.Close() }) if err := os.WriteFile(healthPath, []byte("stale-health-marker"), 0600); err != nil { t.Fatal(err) } - if err := client.writeActionRunnerHealth(); err != nil { + if err := client.writeActionRunnerHealth(true); err != nil { t.Fatal(err) } data, err := os.ReadFile(healthPath) @@ -118,7 +124,7 @@ func TestActionRunnerHealthIsAtomicBoundedAndSecretFree(t *testing.T) { if err := json.Unmarshal(data, &health); err != nil { t.Fatal(err) } - if !health.Registered || health.RuntimeRole != agentexec.RuntimeRoleActionRunner || health.HostID != "agent-1" || health.Server != "https://pulse.example" || health.RegisteredAt.IsZero() { + if !health.Registered || !health.Activated || health.ActivationNonce != strings.Repeat("c", 32) || health.RuntimeRole != agentexec.RuntimeRoleActionRunner || health.HostID != "agent-1" || health.Server != "https://pulse.example" || health.RegisteredAt.IsZero() { t.Fatalf("health = %+v", health) } info, err := os.Lstat(healthPath) @@ -134,6 +140,51 @@ func TestActionRunnerHealthIsAtomicBoundedAndSecretFree(t *testing.T) { } } +func TestActionRunnerActivationFailureLeavesOnlyPendingHealthProof(t *testing.T) { + upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodPatch { + http.Error(w, "persistence unavailable", http.StatusServiceUnavailable) + return + } + conn, err := upgrader.Upgrade(w, request, nil) + if err != nil { + return + } + defer conn.Close() + var message wsMessage + if conn.ReadJSON(&message) != nil { + return + } + ack, _ := json.Marshal(registeredPayload{Success: true}) + _ = conn.WriteJSON(wsMessage{Type: msgTypeRegistered, Timestamp: time.Now(), Payload: ack}) + _, _, _ = conn.ReadMessage() + })) + defer server.Close() + dir := t.TempDir() + healthPath := filepath.Join(dir, "health.json") + logger := zerolog.Nop() + client := NewActionRunnerClient(ActionRunnerClientConfig{ + PulseURL: server.URL, APIToken: "runner-token", StateDir: filepath.Join(dir, "state"), + HealthPath: healthPath, ActivationNonce: strings.Repeat("d", 32), InsecureSkipVerify: true, Logger: &logger, + }, "agent-1", "host-1", "v1") + defer client.Close() + if err := client.connectAndHandle(context.Background()); err == nil || !strings.Contains(err.Error(), "503 Service Unavailable") { + t.Fatalf("activation error = %v", err) + } + data, err := os.ReadFile(healthPath) + if err != nil { + t.Fatal(err) + } + var health actionRunnerHealth + if err := json.Unmarshal(data, &health); err != nil { + t.Fatal(err) + } + if !health.Registered || health.Activated || health.ActivationNonce != strings.Repeat("d", 32) { + t.Fatalf("pending health = %+v", health) + } +} + func jsonContains(data []byte, value string) bool { var decoded any if json.Unmarshal(data, &decoded) != nil { diff --git a/internal/hostagent/commands.go b/internal/hostagent/commands.go index 8f883648d..8789e642e 100644 --- a/internal/hostagent/commands.go +++ b/internal/hostagent/commands.go @@ -137,11 +137,12 @@ type CommandClient struct { // actionRunnerOnly is an immutable constructor-selected protocol ceiling. // It permits only the closed typed action families and their receipt, // preflight, observation, cancellation, and liveness messages. - actionRunnerOnly bool - runtimeRole string - actionCapability string - healthPath string - healthCapabilities []string + actionRunnerOnly bool + runtimeRole string + actionCapability string + healthPath string + healthCapabilities []string + actionActivationNonce string } // NewCommandClient creates a new command execution client @@ -419,9 +420,15 @@ func (c *CommandClient) connectAndHandle(ctx context.Context) error { return fmt.Errorf("registration failed: %w", err) } if c.actionRunnerOnly { - if err := c.writeActionRunnerHealth(); err != nil { + if err := c.writeActionRunnerHealth(false); err != nil { return fmt.Errorf("write action-runner health: %w", err) } + if err := c.activateActionRunnerCredential(ctx); err != nil { + return err + } + if err := c.writeActionRunnerHealth(true); err != nil { + return fmt.Errorf("write activated action-runner health: %w", err) + } } c.logger.Info().Msg("Connected and registered with Pulse command server") diff --git a/internal/hostagent/commands_host_update_test.go b/internal/hostagent/commands_host_update_test.go index ea9bf2a02..a15a93ca1 100644 --- a/internal/hostagent/commands_host_update_test.go +++ b/internal/hostagent/commands_host_update_test.go @@ -142,6 +142,46 @@ func TestCommandClientHandlesTypedHostUpdateWithoutExecuteCommand(t *testing.T) } } +func TestActionRunnerHealthSeparatesRegisteredFromActivated(t *testing.T) { + dir := t.TempDir() + healthPath := filepath.Join(dir, "health.json") + logger := zerolog.Nop() + client := NewActionRunnerClient(ActionRunnerClientConfig{ + PulseURL: "https://pulse.example", APIToken: "runner-secret", + StateDir: filepath.Join(dir, "state"), HealthPath: healthPath, + ActivationNonce: strings.Repeat("e", 32), Logger: &logger, + }, "agent-1", "host-1.local", "v1") + defer client.Close() + if err := client.writeActionRunnerHealth(false); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(healthPath) + if err != nil { + t.Fatal(err) + } + var pending actionRunnerHealth + if err := json.Unmarshal(data, &pending); err != nil { + t.Fatal(err) + } + if !pending.Registered || pending.Activated { + t.Fatalf("pending health = %+v", pending) + } + if err := client.writeActionRunnerHealth(true); err != nil { + t.Fatal(err) + } + data, err = os.ReadFile(healthPath) + if err != nil { + t.Fatal(err) + } + var active actionRunnerHealth + if err := json.Unmarshal(data, &active); err != nil { + t.Fatal(err) + } + if !active.Registered || !active.Activated || active.ActivationNonce != strings.Repeat("e", 32) { + t.Fatalf("activated health = %+v", active) + } +} + func TestHostUpdateInventoryDriftReceiptCompletesAndReplaysWithAdmittedIdentity(t *testing.T) { receipts, err := operationreceipt.Open(filepath.Join(t.TempDir(), "receipts.db"), hostOperationReceiptConfig()) if err != nil { diff --git a/scripts/install.sh b/scripts/install.sh index 43b9aa4c1..21e13dcbe 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -182,6 +182,7 @@ ACTION_RUNNER_ENV_FILE="${ACTION_RUNNER_CONFIG_DIR}/runner.env" ACTION_RUNNER_TOKEN_FILE="${ACTION_RUNNER_CONFIG_DIR}/token" ACTION_RUNNER_STATE_DIR="/var/lib/pulse-agent-runner" ACTION_RUNNER_HEALTH_FILE="${ACTION_RUNNER_STATE_DIR}/health.json" +ACTION_RUNNER_ACTIVATION_NONCE="" ACTION_TOKEN="" ACTION_TOKEN_FILE_PATH="" TMP_ACTION_RUNNER_BIN="" @@ -1355,15 +1356,18 @@ write_action_runner_config() { if [[ -z "$runner_hostname" ]]; then runner_hostname=$(hostname 2>/dev/null || true) fi - [[ -n "$runner_hostname" && "$runner_hostname" != *$'\r'* && "$runner_hostname" != *$'\n'* ]] || - fail "Action runner requires a canonical hostname" "$EXIT_MISSING_ARGS" + [[ -n "$runner_hostname" && "$runner_hostname" != *$'\r'* && "$runner_hostname" != *$'\n'* ]] || + fail "Action runner requires a canonical hostname" "$EXIT_MISSING_ARGS" + [[ "$ACTION_RUNNER_ACTIVATION_NONCE" =~ ^[a-f0-9]{64}$ ]] || + fail "Action runner requires a fresh activation nonce" "$EXIT_GENERAL" : > "$ACTION_RUNNER_ENV_FILE" chmod 0600 "$ACTION_RUNNER_ENV_FILE" write_action_runner_env_value "PULSE_URL" "$PULSE_URL" write_action_runner_env_value "PULSE_AGENT_RUNNER_TOKEN_FILE" "$ACTION_RUNNER_TOKEN_FILE" write_action_runner_env_value "PULSE_AGENT_RUNNER_STATE_DIR" "$ACTION_RUNNER_STATE_DIR" - write_action_runner_env_value "PULSE_AGENT_RUNNER_HEALTH_FILE" "$ACTION_RUNNER_HEALTH_FILE" + write_action_runner_env_value "PULSE_AGENT_RUNNER_HEALTH_FILE" "$ACTION_RUNNER_HEALTH_FILE" + write_action_runner_env_value "PULSE_AGENT_RUNNER_ACTIVATION_NONCE" "$ACTION_RUNNER_ACTIVATION_NONCE" write_action_runner_env_value "PULSE_AGENT_RUNNER_AGENT_ID_FILE" "${STATE_DIR%/}/agent-id" write_action_runner_env_value "PULSE_AGENT_RUNNER_HOSTNAME" "$runner_hostname" if [[ -n "$SERVER_FINGERPRINT" ]]; then @@ -1378,6 +1382,33 @@ write_action_runner_config() { chown root:root "$ACTION_RUNNER_ENV_FILE" } +generate_action_runner_activation_nonce() { + local nonce="" + nonce=$(od -An -N32 -tx1 /dev/urandom 2>/dev/null | tr -d '[:space:]' || true) + [[ "$nonce" =~ ^[a-f0-9]{64}$ ]] || return 1 + printf '%s\n' "$nonce" +} + +action_runner_health_matches_activation() { + local expected_agent_id="$1" + local expected_nonce="$2" + local health_agent_id="" + local health_activation_nonce="" + local health_owner="" + local health_mode="" + + [[ -n "$expected_agent_id" && "$expected_nonce" =~ ^[a-f0-9]{64}$ ]] || return 1 + [[ -f "$ACTION_RUNNER_HEALTH_FILE" && ! -L "$ACTION_RUNNER_HEALTH_FILE" ]] || return 1 + health_owner=$(stat -c '%u' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || true) + health_mode=$(stat -c '%a' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || true) + health_agent_id=$(sed -n 's/.*"host_id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$ACTION_RUNNER_HEALTH_FILE" | head -1) + health_activation_nonce=$(sed -n 's/.*"activation_nonce"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$ACTION_RUNNER_HEALTH_FILE" | head -1) + [[ "$health_owner" == "0" && "$health_mode" =~ ^(400|600)$ ]] || return 1 + grep -Eq '"registered"[[:space:]]*:[[:space:]]*true([[:space:],}]|$)' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || return 1 + grep -Eq '"activated"[[:space:]]*:[[:space:]]*true([[:space:],}]|$)' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || return 1 + [[ "$health_agent_id" == "$expected_agent_id" && "$health_activation_nonce" == "$expected_nonce" ]] +} + write_action_runner_env_value() { local key="$1" local value="$2" @@ -1396,9 +1427,9 @@ provision_action_runner() { local had_unit="false" local had_state_dir="false" local had_config_dir="false" - local runner_active="false" - local apply_succeeded="false" - local activation_started_at="" + local runner_active="false" + local apply_succeeded="false" + local activation_nonce="" if [[ -d "$ACTION_RUNNER_STATE_DIR" ]]; then had_state_dir="true" @@ -1406,7 +1437,7 @@ provision_action_runner() { if [[ -d "$ACTION_RUNNER_CONFIG_DIR" ]]; then had_config_dir="true" fi - for path in "$ACTION_RUNNER_BINARY_PATH" "$ACTION_RUNNER_SERVICE_UNIT" "$ACTION_RUNNER_ENV_FILE" "$ACTION_RUNNER_TOKEN_FILE" "$ACTION_RUNNER_HEALTH_FILE"; do + for path in "$ACTION_RUNNER_BINARY_PATH" "$ACTION_RUNNER_SERVICE_UNIT" "$ACTION_RUNNER_ENV_FILE" "$ACTION_RUNNER_TOKEN_FILE"; do if [[ -e "$path" ]]; then cp -a "$path" "${path}${backup_suffix}" case "$path" in @@ -1414,11 +1445,14 @@ provision_action_runner() { "$ACTION_RUNNER_SERVICE_UNIT") had_unit="true" ;; esac fi - done + done - activation_started_at=$(date +%s) - if ( - set -e + activation_nonce=$(generate_action_runner_activation_nonce) || + fail "Could not generate an action-runner activation nonce" "$EXIT_GENERAL" + ACTION_RUNNER_ACTIVATION_NONCE="$activation_nonce" + if ( + set -e + rm -f "$ACTION_RUNNER_HEALTH_FILE" mkdir -p "$PRIVILEGE_HELPER_DIR" install -o root -g root -m 0755 "$TMP_ACTION_RUNNER_BIN" "${ACTION_RUNNER_BINARY_PATH}.new" mv "${ACTION_RUNNER_BINARY_PATH}.new" "$ACTION_RUNNER_BINARY_PATH" @@ -1438,26 +1472,13 @@ provision_action_runner() { if [[ "$apply_succeeded" == "true" ]]; then local attempt=0 - while [[ "$attempt" -lt 30 ]]; do - local expected_agent_id="${AGENT_ID}" - local health_agent_id="" - local health_mtime="" - local health_owner="" - local health_mode="" - if [[ -s "${STATE_DIR%/}/agent-id" ]]; then - expected_agent_id=$(head -1 "${STATE_DIR%/}/agent-id" 2>/dev/null || true) - fi - if [[ -n "$expected_agent_id" && -f "$ACTION_RUNNER_HEALTH_FILE" && ! -L "$ACTION_RUNNER_HEALTH_FILE" ]]; then - health_mtime=$(stat -c '%Y' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || true) - health_owner=$(stat -c '%u' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || true) - health_mode=$(stat -c '%a' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null || true) - health_agent_id=$(sed -n 's/.*"host_id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$ACTION_RUNNER_HEALTH_FILE" | head -1) - fi - if systemctl is-active --quiet "${ACTION_RUNNER_NAME}.service" && - [[ "$health_mtime" =~ ^[0-9]+$ ]] && [[ "$health_mtime" -ge "$activation_started_at" ]] && - [[ "$health_owner" == "0" ]] && [[ "$health_mode" =~ ^(400|600)$ ]] && - grep -Eq '"registered"[[:space:]]*:[[:space:]]*true([[:space:],}]|$)' "$ACTION_RUNNER_HEALTH_FILE" 2>/dev/null && - [[ "$health_agent_id" == "$expected_agent_id" ]]; then + while [[ "$attempt" -lt 30 ]]; do + local expected_agent_id="${AGENT_ID}" + if [[ -s "${STATE_DIR%/}/agent-id" ]]; then + expected_agent_id=$(head -1 "${STATE_DIR%/}/agent-id" 2>/dev/null || true) + fi + if systemctl is-active --quiet "${ACTION_RUNNER_NAME}.service" && + action_runner_health_matches_activation "$expected_agent_id" "$activation_nonce"; then runner_active="true" break fi @@ -1471,7 +1492,8 @@ provision_action_runner() { systemctl stop "${ACTION_RUNNER_NAME}.service" 2>/dev/null || true systemctl disable "${ACTION_RUNNER_NAME}.service" 2>/dev/null || true rm -f "${ACTION_RUNNER_BINARY_PATH}.new" "${ACTION_RUNNER_SERVICE_UNIT}.new" - for path in "$ACTION_RUNNER_BINARY_PATH" "$ACTION_RUNNER_SERVICE_UNIT" "$ACTION_RUNNER_ENV_FILE" "$ACTION_RUNNER_TOKEN_FILE" "$ACTION_RUNNER_HEALTH_FILE"; do + rm -f "$ACTION_RUNNER_HEALTH_FILE" + for path in "$ACTION_RUNNER_BINARY_PATH" "$ACTION_RUNNER_SERVICE_UNIT" "$ACTION_RUNNER_ENV_FILE" "$ACTION_RUNNER_TOKEN_FILE"; do rm -f "$path" if [[ -e "${path}${backup_suffix}" ]]; then mv "${path}${backup_suffix}" "$path" @@ -1486,16 +1508,17 @@ provision_action_runner() { systemctl daemon-reload 2>/dev/null || true if [[ "$had_unit" == "true" && "$had_binary" == "true" ]]; then systemctl enable --now "${ACTION_RUNNER_NAME}.service" 2>/dev/null || true - fi - fail "Action runner activation failed and its previous installation was restored; collector monitoring was not stopped or removed" "$EXIT_GENERAL" + fi + ACTION_RUNNER_ACTIVATION_NONCE="" + fail "Action runner activation failed and its previous installation was restored; collector monitoring was not stopped or removed" "$EXIT_GENERAL" fi rm -f \ "${ACTION_RUNNER_BINARY_PATH}${backup_suffix}" \ - "${ACTION_RUNNER_SERVICE_UNIT}${backup_suffix}" \ - "${ACTION_RUNNER_ENV_FILE}${backup_suffix}" \ - "${ACTION_RUNNER_TOKEN_FILE}${backup_suffix}" \ - "${ACTION_RUNNER_HEALTH_FILE}${backup_suffix}" + "${ACTION_RUNNER_SERVICE_UNIT}${backup_suffix}" \ + "${ACTION_RUNNER_ENV_FILE}${backup_suffix}" \ + "${ACTION_RUNNER_TOKEN_FILE}${backup_suffix}" + ACTION_RUNNER_ACTIVATION_NONCE="" TMP_ACTION_RUNNER_BIN="" log_info "Typed action runner enabled as a separate root service with its own credential; collector monitoring remains independently active." } diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go index a4a1a50b3..52830cbc0 100644 --- a/scripts/installtests/install_sh_test.go +++ b/scripts/installtests/install_sh_test.go @@ -3,6 +3,7 @@ package installtests import ( "encoding/json" "errors" + "fmt" "io" "net" "net/http" @@ -5953,6 +5954,7 @@ func TestInstallSHActionRunnerIsSeparateOptInLifecycle(t *testing.T) { `write_action_runner_env_value "PULSE_AGENT_RUNNER_TOKEN_FILE" "$ACTION_RUNNER_TOKEN_FILE"`, `write_action_runner_env_value "PULSE_AGENT_RUNNER_AGENT_ID_FILE" "${STATE_DIR%/}/agent-id"`, `write_action_runner_env_value "PULSE_AGENT_RUNNER_HEALTH_FILE" "$ACTION_RUNNER_HEALTH_FILE"`, + `write_action_runner_env_value "PULSE_AGENT_RUNNER_ACTIVATION_NONCE" "$ACTION_RUNNER_ACTIVATION_NONCE"`, `write_action_runner_env_value "PULSE_AGENT_RUNNER_HOSTNAME" "$runner_hostname"`, `DELETE --data-binary "$payload"`, `/api/agents/action-runner/credential`, @@ -5962,10 +5964,8 @@ func TestInstallSHActionRunnerIsSeparateOptInLifecycle(t *testing.T) { `install -o root -g root -m 0755 "$TMP_ACTION_RUNNER_BIN"`, `chmod 0600 "$ACTION_RUNNER_TOKEN_FILE"`, `ACTION_TOKEN=""`, - `health_mtime=$(stat -c '%Y' "$ACTION_RUNNER_HEALTH_FILE"`, - `"registered"[[:space:]]*:[[:space:]]*true`, - `"host_id"[[:space:]]*:[[:space:]]*`, - `[[ "$health_agent_id" == "$expected_agent_id" ]]`, + `action_runner_health_matches_activation "$expected_agent_id" "$activation_nonce"`, + `[[ "$health_agent_id" == "$expected_agent_id" && "$health_activation_nonce" == "$expected_nonce" ]]`, `rolling back runner-only files while leaving monitoring active`, `Pulse action runner removed. Collector monitoring was left installed and running.`, } { @@ -5979,6 +5979,61 @@ func TestInstallSHActionRunnerIsSeparateOptInLifecycle(t *testing.T) { } } +func TestInstallSHActionRunnerReadinessRequiresCurrentActivationNonce(t *testing.T) { + root := t.TempDir() + healthPath := filepath.Join(root, "health.json") + currentNonce := strings.Repeat("a", 64) + priorNonce := strings.Repeat("b", 64) + writeHealth := func(activated bool, nonce string) { + t.Helper() + payload := fmt.Sprintf(`{"registered":true,"activated":%t,"activation_nonce":%q,"host_id":"agent-1"}`, activated, nonce) + if err := os.WriteFile(healthPath, []byte(payload), 0o600); err != nil { + t.Fatal(err) + } + future := time.Now().Add(24 * time.Hour) + if err := os.Chtimes(healthPath, future, future); err != nil { + t.Fatal(err) + } + } + check := func(nonce string) bool { + t.Helper() + script := ` +set -euo pipefail +ACTION_RUNNER_HEALTH_FILE="` + healthPath + `" +stat() { + case "$1" in + -c) case "$2" in '%u') printf '0\n' ;; '%a') printf '600\n' ;; esac ;; + esac +} +` + extractInstallShellFunction(t, "action_runner_health_matches_activation") + ` +action_runner_health_matches_activation "agent-1" "` + nonce + `" +` + return exec.Command("bash", "-c", script).Run() == nil + } + + writeHealth(true, priorNonce) + if check(currentNonce) { + t.Fatal("stale marker from a prior activation nonce was accepted") + } + writeHealth(false, currentNonce) + if check(currentNonce) { + t.Fatal("registered but uncommitted marker was accepted") + } + writeHealth(true, currentNonce) + if !check(currentNonce) { + t.Fatal("current activated marker was rejected because of filesystem clock skew") + } + if err := os.Remove(healthPath); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "missing"), healthPath); err != nil { + t.Fatal(err) + } + if check(currentNonce) { + t.Fatal("symlinked activation marker was accepted") + } +} + func TestInstallSHActionRunnerSelfRevokeUsesPrivateCredential(t *testing.T) { const token = "runner-secret-that-must-not-appear-in-output" var gotAuthorization string