From d06ffc233def84d42c25d857a7bbb2d64cfbbed8 Mon Sep 17 00:00:00 2001 From: Pulse Test Date: Sun, 30 Aug 2026 01:41:57 +0100 Subject: [PATCH] Harden secure agent runtime transitions --- cmd/pulse-agent-runner/main.go | 46 +- cmd/pulse-agent-runner/main_test.go | 30 +- cmd/pulse-agent/main.go | 124 ++- cmd/pulse-agent/main_test.go | 222 +++- .../internal/AGENT_PRIVILEGE_BOUNDARY_PLAN.md | 36 +- ...ime-qualification-foundation-2026-08-30.md | 76 ++ ...nt-runtime-systemd-receipt-2026-08-30.json | 69 ++ .../v6/internal/subsystems/agent-lifecycle.md | 31 + .../v6/internal/subsystems/api-contracts.md | 20 +- .../subsystems/deployment-installability.md | 25 +- .../internal/subsystems/security-privacy.md | 17 + .../internal/subsystems/storage-recovery.md | 9 +- internal/agentexec/server.go | 47 + internal/agentexec/server_websocket_test.go | 67 ++ internal/agenthelper/protocol.go | 1 + internal/agenthelper/providers.go | 1 + internal/agenthelper/registry.go | 14 + internal/agenthelper/server_test.go | 19 +- internal/agenthelper/update_activation.go | 333 +++++- .../agenthelper/update_activation_test.go | 102 ++ internal/agentupdate/coverage_test.go | 32 + internal/agentupdate/privileged_update.go | 189 +++- .../agentupdate/privileged_update_test.go | 105 +- internal/agentupdate/update.go | 39 +- internal/api/action_runner_credentials.go | 125 ++- .../api/action_runner_credentials_test.go | 157 ++- internal/api/agenttokens/install.go | 56 +- internal/api/agenttokens/install_test.go | 22 + internal/api/router_routes_registration.go | 2 +- .../api/security_tokens_lifecycle_test.go | 28 + internal/hostagent/agent.go | 26 +- internal/hostagent/agent_flushbuffer_test.go | 8 +- internal/hostagent/agent_new_test.go | 20 + internal/hostagent/privilege_helper_client.go | 43 + .../hostagent/privilege_helper_client_test.go | 125 +++ scripts/install.sh | 336 +++++- scripts/installtests/install_sh_test.go | 85 ++ .../safe_profile_migration_test.go | 303 +++++- .../secure_runtime_systemd_lab_test.go | 979 ++++++++++++++++++ 39 files changed, 3818 insertions(+), 151 deletions(-) create mode 100644 docs/release-control/v6/internal/records/secure-agent-runtime-qualification-foundation-2026-08-30.md create mode 100644 docs/release-control/v6/internal/records/secure-agent-runtime-systemd-receipt-2026-08-30.json create mode 100644 scripts/installtests/secure_runtime_systemd_lab_test.go diff --git a/cmd/pulse-agent-runner/main.go b/cmd/pulse-agent-runner/main.go index 6e1ec8325..6543005e2 100644 --- a/cmd/pulse-agent-runner/main.go +++ b/cmd/pulse-agent-runner/main.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "net/url" "os" "os/signal" @@ -25,6 +26,7 @@ type runtimeConfig struct { StateDir string HealthFile string AgentIDFile string + Hostname string ServerFingerprint string CAFile string Insecure bool @@ -37,6 +39,7 @@ func loadConfig() (runtimeConfig, error) { StateDir: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_STATE_DIR")), HealthFile: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_HEALTH_FILE")), AgentIDFile: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE")), + Hostname: strings.TrimSpace(os.Getenv("PULSE_AGENT_RUNNER_HOSTNAME")), ServerFingerprint: strings.TrimSpace(os.Getenv("PULSE_SERVER_FINGERPRINT")), CAFile: strings.TrimSpace(os.Getenv("SSL_CERT_FILE")), Insecure: strings.EqualFold(strings.TrimSpace(os.Getenv("PULSE_INSECURE")), "true"), @@ -44,6 +47,13 @@ func loadConfig() (runtimeConfig, error) { if config.PulseURL == "" || config.TokenFile == "" || config.StateDir == "" || config.HealthFile == "" || config.AgentIDFile == "" { return runtimeConfig{}, errors.New("PULSE_URL and the action-runner token, state, health, and agent identity file settings are required") } + if config.Hostname != "" { + hostname, err := normalizeRunnerHostname(config.Hostname) + if err != nil { + return runtimeConfig{}, err + } + config.Hostname = hostname + } parsed, err := url.Parse(config.PulseURL) if err != nil || parsed.Host == "" || (parsed.Scheme != "https" && !(config.Insecure && parsed.Scheme == "http")) { return runtimeConfig{}, errors.New("PULSE_URL must be HTTPS unless PULSE_INSECURE=true explicitly permits HTTP") @@ -83,9 +93,13 @@ func run() error { if err != nil { return err } - hostname, err := os.Hostname() - if err != nil || strings.TrimSpace(hostname) == "" { - return errors.New("determine action-runner hostname") + hostname := config.Hostname + if hostname == "" { + hostname, err = os.Hostname() + hostname = strings.TrimSpace(hostname) + if err != nil || hostname == "" { + return errors.New("determine action-runner hostname") + } } logger := zerolog.New(os.Stderr).With().Timestamp().Str("component", "action-runner").Logger() transportConfig := actionrunner.TransportConfig{ @@ -112,6 +126,32 @@ func run() error { return nil } +func normalizeRunnerHostname(value string) (string, error) { + hostname := strings.ToLower(strings.TrimSpace(value)) + hostname = strings.TrimRight(hostname, ".") + if hostname == "" || len(hostname) > 253 { + return "", errors.New("PULSE_AGENT_RUNNER_HOSTNAME must be a valid hostname or IP address") + } + if net.ParseIP(hostname) != nil { + return hostname, nil + } + for _, label := range strings.Split(hostname, ".") { + if len(label) == 0 || len(label) > 63 || !runnerHostnameLabelByte(label[0]) || !runnerHostnameLabelByte(label[len(label)-1]) { + return "", errors.New("PULSE_AGENT_RUNNER_HOSTNAME must be a valid hostname or IP address") + } + for index := 1; index < len(label)-1; index++ { + if !runnerHostnameLabelByte(label[index]) && label[index] != '-' { + return "", errors.New("PULSE_AGENT_RUNNER_HOSTNAME must be a valid hostname or IP address") + } + } + } + return hostname, nil +} + +func runnerHostnameLabelByte(value byte) bool { + return value >= 'a' && value <= 'z' || value >= '0' && value <= '9' +} + func readPrivateValue(path, label string) (string, error) { info, err := os.Lstat(path) if err != nil { diff --git a/cmd/pulse-agent-runner/main_test.go b/cmd/pulse-agent-runner/main_test.go index 65b6908b1..128c9acf5 100644 --- a/cmd/pulse-agent-runner/main_test.go +++ b/cmd/pulse-agent-runner/main_test.go @@ -22,16 +22,44 @@ func TestLoadConfigUsesDedicatedEnvironmentAndPrivateTokenFile(t *testing.T) { t.Setenv("PULSE_AGENT_RUNNER_STATE_DIR", filepath.Join(dir, "state")) t.Setenv("PULSE_AGENT_RUNNER_HEALTH_FILE", filepath.Join(dir, "state", "health.json")) t.Setenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE", agentID) + t.Setenv("PULSE_AGENT_RUNNER_HOSTNAME", " Node.Example. ") t.Setenv("PULSE_SERVER_FINGERPRINT", "sha256:test") config, err := loadConfig() if err != nil { t.Fatal(err) } - if config.TokenFile != token || config.ServerFingerprint != "sha256:test" || config.Insecure { + if config.TokenFile != token || config.ServerFingerprint != "sha256:test" || config.Hostname != "node.example" || config.Insecure { t.Fatalf("config = %+v", config) } } +func TestLoadConfigRejectsInvalidCanonicalHostnameOverride(t *testing.T) { + dir := t.TempDir() + token := filepath.Join(dir, "runner.token") + if err := os.WriteFile(token, []byte("secret\n"), 0600); err != nil { + t.Fatal(err) + } + t.Setenv("PULSE_URL", "https://pulse.example") + t.Setenv("PULSE_AGENT_RUNNER_TOKEN_FILE", token) + t.Setenv("PULSE_AGENT_RUNNER_STATE_DIR", filepath.Join(dir, "state")) + t.Setenv("PULSE_AGENT_RUNNER_HEALTH_FILE", filepath.Join(dir, "state", "health.json")) + t.Setenv("PULSE_AGENT_RUNNER_AGENT_ID_FILE", filepath.Join(dir, "agent-id")) + for _, hostname := range []string{"bad host", "-node.example", "node/example", strings.Repeat("a", 64) + ".example"} { + t.Run(hostname, func(t *testing.T) { + t.Setenv("PULSE_AGENT_RUNNER_HOSTNAME", hostname) + if _, err := loadConfig(); err == nil || !strings.Contains(err.Error(), "valid hostname") { + t.Fatalf("loadConfig error = %v", err) + } + }) + } +} + +func TestNormalizeRunnerHostnameAcceptsIPLiteral(t *testing.T) { + if got, err := normalizeRunnerHostname(" 2001:DB8::1 "); err != nil || got != "2001:db8::1" { + t.Fatalf("normalizeRunnerHostname = %q, %v", got, err) + } +} + func TestLoadConfigRejectsTokenInArgvEquivalentAndInsecureHTTPByDefault(t *testing.T) { dir := t.TempDir() t.Setenv("PULSE_URL", "http://pulse.example") diff --git a/cmd/pulse-agent/main.go b/cmd/pulse-agent/main.go index efb0c4e6b..922c2c104 100644 --- a/cmd/pulse-agent/main.go +++ b/cmd/pulse-agent/main.go @@ -119,6 +119,87 @@ func wireUpdaterHooks(hostCfg *hostagent.Config, updater *agentupdate.Updater) { hostCfg.OnServerVersion = updater.NudgeVersion } +func pendingUpdatePreviousVersion(pending *agentupdate.PendingPrivilegedUpdate) string { + if pending == nil { + return "" + } + return pending.PreviousVersion +} + +func supervisePendingPrivilegedUpdate( + ctx context.Context, + update agentupdate.PrivilegedUpdate, + pending *agentupdate.PendingPrivilegedUpdate, + stateDir string, + locallyReady func() bool, + reportAccepted <-chan struct{}, + pollInterval time.Duration, + logger *zerolog.Logger, +) error { + if pending == nil { + return nil + } + if update == nil || locallyReady == nil || pollInterval <= 0 { + return errors.New("pending helper update supervisor is not configured") + } + activation := pending.Activation + rollback := func(reason error) error { + rollbackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + result, err := update.Rollback(rollbackCtx, activation) + if err != nil { + return errors.Join(reason, fmt.Errorf("typed helper rollback failed: %w", err)) + } + if result.Action != "rolled_back" || !strings.EqualFold(result.ActiveSHA256, activation.RollbackSHA256) { + return errors.Join(reason, errors.New("typed helper returned an invalid rollback result")) + } + if clearErr := agentupdate.ClearPendingPrivilegedUpdate(stateDir); clearErr != nil { + return errors.Join(fmt.Errorf("%w; pending update rolled back", reason), fmt.Errorf("clear pending update handoff: %w", clearErr)) + } + return fmt.Errorf("%w; pending update rolled back", reason) + } + + deadlineDelay := time.Until(activation.RollbackDeadline) + if deadlineDelay <= 0 { + return rollback(errors.New("pending update health deadline expired before startup")) + } + deadline := time.NewTimer(deadlineDelay) + defer deadline.Stop() + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + reportOK := false + for { + if reportOK && locallyReady() { + commitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + result, err := update.Commit(commitCtx, activation) + cancel() + if err == nil && result.Action == "committed" && strings.EqualFold(result.ActiveSHA256, activation.ActiveSHA256) { + if err := agentupdate.ClearPendingPrivilegedUpdate(stateDir); err != nil { + return fmt.Errorf("clear committed update handoff: %w", err) + } + if logger != nil { + logger.Info().Str("activation_id", activation.ActivationID).Msg("Committed helper-backed agent update after local readiness and server report") + } + return nil + } + if logger != nil && err != nil { + logger.Warn().Err(err).Str("activation_id", activation.ActivationID).Msg("Pending helper update commit failed; retrying until rollback deadline") + } + } + + select { + case <-ctx.Done(): + return rollback(fmt.Errorf("runtime stopped before pending update commit: %w", ctx.Err())) + case <-deadline.C: + return rollback(errors.New("pending update did not reach local readiness and accepted-report health floor before deadline")) + case <-reportAccepted: + reportOK = true + reportAccepted = nil + case <-ticker.C: + } + } +} + type multiValue []string func (m *multiValue) String() string { @@ -383,7 +464,8 @@ func run(ctx context.Context, args []string, getenv func(string) string) error { // 7. Start Auto-Updater var privilegedUpdate agentupdate.PrivilegedUpdate - if helperSocket := strings.TrimSpace(os.Getenv("PULSE_AGENT_HELPER_SOCKET")); helperSocket != "" { + helperSocket := strings.TrimSpace(os.Getenv("PULSE_AGENT_HELPER_SOCKET")) + if helperSocket != "" { privilegedUpdate, err = newPrivilegeHelperUpdate(helperSocket) if err != nil { return fmt.Errorf("configure typed privilege-helper updates: %w", err) @@ -403,6 +485,18 @@ func run(ctx context.Context, args []string, getenv func(string) string) error { Disabled: cfg.DisableAutoUpdate, PrivilegedUpdate: privilegedUpdate, }) + var pendingUpdate *agentupdate.PendingPrivilegedUpdate + if helperSocket != "" { + pendingUpdate, err = agentupdate.LoadPendingPrivilegedUpdate(cfg.StateDir) + if err != nil { + return fmt.Errorf("load pending helper update handoff: %w", err) + } + } + if pendingUpdate != nil && privilegedUpdate == nil { + return errors.New("pending helper update cannot be verified without the typed privilege helper") + } + pendingReportAccepted := make(chan struct{}) + var pendingReportOnce sync.Once g.Go(func() error { updater.RunLoop(ctx) @@ -419,6 +513,14 @@ func run(ctx context.Context, args []string, getenv func(string) string) error { if err != nil { return fmt.Errorf("configure typed privilege helper: %w", err) } + if privilegedTelemetry != nil { + healthCtx, cancelHealth := context.WithTimeout(ctx, 10*time.Second) + healthErr := privilegedTelemetry.Health(healthCtx) + cancelHealth() + if healthErr != nil { + return fmt.Errorf("verify typed privilege helper protocol: %w", healthErr) + } + } hostCfg := hostagent.Config{ PulseURL: cfg.PulseURL, APIToken: cfg.APIToken, @@ -450,10 +552,16 @@ func run(ctx context.Context, args []string, getenv func(string) string) error { ModuleStatus: runtimeStatus.moduleStatuses, Observers: hostObserverTargets(cfg.Observers), PrivilegedTelemetry: privilegedTelemetry, + UpdatedFromVersion: pendingUpdatePreviousVersion(pendingUpdate), DockerContainerUpdater: dockerUpdaterBridge, DockerContainerLifecycleOperator: dockerUpdaterBridge, } + if pendingUpdate != nil { + hostCfg.OnPrimaryReportAccepted = func() { + pendingReportOnce.Do(func() { close(pendingReportAccepted) }) + } + } wireUpdaterHooks(&hostCfg, updater) agent, err := newHostAgent(hostCfg) @@ -592,6 +700,20 @@ func run(ctx context.Context, args []string, getenv func(string) string) error { return nil }) } + if pendingUpdate != nil { + g.Go(func() error { + return supervisePendingPrivilegedUpdate( + ctx, + privilegedUpdate, + pendingUpdate, + cfg.StateDir, + ready.Load, + pendingReportAccepted, + 250*time.Millisecond, + &logger, + ) + }) + } // 11. Wait for all agents to exit if err := g.Wait(); err != nil && err != context.Canceled { diff --git a/cmd/pulse-agent/main_test.go b/cmd/pulse-agent/main_test.go index d7181365c..e322e827e 100644 --- a/cmd/pulse-agent/main_test.go +++ b/cmd/pulse-agent/main_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" + "github.com/rcourtman/pulse-go-rewrite/internal/agenthelper" "github.com/rcourtman/pulse-go-rewrite/internal/agentupdate" "github.com/rcourtman/pulse-go-rewrite/internal/dockeragent" "github.com/rcourtman/pulse-go-rewrite/internal/hostagent" @@ -1754,9 +1755,10 @@ func TestRunConfiguresTypedPrivilegeHelperFromInstallerEnvironment(t *testing.T) t.Setenv("PULSE_AGENT_HELPER_SOCKET", socketPath) configuredPath := "" configuredUpdatePath := "" + helper := &helperHealthStub{} newPrivilegeHelperTelemetry = func(path string) (hostagent.PrivilegedTelemetry, error) { configuredPath = path - return nil, nil + return helper, nil } newPrivilegeHelperUpdate = func(path string) (agentupdate.PrivilegedUpdate, error) { configuredUpdatePath = path @@ -1786,6 +1788,224 @@ func TestRunConfiguresTypedPrivilegeHelperFromInstallerEnvironment(t *testing.T) if configuredUpdatePath != socketPath { t.Fatalf("helper update socket path = %q, want %q", configuredUpdatePath, socketPath) } + if helper.healthCalls != 1 { + t.Fatalf("helper health calls = %d, want 1", helper.healthCalls) + } +} + +type helperHealthStub struct { + hostagent.PrivilegedTelemetry + healthErr error + healthCalls int +} + +func (h *helperHealthStub) Health(context.Context) error { + h.healthCalls++ + return h.healthErr +} + +func TestRunRejectsUnhealthyTypedPrivilegeHelper(t *testing.T) { + originalHelper := newPrivilegeHelperTelemetry + originalUpdate := newPrivilegeHelperUpdate + originalUpdater := newUpdater + originalHost := newHostAgent + defer func() { + newPrivilegeHelperTelemetry = originalHelper + newPrivilegeHelperUpdate = originalUpdate + newUpdater = originalUpdater + newHostAgent = originalHost + }() + + t.Setenv("PULSE_AGENT_HELPER_SOCKET", "/run/pulse-agent/helper.sock") + helper := &helperHealthStub{healthErr: errors.New("incompatible helper")} + newPrivilegeHelperTelemetry = func(string) (hostagent.PrivilegedTelemetry, error) { + return helper, nil + } + newPrivilegeHelperUpdate = func(string) (agentupdate.PrivilegedUpdate, error) { + return nil, nil + } + newUpdater = func(agentupdate.Config) *agentupdate.Updater { + return agentupdate.New(agentupdate.Config{Disabled: true}) + } + hostCreated := false + newHostAgent = func(hostagent.Config) (Runnable, error) { + hostCreated = true + return &mockRunnable{}, nil + } + + err := run(context.Background(), []string{ + "-token", "deadbeef", + "-enable-docker=false", + "-enable-kubernetes=false", + "-health-addr", "", + }, func(string) string { return "" }) + if err == nil || !strings.Contains(err.Error(), "verify typed privilege helper protocol") { + t.Fatalf("run error = %v", err) + } + if helper.healthCalls != 1 { + t.Fatalf("helper health calls = %d, want 1", helper.healthCalls) + } + if hostCreated { + t.Fatal("host agent was created after helper health failed") + } +} + +type pendingUpdateSupervisorStub struct { + commitResult agenthelper.UpdateResult + rollbackResult agenthelper.UpdateResult + commitErr error + rollbackErr error + commitCalls chan agenthelper.UpdateResult + rollbackCalls chan agenthelper.UpdateResult +} + +func (s *pendingUpdateSupervisorStub) CreateQuarantinedArtifact() (string, *os.File, func() error, error) { + return "", nil, func() error { return nil }, errors.New("not implemented") +} + +func (s *pendingUpdateSupervisorStub) WriteQuarantinedSignature(string, string) error { + return errors.New("not implemented") +} + +func (s *pendingUpdateSupervisorStub) Stage(context.Context, string, string) (agenthelper.UpdateStageResult, error) { + return agenthelper.UpdateStageResult{}, errors.New("not implemented") +} + +func (s *pendingUpdateSupervisorStub) Activate(context.Context, string, string) (agenthelper.UpdateResult, error) { + return agenthelper.UpdateResult{}, errors.New("not implemented") +} + +func (s *pendingUpdateSupervisorStub) Commit(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { + s.commitCalls <- activation + return s.commitResult, s.commitErr +} + +func (s *pendingUpdateSupervisorStub) Rollback(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { + s.rollbackCalls <- activation + return s.rollbackResult, s.rollbackErr +} + +func testPendingUpdate(t *testing.T, stateDir string) *agentupdate.PendingPrivilegedUpdate { + t.Helper() + activation := agenthelper.UpdateResult{ + Action: "pending", + ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", + ActiveSHA256: strings.Repeat("a", 64), + RollbackSHA256: strings.Repeat("b", 64), + RollbackDeadline: time.Now().Add(2 * time.Second).UTC(), + } + if err := agentupdate.PersistPendingPrivilegedUpdate(stateDir, "1.0.0", activation); err != nil { + t.Fatal(err) + } + return &agentupdate.PendingPrivilegedUpdate{Activation: activation, PreviousVersion: "1.0.0"} +} + +func TestPendingPrivilegedUpdateCommitsOnlyAfterReadinessAndAcceptedReport(t *testing.T) { + stateDir := t.TempDir() + if err := os.Chmod(stateDir, 0o700); err != nil { + t.Fatal(err) + } + pending := testPendingUpdate(t, stateDir) + stub := &pendingUpdateSupervisorStub{ + commitResult: agenthelper.UpdateResult{ + Action: "committed", + ActivationID: pending.Activation.ActivationID, + ActiveSHA256: pending.Activation.ActiveSHA256, + RollbackSHA256: pending.Activation.RollbackSHA256, + }, + commitCalls: make(chan agenthelper.UpdateResult, 1), + rollbackCalls: make(chan agenthelper.UpdateResult, 1), + } + reportAccepted := make(chan struct{}) + close(reportAccepted) + var ready atomic.Bool + result := make(chan error, 1) + go func() { + result <- supervisePendingPrivilegedUpdate(context.Background(), stub, pending, stateDir, ready.Load, reportAccepted, time.Millisecond, nil) + }() + select { + case <-stub.commitCalls: + t.Fatal("pending update committed before local readiness") + case <-time.After(20 * time.Millisecond): + } + ready.Store(true) + select { + case activation := <-stub.commitCalls: + if activation != pending.Activation { + t.Fatalf("commit activation = %#v", activation) + } + case <-time.After(time.Second): + t.Fatal("pending update was not committed after both health signals") + } + if err := <-result; err != nil { + t.Fatal(err) + } + if loaded, err := agentupdate.LoadPendingPrivilegedUpdate(stateDir); err != nil || loaded != nil { + t.Fatalf("committed handoff = %#v, %v", loaded, err) + } + select { + case <-stub.rollbackCalls: + t.Fatal("healthy pending update rolled back") + default: + } +} + +func TestPendingPrivilegedUpdateCancellationRollsBackAndClearsHandoff(t *testing.T) { + stateDir := t.TempDir() + if err := os.Chmod(stateDir, 0o700); err != nil { + t.Fatal(err) + } + pending := testPendingUpdate(t, stateDir) + stub := &pendingUpdateSupervisorStub{ + rollbackResult: agenthelper.UpdateResult{ + Action: "rolled_back", + ActivationID: pending.Activation.ActivationID, + ActiveSHA256: pending.Activation.RollbackSHA256, + RollbackSHA256: pending.Activation.ActiveSHA256, + }, + commitCalls: make(chan agenthelper.UpdateResult, 1), + rollbackCalls: make(chan agenthelper.UpdateResult, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := supervisePendingPrivilegedUpdate(ctx, stub, pending, stateDir, func() bool { return false }, make(chan struct{}), time.Millisecond, nil) + if err == nil || !strings.Contains(err.Error(), "pending update rolled back") { + t.Fatalf("supervisor error = %v", err) + } + select { + case activation := <-stub.rollbackCalls: + if activation != pending.Activation { + t.Fatalf("rollback activation = %#v", activation) + } + default: + t.Fatal("pending update was not rolled back") + } + if loaded, loadErr := agentupdate.LoadPendingPrivilegedUpdate(stateDir); loadErr != nil || loaded != nil { + t.Fatalf("rolled-back handoff = %#v, %v", loaded, loadErr) + } +} + +func TestPendingPrivilegedUpdateRollbackFailurePreservesHandoff(t *testing.T) { + stateDir := t.TempDir() + if err := os.Chmod(stateDir, 0o700); err != nil { + t.Fatal(err) + } + pending := testPendingUpdate(t, stateDir) + stub := &pendingUpdateSupervisorStub{ + rollbackErr: errors.New("helper unavailable"), + commitCalls: make(chan agenthelper.UpdateResult, 1), + rollbackCalls: make(chan agenthelper.UpdateResult, 1), + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := supervisePendingPrivilegedUpdate(ctx, stub, pending, stateDir, func() bool { return false }, make(chan struct{}), time.Millisecond, nil) + if err == nil || !strings.Contains(err.Error(), "typed helper rollback failed") { + t.Fatalf("supervisor error = %v", err) + } + loaded, loadErr := agentupdate.LoadPendingPrivilegedUpdate(stateDir) + if loadErr != nil || loaded == nil || loaded.Activation != pending.Activation { + t.Fatalf("failed-rollback handoff = %#v, %v", loaded, loadErr) + } } func TestRun_AgentFailure(t *testing.T) { diff --git a/docs/release-control/v6/internal/AGENT_PRIVILEGE_BOUNDARY_PLAN.md b/docs/release-control/v6/internal/AGENT_PRIVILEGE_BOUNDARY_PLAN.md index b5b01be02..f5ac263e5 100644 --- a/docs/release-control/v6/internal/AGENT_PRIVILEGE_BOUNDARY_PLAN.md +++ b/docs/release-control/v6/internal/AGENT_PRIVILEGE_BOUNDARY_PLAN.md @@ -1,6 +1,6 @@ # Unified Agent Privilege-Boundary Plan -Last updated: 2026-08-29 +Last updated: 2026-08-30 Status: PROPOSED Governance surfaces: - `status.json.coverage_gaps.agent-privilege-boundary-separation` @@ -70,6 +70,40 @@ systemd sandboxing, scoped API tokens, and the optional least-privilege profile is retained as useful input. None of it substitutes for separating monitoring from root-equivalent and remediation authority. +## Current Implementation Checkpoint (2026-08-30) + +The optional Linux systemd path now establishes more of the intended boundary, +without changing the product default: + +- a configured helper must answer the exact versioned health protocol before + collector readiness, and failed helper SMART/Proxmox reads do not fall back + into local privileged collection; +- helper-backed collector activation is pending under a bounded deadline until + the replacement is locally ready and a newly collected authoritative report + is accepted; explicit commit, restart/power-loss recovery, watchdog rollback, + and fixed staging/quarantine cleanup preserve the last-known-good binary; +- runner enrollment persists the canonical hostname, durable credential + rotation invalidates only the superseded live session after storage commits, + and uninstall performs an exact bearer self-revoke without exposing the + secret in argv; +- safe migration rejects effective unit overrides, requires a fresh post-start + registration timestamp and live helper protocol response, restores complete + state metadata and Proxmox markers on rollback, and disables rootful Docker + unless a usable collector-owned rootless socket exists. + +This is a qualification foundation, not the Slice E ratchet. The generated +setup default remains the legacy/root profile. A clean disposable arm64 Ubuntu +systemd exercise now proves exact-working-tree install, migration, explicit and +automatic rollback, ordinary update isolation, typed helper health, fresh +registration, and continued reporting; its secret-free receipt is recorded in +`internal/records/secure-agent-runtime-systemd-receipt-2026-08-30.json`. The +repository still needs exact committed release-candidate reproduction, runner +rotation/revocation and typed action-receipt proof, representative Proxmox, +SMART, Docker and rootless Podman telemetry/action parity, appliance profiles, +and the external security review. Until those proofs are recorded, the safe +profile remains opt-in and provider degradation remains an explicit residual +rather than evidence that the candidate lane is complete. + ## Target Architecture ### 1. API-Only Monitoring diff --git a/docs/release-control/v6/internal/records/secure-agent-runtime-qualification-foundation-2026-08-30.md b/docs/release-control/v6/internal/records/secure-agent-runtime-qualification-foundation-2026-08-30.md new file mode 100644 index 000000000..65cc2f1b3 --- /dev/null +++ b/docs/release-control/v6/internal/records/secure-agent-runtime-qualification-foundation-2026-08-30.md @@ -0,0 +1,76 @@ +# Secure agent runtime qualification foundation (2026-08-30) + +## Scope + +This slice closes implementation defects found while preparing a real Linux +systemd qualification of the optional collector/helper/action-runner split. It +also records a managed-development real-systemd qualification of the collector +and helper migration boundary. It does not flip the installer default and is +not exact committed release-candidate or representative live-provider proof. + +## Landed runtime semantics + +- Collector startup exercises `helper.health` over the versioned local + protocol. Once configured, helper failures omit SMART or Proxmox privileged + snapshots instead of falling back to local privileged collection. +- Helper-backed update activation persists a pending identity, active and + last-known-good digests, and a bounded rollback deadline. The replacement + process commits only after local readiness and acceptance of its newly + collected authoritative report. Interrupted, expired, or uncommitted state + recovers to last-known-good, and terminal transitions reap fixed staging and + quarantine artifacts. +- The action runner uses the canonical enrollment hostname. Durable rotation + invalidates exactly the superseded live organization/token/agent/hostname + session after token storage succeeds. Runner teardown attempts exact + bearer-authenticated self-revocation before removing local remediation state; + remote failure remains visible and does not stop local removal. +- Safe-profile apply rejects foreign effective fragments and all systemd + drop-ins before mutation. Commit requires the intended non-root unit, a live + helper protocol response, and a server registration timestamp newer than the + stopped legacy collector. Rollback restores full state-tree metadata and the + Proxmox registration markers touched by installation. Transient runtime + files that disappear while the collector is stopped do not prevent rollback + from restoring every surviving pre-migration entry. Rootful Docker is + disabled unless the collector owns a usable rootless runtime socket. Local + readiness and helper health receive a bounded startup window before the + separate authoritative registration proof. Installed collector binaries are + pinned to mode `0755` rather than inheriting `mktemp` permissions. + +## Managed-development systemd proof + +The disposable arm64 Colima profile `pulse-agent-qual` ran Ubuntu 24.04.4, +kernel 6.8.0-117, and systemd 255. The exact working-tree installer and lab +source are bound by the hashes in +`secure-agent-runtime-systemd-receipt-2026-08-30.json`; the receipt itself has +SHA-256 `c20aee566835ac88f6163085d2d559c3fa493274b1a8eae328ed39063af06e5e`. +The guarded lab passed all of these destructive scenarios from a clean VM: + +- legacy root/command-capable install with rootful Docker enabled; +- read-only profile inspection and fail-closed rejection of a systemd drop-in; +- migration to a non-root `pulse-agent` collector with no ambient capabilities, + monitoring-only authority, a healthy typed helper, fresh server `lastSeen`, + explicit rootful-Docker degradation, and continued reporting; +- exact explicit rollback and automatic rollback when server freshness was + frozen, including stable binary, unit, credential-state, ownership, and mode + identity; +- ordinary update without implicit privilege migration, followed by a final + committed safe-profile migration. + +The receipt is intentionally secret-free and contains source/artifact hashes, +host facts, resulting privilege posture, report timestamps, and the eight +scenario outcomes. This is managed-development proof because it exercised the +working tree before its eventual commit identity existed. + +## Proof classification and residuals + +The focused protocol, update, runner, API, and installer regressions plus the +managed systemd receipt are proof for these semantics. They are not a +substitute for exact committed release-candidate evidence, representative +provider/appliance qualification, or external review. + +The safe profile therefore remains opt-in. The default may not ratchet until +the exact release candidate reproduces the systemd result and proves runner +rotation/revocation plus typed action receipts; representative Proxmox, SMART, +Docker and rootless Podman parity is recorded; supported appliance residuals +are owned; and the helper, update, action credential, and migration boundaries +complete external security review. diff --git a/docs/release-control/v6/internal/records/secure-agent-runtime-systemd-receipt-2026-08-30.json b/docs/release-control/v6/internal/records/secure-agent-runtime-systemd-receipt-2026-08-30.json new file mode 100644 index 000000000..794426a48 --- /dev/null +++ b/docs/release-control/v6/internal/records/secure-agent-runtime-systemd-receipt-2026-08-30.json @@ -0,0 +1,69 @@ +{ + "schema_version": 1, + "completed_at": "2026-08-30T00:39:22.928744899Z", + "source_hashes": { + "scripts/install.sh": "0b557c9394f3f0741cbf4287e5ee16a88fe03e189678bddf754606458c4e276e", + "scripts/installtests/secure_runtime_systemd_lab_test.go": "cebaeca194bd2ef1a3754230026ab73cd7c8b28427c773dbc62c86f4cd69ba28" + }, + "artifact_hashes": { + "collector_v1": "c9c775085178102d7fea5ecc899960a3e27ad7dc41c632a83f01e0f642942379", + "collector_v2": "2b52cbf20884d391acd7c072b7c4af011377ef1f19b97725a76ad18a3d1154fa", + "helper": "9d865a4adb9fdd5e70f36ee4021acefd2b5bfe1010739ab1dd916c701818032d" + }, + "os_release": "PRETTY_NAME=\"Ubuntu 24.04.4 LTS\"\nNAME=\"Ubuntu\"\nVERSION_ID=\"24.04\"\nVERSION=\"24.04.4 LTS (Noble Numbat)\"\nVERSION_CODENAME=noble\nID=ubuntu\nID_LIKE=debian\nHOME_URL=\"https://www.ubuntu.com/\"\nSUPPORT_URL=\"https://help.ubuntu.com/\"\nBUG_REPORT_URL=\"https://bugs.launchpad.net/ubuntu/\"\nPRIVACY_POLICY_URL=\"https://www.ubuntu.com/legal/terms-and-policies/privacy-policy\"\nUBUNTU_CODENAME=noble\nLOGO=ubuntu-logo", + "kernel": "Linux 6.8.0-117-generic #117-Ubuntu SMP PREEMPT_DYNAMIC Thu May 7 17:26:37 UTC 2026 aarch64 GNU/Linux", + "systemd_version": "systemd 255 (255.4-1ubuntu8.15)", + "architecture": "arm64", + "collector_service_user": "pulse-agent", + "collector_process_uid": 996, + "collector_authority": "monitoring-only", + "ambient_capabilities_none": true, + "helper_protocol_healthy": true, + "state_identity_preserved": true, + "docker_degraded": true, + "report_count": 26, + "first_report_at": "2026-08-30T00:38:25.603885842Z", + "last_report_at": "2026-08-30T00:39:22.777779187Z", + "scenarios": [ + { + "name": "legacy_root_command_capable_install", + "passed": true, + "detail": "root collector installed; docker_enabled=true" + }, + { + "name": "read_only_inspect", + "passed": true, + "detail": "stable installer-owned files unchanged" + }, + { + "name": "drop_in_fail_closed_rehearsal", + "passed": true, + "detail": "migration rejected before stable installer-owned files changed" + }, + { + "name": "safe_profile_apply", + "passed": true, + "detail": "fresh server lastSeen, least-privilege identity, typed helper health" + }, + { + "name": "explicit_safe_profile_rollback", + "passed": true, + "detail": "legacy binary, unit, authority, and service identity restored" + }, + { + "name": "automatic_failure_rollback", + "passed": true, + "detail": "frozen lastSeen prevented commit and restored binary/unit/state identity" + }, + { + "name": "ordinary_update_non_migration", + "passed": true, + "detail": "binary updated while root command-capable profile remained unchanged" + }, + { + "name": "final_safe_profile_apply", + "passed": true, + "detail": "collector continued reporting after committed migration" + } + ] +} diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 2e75cfbd0..2594ca482 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -43,6 +43,24 @@ host-storage-cleanup, Proxmox guest lifecycle, and container lifecycle/update operations. Generic shell, exec, unrestricted `read_file`, and deploy requests are forbidden. Removing or disabling the runner leaves collector monitoring and the helper unchanged. +When that helper socket is configured, collector startup must complete a +versioned `helper.health` request before the runtime may become ready. A +listening or active systemd socket alone is not helper health. Once the helper +boundary is selected, SMART and Proxmox LXC filesystem collection fail closed +to an omitted/degraded snapshot when the helper fails; the collector must not +silently retry the privileged read locally. + +Helper-backed collector updates are a durable pending transaction rather than +a successful binary swap. The helper records activation intent before the +root-owned replacement, retains the last-known-good digest, and places the new +binary under a bounded rollback deadline. The replacement process must load +the private pending handoff, pass local readiness, and deliver a newly +collected authoritative primary report before it may issue the identity-bound +typed commit. Buffered or observer reports do not satisfy that floor. An +expired, interrupted, invalid, or uncommitted activation rolls back during +helper recovery or through the deadline watchdog; commit and rollback remove +their fixed staging/quarantine artifacts durably. The first accepted report +retains the previous-version update evidence from that handoff. The runner's server transport currently reuses the combined agent command WebSocket envelope as a migration boundary. That compatibility path may carry @@ -53,6 +71,19 @@ have an action-runner migration path, live action-runner session parity is qualified, and no supported client depends on collector command delivery. Until those criteria are met, the combined collector command path is explicitly legacy/full-trust compatibility, not part of the safe profile. +The runner service persists the same normalized canonical hostname used when +its credential was issued; it must not substitute the machine's incidental OS +hostname when the collector was enrolled under an override. Credential +rotation invalidates exactly the superseded organization/token/agent/hostname +session only after the replacement token inventory is durably stored. That +invalidation removes the session from dispatch, closes its transport, and +unblocks server-side waits; an already-started host mutation remains governed +by its typed receipt and best-effort cancellation semantics rather than being +described as rolled back. Runner uninstall attempts an authenticated +credential self-revoke before deleting local state. The delete route may +remove only the caller's exact host-bound action-runner record; an unreachable +server cannot prevent local runner removal and leaves an explicit operator +revocation residual. Fresh installs carry an explicit local command-authority profile. The closed values are `monitoring-only`, `command-capable`, and `legacy`. A `monitoring-only` service may accept remote configuration that keeps commands diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index b9cdaa4a9..2aca7360f 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -41,10 +41,22 @@ runtime role `action-runner`, binding version 1, and capability non-conflicted, non-integration host in the request tenant. Issuing again atomically replaces only the earlier action-runner record for that tenant and agent ID; a persistence failure restores the complete token inventory and -returns no usable new secret. Monitoring credentials cannot call the issuance -route or authenticate an action-runner session, and action-runner credentials -cannot report, read agent configuration, or manage collectors. Unknown request -fields and ambiguous, mismatched, or conflicted host identities fail closed. +returns no usable new secret. Only after that durable replacement may the +server close the exact superseded live action-runner session, remove it from +new dispatch, and unblock its outstanding server waits. The created response +is non-cacheable. + +`DELETE /api/agents/action-runner/credential` is the runner's narrowly scoped +self-revoke operation. It requires the current `agent:exec` bearer credential, +rejects browser/session authentication, accepts only `agentId` and `hostname`, +and deletes only when the caller token, organization, action-runner metadata, +and canonical host binding all match. The durable token inventory is restored +on persistence failure; after a successful delete, the exact admitted session +is invalidated and the route returns no credential material. Monitoring +credentials cannot call issuance or self-revoke or authenticate an +action-runner session, and action-runner credentials cannot report, read agent +configuration, or manage collectors. Unknown request fields and ambiguous, +mismatched, or conflicted host identities fail closed. The API runtime is decomposed along production domain boundaries so Go can compile and execute domain qualification packages concurrently. Shared tenant diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 0a30cd407..da526c93c 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -130,6 +130,14 @@ solely to atomically replace the protocol-fixed `pulse-agent` target and its last-known-good copy. The collector can select no privileged source, target, path, command, or argument, and direct collector-owned replacement is not a fallback. +Activation persists a pending identity and bounded rollback deadline before +the new binary is trusted. The replacement collector must pass local readiness +and have a fresh authoritative primary report accepted before it sends the +typed commit; helper startup/restart recovery and the watchdog restore the +last-known-good binary when activation is interrupted, expires, or never +commits. Commit and rollback durably reap the fixed staging and quarantine +artifacts. A socket-active check is insufficient: installer migration and the +collector both exercise the versioned helper health protocol. The same supported Linux systemd profile can install `pulse-agent-runner` only through the separate `--enable-action-runner` choice, a private token file, @@ -138,7 +146,12 @@ configuration, credential, health record, and receipt database are root-owned and independent from collector state. Activation is transactional and restores the previous runner-only files if health does not become current; disable and uninstall remove only remediation and leave monitoring running. The action -credential is never placed in argv or reused as the collector token. +credential is never placed in argv or reused as the collector token. The +installer persists the canonical enrollment hostname for runner admission and +uses a private curl configuration for best-effort exact self-revocation before +local teardown, so the bearer secret does not enter argv. Server unreachability +is warned explicitly but does not make local removal depend on the remote +control plane. Safe-profile migration is never an ordinary update side effect. `--safe-profile-inspect` is read-only and reports the current authority, @@ -150,6 +163,16 @@ privilege or changing the independently enrolled runner. These operations fail closed outside Linux systemd. Appliance, non-systemd, Windows, and macOS profiles retain an explicitly named legacy/full-trust path until their service, filesystem, update, helper, and runner boundaries have separate proof. +Apply also fails before mutation when the effective collector fragment differs +from the installer-owned unit or any systemd drop-in is present. Commit requires +the effective non-root/no-ambient-capability unit, a live helper protocol +response, and a registration `lastSeen` newer than the frozen legacy +collector's value. The rollback manifest restores all state-tree ownership and +modes plus the Proxmox registration markers the apply path can mutate. Rootful +Docker is an explicit migration degradation: it is disabled unless a +collector-owned, readable and writable rootless runtime socket is available; +the safe profile never restores Docker by adding the collector to a +root-equivalent group. Release builds and archives carry both helper and runner binaries for the five Linux targets (`amd64`, `arm64`, `armv7`, `armv6`, and `386`) with checksum, diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index 38e1062f8..4d602940b 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -53,6 +53,15 @@ shape, regular-file identity, symlink resistance, and byte ceiling before copying into fixed root-owned staging. Activation and rollback then revalidate the root boundary, perform atomic replacement, and durably bind the transition identity before changing the root-owned collector binary. +Activation is not final authority: the helper records a bounded pending state +and last-known-good digest before replacement, and exposes an exact typed +commit. Only the replacement collector's local readiness plus an accepted new +authoritative report may request commit. Restart/power-loss recovery and the +deadline watchdog roll back an uncommitted activation, while strict private +handoff parsing and fixed artifact cleanup prevent a collector-selected target +or stale quarantine from widening that boundary. Configuring the helper also +requires a successful versioned health exchange at collector startup and +forbids privileged telemetry fallback into the collector process. Remediation credentials belong only to the separately installed `pulse-agent-runner`. They bind organization, canonical host identity, token @@ -72,6 +81,14 @@ secret only after persistence succeeds, and restores the complete prior token inventory if persistence fails. A successful rotation invalidates the previous secret immediately; it never widens the collector credential or turns a monitoring session into an action session. +That invalidation is exact and post-persistence: it matches organization, +token, canonical agent, hostname, runtime role, and typed capability before +closing the session, so a stale rotation cannot evict a replacement. The +runner may also delete only its own matching record using its bearer +credential; browser sessions and a caller-selected token ID are rejected, and +persistence failure restores the prior inventory. Installer teardown keeps the +secret in a private file/config boundary rather than argv and reports remote +revocation failure without retaining local remediation authority. Own Pulse's canonical privacy disclosures, outbound usage-data boundary, and the security-facing settings surfaces that expose authentication posture, diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index b714166a0..952a05166 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -37,12 +37,15 @@ not storage-recovery authority inferred from the credential itself. Runner credential rotation follows the shared token-inventory commit boundary: failed persistence restores the complete prior inventory and returns no new secret, while successful persistence replaces the previous organization/agent -binding and makes its secret unusable. That rollback protects restart-time +binding and invalidates only that superseded live session. Exact bearer +self-revocation uses the same durable boundary and cannot select another token, +organization, or host. That rollback protects restart-time credential truth only; it is not a customer backup, recovery point, restore operation, or storage-cleanup grant. The helper-backed collector updater likewise retains only one fixed -last-known-good executable and an identity-bound activation receipt. That -binary rollback is agent lifecycle state, not a Pulse storage snapshot, +last-known-good executable and an identity-bound pending/commit receipt. A +startup recovery or deadline rollback of an uncommitted activation is agent +lifecycle state, not a Pulse storage snapshot, recovery point, backup retention record, or authorization to read or mutate customer storage/recovery data. diff --git a/internal/agentexec/server.go b/internal/agentexec/server.go index 3f627c734..ddd14af33 100644 --- a/internal/agentexec/server.go +++ b/internal/agentexec/server.go @@ -376,6 +376,53 @@ func (s *Server) connectionForOrganization(organizationID, agentID string) (*age return nil, false } +// InvalidateActionRunnerSession closes exactly the currently admitted typed +// action-runner session identified by admission. A stale rotation result must +// never evict a replacement session that has already registered for the same +// tenant and host identity. +func (s *Server) InvalidateActionRunnerSession(admission AgentAdmission) bool { + if s == nil { + return false + } + expected := AgentAdmission{ + OrganizationID: normalizeOrganizationID(admission.OrganizationID), + TokenID: strings.TrimSpace(admission.TokenID), + AgentID: strings.TrimSpace(admission.AgentID), + Hostname: strings.TrimSpace(admission.Hostname), + RuntimeRole: strings.TrimSpace(admission.RuntimeRole), + ActionCapability: strings.TrimSpace(admission.ActionCapability), + } + if expected.TokenID == "" || expected.AgentID == "" || expected.Hostname == "" || + expected.RuntimeRole != RuntimeRoleActionRunner || expected.ActionCapability != ActionCapabilityTypedV1 { + return false + } + key := agentSessionKey(expected.OrganizationID, expected.AgentID) + s.mu.Lock() + current, ok := s.agents[key] + if !ok || current == nil || !sameActionRunnerAdmission(current.admission, expected) { + s.mu.Unlock() + return false + } + delete(s.agents, key) + s.mu.Unlock() + + current.signalDone() + if current.conn != nil { + _ = current.conn.Close() + } + return true +} + +func sameActionRunnerAdmission(current, expected AgentAdmission) bool { + return normalizeOrganizationID(current.OrganizationID) == expected.OrganizationID && + strings.TrimSpace(current.TokenID) == expected.TokenID && + strings.TrimSpace(current.AgentID) == expected.AgentID && + (strings.EqualFold(strings.TrimSpace(current.Hostname), expected.Hostname) || + unifiedresources.HostnamesEquivalent(current.Hostname, expected.Hostname)) && + strings.TrimSpace(current.RuntimeRole) == RuntimeRoleActionRunner && + strings.TrimSpace(current.ActionCapability) == ActionCapabilityTypedV1 +} + func requireLegacyFullTrustConnection(ac *agentConn, operation string) error { if ac == nil { return fmt.Errorf("agent connection is unavailable") diff --git a/internal/agentexec/server_websocket_test.go b/internal/agentexec/server_websocket_test.go index 334a029e9..361d52ed6 100644 --- a/internal/agentexec/server_websocket_test.go +++ b/internal/agentexec/server_websocket_test.go @@ -1239,6 +1239,73 @@ func TestRevokedAdmissionInvalidatesStaleSocketBeforeDispatch(t *testing.T) { } } +func TestInvalidateActionRunnerSessionClosesExactSessionAndUnblocksInflightDispatch(t *testing.T) { + admission := AgentAdmission{ + OrganizationID: "org-a", + TokenID: "runner-token", + AgentID: "agent-a", + Hostname: "node.example", + RuntimeRole: RuntimeRoleActionRunner, + ActionCapability: ActionCapabilityTypedV1, + } + s := NewServerWithAdmissionValidator(func(token, _, _ string) (AgentAdmission, bool) { + return admission, token == admission.TokenID + }, func(candidate AgentAdmission) bool { return candidate == admission }) + ts := newWSServer(t, s) + defer ts.Close() + + conn, _, err := dialAgentExecWebSocket(t, ts.URL) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + wsWriteMessage(t, conn, mustNewMessage(t, MsgTypeAgentRegister, "", AgentRegisterPayload{ + AgentID: admission.AgentID, Hostname: admission.Hostname, Token: admission.TokenID, + RuntimeRole: admission.RuntimeRole, ActionCapability: admission.ActionCapability, + OperationReceiptVersion: operationreceipt.ProtocolVersion, + })) + if ack := wsReadRegisteredPayload(t, conn); !ack.Success { + t.Fatalf("registration failed: %s", ack.Message) + } + + ctx := WithOrganizationID(context.Background(), admission.OrganizationID) + result := make(chan error, 1) + go func() { + _, dispatchErr := s.ExecuteHostUpdate(ctx, admission.AgentID, HostUpdatePayload{ + RequestID: "rotation-inflight", ActionID: "action-a", Operation: HostUpdateOperationInstall, + ExpectedInventoryHash: "sha256:" + strings.Repeat("a", 64), Timeout: 30, + }) + result <- dispatchErr + }() + message, err := wsReadRawMessageWithTimeout(conn, 2*time.Second) + if err != nil || message.Type != MsgTypeHostUpdate { + t.Fatalf("in-flight dispatch = %#v, %v", message, err) + } + + wrong := admission + wrong.TokenID = "replacement-token" + if s.InvalidateActionRunnerSession(wrong) { + t.Fatal("mismatched replacement identity evicted the current session") + } + if !s.InvalidateActionRunnerSession(admission) { + t.Fatal("exact action-runner admission was not invalidated") + } + select { + case dispatchErr := <-result: + if dispatchErr == nil || !strings.Contains(dispatchErr.Error(), "disconnected") { + t.Fatalf("in-flight dispatch error = %v", dispatchErr) + } + case <-time.After(2 * time.Second): + t.Fatal("in-flight dispatch did not unblock after session invalidation") + } + if _, err := s.ExecuteHostUpdate(ctx, admission.AgentID, HostUpdatePayload{ + RequestID: "after-rotation", ActionID: "action-b", Operation: HostUpdateOperationInstall, + ExpectedInventoryHash: "sha256:" + strings.Repeat("b", 64), Timeout: 1, + }); err == nil || !strings.Contains(err.Error(), "not connected") { + t.Fatalf("new dispatch after invalidation = %v", err) + } +} + // registerCancelTestAgent registers agent "a1" over the websocket harness and // returns after the registration ack has been read. func registerCancelTestAgent(t *testing.T, s *Server, tsURL string) *cancelTestConn { diff --git a/internal/agenthelper/protocol.go b/internal/agenthelper/protocol.go index 1945d6a3f..1a5f2dc36 100644 --- a/internal/agenthelper/protocol.go +++ b/internal/agenthelper/protocol.go @@ -40,6 +40,7 @@ const ( OperationContainerInventory = "container.inventory" OperationAgentUpdateStage = "agent_update.stage" OperationAgentUpdateActivate = "agent_update.activate" + OperationAgentUpdateCommit = "agent_update.commit" OperationAgentUpdateRollback = "agent_update.rollback" OperationVersion1 = 1 ) diff --git a/internal/agenthelper/providers.go b/internal/agenthelper/providers.go index 972194ca5..eb523ccaf 100644 --- a/internal/agenthelper/providers.go +++ b/internal/agenthelper/providers.go @@ -33,6 +33,7 @@ type ContainerProvider interface { type UpdateProvider interface { Stage(context.Context, UpdateStageRequest) (UpdateStageResult, error) Activate(context.Context, UpdateActivateRequest) (UpdateResult, error) + Commit(context.Context, UpdateCommitRequest) (UpdateResult, error) Rollback(context.Context, UpdateRollbackRequest) (UpdateResult, error) } diff --git a/internal/agenthelper/registry.go b/internal/agenthelper/registry.go index a02bd7f55..244a81ec6 100644 --- a/internal/agenthelper/registry.go +++ b/internal/agenthelper/registry.go @@ -131,6 +131,20 @@ func NewRegistryWithProviders(smart SMARTProvider, proxmox ProxmoxProvider, prov } return marshalOperationResult(result) }) + registry.add(OperationAgentUpdateCommit, OperationVersion1, providers.Updates != nil, func(ctx context.Context, payload json.RawMessage) (json.RawMessage, *ResponseError) { + var request UpdateCommitRequest + if err := decodePayload(payload, &request); err != nil { + return nil, invalidPayloadError(err) + } + if providers.Updates == nil { + return nil, unavailableError("agent update provider is not configured") + } + result, err := providers.Updates.Commit(ctx, request) + if err != nil { + return nil, providerError(err) + } + return marshalOperationResult(result) + }) registry.add(OperationAgentUpdateRollback, OperationVersion1, providers.Updates != nil, func(ctx context.Context, payload json.RawMessage) (json.RawMessage, *ResponseError) { var request UpdateRollbackRequest if err := decodePayload(payload, &request); err != nil { diff --git a/internal/agenthelper/server_test.go b/internal/agenthelper/server_test.go index 9d75afd2c..c964790d5 100644 --- a/internal/agenthelper/server_test.go +++ b/internal/agenthelper/server_test.go @@ -34,6 +34,7 @@ func (f fakeContainerProvider) Inventory(ctx context.Context) (json.RawMessage, type fakeUpdateProvider struct { stage func(context.Context, UpdateStageRequest) (UpdateStageResult, error) activate func(context.Context, UpdateActivateRequest) (UpdateResult, error) + commit func(context.Context, UpdateCommitRequest) (UpdateResult, error) rollback func(context.Context, UpdateRollbackRequest) (UpdateResult, error) } @@ -45,6 +46,10 @@ func (f fakeUpdateProvider) Activate(ctx context.Context, request UpdateActivate return f.activate(ctx, request) } +func (f fakeUpdateProvider) Commit(ctx context.Context, request UpdateCommitRequest) (UpdateResult, error) { + return f.commit(ctx, request) +} + func (f fakeUpdateProvider) Rollback(ctx context.Context, request UpdateRollbackRequest) (UpdateResult, error) { return f.rollback(ctx, request) } @@ -187,6 +192,7 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) { stageCalled := false activateCalled := false rollbackCalled := false + commitCalled := false updates := fakeUpdateProvider{ stage: func(_ context.Context, request UpdateStageRequest) (UpdateStageResult, error) { stageCalled = request.ArtifactID == "release-1" @@ -194,7 +200,11 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) { }, activate: func(_ context.Context, request UpdateActivateRequest) (UpdateResult, error) { activateCalled = request.ArtifactID == "release-1" - return UpdateResult{Action: "activated", ActivationID: "release-1:0123456789abcdef"}, nil + return UpdateResult{Action: "pending", ActivationID: "release-1:0123456789abcdef"}, nil + }, + commit: func(_ context.Context, request UpdateCommitRequest) (UpdateResult, error) { + commitCalled = request.ActivationID == "release-1:0123456789abcdef" + return UpdateResult{Action: "committed", ActivationID: request.ActivationID}, nil }, rollback: func(_ context.Context, request UpdateRollbackRequest) (UpdateResult, error) { rollbackCalled = request.ActivationID == "release-1:0123456789abcdef" @@ -222,13 +232,18 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) { if response := exchange(t, server, activate); !response.Success || !activateCalled { t.Fatalf("activate response=%#v called=%t", response, activateCalled) } + commit := validRequest(OperationAgentUpdateCommit) + commit.Payload = json.RawMessage(`{"activationId":"release-1:0123456789abcdef","currentSha256":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}`) + if response := exchange(t, server, commit); !response.Success || !commitCalled { + t.Fatalf("commit response=%#v called=%t", response, commitCalled) + } rollback := validRequest(OperationAgentUpdateRollback) rollback.Payload = json.RawMessage(`{"activationId":"release-1:0123456789abcdef","currentSha256":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","rollbackSha256":"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}`) if response := exchange(t, server, rollback); !response.Success || !rollbackCalled { t.Fatalf("rollback response=%#v called=%t", response, rollbackCalled) } - for _, operation := range []string{OperationContainerInventory, OperationAgentUpdateStage, OperationAgentUpdateActivate, OperationAgentUpdateRollback} { + for _, operation := range []string{OperationContainerInventory, OperationAgentUpdateStage, OperationAgentUpdateActivate, OperationAgentUpdateCommit, OperationAgentUpdateRollback} { request := validRequest(operation) request.Payload = json.RawMessage(`{"path":"/tmp/attacker","args":["sh"]}`) requireErrorCode(t, exchange(t, server, request), ErrorInvalidRequest) diff --git a/internal/agenthelper/update_activation.go b/internal/agenthelper/update_activation.go index 6c5f0f303..2ebf2a1d8 100644 --- a/internal/agenthelper/update_activation.go +++ b/internal/agenthelper/update_activation.go @@ -16,6 +16,12 @@ import ( const maxUpdateArtifactBytes = 100 * 1024 * 1024 +const ( + defaultUpdateRollbackWindow = 2 * time.Minute + minUpdateRollbackWindow = 5 * time.Second + maxUpdateRollbackWindow = 10 * time.Minute +) + type UpdateActivateRequest struct { ArtifactID string `json:"artifactId"` SHA256 string `json:"sha256"` @@ -39,12 +45,18 @@ type UpdateRollbackRequest struct { RollbackSHA256 string `json:"rollbackSha256"` } +type UpdateCommitRequest struct { + ActivationID string `json:"activationId"` + CurrentSHA256 string `json:"currentSha256"` +} + type UpdateResult struct { - Action string `json:"action"` - ActivationID string `json:"activationId"` - ActiveSHA256 string `json:"activeSha256"` - RollbackSHA256 string `json:"rollbackSha256"` - DurableAt time.Time `json:"durableAt"` + Action string `json:"action"` + ActivationID string `json:"activationId"` + ActiveSHA256 string `json:"activeSha256"` + RollbackSHA256 string `json:"rollbackSha256"` + RollbackDeadline time.Time `json:"rollbackDeadline,omitempty"` + DurableAt time.Time `json:"durableAt"` } type UpdateActivatorConfig struct { @@ -56,6 +68,8 @@ type UpdateActivatorConfig struct { ValidateOwner func(*os.File) error ValidateQuarantineOwner func(*os.File) error Now func() time.Time + RollbackWindow time.Duration + ScheduleRollback func(time.Duration, func()) func() } type updateActivator struct { @@ -69,14 +83,18 @@ type updateActivator struct { validateOwner func(*os.File) error validateQuarantineOwner func(*os.File) error now func() time.Time + rollbackWindow time.Duration + scheduleRollback func(time.Duration, func()) func() + cancelRollback func() } type durableUpdateState struct { - Action string `json:"action"` - ActivationID string `json:"activationId"` - ActiveSHA256 string `json:"activeSha256"` - RollbackSHA256 string `json:"rollbackSha256"` - UpdatedAt time.Time `json:"updatedAt"` + Action string `json:"action"` + ActivationID string `json:"activationId"` + ActiveSHA256 string `json:"activeSha256"` + RollbackSHA256 string `json:"rollbackSha256"` + RollbackDeadline time.Time `json:"rollbackDeadline,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` } func NewUpdateActivator(config UpdateActivatorConfig) (UpdateProvider, error) { @@ -98,12 +116,31 @@ func NewUpdateActivator(config UpdateActivatorConfig) (UpdateProvider, error) { if now == nil { now = time.Now } - return &updateActivator{ + rollbackWindow := config.RollbackWindow + if rollbackWindow == 0 { + rollbackWindow = defaultUpdateRollbackWindow + } + if rollbackWindow < minUpdateRollbackWindow || rollbackWindow > maxUpdateRollbackWindow { + return nil, errors.New("update rollback window must be between 5 seconds and 10 minutes") + } + scheduleRollback := config.ScheduleRollback + if scheduleRollback == nil { + scheduleRollback = func(delay time.Duration, callback func()) func() { + timer := time.AfterFunc(delay, callback) + return func() { timer.Stop() } + } + } + activator := &updateActivator{ quarantineDir: config.QuarantineDir, stagingDir: config.StagingDir, targetPath: config.TargetPath, rollbackPath: config.TargetPath + ".last-known-good", statePath: config.StatePath, verifySignature: config.VerifySignature, validateOwner: config.ValidateOwner, validateQuarantineOwner: config.ValidateQuarantineOwner, now: now, - }, nil + rollbackWindow: rollbackWindow, scheduleRollback: scheduleRollback, + } + if err := activator.recoverUncommittedLocked(); err != nil { + return nil, err + } + return activator, nil } // Stage promotes exactly one signed collector-downloaded artifact from the @@ -186,17 +223,30 @@ func (u *updateActivator) Activate(ctx context.Context, request UpdateActivateRe return UpdateResult{}, invalidArtifact("staged artifact signature is invalid") } activationID := request.ArtifactID + ":" + actual[:16] - if state, err := u.readState(); err == nil && state.ActivationID == activationID { - if state.Action != "activated" || !strings.EqualFold(state.ActiveSHA256, actual) { - return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed a different update transition"} + if state, err := u.readState(); err == nil { + if state.Action == "preparing" || state.Action == "pending" { + if state.ActivationID != activationID { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "another agent update is still awaiting commit or rollback"} + } + if state.Action == "preparing" { + if err := u.recoverStateLocked(state); err != nil { + return UpdateResult{}, err + } + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "interrupted activation was rolled back and cannot be replayed"} + } + if !strings.EqualFold(state.ActiveSHA256, actual) { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed a different update transition"} + } + if err := u.validateInstalledState(state); err != nil { + return UpdateResult{}, err + } + u.schedulePendingRollbackLocked(state) + return updateResultFromState(state), nil } - current, currentErr := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes) - rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes) - if currentErr != nil || rollbackErr != nil || !strings.EqualFold(sha256Hex(current), state.ActiveSHA256) || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) { - return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "durable activation no longer matches installed binaries"} + if state.ActivationID == activationID { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed its update transition"} } - return UpdateResult{Action: state.Action, ActivationID: state.ActivationID, ActiveSHA256: state.ActiveSHA256, RollbackSHA256: state.RollbackSHA256, DurableAt: state.UpdatedAt}, nil - } else if err != nil && !errors.Is(err, os.ErrNotExist) { + } else if !errors.Is(err, os.ErrNotExist) { return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "durable update state is invalid"} } if err := ctx.Err(); err != nil { @@ -207,20 +257,76 @@ func (u *updateActivator) Activate(ctx context.Context, request UpdateActivateRe return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "installed agent binary is unavailable or unsafe"} } rollbackDigest := sha256Hex(current) + deadline := u.now().UTC().Add(u.rollbackWindow) + preparing := durableUpdateState{ + Action: "preparing", ActivationID: activationID, ActiveSHA256: actual, + RollbackSHA256: rollbackDigest, RollbackDeadline: deadline, UpdatedAt: u.now().UTC(), + } + if err := u.writeDurableState(preparing); err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist activation intent"} + } if err := u.installDurably(artifact, current); err != nil { + _ = u.recoverStateLocked(preparing) return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "activate staged agent binary"} } - result := UpdateResult{ - Action: "activated", ActivationID: activationID, - ActiveSHA256: actual, RollbackSHA256: rollbackDigest, DurableAt: u.now().UTC(), + pending := durableUpdateState{ + Action: "pending", ActivationID: activationID, + ActiveSHA256: actual, RollbackSHA256: rollbackDigest, + RollbackDeadline: deadline, UpdatedAt: u.now().UTC(), } - if err := u.writeState(result); err != nil { - // Restore the pre-activation binary if the durable receipt cannot - // commit. A failed operation must not leave an unjournaled activation. - _ = u.installDurably(current, artifact) - return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist activation result"} + if err := u.writeDurableState(pending); err != nil { + if recoveryErr := u.recoverStateLocked(preparing); recoveryErr != nil { + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist pending activation and restore last-known-good binary"} + } + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist pending activation"} } - return result, nil + u.schedulePendingRollbackLocked(pending) + return updateResultFromState(pending), nil +} + +func (u *updateActivator) Commit(ctx context.Context, request UpdateCommitRequest) (UpdateResult, error) { + u.mu.Lock() + defer u.mu.Unlock() + if !validActivationID(request.ActivationID) || !validSHA256(request.CurrentSHA256) { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity is invalid"} + } + state, err := u.readState() + if err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity does not match a durable activation"} + } + if state.Action == "committed" && state.ActivationID == request.ActivationID && strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) { + return updateResultFromState(state), nil + } + if state.Action != "pending" || state.ActivationID != request.ActivationID || !strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity does not match the pending activation"} + } + if !u.now().UTC().Before(state.RollbackDeadline) { + if err := u.recoverStateLocked(state); err != nil { + return UpdateResult{}, err + } + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "activation deadline expired and the update was rolled back"} + } + if err := ctx.Err(); err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorDeadlineExceeded, Message: "update commit deadline exceeded", Retryable: true} + } + if err := u.validateInstalledState(state); err != nil { + return UpdateResult{}, err + } + if err := u.removeStagedArtifact(artifactIDFromActivation(state.ActivationID)); err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "remove committed update staging"} + } + pending := state + state.Action = "committed" + state.RollbackDeadline = time.Time{} + state.UpdatedAt = u.now().UTC() + if err := u.writeDurableState(state); err != nil { + if recoveryErr := u.recoverStateLocked(pending); recoveryErr != nil { + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist update commit and restore last-known-good binary"} + } + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist update commit"} + } + u.cancelPendingRollbackLocked() + return updateResultFromState(state), nil } func (u *updateActivator) Rollback(ctx context.Context, request UpdateRollbackRequest) (UpdateResult, error) { @@ -230,34 +336,162 @@ func (u *updateActivator) Rollback(ctx context.Context, request UpdateRollbackRe return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity is invalid"} } state, err := u.readState() - if err != nil || state.Action != "activated" || state.ActivationID != request.ActivationID || + if err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match a durable activation"} + } + if state.Action == "rolled_back" && state.ActivationID == request.ActivationID && + strings.EqualFold(state.RollbackSHA256, request.CurrentSHA256) && strings.EqualFold(state.ActiveSHA256, request.RollbackSHA256) { + return updateResultFromState(state), nil + } + if (state.Action != "pending" && state.Action != "preparing") || state.ActivationID != request.ActivationID || !strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) || !strings.EqualFold(state.RollbackSHA256, request.RollbackSHA256) { - return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match the durable activation"} - } - current, err := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes) - if err != nil || !strings.EqualFold(sha256Hex(current), state.ActiveSHA256) { - return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "installed agent identity changed after activation"} - } - rollback, err := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes) - if err != nil || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) { - return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "last-known-good identity changed after activation"} + return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match the pending activation"} } if err := ctx.Err(); err != nil { return UpdateResult{}, &ProviderError{Code: ErrorDeadlineExceeded, Message: "update rollback deadline exceeded", Retryable: true} } - if err := u.installDurably(rollback, current); err != nil { - return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "restore last-known-good agent binary"} + if err := u.recoverStateLocked(state); err != nil { + return UpdateResult{}, err } - result := UpdateResult{ + rolledBack, err := u.readState() + if err != nil { + return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "read durable rollback result"} + } + return updateResultFromState(rolledBack), nil +} + +func (u *updateActivator) recoverUncommittedLocked() error { + state, err := u.readState() + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return &ProviderError{Code: ErrorStateConflict, Message: "durable update state is invalid"} + } + if state.Action != "preparing" && state.Action != "pending" { + return nil + } + return u.recoverStateLocked(state) +} + +func (u *updateActivator) recoverStateLocked(state durableUpdateState) error { + if state.Action != "preparing" && state.Action != "pending" { + return &ProviderError{Code: ErrorStateConflict, Message: "update state is not recoverable"} + } + current, err := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes) + if err != nil { + return &ProviderError{Code: ErrorStateConflict, Message: "installed agent binary is unavailable during recovery"} + } + currentDigest := sha256Hex(current) + if strings.EqualFold(currentDigest, state.ActiveSHA256) { + rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes) + if rollbackErr != nil || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) { + return &ProviderError{Code: ErrorStateConflict, Message: "last-known-good binary is unavailable during recovery"} + } + if err := u.installDurably(rollback, current); err != nil { + return &ProviderError{Code: ErrorInternal, Message: "restore last-known-good agent binary"} + } + } else if !strings.EqualFold(currentDigest, state.RollbackSHA256) { + return &ProviderError{Code: ErrorStateConflict, Message: "installed agent identity changed during pending activation"} + } + + rolledBack := durableUpdateState{ Action: "rolled_back", ActivationID: state.ActivationID, - ActiveSHA256: state.RollbackSHA256, RollbackSHA256: state.ActiveSHA256, DurableAt: u.now().UTC(), + ActiveSHA256: state.RollbackSHA256, RollbackSHA256: state.ActiveSHA256, + UpdatedAt: u.now().UTC(), } - if err := u.writeState(result); err != nil { - _ = u.installDurably(current, rollback) - return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist rollback result"} + // Keep the durable state recoverable until the staged payload has also + // been removed. A crash anywhere before the terminal state write will + // replay this same bounded rollback and cleanup on helper startup. + if err := u.removeStagedArtifact(artifactIDFromActivation(state.ActivationID)); err != nil { + return &ProviderError{Code: ErrorInternal, Message: "remove rolled-back update staging"} } - return result, nil + if err := u.writeDurableState(rolledBack); err != nil { + return &ProviderError{Code: ErrorInternal, Message: "persist rollback result"} + } + u.cancelPendingRollbackLocked() + return nil +} + +func (u *updateActivator) validateInstalledState(state durableUpdateState) error { + current, currentErr := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes) + rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes) + if currentErr != nil || rollbackErr != nil || + !strings.EqualFold(sha256Hex(current), state.ActiveSHA256) || + !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) { + return &ProviderError{Code: ErrorStateConflict, Message: "durable activation no longer matches installed binaries"} + } + return nil +} + +func (u *updateActivator) schedulePendingRollbackLocked(state durableUpdateState) { + u.cancelPendingRollbackLocked() + delay := state.RollbackDeadline.Sub(u.now().UTC()) + if delay < 0 { + delay = 0 + } + u.cancelRollback = u.scheduleRollback(delay, func() { + u.mu.Lock() + defer u.mu.Unlock() + current, err := u.readState() + if err != nil || current.Action != "pending" || current.ActivationID != state.ActivationID { + return + } + _ = u.recoverStateLocked(current) + }) +} + +func (u *updateActivator) cancelPendingRollbackLocked() { + if u.cancelRollback != nil { + u.cancelRollback() + u.cancelRollback = nil + } +} + +func updateResultFromState(state durableUpdateState) UpdateResult { + return UpdateResult{ + Action: state.Action, ActivationID: state.ActivationID, + ActiveSHA256: state.ActiveSHA256, RollbackSHA256: state.RollbackSHA256, + RollbackDeadline: state.RollbackDeadline, DurableAt: state.UpdatedAt, + } +} + +func artifactIDFromActivation(activationID string) string { + parts := strings.SplitN(activationID, ":", 2) + if len(parts) != 2 { + return "" + } + return parts[0] +} + +func (u *updateActivator) removeStagedArtifact(artifactID string) error { + if !validArtifactID(artifactID) { + return errors.New("invalid staged artifact identity") + } + destination := filepath.Join(u.stagingDir, artifactID) + if _, err := os.Lstat(destination); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + if err := u.validateDirectory(destination); err != nil { + return err + } + for _, name := range []string{"pulse-agent", "pulse-agent.sig"} { + path := filepath.Join(destination, name) + info, err := os.Lstat(path) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return errors.New("unsafe staged artifact cleanup target") + } + if err := os.Remove(path); err != nil { + return err + } + } + if err := os.Remove(destination); err != nil { + return err + } + return syncDirectory(u.stagingDir) } func (u *updateActivator) validateDirectory(path string) error { @@ -386,8 +620,7 @@ func (u *updateActivator) installDurably(active, lastKnownGood []byte) error { return syncDirectory(targetDir) } -func (u *updateActivator) writeState(result UpdateResult) error { - state := durableUpdateState{Action: result.Action, ActivationID: result.ActivationID, ActiveSHA256: result.ActiveSHA256, RollbackSHA256: result.RollbackSHA256, UpdatedAt: result.DurableAt} +func (u *updateActivator) writeDurableState(state durableUpdateState) error { data, err := json.Marshal(state) if err != nil { return err diff --git a/internal/agenthelper/update_activation_test.go b/internal/agenthelper/update_activation_test.go index c711dd549..5889c4412 100644 --- a/internal/agenthelper/update_activation_test.go +++ b/internal/agenthelper/update_activation_test.go @@ -32,6 +32,7 @@ func testUpdateActivator(t *testing.T, verifier func([]byte, string) error) (Upd ValidateOwner: func(*os.File) error { return nil }, ValidateQuarantineOwner: func(*os.File) error { return nil }, Now: func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) }, + ScheduleRollback: func(time.Duration, func()) func() { return func() {} }, }) if err != nil { t.Fatal(err) @@ -86,6 +87,9 @@ func TestUpdateActivationAndIdentityBoundRollbackAreDurable(t *testing.T) { if activated.ActiveSHA256 != newDigest || activated.RollbackSHA256 != oldDigest || activated.ActivationID == "" { t.Fatalf("activation result = %#v", activated) } + if activated.Action != "pending" || activated.RollbackDeadline.IsZero() { + t.Fatalf("activation is not durably pending: %#v", activated) + } if installed, _ := os.ReadFile(target); string(installed) != string(testELF("new-signed-binary")) { t.Fatalf("installed binary = %q", installed) } @@ -115,6 +119,104 @@ func TestUpdateActivationAndIdentityBoundRollbackAreDurable(t *testing.T) { } } +func TestUpdateCommitClosesRollbackWindowAndCleansStaging(t *testing.T) { + provider, quarantine, target, _ := testUpdateActivator(t, func([]byte, string) error { return nil }) + digest := stageUpdate(t, quarantine, "release-commit", testELF("committed")) + promoteUpdate(t, provider, "release-commit", digest) + activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-commit", SHA256: digest}) + if err != nil { + t.Fatal(err) + } + committed, err := provider.Commit(context.Background(), UpdateCommitRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, + }) + if err != nil { + t.Fatal(err) + } + if committed.Action != "committed" || committed.RollbackDeadline != (time.Time{}) { + t.Fatalf("commit result = %#v", committed) + } + if installed, _ := os.ReadFile(target); string(installed) != string(testELF("committed")) { + t.Fatalf("committed binary = %q", installed) + } + staged := filepath.Join(filepath.Dir(quarantine), "staging", "release-commit") + if _, err := os.Stat(staged); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("committed staging still exists: %v", err) + } + if retried, err := provider.Commit(context.Background(), UpdateCommitRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, + }); err != nil || retried.Action != "committed" { + t.Fatalf("idempotent commit = %#v, %v", retried, err) + } + if _, err := provider.Rollback(context.Background(), UpdateRollbackRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256, + }); err == nil { + t.Fatal("committed update remained rollback-eligible") + } +} + +func TestUpdateActivatorRestartRollsBackUncommittedActivation(t *testing.T) { + provider, quarantine, target, state := testUpdateActivator(t, func([]byte, string) error { return nil }) + digest := stageUpdate(t, quarantine, "release-restart", testELF("uncommitted")) + promoteUpdate(t, provider, "release-restart", digest) + activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-restart", SHA256: digest}) + if err != nil { + t.Fatal(err) + } + root := filepath.Dir(quarantine) + recovered, err := NewUpdateActivator(UpdateActivatorConfig{ + QuarantineDir: quarantine, + StagingDir: filepath.Join(root, "staging"), + TargetPath: target, + StatePath: state, + VerifySignature: func([]byte, string) error { + return nil + }, + ValidateOwner: func(*os.File) error { return nil }, + ValidateQuarantineOwner: func(*os.File) error { return nil }, + Now: func() time.Time { return activation.RollbackDeadline.Add(-time.Second) }, + ScheduleRollback: func(time.Duration, func()) func() { return func() {} }, + }) + if err != nil { + t.Fatalf("restart recovery: %v", err) + } + if installed, _ := os.ReadFile(target); string(installed) != string(testELF("old-signed-binary")) { + t.Fatalf("restart recovery installed binary = %q", installed) + } + result, err := recovered.Rollback(context.Background(), UpdateRollbackRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256, + }) + if err != nil || result.Action != "rolled_back" { + t.Fatalf("durable restart result = %#v, %v", result, err) + } +} + +func TestUpdateRollbackWatchdogRestoresUncommittedActivation(t *testing.T) { + provider, quarantine, target, _ := testUpdateActivator(t, func([]byte, string) error { return nil }) + activator := provider.(*updateActivator) + var watchdog func() + activator.scheduleRollback = func(_ time.Duration, callback func()) func() { + watchdog = callback + return func() {} + } + digest := stageUpdate(t, quarantine, "release-watchdog", testELF("uncommitted")) + promoteUpdate(t, provider, "release-watchdog", digest) + activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-watchdog", SHA256: digest}) + if err != nil || watchdog == nil { + t.Fatalf("Activate = %#v, %v; watchdog=%v", activation, err, watchdog != nil) + } + watchdog() + if installed, _ := os.ReadFile(target); string(installed) != string(testELF("old-signed-binary")) { + t.Fatalf("watchdog recovery installed binary = %q", installed) + } + result, err := provider.Rollback(context.Background(), UpdateRollbackRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256, + }) + if err != nil || result.Action != "rolled_back" { + t.Fatalf("watchdog durable result = %#v, %v", result, err) + } +} + func TestUpdateActivationRejectsInvalidSignatureAndExecutable(t *testing.T) { provider, staging, _, _ := testUpdateActivator(t, func([]byte, string) error { return errors.New("untrusted") }) digest := stageUpdate(t, staging, "bad-signature", testELF("binary")) diff --git a/internal/agentupdate/coverage_test.go b/internal/agentupdate/coverage_test.go index 34f2a6c14..db817ad40 100644 --- a/internal/agentupdate/coverage_test.go +++ b/internal/agentupdate/coverage_test.go @@ -27,6 +27,38 @@ func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } +func TestPrivilegedUpdateRollbackFailurePreservesPendingHandoff(t *testing.T) { + originalGOOS := runtimeGOOS + originalRestart := restartProcessFn + t.Cleanup(func() { + runtimeGOOS = originalGOOS + restartProcessFn = originalRestart + }) + runtimeGOOS = goOSLinux + restartProcessFn = func(string) error { return errors.New("exec refused") } + + binary := append([]byte{0x7f, 'E', 'L', 'F'}, bytes.Repeat([]byte("x"), 128)...) + sum := sha256.Sum256(binary) + digest := hex.EncodeToString(sum[:]) + helper := &fakePrivilegedUpdate{root: t.TempDir(), rollbackErr: errors.New("helper unavailable")} + if err := os.Chmod(helper.root, 0o700); err != nil { + t.Fatal(err) + } + u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"}) + u.selfTestFn = func(context.Context, string) error { return nil } + + err := u.performPrivilegedUpdate(context.Background(), "/usr/local/bin/pulse-agent", bytes.NewReader(binary), int64(len(binary)), digest, "signed-update") + if err == nil || !strings.Contains(err.Error(), "typed helper rollback failed") { + t.Fatalf("performPrivilegedUpdate error = %v", err) + } + loaded, loadErr := LoadPendingPrivilegedUpdate(helper.root) + if loadErr != nil || loaded == nil || loaded.Activation.ActivationID != helper.activation.ActivationID || + loaded.Activation.ActiveSHA256 != helper.activation.ActiveSHA256 || + loaded.Activation.RollbackSHA256 != helper.activation.RollbackSHA256 { + t.Fatalf("failed-rollback handoff = %#v, %v", loaded, loadErr) + } +} + func TestUpdateRetryBackoffUsesOperationalDelays(t *testing.T) { if updateRetryBaseDelay < time.Second { t.Fatalf("updateRetryBaseDelay = %s, want at least 1s", updateRetryBaseDelay) diff --git a/internal/agentupdate/privileged_update.go b/internal/agentupdate/privileged_update.go index eb1ea7af2..09f7892e3 100644 --- a/internal/agentupdate/privileged_update.go +++ b/internal/agentupdate/privileged_update.go @@ -4,8 +4,10 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -15,17 +17,24 @@ import ( ) const privilegedUpdateQuarantineDir = "/var/lib/pulse-agent/update-quarantine" +const pendingPrivilegedUpdateFile = ".pulse-agent-update-pending.json" // PrivilegedUpdate is the updater's closed view of helper-backed activation. // Implementations cannot select a target path, staging path, command, or URL. type PrivilegedUpdate interface { - CreateQuarantinedArtifact() (artifactID string, file *os.File, cleanup func(), err error) + CreateQuarantinedArtifact() (artifactID string, file *os.File, cleanup func() error, err error) WriteQuarantinedSignature(artifactID, signature string) error Stage(context.Context, string, string) (agenthelper.UpdateStageResult, error) Activate(context.Context, string, string) (agenthelper.UpdateResult, error) + Commit(context.Context, agenthelper.UpdateResult) (agenthelper.UpdateResult, error) Rollback(context.Context, agenthelper.UpdateResult) (agenthelper.UpdateResult, error) } +type PendingPrivilegedUpdate struct { + Activation agenthelper.UpdateResult `json:"activation"` + PreviousVersion string `json:"previousVersion"` +} + type privilegeHelperUpdate struct { client *agenthelper.Client quarantineDir string @@ -39,28 +48,38 @@ func NewPrivilegeHelperUpdate(socketPath string) (PrivilegedUpdate, error) { return &privilegeHelperUpdate{client: client, quarantineDir: privilegedUpdateQuarantineDir}, nil } -func (p *privilegeHelperUpdate) CreateQuarantinedArtifact() (string, *os.File, func(), error) { +func (p *privilegeHelperUpdate) CreateQuarantinedArtifact() (string, *os.File, func() error, error) { if err := validateCollectorQuarantineRoot(p.quarantineDir); err != nil { - return "", nil, func() {}, err + return "", nil, func() error { return nil }, err } var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { - return "", nil, func() {}, fmt.Errorf("create update artifact identity: %w", err) + return "", nil, func() error { return nil }, fmt.Errorf("create update artifact identity: %w", err) } artifactID := "pulse-agent-" + hex.EncodeToString(nonce[:]) artifactDir := filepath.Join(p.quarantineDir, artifactID) if err := os.Mkdir(artifactDir, 0o700); err != nil { - return "", nil, func() {}, fmt.Errorf("create quarantined artifact directory: %w", err) + return "", nil, func() error { return nil }, fmt.Errorf("create quarantined artifact directory: %w", err) } - cleanup := func() { - _ = os.Remove(filepath.Join(artifactDir, "pulse-agent")) - _ = os.Remove(filepath.Join(artifactDir, "pulse-agent.sig")) - _ = os.Remove(artifactDir) + cleanup := func() error { + var cleanupErr error + for _, name := range []string{"pulse-agent", "pulse-agent.sig"} { + if err := os.Remove(filepath.Join(artifactDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, err) + } + } + if err := os.Remove(artifactDir); err != nil && !errors.Is(err, os.ErrNotExist) { + cleanupErr = errors.Join(cleanupErr, err) + } + if err := syncUpdateDirectory(p.quarantineDir); err != nil { + cleanupErr = errors.Join(cleanupErr, err) + } + return cleanupErr } file, err := os.OpenFile(filepath.Join(artifactDir, "pulse-agent"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) if err != nil { - cleanup() - return "", nil, func() {}, fmt.Errorf("create quarantined agent binary: %w", err) + _ = cleanup() + return "", nil, func() error { return nil }, fmt.Errorf("create quarantined agent binary: %w", err) } return artifactID, file, cleanup, nil } @@ -104,6 +123,14 @@ func (p *privilegeHelperUpdate) Activate(ctx context.Context, artifactID, digest return result, err } +func (p *privilegeHelperUpdate) Commit(ctx context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { + var result agenthelper.UpdateResult + _, err := p.client.Call(ctx, agenthelper.OperationAgentUpdateCommit, agenthelper.OperationVersion1, 30*time.Second, agenthelper.UpdateCommitRequest{ + ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, + }, &result) + return result, err +} + func (p *privilegeHelperUpdate) Rollback(ctx context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { var result agenthelper.UpdateResult _, err := p.client.Call(ctx, agenthelper.OperationAgentUpdateRollback, agenthelper.OperationVersion1, 30*time.Second, agenthelper.UpdateRollbackRequest{ @@ -145,3 +172,143 @@ func syncUpdateDirectory(path string) error { defer dir.Close() return dir.Sync() } + +func PersistPendingPrivilegedUpdate(stateDir, previousVersion string, activation agenthelper.UpdateResult) error { + path, err := pendingPrivilegedUpdatePath(stateDir) + if err != nil { + return err + } + if err := validatePendingUpdateStateDir(stateDir); err != nil { + return err + } + previousVersion = strings.TrimSpace(previousVersion) + if previousVersion == "" || len(previousVersion) > 128 || strings.ContainsAny(previousVersion, "\x00\r\n") { + return errors.New("previous update version is invalid") + } + if !validPendingActivation(activation) { + return errors.New("pending helper activation is invalid") + } + data, err := json.Marshal(PendingPrivilegedUpdate{Activation: activation, PreviousVersion: previousVersion}) + if err != nil { + return err + } + temp, err := os.CreateTemp(stateDir, ".pulse-agent-update-pending-*") + if err != nil { + return fmt.Errorf("create pending update handoff: %w", err) + } + tempPath := temp.Name() + defer os.Remove(tempPath) + if err := temp.Chmod(0o600); err != nil { + _ = temp.Close() + return err + } + if _, err := temp.Write(data); err != nil { + _ = temp.Close() + return err + } + if err := temp.Sync(); err != nil { + _ = temp.Close() + return err + } + if err := temp.Close(); err != nil { + return err + } + if err := os.Rename(tempPath, path); err != nil { + return err + } + return syncUpdateDirectory(stateDir) +} + +func LoadPendingPrivilegedUpdate(stateDir string) (*PendingPrivilegedUpdate, error) { + path, err := pendingPrivilegedUpdatePath(stateDir) + if err != nil { + return nil, err + } + before, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + if err := validatePendingUpdateStateDir(stateDir); err != nil { + return nil, err + } + if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() || before.Mode().Perm()&0o077 != 0 { + return nil, errors.New("pending update handoff is not a private regular file") + } + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + after, err := file.Stat() + if err != nil || !os.SameFile(before, after) { + return nil, errors.New("pending update handoff identity changed") + } + decoder := json.NewDecoder(file) + decoder.DisallowUnknownFields() + var pending PendingPrivilegedUpdate + if err := decoder.Decode(&pending); err != nil { + return nil, fmt.Errorf("decode pending update handoff: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err != nil { + return nil, fmt.Errorf("decode pending update handoff trailing data: %w", err) + } + return nil, errors.New("pending update handoff contains trailing JSON") + } + if !validPendingActivation(pending.Activation) || strings.TrimSpace(pending.PreviousVersion) == "" { + return nil, errors.New("pending update handoff is invalid") + } + return &pending, nil +} + +func ClearPendingPrivilegedUpdate(stateDir string) error { + path, err := pendingPrivilegedUpdatePath(stateDir) + if err != nil { + return err + } + if err := validatePendingUpdateStateDir(stateDir); err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return syncUpdateDirectory(stateDir) +} + +func pendingPrivilegedUpdatePath(stateDir string) (string, error) { + if stateDir == "" || !filepath.IsAbs(stateDir) || filepath.Clean(stateDir) != stateDir { + return "", errors.New("pending update state directory must be clean and absolute") + } + return filepath.Join(stateDir, pendingPrivilegedUpdateFile), nil +} + +func validatePendingUpdateStateDir(stateDir string) error { + info, err := os.Lstat(stateDir) + if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return errors.New("pending update state directory must be a private real directory") + } + return nil +} + +func validPendingActivation(activation agenthelper.UpdateResult) bool { + parts := strings.Split(activation.ActivationID, ":") + if activation.Action != "pending" || len(parts) != 2 || !validPrivilegedArtifactID(parts[0]) || len(parts[1]) != 16 { + return false + } + if _, err := hex.DecodeString(parts[1]); err != nil { + return false + } + for _, digest := range []string{activation.ActiveSHA256, activation.RollbackSHA256} { + if len(digest) != 64 { + return false + } + if _, err := hex.DecodeString(digest); err != nil { + return false + } + } + return !activation.RollbackDeadline.IsZero() +} diff --git a/internal/agentupdate/privileged_update_test.go b/internal/agentupdate/privileged_update_test.go index 451da99ef..e327d8fef 100644 --- a/internal/agentupdate/privileged_update_test.go +++ b/internal/agentupdate/privileged_update_test.go @@ -17,19 +17,23 @@ import ( ) type fakePrivilegedUpdate struct { - root string - events []string - activation agenthelper.UpdateResult + root string + events []string + activation agenthelper.UpdateResult + rollbackErr error } -func (f *fakePrivilegedUpdate) CreateQuarantinedArtifact() (string, *os.File, func(), error) { +func (f *fakePrivilegedUpdate) CreateQuarantinedArtifact() (string, *os.File, func() error, error) { f.events = append(f.events, "quarantine") dir := filepath.Join(f.root, "pulse-agent-0123456789abcdef0123456789abcdef") if err := os.Mkdir(dir, 0o700); err != nil { - return "", nil, func() {}, err + return "", nil, func() error { return nil }, err } file, err := os.OpenFile(filepath.Join(dir, "pulse-agent"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) - return filepath.Base(dir), file, func() { _ = os.RemoveAll(dir) }, err + return filepath.Base(dir), file, func() error { + f.events = append(f.events, "cleanup") + return os.RemoveAll(dir) + }, err } func (f *fakePrivilegedUpdate) WriteQuarantinedSignature(_ string, signature string) error { @@ -44,12 +48,20 @@ func (f *fakePrivilegedUpdate) Stage(_ context.Context, artifactID, digest strin func (f *fakePrivilegedUpdate) Activate(_ context.Context, _ string, digest string) (agenthelper.UpdateResult, error) { f.events = append(f.events, "activate") - f.activation = agenthelper.UpdateResult{Action: "activated", ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", ActiveSHA256: digest, RollbackSHA256: strings.Repeat("a", 64)} + f.activation = agenthelper.UpdateResult{Action: "pending", ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", ActiveSHA256: digest, RollbackSHA256: strings.Repeat("a", 64), RollbackDeadline: time.Now().Add(time.Minute)} return f.activation, nil } +func (f *fakePrivilegedUpdate) Commit(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { + f.events = append(f.events, "commit") + return agenthelper.UpdateResult{Action: "committed", ActivationID: activation.ActivationID, ActiveSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256}, nil +} + func (f *fakePrivilegedUpdate) Rollback(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) { f.events = append(f.events, "rollback") + if f.rollbackErr != nil { + return agenthelper.UpdateResult{}, f.rollbackErr + } if activation != f.activation { return agenthelper.UpdateResult{}, errors.New("wrong activation identity") } @@ -70,7 +82,10 @@ func TestPrivilegedUpdateStagesActivatesAndRollsBackRestartFailure(t *testing.T) sum := sha256.Sum256(binary) digest := hex.EncodeToString(sum[:]) helper := &fakePrivilegedUpdate{root: t.TempDir()} - u := New(Config{PrivilegedUpdate: helper, Disabled: true}) + if err := os.Chmod(helper.root, 0o700); err != nil { + t.Fatal(err) + } + u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"}) u.selfTestFn = func(_ context.Context, path string) error { helper.events = append(helper.events, "self-test") data, err := os.ReadFile(path) @@ -84,7 +99,7 @@ func TestPrivilegedUpdateStagesActivatesAndRollsBackRestartFailure(t *testing.T) if err == nil || !strings.Contains(err.Error(), "update rolled back") { t.Fatalf("performPrivilegedUpdate error = %v", err) } - want := []string{"quarantine", "self-test", "signature:signed-update", "stage", "activate", "rollback"} + want := []string{"quarantine", "self-test", "signature:signed-update", "stage", "activate", "cleanup", "rollback"} if !reflect.DeepEqual(helper.events, want) { t.Fatalf("helper events = %#v, want %#v", helper.events, want) } @@ -96,14 +111,80 @@ func TestPrivilegedUpdateFailsClosedBeforeActivation(t *testing.T) { runtimeGOOS = goOSLinux binary := append([]byte{0x7f, 'E', 'L', 'F'}, []byte("update")...) helper := &fakePrivilegedUpdate{root: t.TempDir()} - u := New(Config{PrivilegedUpdate: helper, Disabled: true}) + if err := os.Chmod(helper.root, 0o700); err != nil { + t.Fatal(err) + } + u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"}) u.selfTestFn = func(context.Context, string) error { return nil } err := u.performPrivilegedUpdate(context.Background(), "/usr/local/bin/pulse-agent", bytes.NewReader(binary), int64(len(binary)), strings.Repeat("0", 64), "signed-update") if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { t.Fatalf("performPrivilegedUpdate error = %v", err) } - if !reflect.DeepEqual(helper.events, []string{"quarantine"}) { - t.Fatalf("helper events = %#v, want quarantine only", helper.events) + if !reflect.DeepEqual(helper.events, []string{"quarantine", "cleanup"}) { + t.Fatalf("helper events = %#v, want quarantine cleanup", helper.events) + } +} + +func TestPendingPrivilegedUpdateHandoffIsDurableAndStrict(t *testing.T) { + stateDir := t.TempDir() + if err := os.Chmod(stateDir, 0o700); err != nil { + t.Fatal(err) + } + activation := agenthelper.UpdateResult{ + Action: "pending", + ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", + ActiveSHA256: strings.Repeat("a", 64), + RollbackSHA256: strings.Repeat("b", 64), + RollbackDeadline: time.Now().Add(time.Minute).UTC(), + DurableAt: time.Now().UTC(), + } + if err := PersistPendingPrivilegedUpdate(stateDir, " 1.0.0 ", activation); err != nil { + t.Fatal(err) + } + path := filepath.Join(stateDir, pendingPrivilegedUpdateFile) + if info, err := os.Stat(path); err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("handoff mode=%v err=%v", info, err) + } + loaded, err := LoadPendingPrivilegedUpdate(stateDir) + if err != nil { + t.Fatal(err) + } + if loaded == nil || loaded.PreviousVersion != "1.0.0" || loaded.Activation != activation { + t.Fatalf("loaded handoff = %#v", loaded) + } + if err := ClearPendingPrivilegedUpdate(stateDir); err != nil { + t.Fatal(err) + } + if loaded, err := LoadPendingPrivilegedUpdate(stateDir); err != nil || loaded != nil { + t.Fatalf("cleared handoff = %#v, %v", loaded, err) + } +} + +func TestPendingPrivilegedUpdateHandoffRejectsUnsafeState(t *testing.T) { + stateDir := t.TempDir() + if err := os.Chmod(stateDir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(stateDir, pendingPrivilegedUpdateFile) + outside := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(outside, []byte(`{"activation":{},"previousVersion":"1.0.0"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, path); err != nil { + t.Fatal(err) + } + if _, err := LoadPendingPrivilegedUpdate(stateDir); err == nil { + t.Fatal("symlinked pending handoff accepted") + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + unsafe := `{"activation":{"action":"pending","activationId":"pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef","activeSha256":"` + strings.Repeat("a", 64) + `","rollbackSha256":"` + strings.Repeat("b", 64) + `","rollbackDeadline":"2030-01-01T00:00:00Z"},"previousVersion":"1.0.0","unexpected":true}` + if err := os.WriteFile(path, []byte(unsafe), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadPendingPrivilegedUpdate(stateDir); err == nil { + t.Fatal("unknown handoff field accepted") } } diff --git a/internal/agentupdate/update.go b/internal/agentupdate/update.go index 84540bfa7..fc2659fa6 100644 --- a/internal/agentupdate/update.go +++ b/internal/agentupdate/update.go @@ -1093,7 +1093,14 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string, if err != nil { return fmt.Errorf("prepare fixed update quarantine: %w", err) } - defer cleanup() + cleaned := false + defer func() { + if !cleaned { + if cleanupErr := cleanup(); cleanupErr != nil { + u.logger.Warn().Err(cleanupErr).Msg("failed to remove quarantined helper update artifact") + } + } + }() artifactPath := file.Name() hasher := sha256.New() written, copyErr := io.Copy(file, io.TeeReader(io.LimitReader(body, maxBinarySizeBytes+1), hasher)) @@ -1147,22 +1154,36 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string, if err != nil { return fmt.Errorf("activate signed update through typed helper: %w", err) } - if activation.Action != "activated" || !strings.EqualFold(activation.ActiveSHA256, digest) || activation.ActivationID == "" || activation.RollbackSHA256 == "" { - return errors.New("typed helper returned an invalid activation result") - } - if err := restartProcessFn(execPath); err != nil { + rollbackActivation := func(cause error) error { rollbackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() rolledBack, rollbackErr := u.cfg.PrivilegedUpdate.Rollback(rollbackCtx, activation) if rollbackErr != nil { - return errors.Join(fmt.Errorf("failed to restart after helper activation: %w", err), fmt.Errorf("typed helper rollback failed: %w", rollbackErr)) + return errors.Join(cause, fmt.Errorf("typed helper rollback failed: %w", rollbackErr)) } if rolledBack.Action != "rolled_back" || !strings.EqualFold(rolledBack.ActiveSHA256, activation.RollbackSHA256) { - return errors.Join(fmt.Errorf("failed to restart after helper activation: %w", err), errors.New("typed helper returned an invalid rollback result")) + return errors.Join(cause, errors.New("typed helper returned an invalid rollback result")) } - return fmt.Errorf("failed to restart after helper activation; update rolled back: %w", err) + if clearErr := ClearPendingPrivilegedUpdate(u.cfg.StateDir); clearErr != nil { + return errors.Join(fmt.Errorf("%w; update rolled back", cause), fmt.Errorf("clear pending update handoff: %w", clearErr)) + } + return fmt.Errorf("%w; update rolled back", cause) } - return nil + if activation.Action != "pending" || !strings.EqualFold(activation.ActiveSHA256, digest) || activation.ActivationID == "" || + activation.RollbackSHA256 == "" || activation.RollbackDeadline.IsZero() || !time.Now().Before(activation.RollbackDeadline) { + return rollbackActivation(errors.New("typed helper returned an invalid pending activation result")) + } + if err := PersistPendingPrivilegedUpdate(u.cfg.StateDir, u.cfg.CurrentVersion, activation); err != nil { + return rollbackActivation(fmt.Errorf("persist pending update handoff: %w", err)) + } + if err := cleanup(); err != nil { + return rollbackActivation(fmt.Errorf("remove quarantined update after helper staging: %w", err)) + } + cleaned = true + if err := restartProcessFn(execPath); err != nil { + return rollbackActivation(fmt.Errorf("failed to restart after helper activation: %w", err)) + } + return rollbackActivation(errors.New("restart returned without replacing the current process")) } func (u *Updater) syncPersistentBinaryCopy(execPath, persistPath, platform string) { diff --git a/internal/api/action_runner_credentials.go b/internal/api/action_runner_credentials.go index 07406aec3..daf495e23 100644 --- a/internal/api/action_runner_credentials.go +++ b/internal/api/action_runner_credentials.go @@ -7,7 +7,10 @@ import ( "net/http" "strings" + "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" + "github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding" "github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens" + "github.com/rcourtman/pulse-go-rewrite/internal/config" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" "github.com/rcourtman/pulse-go-rewrite/pkg/auth" ) @@ -30,6 +33,27 @@ type actionRunnerCredentialResponse struct { ActionCapability string `json:"actionCapability"` } +type actionRunnerCredentialSelfRevokeRequest struct { + AgentID string `json:"agentId"` + Hostname string `json:"hostname"` +} + +func actionRunnerCredentialRoute(cfg *config.Config, issue, selfRevoke http.HandlerFunc) http.HandlerFunc { + issue = RequireAdmin(cfg, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, issue))) + selfRevoke = RequireAuth(cfg, RequireScope(config.ScopeAgentExec, selfRevoke)) + return func(w http.ResponseWriter, req *http.Request) { + switch req.Method { + case http.MethodPost: + issue(w, req) + case http.MethodDelete: + selfRevoke(w, req) + default: + w.Header().Set("Allow", http.MethodPost+", "+http.MethodDelete) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + } + } +} + // handleIssueActionRunnerCredential creates the separately scoped credential // consumed by pulse-agent-runner. The route is operator-only; a monitoring // collector credential cannot mint or upgrade itself into remediation @@ -63,7 +87,7 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h http.Error(w, "Canonical monitored host identity not found", http.StatusNotFound) return } - rawToken, record, err := agenttokens.IssueActionRunnerAndPersist(r.config, r.persistence, agenttokens.ActionRunnerIssueOptions{ + issued, err := agenttokens.IssueActionRunnerAndPersistDetailed(r.config, r.persistence, agenttokens.ActionRunnerIssueOptions{ TokenName: payload.Name, OrgID: organizationID, OwnerUserID: apiTokenOwnerUserIDForRequest(r.config, req), @@ -78,12 +102,17 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h http.Error(w, "Failed to issue action runner credential", status) return } + record := issued.Record + for _, replaced := range issued.Replaced { + r.invalidateActionRunnerRecord(replaced) + } LogAuditEventForTenant(organizationID, "action_runner_credential_issued", auth.GetUser(req.Context()), GetClientIP(req), req.URL.Path, true, "Issued host-bound typed action runner credential") + w.Header().Set("Cache-Control", "no-store") w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) _ = json.NewEncoder(w).Encode(actionRunnerCredentialResponse{ - Token: rawToken, + Token: issued.Token, TokenID: record.ID, OrganizationID: record.OrgID, AgentID: record.Metadata["bound_agent_id"], @@ -93,6 +122,98 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h }) } +// handleSelfRevokeActionRunnerCredential lets the separately credentialed +// runner revoke only its own exact tenant/host binding. It cannot select a +// token ID or another host, and browser/session authentication is rejected. +func (r *Router) handleSelfRevokeActionRunnerCredential(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodDelete { + w.Header().Set("Allow", http.MethodDelete) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + if r == nil || r.config == nil { + http.Error(w, "Action runner credential service unavailable", http.StatusServiceUnavailable) + return + } + caller := getAPITokenRecordFromRequest(req) + if caller == nil { + http.Error(w, "Action runner bearer credential required", http.StatusForbidden) + return + } + decoder := json.NewDecoder(io.LimitReader(req.Body, maxActionRunnerCredentialRequestBytes+1)) + decoder.DisallowUnknownFields() + var payload actionRunnerCredentialSelfRevokeRequest + if err := decoder.Decode(&payload); err != nil { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + + organizationID := strings.TrimSpace(GetOrgID(req.Context())) + config.Mu.Lock() + previousTokens := append([]config.APITokenRecord(nil), r.config.APITokens...) + index := -1 + var removed config.APITokenRecord + for candidateIndex := range r.config.APITokens { + candidate := &r.config.APITokens[candidateIndex] + if candidate.ID != caller.ID { + continue + } + orgs := candidate.GetBoundOrgs() + if len(orgs) != 1 || strings.TrimSpace(orgs[0]) != organizationID || + !agentbinding.EvaluateActionRunner(candidate, payload.AgentID, payload.Hostname).Admit { + config.Mu.Unlock() + http.Error(w, "Action runner credential binding mismatch", http.StatusForbidden) + return + } + index = candidateIndex + removed = candidate.Clone() + break + } + if index < 0 { + config.Mu.Unlock() + http.Error(w, "Action runner credential not found", http.StatusUnauthorized) + return + } + r.config.APITokens = append(r.config.APITokens[:index], r.config.APITokens[index+1:]...) + r.config.SortAPITokens() + if r.persistence != nil { + if err := r.persistence.SaveAPITokens(r.config.APITokens); err != nil { + r.config.APITokens = previousTokens + r.config.SortAPITokens() + config.Mu.Unlock() + http.Error(w, "Failed to revoke action runner credential", http.StatusInternalServerError) + return + } + } + config.Mu.Unlock() + + r.invalidateActionRunnerRecord(removed) + LogAuditEventForTenant(organizationID, "action_runner_credential_revoked", auth.GetUser(req.Context()), GetClientIP(req), req.URL.Path, true, "Revoked exact host-bound typed action runner credential") + w.WriteHeader(http.StatusNoContent) +} + +func (r *Router) invalidateActionRunnerRecord(record config.APITokenRecord) bool { + if r == nil || r.agentExecServer == nil { + return false + } + orgs := record.GetBoundOrgs() + if len(orgs) != 1 { + return false + } + return r.agentExecServer.InvalidateActionRunnerSession(agentexec.AgentAdmission{ + OrganizationID: strings.TrimSpace(orgs[0]), + TokenID: strings.TrimSpace(record.ID), + AgentID: strings.TrimSpace(record.Metadata["bound_agent_id"]), + Hostname: strings.TrimSpace(record.Metadata["bound_hostname"]), + RuntimeRole: strings.TrimSpace(record.Metadata[agenttokens.RuntimeRoleMetadataKey]), + ActionCapability: strings.TrimSpace(record.Metadata[agenttokens.ActionCapabilityMetadataKey]), + }) +} + func (r *Router) resolveActionRunnerHostIdentity(req *http.Request, requestedID, requestedHostname string) (string, string, bool) { requestedID = strings.TrimSpace(requestedID) requestedHostname = strings.TrimSpace(requestedHostname) diff --git a/internal/api/action_runner_credentials_test.go b/internal/api/action_runner_credentials_test.go index 2c96c2d11..672302dcd 100644 --- a/internal/api/action_runner_credentials_test.go +++ b/internal/api/action_runner_credentials_test.go @@ -2,16 +2,41 @@ package api import ( "bytes" + "context" "encoding/json" + "errors" "net/http" "net/http/httptest" + "os" "testing" "time" + "github.com/gorilla/websocket" + "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" "github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens" "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt" ) +type actionRunnerFailingPersistenceFS struct{} + +func (actionRunnerFailingPersistenceFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} +func (actionRunnerFailingPersistenceFS) WriteFile(string, []byte, os.FileMode) error { + return errors.New("injected action-runner persistence failure") +} +func (actionRunnerFailingPersistenceFS) Rename(oldPath, newPath string) error { + return os.Rename(oldPath, newPath) +} +func (actionRunnerFailingPersistenceFS) Remove(name string) error { return os.Remove(name) } +func (actionRunnerFailingPersistenceFS) Stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} +func (actionRunnerFailingPersistenceFS) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + func newActionRunnerCredentialTestRouter(t *testing.T) (*Router, *config.Config, string) { t.Helper() cfg := &config.Config{DataPath: t.TempDir(), AuthUser: "admin", AuthPass: "$2a$10$dummy"} @@ -29,6 +54,53 @@ func actionRunnerCredentialBody(hostID, hostname string) *bytes.Reader { return bytes.NewReader(body) } +func issueActionRunnerCredentialForTest(t *testing.T, router *Router, hostID, hostname string) actionRunnerCredentialResponse { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, hostname)) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + router.handleIssueActionRunnerCredential(rec, req) + if rec.Code != http.StatusCreated { + t.Fatalf("issue status = %d, body=%s", rec.Code, rec.Body.String()) + } + var response actionRunnerCredentialResponse + if err := json.NewDecoder(rec.Body).Decode(&response); err != nil { + t.Fatal(err) + } + return response +} + +func connectActionRunnerCredentialForTest(t *testing.T, server *agentexec.Server, credential actionRunnerCredentialResponse) (*websocket.Conn, *httptest.Server) { + t.Helper() + ts := httptest.NewServer(http.HandlerFunc(server.HandleWebSocket)) + conn, _, err := websocket.DefaultDialer.Dial(wsURLForHTTP(ts.URL), wsHeadersForHTTP(t, ts.URL)) + if err != nil { + ts.Close() + t.Fatal(err) + } + message, err := agentexec.NewMessage(agentexec.MsgTypeAgentRegister, "", agentexec.AgentRegisterPayload{ + AgentID: credential.AgentID, Hostname: credential.Hostname, Token: credential.Token, + RuntimeRole: credential.RuntimeRole, ActionCapability: credential.ActionCapability, + OperationReceiptVersion: operationreceipt.ProtocolVersion, + }) + if err != nil { + conn.Close() + ts.Close() + t.Fatal(err) + } + if err := conn.WriteJSON(message); err != nil { + conn.Close() + ts.Close() + t.Fatal(err) + } + if registered := readRegisteredPayload(t, conn); !registered.Success { + conn.Close() + ts.Close() + t.Fatalf("runner registration failed: %s", registered.Message) + } + return conn, ts +} + func TestIssueActionRunnerCredentialResolvesCanonicalMonitoredHost(t *testing.T) { router, cfg, hostID := newActionRunnerCredentialTestRouter(t) req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "HOST-1")) @@ -106,11 +178,94 @@ func TestIssueActionRunnerCredentialRouteRotatesExistingHostBinding(t *testing.T } } +func TestIssueActionRunnerCredentialRotationClosesReplacedLiveSessionAfterPersistence(t *testing.T) { + router, _, hostID := newActionRunnerCredentialTestRouter(t) + router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) + first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first) + defer conn.Close() + defer ts.Close() + if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("runner session was not connected") + } + + second := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + if second.TokenID == first.TokenID { + t.Fatal("credential did not rotate") + } + if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("replaced action-runner session remained connected") + } +} + +func TestIssueActionRunnerCredentialPersistenceFailureKeepsPriorLiveSession(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) + router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) + first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first) + defer conn.Close() + defer ts.Close() + + router.persistence.SetFileSystem(actionRunnerFailingPersistenceFS{}) + req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "host-1.local")) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + router.handleIssueActionRunnerCredential(rec, req) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, body=%s", rec.Code, rec.Body.String()) + } + if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != first.TokenID { + t.Fatalf("prior credential was not restored: %#v", cfg.APITokens) + } + if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("failed persistence invalidated the prior live session") + } +} + +func TestSelfRevokeActionRunnerCredentialRequiresExactBearerBindingAndClosesSession(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) + router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession) + issued := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local") + conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, issued) + defer conn.Close() + defer ts.Close() + + handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) + request := func(organizationID, agentID, hostname, token string) *httptest.ResponseRecorder { + t.Helper() + body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: agentID, Hostname: hostname}) + req := httptest.NewRequest(http.MethodDelete, "/api/agents/action-runner/credential", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, organizationID)) + rec := httptest.NewRecorder() + handler(rec, req) + return rec + } + if rec := request("other", hostID, issued.Hostname, issued.Token); rec.Code != http.StatusForbidden { + t.Fatalf("mismatched organization status = %d, body=%s", rec.Code, rec.Body.String()) + } + if rec := request("default", hostID, "other.example", issued.Token); rec.Code != http.StatusForbidden { + t.Fatalf("mismatched hostname status = %d, body=%s", rec.Code, rec.Body.String()) + } + if len(cfg.APITokens) != 1 || !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("binding mismatch changed credential or session state") + } + if rec := request("default", hostID, issued.Hostname, issued.Token); rec.Code != http.StatusNoContent { + t.Fatalf("self revoke status = %d, body=%s", rec.Code, rec.Body.String()) + } + if len(cfg.APITokens) != 0 { + t.Fatalf("self-revoked credential remained stored: %#v", cfg.APITokens) + } + if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) { + t.Fatal("self-revoked session remained connected") + } +} + func TestActionRunnerCredentialRouteAuthSupportsAdminSessionAndScopedToken(t *testing.T) { for _, mode := range []string{"api-token", "admin-session"} { t.Run(mode, func(t *testing.T) { router, cfg, hostID := newActionRunnerCredentialTestRouter(t) - handler := RequireAdmin(cfg, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, router.handleIssueActionRunnerCredential))) + handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential) req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "host-1.local")) switch mode { case "api-token": diff --git a/internal/api/agenttokens/install.go b/internal/api/agenttokens/install.go index 11d6f877f..0059c5ce7 100644 --- a/internal/api/agenttokens/install.go +++ b/internal/api/agenttokens/install.go @@ -55,6 +55,16 @@ type ActionRunnerIssueOptions struct { Hostname string } +// ActionRunnerIssueResult carries the newly issued credential together with +// the prior host-bound records it durably replaced. Callers use the replaced +// non-secret identities to invalidate only the superseded live sessions after +// persistence succeeds. +type ActionRunnerIssueResult struct { + Token string + Record *config.APITokenRecord + Replaced []config.APITokenRecord +} + func ProxmoxScopes(enableCommands bool) []string { scopes := []string{ config.ScopeAgentReport, @@ -109,7 +119,8 @@ func ParseCommandPolicyIntent(record *config.APITokenRecord) (bool, bool) { } func IssueAndPersist(cfg *config.Config, persistence *config.ConfigPersistence, opts IssueOptions) (string, *config.APITokenRecord, error) { - return issueAndPersistReplacing(cfg, persistence, opts, nil) + rawToken, record, _, err := issueAndPersistReplacing(cfg, persistence, opts, nil) + return rawToken, record, err } func issueAndPersistReplacing( @@ -117,14 +128,14 @@ func issueAndPersistReplacing( persistence *config.ConfigPersistence, opts IssueOptions, replace func(config.APITokenRecord) bool, -) (string, *config.APITokenRecord, error) { +) (string, *config.APITokenRecord, []config.APITokenRecord, error) { if cfg == nil { - return "", nil, fmt.Errorf("config is required") + return "", nil, nil, fmt.Errorf("config is required") } rawToken, err := internalauth.GenerateAPIToken() if err != nil { - return "", nil, fmt.Errorf("%w: %w", ErrGeneration, err) + return "", nil, nil, fmt.Errorf("%w: %w", ErrGeneration, err) } scopes := opts.Scopes @@ -136,19 +147,19 @@ func issueAndPersistReplacing( } record, err := config.NewAPITokenRecord(rawToken, opts.TokenName, scopes) if err != nil { - return "", nil, fmt.Errorf("%w: %w", ErrRecord, err) + return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err) } record.OrgID = strings.TrimSpace(opts.OrgID) setOwnerUserID(record, opts.OwnerUserID) if err := mergeMetadata(record, opts.Metadata); err != nil { - return "", nil, fmt.Errorf("%w: %w", ErrRecord, err) + return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err) } if record.Metadata == nil { record.Metadata = make(map[string]string) } if err := normalizeCredentialKind(record); err != nil { - return "", nil, fmt.Errorf("%w: %w", ErrRecord, err) + return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err) } record.Metadata[IssuedAtMetadataKey] = record.CreatedAt.UTC().Format(time.RFC3339) @@ -156,12 +167,15 @@ func issueAndPersistReplacing( defer config.Mu.Unlock() previousTokens := append([]config.APITokenRecord(nil), cfg.APITokens...) + replaced := make([]config.APITokenRecord, 0, 1) if replace == nil { cfg.APITokens = append(cfg.APITokens, *record) } else { nextTokens := make([]config.APITokenRecord, 0, len(cfg.APITokens)+1) for _, existing := range cfg.APITokens { - if !replace(existing) { + if replace(existing) { + replaced = append(replaced, existing.Clone()) + } else { nextTokens = append(nextTokens, existing) } } @@ -175,38 +189,46 @@ func issueAndPersistReplacing( // that the failed request never returned. Restore the full snapshot. cfg.APITokens = previousTokens cfg.SortAPITokens() - return "", nil, fmt.Errorf("%w: %w", ErrPersist, err) + return "", nil, nil, fmt.Errorf("%w: %w", ErrPersist, err) } } - return rawToken, record, nil + return rawToken, record, replaced, nil } // IssueActionRunnerAndPersist mints the host-bound credential used by the // separate action runner. It fails closed on missing tenant/host identity and // never grants collector report, lookup, configuration, or management scopes. func IssueActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigPersistence, opts ActionRunnerIssueOptions) (string, *config.APITokenRecord, error) { + result, err := IssueActionRunnerAndPersistDetailed(cfg, persistence, opts) + return result.Token, result.Record, err +} + +// IssueActionRunnerAndPersistDetailed is the rotation-aware form used by the +// API boundary. Replaced contains records only when the new credential was +// durably committed; persistence failure returns an empty result. +func IssueActionRunnerAndPersistDetailed(cfg *config.Config, persistence *config.ConfigPersistence, opts ActionRunnerIssueOptions) (ActionRunnerIssueResult, error) { agentID := strings.TrimSpace(opts.AgentID) hostname := unifiedresources.NormalizeFullHostname(opts.Hostname) organizationID := strings.TrimSpace(opts.OrgID) if organizationID == "" { - return "", nil, fmt.Errorf("%w: organization id is required", ErrRecord) + return ActionRunnerIssueResult{}, fmt.Errorf("%w: organization id is required", ErrRecord) } if agentID == "" { - return "", nil, fmt.Errorf("%w: canonical agent id is required", ErrRecord) + return ActionRunnerIssueResult{}, fmt.Errorf("%w: canonical agent id is required", ErrRecord) } if hostname == "" { - return "", nil, fmt.Errorf("%w: canonical hostname is required", ErrRecord) + return ActionRunnerIssueResult{}, fmt.Errorf("%w: canonical hostname is required", ErrRecord) } if len(agentID) > 128 || len(hostname) > 253 { - return "", nil, fmt.Errorf("%w: action runner identity exceeds maximum length", ErrRecord) + return ActionRunnerIssueResult{}, fmt.Errorf("%w: action runner identity exceeds maximum length", ErrRecord) } tokenName := strings.TrimSpace(opts.TokenName) if tokenName == "" { tokenName = "action-runner:" + hostname } - return issueAndPersistReplacing(cfg, persistence, IssueOptions{ + rawToken, record, replaced, err := issueAndPersistReplacing(cfg, persistence, IssueOptions{ TokenName: tokenName, OrgID: organizationID, OwnerUserID: opts.OwnerUserID, @@ -224,6 +246,10 @@ func IssueActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigP strings.TrimSpace(record.Metadata[CredentialKindMetadataKey]) == CredentialKindActionRunner && strings.TrimSpace(record.Metadata["bound_agent_id"]) == agentID }) + if err != nil { + return ActionRunnerIssueResult{}, err + } + return ActionRunnerIssueResult{Token: rawToken, Record: record, Replaced: replaced}, nil } func normalizeCredentialKind(record *config.APITokenRecord) error { diff --git a/internal/api/agenttokens/install_test.go b/internal/api/agenttokens/install_test.go index 4cf012115..bfc7a9b86 100644 --- a/internal/api/agenttokens/install_test.go +++ b/internal/api/agenttokens/install_test.go @@ -113,6 +113,28 @@ func TestIssueActionRunnerAndPersistReplacesMatchingBoundCredential(t *testing.T } } +func TestIssueActionRunnerAndPersistDetailedReturnsOnlyDurablyReplacedRecords(t *testing.T) { + cfg := &config.Config{DataPath: t.TempDir()} + _, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ + OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example", + }) + if err != nil { + t.Fatal(err) + } + result, err := IssueActionRunnerAndPersistDetailed(cfg, config.NewConfigPersistence(cfg.DataPath), ActionRunnerIssueOptions{ + OrgID: "org-a", AgentID: "machine-123", Hostname: "renamed.example", + }) + if err != nil { + t.Fatal(err) + } + if result.Token == "" || result.Record == nil || len(result.Replaced) != 1 || result.Replaced[0].ID != prior.ID { + t.Fatalf("detailed issue result = %#v", result) + } + if result.Record.ID == prior.ID || len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != result.Record.ID { + t.Fatalf("persisted replacement = %#v", cfg.APITokens) + } +} + func TestIssueActionRunnerAndPersistRestoresReplacedCredentialOnPersistenceFailure(t *testing.T) { cfg := &config.Config{DataPath: t.TempDir()} _, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{ diff --git a/internal/api/router_routes_registration.go b/internal/api/router_routes_registration.go index 38b31bdb5..96a9f28c9 100644 --- a/internal/api/router_routes_registration.go +++ b/internal/api/router_routes_registration.go @@ -55,7 +55,7 @@ func (r *Router) registerConfigSystemRoutes(updateHandlers *UpdateHandlers) { r.mux.HandleFunc("/api/agents/docker/report", RequireAuth(r.config, RequireScope(config.ScopeDockerReport, r.dockerAgentHandlers.HandleReport))) r.mux.HandleFunc("/api/agents/kubernetes/report", RequireAuth(r.config, RequireScope(config.ScopeKubernetesReport, r.kubernetesAgentHandlers.HandleReport))) r.mux.HandleFunc("/api/agents/agent/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport))) - r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, r.handleIssueActionRunnerCredential)))) + r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAuth(r.config, actionRunnerCredentialRoute(r.config, r.handleIssueActionRunnerCredential, r.handleSelfRevokeActionRunnerCredential))) r.mux.HandleFunc("/api/agents/host/report", wrapLegacyHostAlias("/api/agents/host/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport)))) r.mux.HandleFunc("/api/agents/agent/lookup", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleLookup))) r.mux.HandleFunc("/api/agents/host/lookup", wrapLegacyHostAlias("/api/agents/host/lookup", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleLookup)))) diff --git a/internal/api/security_tokens_lifecycle_test.go b/internal/api/security_tokens_lifecycle_test.go index 26d707bea..1c7ffeb0b 100644 --- a/internal/api/security_tokens_lifecycle_test.go +++ b/internal/api/security_tokens_lifecycle_test.go @@ -2,6 +2,8 @@ package api import ( "bytes" + "context" + "encoding/json" "net/http" "net/http/httptest" "os" @@ -10,9 +12,35 @@ import ( "testing" "time" + "github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens" "github.com/rcourtman/pulse-go-rewrite/internal/config" ) +func TestSelfRevokeActionRunnerCredentialRejectsNonRunnerExecBearer(t *testing.T) { + router, cfg, hostID := newActionRunnerCredentialTestRouter(t) + raw := "legacy-exec-token-1234567890.12345678" + record, err := config.NewAPITokenRecord(raw, "legacy", []string{config.ScopeAgentExec}) + if err != nil { + t.Fatal(err) + } + record.OrgID = "default" + record.Metadata = map[string]string{ + agenttokens.RuntimeRoleMetadataKey: agenttokens.CredentialKindLegacyFullTrust, + "bound_agent_id": hostID, + "bound_hostname": "host-1.local", + } + cfg.APITokens = append(cfg.APITokens, *record) + body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: hostID, Hostname: "host-1.local"}) + req := httptest.NewRequest(http.MethodDelete, "/api/agents/action-runner/credential", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+raw) + req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default")) + rec := httptest.NewRecorder() + actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)(rec, req) + if rec.Code != http.StatusForbidden || len(cfg.APITokens) != 1 { + t.Fatalf("legacy self revoke = status %d tokens %#v body=%s", rec.Code, cfg.APITokens, rec.Body.String()) + } +} + func TestSecurityTokensCreateRollsBackCompleteInventoryWhenPersistenceFails(t *testing.T) { now := time.Now().UTC() tokens := []config.APITokenRecord{ diff --git a/internal/hostagent/agent.go b/internal/hostagent/agent.go index be2a07a30..df98fe49a 100644 --- a/internal/hostagent/agent.go +++ b/internal/hostagent/agent.go @@ -99,6 +99,13 @@ type Config struct { // server upgrade within one report cycle instead of its next hourly check. // Nil disables the hook; acks from observer destinations never invoke it. OnServerVersion func(version string) + // OnPrimaryReportAccepted is called only after the current process's newly + // collected authoritative report is accepted. Buffered reports and observer + // acknowledgements do not satisfy this startup-health signal. + OnPrimaryReportAccepted func() + // UpdatedFromVersion is supplied by the durable helper-update handoff and + // is carried by the first report built by the replacement process. + UpdatedFromVersion string Collector SystemCollector // Optional: override default system information collector (for testing) // PrivilegedTelemetry routes the exceptional SMART and Proxmox LXC @@ -400,7 +407,10 @@ func New(cfg Config) (*Agent, error) { const bufferCapacity = 60 // Check if agent was recently auto-updated (only reported once per restart) - updatedFrom := updatedFromVersionFn() + updatedFrom := strings.TrimSpace(cfg.UpdatedFromVersion) + if updatedFrom == "" { + updatedFrom = updatedFromVersionFn() + } if updatedFrom != "" { logger.Info(). Str("previousVersion", updatedFrom). @@ -854,6 +864,9 @@ func (a *Agent) deliverPrimaryReport(ctx context.Context, report agentshost.Repo return nil } agenttarget.MarkDelivery("host", "primary", "primary", true) + if a.cfg.OnPrimaryReportAccepted != nil { + a.cfg.OnPrimaryReportAccepted() + } // The server has counted every availability result in this report, so they // must never be offered to it again. @@ -1167,16 +1180,7 @@ func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) { // Proxmox VE exposes per-mount LXC usage only through the node-local pct // CLI. Query running containers on an independent bounded deadline. - var proxmoxLXCData *agentshost.ProxmoxLXCInventory - if a.privilegedTelemetry != nil { - proxmoxLXCData, err = a.privilegedTelemetry.ProxmoxLXCFilesystems(ctx) - if err != nil { - a.logger.Debug().Err(err).Msg("Typed helper could not collect Proxmox LXC filesystems") - proxmoxLXCData = nil - } - } else { - proxmoxLXCData = a.collectProxmoxLXCFilesystems(ctx) - } + proxmoxLXCData := a.collectProxmoxLXCFilesystemsForReport(ctx) // Collect S.M.A.R.T. disk data after topology owners. Enumeration and each // device probe have their own deadlines; a single shared 10-second budget diff --git a/internal/hostagent/agent_flushbuffer_test.go b/internal/hostagent/agent_flushbuffer_test.go index 131c37630..b70f39813 100644 --- a/internal/hostagent/agent_flushbuffer_test.go +++ b/internal/hostagent/agent_flushbuffer_test.go @@ -20,6 +20,7 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin var ( mu sync.Mutex received []string + accepted atomic.Int32 ) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var body io.Reader = r.Body @@ -45,7 +46,9 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin defer server.Close() a := &Agent{ - cfg: Config{APIToken: "token"}, + cfg: Config{APIToken: "token", OnPrimaryReportAccepted: func() { + accepted.Add(1) + }}, logger: zerolog.Nop(), httpClient: server.Client(), trimmedPulseURL: server.URL, @@ -69,6 +72,9 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin t.Fatalf("received = %v, want %v", received, want) } } + if got := accepted.Load(); got != 1 { + t.Fatalf("accepted current-report callbacks = %d, want 1", got) + } } func TestAgent_flushBuffer_StopsOnFailureAndDoesNotDropReport(t *testing.T) { diff --git a/internal/hostagent/agent_new_test.go b/internal/hostagent/agent_new_test.go index 7589a42ce..426eda0bb 100644 --- a/internal/hostagent/agent_new_test.go +++ b/internal/hostagent/agent_new_test.go @@ -1043,6 +1043,26 @@ func TestNew_CarriesUpdatedFromIntoFirstV6Report(t *testing.T) { } } +func TestNew_PrefersDurableHelperUpdatedFromHandoff(t *testing.T) { + mc := &mockCollector{ + hostInfoFn: func(context.Context) (*gohost.InfoStat, error) { + return &gohost.InfoStat{Hostname: "upgraded-host", HostID: "machine-id-1", KernelArch: runtime.GOARCH}, nil + }, + } + agent, err := New(Config{ + APIToken: "token", + Collector: mc, + UpdatedFromVersion: "6.0.0", + updatedFromVersionFn: func() string { return "stale-local-marker" }, + }) + if err != nil { + t.Fatal(err) + } + if agent.updatedFrom != "6.0.0" { + t.Fatalf("updatedFrom = %q, want durable helper handoff", agent.updatedFrom) + } +} + func TestNew_RejectsInvalidPulseURL(t *testing.T) { mc := &mockCollector{ hostInfoFn: func(context.Context) (*gohost.InfoStat, error) { diff --git a/internal/hostagent/privilege_helper_client.go b/internal/hostagent/privilege_helper_client.go index a877be76f..6fd5e1d68 100644 --- a/internal/hostagent/privilege_helper_client.go +++ b/internal/hostagent/privilege_helper_client.go @@ -16,6 +16,7 @@ const privilegeHelperOperationDeadline = 30 * time.Second // It intentionally exposes complete typed snapshots rather than commands, // executable paths, VMIDs, device paths, or caller-selected arguments. type PrivilegedTelemetry interface { + Health(context.Context) error SMARTSnapshot(context.Context) ([]DiskSMART, error) ProxmoxLXCFilesystems(context.Context) (*agentshost.ProxmoxLXCInventory, error) } @@ -40,6 +41,32 @@ func NewPrivilegeHelperTelemetry(socketPath string) (PrivilegedTelemetry, error) return &privilegeHelperTelemetry{client: client}, nil } +// Health proves that the configured socket is serving the exact typed-helper +// protocol expected by this collector. A listening systemd socket alone is +// not sufficient: socket activation can succeed while the helper binary is +// missing, incompatible, or unable to handle requests. +func (c *privilegeHelperTelemetry) Health(ctx context.Context) error { + var response agenthelper.HealthResult + _, err := c.client.Call( + ctx, + agenthelper.OperationHealth, + agenthelper.OperationVersion1, + privilegeHelperOperationDeadline, + struct{}{}, + &response, + ) + if err != nil { + return err + } + if response.Status != "ok" { + return errors.New("helper health response is not ok") + } + if response.ProtocolVersion != agenthelper.ProtocolVersion { + return errors.New("helper health protocol version does not match") + } + return nil +} + func (c *privilegeHelperTelemetry) SMARTSnapshot(ctx context.Context) ([]DiskSMART, error) { var response struct { Disks []DiskSMART `json:"disks"` @@ -78,3 +105,19 @@ func (c *privilegeHelperTelemetry) ProxmoxLXCFilesystems(ctx context.Context) (* } return response.Inventory, nil } + +// collectProxmoxLXCFilesystemsForReport preserves the privilege boundary once +// a helper is configured. A helper failure must omit this best-effort snapshot +// rather than silently retrying the same collection in the unprivileged +// collector process (or widening that process's privileges to make it work). +func (a *Agent) collectProxmoxLXCFilesystemsForReport(ctx context.Context) *agentshost.ProxmoxLXCInventory { + if a.privilegedTelemetry == nil { + return a.collectProxmoxLXCFilesystems(ctx) + } + inventory, err := a.privilegedTelemetry.ProxmoxLXCFilesystems(ctx) + if err != nil { + a.logger.Debug().Err(err).Msg("Typed helper could not collect Proxmox LXC filesystems") + return nil + } + return inventory +} diff --git a/internal/hostagent/privilege_helper_client_test.go b/internal/hostagent/privilege_helper_client_test.go index 5744792b5..e5ac9df2f 100644 --- a/internal/hostagent/privilege_helper_client_test.go +++ b/internal/hostagent/privilege_helper_client_test.go @@ -2,15 +2,22 @@ package hostagent import ( "context" + "encoding/binary" + "encoding/json" "errors" + "io" + "net" "os/exec" "testing" + "time" + "github.com/rcourtman/pulse-go-rewrite/internal/agenthelper" agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host" "github.com/rs/zerolog" ) type fakePrivilegedTelemetry struct { + health error smart []DiskSMART smartErr error proxmox *agentshost.ProxmoxLXCInventory @@ -19,6 +26,10 @@ type fakePrivilegedTelemetry struct { proxmoxCalls int } +func (f *fakePrivilegedTelemetry) Health(context.Context) error { + return f.health +} + func (f *fakePrivilegedTelemetry) SMARTSnapshot(context.Context) ([]DiskSMART, error) { f.smartCalls++ return f.smart, f.smartErr @@ -29,6 +40,93 @@ func (f *fakePrivilegedTelemetry) ProxmoxLXCFilesystems(context.Context) (*agent return f.proxmox, f.proxmoxErr } +func testPrivilegeHelperTelemetry(t *testing.T, result agenthelper.HealthResult) *privilegeHelperTelemetry { + t.Helper() + client, err := agenthelper.NewClient(agenthelper.ClientConfig{ + SocketPath: "/run/pulse-agent/helper.sock", + MaxDeadline: privilegeHelperOperationDeadline, + NewRequestID: func() (string, error) { + return "health-request", nil + }, + DialContext: func(context.Context, string, string) (net.Conn, error) { + clientConn, serverConn := net.Pipe() + go func() { + defer serverConn.Close() + var header [4]byte + if _, readErr := io.ReadFull(serverConn, header[:]); readErr != nil { + return + } + requestBytes := make([]byte, binary.BigEndian.Uint32(header[:])) + if _, readErr := io.ReadFull(serverConn, requestBytes); readErr != nil { + return + } + var request agenthelper.Request + if unmarshalErr := json.Unmarshal(requestBytes, &request); unmarshalErr != nil { + return + } + resultBytes, marshalErr := json.Marshal(result) + if marshalErr != nil { + return + } + responseBytes, marshalErr := json.Marshal(agenthelper.Response{ + ProtocolVersion: agenthelper.ProtocolVersion, + RequestID: request.RequestID, + Operation: request.Operation, + OperationVersion: request.OperationVersion, + Success: true, + Result: resultBytes, + }) + if marshalErr != nil { + return + } + frame := make([]byte, 4+len(responseBytes)) + binary.BigEndian.PutUint32(frame[:4], uint32(len(responseBytes))) + copy(frame[4:], responseBytes) + _, _ = serverConn.Write(frame) + }() + return clientConn, nil + }, + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + return &privilegeHelperTelemetry{client: client} +} + +func TestPrivilegeHelperHealthRequiresExactProtocolResponse(t *testing.T) { + tests := []struct { + name string + result agenthelper.HealthResult + wantErr bool + }{ + { + name: "healthy", + result: agenthelper.HealthResult{Status: "ok", ProtocolVersion: agenthelper.ProtocolVersion}, + }, + { + name: "unhealthy status", + result: agenthelper.HealthResult{Status: "degraded", ProtocolVersion: agenthelper.ProtocolVersion}, + wantErr: true, + }, + { + name: "protocol mismatch", + result: agenthelper.HealthResult{Status: "ok", ProtocolVersion: agenthelper.ProtocolVersion + 1}, + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + telemetry := testPrivilegeHelperTelemetry(t, test.result) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := telemetry.Health(ctx) + if (err != nil) != test.wantErr { + t.Fatalf("Health error = %v, wantErr %v", err, test.wantErr) + } + }) + } +} + func TestCollectSMARTDataUsesTypedHelperWithoutLocalFallback(t *testing.T) { originalLookPath := zpoolLookPath zpoolLookPath = func(string) (string, error) { return "", exec.ErrNotFound } @@ -87,3 +185,30 @@ func TestCollectSMARTDataDoesNotWidenPrivilegeAfterHelperFailure(t *testing.T) { t.Fatalf("local SMART fallback calls = %d, want 0", localCalls) } } + +func TestCollectProxmoxLXCFilesystemsDoesNotWidenPrivilegeAfterHelperFailure(t *testing.T) { + localCalls := 0 + collector := &mockCollector{ + goos: "linux", + lookPathFn: func(string) (string, error) { + localCalls++ + return "/usr/sbin/pct", nil + }, + } + helper := &fakePrivilegedTelemetry{proxmoxErr: errors.New("helper unavailable")} + agent := &Agent{ + collector: collector, + privilegedTelemetry: helper, + logger: zerolog.Nop(), + } + + if got := agent.collectProxmoxLXCFilesystemsForReport(context.Background()); got != nil { + t.Fatalf("Proxmox inventory after helper failure = %+v, want nil", got) + } + if helper.proxmoxCalls != 1 { + t.Fatalf("helper Proxmox calls = %d, want 1", helper.proxmoxCalls) + } + if localCalls != 0 { + t.Fatalf("local Proxmox fallback calls = %d, want 0", localCalls) + } +} diff --git a/scripts/install.sh b/scripts/install.sh index 52eff1b57..fe2345be4 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -193,6 +193,8 @@ SAFE_PROFILE_COLLECTOR_UNIT="/etc/systemd/system/${AGENT_NAME}.service" SAFE_PROFILE_TRANSACTION_DIR="" SAFE_PROFILE_TRANSACTION_ACTIVE="false" SAFE_PROFILE_TRANSACTION_COMMITTED="false" +SAFE_PROFILE_PRIOR_REGISTRATION_LAST_SEEN="" +AGENT_REGISTRATION_LAST_SEEN="" SYSTEMD_ENV_LINES="" SHELL_EXPORT_LINES="" @@ -642,12 +644,14 @@ warn_agent_token_rejected() { # 2 - the server rejected the credential (401, or 403 other than a stale # hostname ownership match - actionable, permanent) verify_agent_server_registration() { + local required_previous_last_seen="${1:-}" local lookup_id="${AGENT_ID}" local lookup_hostname="${HOSTNAME_OVERRIDE}" local lookup_query="" local lookup_out="" local lookup_status="" local lookup_body="" + local lookup_last_seen="" # No -f: we need the response body AND the HTTP status even on 4xx so a # rejected token (401/403) can be told apart from "not reported yet". The # -w format appends "\n" after the body. @@ -694,7 +698,15 @@ verify_agent_server_registration() { ;; esac + AGENT_REGISTRATION_LAST_SEEN="" if echo "$lookup_body" | grep -q '"agent"[[:space:]]*:' && echo "$lookup_body" | grep -q '"id"[[:space:]]*:'; then + lookup_last_seen=$(printf '%s' "$lookup_body" | tr -d '\r\n' | + sed -n 's/.*"lastSeen"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + AGENT_REGISTRATION_LAST_SEEN="$lookup_last_seen" + if [[ -n "$required_previous_last_seen" ]] && + [[ -z "$lookup_last_seen" || "$lookup_last_seen" == "$required_previous_last_seen" ]]; then + return 1 + fi return 0 fi return 1 @@ -706,6 +718,7 @@ verify_agent_server_registration() { # immediate lookup routinely misses a perfectly healthy registration (#1644). # Return codes mirror verify_agent_server_registration. verify_agent_server_registration_with_retry() { + local required_previous_last_seen="${1:-}" local max_attempts=10 local interval=3 local attempt=0 @@ -716,7 +729,7 @@ verify_agent_server_registration_with_retry() { fi while [ $attempt -lt $max_attempts ]; do - verify_agent_server_registration + verify_agent_server_registration "$required_previous_last_seen" reg_rc=$? # 0 = confirmed; 2 = token rejected, which is definitive and will not # change with more polling. @@ -930,7 +943,81 @@ teardown_privileged_helper_service() { systemctl reset-failed "${PRIVILEGED_HELPER_NAME}.service" 2>/dev/null || true } +read_action_runner_env_value() { + local key="$1" + local value="" + [[ "$key" =~ ^[A-Z0-9_]+$ ]] || return 1 + [[ -f "$ACTION_RUNNER_ENV_FILE" && ! -L "$ACTION_RUNNER_ENV_FILE" ]] || return 1 + value=$(sed -n "s/^${key}=\"\(.*\)\"$/\1/p" "$ACTION_RUNNER_ENV_FILE" | tail -1) + # Values emitted by this installer escape backslashes and quotes. The + # revoke contract only needs URL, hostname, and an absolute state path; + # refuse an escaped value instead of evaluating shell syntax to recover it. + [[ -n "$value" && "$value" != *'\'* && "$value" != *$'\r'* && "$value" != *$'\n'* ]] || return 1 + printf '%s\n' "$value" +} + +revoke_action_runner_credential() { + local runner_url="" + local runner_hostname="" + local runner_agent_id_file="" + local runner_agent_id="" + local runner_token_file="" + local runner_token="" + local token_mode="" + local token_size="" + local curl_config="" + local payload="" + local -a revoke_args + + runner_url=$(read_action_runner_env_value "PULSE_URL" || true) + runner_hostname=$(read_action_runner_env_value "PULSE_AGENT_RUNNER_HOSTNAME" || true) + runner_agent_id_file=$(read_action_runner_env_value "PULSE_AGENT_RUNNER_AGENT_ID_FILE" || true) + runner_token_file=$(read_action_runner_env_value "PULSE_AGENT_RUNNER_TOKEN_FILE" || true) + [[ "$runner_url" =~ ^https?://[^[:space:]]+$ && -n "$runner_hostname" ]] || return 1 + [[ "$runner_agent_id_file" == /* && "$runner_agent_id_file" != *'/../'* && + -f "$runner_agent_id_file" && ! -L "$runner_agent_id_file" ]] || return 1 + [[ "$runner_token_file" == /* && "$runner_token_file" != *'/../'* && + -f "$runner_token_file" && ! -L "$runner_token_file" ]] || return 1 + runner_agent_id=$(head -1 "$runner_agent_id_file" 2>/dev/null || true) + (( ${#runner_agent_id} >= 1 && ${#runner_agent_id} <= 256 && + ${#runner_hostname} >= 1 && ${#runner_hostname} <= 253 )) || return 1 + [[ "$runner_agent_id" =~ ^[A-Za-z0-9._:-]+$ && + "$runner_hostname" =~ ^[A-Za-z0-9._:-]+$ ]] || return 1 + token_mode=$(stat -c '%a' "$runner_token_file" 2>/dev/null || true) + token_size=$(wc -c < "$runner_token_file" 2>/dev/null | tr -d ' ' || true) + [[ "$token_mode" =~ ^[0-7]{3,4}$ && "$token_size" =~ ^[0-9]+$ ]] || return 1 + (( (8#$token_mode & 8#077) == 0 && token_size >= 1 && token_size <= 4096 )) || return 1 + runner_token=$(head -1 "$runner_token_file" 2>/dev/null || true) + [[ -n "$runner_token" && "$runner_token" != *$'\r'* && "$runner_token" != *$'\n'* ]] || return 1 + + curl_config=$(mktemp) + chmod 0600 "$curl_config" + printf 'header = "Authorization: Bearer %s"\nheader = "Content-Type: application/json"\n' \ + "$runner_token" > "$curl_config" + runner_token="" + payload="{\"agentId\":\"${runner_agent_id}\",\"hostname\":\"${runner_hostname}\"}" + revoke_args=(--config "$curl_config" -fsS --connect-timeout 5 --max-time 10 -X DELETE --data-binary "$payload") + if grep -q '^PULSE_INSECURE="true"$' "$ACTION_RUNNER_ENV_FILE" 2>/dev/null; then + revoke_args+=(-k) + fi + if curl "${revoke_args[@]}" "${runner_url%/}/api/agents/action-runner/credential" >/dev/null 2>&1; then + rm -f "$curl_config" + return 0 + fi + rm -f "$curl_config" + return 1 +} + teardown_action_runner_service() { + local had_runner_config="false" + if [[ -f "$ACTION_RUNNER_ENV_FILE" || -f "$ACTION_RUNNER_TOKEN_FILE" ]]; then + had_runner_config="true" + fi + if revoke_action_runner_credential; then + log_info "Revoked the action-runner credential before removing local runner state." + elif [[ "$had_runner_config" == "true" ]]; then + log_warn "Could not self-revoke the action-runner credential; local runner removal will continue. Revoke the credential from Pulse if the server was unreachable." + fi if command -v systemctl >/dev/null 2>&1; then systemctl stop "${ACTION_RUNNER_NAME}.service" 2>/dev/null || true systemctl disable "${ACTION_RUNNER_NAME}.service" 2>/dev/null || true @@ -1206,6 +1293,7 @@ EOF } write_action_runner_config() { + local runner_hostname="$HOSTNAME_OVERRIDE" if [[ -L "$ACTION_RUNNER_CONFIG_DIR" || -L "$ACTION_RUNNER_STATE_DIR" || -L "$ACTION_RUNNER_TOKEN_FILE" || -L "$ACTION_RUNNER_ENV_FILE" ]]; then fail "Refusing unsafe symlink in the action-runner config or state boundary" "$EXIT_GENERAL" @@ -1222,6 +1310,12 @@ write_action_runner_config() { chmod 0600 "$ACTION_RUNNER_TOKEN_FILE" ACTION_TOKEN="" + if [[ -z "$runner_hostname" ]]; then + runner_hostname=$(hostname 2>/dev/null || true) + fi + [[ -n "$runner_hostname" && "$runner_hostname" != *$'\r'* && "$runner_hostname" != *$'\n'* ]] || + fail "Action runner requires a canonical hostname" "$EXIT_MISSING_ARGS" + : > "$ACTION_RUNNER_ENV_FILE" chmod 0600 "$ACTION_RUNNER_ENV_FILE" write_action_runner_env_value "PULSE_URL" "$PULSE_URL" @@ -1229,6 +1323,7 @@ write_action_runner_config() { write_action_runner_env_value "PULSE_AGENT_RUNNER_STATE_DIR" "$ACTION_RUNNER_STATE_DIR" write_action_runner_env_value "PULSE_AGENT_RUNNER_HEALTH_FILE" "$ACTION_RUNNER_HEALTH_FILE" write_action_runner_env_value "PULSE_AGENT_RUNNER_AGENT_ID_FILE" "${STATE_DIR%/}/agent-id" + write_action_runner_env_value "PULSE_AGENT_RUNNER_HOSTNAME" "$runner_hostname" if [[ -n "$SERVER_FINGERPRINT" ]]; then write_action_runner_env_value "PULSE_SERVER_FINGERPRINT" "$SERVER_FINGERPRINT" fi @@ -1415,6 +1510,31 @@ safe_profile_unit_property() { esac } +safe_profile_effective_unit_unoverridden() { + local fragment_path="" + local drop_in_paths="" + fragment_path=$(systemctl show "${AGENT_NAME}.service" --property FragmentPath --value 2>/dev/null) || return 1 + drop_in_paths=$(systemctl show "${AGENT_NAME}.service" --property DropInPaths --value 2>/dev/null) || return 1 + [[ "$fragment_path" == "$SAFE_PROFILE_COLLECTOR_UNIT" && -z "$drop_in_paths" ]] +} + +safe_profile_verify_effective_target() { + local unit_user="" + local ambient="" + local exec_start="" + local environment="" + + safe_profile_effective_unit_unoverridden || return 1 + unit_user=$(safe_profile_unit_property User) + ambient=$(safe_profile_unit_property AmbientCapabilities) + exec_start=$(safe_profile_unit_property ExecStart) + environment=$(safe_profile_unit_property Environment) + [[ "$unit_user" == "$LEAST_PRIVILEGE_USER" ]] || return 1 + [[ -z "$ambient" ]] || return 1 + [[ "$exec_start" != *"--enable-commands"* ]] || return 1 + [[ "$environment" == *"PULSE_AGENT_HELPER_SOCKET=${PRIVILEGED_HELPER_SOCKET_PATH}"* ]] || return 1 +} + safe_profile_inspect() { local unit_path="$SAFE_PROFILE_COLLECTOR_UNIT" local binary_path="${INSTALL_DIR}/${BINARY_NAME}" @@ -1427,10 +1547,16 @@ safe_profile_inspect() { local binary_mode="missing" local host="false" docker="false" kubernetes="false" proxmox="false" local helper="false" commands="false" runner="false" + local fragment_path="unavailable" drop_in_paths="unavailable" unit_unoverridden="false" if safe_profile_platform_supported; then supported="true"; fi profile=$(safe_profile_detect_current_profile) if [[ -f "$unit_path" ]]; then + fragment_path=$(safe_profile_unit_property FragmentPath) + fragment_path="${fragment_path:-unavailable}" + drop_in_paths=$(safe_profile_unit_property DropInPaths) + drop_in_paths="${drop_in_paths:-none}" + if safe_profile_effective_unit_unoverridden; then unit_unoverridden="true"; fi unit_user=$(safe_profile_unit_property User) unit_user="${unit_user:-root}" ambient=$(safe_profile_unit_property AmbientCapabilities) @@ -1455,6 +1581,9 @@ safe_profile_inspect() { "Pulse safe-profile migration inspection (read-only)" \ "platform_supported=${supported}" \ "current_profile=${profile}" \ + "unit_fragment_path=${fragment_path}" \ + "unit_drop_in_paths=${drop_in_paths}" \ + "unit_unoverridden=${unit_unoverridden}" \ "unit_user=${unit_user}" \ "unit_groups=${groups}" \ "ambient_capabilities=${ambient}" \ @@ -1516,6 +1645,69 @@ safe_profile_manifest_value() { sed -n "s/^${key}=//p" "$manifest_file" 2>/dev/null | tail -1 } +safe_profile_snapshot_state_metadata() { + local metadata_file="${SAFE_PROFILE_TRANSACTION_DIR}/state-metadata.bin" + local path="" + local relative_path="" + local entry_type="file" + local uid="" gid="" mode="" + + [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || + fail "Refusing unsafe safe-profile state directory: ${STATE_DIR}" "$EXIT_GENERAL" + : > "$metadata_file" + chmod 0600 "$metadata_file" + while IFS= read -r -d '' path; do + if [[ "$path" == "$STATE_DIR" ]]; then + relative_path="." + else + relative_path="${path#${STATE_DIR%/}/}" + fi + entry_type="file" + [[ -d "$path" && ! -L "$path" ]] && entry_type="directory" + [[ -L "$path" ]] && entry_type="symlink" + uid=$(stat -c '%u' "$path") || return 1 + gid=$(stat -c '%g' "$path") || return 1 + mode=$(stat -c '%a' "$path") || return 1 + printf '%s\0%s\0%s\0%s\0%s\0' \ + "$relative_path" \ + "$uid" \ + "$gid" \ + "$mode" \ + "$entry_type" >> "$metadata_file" + done < <(find "$STATE_DIR" -xdev -print0) +} + +safe_profile_restore_state_metadata() { + local transaction_dir="$1" + local snapshot_state_dir="$2" + local metadata_file="${transaction_dir}/state-metadata.bin" + local relative_path="" uid="" gid="" mode="" entry_type="" destination="" + + [[ -f "$metadata_file" && ! -L "$metadata_file" ]] || return 1 + while IFS= read -r -d '' relative_path && + IFS= read -r -d '' uid && + IFS= read -r -d '' gid && + IFS= read -r -d '' mode && + IFS= read -r -d '' entry_type; do + if [[ "$relative_path" == "." ]]; then + destination="$snapshot_state_dir" + else + [[ -n "$relative_path" && "$relative_path" != /* && + "/${relative_path}/" != *"/../"* ]] || return 1 + destination="${snapshot_state_dir%/}/${relative_path}" + fi + # Runtime-owned state such as SQLite WAL/SHM files can legitimately + # disappear while the collector is stopped. Restore the identity of + # every surviving pre-migration entry without turning a vanished + # transient file into a rollback failure. + [[ -e "$destination" || -L "$destination" ]] || continue + chown -h "${uid}:${gid}" "$destination" || return 1 + if [[ "$entry_type" != "symlink" ]]; then + chmod "$mode" "$destination" || return 1 + fi + done < "$metadata_file" +} + safe_profile_begin_transaction() { local unit_path="$SAFE_PROFILE_COLLECTOR_UNIT" local prior_profile="" @@ -1525,6 +1717,8 @@ safe_profile_begin_transaction() { [[ -x "${INSTALL_DIR}/${BINARY_NAME}" && -f "$unit_path" ]] || fail "--safe-profile-apply requires an existing Linux systemd Pulse collector installation" "$EXIT_MISSING_ARGS" + safe_profile_effective_unit_unoverridden || + fail "Refusing safe-profile migration while the collector has a different effective FragmentPath or any systemd drop-in override; consolidate the effective unit first" "$EXIT_GENERAL" [[ ! -L "$SAFE_PROFILE_STATE_DIR" ]] || fail "Refusing symlinked safe-profile transaction directory: ${SAFE_PROFILE_STATE_DIR}" "$EXIT_GENERAL" mkdir -p "$SAFE_PROFILE_STATE_DIR" @@ -1539,7 +1733,7 @@ safe_profile_begin_transaction() { prior_profile=$(safe_profile_detect_current_profile) printf '%s\n' \ - "FORMAT_VERSION=1" \ + "FORMAT_VERSION=2" \ "PRIOR_PROFILE=${prior_profile}" \ "TARGET_PROFILE=typed-helper-monitoring-only" \ "STATE_DIR=${STATE_DIR}" \ @@ -1575,7 +1769,14 @@ safe_profile_begin_transaction() { safe_profile_snapshot_entry "${STATE_DIR%/}/runtime.token" runtime-token RUNTIME_TOKEN safe_profile_snapshot_entry "${STATE_DIR%/}/agent-id" agent-id AGENT_ID_FILE safe_profile_snapshot_entry "${STATE_DIR%/}/connection.env" connection-env CONNECTION_ENV + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-registered" proxmox-registered PROXMOX_REGISTERED + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-pve-registered" proxmox-pve-registered PROXMOX_PVE_REGISTERED + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-pbs-registered" proxmox-pbs-registered PROXMOX_PBS_REGISTERED + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-pve-registration-blocked" proxmox-pve-registration-blocked PROXMOX_PVE_REGISTRATION_BLOCKED + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-pbs-registration-blocked" proxmox-pbs-registration-blocked PROXMOX_PBS_REGISTRATION_BLOCKED + safe_profile_snapshot_entry "${STATE_DIR%/}/proxmox-detected-types" proxmox-detected-types PROXMOX_DETECTED_TYPES safe_profile_snapshot_entry "${PRIVILEGED_HELPER_CREDENTIAL_DIR}/token" protected-token PROTECTED_TOKEN + safe_profile_snapshot_state_metadata SAFE_PROFILE_TRANSACTION_ACTIVE="true" SAFE_PROFILE_TRANSACTION_COMMITTED="false" @@ -1604,6 +1805,7 @@ safe_profile_restore_transaction() { local prior_profile="" local docker_member="false" local current_tmp="" + local format_version="" SAFE_PROFILE_TRANSACTION_ACTIVE="false" case "$transaction_dir" in @@ -1611,7 +1813,8 @@ safe_profile_restore_transaction() { *) log_error "Refusing untrusted safe-profile transaction path: ${transaction_dir}"; return 1 ;; esac [[ -d "$transaction_dir" && ! -L "$transaction_dir" && -f "$manifest_file" && ! -L "$manifest_file" ]] || return 1 - [[ "$(safe_profile_manifest_value "$manifest_file" FORMAT_VERSION)" == "1" ]] || return 1 + format_version=$(safe_profile_manifest_value "$manifest_file" FORMAT_VERSION) + [[ "$format_version" == "1" || "$format_version" == "2" ]] || return 1 snapshot_state_dir=$(safe_profile_manifest_value "$manifest_file" STATE_DIR) [[ -n "$snapshot_state_dir" && "$snapshot_state_dir" == /* && "$snapshot_state_dir" != "/" ]] || return 1 prior_profile=$(safe_profile_manifest_value "$manifest_file" PRIOR_PROFILE) @@ -1631,6 +1834,14 @@ safe_profile_restore_transaction() { safe_profile_restore_entry "$transaction_dir" runtime-token "${snapshot_state_dir%/}/runtime.token" RUNTIME_TOKEN safe_profile_restore_entry "$transaction_dir" agent-id "${snapshot_state_dir%/}/agent-id" AGENT_ID_FILE safe_profile_restore_entry "$transaction_dir" connection-env "${snapshot_state_dir%/}/connection.env" CONNECTION_ENV + if [[ "$format_version" == "2" ]]; then + safe_profile_restore_entry "$transaction_dir" proxmox-registered "${snapshot_state_dir%/}/proxmox-registered" PROXMOX_REGISTERED + safe_profile_restore_entry "$transaction_dir" proxmox-pve-registered "${snapshot_state_dir%/}/proxmox-pve-registered" PROXMOX_PVE_REGISTERED + safe_profile_restore_entry "$transaction_dir" proxmox-pbs-registered "${snapshot_state_dir%/}/proxmox-pbs-registered" PROXMOX_PBS_REGISTERED + safe_profile_restore_entry "$transaction_dir" proxmox-pve-registration-blocked "${snapshot_state_dir%/}/proxmox-pve-registration-blocked" PROXMOX_PVE_REGISTRATION_BLOCKED + safe_profile_restore_entry "$transaction_dir" proxmox-pbs-registration-blocked "${snapshot_state_dir%/}/proxmox-pbs-registration-blocked" PROXMOX_PBS_REGISTRATION_BLOCKED + safe_profile_restore_entry "$transaction_dir" proxmox-detected-types "${snapshot_state_dir%/}/proxmox-detected-types" PROXMOX_DETECTED_TYPES + fi safe_profile_restore_entry "$transaction_dir" protected-token "${PRIVILEGED_HELPER_CREDENTIAL_DIR}/token" PROTECTED_TOKEN if [[ "$(safe_profile_manifest_value "$manifest_file" PROTECTED_DIR)" == "true" ]]; then chown "$(safe_profile_manifest_value "$manifest_file" PROTECTED_DIR_UID):$(safe_profile_manifest_value "$manifest_file" PROTECTED_DIR_GID)" "$PRIVILEGED_HELPER_CREDENTIAL_DIR" || return 1 @@ -1638,6 +1849,9 @@ safe_profile_restore_transaction() { else rmdir "$PRIVILEGED_HELPER_CREDENTIAL_DIR" 2>/dev/null || true fi + if [[ "$format_version" == "2" ]]; then + safe_profile_restore_state_metadata "$transaction_dir" "$snapshot_state_dir" || return 1 + fi rm -f "$PRIVILEGED_HELPER_SOCKET_PATH" docker_member=$(safe_profile_manifest_value "$manifest_file" DOCKER_MEMBER) @@ -1690,14 +1904,80 @@ safe_profile_remove_legacy_authority() { fi } +safe_profile_probe_helper_protocol() { + local request_id="installer-health-$$" + local request="" + local request_length=0 + local response_file="" + local response_size=0 + local response_length=0 + local response_body="" + local header_bytes="" + local header_one=0 header_two=0 header_three=0 header_four=0 + + command -v runuser >/dev/null 2>&1 || return 1 + id "$LEAST_PRIVILEGE_USER" >/dev/null 2>&1 || return 1 + [[ -S "$PRIVILEGED_HELPER_SOCKET_PATH" ]] || return 1 + request="{\"protocolVersion\":1,\"requestId\":\"${request_id}\",\"operation\":\"helper.health\",\"operationVersion\":1,\"deadlineMillis\":2000,\"payload\":{}}" + request_length=${#request} + [[ $request_length -gt 0 && $request_length -le 65536 ]] || return 1 + response_file=$(mktemp "${SAFE_PROFILE_TRANSACTION_DIR}/.helper-health.XXXXXX") || return 1 + chmod 0600 "$response_file" + if ! { + printf "\\$(printf '%03o' $((request_length / 16777216 % 256)))" + printf "\\$(printf '%03o' $((request_length / 65536 % 256)))" + printf "\\$(printf '%03o' $((request_length / 256 % 256)))" + printf "\\$(printf '%03o' $((request_length % 256)))" + printf '%s' "$request" + } | runuser -u "$LEAST_PRIVILEGE_USER" -- curl -sS --max-time 5 \ + --unix-socket "$PRIVILEGED_HELPER_SOCKET_PATH" --upload-file - telnet://localhost > "$response_file"; then + rm -f "$response_file" + return 1 + fi + response_size=$(wc -c < "$response_file" | tr -d ' ') + [[ "$response_size" =~ ^[0-9]+$ && $response_size -ge 5 ]] || { rm -f "$response_file"; return 1; } + header_bytes=$(od -An -tu1 -N4 "$response_file") + read -r header_one header_two header_three header_four <<< "$header_bytes" + response_length=$((header_one * 16777216 + header_two * 65536 + header_three * 256 + header_four)) + [[ $response_length -gt 0 && $response_length -le 1048576 && $response_size -eq $((response_length + 4)) ]] || { + rm -f "$response_file" + return 1 + } + response_body=$(dd if="$response_file" bs=1 skip=4 count="$response_length" 2>/dev/null) + rm -f "$response_file" + printf '%s' "$response_body" | grep -q '"protocolVersion"[[:space:]]*:[[:space:]]*1' || return 1 + printf '%s' "$response_body" | grep -q "\"requestId\"[[:space:]]*:[[:space:]]*\"${request_id}\"" || return 1 + printf '%s' "$response_body" | grep -q '"operation"[[:space:]]*:[[:space:]]*"helper.health"' || return 1 + printf '%s' "$response_body" | grep -q '"operationVersion"[[:space:]]*:[[:space:]]*1' || return 1 + printf '%s' "$response_body" | grep -q '"success"[[:space:]]*:[[:space:]]*true' || return 1 + printf '%s' "$response_body" | grep -q '"status"[[:space:]]*:[[:space:]]*"ok"' || return 1 +} + safe_profile_verify_declared_health() { local health_url="" + local attempt=0 + local local_health_ready="false" health_url=$(resolve_agent_health_url || true) [[ -n "$health_url" ]] || return 1 - curl -sf --max-time 2 "$health_url" >/dev/null 2>&1 || return 1 - systemctl is-active --quiet "${AGENT_NAME}.service" || return 1 - systemctl is-active --quiet "${PRIVILEGED_HELPER_NAME}.socket" || return 1 - verify_agent_server_registration_with_retry + + # systemd can report the new units active before the collector readiness + # endpoint and socket-activated helper have finished starting. Give that + # local floor a bounded window, then perform the (separately retried) + # authoritative server-registration proof exactly once. + for ((attempt = 1; attempt <= 30; attempt++)); do + if curl -sf --max-time 2 "$health_url" >/dev/null 2>&1 && + systemctl is-active --quiet "${AGENT_NAME}.service" && + systemctl is-active --quiet "${PRIVILEGED_HELPER_NAME}.socket" && + safe_profile_verify_effective_target && + safe_profile_probe_helper_protocol; then + local_health_ready="true" + break + fi + sleep 1 + done + [[ "$local_health_ready" == "true" ]] || return 1 + [[ -n "$SAFE_PROFILE_PRIOR_REGISTRATION_LAST_SEEN" ]] || return 1 + verify_agent_server_registration_with_retry "$SAFE_PROFILE_PRIOR_REGISTRATION_LAST_SEEN" } safe_profile_commit_transaction() { @@ -2574,6 +2854,29 @@ discover_rootless_container_runtime() { return 1 } +safe_profile_selected_rootless_runtime_usable() { + local collector_uid="" + local socket_uid="" + [[ -n "$ROOTLESS_RUNTIME_KIND" && -S "$ROOTLESS_RUNTIME_SOCKET_PATH" ]] || return 1 + collector_uid=$(id -u "$LEAST_PRIVILEGE_USER" 2>/dev/null || true) + socket_uid=$(stat -c '%u' "$ROOTLESS_RUNTIME_SOCKET_PATH" 2>/dev/null || true) + [[ -n "$collector_uid" && "$socket_uid" == "$collector_uid" ]] || return 1 + command -v runuser >/dev/null 2>&1 || return 1 + runuser -u "$LEAST_PRIVILEGE_USER" -- test -r "$ROOTLESS_RUNTIME_SOCKET_PATH" || return 1 + runuser -u "$LEAST_PRIVILEGE_USER" -- test -w "$ROOTLESS_RUNTIME_SOCKET_PATH" || return 1 +} + +safe_profile_apply_docker_degradation() { + [[ "$SAFE_PROFILE_ACTION" == "apply" && "$ENABLE_DOCKER" == "true" ]] || return 0 + if safe_profile_selected_rootless_runtime_usable; then + log_info "Safe-profile migration preserved container monitoring through the collector-owned ${ROOTLESS_RUNTIME_KIND} socket: ${ROOTLESS_RUNTIME_SOCKET_PATH}" + return 0 + fi + ENABLE_DOCKER="false" + DOCKER_EXPLICIT="true" + log_warn "Safe-profile migration disabled rootful Docker monitoring: the collector has no usable collector-owned rootless runtime. Container monitoring is an explicit migration degradation, not helper parity." +} + detect_kubernetes() { # If user already specified a kubeconfig path, just verify it exists if [[ -n "$KUBECONFIG_PATH" ]]; then @@ -4169,6 +4472,8 @@ if [[ "$ENABLE_DOCKER" == "true" ]] && discover_rootless_container_runtime; then fi fi +safe_profile_apply_docker_degradation + finalize_plist_env_block # --- Uninstall Logic --- @@ -5023,7 +5328,7 @@ if [[ "$ACTION_RUNNER_ENABLED" == "true" ]]; then download_verified_action_runner fi -chmod +x "$TMP_BIN" +chmod 0755 "$TMP_BIN" NEW_VERSION=$("$TMP_BIN" --version 2>/dev/null | head -1 || echo "unknown") # Compare versions with any leading "v" stripped so the agent binary's "v6.0.4" @@ -5079,6 +5384,19 @@ elif command -v systemctl >/dev/null 2>&1 && systemctl is-enabled --quiet "${AGE systemctl stop "${AGENT_NAME}" 2>/dev/null || true fi +if [[ "$SAFE_PROFILE_ACTION" == "apply" ]]; then + # Freeze the legacy collector before recording its server-side freshness + # marker. The replacement must advance this exact registration row after + # activation; an old row that merely still exists cannot commit migration. + stop_existing_agent_service || true + pkill -f "^${INSTALL_DIR}/${BINARY_NAME}([[:space:]]|$)" 2>/dev/null || true + AGENT_REGISTRATION_LAST_SEEN="" + if ! verify_agent_server_registration_with_retry || [[ -z "$AGENT_REGISTRATION_LAST_SEEN" ]]; then + fail "Safe-profile migration could not capture the stopped collector's server registration freshness marker; restoring the previous profile" "$EXIT_GENERAL" + fi + SAFE_PROFILE_PRIOR_REGISTRATION_LAST_SEEN="$AGENT_REGISTRATION_LAST_SEEN" +fi + if [[ "$UPDATE_ONLY" == "true" && "$UPGRADE_MODE" != "true" ]]; then fail "No existing Pulse Agent installation found to update. Use the install command instead." "$EXIT_MISSING_ARGS" fi @@ -5093,7 +5411,7 @@ if [[ "$SAFE_PROFILE_ACTION" == "apply" ]]; then mv "$SAFE_PROFILE_STAGED_COLLECTOR" "${INSTALL_DIR}/${BINARY_NAME}" else mv "$TMP_BIN" "${INSTALL_DIR}/${BINARY_NAME}" - chmod +x "${INSTALL_DIR}/${BINARY_NAME}" + chmod 0755 "${INSTALL_DIR}/${BINARY_NAME}" fi if [[ "$PRIVILEGED_HELPER_ENABLED" == "true" ]]; then diff --git a/scripts/installtests/install_sh_test.go b/scripts/installtests/install_sh_test.go index bf184cc4e..167184f5d 100644 --- a/scripts/installtests/install_sh_test.go +++ b/scripts/installtests/install_sh_test.go @@ -1802,6 +1802,23 @@ func TestInstallSHChecksDiskHeadroomBeforeDownload(t *testing.T) { } } +func TestInstallSHSetsAgentBinaryModeExplicitly(t *testing.T) { + content, err := os.ReadFile(repoFile("scripts", "install.sh")) + if err != nil { + t.Fatalf("read install.sh: %v", err) + } + + script := string(content) + for _, want := range []string{ + `chmod 0755 "$TMP_BIN"`, + `chmod 0755 "${INSTALL_DIR}/${BINARY_NAME}"`, + } { + if !strings.Contains(script, want) { + t.Fatalf("install.sh does not pin executable mode with %q", want) + } + } +} + // TestInstallSHWatchdogPathsUseRotatingAgentLog pins the logging half of issue // #1617: the QNAP and Unraid watchdog loops must not shell-append agent stdout // to an unrotated file on the RAM-backed root; the agent's own rotating writer @@ -5780,6 +5797,10 @@ func TestInstallSHActionRunnerIsSeparateOptInLifecycle(t *testing.T) { `write_action_runner_env_value "PULSE_AGENT_RUNNER_TOKEN_FILE" "$ACTION_RUNNER_TOKEN_FILE"`, `write_action_runner_env_value "PULSE_AGENT_RUNNER_AGENT_ID_FILE" "${STATE_DIR%/}/agent-id"`, `write_action_runner_env_value "PULSE_AGENT_RUNNER_HEALTH_FILE" "$ACTION_RUNNER_HEALTH_FILE"`, + `write_action_runner_env_value "PULSE_AGENT_RUNNER_HOSTNAME" "$runner_hostname"`, + `DELETE --data-binary "$payload"`, + `/api/agents/action-runner/credential`, + `Could not self-revoke the action-runner credential`, `/download/${ACTION_RUNNER_BINARY_NAME}?${DOWNLOAD_QUERY}`, `verify_download_signature "$TMP_ACTION_RUNNER_BIN" "$runner_signature"`, `install -o root -g root -m 0755 "$TMP_ACTION_RUNNER_BIN"`, @@ -5802,6 +5823,70 @@ func TestInstallSHActionRunnerIsSeparateOptInLifecycle(t *testing.T) { } } +func TestInstallSHActionRunnerSelfRevokeUsesPrivateCredential(t *testing.T) { + const token = "runner-secret-that-must-not-appear-in-output" + var gotAuthorization string + var gotMethod string + var gotBody map[string]string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuthorization = r.Header.Get("Authorization") + gotMethod = r.Method + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Errorf("decode revoke body: %v", err) + http.Error(w, "bad request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"success":true}`) + })) + defer server.Close() + + root := t.TempDir() + envFile := filepath.Join(root, "runner.env") + tokenFile := filepath.Join(root, "token") + agentIDFile := filepath.Join(root, "agent-id") + mustWrite(t, tokenFile, token+"\n") + mustWrite(t, agentIDFile, "agent-secure-runtime\n") + if err := os.Chmod(tokenFile, 0o600); err != nil { + t.Fatal(err) + } + mustWrite(t, envFile, strings.Join([]string{ + `PULSE_URL="` + server.URL + `"`, + `PULSE_AGENT_RUNNER_HOSTNAME="secure-runtime.lab"`, + `PULSE_AGENT_RUNNER_AGENT_ID_FILE="` + agentIDFile + `"`, + `PULSE_AGENT_RUNNER_TOKEN_FILE="` + tokenFile + `"`, + }, "\n")+"\n") + + script := ` +set -euo pipefail +ACTION_RUNNER_ENV_FILE="` + envFile + `" +stat() { + if [[ "$1" == "-c" && "$2" == "%a" ]]; then printf '600\n'; return 0; fi + command stat "$@" +} +` + extractInstallShellFunction(t, "read_action_runner_env_value") + ` +` + extractInstallShellFunction(t, "revoke_action_runner_credential") + ` +revoke_action_runner_credential +printf 'revoked\n' +` + out, err := exec.Command("bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("runner self-revoke: %v\n%s", err, out) + } + if strings.Contains(string(out), token) { + t.Fatalf("runner credential leaked in output: %s", out) + } + if gotMethod != http.MethodDelete { + t.Fatalf("revoke method = %q, want DELETE", gotMethod) + } + if gotAuthorization != "Bearer "+token { + t.Fatalf("authorization header mismatch") + } + if gotBody["agentId"] != "agent-secure-runtime" || gotBody["hostname"] != "secure-runtime.lab" { + t.Fatalf("revoke body = %#v", gotBody) + } +} + func TestInstallSHRendersHardenedActionRunnerUnit(t *testing.T) { root := t.TempDir() unitPath := filepath.Join(root, "pulse-agent-runner.service") diff --git a/scripts/installtests/safe_profile_migration_test.go b/scripts/installtests/safe_profile_migration_test.go index 1005d13ee..97f177065 100644 --- a/scripts/installtests/safe_profile_migration_test.go +++ b/scripts/installtests/safe_profile_migration_test.go @@ -1,6 +1,10 @@ package installtests import ( + "encoding/binary" + "encoding/json" + "io" + "net" "os" "os/exec" "path/filepath" @@ -13,14 +17,18 @@ func safeProfileInspectFunctions(t *testing.T) string { return extractInstallShellFunction(t, "safe_profile_platform_supported") + "\n" + extractInstallShellFunction(t, "safe_profile_detect_current_profile") + "\n" + extractInstallShellFunction(t, "safe_profile_unit_property") + "\n" + + extractInstallShellFunction(t, "safe_profile_effective_unit_unoverridden") + "\n" + extractInstallShellFunction(t, "safe_profile_inspect") } func safeProfileTransactionFunctions(t *testing.T) string { t.Helper() return extractInstallShellFunction(t, "safe_profile_detect_current_profile") + "\n" + + extractInstallShellFunction(t, "safe_profile_effective_unit_unoverridden") + "\n" + extractInstallShellFunction(t, "safe_profile_snapshot_entry") + "\n" + extractInstallShellFunction(t, "safe_profile_manifest_value") + "\n" + + extractInstallShellFunction(t, "safe_profile_snapshot_state_metadata") + "\n" + + extractInstallShellFunction(t, "safe_profile_restore_state_metadata") + "\n" + extractInstallShellFunction(t, "safe_profile_begin_transaction") + "\n" + extractInstallShellFunction(t, "safe_profile_restore_entry") + "\n" + extractInstallShellFunction(t, "safe_profile_restore_transaction") + "\n" + @@ -52,7 +60,12 @@ log_error() { printf 'ERROR:%s\n' "$*" >&2; } uname() { printf 'Linux\n'; } systemctl() { if [[ "$1" == show ]]; then - case "$4" in User) printf 'root\n' ;; AmbientCapabilities) printf 'CAP_SETUID CAP_SETGID\n' ;; esac + case "$4" in + User) printf 'root\n' ;; + AmbientCapabilities) printf 'CAP_SETUID CAP_SETGID\n' ;; + FragmentPath) printf '%s\n' "$SAFE_PROFILE_COLLECTOR_UNIT" ;; + DropInPaths) printf '\n' ;; + esac fi } id() { if [[ "${1:-}" == -nG ]]; then printf 'root docker\n'; fi; return 0; } @@ -65,6 +78,7 @@ safe_profile_inspect } for _, want := range []string{ "platform_supported=true", "current_profile=legacy-root-command-capable", + "unit_fragment_path=" + unit, "unit_drop_in_paths=none", "unit_unoverridden=true", "unit_user=root", "unit_groups=root docker", "ambient_capabilities=CAP_SETUID CAP_SETGID", "provider_docker=true", "provider_proxmox=true", "collector_commands=true", "action_runner_independent=true", "target_profile=typed-helper-monitoring-only", @@ -101,16 +115,22 @@ test "$SAFE_PROFILE_TRANSACTION_COMMITTED" = true t.Run("failure rollback", func(t *testing.T) { root := t.TempDir() harness := safeProfileHarness(t, root, true) + ` +printf 'transient-wal\n' > "$STATE_DIR/cache/runtime.db-wal" +chmod 0600 "$STATE_DIR/cache/runtime.db-wal" safe_profile_begin_transaction transaction="$SAFE_PROFILE_TRANSACTION_DIR" +rm -f "$STATE_DIR/cache/runtime.db-wal" printf 'new-binary\n' > "$INSTALL_DIR/$BINARY_NAME" printf '[Service]\nUser=pulse-agent\nEnvironment=PULSE_AGENT_HELPER_SOCKET=/run/pulse-agent/helper.sock\n' > "$SAFE_PROFILE_COLLECTOR_UNIT" printf 'typed-helper\n' > "$PRIVILEGED_HELPER_BINARY_PATH" printf 'helper-unit\n' > "$PRIVILEGED_HELPER_SERVICE_UNIT" printf 'helper-socket\n' > "$PRIVILEGED_HELPER_SOCKET_UNIT" rm -f "$STATE_DIR/token" "$STATE_DIR/runtime.token" +rm -f "$STATE_DIR/proxmox-registered" "$STATE_DIR/proxmox-pve-registered" "$STATE_DIR/proxmox-pbs-registered" +rm -f "$STATE_DIR/proxmox-pve-registration-blocked" "$STATE_DIR/proxmox-pbs-registration-blocked" "$STATE_DIR/proxmox-detected-types" printf 'changed-agent-id\n' > "$STATE_DIR/agent-id" printf 'changed-connection\n' > "$STATE_DIR/connection.env" +chmod 0777 "$STATE_DIR" "$STATE_DIR/cache" "$STATE_DIR/cache/sample" mkdir -p "$PRIVILEGED_HELPER_CREDENTIAL_DIR" printf 'moved-monitoring-token\n' > "$PRIVILEGED_HELPER_CREDENTIAL_DIR/token" printf 'runner-still-independent\n' > "$ACTION_RUNNER_SENTINEL" @@ -121,6 +141,16 @@ cmp "$STATE_DIR/token" "$EXPECTED_DIR/state-token" cmp "$STATE_DIR/runtime.token" "$EXPECTED_DIR/runtime-token" cmp "$STATE_DIR/agent-id" "$EXPECTED_DIR/agent-id" cmp "$STATE_DIR/connection.env" "$EXPECTED_DIR/connection-env" +grep -q '^legacy-generic$' "$STATE_DIR/proxmox-registered" +grep -q '^legacy-pve$' "$STATE_DIR/proxmox-pve-registered" +grep -q '^legacy-pbs$' "$STATE_DIR/proxmox-pbs-registered" +grep -q '^legacy-pve-blocked$' "$STATE_DIR/proxmox-pve-registration-blocked" +grep -q '^legacy-pbs-blocked$' "$STATE_DIR/proxmox-pbs-registration-blocked" +grep -q '^pve,pbs$' "$STATE_DIR/proxmox-detected-types" +test "$(stat -c '%a' "$STATE_DIR")" = 750 +test "$(stat -c '%a' "$STATE_DIR/cache")" = 710 +test "$(stat -c '%a' "$STATE_DIR/cache/sample")" = 640 +test ! -e "$STATE_DIR/cache/runtime.db-wal" test ! -e "$PRIVILEGED_HELPER_BINARY_PATH" test ! -e "$PRIVILEGED_HELPER_SERVICE_UNIT" test ! -e "$PRIVILEGED_HELPER_SOCKET_UNIT" @@ -170,12 +200,35 @@ func TestSafeProfileApplyRequiresReadinessHelperAndRegistration(t *testing.T) { set -euo pipefail AGENT_NAME=pulse-agent PRIVILEGED_HELPER_NAME=pulse-agent-helper +SAFE_PROFILE_PRIOR_REGISTRATION_LAST_SEEN=before resolve_agent_health_url() { printf 'http://127.0.0.1:9191/readyz\n'; } +sleep() { :; } curl() { return 0; } systemctl() { return 0; } -verify_agent_server_registration_with_retry() { return 0; } +safe_profile_verify_effective_target() { return 0; } +safe_profile_probe_helper_protocol() { return 0; } +verify_agent_server_registration_with_retry() { [[ "${1:-}" == before ]]; } ` + gate + ` safe_profile_verify_declared_health +curl_attempts=0 +helper_attempts=0 +curl() { + curl_attempts=$((curl_attempts + 1)) + (( curl_attempts >= 3 )) +} +safe_profile_probe_helper_protocol() { + helper_attempts=$((helper_attempts + 1)) + (( helper_attempts >= 2 )) +} +safe_profile_verify_declared_health +test "$curl_attempts" = 4 +test "$helper_attempts" = 2 +safe_profile_probe_helper_protocol() { return 1; } +if safe_profile_verify_declared_health; then + echo 'helper protocol failure was accepted' >&2 + exit 1 +fi +safe_profile_probe_helper_protocol() { return 0; } verify_agent_server_registration_with_retry() { return 1; } if safe_profile_verify_declared_health; then echo 'registration failure was accepted' >&2 @@ -187,6 +240,189 @@ fi } } +func TestSafeProfileFailsClosedOnEffectiveSystemdOverrides(t *testing.T) { + for _, tc := range []struct { + name, property, value string + }{ + {name: "drop-in", property: "DropInPaths", value: "/etc/systemd/system/pulse-agent.service.d/override.conf"}, + {name: "different fragment", property: "FragmentPath", value: "/usr/lib/systemd/system/pulse-agent.service"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + harness := safeProfileHarness(t, root, false) + old := tc.property + `) printf '%s\n' "$SAFE_PROFILE_COLLECTOR_UNIT" ;;` + if tc.property == "DropInPaths" { + old = `DropInPaths) printf '\n' ;;` + } + replacement := tc.property + `) printf '%s\n' '` + tc.value + `' ;;` + harness = strings.Replace(harness, old, replacement, 1) + "\nsafe_profile_begin_transaction\n" + out, err := exec.Command("bash", "-c", harness).CombinedOutput() + if err == nil || !strings.Contains(string(out), "Refusing safe-profile migration") { + t.Fatalf("override was not rejected: err=%v\n%s", err, out) + } + }) + } +} + +func TestSafeProfileRegistrationMustAdvanceLastSeen(t *testing.T) { + functions := extractInstallShellFunction(t, "verify_agent_server_registration") + script := ` +set -euo pipefail +AGENT_ID=agent-1 +HOSTNAME_OVERRIDE= +PULSE_URL=https://pulse.example +INSECURE=false +CURL_CA_BUNDLE= +AGENT_REGISTRATION_LAST_SEEN= +url_encode() { printf '%s' "$1"; } +curl_with_pulse_token() { printf '%s\n200\n' "$LOOKUP_BODY"; } +` + functions + ` +LOOKUP_BODY='{"agent":{"id":"agent-1","lastSeen":"2026-08-30T10:00:00Z"}}' +if verify_agent_server_registration '2026-08-30T10:00:00Z'; then + echo 'stale registration was accepted' >&2 + exit 1 +fi +LOOKUP_BODY='{"agent":{"id":"agent-1","lastSeen":"2026-08-30T10:00:31Z"}}' +verify_agent_server_registration '2026-08-30T10:00:00Z' +test "$AGENT_REGISTRATION_LAST_SEEN" = '2026-08-30T10:00:31Z' +` + if out, err := exec.Command("bash", "-c", script).CombinedOutput(); err != nil { + t.Fatalf("fresh registration rehearsal: %v\n%s", err, out) + } +} + +func TestSafeProfileHelperProtocolProbeValidatesFramedHealth(t *testing.T) { + for _, success := range []bool{true, false} { + t.Run(map[bool]string{true: "healthy", false: "typed failure"}[success], func(t *testing.T) { + root, err := os.MkdirTemp("/tmp", "pulse-helper-probe-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + socketPath := filepath.Join(root, "helper.sock") + listener, err := net.Listen("unix", socketPath) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + serverErr := make(chan error, 1) + go func() { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + serverErr <- acceptErr + return + } + defer conn.Close() + var header [4]byte + if _, readErr := io.ReadFull(conn, header[:]); readErr != nil { + serverErr <- readErr + return + } + requestBody := make([]byte, binary.BigEndian.Uint32(header[:])) + if _, readErr := io.ReadFull(conn, requestBody); readErr != nil { + serverErr <- readErr + return + } + var request map[string]any + if decodeErr := json.Unmarshal(requestBody, &request); decodeErr != nil { + serverErr <- decodeErr + return + } + response, marshalErr := json.Marshal(map[string]any{ + "protocolVersion": 1, "requestId": request["requestId"], + "operation": "helper.health", "operationVersion": 1, + "success": success, "result": map[string]any{"status": "ok", "protocolVersion": 1}, + }) + if marshalErr != nil { + serverErr <- marshalErr + return + } + binary.BigEndian.PutUint32(header[:], uint32(len(response))) + _, writeErr := conn.Write(append(header[:], response...)) + serverErr <- writeErr + }() + + fakeBin := filepath.Join(root, "bin") + mustMkdirAll(t, fakeBin) + runuser := filepath.Join(fakeBin, "runuser") + if err := os.WriteFile(runuser, []byte("#!/bin/sh\nwhile [ \"$#\" -gt 0 ] && [ \"$1\" != -- ]; do shift; done\nshift\nexec \"$@\"\n"), 0o755); err != nil { + t.Fatal(err) + } + script := ` +set -euo pipefail +PATH="` + fakeBin + `:$PATH" +LEAST_PRIVILEGE_USER=pulse-agent +PRIVILEGED_HELPER_SOCKET_PATH="` + socketPath + `" +SAFE_PROFILE_TRANSACTION_DIR="` + root + `" +id() { return 0; } +` + extractInstallShellFunction(t, "safe_profile_probe_helper_protocol") + "\n" + if success { + script += "safe_profile_probe_helper_protocol\n" + } else { + script += "if safe_profile_probe_helper_protocol; then exit 91; fi\n" + } + out, runErr := exec.Command("bash", "-c", script).CombinedOutput() + if runErr != nil { + t.Fatalf("protocol probe: %v\n%s", runErr, out) + } + if err := <-serverErr; err != nil { + t.Fatal(err) + } + }) + } +} + +func TestSafeProfileExitTrapRestoresUncommittedTransaction(t *testing.T) { + root := t.TempDir() + harness := safeProfileHarness(t, root, true) + "\n" + extractInstallShellFunction(t, "cleanup") + ` +TMP_FILES=() +trap cleanup EXIT +safe_profile_begin_transaction +printf 'uncommitted-binary\n' > "$INSTALL_DIR/$BINARY_NAME" +rm -f "$STATE_DIR/proxmox-registered" +printf 'runner-unchanged\n' > "$ACTION_RUNNER_SENTINEL" +exit 23 +` + out, err := exec.Command("bash", "-c", harness).CombinedOutput() + if exitErr, ok := err.(*exec.ExitError); !ok || exitErr.ExitCode() != 23 { + t.Fatalf("exit trap status=%v\n%s", err, out) + } + assertFileBody(t, filepath.Join(root, "bin", "pulse-agent"), "old-binary\n") + assertFileBody(t, filepath.Join(root, "state", "proxmox-registered"), "legacy-generic\n") + assertFileBody(t, filepath.Join(root, "runner-sentinel"), "runner-unchanged\n") +} + +func TestSafeProfileDockerDegradationRequiresCollectorOwnedRootlessRuntime(t *testing.T) { + function := extractInstallShellFunction(t, "safe_profile_apply_docker_degradation") + for _, usable := range []bool{true, false} { + script := ` +set -euo pipefail +SAFE_PROFILE_ACTION=apply +ENABLE_DOCKER=true +DOCKER_EXPLICIT=false +ROOTLESS_RUNTIME_KIND=docker +ROOTLESS_RUNTIME_SOCKET_PATH=/run/user/991/docker.sock +log_info() { :; } +log_warn() { printf '%s\n' "$*"; } +safe_profile_selected_rootless_runtime_usable() { return ` + map[bool]string{true: "0", false: "1"}[usable] + `; } +` + function + ` +safe_profile_apply_docker_degradation +printf 'enabled=%s explicit=%s\n' "$ENABLE_DOCKER" "$DOCKER_EXPLICIT" +` + out, err := exec.Command("bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("docker degradation: %v\n%s", err, out) + } + want := "enabled=false explicit=true" + if usable { + want = "enabled=true explicit=false" + } + if !strings.Contains(string(out), want) { + t.Fatalf("usable=%v output missing %q:\n%s", usable, want, out) + } + } +} + func safeProfileHarness(t *testing.T, root string, dockerMember bool) string { t.Helper() binDir := filepath.Join(root, "bin") @@ -197,19 +433,36 @@ func safeProfileHarness(t *testing.T, root string, dockerMember bool) string { expectedDir := filepath.Join(root, "expected") mustMkdirAll(t, binDir, unitDir, helperDir, stateDir, expectedDir) files := map[string]string{ - filepath.Join(binDir, "pulse-agent"): "old-binary\n", - filepath.Join(unitDir, "pulse-agent.service"): "[Service]\nUser=root\nAmbientCapabilities=CAP_SETUID CAP_SETGID\nExecStart=/bin/pulse-agent --enable-commands\n", - filepath.Join(root, "sudoers"): "legacy sudo grant\n", - filepath.Join(helperDir, "smartctl"): "legacy smart wrapper\n", - filepath.Join(helperDir, "pct"): "legacy pct wrapper\n", - filepath.Join(stateDir, "token"): "monitoring-token\n", - filepath.Join(stateDir, "runtime.token"): "runtime-monitoring-token\n", - filepath.Join(stateDir, "agent-id"): "stable-agent-id\n", - filepath.Join(stateDir, "connection.env"): "PULSE_URL='https://pulse.example'\n", + filepath.Join(binDir, "pulse-agent"): "old-binary\n", + filepath.Join(unitDir, "pulse-agent.service"): "[Service]\nUser=root\nAmbientCapabilities=CAP_SETUID CAP_SETGID\nExecStart=/bin/pulse-agent --enable-commands\n", + filepath.Join(root, "sudoers"): "legacy sudo grant\n", + filepath.Join(helperDir, "smartctl"): "legacy smart wrapper\n", + filepath.Join(helperDir, "pct"): "legacy pct wrapper\n", + filepath.Join(stateDir, "token"): "monitoring-token\n", + filepath.Join(stateDir, "runtime.token"): "runtime-monitoring-token\n", + filepath.Join(stateDir, "agent-id"): "stable-agent-id\n", + filepath.Join(stateDir, "connection.env"): "PULSE_URL='https://pulse.example'\n", + filepath.Join(stateDir, "cache", "sample"): "cached-state\n", + filepath.Join(stateDir, "proxmox-registered"): "legacy-generic\n", + filepath.Join(stateDir, "proxmox-pve-registered"): "legacy-pve\n", + filepath.Join(stateDir, "proxmox-pbs-registered"): "legacy-pbs\n", + filepath.Join(stateDir, "proxmox-pve-registration-blocked"): "legacy-pve-blocked\n", + filepath.Join(stateDir, "proxmox-pbs-registration-blocked"): "legacy-pbs-blocked\n", + filepath.Join(stateDir, "proxmox-detected-types"): "pve,pbs\n", } for path, body := range files { + mustMkdirAll(t, filepath.Dir(path)) mustWrite(t, path, body) } + if err := os.Chmod(stateDir, 0o750); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(stateDir, "cache"), 0o710); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(stateDir, "cache", "sample"), 0o640); err != nil { + t.Fatal(err) + } for source, name := range map[string]string{ filepath.Join(binDir, "pulse-agent"): "collector-binary", filepath.Join(unitDir, "pulse-agent.service"): "collector-unit", @@ -257,10 +510,36 @@ EXIT_MISSING_ARGS=2 log_info() { :; } log_error() { printf 'ERROR:%s\n' "$*" >&2; } fail() { printf 'FAIL:%s\n' "$1" >&2; return "${2:-1}"; } -systemctl() { case "${1:-}" in is-active|is-enabled) return 0 ;; *) return 0 ;; esac; } +systemctl() { + case "${1:-}" in + show) + case "${4:-}" in + FragmentPath) printf '%s\n' "$SAFE_PROFILE_COLLECTOR_UNIT" ;; + DropInPaths) printf '\n' ;; + esac + ;; + is-active|is-enabled) return 0 ;; + esac + return 0 +} getent() { [[ "${1:-}" == group && "${2:-}" == docker ]]; } id() { if [[ "${1:-}" == -nG ]]; then printf '` + membership + `\n'; fi; return 0; } gpasswd() { printf 'gpasswd %s\n' "$*" >> "$CALL_LOG"; } +stat() { + if [[ "${1:-}" == -c ]]; then + if /usr/bin/stat -c "$2" "$3" 2>/dev/null; then + return + fi + case "$2" in + %u) /usr/bin/stat -f '%u' "$3" ;; + %g) /usr/bin/stat -f '%g' "$3" ;; + %a) /usr/bin/stat -f '%Lp' "$3" ;; + *) return 1 ;; + esac + return + fi + /usr/bin/stat "$@" +} ` + safeProfileTransactionFunctions(t) + "\n" } diff --git a/scripts/installtests/secure_runtime_systemd_lab_test.go b/scripts/installtests/secure_runtime_systemd_lab_test.go new file mode 100644 index 000000000..7ce442f3e --- /dev/null +++ b/scripts/installtests/secure_runtime_systemd_lab_test.go @@ -0,0 +1,979 @@ +//go:build !windows + +package installtests + +// TestSecureRuntimeSystemdLab is intentionally excluded from ordinary test +// runs. It installs services and users into a disposable Linux systemd host. +// Build the release-shaped inputs on the host, then copy or mount them into a +// dedicated Colima profile before opting in: +// +// mkdir -p .lab-artifacts +// v1_ldflags="$(./scripts/release_ldflags.sh agent --version 6.2.0-lab.1)" +// v2_ldflags="$(./scripts/release_ldflags.sh agent --version 6.2.0-lab.2)" +// CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$v1_ldflags" -o .lab-artifacts/pulse-agent-v1 ./cmd/pulse-agent +// CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$v2_ldflags" -o .lab-artifacts/pulse-agent-v2 ./cmd/pulse-agent +// CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o .lab-artifacts/pulse-agent-helper ./cmd/pulse-agent-helper +// CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go test -c -o .lab-artifacts/installtests-linux-arm64.test ./scripts/installtests +// colima start pulse-agent-qual --activate=false --mount "$PWD:w" +// colima ssh -p pulse-agent-qual -- sudo sh -c \ +// 'printf "%s\n" "PULSE_SECURE_RUNTIME_SYSTEMD_LAB=disposable-v1" > /etc/pulse-secure-runtime-lab' +// repo="$PWD" +// colima ssh -p pulse-agent-qual -- sudo env \ +// PULSE_SECURE_RUNTIME_SYSTEMD_LAB=1 \ +// PULSE_SECURE_RUNTIME_COLLECTOR_V1="$repo/.lab-artifacts/pulse-agent-v1" \ +// PULSE_SECURE_RUNTIME_COLLECTOR_V2="$repo/.lab-artifacts/pulse-agent-v2" \ +// PULSE_SECURE_RUNTIME_HELPER="$repo/.lab-artifacts/pulse-agent-helper" \ +// PULSE_SECURE_RUNTIME_RECEIPT=/tmp/secure-runtime-receipt.json \ +// sh -c 'cd "$1/scripts/installtests" && exec "$1/.lab-artifacts/installtests-linux-arm64.test" -test.run "^TestSecureRuntimeSystemdLab$" -test.count=1 -test.v' sh "$repo" +// +// Use the VM's GOARCH in place of arm64 when qualifying another architecture. +// The test never creates or deletes a VM; lifecycle remains a host-side choice. + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +const ( + secureRuntimeLabOptIn = "PULSE_SECURE_RUNTIME_SYSTEMD_LAB" + secureRuntimeLabMarkerPath = "/etc/pulse-secure-runtime-lab" + secureRuntimeLabMarkerValue = "PULSE_SECURE_RUNTIME_SYSTEMD_LAB=disposable-v1" + secureRuntimeLabToken = "8fa728ed4cbfa7466947e747627b51e2464ed3f69966ed6394e36a82d85d7d31" + secureRuntimeLabAgentID = "secure-runtime-systemd-lab" + secureRuntimeLabHostname = "pulse-secure-runtime-lab" +) + +var secureRuntimeInstalledPaths = []string{ + "/usr/local/bin/pulse-agent", + "/usr/local/lib/pulse-agent/pulse-agent-helper", + "/etc/systemd/system/pulse-agent.service", + "/etc/systemd/system/pulse-agent.service.d", + "/etc/systemd/system/pulse-agent-helper.service", + "/etc/systemd/system/pulse-agent-helper.socket", + "/var/lib/pulse-agent", + "/var/lib/pulse-agent-helper", + "/var/lib/pulse-agent-profile", +} + +type secureRuntimeLabReport struct { + ReceivedAt time.Time + AgentID string + AgentVersion string + Hostname string + RunningAsRoot bool + ServiceUser string + Authority string + TypedHelper bool + CommandsEnabled bool +} + +type secureRuntimeLabFixture struct { + mu sync.Mutex + collector []byte + helper []byte + serverVersion string + reports []secureRuntimeLabReport + lastSeen time.Time + freezeLastSeen bool + authFailures int + requestFailures []string +} + +func (f *secureRuntimeLabFixture) setCollector(artifact []byte) { + f.mu.Lock() + defer f.mu.Unlock() + f.collector = append([]byte(nil), artifact...) +} + +func (f *secureRuntimeLabFixture) setServerVersion(version string) { + f.mu.Lock() + defer f.mu.Unlock() + f.serverVersion = version +} + +func (f *secureRuntimeLabFixture) setFrozen(frozen bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.freezeLastSeen = frozen +} + +func (f *secureRuntimeLabFixture) snapshot() ([]secureRuntimeLabReport, time.Time, int, []string) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]secureRuntimeLabReport(nil), f.reports...), f.lastSeen, f.authFailures, append([]string(nil), f.requestFailures...) +} + +func (f *secureRuntimeLabFixture) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == "/api/version": + f.mu.Lock() + version := f.serverVersion + f.mu.Unlock() + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{"version": version}) + case r.URL.Path == "/download/pulse-agent": + f.serveArtifact(w, r, false) + case r.URL.Path == "/download/pulse-agent-helper": + f.serveArtifact(w, r, true) + case r.URL.Path == "/api/health": + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{"status": "ok"}) + case r.URL.Path == "/api/agents/agent/report": + f.handleReport(w, r) + case r.URL.Path == "/api/agents/docker/report": + if !f.authorized(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{"success": true}) + case r.URL.Path == "/api/agents/agent/lookup": + f.handleLookup(w, r) + case strings.HasPrefix(r.URL.Path, "/api/agents/agent/") && strings.HasSuffix(r.URL.Path, "/config"): + if !f.authorized(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{ + "success": true, + "agentId": secureRuntimeLabAgentID, + "config": map[string]any{"settings": map[string]any{}}, + }) + default: + http.NotFound(w, r) + } +} + +func (f *secureRuntimeLabFixture) authorized(r *http.Request) bool { + bearer := r.Header.Get("Authorization") + valid := r.Header.Get("X-API-Token") == secureRuntimeLabToken && + (bearer == "" || bearer == "Bearer "+secureRuntimeLabToken) && + r.URL.Query().Get("token") == "" + if !valid { + f.mu.Lock() + f.authFailures++ + f.requestFailures = append(f.requestFailures, r.Method+" "+r.URL.Path+": invalid credential transport") + f.mu.Unlock() + } + return valid +} + +func (f *secureRuntimeLabFixture) serveArtifact(w http.ResponseWriter, r *http.Request, helper bool) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + f.mu.Lock() + artifact := append([]byte(nil), f.collector...) + if helper { + artifact = append([]byte(nil), f.helper...) + } + f.mu.Unlock() + sum := sha256.Sum256(artifact) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("X-Checksum-Sha256", hex.EncodeToString(sum[:])) + w.Header().Set("Content-Length", strconv.Itoa(len(artifact))) + if r.Method == http.MethodGet { + _, _ = w.Write(artifact) + } +} + +func (f *secureRuntimeLabFixture) handleReport(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !f.authorized(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + body := io.Reader(http.MaxBytesReader(w, r.Body, 32<<20)) + if r.Header.Get("Content-Encoding") == "gzip" { + compressed, err := gzip.NewReader(body) + if err != nil { + http.Error(w, "invalid compressed report", http.StatusBadRequest) + return + } + defer compressed.Close() + body = compressed + } + var payload struct { + Agent struct { + ID string `json:"id"` + Version string `json:"version"` + Hostname string `json:"hostname"` + CommandsEnabled bool `json:"commandsEnabled"` + Privilege *struct { + RunningAsRoot bool `json:"runningAsRoot"` + ServiceUser string `json:"serviceUser"` + Authority string `json:"commandAuthority"` + TypedHelper bool `json:"typedHelper"` + } `json:"privilege"` + } `json:"agent"` + Host struct { + Hostname string `json:"hostname"` + } `json:"host"` + } + if err := json.NewDecoder(body).Decode(&payload); err != nil { + http.Error(w, "invalid report", http.StatusBadRequest) + return + } + report := secureRuntimeLabReport{ + ReceivedAt: time.Now().UTC(), + AgentID: strings.TrimSpace(payload.Agent.ID), + AgentVersion: strings.TrimSpace(payload.Agent.Version), + Hostname: strings.TrimSpace(payload.Agent.Hostname), + CommandsEnabled: payload.Agent.CommandsEnabled, + } + if report.Hostname == "" { + report.Hostname = strings.TrimSpace(payload.Host.Hostname) + } + if payload.Agent.Privilege != nil { + report.RunningAsRoot = payload.Agent.Privilege.RunningAsRoot + report.ServiceUser = payload.Agent.Privilege.ServiceUser + report.Authority = payload.Agent.Privilege.Authority + report.TypedHelper = payload.Agent.Privilege.TypedHelper + } + f.mu.Lock() + f.reports = append(f.reports, report) + if !f.freezeLastSeen { + f.lastSeen = report.ReceivedAt + } + serverVersion := f.serverVersion + f.mu.Unlock() + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{ + "success": true, + "agentId": secureRuntimeLabAgentID, + "serverVersion": serverVersion, + }) +} + +func (f *secureRuntimeLabFixture) handleLookup(w http.ResponseWriter, r *http.Request) { + if !f.authorized(r) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + f.mu.Lock() + lastSeen := f.lastSeen + f.mu.Unlock() + if lastSeen.IsZero() { + writeSecureRuntimeJSON(w, http.StatusNotFound, map[string]any{"success": false}) + return + } + writeSecureRuntimeJSON(w, http.StatusOK, map[string]any{ + "success": true, + "agent": map[string]any{ + "id": secureRuntimeLabAgentID, + "hostname": secureRuntimeLabHostname, + "lastSeen": lastSeen.Format(time.RFC3339Nano), + }, + }) +} + +func writeSecureRuntimeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +type secureRuntimeFileIdentity struct { + Present bool + Mode os.FileMode + UID uint32 + GID uint32 + Hash string +} + +type secureRuntimeStableIdentity map[string]secureRuntimeFileIdentity + +func secureRuntimeStableSnapshot(t *testing.T) secureRuntimeStableIdentity { + t.Helper() + paths := []string{ + "/usr/local/bin/pulse-agent", + "/etc/systemd/system/pulse-agent.service", + "/var/lib/pulse-agent/agent-id", + "/var/lib/pulse-agent/connection.env", + "/var/lib/pulse-agent/token", + "/var/lib/pulse-agent/runtime.token", + } + result := make(secureRuntimeStableIdentity, len(paths)) + for _, path := range paths { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + result[path] = secureRuntimeFileIdentity{} + continue + } + if err != nil { + t.Fatalf("inspect stable identity %s: %v", path, err) + } + if !info.Mode().IsRegular() { + t.Fatalf("stable identity path is not a regular file: %s (%s)", path, info.Mode()) + } + content, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read stable identity %s: %v", path, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("read ownership for %s", path) + } + sum := sha256.Sum256(content) + result[path] = secureRuntimeFileIdentity{ + Present: true, + Mode: info.Mode().Perm(), + UID: stat.Uid, + GID: stat.Gid, + Hash: hex.EncodeToString(sum[:]), + } + } + return result +} + +type secureRuntimeScenarioResult struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Detail string `json:"detail,omitempty"` +} + +type secureRuntimeLabReceipt struct { + SchemaVersion int `json:"schema_version"` + CompletedAt string `json:"completed_at"` + SourceHashes map[string]string `json:"source_hashes"` + ArtifactHashes map[string]string `json:"artifact_hashes"` + OSRelease string `json:"os_release"` + Kernel string `json:"kernel"` + SystemdVersion string `json:"systemd_version"` + Architecture string `json:"architecture"` + CollectorServiceUser string `json:"collector_service_user"` + CollectorProcessUID int `json:"collector_process_uid"` + CollectorAuthority string `json:"collector_authority"` + AmbientCapabilitiesNone bool `json:"ambient_capabilities_none"` + HelperProtocolHealthy bool `json:"helper_protocol_healthy"` + StateIdentityPreserved bool `json:"state_identity_preserved"` + DockerDegraded bool `json:"docker_degraded"` + ReportCount int `json:"report_count"` + FirstReportAt string `json:"first_report_at"` + LastReportAt string `json:"last_report_at"` + Scenarios []secureRuntimeScenarioResult `json:"scenarios"` +} + +func TestSecureRuntimeSystemdLab(t *testing.T) { + if os.Getenv(secureRuntimeLabOptIn) != "1" { + t.Skip("set PULSE_SECURE_RUNTIME_SYSTEMD_LAB=1 only inside a disposable systemd VM") + } + secureRuntimeRequireDisposableHost(t) + + collectorV1 := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_COLLECTOR_V1") + collectorV2 := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_COLLECTOR_V2") + helper := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_HELPER") + collectorV1Version := secureRuntimeArtifactVersion(t, "PULSE_SECURE_RUNTIME_COLLECTOR_V1") + collectorV2Version := secureRuntimeArtifactVersion(t, "PULSE_SECURE_RUNTIME_COLLECTOR_V2") + if collectorV1Version == collectorV2Version { + t.Fatalf("collector V1 and V2 must have distinct --version output, both reported %q", collectorV1Version) + } + installerPath, err := filepath.Abs(repoFile("scripts", "install.sh")) + if err != nil { + t.Fatalf("resolve installer path: %v", err) + } + installer := secureRuntimeReadFile(t, installerPath) + + fixture := &secureRuntimeLabFixture{collector: collectorV1, helper: helper, serverVersion: collectorV1Version} + server := httptest.NewServer(fixture) + defer server.Close() + + receipt := secureRuntimeLabReceipt{ + SchemaVersion: 1, + SourceHashes: map[string]string{ + "scripts/install.sh": secureRuntimeHash(installer), + "scripts/installtests/secure_runtime_systemd_lab_test.go": secureRuntimeHash(secureRuntimeReadFile(t, repoFile("scripts", "installtests", "secure_runtime_systemd_lab_test.go"))), + }, + ArtifactHashes: map[string]string{"collector_v1": secureRuntimeHash(collectorV1), "collector_v2": secureRuntimeHash(collectorV2), "helper": secureRuntimeHash(helper)}, + Architecture: runtime.GOARCH, + } + receipt.OSRelease = strings.TrimSpace(string(secureRuntimeReadFile(t, "/etc/os-release"))) + receipt.Kernel = secureRuntimeCommand(t, 10*time.Second, "uname", "-srvmo") + receipt.SystemdVersion = strings.SplitN(secureRuntimeCommand(t, 10*time.Second, "systemctl", "--version"), "\n", 2)[0] + pass := func(name, detail string) { + receipt.Scenarios = append(receipt.Scenarios, secureRuntimeScenarioResult{Name: name, Passed: true, Detail: detail}) + } + + initialArgs := []string{"--enable-commands", "--command-authority", "command-capable"} + dockerInitiallyAvailable := secureRuntimeRootfulDockerAvailable() + if dockerInitiallyAvailable { + initialArgs = append(initialArgs, "--enable-docker") + } + secureRuntimeRunInstaller(t, installerPath, server.URL, initialArgs...) + secureRuntimeWaitForReports(t, fixture, 1, 45*time.Second) + secureRuntimeAssertRootCommandProfile(t) + dockerInitiallyEnabled := secureRuntimeCollectorHasArgument("--enable-docker") + if dockerInitiallyAvailable && !dockerInitiallyEnabled { + t.Fatal("rootful Docker was available but the requested legacy profile did not enable Docker monitoring") + } + pass("legacy_root_command_capable_install", fmt.Sprintf("root collector installed; docker_enabled=%t", dockerInitiallyEnabled)) + + beforeInspect := secureRuntimeStableSnapshot(t) + inspectOutput := secureRuntimeRunInstaller(t, installerPath, server.URL, "--safe-profile-inspect") + if !strings.Contains(inspectOutput, "current_profile=legacy-root-command-capable") || + !strings.Contains(inspectOutput, "target_profile=typed-helper-monitoring-only") { + t.Fatalf("read-only inspection did not describe legacy and target profiles:\n%s", inspectOutput) + } + if afterInspect := secureRuntimeStableSnapshot(t); !secureRuntimeIdentitiesEqual(beforeInspect, afterInspect) { + t.Fatal("--safe-profile-inspect mutated installer-owned stable files") + } + pass("read_only_inspect", "stable installer-owned files unchanged") + + dropInDir := "/etc/systemd/system/pulse-agent.service.d" + dropInPath := filepath.Join(dropInDir, "secure-runtime-lab.conf") + if err := os.MkdirAll(dropInDir, 0o755); err != nil { + t.Fatalf("create systemd drop-in directory: %v", err) + } + if err := os.WriteFile(dropInPath, []byte("[Service]\nUser=root\n"), 0o644); err != nil { + t.Fatalf("write systemd drop-in: %v", err) + } + dropInPresent := true + t.Cleanup(func() { + if dropInPresent { + _ = os.Remove(dropInPath) + _ = os.Remove(dropInDir) + _ = exec.Command("systemctl", "daemon-reload").Run() + } + }) + secureRuntimeCommand(t, 20*time.Second, "systemctl", "daemon-reload") + beforeDropIn := secureRuntimeStableSnapshot(t) + failedOutput, failedErr := secureRuntimeRunInstallerError(t, installerPath, server.URL, "--safe-profile-apply") + if failedErr == nil || !strings.Contains(failedOutput, "systemd drop-in override") { + t.Fatalf("safe apply did not fail closed on a systemd drop-in: err=%v\n%s", failedErr, failedOutput) + } + if afterDropIn := secureRuntimeStableSnapshot(t); !secureRuntimeIdentitiesEqual(beforeDropIn, afterDropIn) { + t.Fatal("drop-in rejection mutated installer-owned stable files") + } + if err := os.Remove(dropInPath); err != nil { + t.Fatalf("remove systemd drop-in: %v", err) + } + _ = os.Remove(dropInDir) + dropInPresent = false + secureRuntimeCommand(t, 20*time.Second, "systemctl", "daemon-reload") + pass("drop_in_fail_closed_rehearsal", "migration rejected before stable installer-owned files changed") + + legacyBaseline := secureRuntimeStableSnapshot(t) + reportsBeforeApply, preApplyLastSeen, _, _ := fixture.snapshot() + fixture.setCollector(collectorV2) + applyOutput := secureRuntimeRunInstaller(t, installerPath, server.URL, "--safe-profile-apply") + secureRuntimeWaitForReports(t, fixture, len(reportsBeforeApply)+1, 45*time.Second) + _, postApplyLastSeen, _, _ := fixture.snapshot() + if !postApplyLastSeen.After(preApplyLastSeen) { + t.Fatalf("safe apply committed without fresh server registration: before=%s after=%s", preApplyLastSeen, postApplyLastSeen) + } + secureRuntimeAssertSafeProfile(t) + secureRuntimeAssertHelperProtocol(t) + dockerDegraded := dockerInitiallyEnabled && !secureRuntimeCollectorHasArgument("--enable-docker") + if dockerInitiallyEnabled && !secureRuntimeCollectorOwnedRootlessAvailable(t) { + if !dockerDegraded || !strings.Contains(applyOutput, "disabled rootful Docker monitoring") { + t.Fatalf("safe migration did not make rootful Docker degradation explicit:\n%s", applyOutput) + } + } + pass("safe_profile_apply", "fresh server lastSeen, least-privilege identity, typed helper health") + + reportsBeforeRollback, _, _, _ := fixture.snapshot() + secureRuntimeRunInstaller(t, installerPath, server.URL, "--safe-profile-rollback") + secureRuntimeWaitForReports(t, fixture, len(reportsBeforeRollback)+1, 45*time.Second) + secureRuntimeAssertRootCommandProfile(t) + secureRuntimeWaitForStableIdentity(t, legacyBaseline, 10*time.Second, "explicit rollback") + pass("explicit_safe_profile_rollback", "legacy binary, unit, authority, and service identity restored") + + automaticRollbackBaseline := secureRuntimeStableSnapshot(t) + fixture.setFrozen(true) + failedOutput, failedErr = secureRuntimeRunInstallerError(t, installerPath, server.URL, "--safe-profile-apply") + fixture.setFrozen(false) + if failedErr == nil || !strings.Contains(failedOutput, "restoring the previous profile") { + t.Fatalf("stale lastSeen safe apply unexpectedly succeeded: err=%v\n%s", failedErr, failedOutput) + } + secureRuntimeWaitForStableIdentity(t, automaticRollbackBaseline, 10*time.Second, "automatic failure rollback") + secureRuntimeAssertRootCommandProfile(t) + pass("automatic_failure_rollback", "frozen lastSeen prevented commit and restored binary/unit/state identity") + + fixture.setCollector(collectorV2) + reportsBeforeUpdate, _, _, _ := fixture.snapshot() + secureRuntimeRunInstaller(t, installerPath, server.URL, "--update") + secureRuntimeWaitForReports(t, fixture, len(reportsBeforeUpdate)+1, 45*time.Second) + secureRuntimeAssertRootCommandProfile(t) + if got := secureRuntimeHash(secureRuntimeReadFile(t, "/usr/local/bin/pulse-agent")); got != secureRuntimeHash(collectorV2) { + t.Fatalf("ordinary update collector hash = %s, want v2 hash", got) + } + pass("ordinary_update_non_migration", "binary updated while root command-capable profile remained unchanged") + + fixture.setServerVersion(collectorV2Version) + reportsBeforeFinalApply, preFinalLastSeen, _, _ := fixture.snapshot() + secureRuntimeRunInstaller(t, installerPath, server.URL, "--safe-profile-apply") + secureRuntimeWaitForReports(t, fixture, len(reportsBeforeFinalApply)+1, 45*time.Second) + _, finalLastSeen, authFailures, requestFailures := fixture.snapshot() + if !finalLastSeen.After(preFinalLastSeen) { + t.Fatalf("final safe apply did not advance server lastSeen: before=%s after=%s", preFinalLastSeen, finalLastSeen) + } + if authFailures != 0 || len(requestFailures) != 0 { + t.Fatalf("fixture observed credential/request failures: auth=%d failures=%v", authFailures, requestFailures) + } + secureRuntimeAssertSafeProfile(t) + secureRuntimeAssertHelperProtocol(t) + reportsBeforeContinuity, _, _, _ := fixture.snapshot() + secureRuntimeWaitForReports(t, fixture, len(reportsBeforeContinuity)+1, 20*time.Second) + pass("final_safe_profile_apply", "collector continued reporting after committed migration") + + reports, _, _, _ := fixture.snapshot() + if len(reports) == 0 { + t.Fatal("no reports recorded") + } + for i, report := range reports { + if report.AgentID != secureRuntimeLabAgentID { + t.Fatalf("report %d changed collector identity: %q", i, report.AgentID) + } + if report.Hostname != secureRuntimeLabHostname { + t.Fatalf("report %d changed collector hostname: %q", i, report.Hostname) + } + } + latest := reports[len(reports)-1] + if latest.RunningAsRoot || latest.ServiceUser != "pulse-agent" || latest.Authority != "monitoring-only" || !latest.TypedHelper || latest.CommandsEnabled { + t.Fatalf("final report privilege posture = %+v", latest) + } + + receipt.CompletedAt = time.Now().UTC().Format(time.RFC3339Nano) + receipt.CollectorServiceUser = secureRuntimeSystemdProperty(t, "User") + receipt.CollectorProcessUID = secureRuntimeCollectorProcessUID(t) + receipt.CollectorAuthority = latest.Authority + receipt.AmbientCapabilitiesNone = strings.TrimSpace(secureRuntimeSystemdProperty(t, "AmbientCapabilities")) == "" + receipt.HelperProtocolHealthy = true + receipt.StateIdentityPreserved = true + receipt.DockerDegraded = dockerDegraded + receipt.ReportCount = len(reports) + receipt.FirstReportAt = reports[0].ReceivedAt.Format(time.RFC3339Nano) + receipt.LastReportAt = reports[len(reports)-1].ReceivedAt.Format(time.RFC3339Nano) + secureRuntimeWriteReceipt(t, receipt) +} + +func secureRuntimeRequireDisposableHost(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" { + t.Fatalf("destructive secure-runtime lab requires Linux, got %s", runtime.GOOS) + } + if os.Geteuid() != 0 { + t.Fatal("destructive secure-runtime lab requires EUID 0") + } + comm, err := os.ReadFile("/proc/1/comm") + if err != nil || strings.TrimSpace(string(comm)) != "systemd" { + t.Fatalf("destructive secure-runtime lab requires systemd as PID 1: comm=%q err=%v", strings.TrimSpace(string(comm)), err) + } + marker, err := os.ReadFile(secureRuntimeLabMarkerPath) + if err != nil || strings.TrimSpace(string(marker)) != secureRuntimeLabMarkerValue { + t.Fatalf("refusing host mutation: %s must contain exactly %q", secureRuntimeLabMarkerPath, secureRuntimeLabMarkerValue) + } + for _, path := range secureRuntimeInstalledPaths { + if _, err := os.Lstat(path); err == nil { + t.Fatalf("refusing already-installed host: %s exists", path) + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("inspect existing Pulse path %s: %v", path, err) + } + } + if out, err := exec.Command("systemctl", "show", "pulse-agent.service", "--property=LoadState", "--value").CombinedOutput(); err == nil && strings.TrimSpace(string(out)) != "not-found" { + t.Fatalf("refusing host with loaded pulse-agent.service: %s", strings.TrimSpace(string(out))) + } +} + +func secureRuntimeReadArtifact(t *testing.T, envName string) []byte { + t.Helper() + path := strings.TrimSpace(os.Getenv(envName)) + if path == "" || !filepath.IsAbs(path) { + t.Fatalf("%s must name an absolute caller-built artifact path", envName) + } + info, err := os.Lstat(path) + if err != nil { + t.Fatalf("inspect %s: %v", envName, err) + } + if !info.Mode().IsRegular() || info.Mode()&0o111 == 0 { + t.Fatalf("%s must be a regular executable file: %s (%s)", envName, path, info.Mode()) + } + artifact := secureRuntimeReadFile(t, path) + if len(artifact) < 4 || !bytes.Equal(artifact[:4], []byte{0x7f, 'E', 'L', 'F'}) { + t.Fatalf("%s is not an ELF executable: %s", envName, path) + } + return artifact +} + +func secureRuntimeArtifactVersion(t *testing.T, envName string) string { + t.Helper() + path := strings.TrimSpace(os.Getenv(envName)) + if path == "" || !filepath.IsAbs(path) { + t.Fatalf("%s must name an absolute caller-built artifact path", envName) + } + version := secureRuntimeCommand(t, 10*time.Second, path, "--version") + version = strings.TrimSpace(strings.SplitN(version, "\n", 2)[0]) + if version == "" || version == "unknown" { + t.Fatalf("%s returned unusable --version output %q", envName, version) + } + return version +} + +func secureRuntimeReadFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return data +} + +func secureRuntimeHash(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func secureRuntimeRunInstaller(t *testing.T, installerPath, fixtureURL string, scenarioArgs ...string) string { + t.Helper() + out, err := secureRuntimeRunInstallerError(t, installerPath, fixtureURL, scenarioArgs...) + if err != nil { + t.Fatalf("installer %s failed: %v\n%s", strings.Join(scenarioArgs, " "), err, out) + } + return out +} + +func secureRuntimeRunInstallerError(t *testing.T, installerPath, fixtureURL string, scenarioArgs ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + args := []string{installerPath} + switch { + case len(scenarioArgs) == 1 && scenarioArgs[0] == "--safe-profile-inspect": + args = append(args, scenarioArgs...) + case len(scenarioArgs) == 1 && scenarioArgs[0] == "--safe-profile-rollback": + args = append(args, scenarioArgs...) + default: + tokenFile := filepath.Join(t.TempDir(), "monitoring-token") + if err := os.WriteFile(tokenFile, []byte(secureRuntimeLabToken+"\n"), 0o600); err != nil { + t.Fatalf("write private fixture token: %v", err) + } + args = append(args, + "--url", fixtureURL, + "--token-file", tokenFile, + "--interval", "2s", + "--agent-id", secureRuntimeLabAgentID, + "--hostname", secureRuntimeLabHostname, + "--state-dir", "/var/lib/pulse-agent", + "--insecure", + "--non-interactive", + ) + args = append(args, scenarioArgs...) + } + cmd := exec.CommandContext(ctx, "bash", args...) + cmd.Env = os.Environ() + out, err := cmd.CombinedOutput() + if ctx.Err() != nil { + return string(out), fmt.Errorf("installer timed out: %w", ctx.Err()) + } + if bytes.Contains(out, []byte(secureRuntimeLabToken)) { + t.Fatal("installer output exposed the monitoring credential") + } + return string(out), err +} + +func secureRuntimeWaitForReports(t *testing.T, fixture *secureRuntimeLabFixture, count int, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + reports, _, _, _ := fixture.snapshot() + if len(reports) >= count { + return + } + time.Sleep(200 * time.Millisecond) + } + reports, _, _, failures := fixture.snapshot() + t.Fatalf("timed out waiting for %d reports; got %d (fixture failures: %v)", count, len(reports), failures) +} + +func secureRuntimeCommand(t *testing.T, timeout time.Duration, name string, args ...string) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s %s: %v\n%s", name, strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +func secureRuntimeSystemdProperty(t *testing.T, property string) string { + t.Helper() + return secureRuntimeCommand(t, 10*time.Second, "systemctl", "show", "pulse-agent.service", "--property="+property, "--value") +} + +func secureRuntimeCollectorHasArgument(argument string) bool { + out, err := exec.Command("systemctl", "show", "pulse-agent.service", "--property=ExecStart", "--value").Output() + if err != nil { + return false + } + for _, field := range strings.Fields(string(out)) { + if strings.Trim(field, `{ };"`) == argument { + return true + } + } + return false +} + +func secureRuntimeCollectorProcessUID(t *testing.T) int { + t.Helper() + pidText := secureRuntimeSystemdProperty(t, "MainPID") + pid, err := strconv.Atoi(strings.TrimSpace(pidText)) + if err != nil || pid <= 1 { + t.Fatalf("invalid pulse-agent MainPID %q", pidText) + } + status := string(secureRuntimeReadFile(t, fmt.Sprintf("/proc/%d/status", pid))) + for _, line := range strings.Split(status, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "Uid:" { + uid, err := strconv.Atoi(fields[1]) + if err != nil { + t.Fatalf("parse collector UID from %q: %v", line, err) + } + return uid + } + } + t.Fatalf("collector process status omitted UID: %s", status) + return -1 +} + +func secureRuntimeAssertRootCommandProfile(t *testing.T) { + t.Helper() + if user := secureRuntimeSystemdProperty(t, "User"); user != "root" && user != "" { + t.Fatalf("legacy collector systemd User = %q, want root", user) + } + if !secureRuntimeCollectorHasArgument("--enable-commands") { + t.Fatal("legacy collector is not command-capable") + } + if uid := secureRuntimeCollectorProcessUID(t); uid != 0 { + t.Fatalf("legacy collector process UID = %d, want 0", uid) + } + secureRuntimeCommand(t, 10*time.Second, "systemctl", "is-active", "pulse-agent.service") +} + +func secureRuntimeAssertSafeProfile(t *testing.T) { + t.Helper() + if user := secureRuntimeSystemdProperty(t, "User"); user != "pulse-agent" { + t.Fatalf("safe collector systemd User = %q, want pulse-agent", user) + } + if ambient := strings.TrimSpace(secureRuntimeSystemdProperty(t, "AmbientCapabilities")); ambient != "" { + t.Fatalf("safe collector AmbientCapabilities = %q, want none", ambient) + } + if secureRuntimeCollectorHasArgument("--enable-commands") { + t.Fatal("safe collector retained --enable-commands") + } + if environment := secureRuntimeSystemdProperty(t, "Environment"); !strings.Contains(environment, "PULSE_AGENT_HELPER_SOCKET=/run/pulse-agent/helper.sock") { + t.Fatalf("safe collector lacks typed-helper environment: %s", environment) + } + uidText := secureRuntimeCommand(t, 10*time.Second, "id", "-u", "pulse-agent") + wantUID, err := strconv.Atoi(uidText) + if err != nil { + t.Fatalf("parse pulse-agent UID %q: %v", uidText, err) + } + if uid := secureRuntimeCollectorProcessUID(t); uid != wantUID { + t.Fatalf("safe collector process UID = %d, want %d", uid, wantUID) + } + groups := strings.Fields(secureRuntimeCommand(t, 10*time.Second, "id", "-nG", "pulse-agent")) + for _, group := range groups { + if group == "docker" { + t.Fatal("safe collector retained rootful docker group membership") + } + } + for _, path := range []string{"/usr/local/bin/pulse-agent", "/usr/local/lib/pulse-agent/pulse-agent-helper"} { + identity := secureRuntimeStableFileIdentity(t, path) + if identity.UID != 0 || identity.GID != 0 || identity.Mode != 0o755 { + t.Fatalf("%s identity = uid:%d gid:%d mode:%#o, want root:root 0755", path, identity.UID, identity.GID, identity.Mode) + } + } + secureRuntimeCommand(t, 10*time.Second, "systemctl", "is-active", "pulse-agent.service") + secureRuntimeCommand(t, 10*time.Second, "systemctl", "is-active", "pulse-agent-helper.socket") +} + +func secureRuntimeStableFileIdentity(t *testing.T, path string) secureRuntimeFileIdentity { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("read ownership for %s", path) + } + return secureRuntimeFileIdentity{Present: true, Mode: info.Mode().Perm(), UID: stat.Uid, GID: stat.Gid} +} + +func secureRuntimeAssertHelperProtocol(t *testing.T) { + t.Helper() + requestID := "secure-runtime-lab-health" + request, err := json.Marshal(map[string]any{ + "protocolVersion": 1, + "requestId": requestID, + "operation": "helper.health", + "operationVersion": 1, + "deadlineMillis": 2000, + "payload": map[string]any{}, + }) + if err != nil { + t.Fatalf("marshal helper health request: %v", err) + } + frame := make([]byte, 4+len(request)) + binary.BigEndian.PutUint32(frame[:4], uint32(len(request))) + copy(frame[4:], request) + requestHandle, err := os.CreateTemp("/tmp", "pulse-helper-health-*.frame") + if err != nil { + t.Fatalf("create helper health frame: %v", err) + } + requestFile := requestHandle.Name() + t.Cleanup(func() { _ = os.Remove(requestFile) }) + if _, err := requestHandle.Write(frame); err != nil { + _ = requestHandle.Close() + t.Fatalf("write helper health frame: %v", err) + } + if err := requestHandle.Chmod(0o644); err != nil { + _ = requestHandle.Close() + t.Fatalf("make helper health frame collector-readable: %v", err) + } + if err := requestHandle.Close(); err != nil { + t.Fatalf("close helper health frame: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + output, err := exec.CommandContext(ctx, "runuser", "-u", "pulse-agent", "--", "curl", "-sS", "--max-time", "5", "--unix-socket", "/run/pulse-agent/helper.sock", "--upload-file", requestFile, "telnet://localhost").Output() + if err != nil { + t.Fatalf("typed helper protocol health: %v", err) + } + if len(output) < 5 { + t.Fatalf("typed helper returned short frame: %d bytes", len(output)) + } + length := int(binary.BigEndian.Uint32(output[:4])) + if length != len(output)-4 { + t.Fatalf("typed helper frame length = %d, payload bytes = %d", length, len(output)-4) + } + var response struct { + ProtocolVersion int `json:"protocolVersion"` + RequestID string `json:"requestId"` + Operation string `json:"operation"` + OperationVersion int `json:"operationVersion"` + Success bool `json:"success"` + Result struct { + Status string `json:"status"` + } `json:"result"` + } + if err := json.Unmarshal(output[4:], &response); err != nil { + t.Fatalf("decode typed helper health: %v", err) + } + if response.ProtocolVersion != 1 || response.RequestID != requestID || response.Operation != "helper.health" || response.OperationVersion != 1 || !response.Success || response.Result.Status != "ok" { + t.Fatalf("typed helper health response failed correlation/health: %+v", response) + } + secureRuntimeCommand(t, 10*time.Second, "systemctl", "is-active", "pulse-agent-helper.service") +} + +func secureRuntimeIdentitiesEqual(left, right secureRuntimeStableIdentity) bool { + if len(left) != len(right) { + return false + } + for path, identity := range left { + if right[path] != identity { + return false + } + } + return true +} + +func secureRuntimeWaitForStableIdentity(t *testing.T, want secureRuntimeStableIdentity, timeout time.Duration, scenario string) { + t.Helper() + deadline := time.Now().Add(timeout) + var got secureRuntimeStableIdentity + for time.Now().Before(deadline) { + got = secureRuntimeStableSnapshot(t) + if secureRuntimeIdentitiesEqual(want, got) { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("%s did not converge to the pre-migration binary/unit/state identity: want=%#v got=%#v", scenario, want, got) +} + +func secureRuntimeRootfulDockerAvailable() bool { + if info, err := os.Stat("/var/run/docker.sock"); err != nil || info.Mode()&os.ModeSocket == 0 { + return false + } + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock") + }, + } + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 2 * time.Second} + response, err := client.Get("http://docker/_ping") + if err != nil { + return false + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 16)) + return err == nil && response.StatusCode == http.StatusOK && strings.TrimSpace(string(body)) == "OK" +} + +func secureRuntimeCollectorOwnedRootlessAvailable(t *testing.T) bool { + t.Helper() + uidText, err := exec.Command("id", "-u", "pulse-agent").Output() + if err != nil { + return false + } + uid := strings.TrimSpace(string(uidText)) + for _, path := range []string{filepath.Join("/run/user", uid, "docker.sock"), filepath.Join("/run/user", uid, "podman", "podman.sock")} { + if info, err := os.Stat(path); err == nil && info.Mode()&os.ModeSocket != 0 { + return true + } + } + return false +} + +func secureRuntimeWriteReceipt(t *testing.T, receipt secureRuntimeLabReceipt) { + t.Helper() + path := strings.TrimSpace(os.Getenv("PULSE_SECURE_RUNTIME_RECEIPT")) + if path == "" { + return + } + if !filepath.IsAbs(path) { + t.Fatalf("PULSE_SECURE_RUNTIME_RECEIPT must be an absolute path: %s", path) + } + encoded, err := json.MarshalIndent(receipt, "", " ") + if err != nil { + t.Fatalf("marshal secure-runtime receipt: %v", err) + } + if bytes.Contains(encoded, []byte(secureRuntimeLabToken)) || bytes.Contains(bytes.ToLower(encoded), []byte("token")) { + t.Fatal("refusing to write a receipt containing credential material or token-labelled fields") + } + encoded = append(encoded, '\n') + temporary := path + ".tmp" + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create receipt directory: %v", err) + } + if err := os.WriteFile(temporary, encoded, 0o600); err != nil { + t.Fatalf("write receipt temporary file: %v", err) + } + if err := os.Rename(temporary, path); err != nil { + t.Fatalf("publish receipt: %v", err) + } +}