Harden secure agent runtime transitions

This commit is contained in:
Pulse Test
2026-08-30 01:41:57 +01:00
parent 5d3571dbe8
commit d06ffc233d
39 changed files with 3818 additions and 151 deletions
+47
View File
@@ -376,6 +376,53 @@ func (s *Server) connectionForOrganization(organizationID, agentID string) (*age
return nil, false
}
// InvalidateActionRunnerSession closes exactly the currently admitted typed
// action-runner session identified by admission. A stale rotation result must
// never evict a replacement session that has already registered for the same
// tenant and host identity.
func (s *Server) InvalidateActionRunnerSession(admission AgentAdmission) bool {
if s == nil {
return false
}
expected := AgentAdmission{
OrganizationID: normalizeOrganizationID(admission.OrganizationID),
TokenID: strings.TrimSpace(admission.TokenID),
AgentID: strings.TrimSpace(admission.AgentID),
Hostname: strings.TrimSpace(admission.Hostname),
RuntimeRole: strings.TrimSpace(admission.RuntimeRole),
ActionCapability: strings.TrimSpace(admission.ActionCapability),
}
if expected.TokenID == "" || expected.AgentID == "" || expected.Hostname == "" ||
expected.RuntimeRole != RuntimeRoleActionRunner || expected.ActionCapability != ActionCapabilityTypedV1 {
return false
}
key := agentSessionKey(expected.OrganizationID, expected.AgentID)
s.mu.Lock()
current, ok := s.agents[key]
if !ok || current == nil || !sameActionRunnerAdmission(current.admission, expected) {
s.mu.Unlock()
return false
}
delete(s.agents, key)
s.mu.Unlock()
current.signalDone()
if current.conn != nil {
_ = current.conn.Close()
}
return true
}
func sameActionRunnerAdmission(current, expected AgentAdmission) bool {
return normalizeOrganizationID(current.OrganizationID) == expected.OrganizationID &&
strings.TrimSpace(current.TokenID) == expected.TokenID &&
strings.TrimSpace(current.AgentID) == expected.AgentID &&
(strings.EqualFold(strings.TrimSpace(current.Hostname), expected.Hostname) ||
unifiedresources.HostnamesEquivalent(current.Hostname, expected.Hostname)) &&
strings.TrimSpace(current.RuntimeRole) == RuntimeRoleActionRunner &&
strings.TrimSpace(current.ActionCapability) == ActionCapabilityTypedV1
}
func requireLegacyFullTrustConnection(ac *agentConn, operation string) error {
if ac == nil {
return fmt.Errorf("agent connection is unavailable")
@@ -1239,6 +1239,73 @@ func TestRevokedAdmissionInvalidatesStaleSocketBeforeDispatch(t *testing.T) {
}
}
func TestInvalidateActionRunnerSessionClosesExactSessionAndUnblocksInflightDispatch(t *testing.T) {
admission := AgentAdmission{
OrganizationID: "org-a",
TokenID: "runner-token",
AgentID: "agent-a",
Hostname: "node.example",
RuntimeRole: RuntimeRoleActionRunner,
ActionCapability: ActionCapabilityTypedV1,
}
s := NewServerWithAdmissionValidator(func(token, _, _ string) (AgentAdmission, bool) {
return admission, token == admission.TokenID
}, func(candidate AgentAdmission) bool { return candidate == admission })
ts := newWSServer(t, s)
defer ts.Close()
conn, _, err := dialAgentExecWebSocket(t, ts.URL)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
wsWriteMessage(t, conn, mustNewMessage(t, MsgTypeAgentRegister, "", AgentRegisterPayload{
AgentID: admission.AgentID, Hostname: admission.Hostname, Token: admission.TokenID,
RuntimeRole: admission.RuntimeRole, ActionCapability: admission.ActionCapability,
OperationReceiptVersion: operationreceipt.ProtocolVersion,
}))
if ack := wsReadRegisteredPayload(t, conn); !ack.Success {
t.Fatalf("registration failed: %s", ack.Message)
}
ctx := WithOrganizationID(context.Background(), admission.OrganizationID)
result := make(chan error, 1)
go func() {
_, dispatchErr := s.ExecuteHostUpdate(ctx, admission.AgentID, HostUpdatePayload{
RequestID: "rotation-inflight", ActionID: "action-a", Operation: HostUpdateOperationInstall,
ExpectedInventoryHash: "sha256:" + strings.Repeat("a", 64), Timeout: 30,
})
result <- dispatchErr
}()
message, err := wsReadRawMessageWithTimeout(conn, 2*time.Second)
if err != nil || message.Type != MsgTypeHostUpdate {
t.Fatalf("in-flight dispatch = %#v, %v", message, err)
}
wrong := admission
wrong.TokenID = "replacement-token"
if s.InvalidateActionRunnerSession(wrong) {
t.Fatal("mismatched replacement identity evicted the current session")
}
if !s.InvalidateActionRunnerSession(admission) {
t.Fatal("exact action-runner admission was not invalidated")
}
select {
case dispatchErr := <-result:
if dispatchErr == nil || !strings.Contains(dispatchErr.Error(), "disconnected") {
t.Fatalf("in-flight dispatch error = %v", dispatchErr)
}
case <-time.After(2 * time.Second):
t.Fatal("in-flight dispatch did not unblock after session invalidation")
}
if _, err := s.ExecuteHostUpdate(ctx, admission.AgentID, HostUpdatePayload{
RequestID: "after-rotation", ActionID: "action-b", Operation: HostUpdateOperationInstall,
ExpectedInventoryHash: "sha256:" + strings.Repeat("b", 64), Timeout: 1,
}); err == nil || !strings.Contains(err.Error(), "not connected") {
t.Fatalf("new dispatch after invalidation = %v", err)
}
}
// registerCancelTestAgent registers agent "a1" over the websocket harness and
// returns after the registration ack has been read.
func registerCancelTestAgent(t *testing.T, s *Server, tsURL string) *cancelTestConn {
+1
View File
@@ -40,6 +40,7 @@ const (
OperationContainerInventory = "container.inventory"
OperationAgentUpdateStage = "agent_update.stage"
OperationAgentUpdateActivate = "agent_update.activate"
OperationAgentUpdateCommit = "agent_update.commit"
OperationAgentUpdateRollback = "agent_update.rollback"
OperationVersion1 = 1
)
+1
View File
@@ -33,6 +33,7 @@ type ContainerProvider interface {
type UpdateProvider interface {
Stage(context.Context, UpdateStageRequest) (UpdateStageResult, error)
Activate(context.Context, UpdateActivateRequest) (UpdateResult, error)
Commit(context.Context, UpdateCommitRequest) (UpdateResult, error)
Rollback(context.Context, UpdateRollbackRequest) (UpdateResult, error)
}
+14
View File
@@ -131,6 +131,20 @@ func NewRegistryWithProviders(smart SMARTProvider, proxmox ProxmoxProvider, prov
}
return marshalOperationResult(result)
})
registry.add(OperationAgentUpdateCommit, OperationVersion1, providers.Updates != nil, func(ctx context.Context, payload json.RawMessage) (json.RawMessage, *ResponseError) {
var request UpdateCommitRequest
if err := decodePayload(payload, &request); err != nil {
return nil, invalidPayloadError(err)
}
if providers.Updates == nil {
return nil, unavailableError("agent update provider is not configured")
}
result, err := providers.Updates.Commit(ctx, request)
if err != nil {
return nil, providerError(err)
}
return marshalOperationResult(result)
})
registry.add(OperationAgentUpdateRollback, OperationVersion1, providers.Updates != nil, func(ctx context.Context, payload json.RawMessage) (json.RawMessage, *ResponseError) {
var request UpdateRollbackRequest
if err := decodePayload(payload, &request); err != nil {
+17 -2
View File
@@ -34,6 +34,7 @@ func (f fakeContainerProvider) Inventory(ctx context.Context) (json.RawMessage,
type fakeUpdateProvider struct {
stage func(context.Context, UpdateStageRequest) (UpdateStageResult, error)
activate func(context.Context, UpdateActivateRequest) (UpdateResult, error)
commit func(context.Context, UpdateCommitRequest) (UpdateResult, error)
rollback func(context.Context, UpdateRollbackRequest) (UpdateResult, error)
}
@@ -45,6 +46,10 @@ func (f fakeUpdateProvider) Activate(ctx context.Context, request UpdateActivate
return f.activate(ctx, request)
}
func (f fakeUpdateProvider) Commit(ctx context.Context, request UpdateCommitRequest) (UpdateResult, error) {
return f.commit(ctx, request)
}
func (f fakeUpdateProvider) Rollback(ctx context.Context, request UpdateRollbackRequest) (UpdateResult, error) {
return f.rollback(ctx, request)
}
@@ -187,6 +192,7 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) {
stageCalled := false
activateCalled := false
rollbackCalled := false
commitCalled := false
updates := fakeUpdateProvider{
stage: func(_ context.Context, request UpdateStageRequest) (UpdateStageResult, error) {
stageCalled = request.ArtifactID == "release-1"
@@ -194,7 +200,11 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) {
},
activate: func(_ context.Context, request UpdateActivateRequest) (UpdateResult, error) {
activateCalled = request.ArtifactID == "release-1"
return UpdateResult{Action: "activated", ActivationID: "release-1:0123456789abcdef"}, nil
return UpdateResult{Action: "pending", ActivationID: "release-1:0123456789abcdef"}, nil
},
commit: func(_ context.Context, request UpdateCommitRequest) (UpdateResult, error) {
commitCalled = request.ActivationID == "release-1:0123456789abcdef"
return UpdateResult{Action: "committed", ActivationID: request.ActivationID}, nil
},
rollback: func(_ context.Context, request UpdateRollbackRequest) (UpdateResult, error) {
rollbackCalled = request.ActivationID == "release-1:0123456789abcdef"
@@ -222,13 +232,18 @@ func TestServerDispatchesClosedContainerAndUpdateOperations(t *testing.T) {
if response := exchange(t, server, activate); !response.Success || !activateCalled {
t.Fatalf("activate response=%#v called=%t", response, activateCalled)
}
commit := validRequest(OperationAgentUpdateCommit)
commit.Payload = json.RawMessage(`{"activationId":"release-1:0123456789abcdef","currentSha256":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}`)
if response := exchange(t, server, commit); !response.Success || !commitCalled {
t.Fatalf("commit response=%#v called=%t", response, commitCalled)
}
rollback := validRequest(OperationAgentUpdateRollback)
rollback.Payload = json.RawMessage(`{"activationId":"release-1:0123456789abcdef","currentSha256":"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","rollbackSha256":"abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}`)
if response := exchange(t, server, rollback); !response.Success || !rollbackCalled {
t.Fatalf("rollback response=%#v called=%t", response, rollbackCalled)
}
for _, operation := range []string{OperationContainerInventory, OperationAgentUpdateStage, OperationAgentUpdateActivate, OperationAgentUpdateRollback} {
for _, operation := range []string{OperationContainerInventory, OperationAgentUpdateStage, OperationAgentUpdateActivate, OperationAgentUpdateCommit, OperationAgentUpdateRollback} {
request := validRequest(operation)
request.Payload = json.RawMessage(`{"path":"/tmp/attacker","args":["sh"]}`)
requireErrorCode(t, exchange(t, server, request), ErrorInvalidRequest)
+283 -50
View File
@@ -16,6 +16,12 @@ import (
const maxUpdateArtifactBytes = 100 * 1024 * 1024
const (
defaultUpdateRollbackWindow = 2 * time.Minute
minUpdateRollbackWindow = 5 * time.Second
maxUpdateRollbackWindow = 10 * time.Minute
)
type UpdateActivateRequest struct {
ArtifactID string `json:"artifactId"`
SHA256 string `json:"sha256"`
@@ -39,12 +45,18 @@ type UpdateRollbackRequest struct {
RollbackSHA256 string `json:"rollbackSha256"`
}
type UpdateCommitRequest struct {
ActivationID string `json:"activationId"`
CurrentSHA256 string `json:"currentSha256"`
}
type UpdateResult struct {
Action string `json:"action"`
ActivationID string `json:"activationId"`
ActiveSHA256 string `json:"activeSha256"`
RollbackSHA256 string `json:"rollbackSha256"`
DurableAt time.Time `json:"durableAt"`
Action string `json:"action"`
ActivationID string `json:"activationId"`
ActiveSHA256 string `json:"activeSha256"`
RollbackSHA256 string `json:"rollbackSha256"`
RollbackDeadline time.Time `json:"rollbackDeadline,omitempty"`
DurableAt time.Time `json:"durableAt"`
}
type UpdateActivatorConfig struct {
@@ -56,6 +68,8 @@ type UpdateActivatorConfig struct {
ValidateOwner func(*os.File) error
ValidateQuarantineOwner func(*os.File) error
Now func() time.Time
RollbackWindow time.Duration
ScheduleRollback func(time.Duration, func()) func()
}
type updateActivator struct {
@@ -69,14 +83,18 @@ type updateActivator struct {
validateOwner func(*os.File) error
validateQuarantineOwner func(*os.File) error
now func() time.Time
rollbackWindow time.Duration
scheduleRollback func(time.Duration, func()) func()
cancelRollback func()
}
type durableUpdateState struct {
Action string `json:"action"`
ActivationID string `json:"activationId"`
ActiveSHA256 string `json:"activeSha256"`
RollbackSHA256 string `json:"rollbackSha256"`
UpdatedAt time.Time `json:"updatedAt"`
Action string `json:"action"`
ActivationID string `json:"activationId"`
ActiveSHA256 string `json:"activeSha256"`
RollbackSHA256 string `json:"rollbackSha256"`
RollbackDeadline time.Time `json:"rollbackDeadline,omitempty"`
UpdatedAt time.Time `json:"updatedAt"`
}
func NewUpdateActivator(config UpdateActivatorConfig) (UpdateProvider, error) {
@@ -98,12 +116,31 @@ func NewUpdateActivator(config UpdateActivatorConfig) (UpdateProvider, error) {
if now == nil {
now = time.Now
}
return &updateActivator{
rollbackWindow := config.RollbackWindow
if rollbackWindow == 0 {
rollbackWindow = defaultUpdateRollbackWindow
}
if rollbackWindow < minUpdateRollbackWindow || rollbackWindow > maxUpdateRollbackWindow {
return nil, errors.New("update rollback window must be between 5 seconds and 10 minutes")
}
scheduleRollback := config.ScheduleRollback
if scheduleRollback == nil {
scheduleRollback = func(delay time.Duration, callback func()) func() {
timer := time.AfterFunc(delay, callback)
return func() { timer.Stop() }
}
}
activator := &updateActivator{
quarantineDir: config.QuarantineDir, stagingDir: config.StagingDir, targetPath: config.TargetPath,
rollbackPath: config.TargetPath + ".last-known-good", statePath: config.StatePath,
verifySignature: config.VerifySignature, validateOwner: config.ValidateOwner,
validateQuarantineOwner: config.ValidateQuarantineOwner, now: now,
}, nil
rollbackWindow: rollbackWindow, scheduleRollback: scheduleRollback,
}
if err := activator.recoverUncommittedLocked(); err != nil {
return nil, err
}
return activator, nil
}
// Stage promotes exactly one signed collector-downloaded artifact from the
@@ -186,17 +223,30 @@ func (u *updateActivator) Activate(ctx context.Context, request UpdateActivateRe
return UpdateResult{}, invalidArtifact("staged artifact signature is invalid")
}
activationID := request.ArtifactID + ":" + actual[:16]
if state, err := u.readState(); err == nil && state.ActivationID == activationID {
if state.Action != "activated" || !strings.EqualFold(state.ActiveSHA256, actual) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed a different update transition"}
if state, err := u.readState(); err == nil {
if state.Action == "preparing" || state.Action == "pending" {
if state.ActivationID != activationID {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "another agent update is still awaiting commit or rollback"}
}
if state.Action == "preparing" {
if err := u.recoverStateLocked(state); err != nil {
return UpdateResult{}, err
}
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "interrupted activation was rolled back and cannot be replayed"}
}
if !strings.EqualFold(state.ActiveSHA256, actual) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed a different update transition"}
}
if err := u.validateInstalledState(state); err != nil {
return UpdateResult{}, err
}
u.schedulePendingRollbackLocked(state)
return updateResultFromState(state), nil
}
current, currentErr := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes)
rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes)
if currentErr != nil || rollbackErr != nil || !strings.EqualFold(sha256Hex(current), state.ActiveSHA256) || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "durable activation no longer matches installed binaries"}
if state.ActivationID == activationID {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "artifact identity has already completed its update transition"}
}
return UpdateResult{Action: state.Action, ActivationID: state.ActivationID, ActiveSHA256: state.ActiveSHA256, RollbackSHA256: state.RollbackSHA256, DurableAt: state.UpdatedAt}, nil
} else if err != nil && !errors.Is(err, os.ErrNotExist) {
} else if !errors.Is(err, os.ErrNotExist) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "durable update state is invalid"}
}
if err := ctx.Err(); err != nil {
@@ -207,20 +257,76 @@ func (u *updateActivator) Activate(ctx context.Context, request UpdateActivateRe
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "installed agent binary is unavailable or unsafe"}
}
rollbackDigest := sha256Hex(current)
deadline := u.now().UTC().Add(u.rollbackWindow)
preparing := durableUpdateState{
Action: "preparing", ActivationID: activationID, ActiveSHA256: actual,
RollbackSHA256: rollbackDigest, RollbackDeadline: deadline, UpdatedAt: u.now().UTC(),
}
if err := u.writeDurableState(preparing); err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist activation intent"}
}
if err := u.installDurably(artifact, current); err != nil {
_ = u.recoverStateLocked(preparing)
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "activate staged agent binary"}
}
result := UpdateResult{
Action: "activated", ActivationID: activationID,
ActiveSHA256: actual, RollbackSHA256: rollbackDigest, DurableAt: u.now().UTC(),
pending := durableUpdateState{
Action: "pending", ActivationID: activationID,
ActiveSHA256: actual, RollbackSHA256: rollbackDigest,
RollbackDeadline: deadline, UpdatedAt: u.now().UTC(),
}
if err := u.writeState(result); err != nil {
// Restore the pre-activation binary if the durable receipt cannot
// commit. A failed operation must not leave an unjournaled activation.
_ = u.installDurably(current, artifact)
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist activation result"}
if err := u.writeDurableState(pending); err != nil {
if recoveryErr := u.recoverStateLocked(preparing); recoveryErr != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist pending activation and restore last-known-good binary"}
}
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist pending activation"}
}
return result, nil
u.schedulePendingRollbackLocked(pending)
return updateResultFromState(pending), nil
}
func (u *updateActivator) Commit(ctx context.Context, request UpdateCommitRequest) (UpdateResult, error) {
u.mu.Lock()
defer u.mu.Unlock()
if !validActivationID(request.ActivationID) || !validSHA256(request.CurrentSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity is invalid"}
}
state, err := u.readState()
if err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity does not match a durable activation"}
}
if state.Action == "committed" && state.ActivationID == request.ActivationID && strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) {
return updateResultFromState(state), nil
}
if state.Action != "pending" || state.ActivationID != request.ActivationID || !strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "commit identity does not match the pending activation"}
}
if !u.now().UTC().Before(state.RollbackDeadline) {
if err := u.recoverStateLocked(state); err != nil {
return UpdateResult{}, err
}
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "activation deadline expired and the update was rolled back"}
}
if err := ctx.Err(); err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorDeadlineExceeded, Message: "update commit deadline exceeded", Retryable: true}
}
if err := u.validateInstalledState(state); err != nil {
return UpdateResult{}, err
}
if err := u.removeStagedArtifact(artifactIDFromActivation(state.ActivationID)); err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "remove committed update staging"}
}
pending := state
state.Action = "committed"
state.RollbackDeadline = time.Time{}
state.UpdatedAt = u.now().UTC()
if err := u.writeDurableState(state); err != nil {
if recoveryErr := u.recoverStateLocked(pending); recoveryErr != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist update commit and restore last-known-good binary"}
}
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist update commit"}
}
u.cancelPendingRollbackLocked()
return updateResultFromState(state), nil
}
func (u *updateActivator) Rollback(ctx context.Context, request UpdateRollbackRequest) (UpdateResult, error) {
@@ -230,34 +336,162 @@ func (u *updateActivator) Rollback(ctx context.Context, request UpdateRollbackRe
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity is invalid"}
}
state, err := u.readState()
if err != nil || state.Action != "activated" || state.ActivationID != request.ActivationID ||
if err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match a durable activation"}
}
if state.Action == "rolled_back" && state.ActivationID == request.ActivationID &&
strings.EqualFold(state.RollbackSHA256, request.CurrentSHA256) && strings.EqualFold(state.ActiveSHA256, request.RollbackSHA256) {
return updateResultFromState(state), nil
}
if (state.Action != "pending" && state.Action != "preparing") || state.ActivationID != request.ActivationID ||
!strings.EqualFold(state.ActiveSHA256, request.CurrentSHA256) ||
!strings.EqualFold(state.RollbackSHA256, request.RollbackSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match the durable activation"}
}
current, err := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes)
if err != nil || !strings.EqualFold(sha256Hex(current), state.ActiveSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "installed agent identity changed after activation"}
}
rollback, err := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes)
if err != nil || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) {
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "last-known-good identity changed after activation"}
return UpdateResult{}, &ProviderError{Code: ErrorStateConflict, Message: "rollback identity does not match the pending activation"}
}
if err := ctx.Err(); err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorDeadlineExceeded, Message: "update rollback deadline exceeded", Retryable: true}
}
if err := u.installDurably(rollback, current); err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "restore last-known-good agent binary"}
if err := u.recoverStateLocked(state); err != nil {
return UpdateResult{}, err
}
result := UpdateResult{
rolledBack, err := u.readState()
if err != nil {
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "read durable rollback result"}
}
return updateResultFromState(rolledBack), nil
}
func (u *updateActivator) recoverUncommittedLocked() error {
state, err := u.readState()
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return &ProviderError{Code: ErrorStateConflict, Message: "durable update state is invalid"}
}
if state.Action != "preparing" && state.Action != "pending" {
return nil
}
return u.recoverStateLocked(state)
}
func (u *updateActivator) recoverStateLocked(state durableUpdateState) error {
if state.Action != "preparing" && state.Action != "pending" {
return &ProviderError{Code: ErrorStateConflict, Message: "update state is not recoverable"}
}
current, err := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes)
if err != nil {
return &ProviderError{Code: ErrorStateConflict, Message: "installed agent binary is unavailable during recovery"}
}
currentDigest := sha256Hex(current)
if strings.EqualFold(currentDigest, state.ActiveSHA256) {
rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes)
if rollbackErr != nil || !strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) {
return &ProviderError{Code: ErrorStateConflict, Message: "last-known-good binary is unavailable during recovery"}
}
if err := u.installDurably(rollback, current); err != nil {
return &ProviderError{Code: ErrorInternal, Message: "restore last-known-good agent binary"}
}
} else if !strings.EqualFold(currentDigest, state.RollbackSHA256) {
return &ProviderError{Code: ErrorStateConflict, Message: "installed agent identity changed during pending activation"}
}
rolledBack := durableUpdateState{
Action: "rolled_back", ActivationID: state.ActivationID,
ActiveSHA256: state.RollbackSHA256, RollbackSHA256: state.ActiveSHA256, DurableAt: u.now().UTC(),
ActiveSHA256: state.RollbackSHA256, RollbackSHA256: state.ActiveSHA256,
UpdatedAt: u.now().UTC(),
}
if err := u.writeState(result); err != nil {
_ = u.installDurably(current, rollback)
return UpdateResult{}, &ProviderError{Code: ErrorInternal, Message: "persist rollback result"}
// Keep the durable state recoverable until the staged payload has also
// been removed. A crash anywhere before the terminal state write will
// replay this same bounded rollback and cleanup on helper startup.
if err := u.removeStagedArtifact(artifactIDFromActivation(state.ActivationID)); err != nil {
return &ProviderError{Code: ErrorInternal, Message: "remove rolled-back update staging"}
}
return result, nil
if err := u.writeDurableState(rolledBack); err != nil {
return &ProviderError{Code: ErrorInternal, Message: "persist rollback result"}
}
u.cancelPendingRollbackLocked()
return nil
}
func (u *updateActivator) validateInstalledState(state durableUpdateState) error {
current, currentErr := u.readOwnedBounded(u.targetPath, maxUpdateArtifactBytes)
rollback, rollbackErr := u.readOwnedBounded(u.rollbackPath, maxUpdateArtifactBytes)
if currentErr != nil || rollbackErr != nil ||
!strings.EqualFold(sha256Hex(current), state.ActiveSHA256) ||
!strings.EqualFold(sha256Hex(rollback), state.RollbackSHA256) {
return &ProviderError{Code: ErrorStateConflict, Message: "durable activation no longer matches installed binaries"}
}
return nil
}
func (u *updateActivator) schedulePendingRollbackLocked(state durableUpdateState) {
u.cancelPendingRollbackLocked()
delay := state.RollbackDeadline.Sub(u.now().UTC())
if delay < 0 {
delay = 0
}
u.cancelRollback = u.scheduleRollback(delay, func() {
u.mu.Lock()
defer u.mu.Unlock()
current, err := u.readState()
if err != nil || current.Action != "pending" || current.ActivationID != state.ActivationID {
return
}
_ = u.recoverStateLocked(current)
})
}
func (u *updateActivator) cancelPendingRollbackLocked() {
if u.cancelRollback != nil {
u.cancelRollback()
u.cancelRollback = nil
}
}
func updateResultFromState(state durableUpdateState) UpdateResult {
return UpdateResult{
Action: state.Action, ActivationID: state.ActivationID,
ActiveSHA256: state.ActiveSHA256, RollbackSHA256: state.RollbackSHA256,
RollbackDeadline: state.RollbackDeadline, DurableAt: state.UpdatedAt,
}
}
func artifactIDFromActivation(activationID string) string {
parts := strings.SplitN(activationID, ":", 2)
if len(parts) != 2 {
return ""
}
return parts[0]
}
func (u *updateActivator) removeStagedArtifact(artifactID string) error {
if !validArtifactID(artifactID) {
return errors.New("invalid staged artifact identity")
}
destination := filepath.Join(u.stagingDir, artifactID)
if _, err := os.Lstat(destination); errors.Is(err, os.ErrNotExist) {
return nil
} else if err != nil {
return err
}
if err := u.validateDirectory(destination); err != nil {
return err
}
for _, name := range []string{"pulse-agent", "pulse-agent.sig"} {
path := filepath.Join(destination, name)
info, err := os.Lstat(path)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
return errors.New("unsafe staged artifact cleanup target")
}
if err := os.Remove(path); err != nil {
return err
}
}
if err := os.Remove(destination); err != nil {
return err
}
return syncDirectory(u.stagingDir)
}
func (u *updateActivator) validateDirectory(path string) error {
@@ -386,8 +620,7 @@ func (u *updateActivator) installDurably(active, lastKnownGood []byte) error {
return syncDirectory(targetDir)
}
func (u *updateActivator) writeState(result UpdateResult) error {
state := durableUpdateState{Action: result.Action, ActivationID: result.ActivationID, ActiveSHA256: result.ActiveSHA256, RollbackSHA256: result.RollbackSHA256, UpdatedAt: result.DurableAt}
func (u *updateActivator) writeDurableState(state durableUpdateState) error {
data, err := json.Marshal(state)
if err != nil {
return err
@@ -32,6 +32,7 @@ func testUpdateActivator(t *testing.T, verifier func([]byte, string) error) (Upd
ValidateOwner: func(*os.File) error { return nil },
ValidateQuarantineOwner: func(*os.File) error { return nil },
Now: func() time.Time { return time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) },
ScheduleRollback: func(time.Duration, func()) func() { return func() {} },
})
if err != nil {
t.Fatal(err)
@@ -86,6 +87,9 @@ func TestUpdateActivationAndIdentityBoundRollbackAreDurable(t *testing.T) {
if activated.ActiveSHA256 != newDigest || activated.RollbackSHA256 != oldDigest || activated.ActivationID == "" {
t.Fatalf("activation result = %#v", activated)
}
if activated.Action != "pending" || activated.RollbackDeadline.IsZero() {
t.Fatalf("activation is not durably pending: %#v", activated)
}
if installed, _ := os.ReadFile(target); string(installed) != string(testELF("new-signed-binary")) {
t.Fatalf("installed binary = %q", installed)
}
@@ -115,6 +119,104 @@ func TestUpdateActivationAndIdentityBoundRollbackAreDurable(t *testing.T) {
}
}
func TestUpdateCommitClosesRollbackWindowAndCleansStaging(t *testing.T) {
provider, quarantine, target, _ := testUpdateActivator(t, func([]byte, string) error { return nil })
digest := stageUpdate(t, quarantine, "release-commit", testELF("committed"))
promoteUpdate(t, provider, "release-commit", digest)
activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-commit", SHA256: digest})
if err != nil {
t.Fatal(err)
}
committed, err := provider.Commit(context.Background(), UpdateCommitRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256,
})
if err != nil {
t.Fatal(err)
}
if committed.Action != "committed" || committed.RollbackDeadline != (time.Time{}) {
t.Fatalf("commit result = %#v", committed)
}
if installed, _ := os.ReadFile(target); string(installed) != string(testELF("committed")) {
t.Fatalf("committed binary = %q", installed)
}
staged := filepath.Join(filepath.Dir(quarantine), "staging", "release-commit")
if _, err := os.Stat(staged); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("committed staging still exists: %v", err)
}
if retried, err := provider.Commit(context.Background(), UpdateCommitRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256,
}); err != nil || retried.Action != "committed" {
t.Fatalf("idempotent commit = %#v, %v", retried, err)
}
if _, err := provider.Rollback(context.Background(), UpdateRollbackRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256,
}); err == nil {
t.Fatal("committed update remained rollback-eligible")
}
}
func TestUpdateActivatorRestartRollsBackUncommittedActivation(t *testing.T) {
provider, quarantine, target, state := testUpdateActivator(t, func([]byte, string) error { return nil })
digest := stageUpdate(t, quarantine, "release-restart", testELF("uncommitted"))
promoteUpdate(t, provider, "release-restart", digest)
activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-restart", SHA256: digest})
if err != nil {
t.Fatal(err)
}
root := filepath.Dir(quarantine)
recovered, err := NewUpdateActivator(UpdateActivatorConfig{
QuarantineDir: quarantine,
StagingDir: filepath.Join(root, "staging"),
TargetPath: target,
StatePath: state,
VerifySignature: func([]byte, string) error {
return nil
},
ValidateOwner: func(*os.File) error { return nil },
ValidateQuarantineOwner: func(*os.File) error { return nil },
Now: func() time.Time { return activation.RollbackDeadline.Add(-time.Second) },
ScheduleRollback: func(time.Duration, func()) func() { return func() {} },
})
if err != nil {
t.Fatalf("restart recovery: %v", err)
}
if installed, _ := os.ReadFile(target); string(installed) != string(testELF("old-signed-binary")) {
t.Fatalf("restart recovery installed binary = %q", installed)
}
result, err := recovered.Rollback(context.Background(), UpdateRollbackRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256,
})
if err != nil || result.Action != "rolled_back" {
t.Fatalf("durable restart result = %#v, %v", result, err)
}
}
func TestUpdateRollbackWatchdogRestoresUncommittedActivation(t *testing.T) {
provider, quarantine, target, _ := testUpdateActivator(t, func([]byte, string) error { return nil })
activator := provider.(*updateActivator)
var watchdog func()
activator.scheduleRollback = func(_ time.Duration, callback func()) func() {
watchdog = callback
return func() {}
}
digest := stageUpdate(t, quarantine, "release-watchdog", testELF("uncommitted"))
promoteUpdate(t, provider, "release-watchdog", digest)
activation, err := provider.Activate(context.Background(), UpdateActivateRequest{ArtifactID: "release-watchdog", SHA256: digest})
if err != nil || watchdog == nil {
t.Fatalf("Activate = %#v, %v; watchdog=%v", activation, err, watchdog != nil)
}
watchdog()
if installed, _ := os.ReadFile(target); string(installed) != string(testELF("old-signed-binary")) {
t.Fatalf("watchdog recovery installed binary = %q", installed)
}
result, err := provider.Rollback(context.Background(), UpdateRollbackRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256,
})
if err != nil || result.Action != "rolled_back" {
t.Fatalf("watchdog durable result = %#v, %v", result, err)
}
}
func TestUpdateActivationRejectsInvalidSignatureAndExecutable(t *testing.T) {
provider, staging, _, _ := testUpdateActivator(t, func([]byte, string) error { return errors.New("untrusted") })
digest := stageUpdate(t, staging, "bad-signature", testELF("binary"))
+32
View File
@@ -27,6 +27,38 @@ func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
return f(r)
}
func TestPrivilegedUpdateRollbackFailurePreservesPendingHandoff(t *testing.T) {
originalGOOS := runtimeGOOS
originalRestart := restartProcessFn
t.Cleanup(func() {
runtimeGOOS = originalGOOS
restartProcessFn = originalRestart
})
runtimeGOOS = goOSLinux
restartProcessFn = func(string) error { return errors.New("exec refused") }
binary := append([]byte{0x7f, 'E', 'L', 'F'}, bytes.Repeat([]byte("x"), 128)...)
sum := sha256.Sum256(binary)
digest := hex.EncodeToString(sum[:])
helper := &fakePrivilegedUpdate{root: t.TempDir(), rollbackErr: errors.New("helper unavailable")}
if err := os.Chmod(helper.root, 0o700); err != nil {
t.Fatal(err)
}
u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"})
u.selfTestFn = func(context.Context, string) error { return nil }
err := u.performPrivilegedUpdate(context.Background(), "/usr/local/bin/pulse-agent", bytes.NewReader(binary), int64(len(binary)), digest, "signed-update")
if err == nil || !strings.Contains(err.Error(), "typed helper rollback failed") {
t.Fatalf("performPrivilegedUpdate error = %v", err)
}
loaded, loadErr := LoadPendingPrivilegedUpdate(helper.root)
if loadErr != nil || loaded == nil || loaded.Activation.ActivationID != helper.activation.ActivationID ||
loaded.Activation.ActiveSHA256 != helper.activation.ActiveSHA256 ||
loaded.Activation.RollbackSHA256 != helper.activation.RollbackSHA256 {
t.Fatalf("failed-rollback handoff = %#v, %v", loaded, loadErr)
}
}
func TestUpdateRetryBackoffUsesOperationalDelays(t *testing.T) {
if updateRetryBaseDelay < time.Second {
t.Fatalf("updateRetryBaseDelay = %s, want at least 1s", updateRetryBaseDelay)
+178 -11
View File
@@ -4,8 +4,10 @@ import (
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
@@ -15,17 +17,24 @@ import (
)
const privilegedUpdateQuarantineDir = "/var/lib/pulse-agent/update-quarantine"
const pendingPrivilegedUpdateFile = ".pulse-agent-update-pending.json"
// PrivilegedUpdate is the updater's closed view of helper-backed activation.
// Implementations cannot select a target path, staging path, command, or URL.
type PrivilegedUpdate interface {
CreateQuarantinedArtifact() (artifactID string, file *os.File, cleanup func(), err error)
CreateQuarantinedArtifact() (artifactID string, file *os.File, cleanup func() error, err error)
WriteQuarantinedSignature(artifactID, signature string) error
Stage(context.Context, string, string) (agenthelper.UpdateStageResult, error)
Activate(context.Context, string, string) (agenthelper.UpdateResult, error)
Commit(context.Context, agenthelper.UpdateResult) (agenthelper.UpdateResult, error)
Rollback(context.Context, agenthelper.UpdateResult) (agenthelper.UpdateResult, error)
}
type PendingPrivilegedUpdate struct {
Activation agenthelper.UpdateResult `json:"activation"`
PreviousVersion string `json:"previousVersion"`
}
type privilegeHelperUpdate struct {
client *agenthelper.Client
quarantineDir string
@@ -39,28 +48,38 @@ func NewPrivilegeHelperUpdate(socketPath string) (PrivilegedUpdate, error) {
return &privilegeHelperUpdate{client: client, quarantineDir: privilegedUpdateQuarantineDir}, nil
}
func (p *privilegeHelperUpdate) CreateQuarantinedArtifact() (string, *os.File, func(), error) {
func (p *privilegeHelperUpdate) CreateQuarantinedArtifact() (string, *os.File, func() error, error) {
if err := validateCollectorQuarantineRoot(p.quarantineDir); err != nil {
return "", nil, func() {}, err
return "", nil, func() error { return nil }, err
}
var nonce [16]byte
if _, err := rand.Read(nonce[:]); err != nil {
return "", nil, func() {}, fmt.Errorf("create update artifact identity: %w", err)
return "", nil, func() error { return nil }, fmt.Errorf("create update artifact identity: %w", err)
}
artifactID := "pulse-agent-" + hex.EncodeToString(nonce[:])
artifactDir := filepath.Join(p.quarantineDir, artifactID)
if err := os.Mkdir(artifactDir, 0o700); err != nil {
return "", nil, func() {}, fmt.Errorf("create quarantined artifact directory: %w", err)
return "", nil, func() error { return nil }, fmt.Errorf("create quarantined artifact directory: %w", err)
}
cleanup := func() {
_ = os.Remove(filepath.Join(artifactDir, "pulse-agent"))
_ = os.Remove(filepath.Join(artifactDir, "pulse-agent.sig"))
_ = os.Remove(artifactDir)
cleanup := func() error {
var cleanupErr error
for _, name := range []string{"pulse-agent", "pulse-agent.sig"} {
if err := os.Remove(filepath.Join(artifactDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
cleanupErr = errors.Join(cleanupErr, err)
}
}
if err := os.Remove(artifactDir); err != nil && !errors.Is(err, os.ErrNotExist) {
cleanupErr = errors.Join(cleanupErr, err)
}
if err := syncUpdateDirectory(p.quarantineDir); err != nil {
cleanupErr = errors.Join(cleanupErr, err)
}
return cleanupErr
}
file, err := os.OpenFile(filepath.Join(artifactDir, "pulse-agent"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
cleanup()
return "", nil, func() {}, fmt.Errorf("create quarantined agent binary: %w", err)
_ = cleanup()
return "", nil, func() error { return nil }, fmt.Errorf("create quarantined agent binary: %w", err)
}
return artifactID, file, cleanup, nil
}
@@ -104,6 +123,14 @@ func (p *privilegeHelperUpdate) Activate(ctx context.Context, artifactID, digest
return result, err
}
func (p *privilegeHelperUpdate) Commit(ctx context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) {
var result agenthelper.UpdateResult
_, err := p.client.Call(ctx, agenthelper.OperationAgentUpdateCommit, agenthelper.OperationVersion1, 30*time.Second, agenthelper.UpdateCommitRequest{
ActivationID: activation.ActivationID, CurrentSHA256: activation.ActiveSHA256,
}, &result)
return result, err
}
func (p *privilegeHelperUpdate) Rollback(ctx context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) {
var result agenthelper.UpdateResult
_, err := p.client.Call(ctx, agenthelper.OperationAgentUpdateRollback, agenthelper.OperationVersion1, 30*time.Second, agenthelper.UpdateRollbackRequest{
@@ -145,3 +172,143 @@ func syncUpdateDirectory(path string) error {
defer dir.Close()
return dir.Sync()
}
func PersistPendingPrivilegedUpdate(stateDir, previousVersion string, activation agenthelper.UpdateResult) error {
path, err := pendingPrivilegedUpdatePath(stateDir)
if err != nil {
return err
}
if err := validatePendingUpdateStateDir(stateDir); err != nil {
return err
}
previousVersion = strings.TrimSpace(previousVersion)
if previousVersion == "" || len(previousVersion) > 128 || strings.ContainsAny(previousVersion, "\x00\r\n") {
return errors.New("previous update version is invalid")
}
if !validPendingActivation(activation) {
return errors.New("pending helper activation is invalid")
}
data, err := json.Marshal(PendingPrivilegedUpdate{Activation: activation, PreviousVersion: previousVersion})
if err != nil {
return err
}
temp, err := os.CreateTemp(stateDir, ".pulse-agent-update-pending-*")
if err != nil {
return fmt.Errorf("create pending update handoff: %w", err)
}
tempPath := temp.Name()
defer os.Remove(tempPath)
if err := temp.Chmod(0o600); err != nil {
_ = temp.Close()
return err
}
if _, err := temp.Write(data); err != nil {
_ = temp.Close()
return err
}
if err := temp.Sync(); err != nil {
_ = temp.Close()
return err
}
if err := temp.Close(); err != nil {
return err
}
if err := os.Rename(tempPath, path); err != nil {
return err
}
return syncUpdateDirectory(stateDir)
}
func LoadPendingPrivilegedUpdate(stateDir string) (*PendingPrivilegedUpdate, error) {
path, err := pendingPrivilegedUpdatePath(stateDir)
if err != nil {
return nil, err
}
before, err := os.Lstat(path)
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
if err := validatePendingUpdateStateDir(stateDir); err != nil {
return nil, err
}
if before.Mode()&os.ModeSymlink != 0 || !before.Mode().IsRegular() || before.Mode().Perm()&0o077 != 0 {
return nil, errors.New("pending update handoff is not a private regular file")
}
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
after, err := file.Stat()
if err != nil || !os.SameFile(before, after) {
return nil, errors.New("pending update handoff identity changed")
}
decoder := json.NewDecoder(file)
decoder.DisallowUnknownFields()
var pending PendingPrivilegedUpdate
if err := decoder.Decode(&pending); err != nil {
return nil, fmt.Errorf("decode pending update handoff: %w", err)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if err != nil {
return nil, fmt.Errorf("decode pending update handoff trailing data: %w", err)
}
return nil, errors.New("pending update handoff contains trailing JSON")
}
if !validPendingActivation(pending.Activation) || strings.TrimSpace(pending.PreviousVersion) == "" {
return nil, errors.New("pending update handoff is invalid")
}
return &pending, nil
}
func ClearPendingPrivilegedUpdate(stateDir string) error {
path, err := pendingPrivilegedUpdatePath(stateDir)
if err != nil {
return err
}
if err := validatePendingUpdateStateDir(stateDir); err != nil {
return err
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return syncUpdateDirectory(stateDir)
}
func pendingPrivilegedUpdatePath(stateDir string) (string, error) {
if stateDir == "" || !filepath.IsAbs(stateDir) || filepath.Clean(stateDir) != stateDir {
return "", errors.New("pending update state directory must be clean and absolute")
}
return filepath.Join(stateDir, pendingPrivilegedUpdateFile), nil
}
func validatePendingUpdateStateDir(stateDir string) error {
info, err := os.Lstat(stateDir)
if err != nil || info.Mode()&os.ModeSymlink != 0 || !info.IsDir() || info.Mode().Perm()&0o077 != 0 {
return errors.New("pending update state directory must be a private real directory")
}
return nil
}
func validPendingActivation(activation agenthelper.UpdateResult) bool {
parts := strings.Split(activation.ActivationID, ":")
if activation.Action != "pending" || len(parts) != 2 || !validPrivilegedArtifactID(parts[0]) || len(parts[1]) != 16 {
return false
}
if _, err := hex.DecodeString(parts[1]); err != nil {
return false
}
for _, digest := range []string{activation.ActiveSHA256, activation.RollbackSHA256} {
if len(digest) != 64 {
return false
}
if _, err := hex.DecodeString(digest); err != nil {
return false
}
}
return !activation.RollbackDeadline.IsZero()
}
+93 -12
View File
@@ -17,19 +17,23 @@ import (
)
type fakePrivilegedUpdate struct {
root string
events []string
activation agenthelper.UpdateResult
root string
events []string
activation agenthelper.UpdateResult
rollbackErr error
}
func (f *fakePrivilegedUpdate) CreateQuarantinedArtifact() (string, *os.File, func(), error) {
func (f *fakePrivilegedUpdate) CreateQuarantinedArtifact() (string, *os.File, func() error, error) {
f.events = append(f.events, "quarantine")
dir := filepath.Join(f.root, "pulse-agent-0123456789abcdef0123456789abcdef")
if err := os.Mkdir(dir, 0o700); err != nil {
return "", nil, func() {}, err
return "", nil, func() error { return nil }, err
}
file, err := os.OpenFile(filepath.Join(dir, "pulse-agent"), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
return filepath.Base(dir), file, func() { _ = os.RemoveAll(dir) }, err
return filepath.Base(dir), file, func() error {
f.events = append(f.events, "cleanup")
return os.RemoveAll(dir)
}, err
}
func (f *fakePrivilegedUpdate) WriteQuarantinedSignature(_ string, signature string) error {
@@ -44,12 +48,20 @@ func (f *fakePrivilegedUpdate) Stage(_ context.Context, artifactID, digest strin
func (f *fakePrivilegedUpdate) Activate(_ context.Context, _ string, digest string) (agenthelper.UpdateResult, error) {
f.events = append(f.events, "activate")
f.activation = agenthelper.UpdateResult{Action: "activated", ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", ActiveSHA256: digest, RollbackSHA256: strings.Repeat("a", 64)}
f.activation = agenthelper.UpdateResult{Action: "pending", ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef", ActiveSHA256: digest, RollbackSHA256: strings.Repeat("a", 64), RollbackDeadline: time.Now().Add(time.Minute)}
return f.activation, nil
}
func (f *fakePrivilegedUpdate) Commit(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) {
f.events = append(f.events, "commit")
return agenthelper.UpdateResult{Action: "committed", ActivationID: activation.ActivationID, ActiveSHA256: activation.ActiveSHA256, RollbackSHA256: activation.RollbackSHA256}, nil
}
func (f *fakePrivilegedUpdate) Rollback(_ context.Context, activation agenthelper.UpdateResult) (agenthelper.UpdateResult, error) {
f.events = append(f.events, "rollback")
if f.rollbackErr != nil {
return agenthelper.UpdateResult{}, f.rollbackErr
}
if activation != f.activation {
return agenthelper.UpdateResult{}, errors.New("wrong activation identity")
}
@@ -70,7 +82,10 @@ func TestPrivilegedUpdateStagesActivatesAndRollsBackRestartFailure(t *testing.T)
sum := sha256.Sum256(binary)
digest := hex.EncodeToString(sum[:])
helper := &fakePrivilegedUpdate{root: t.TempDir()}
u := New(Config{PrivilegedUpdate: helper, Disabled: true})
if err := os.Chmod(helper.root, 0o700); err != nil {
t.Fatal(err)
}
u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"})
u.selfTestFn = func(_ context.Context, path string) error {
helper.events = append(helper.events, "self-test")
data, err := os.ReadFile(path)
@@ -84,7 +99,7 @@ func TestPrivilegedUpdateStagesActivatesAndRollsBackRestartFailure(t *testing.T)
if err == nil || !strings.Contains(err.Error(), "update rolled back") {
t.Fatalf("performPrivilegedUpdate error = %v", err)
}
want := []string{"quarantine", "self-test", "signature:signed-update", "stage", "activate", "rollback"}
want := []string{"quarantine", "self-test", "signature:signed-update", "stage", "activate", "cleanup", "rollback"}
if !reflect.DeepEqual(helper.events, want) {
t.Fatalf("helper events = %#v, want %#v", helper.events, want)
}
@@ -96,14 +111,80 @@ func TestPrivilegedUpdateFailsClosedBeforeActivation(t *testing.T) {
runtimeGOOS = goOSLinux
binary := append([]byte{0x7f, 'E', 'L', 'F'}, []byte("update")...)
helper := &fakePrivilegedUpdate{root: t.TempDir()}
u := New(Config{PrivilegedUpdate: helper, Disabled: true})
if err := os.Chmod(helper.root, 0o700); err != nil {
t.Fatal(err)
}
u := New(Config{PrivilegedUpdate: helper, Disabled: true, StateDir: helper.root, CurrentVersion: "1.0.0"})
u.selfTestFn = func(context.Context, string) error { return nil }
err := u.performPrivilegedUpdate(context.Background(), "/usr/local/bin/pulse-agent", bytes.NewReader(binary), int64(len(binary)), strings.Repeat("0", 64), "signed-update")
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
t.Fatalf("performPrivilegedUpdate error = %v", err)
}
if !reflect.DeepEqual(helper.events, []string{"quarantine"}) {
t.Fatalf("helper events = %#v, want quarantine only", helper.events)
if !reflect.DeepEqual(helper.events, []string{"quarantine", "cleanup"}) {
t.Fatalf("helper events = %#v, want quarantine cleanup", helper.events)
}
}
func TestPendingPrivilegedUpdateHandoffIsDurableAndStrict(t *testing.T) {
stateDir := t.TempDir()
if err := os.Chmod(stateDir, 0o700); err != nil {
t.Fatal(err)
}
activation := agenthelper.UpdateResult{
Action: "pending",
ActivationID: "pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef",
ActiveSHA256: strings.Repeat("a", 64),
RollbackSHA256: strings.Repeat("b", 64),
RollbackDeadline: time.Now().Add(time.Minute).UTC(),
DurableAt: time.Now().UTC(),
}
if err := PersistPendingPrivilegedUpdate(stateDir, " 1.0.0 ", activation); err != nil {
t.Fatal(err)
}
path := filepath.Join(stateDir, pendingPrivilegedUpdateFile)
if info, err := os.Stat(path); err != nil || info.Mode().Perm() != 0o600 {
t.Fatalf("handoff mode=%v err=%v", info, err)
}
loaded, err := LoadPendingPrivilegedUpdate(stateDir)
if err != nil {
t.Fatal(err)
}
if loaded == nil || loaded.PreviousVersion != "1.0.0" || loaded.Activation != activation {
t.Fatalf("loaded handoff = %#v", loaded)
}
if err := ClearPendingPrivilegedUpdate(stateDir); err != nil {
t.Fatal(err)
}
if loaded, err := LoadPendingPrivilegedUpdate(stateDir); err != nil || loaded != nil {
t.Fatalf("cleared handoff = %#v, %v", loaded, err)
}
}
func TestPendingPrivilegedUpdateHandoffRejectsUnsafeState(t *testing.T) {
stateDir := t.TempDir()
if err := os.Chmod(stateDir, 0o700); err != nil {
t.Fatal(err)
}
path := filepath.Join(stateDir, pendingPrivilegedUpdateFile)
outside := filepath.Join(t.TempDir(), "outside")
if err := os.WriteFile(outside, []byte(`{"activation":{},"previousVersion":"1.0.0"}`), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, path); err != nil {
t.Fatal(err)
}
if _, err := LoadPendingPrivilegedUpdate(stateDir); err == nil {
t.Fatal("symlinked pending handoff accepted")
}
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
unsafe := `{"activation":{"action":"pending","activationId":"pulse-agent-0123456789abcdef0123456789abcdef:0123456789abcdef","activeSha256":"` + strings.Repeat("a", 64) + `","rollbackSha256":"` + strings.Repeat("b", 64) + `","rollbackDeadline":"2030-01-01T00:00:00Z"},"previousVersion":"1.0.0","unexpected":true}`
if err := os.WriteFile(path, []byte(unsafe), 0o600); err != nil {
t.Fatal(err)
}
if _, err := LoadPendingPrivilegedUpdate(stateDir); err == nil {
t.Fatal("unknown handoff field accepted")
}
}
+30 -9
View File
@@ -1093,7 +1093,14 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string,
if err != nil {
return fmt.Errorf("prepare fixed update quarantine: %w", err)
}
defer cleanup()
cleaned := false
defer func() {
if !cleaned {
if cleanupErr := cleanup(); cleanupErr != nil {
u.logger.Warn().Err(cleanupErr).Msg("failed to remove quarantined helper update artifact")
}
}
}()
artifactPath := file.Name()
hasher := sha256.New()
written, copyErr := io.Copy(file, io.TeeReader(io.LimitReader(body, maxBinarySizeBytes+1), hasher))
@@ -1147,22 +1154,36 @@ func (u *Updater) performPrivilegedUpdate(ctx context.Context, execPath string,
if err != nil {
return fmt.Errorf("activate signed update through typed helper: %w", err)
}
if activation.Action != "activated" || !strings.EqualFold(activation.ActiveSHA256, digest) || activation.ActivationID == "" || activation.RollbackSHA256 == "" {
return errors.New("typed helper returned an invalid activation result")
}
if err := restartProcessFn(execPath); err != nil {
rollbackActivation := func(cause error) error {
rollbackCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
rolledBack, rollbackErr := u.cfg.PrivilegedUpdate.Rollback(rollbackCtx, activation)
if rollbackErr != nil {
return errors.Join(fmt.Errorf("failed to restart after helper activation: %w", err), fmt.Errorf("typed helper rollback failed: %w", rollbackErr))
return errors.Join(cause, fmt.Errorf("typed helper rollback failed: %w", rollbackErr))
}
if rolledBack.Action != "rolled_back" || !strings.EqualFold(rolledBack.ActiveSHA256, activation.RollbackSHA256) {
return errors.Join(fmt.Errorf("failed to restart after helper activation: %w", err), errors.New("typed helper returned an invalid rollback result"))
return errors.Join(cause, errors.New("typed helper returned an invalid rollback result"))
}
return fmt.Errorf("failed to restart after helper activation; update rolled back: %w", err)
if clearErr := ClearPendingPrivilegedUpdate(u.cfg.StateDir); clearErr != nil {
return errors.Join(fmt.Errorf("%w; update rolled back", cause), fmt.Errorf("clear pending update handoff: %w", clearErr))
}
return fmt.Errorf("%w; update rolled back", cause)
}
return nil
if activation.Action != "pending" || !strings.EqualFold(activation.ActiveSHA256, digest) || activation.ActivationID == "" ||
activation.RollbackSHA256 == "" || activation.RollbackDeadline.IsZero() || !time.Now().Before(activation.RollbackDeadline) {
return rollbackActivation(errors.New("typed helper returned an invalid pending activation result"))
}
if err := PersistPendingPrivilegedUpdate(u.cfg.StateDir, u.cfg.CurrentVersion, activation); err != nil {
return rollbackActivation(fmt.Errorf("persist pending update handoff: %w", err))
}
if err := cleanup(); err != nil {
return rollbackActivation(fmt.Errorf("remove quarantined update after helper staging: %w", err))
}
cleaned = true
if err := restartProcessFn(execPath); err != nil {
return rollbackActivation(fmt.Errorf("failed to restart after helper activation: %w", err))
}
return rollbackActivation(errors.New("restart returned without replacing the current process"))
}
func (u *Updater) syncPersistentBinaryCopy(execPath, persistPath, platform string) {
+123 -2
View File
@@ -7,7 +7,10 @@ import (
"net/http"
"strings"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/api/agentbinding"
"github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rcourtman/pulse-go-rewrite/pkg/auth"
)
@@ -30,6 +33,27 @@ type actionRunnerCredentialResponse struct {
ActionCapability string `json:"actionCapability"`
}
type actionRunnerCredentialSelfRevokeRequest struct {
AgentID string `json:"agentId"`
Hostname string `json:"hostname"`
}
func actionRunnerCredentialRoute(cfg *config.Config, issue, selfRevoke http.HandlerFunc) http.HandlerFunc {
issue = RequireAdmin(cfg, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, issue)))
selfRevoke = RequireAuth(cfg, RequireScope(config.ScopeAgentExec, selfRevoke))
return func(w http.ResponseWriter, req *http.Request) {
switch req.Method {
case http.MethodPost:
issue(w, req)
case http.MethodDelete:
selfRevoke(w, req)
default:
w.Header().Set("Allow", http.MethodPost+", "+http.MethodDelete)
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
}
}
// handleIssueActionRunnerCredential creates the separately scoped credential
// consumed by pulse-agent-runner. The route is operator-only; a monitoring
// collector credential cannot mint or upgrade itself into remediation
@@ -63,7 +87,7 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h
http.Error(w, "Canonical monitored host identity not found", http.StatusNotFound)
return
}
rawToken, record, err := agenttokens.IssueActionRunnerAndPersist(r.config, r.persistence, agenttokens.ActionRunnerIssueOptions{
issued, err := agenttokens.IssueActionRunnerAndPersistDetailed(r.config, r.persistence, agenttokens.ActionRunnerIssueOptions{
TokenName: payload.Name,
OrgID: organizationID,
OwnerUserID: apiTokenOwnerUserIDForRequest(r.config, req),
@@ -78,12 +102,17 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h
http.Error(w, "Failed to issue action runner credential", status)
return
}
record := issued.Record
for _, replaced := range issued.Replaced {
r.invalidateActionRunnerRecord(replaced)
}
LogAuditEventForTenant(organizationID, "action_runner_credential_issued", auth.GetUser(req.Context()), GetClientIP(req), req.URL.Path, true, "Issued host-bound typed action runner credential")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(actionRunnerCredentialResponse{
Token: rawToken,
Token: issued.Token,
TokenID: record.ID,
OrganizationID: record.OrgID,
AgentID: record.Metadata["bound_agent_id"],
@@ -93,6 +122,98 @@ func (r *Router) handleIssueActionRunnerCredential(w http.ResponseWriter, req *h
})
}
// handleSelfRevokeActionRunnerCredential lets the separately credentialed
// runner revoke only its own exact tenant/host binding. It cannot select a
// token ID or another host, and browser/session authentication is rejected.
func (r *Router) handleSelfRevokeActionRunnerCredential(w http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodDelete {
w.Header().Set("Allow", http.MethodDelete)
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
if r == nil || r.config == nil {
http.Error(w, "Action runner credential service unavailable", http.StatusServiceUnavailable)
return
}
caller := getAPITokenRecordFromRequest(req)
if caller == nil {
http.Error(w, "Action runner bearer credential required", http.StatusForbidden)
return
}
decoder := json.NewDecoder(io.LimitReader(req.Body, maxActionRunnerCredentialRequestBytes+1))
decoder.DisallowUnknownFields()
var payload actionRunnerCredentialSelfRevokeRequest
if err := decoder.Decode(&payload); err != nil {
http.Error(w, "Invalid request body", http.StatusBadRequest)
return
}
if err := decoder.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
http.Error(w, "Invalid request body", http.StatusBadRequest)
return
}
organizationID := strings.TrimSpace(GetOrgID(req.Context()))
config.Mu.Lock()
previousTokens := append([]config.APITokenRecord(nil), r.config.APITokens...)
index := -1
var removed config.APITokenRecord
for candidateIndex := range r.config.APITokens {
candidate := &r.config.APITokens[candidateIndex]
if candidate.ID != caller.ID {
continue
}
orgs := candidate.GetBoundOrgs()
if len(orgs) != 1 || strings.TrimSpace(orgs[0]) != organizationID ||
!agentbinding.EvaluateActionRunner(candidate, payload.AgentID, payload.Hostname).Admit {
config.Mu.Unlock()
http.Error(w, "Action runner credential binding mismatch", http.StatusForbidden)
return
}
index = candidateIndex
removed = candidate.Clone()
break
}
if index < 0 {
config.Mu.Unlock()
http.Error(w, "Action runner credential not found", http.StatusUnauthorized)
return
}
r.config.APITokens = append(r.config.APITokens[:index], r.config.APITokens[index+1:]...)
r.config.SortAPITokens()
if r.persistence != nil {
if err := r.persistence.SaveAPITokens(r.config.APITokens); err != nil {
r.config.APITokens = previousTokens
r.config.SortAPITokens()
config.Mu.Unlock()
http.Error(w, "Failed to revoke action runner credential", http.StatusInternalServerError)
return
}
}
config.Mu.Unlock()
r.invalidateActionRunnerRecord(removed)
LogAuditEventForTenant(organizationID, "action_runner_credential_revoked", auth.GetUser(req.Context()), GetClientIP(req), req.URL.Path, true, "Revoked exact host-bound typed action runner credential")
w.WriteHeader(http.StatusNoContent)
}
func (r *Router) invalidateActionRunnerRecord(record config.APITokenRecord) bool {
if r == nil || r.agentExecServer == nil {
return false
}
orgs := record.GetBoundOrgs()
if len(orgs) != 1 {
return false
}
return r.agentExecServer.InvalidateActionRunnerSession(agentexec.AgentAdmission{
OrganizationID: strings.TrimSpace(orgs[0]),
TokenID: strings.TrimSpace(record.ID),
AgentID: strings.TrimSpace(record.Metadata["bound_agent_id"]),
Hostname: strings.TrimSpace(record.Metadata["bound_hostname"]),
RuntimeRole: strings.TrimSpace(record.Metadata[agenttokens.RuntimeRoleMetadataKey]),
ActionCapability: strings.TrimSpace(record.Metadata[agenttokens.ActionCapabilityMetadataKey]),
})
}
func (r *Router) resolveActionRunnerHostIdentity(req *http.Request, requestedID, requestedHostname string) (string, string, bool) {
requestedID = strings.TrimSpace(requestedID)
requestedHostname = strings.TrimSpace(requestedHostname)
+156 -1
View File
@@ -2,16 +2,41 @@ package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"os"
"testing"
"time"
"github.com/gorilla/websocket"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
)
type actionRunnerFailingPersistenceFS struct{}
func (actionRunnerFailingPersistenceFS) ReadFile(name string) ([]byte, error) {
return os.ReadFile(name)
}
func (actionRunnerFailingPersistenceFS) WriteFile(string, []byte, os.FileMode) error {
return errors.New("injected action-runner persistence failure")
}
func (actionRunnerFailingPersistenceFS) Rename(oldPath, newPath string) error {
return os.Rename(oldPath, newPath)
}
func (actionRunnerFailingPersistenceFS) Remove(name string) error { return os.Remove(name) }
func (actionRunnerFailingPersistenceFS) Stat(name string) (os.FileInfo, error) {
return os.Stat(name)
}
func (actionRunnerFailingPersistenceFS) MkdirAll(path string, perm os.FileMode) error {
return os.MkdirAll(path, perm)
}
func newActionRunnerCredentialTestRouter(t *testing.T) (*Router, *config.Config, string) {
t.Helper()
cfg := &config.Config{DataPath: t.TempDir(), AuthUser: "admin", AuthPass: "$2a$10$dummy"}
@@ -29,6 +54,53 @@ func actionRunnerCredentialBody(hostID, hostname string) *bytes.Reader {
return bytes.NewReader(body)
}
func issueActionRunnerCredentialForTest(t *testing.T, router *Router, hostID, hostname string) actionRunnerCredentialResponse {
t.Helper()
req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, hostname))
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default"))
rec := httptest.NewRecorder()
router.handleIssueActionRunnerCredential(rec, req)
if rec.Code != http.StatusCreated {
t.Fatalf("issue status = %d, body=%s", rec.Code, rec.Body.String())
}
var response actionRunnerCredentialResponse
if err := json.NewDecoder(rec.Body).Decode(&response); err != nil {
t.Fatal(err)
}
return response
}
func connectActionRunnerCredentialForTest(t *testing.T, server *agentexec.Server, credential actionRunnerCredentialResponse) (*websocket.Conn, *httptest.Server) {
t.Helper()
ts := httptest.NewServer(http.HandlerFunc(server.HandleWebSocket))
conn, _, err := websocket.DefaultDialer.Dial(wsURLForHTTP(ts.URL), wsHeadersForHTTP(t, ts.URL))
if err != nil {
ts.Close()
t.Fatal(err)
}
message, err := agentexec.NewMessage(agentexec.MsgTypeAgentRegister, "", agentexec.AgentRegisterPayload{
AgentID: credential.AgentID, Hostname: credential.Hostname, Token: credential.Token,
RuntimeRole: credential.RuntimeRole, ActionCapability: credential.ActionCapability,
OperationReceiptVersion: operationreceipt.ProtocolVersion,
})
if err != nil {
conn.Close()
ts.Close()
t.Fatal(err)
}
if err := conn.WriteJSON(message); err != nil {
conn.Close()
ts.Close()
t.Fatal(err)
}
if registered := readRegisteredPayload(t, conn); !registered.Success {
conn.Close()
ts.Close()
t.Fatalf("runner registration failed: %s", registered.Message)
}
return conn, ts
}
func TestIssueActionRunnerCredentialResolvesCanonicalMonitoredHost(t *testing.T) {
router, cfg, hostID := newActionRunnerCredentialTestRouter(t)
req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "HOST-1"))
@@ -106,11 +178,94 @@ func TestIssueActionRunnerCredentialRouteRotatesExistingHostBinding(t *testing.T
}
}
func TestIssueActionRunnerCredentialRotationClosesReplacedLiveSessionAfterPersistence(t *testing.T) {
router, _, hostID := newActionRunnerCredentialTestRouter(t)
router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession)
first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local")
conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first)
defer conn.Close()
defer ts.Close()
if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) {
t.Fatal("runner session was not connected")
}
second := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local")
if second.TokenID == first.TokenID {
t.Fatal("credential did not rotate")
}
if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) {
t.Fatal("replaced action-runner session remained connected")
}
}
func TestIssueActionRunnerCredentialPersistenceFailureKeepsPriorLiveSession(t *testing.T) {
router, cfg, hostID := newActionRunnerCredentialTestRouter(t)
router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession)
first := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local")
conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, first)
defer conn.Close()
defer ts.Close()
router.persistence.SetFileSystem(actionRunnerFailingPersistenceFS{})
req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "host-1.local"))
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default"))
rec := httptest.NewRecorder()
router.handleIssueActionRunnerCredential(rec, req)
if rec.Code != http.StatusInternalServerError {
t.Fatalf("status = %d, body=%s", rec.Code, rec.Body.String())
}
if len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != first.TokenID {
t.Fatalf("prior credential was not restored: %#v", cfg.APITokens)
}
if !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) {
t.Fatal("failed persistence invalidated the prior live session")
}
}
func TestSelfRevokeActionRunnerCredentialRequiresExactBearerBindingAndClosesSession(t *testing.T) {
router, cfg, hostID := newActionRunnerCredentialTestRouter(t)
router.agentExecServer = agentexec.NewServerWithAdmissionValidator(router.admitAgentExecToken, router.validateAgentExecSession)
issued := issueActionRunnerCredentialForTest(t, router, hostID, "host-1.local")
conn, ts := connectActionRunnerCredentialForTest(t, router.agentExecServer, issued)
defer conn.Close()
defer ts.Close()
handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)
request := func(organizationID, agentID, hostname, token string) *httptest.ResponseRecorder {
t.Helper()
body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: agentID, Hostname: hostname})
req := httptest.NewRequest(http.MethodDelete, "/api/agents/action-runner/credential", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+token)
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, organizationID))
rec := httptest.NewRecorder()
handler(rec, req)
return rec
}
if rec := request("other", hostID, issued.Hostname, issued.Token); rec.Code != http.StatusForbidden {
t.Fatalf("mismatched organization status = %d, body=%s", rec.Code, rec.Body.String())
}
if rec := request("default", hostID, "other.example", issued.Token); rec.Code != http.StatusForbidden {
t.Fatalf("mismatched hostname status = %d, body=%s", rec.Code, rec.Body.String())
}
if len(cfg.APITokens) != 1 || !router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) {
t.Fatal("binding mismatch changed credential or session state")
}
if rec := request("default", hostID, issued.Hostname, issued.Token); rec.Code != http.StatusNoContent {
t.Fatalf("self revoke status = %d, body=%s", rec.Code, rec.Body.String())
}
if len(cfg.APITokens) != 0 {
t.Fatalf("self-revoked credential remained stored: %#v", cfg.APITokens)
}
if router.agentExecServer.IsAgentConnectedForOrganization("default", hostID) {
t.Fatal("self-revoked session remained connected")
}
}
func TestActionRunnerCredentialRouteAuthSupportsAdminSessionAndScopedToken(t *testing.T) {
for _, mode := range []string{"api-token", "admin-session"} {
t.Run(mode, func(t *testing.T) {
router, cfg, hostID := newActionRunnerCredentialTestRouter(t)
handler := RequireAdmin(cfg, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, router.handleIssueActionRunnerCredential)))
handler := actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)
req := httptest.NewRequest(http.MethodPost, "/api/agents/action-runner/credential", actionRunnerCredentialBody(hostID, "host-1.local"))
switch mode {
case "api-token":
+41 -15
View File
@@ -55,6 +55,16 @@ type ActionRunnerIssueOptions struct {
Hostname string
}
// ActionRunnerIssueResult carries the newly issued credential together with
// the prior host-bound records it durably replaced. Callers use the replaced
// non-secret identities to invalidate only the superseded live sessions after
// persistence succeeds.
type ActionRunnerIssueResult struct {
Token string
Record *config.APITokenRecord
Replaced []config.APITokenRecord
}
func ProxmoxScopes(enableCommands bool) []string {
scopes := []string{
config.ScopeAgentReport,
@@ -109,7 +119,8 @@ func ParseCommandPolicyIntent(record *config.APITokenRecord) (bool, bool) {
}
func IssueAndPersist(cfg *config.Config, persistence *config.ConfigPersistence, opts IssueOptions) (string, *config.APITokenRecord, error) {
return issueAndPersistReplacing(cfg, persistence, opts, nil)
rawToken, record, _, err := issueAndPersistReplacing(cfg, persistence, opts, nil)
return rawToken, record, err
}
func issueAndPersistReplacing(
@@ -117,14 +128,14 @@ func issueAndPersistReplacing(
persistence *config.ConfigPersistence,
opts IssueOptions,
replace func(config.APITokenRecord) bool,
) (string, *config.APITokenRecord, error) {
) (string, *config.APITokenRecord, []config.APITokenRecord, error) {
if cfg == nil {
return "", nil, fmt.Errorf("config is required")
return "", nil, nil, fmt.Errorf("config is required")
}
rawToken, err := internalauth.GenerateAPIToken()
if err != nil {
return "", nil, fmt.Errorf("%w: %w", ErrGeneration, err)
return "", nil, nil, fmt.Errorf("%w: %w", ErrGeneration, err)
}
scopes := opts.Scopes
@@ -136,19 +147,19 @@ func issueAndPersistReplacing(
}
record, err := config.NewAPITokenRecord(rawToken, opts.TokenName, scopes)
if err != nil {
return "", nil, fmt.Errorf("%w: %w", ErrRecord, err)
return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err)
}
record.OrgID = strings.TrimSpace(opts.OrgID)
setOwnerUserID(record, opts.OwnerUserID)
if err := mergeMetadata(record, opts.Metadata); err != nil {
return "", nil, fmt.Errorf("%w: %w", ErrRecord, err)
return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err)
}
if record.Metadata == nil {
record.Metadata = make(map[string]string)
}
if err := normalizeCredentialKind(record); err != nil {
return "", nil, fmt.Errorf("%w: %w", ErrRecord, err)
return "", nil, nil, fmt.Errorf("%w: %w", ErrRecord, err)
}
record.Metadata[IssuedAtMetadataKey] = record.CreatedAt.UTC().Format(time.RFC3339)
@@ -156,12 +167,15 @@ func issueAndPersistReplacing(
defer config.Mu.Unlock()
previousTokens := append([]config.APITokenRecord(nil), cfg.APITokens...)
replaced := make([]config.APITokenRecord, 0, 1)
if replace == nil {
cfg.APITokens = append(cfg.APITokens, *record)
} else {
nextTokens := make([]config.APITokenRecord, 0, len(cfg.APITokens)+1)
for _, existing := range cfg.APITokens {
if !replace(existing) {
if replace(existing) {
replaced = append(replaced, existing.Clone())
} else {
nextTokens = append(nextTokens, existing)
}
}
@@ -175,38 +189,46 @@ func issueAndPersistReplacing(
// that the failed request never returned. Restore the full snapshot.
cfg.APITokens = previousTokens
cfg.SortAPITokens()
return "", nil, fmt.Errorf("%w: %w", ErrPersist, err)
return "", nil, nil, fmt.Errorf("%w: %w", ErrPersist, err)
}
}
return rawToken, record, nil
return rawToken, record, replaced, nil
}
// IssueActionRunnerAndPersist mints the host-bound credential used by the
// separate action runner. It fails closed on missing tenant/host identity and
// never grants collector report, lookup, configuration, or management scopes.
func IssueActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigPersistence, opts ActionRunnerIssueOptions) (string, *config.APITokenRecord, error) {
result, err := IssueActionRunnerAndPersistDetailed(cfg, persistence, opts)
return result.Token, result.Record, err
}
// IssueActionRunnerAndPersistDetailed is the rotation-aware form used by the
// API boundary. Replaced contains records only when the new credential was
// durably committed; persistence failure returns an empty result.
func IssueActionRunnerAndPersistDetailed(cfg *config.Config, persistence *config.ConfigPersistence, opts ActionRunnerIssueOptions) (ActionRunnerIssueResult, error) {
agentID := strings.TrimSpace(opts.AgentID)
hostname := unifiedresources.NormalizeFullHostname(opts.Hostname)
organizationID := strings.TrimSpace(opts.OrgID)
if organizationID == "" {
return "", nil, fmt.Errorf("%w: organization id is required", ErrRecord)
return ActionRunnerIssueResult{}, fmt.Errorf("%w: organization id is required", ErrRecord)
}
if agentID == "" {
return "", nil, fmt.Errorf("%w: canonical agent id is required", ErrRecord)
return ActionRunnerIssueResult{}, fmt.Errorf("%w: canonical agent id is required", ErrRecord)
}
if hostname == "" {
return "", nil, fmt.Errorf("%w: canonical hostname is required", ErrRecord)
return ActionRunnerIssueResult{}, fmt.Errorf("%w: canonical hostname is required", ErrRecord)
}
if len(agentID) > 128 || len(hostname) > 253 {
return "", nil, fmt.Errorf("%w: action runner identity exceeds maximum length", ErrRecord)
return ActionRunnerIssueResult{}, fmt.Errorf("%w: action runner identity exceeds maximum length", ErrRecord)
}
tokenName := strings.TrimSpace(opts.TokenName)
if tokenName == "" {
tokenName = "action-runner:" + hostname
}
return issueAndPersistReplacing(cfg, persistence, IssueOptions{
rawToken, record, replaced, err := issueAndPersistReplacing(cfg, persistence, IssueOptions{
TokenName: tokenName,
OrgID: organizationID,
OwnerUserID: opts.OwnerUserID,
@@ -224,6 +246,10 @@ func IssueActionRunnerAndPersist(cfg *config.Config, persistence *config.ConfigP
strings.TrimSpace(record.Metadata[CredentialKindMetadataKey]) == CredentialKindActionRunner &&
strings.TrimSpace(record.Metadata["bound_agent_id"]) == agentID
})
if err != nil {
return ActionRunnerIssueResult{}, err
}
return ActionRunnerIssueResult{Token: rawToken, Record: record, Replaced: replaced}, nil
}
func normalizeCredentialKind(record *config.APITokenRecord) error {
+22
View File
@@ -113,6 +113,28 @@ func TestIssueActionRunnerAndPersistReplacesMatchingBoundCredential(t *testing.T
}
}
func TestIssueActionRunnerAndPersistDetailedReturnsOnlyDurablyReplacedRecords(t *testing.T) {
cfg := &config.Config{DataPath: t.TempDir()}
_, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
OrgID: "org-a", AgentID: "machine-123", Hostname: "node.example",
})
if err != nil {
t.Fatal(err)
}
result, err := IssueActionRunnerAndPersistDetailed(cfg, config.NewConfigPersistence(cfg.DataPath), ActionRunnerIssueOptions{
OrgID: "org-a", AgentID: "machine-123", Hostname: "renamed.example",
})
if err != nil {
t.Fatal(err)
}
if result.Token == "" || result.Record == nil || len(result.Replaced) != 1 || result.Replaced[0].ID != prior.ID {
t.Fatalf("detailed issue result = %#v", result)
}
if result.Record.ID == prior.ID || len(cfg.APITokens) != 1 || cfg.APITokens[0].ID != result.Record.ID {
t.Fatalf("persisted replacement = %#v", cfg.APITokens)
}
}
func TestIssueActionRunnerAndPersistRestoresReplacedCredentialOnPersistenceFailure(t *testing.T) {
cfg := &config.Config{DataPath: t.TempDir()}
_, prior, err := IssueActionRunnerAndPersist(cfg, nil, ActionRunnerIssueOptions{
+1 -1
View File
@@ -55,7 +55,7 @@ func (r *Router) registerConfigSystemRoutes(updateHandlers *UpdateHandlers) {
r.mux.HandleFunc("/api/agents/docker/report", RequireAuth(r.config, RequireScope(config.ScopeDockerReport, r.dockerAgentHandlers.HandleReport)))
r.mux.HandleFunc("/api/agents/kubernetes/report", RequireAuth(r.config, RequireScope(config.ScopeKubernetesReport, r.kubernetesAgentHandlers.HandleReport)))
r.mux.HandleFunc("/api/agents/agent/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport)))
r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAdmin(r.config, RequireScope(config.ScopeSettingsWrite, RequireScope(config.ScopeActionsExecute, r.handleIssueActionRunnerCredential))))
r.mux.HandleFunc("/api/agents/action-runner/credential", RequireAuth(r.config, actionRunnerCredentialRoute(r.config, r.handleIssueActionRunnerCredential, r.handleSelfRevokeActionRunnerCredential)))
r.mux.HandleFunc("/api/agents/host/report", wrapLegacyHostAlias("/api/agents/host/report", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleReport))))
r.mux.HandleFunc("/api/agents/agent/lookup", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleLookup)))
r.mux.HandleFunc("/api/agents/host/lookup", wrapLegacyHostAlias("/api/agents/host/lookup", RequireAuth(r.config, RequireScope(config.ScopeAgentReport, r.unifiedAgentHandlers.HandleLookup))))
@@ -2,6 +2,8 @@ package api
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
@@ -10,9 +12,35 @@ import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/api/agenttokens"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
)
func TestSelfRevokeActionRunnerCredentialRejectsNonRunnerExecBearer(t *testing.T) {
router, cfg, hostID := newActionRunnerCredentialTestRouter(t)
raw := "legacy-exec-token-1234567890.12345678"
record, err := config.NewAPITokenRecord(raw, "legacy", []string{config.ScopeAgentExec})
if err != nil {
t.Fatal(err)
}
record.OrgID = "default"
record.Metadata = map[string]string{
agenttokens.RuntimeRoleMetadataKey: agenttokens.CredentialKindLegacyFullTrust,
"bound_agent_id": hostID,
"bound_hostname": "host-1.local",
}
cfg.APITokens = append(cfg.APITokens, *record)
body, _ := json.Marshal(actionRunnerCredentialSelfRevokeRequest{AgentID: hostID, Hostname: "host-1.local"})
req := httptest.NewRequest(http.MethodDelete, "/api/agents/action-runner/credential", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+raw)
req = req.WithContext(context.WithValue(req.Context(), OrgIDContextKey, "default"))
rec := httptest.NewRecorder()
actionRunnerCredentialRoute(cfg, router.handleIssueActionRunnerCredential, router.handleSelfRevokeActionRunnerCredential)(rec, req)
if rec.Code != http.StatusForbidden || len(cfg.APITokens) != 1 {
t.Fatalf("legacy self revoke = status %d tokens %#v body=%s", rec.Code, cfg.APITokens, rec.Body.String())
}
}
func TestSecurityTokensCreateRollsBackCompleteInventoryWhenPersistenceFails(t *testing.T) {
now := time.Now().UTC()
tokens := []config.APITokenRecord{
+15 -11
View File
@@ -99,6 +99,13 @@ type Config struct {
// server upgrade within one report cycle instead of its next hourly check.
// Nil disables the hook; acks from observer destinations never invoke it.
OnServerVersion func(version string)
// OnPrimaryReportAccepted is called only after the current process's newly
// collected authoritative report is accepted. Buffered reports and observer
// acknowledgements do not satisfy this startup-health signal.
OnPrimaryReportAccepted func()
// UpdatedFromVersion is supplied by the durable helper-update handoff and
// is carried by the first report built by the replacement process.
UpdatedFromVersion string
Collector SystemCollector // Optional: override default system information collector (for testing)
// PrivilegedTelemetry routes the exceptional SMART and Proxmox LXC
@@ -400,7 +407,10 @@ func New(cfg Config) (*Agent, error) {
const bufferCapacity = 60
// Check if agent was recently auto-updated (only reported once per restart)
updatedFrom := updatedFromVersionFn()
updatedFrom := strings.TrimSpace(cfg.UpdatedFromVersion)
if updatedFrom == "" {
updatedFrom = updatedFromVersionFn()
}
if updatedFrom != "" {
logger.Info().
Str("previousVersion", updatedFrom).
@@ -854,6 +864,9 @@ func (a *Agent) deliverPrimaryReport(ctx context.Context, report agentshost.Repo
return nil
}
agenttarget.MarkDelivery("host", "primary", "primary", true)
if a.cfg.OnPrimaryReportAccepted != nil {
a.cfg.OnPrimaryReportAccepted()
}
// The server has counted every availability result in this report, so they
// must never be offered to it again.
@@ -1167,16 +1180,7 @@ func (a *Agent) buildReport(ctx context.Context) (agentshost.Report, error) {
// Proxmox VE exposes per-mount LXC usage only through the node-local pct
// CLI. Query running containers on an independent bounded deadline.
var proxmoxLXCData *agentshost.ProxmoxLXCInventory
if a.privilegedTelemetry != nil {
proxmoxLXCData, err = a.privilegedTelemetry.ProxmoxLXCFilesystems(ctx)
if err != nil {
a.logger.Debug().Err(err).Msg("Typed helper could not collect Proxmox LXC filesystems")
proxmoxLXCData = nil
}
} else {
proxmoxLXCData = a.collectProxmoxLXCFilesystems(ctx)
}
proxmoxLXCData := a.collectProxmoxLXCFilesystemsForReport(ctx)
// Collect S.M.A.R.T. disk data after topology owners. Enumeration and each
// device probe have their own deadlines; a single shared 10-second budget
+7 -1
View File
@@ -20,6 +20,7 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin
var (
mu sync.Mutex
received []string
accepted atomic.Int32
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body io.Reader = r.Body
@@ -45,7 +46,9 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin
defer server.Close()
a := &Agent{
cfg: Config{APIToken: "token"},
cfg: Config{APIToken: "token", OnPrimaryReportAccepted: func() {
accepted.Add(1)
}},
logger: zerolog.Nop(),
httpClient: server.Client(),
trimmedPulseURL: server.URL,
@@ -69,6 +72,9 @@ func TestAgent_deliverPrimaryReport_DrainsBufferedReportsBeforeCurrent(t *testin
t.Fatalf("received = %v, want %v", received, want)
}
}
if got := accepted.Load(); got != 1 {
t.Fatalf("accepted current-report callbacks = %d, want 1", got)
}
}
func TestAgent_flushBuffer_StopsOnFailureAndDoesNotDropReport(t *testing.T) {
+20
View File
@@ -1043,6 +1043,26 @@ func TestNew_CarriesUpdatedFromIntoFirstV6Report(t *testing.T) {
}
}
func TestNew_PrefersDurableHelperUpdatedFromHandoff(t *testing.T) {
mc := &mockCollector{
hostInfoFn: func(context.Context) (*gohost.InfoStat, error) {
return &gohost.InfoStat{Hostname: "upgraded-host", HostID: "machine-id-1", KernelArch: runtime.GOARCH}, nil
},
}
agent, err := New(Config{
APIToken: "token",
Collector: mc,
UpdatedFromVersion: "6.0.0",
updatedFromVersionFn: func() string { return "stale-local-marker" },
})
if err != nil {
t.Fatal(err)
}
if agent.updatedFrom != "6.0.0" {
t.Fatalf("updatedFrom = %q, want durable helper handoff", agent.updatedFrom)
}
}
func TestNew_RejectsInvalidPulseURL(t *testing.T) {
mc := &mockCollector{
hostInfoFn: func(context.Context) (*gohost.InfoStat, error) {
@@ -16,6 +16,7 @@ const privilegeHelperOperationDeadline = 30 * time.Second
// It intentionally exposes complete typed snapshots rather than commands,
// executable paths, VMIDs, device paths, or caller-selected arguments.
type PrivilegedTelemetry interface {
Health(context.Context) error
SMARTSnapshot(context.Context) ([]DiskSMART, error)
ProxmoxLXCFilesystems(context.Context) (*agentshost.ProxmoxLXCInventory, error)
}
@@ -40,6 +41,32 @@ func NewPrivilegeHelperTelemetry(socketPath string) (PrivilegedTelemetry, error)
return &privilegeHelperTelemetry{client: client}, nil
}
// Health proves that the configured socket is serving the exact typed-helper
// protocol expected by this collector. A listening systemd socket alone is
// not sufficient: socket activation can succeed while the helper binary is
// missing, incompatible, or unable to handle requests.
func (c *privilegeHelperTelemetry) Health(ctx context.Context) error {
var response agenthelper.HealthResult
_, err := c.client.Call(
ctx,
agenthelper.OperationHealth,
agenthelper.OperationVersion1,
privilegeHelperOperationDeadline,
struct{}{},
&response,
)
if err != nil {
return err
}
if response.Status != "ok" {
return errors.New("helper health response is not ok")
}
if response.ProtocolVersion != agenthelper.ProtocolVersion {
return errors.New("helper health protocol version does not match")
}
return nil
}
func (c *privilegeHelperTelemetry) SMARTSnapshot(ctx context.Context) ([]DiskSMART, error) {
var response struct {
Disks []DiskSMART `json:"disks"`
@@ -78,3 +105,19 @@ func (c *privilegeHelperTelemetry) ProxmoxLXCFilesystems(ctx context.Context) (*
}
return response.Inventory, nil
}
// collectProxmoxLXCFilesystemsForReport preserves the privilege boundary once
// a helper is configured. A helper failure must omit this best-effort snapshot
// rather than silently retrying the same collection in the unprivileged
// collector process (or widening that process's privileges to make it work).
func (a *Agent) collectProxmoxLXCFilesystemsForReport(ctx context.Context) *agentshost.ProxmoxLXCInventory {
if a.privilegedTelemetry == nil {
return a.collectProxmoxLXCFilesystems(ctx)
}
inventory, err := a.privilegedTelemetry.ProxmoxLXCFilesystems(ctx)
if err != nil {
a.logger.Debug().Err(err).Msg("Typed helper could not collect Proxmox LXC filesystems")
return nil
}
return inventory
}
@@ -2,15 +2,22 @@ package hostagent
import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"io"
"net"
"os/exec"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/agenthelper"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
"github.com/rs/zerolog"
)
type fakePrivilegedTelemetry struct {
health error
smart []DiskSMART
smartErr error
proxmox *agentshost.ProxmoxLXCInventory
@@ -19,6 +26,10 @@ type fakePrivilegedTelemetry struct {
proxmoxCalls int
}
func (f *fakePrivilegedTelemetry) Health(context.Context) error {
return f.health
}
func (f *fakePrivilegedTelemetry) SMARTSnapshot(context.Context) ([]DiskSMART, error) {
f.smartCalls++
return f.smart, f.smartErr
@@ -29,6 +40,93 @@ func (f *fakePrivilegedTelemetry) ProxmoxLXCFilesystems(context.Context) (*agent
return f.proxmox, f.proxmoxErr
}
func testPrivilegeHelperTelemetry(t *testing.T, result agenthelper.HealthResult) *privilegeHelperTelemetry {
t.Helper()
client, err := agenthelper.NewClient(agenthelper.ClientConfig{
SocketPath: "/run/pulse-agent/helper.sock",
MaxDeadline: privilegeHelperOperationDeadline,
NewRequestID: func() (string, error) {
return "health-request", nil
},
DialContext: func(context.Context, string, string) (net.Conn, error) {
clientConn, serverConn := net.Pipe()
go func() {
defer serverConn.Close()
var header [4]byte
if _, readErr := io.ReadFull(serverConn, header[:]); readErr != nil {
return
}
requestBytes := make([]byte, binary.BigEndian.Uint32(header[:]))
if _, readErr := io.ReadFull(serverConn, requestBytes); readErr != nil {
return
}
var request agenthelper.Request
if unmarshalErr := json.Unmarshal(requestBytes, &request); unmarshalErr != nil {
return
}
resultBytes, marshalErr := json.Marshal(result)
if marshalErr != nil {
return
}
responseBytes, marshalErr := json.Marshal(agenthelper.Response{
ProtocolVersion: agenthelper.ProtocolVersion,
RequestID: request.RequestID,
Operation: request.Operation,
OperationVersion: request.OperationVersion,
Success: true,
Result: resultBytes,
})
if marshalErr != nil {
return
}
frame := make([]byte, 4+len(responseBytes))
binary.BigEndian.PutUint32(frame[:4], uint32(len(responseBytes)))
copy(frame[4:], responseBytes)
_, _ = serverConn.Write(frame)
}()
return clientConn, nil
},
})
if err != nil {
t.Fatalf("NewClient: %v", err)
}
return &privilegeHelperTelemetry{client: client}
}
func TestPrivilegeHelperHealthRequiresExactProtocolResponse(t *testing.T) {
tests := []struct {
name string
result agenthelper.HealthResult
wantErr bool
}{
{
name: "healthy",
result: agenthelper.HealthResult{Status: "ok", ProtocolVersion: agenthelper.ProtocolVersion},
},
{
name: "unhealthy status",
result: agenthelper.HealthResult{Status: "degraded", ProtocolVersion: agenthelper.ProtocolVersion},
wantErr: true,
},
{
name: "protocol mismatch",
result: agenthelper.HealthResult{Status: "ok", ProtocolVersion: agenthelper.ProtocolVersion + 1},
wantErr: true,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
telemetry := testPrivilegeHelperTelemetry(t, test.result)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
err := telemetry.Health(ctx)
if (err != nil) != test.wantErr {
t.Fatalf("Health error = %v, wantErr %v", err, test.wantErr)
}
})
}
}
func TestCollectSMARTDataUsesTypedHelperWithoutLocalFallback(t *testing.T) {
originalLookPath := zpoolLookPath
zpoolLookPath = func(string) (string, error) { return "", exec.ErrNotFound }
@@ -87,3 +185,30 @@ func TestCollectSMARTDataDoesNotWidenPrivilegeAfterHelperFailure(t *testing.T) {
t.Fatalf("local SMART fallback calls = %d, want 0", localCalls)
}
}
func TestCollectProxmoxLXCFilesystemsDoesNotWidenPrivilegeAfterHelperFailure(t *testing.T) {
localCalls := 0
collector := &mockCollector{
goos: "linux",
lookPathFn: func(string) (string, error) {
localCalls++
return "/usr/sbin/pct", nil
},
}
helper := &fakePrivilegedTelemetry{proxmoxErr: errors.New("helper unavailable")}
agent := &Agent{
collector: collector,
privilegedTelemetry: helper,
logger: zerolog.Nop(),
}
if got := agent.collectProxmoxLXCFilesystemsForReport(context.Background()); got != nil {
t.Fatalf("Proxmox inventory after helper failure = %+v, want nil", got)
}
if helper.proxmoxCalls != 1 {
t.Fatalf("helper Proxmox calls = %d, want 1", helper.proxmoxCalls)
}
if localCalls != 0 {
t.Fatalf("local Proxmox fallback calls = %d, want 0", localCalls)
}
}