mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-21 11:15:36 +00:00
Richard Courtman
ccea5ba761
An action that reached executing had exactly one way out: a terminal
operation receipt from the agent. Expiry skips executing rows on purpose,
and reconciliation bailed for every non-terminal query answer, so a Docker
update whose agent restarted mid-pull sat in executing forever with no
operator route out of it.
The agent-side receipt store rewrites accepted and started receipts to
interrupted on every Open(), and nothing can move an interrupted or
tombstoned receipt back to terminal. An identity-correlated answer of
either kind is therefore proof the operation will never report, and the
three executors now settle the action immediately as inconclusive with a
message telling the operator Pulse cannot confirm the effect and the
resource needs checking by hand.
A not_found answer, or a receipt still merely accepted or started, may
still be completed, so those keep waiting and only settle once the
dispatch attempt is older than one hour, the same threshold the
pulse-intelligence telemetry already uses to call an executing action
stuck. Every typed operation timeout is far shorter than that, so an
in-flight mutation is never cut short. A transport error answering the
query is still not evidence and preserves receipt_pending unchanged.
RecoverExecutingActions drives all of this on the existing two-minute
recovery loop, so rows already wedged before the upgrade heal themselves
without anyone touching them.
For the residue that reconciliation cannot reach, an agent that was
reinstalled or a legacy executing row with no dispatch attempt, POST
/api/actions/{id}/force-fail writes the same inconclusive terminal truth
under an operator attribution. It never touches the transport, refuses
anything already terminal, and is gated on admin plus settings:write on
top of the execute capability check.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1658 lines
71 KiB
Go
1658 lines
71 KiB
Go
package actionlifecycle
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/actionplanner"
|
|
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
)
|
|
|
|
type delayedCreateStore struct {
|
|
unified.ResourceStore
|
|
calls atomic.Int32
|
|
secondArrived chan struct{}
|
|
releaseSecond chan struct{}
|
|
}
|
|
|
|
func (s *delayedCreateStore) CreateActionAudit(record unified.ActionAuditRecord, events []unified.ActionLifecycleEvent) (unified.ActionAuditRecord, bool, error) {
|
|
if s.calls.Add(1) == 2 {
|
|
close(s.secondArrived)
|
|
<-s.releaseSecond
|
|
}
|
|
return s.ResourceStore.CreateActionAudit(record, events)
|
|
}
|
|
|
|
type blockingExecutor struct {
|
|
calls atomic.Int32
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
type claimBoundaryStore struct {
|
|
unified.ResourceStore
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
once atomic.Bool
|
|
}
|
|
|
|
func (s *claimBoundaryStore) ClaimActionDispatch(actionID, owner string, now time.Time, lease time.Duration) (unified.ActionDispatchAttempt, bool, error) {
|
|
attempt, claimed, err := s.ResourceStore.ClaimActionDispatch(actionID, owner, now, lease)
|
|
if claimed && s.once.CompareAndSwap(false, true) {
|
|
close(s.entered)
|
|
<-s.release
|
|
}
|
|
return attempt, claimed, err
|
|
}
|
|
|
|
func (e *blockingExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
|
|
if e.calls.Add(1) == 1 {
|
|
close(e.entered)
|
|
}
|
|
<-e.release
|
|
return &unified.ExecutionResult{Success: true, Output: record.ID}, nil
|
|
}
|
|
|
|
func serviceForStore(t *testing.T, store unified.ResourceStore, resource unified.Resource, executor Executor) *Service {
|
|
t.Helper()
|
|
registry := unified.NewRegistry(store)
|
|
registry.IngestResources([]unified.Resource{resource})
|
|
return &Service{
|
|
Registry: func(string) (*unified.ResourceRegistry, error) { return registry, nil },
|
|
Store: func(string) (Store, error) { return store, nil },
|
|
Executor: executor,
|
|
DecisionAuthorizer: DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
|
|
ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
|
|
}
|
|
}
|
|
|
|
func TestWithPolicyMutationConcurrentFirstUseSharesCoordinator(t *testing.T) {
|
|
const writers = 64
|
|
|
|
service := &Service{}
|
|
start := make(chan struct{})
|
|
coordinators := make(chan *PolicyAdmissionCoordinator, writers)
|
|
var ready sync.WaitGroup
|
|
var done sync.WaitGroup
|
|
ready.Add(writers)
|
|
done.Add(writers)
|
|
|
|
for range writers {
|
|
go func() {
|
|
defer done.Done()
|
|
ready.Done()
|
|
<-start
|
|
coordinators <- service.admissionCoordinator()
|
|
}()
|
|
}
|
|
|
|
ready.Wait()
|
|
close(start)
|
|
done.Wait()
|
|
close(coordinators)
|
|
|
|
var shared *PolicyAdmissionCoordinator
|
|
for coordinator := range coordinators {
|
|
if shared == nil {
|
|
shared = coordinator
|
|
continue
|
|
}
|
|
if coordinator != shared {
|
|
t.Fatal("concurrent first use returned more than one policy admission coordinator")
|
|
}
|
|
}
|
|
if shared == nil {
|
|
t.Fatal("concurrent first use did not initialize a policy admission coordinator")
|
|
}
|
|
}
|
|
|
|
func testActionActor(subject, orgID string) unified.ActionActor {
|
|
if strings.TrimSpace(subject) == "" {
|
|
subject = "operator"
|
|
}
|
|
return unified.ActionActor{SubjectID: subject, Kind: unified.ActionActorUser, CredentialID: "session:test", OrgID: orgID}
|
|
}
|
|
|
|
func testActionDecision(t *testing.T, service *Service, orgID, actionID string, approval unified.ActionApprovalRecord) unified.ActionDecision {
|
|
t.Helper()
|
|
record, found, _ := service.Get(orgID, actionID)
|
|
planHash := "sha256:test"
|
|
if found {
|
|
planHash = record.Plan.PlanHash
|
|
}
|
|
actor := testActionActor(approval.Actor, orgID)
|
|
return unified.ActionDecision{
|
|
Actor: actor,
|
|
Outcome: approval.Outcome,
|
|
Reason: approval.Reason,
|
|
Evidence: unified.ApprovalEvidence{
|
|
Version: 1, Method: unified.MethodSession, Actor: actor, OrgID: orgID,
|
|
ActionID: actionID, PlanHash: planHash, Outcome: approval.Outcome, IssuedAt: time.Now().UTC(),
|
|
},
|
|
}
|
|
}
|
|
|
|
func testBoundLifecycleApproval(record unified.ActionAuditRecord, subject string, method unified.ApprovalMethod, outcome unified.ApprovalOutcome, at time.Time) unified.ActionApprovalRecord {
|
|
kind := unified.ActionActorUser
|
|
credential := "session:test"
|
|
if method == unified.MethodPolicy {
|
|
kind = unified.ActionActorPolicy
|
|
credential = "policy:test"
|
|
}
|
|
actor := unified.ActionActor{SubjectID: subject, Kind: kind, CredentialID: credential, OrgID: "default"}
|
|
evidence := unified.ApprovalEvidence{Version: 1, Method: method, Actor: actor, OrgID: "default", ActionID: record.ID, PlanHash: record.Plan.PlanHash, Outcome: outcome, IssuedAt: at}
|
|
return unified.ActionApprovalRecord{Actor: subject, ActorBinding: actor, Method: method, Timestamp: at, Outcome: outcome, Evidence: &evidence}
|
|
}
|
|
|
|
func runConcurrentPlanReplayCannotRewindTerminalAction(t *testing.T, store unified.ResourceStore) {
|
|
t.Helper()
|
|
delayed := &delayedCreateStore{ResourceStore: store, secondArrived: make(chan struct{}), releaseSecond: make(chan struct{})}
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, delayed, testResource(time.Now().UTC(), unified.ApprovalAdmin), executor)
|
|
firstPlan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secondResult := make(chan error, 1)
|
|
go func() {
|
|
_, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
secondResult <- err
|
|
}()
|
|
<-delayed.secondArrived
|
|
if _, err := service.Decide(context.Background(), "default", firstPlan.ActionID, testActionDecision(t, service, "default", firstPlan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.Execute(context.Background(), "default", firstPlan.ActionID, testActionActor("operator", "default"), "proof"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
close(delayed.releaseSecond)
|
|
if err := <-secondResult; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, found, err := store.GetActionAudit(firstPlan.ActionID)
|
|
if err != nil || !found || current.State != unified.ActionStateCompleted {
|
|
t.Fatalf("current=%#v found=%v err=%v", current, found, err)
|
|
}
|
|
if executor.calls != 1 {
|
|
t.Fatalf("executor calls=%d, want 1", executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestConcurrentPlanReplayCannotRewindTerminalActionMemoryStore(t *testing.T) {
|
|
runConcurrentPlanReplayCannotRewindTerminalAction(t, unified.NewMemoryStore())
|
|
}
|
|
|
|
func TestConcurrentPlanReplayCannotRewindTerminalActionSQLiteStore(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runConcurrentPlanReplayCannotRewindTerminalAction(t, store)
|
|
}
|
|
|
|
func runConcurrentExecuteAdmitsExecutorExactlyOnce(t *testing.T, store unified.ResourceStore) {
|
|
t.Helper()
|
|
executor := &blockingExecutor{entered: make(chan struct{}), release: make(chan struct{})}
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := make(chan error, 1)
|
|
go func() {
|
|
_, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
|
|
first <- err
|
|
}()
|
|
<-executor.entered
|
|
secondRecord, secondErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "second")
|
|
if secondErr != nil {
|
|
t.Fatalf("duplicate Execute: %v", secondErr)
|
|
}
|
|
if secondRecord.State != unified.ActionStateExecuting {
|
|
t.Fatalf("duplicate state=%q", secondRecord.State)
|
|
}
|
|
close(executor.release)
|
|
if err := <-first; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if executor.calls.Load() != 1 {
|
|
t.Fatalf("executor calls=%d, want 1", executor.calls.Load())
|
|
}
|
|
}
|
|
|
|
func TestConcurrentExecuteAdmitsExecutorExactlyOnceMemoryStore(t *testing.T) {
|
|
runConcurrentExecuteAdmitsExecutorExactlyOnce(t, unified.NewMemoryStore())
|
|
}
|
|
|
|
func TestConcurrentExecuteAdmitsExecutorExactlyOnceSQLiteStore(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runConcurrentExecuteAdmitsExecutorExactlyOnce(t, store)
|
|
}
|
|
|
|
func runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T, store unified.ResourceStore) {
|
|
t.Helper()
|
|
blocked := &claimBoundaryStore{ResourceStore: store, entered: make(chan struct{}), release: make(chan struct{})}
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, blocked, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first := make(chan error, 1)
|
|
go func() {
|
|
_, executeErr := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "first")
|
|
first <- executeErr
|
|
}()
|
|
<-blocked.entered
|
|
second, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate")
|
|
if err != nil || second.State != unified.ActionStateExecuting {
|
|
t.Fatalf("duplicate=%#v err=%v", second, err)
|
|
}
|
|
close(blocked.release)
|
|
if err := <-first; err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if executor.calls != 1 {
|
|
t.Fatalf("executor calls=%d, want 1", executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestMemoryStoreConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
|
|
runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, unified.NewMemoryStore())
|
|
}
|
|
|
|
func TestSQLiteConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runConcurrentDuplicateExecuteAtClaimBoundaryCallsExecutorOnce(t, store)
|
|
}
|
|
|
|
func TestPlanReplayReturnsAuthoritativeApprovedExecutingAndTerminalRecords(t *testing.T) {
|
|
states := []unified.ActionState{unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateRejected, unified.ActionStateCompleted, unified.ActionStateFailed}
|
|
for _, state := range states {
|
|
t.Run(string(state), func(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, _, _ := store.GetActionAudit(plan.ActionID)
|
|
decisionOutcome := unified.OutcomeApproved
|
|
if state == unified.ActionStateRejected {
|
|
decisionOutcome = unified.OutcomeRejected
|
|
}
|
|
decisionAt := time.Now().UTC()
|
|
decided, decisionEvent, err := unified.ApplyActionDecision(record, testBoundLifecycleApproval(record, "operator", unified.MethodSession, decisionOutcome, decisionAt), decisionAt)
|
|
if err != nil || store.RecordActionDecision(decided, decisionEvent) != nil {
|
|
t.Fatalf("decision: %v", err)
|
|
}
|
|
if state != unified.ActionStateApproved && state != unified.ActionStateRejected {
|
|
started, startEvent, err := unified.BeginActionExecution(decided, "operator", time.Now().UTC())
|
|
if err != nil || store.RecordActionExecutionStart(started, startEvent) != nil {
|
|
t.Fatalf("start: %v", err)
|
|
}
|
|
if state == unified.ActionStateCompleted || state == unified.ActionStateFailed {
|
|
result := &unified.ExecutionResult{Success: state == unified.ActionStateCompleted}
|
|
completed, doneEvent, err := unified.CompleteActionExecution(started, result, "operator", time.Now().UTC())
|
|
if err != nil || store.RecordActionExecutionResult(completed, doneEvent) != nil {
|
|
t.Fatalf("complete: %v", err)
|
|
}
|
|
}
|
|
}
|
|
returned, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if returned.ActionID != plan.ActionID {
|
|
t.Fatalf("action id=%q", returned.ActionID)
|
|
}
|
|
current, _, _ := store.GetActionAudit(plan.ActionID)
|
|
if current.State != state {
|
|
t.Fatalf("state=%q want=%q", current.State, state)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPlanReplayRejectsConflictingOriginForDeterministicActionID(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalAdmin), &stubExecutor{})
|
|
first := PlanOptions{Actor: testActionActor("requester", "default"), Origin: &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-1", InvestigationID: "inv-1", ProposalID: "proposal-1"}}
|
|
if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), first); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
conflict := first
|
|
conflict.Origin = &unified.ActionOrigin{Surface: "patrol", FindingID: "finding-2", InvestigationID: "inv-2", ProposalID: "proposal-1"}
|
|
if _, err := service.PlanWithOptions(context.Background(), "default", restartRequest(), conflict); !errors.Is(err, unified.ErrActionIdentityConflict) {
|
|
t.Fatalf("error=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestExecutePublishesPersistedExecutingTransition(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
var states []unified.ActionState
|
|
service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { states = append(states, record.State) }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateExecuting, unified.ActionStateCompleted}
|
|
if len(states) != len(want) {
|
|
t.Fatalf("states=%v", states)
|
|
}
|
|
for i := range want {
|
|
if states[i] != want[i] {
|
|
t.Fatalf("states=%v want=%v", states, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExecuteAfterSQLiteRestartDoesNotReadmitExecutingAction(t *testing.T) {
|
|
dir := t.TempDir()
|
|
first, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, first, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, _, _ := first.GetActionAudit(plan.ActionID)
|
|
started, event, _ := unified.BeginActionExecution(record, "operator", time.Now().UTC())
|
|
if err := first.RecordActionExecutionStart(started, event); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
first.Close()
|
|
second, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer second.Close()
|
|
service = serviceForStore(t, second, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "retry")
|
|
if err != nil || current.State != unified.ActionStateExecuting || executor.calls != 0 {
|
|
t.Fatalf("current=%#v err=%v calls=%d", current, err, executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestExecuteTimeoutAfterSendWaitsForCorrelatedLateResponseWithoutResend(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "timeout proof")
|
|
if !errors.Is(err, context.DeadlineExceeded) || current.State != unified.ActionStateExecuting {
|
|
t.Fatalf("current=%#v err=%v", current, err)
|
|
}
|
|
attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
|
|
if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending || attempt.DispatchCount != 1 {
|
|
t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
|
|
}
|
|
if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
|
|
t.Fatalf("receipt found=%v err=%v", found, err)
|
|
}
|
|
// Duplicate execute is a query-only resume and cannot call ExecuteAction.
|
|
current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate resume")
|
|
if err != nil || current.State != unified.ActionStateExecuting || executor.executeCalls != 1 || executor.reconcileCalls != 1 {
|
|
t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
|
|
}
|
|
executor.found = true
|
|
current, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "late response")
|
|
if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 || executor.reconcileCalls != 2 {
|
|
t.Fatalf("current=%#v err=%v execute=%d reconcile=%d", current, err, executor.executeCalls, executor.reconcileCalls)
|
|
}
|
|
if receipt, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || !found || receipt.AttemptID != attempt.ID {
|
|
t.Fatalf("receipt=%#v found=%v err=%v", receipt, found, err)
|
|
}
|
|
}
|
|
|
|
func TestSQLiteRestartRecoveryReconcilesReceiptPendingWithoutResend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
now := time.Now().UTC()
|
|
first, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, first, testResource(now, unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "before restart"); !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("Execute error=%v", err)
|
|
}
|
|
if err := first.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer second.Close()
|
|
executor.found = true
|
|
service = serviceForStore(t, second, testResource(now, unified.ApprovalNone), executor)
|
|
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
|
|
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateCompleted {
|
|
t.Fatalf("recovered=%#v err=%v", recovered, err)
|
|
}
|
|
if executor.executeCalls != 1 || executor.reconcileCalls != 1 {
|
|
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
|
|
}
|
|
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "duplicate after recovery")
|
|
if err != nil || current.State != unified.ActionStateCompleted || executor.executeCalls != 1 {
|
|
t.Fatalf("current=%#v err=%v execute=%d", current, err, executor.executeCalls)
|
|
}
|
|
}
|
|
|
|
func TestSQLiteRestartRecoveryNotFoundRemainsReceiptPendingWithoutResend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
now := time.Now().UTC()
|
|
store, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "initial"); !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("execute err=%v", err)
|
|
}
|
|
for n := 0; n < 2; n++ {
|
|
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
|
|
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
|
|
t.Fatalf("pass=%d recovered=%#v err=%v", n, recovered, err)
|
|
}
|
|
}
|
|
attempt, found, err := store.GetActionDispatchAttempt(plan.ActionID)
|
|
if err != nil || !found || attempt.State != unified.ActionDispatchReceiptPending {
|
|
t.Fatalf("attempt=%#v found=%v err=%v", attempt, found, err)
|
|
}
|
|
if _, found, err := store.GetActionDispatchReceipt(attempt.ID); err != nil || found {
|
|
t.Fatalf("receipt found=%v err=%v", found, err)
|
|
}
|
|
audit, found, err := store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !found || audit.State != unified.ActionStateExecuting || audit.Result != nil {
|
|
t.Fatalf("audit=%#v found=%v err=%v", audit, found, err)
|
|
}
|
|
if executor.executeCalls != 1 || executor.reconcileCalls != 2 {
|
|
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
|
|
}
|
|
}
|
|
|
|
func TestActionDetailAndInboxMaterializeExplicitExpiry(t *testing.T) {
|
|
now := time.Date(2026, 7, 11, 14, 0, 0, 0, time.UTC)
|
|
store := unified.NewMemoryStore()
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
|
|
service.Now = func() time.Time { return now }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
service.Now = func() time.Time { return plan.ExpiresAt.Add(time.Second) }
|
|
detail, found, err := service.Detail("default", plan.ActionID)
|
|
if err != nil || !found || detail.Audit.State != unified.ActionStateExpired {
|
|
t.Fatalf("detail=%#v found=%v err=%v", detail, found, err)
|
|
}
|
|
if len(detail.Events) == 0 || detail.Events[0].State != unified.ActionStateExpired {
|
|
t.Fatalf("events=%#v", detail.Events)
|
|
}
|
|
pending, err := service.List("default", ActionListPending, 100)
|
|
if err != nil || len(pending) != 0 {
|
|
t.Fatalf("pending=%#v err=%v", pending, err)
|
|
}
|
|
settled, err := service.List("default", ActionListSettled, 100)
|
|
if err != nil || len(settled) != 1 || settled[0].State != unified.ActionStateExpired {
|
|
t.Fatalf("settled=%#v err=%v", settled, err)
|
|
}
|
|
if _, err := service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Outcome: unified.OutcomeApproved})); !errors.Is(err, unified.ErrActionPlanExpired) {
|
|
t.Fatalf("Decide error=%v", err)
|
|
}
|
|
}
|
|
|
|
func policyTestLease(record unified.ActionAuditRecord, now time.Time) unified.ActionPolicyAuthorizationLease {
|
|
lease := unified.ActionPolicyAuthorizationLease{
|
|
Version: 1, OrgID: "default", ActionID: record.ID, ResourceID: record.Request.ResourceID,
|
|
CapabilityName: record.Request.CapabilityName, PlanHash: record.Plan.PlanHash,
|
|
CapabilityPolicyVersion: record.Plan.PolicyVersion, AutoAuthorization: unified.AutoAuthorizeLowRisk,
|
|
ApprovalPolicy: record.Plan.ApprovalPolicy, TenantPolicyVersion: "tenant:v1",
|
|
EffectiveAutonomyLevel: "assisted", LicenseAllowsAutoFix: true,
|
|
ResourcePolicyVersion: "resource:v1", CapabilityNames: []string{"restart"},
|
|
IssuedAt: now, ExpiresAt: now.Add(time.Minute),
|
|
}
|
|
lease.Digest = unified.ActionPolicyAuthorizationDigest(lease)
|
|
return lease
|
|
}
|
|
|
|
func runPolicyAdmissionCommitsAtomically(t *testing.T, store unified.ResourceStore) {
|
|
now := time.Now().UTC()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
|
|
service.Now = func() time.Time { return now }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
completed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
return policyTestLease(record, at), "bounded policy", nil
|
|
})
|
|
if err != nil || completed.State != unified.ActionStateCompleted || executor.calls != 1 {
|
|
t.Fatalf("completed=%#v err=%v calls=%d", completed, err, executor.calls)
|
|
}
|
|
if len(completed.Approvals) != 1 || completed.Approvals[0].Method != unified.MethodPolicy || completed.Approvals[0].PolicyLease == nil {
|
|
t.Fatalf("approvals=%#v", completed.Approvals)
|
|
}
|
|
events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
seenApproved, seenExecuting := false, false
|
|
for _, event := range events {
|
|
seenApproved = seenApproved || event.State == unified.ActionStateApproved
|
|
seenExecuting = seenExecuting || event.State == unified.ActionStateExecuting
|
|
}
|
|
if !seenApproved || !seenExecuting {
|
|
t.Fatalf("events=%#v", events)
|
|
}
|
|
}
|
|
|
|
func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallyMemoryStore(t *testing.T) {
|
|
runPolicyAdmissionCommitsAtomically(t, unified.NewMemoryStore())
|
|
}
|
|
func TestPolicyAdmissionCommitsApprovalAndExecutingAtomicallySQLite(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runPolicyAdmissionCommitsAtomically(t, store)
|
|
}
|
|
|
|
func TestExecuteUnderPolicyRefusesLostReadinessBeforeAutomaticAdmission(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
store := unified.NewMemoryStore()
|
|
executor := &stubExecutor{
|
|
result: &unified.ExecutionResult{Success: true},
|
|
readiness: &unified.ResourceActionReadiness{Name: "restart", Available: true},
|
|
}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
|
|
service.Now = func() time.Time { return now }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
executor.readiness = &unified.ResourceActionReadiness{
|
|
Name: "restart",
|
|
Available: false,
|
|
ReasonCode: "command_agent_disconnected",
|
|
Reason: "command agent disconnected",
|
|
}
|
|
|
|
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
return policyTestLease(record, at), "bounded policy", nil
|
|
})
|
|
if !errors.Is(err, unified.ErrActionExecutionUnavailable) || failed.State != unified.ActionStateFailed {
|
|
t.Fatalf("failed=%#v err=%v", failed, err)
|
|
}
|
|
if executor.calls != 0 || len(failed.Approvals) != 0 {
|
|
t.Fatalf("executor calls=%d approvals=%#v, want no dispatch or policy approval", executor.calls, failed.Approvals)
|
|
}
|
|
if _, found, getErr := store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
|
|
t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
|
|
}
|
|
}
|
|
|
|
func runPolicyBarrierRevocations(t *testing.T, storeFactory func(t *testing.T) unified.ResourceStore) {
|
|
cases := []struct {
|
|
name string
|
|
mutate func(*unified.ActionPolicyAuthorizationLease) error
|
|
}{
|
|
{"policy_row_deleted", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationInvalid
|
|
}},
|
|
{"policy_disabled", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"capability_removed_from_allowlist", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"recurring_window_closed", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"mode_downgraded_to_approval", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"mode_downgraded_to_monitor", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"full_mode_unlock_removed", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"license_removes_effective_mode", func(lease *unified.ActionPolicyAuthorizationLease) error {
|
|
lease.LicenseAllowsAutoFix = false
|
|
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
|
|
return nil
|
|
}},
|
|
{"capability_removed", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrActionPlanDrift }},
|
|
{"safety_class_changed_to_never", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"approval_floor_changed_to_dry_run", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"approval_floor_changed_to_mfa", func(*unified.ActionPolicyAuthorizationLease) error {
|
|
return unified.ErrActionPolicyAuthorizationRevoked
|
|
}},
|
|
{"never_auto_remediate_enabled", func(*unified.ActionPolicyAuthorizationLease) error { return unified.ErrResourceRemediationLocked }},
|
|
{"emergency_stop_enabled", func(lease *unified.ActionPolicyAuthorizationLease) error {
|
|
lease.EmergencyStop = true
|
|
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
|
|
return nil
|
|
}},
|
|
{"policy_store_unreadable", func(*unified.ActionPolicyAuthorizationLease) error { return errors.New("policy store unavailable") }},
|
|
{"lease_wrong_org", func(lease *unified.ActionPolicyAuthorizationLease) error {
|
|
lease.OrgID = "other"
|
|
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
|
|
return nil
|
|
}},
|
|
{"lease_expired", func(lease *unified.ActionPolicyAuthorizationLease) error {
|
|
lease.ExpiresAt = lease.IssuedAt
|
|
lease.Digest = unified.ActionPolicyAuthorizationDigest(*lease)
|
|
return nil
|
|
}},
|
|
{"lease_digest_tampered", func(lease *unified.ActionPolicyAuthorizationLease) error {
|
|
lease.Digest = "sha256:tampered"
|
|
return nil
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
store := storeFactory(t)
|
|
now := time.Now().UTC()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
|
|
service.Now = func() time.Time { return now }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
lease := policyTestLease(record, at)
|
|
if mutationErr := tc.mutate(&lease); mutationErr != nil {
|
|
return unified.ActionPolicyAuthorizationLease{}, "", mutationErr
|
|
}
|
|
return lease, "policy", nil
|
|
})
|
|
if err == nil || failed.State != unified.ActionStateFailed || executor.calls != 0 {
|
|
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
|
|
}
|
|
if len(failed.Approvals) != 0 {
|
|
t.Fatalf("policy approval persisted: %#v", failed.Approvals)
|
|
}
|
|
events, _ := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 20)
|
|
for _, event := range events {
|
|
if event.State == unified.ActionStateExecuting {
|
|
t.Fatalf("executing event persisted: %#v", events)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExecuteUnderPolicyBarrierRevocationsMemoryStore(t *testing.T) {
|
|
runPolicyBarrierRevocations(t, func(*testing.T) unified.ResourceStore { return unified.NewMemoryStore() })
|
|
}
|
|
func TestExecuteUnderPolicyBarrierRevocationsSQLite(t *testing.T) {
|
|
runPolicyBarrierRevocations(t, func(t *testing.T) unified.ResourceStore {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = store.Close() })
|
|
return store
|
|
})
|
|
}
|
|
|
|
func runHumanApprovalSurvivesPolicyRevocation(t *testing.T, store unified.ResourceStore) {
|
|
now := time.Now().UTC()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); err != nil || executor.calls != 1 {
|
|
t.Fatalf("err=%v calls=%d", err, executor.calls)
|
|
}
|
|
}
|
|
func TestHumanApprovalSurvivesAutomaticPolicyRevocationMemoryStore(t *testing.T) {
|
|
runHumanApprovalSurvivesPolicyRevocation(t, unified.NewMemoryStore())
|
|
}
|
|
func TestHumanApprovalSurvivesAutomaticPolicyRevocationSQLite(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runHumanApprovalSurvivesPolicyRevocation(t, store)
|
|
}
|
|
|
|
func runEmergencyStopBlocksHumanAndPolicy(t *testing.T, store unified.ResourceStore) {
|
|
now := time.Now().UTC()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalAdmin), executor)
|
|
service.EmergencyStop = func(string) (bool, error) { return true, nil }
|
|
human := restartRequest()
|
|
human.RequestID = "human-stop"
|
|
plan, err := service.Plan(context.Background(), "default", human, testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, service, "default", plan.ActionID, unified.ActionApprovalRecord{Actor: "operator", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved})); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), ""); !errors.Is(err, unified.ErrActionEmergencyStop) {
|
|
t.Fatalf("human error=%v", err)
|
|
}
|
|
policy := restartRequest()
|
|
policy.RequestID = "policy-stop"
|
|
policyPlan, err := service.Plan(context.Background(), "default", policy, testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = service.ExecuteUnderPolicy(context.Background(), "default", policyPlan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
return policyTestLease(record, at), "policy", nil
|
|
}); !errors.Is(err, unified.ErrActionEmergencyStop) {
|
|
t.Fatalf("policy error=%v", err)
|
|
}
|
|
if executor.calls != 0 {
|
|
t.Fatalf("executor calls=%d", executor.calls)
|
|
}
|
|
}
|
|
func TestEmergencyStopBlocksHumanAndPolicyAdmissionMemoryStore(t *testing.T) {
|
|
runEmergencyStopBlocksHumanAndPolicy(t, unified.NewMemoryStore())
|
|
}
|
|
func TestEmergencyStopBlocksHumanAndPolicyAdmissionSQLite(t *testing.T) {
|
|
store, err := unified.NewSQLiteResourceStore(t.TempDir(), "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runEmergencyStopBlocksHumanAndPolicy(t, store)
|
|
}
|
|
|
|
func TestLegacyPolicyApprovalWithoutLeaseFailsClosedAfterRestart(t *testing.T) {
|
|
dir := t.TempDir()
|
|
first, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Now().UTC()
|
|
request := restartRequest()
|
|
request.Actor = testActionActor("requester", "default")
|
|
request.RequestedBy = request.Actor.SubjectID
|
|
plan, err := (actionplanner.Planner{Now: func() time.Time { return now }}).Plan(request, testResource(now, unified.ApprovalAdmin))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
legacyApproved := unified.ActionAuditRecord{
|
|
ID: plan.ActionID, CreatedAt: now, UpdatedAt: now, State: unified.ActionStateApproved,
|
|
Request: request, Plan: plan,
|
|
Approvals: []unified.ActionApprovalRecord{{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Timestamp: now, Outcome: unified.OutcomeApproved}},
|
|
}
|
|
if err := first.RecordActionAudit(legacyApproved); err != nil {
|
|
t.Fatalf("legacy audit: %v", err)
|
|
}
|
|
if err = first.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer second.Close()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service := serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
|
|
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(_ context.Context, record unified.ActionAuditRecord, at time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
return policyTestLease(record, at), "policy", nil
|
|
})
|
|
if !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
|
|
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestQueuedPolicyActionRevalidatesAfterSQLiteRestart(t *testing.T) {
|
|
dir := t.TempDir()
|
|
first, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
now := time.Now().UTC()
|
|
service := serviceForStore(t, first, testResource(now, unified.ApprovalAdmin), &stubExecutor{})
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = first.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := unified.NewSQLiteResourceStore(dir, "default")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer second.Close()
|
|
executor := &stubExecutor{result: &unified.ExecutionResult{Success: true}}
|
|
service = serviceForStore(t, second, testResource(now, unified.ApprovalAdmin), executor)
|
|
failed, err := service.ExecuteUnderPolicy(context.Background(), "default", plan.ActionID, "pulse_patrol_policy", func(context.Context, unified.ActionAuditRecord, time.Time) (unified.ActionPolicyAuthorizationLease, string, error) {
|
|
return unified.ActionPolicyAuthorizationLease{}, "", unified.ErrActionPolicyAuthorizationRevoked
|
|
})
|
|
if !errors.Is(err, unified.ErrActionPolicyAuthorizationRevoked) || failed.State != unified.ActionStateFailed || executor.calls != 0 {
|
|
t.Fatalf("failed=%#v err=%v calls=%d", failed, err, executor.calls)
|
|
}
|
|
}
|
|
|
|
type stubExecutor struct {
|
|
result *unified.ExecutionResult
|
|
err error
|
|
calls int
|
|
received unified.ActionAuditRecord
|
|
readiness *unified.ResourceActionReadiness
|
|
}
|
|
|
|
type reconcilingExecutor struct {
|
|
executeCalls int
|
|
reconcileCalls int
|
|
executeErr error
|
|
found bool
|
|
result *unified.ExecutionResult
|
|
}
|
|
type bindingReconcilingExecutor struct{ reconcilingExecutor }
|
|
|
|
func (e *bindingReconcilingExecutor) BindActionDispatch(_ context.Context, _ unified.ActionAuditRecord, a unified.ActionDispatchAttempt) (unified.ActionDispatchAttempt, error) {
|
|
return a, nil
|
|
}
|
|
|
|
func (e *reconcilingExecutor) ExecuteAction(_ context.Context, _ unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
|
|
e.executeCalls++
|
|
return nil, e.executeErr
|
|
}
|
|
|
|
func (e *reconcilingExecutor) ReconcileActionDispatch(_ context.Context, record unified.ActionAuditRecord, attempt unified.ActionDispatchAttempt) (*unified.ExecutionResult, unified.ActionDispatchReceipt, bool, error) {
|
|
e.reconcileCalls++
|
|
if !e.found {
|
|
return nil, unified.ActionDispatchReceipt{}, false, nil
|
|
}
|
|
return e.result, unified.ActionDispatchReceipt{
|
|
AttemptID: attempt.ID, ActionID: record.ID, TransportRequestID: attempt.ID, ReceivedAt: time.Now().UTC(),
|
|
}, true, nil
|
|
}
|
|
|
|
func TestLegacyUnboundReceiptPendingAttemptIsInertForBindingOwnedExecutor(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
store := unified.NewMemoryStore()
|
|
executor := &bindingReconcilingExecutor{}
|
|
service := serviceForStore(t, store, testResource(now, unified.ApprovalNone), executor)
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, found, err := store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatal(err)
|
|
}
|
|
started, event, err := unified.BeginActionExecution(record, "operator", now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
attempt, err := unified.NewActionDispatchAttempt(started.ID, now)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RecordActionExecutionAdmission(started, event, attempt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, won, err := store.ClaimActionDispatch(started.ID, "legacy-worker", now, time.Minute)
|
|
if err != nil || !won {
|
|
t.Fatalf("claim=%#v won=%v err=%v", claimed, won, err)
|
|
}
|
|
if _, err := store.MarkActionDispatchStarted(claimed.ID, "legacy-worker", now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:recovery", 10)
|
|
if err != nil || len(recovered) != 1 || recovered[0].State != unified.ActionStateExecuting {
|
|
t.Fatalf("recovered=%#v err=%v", recovered, err)
|
|
}
|
|
if executor.executeCalls != 0 || executor.reconcileCalls != 0 {
|
|
t.Fatalf("execute=%d reconcile=%d", executor.executeCalls, executor.reconcileCalls)
|
|
}
|
|
}
|
|
|
|
func (s *stubExecutor) ExecuteAction(_ context.Context, record unified.ActionAuditRecord) (*unified.ExecutionResult, error) {
|
|
s.calls++
|
|
s.received = record
|
|
return s.result, s.err
|
|
}
|
|
|
|
func (s *stubExecutor) CheckActionAvailable(_ context.Context, _ unified.ActionRequest, _ unified.Resource) unified.ResourceActionReadiness {
|
|
if s.readiness == nil {
|
|
return unified.ResourceActionReadiness{}
|
|
}
|
|
return *s.readiness
|
|
}
|
|
|
|
type serviceEnv struct {
|
|
store unified.ResourceStore
|
|
registry *unified.ResourceRegistry
|
|
executor *stubExecutor
|
|
completed []unified.ActionAuditRecord
|
|
service *Service
|
|
}
|
|
|
|
func testResource(now time.Time, minimumApproval unified.ActionApprovalLevel) unified.Resource {
|
|
return unified.Resource{
|
|
ID: "vm:42",
|
|
Type: unified.ResourceTypeVM,
|
|
Name: "web-42",
|
|
Status: unified.StatusWarning,
|
|
LastSeen: now,
|
|
UpdatedAt: now,
|
|
Sources: []unified.DataSource{unified.SourceProxmox},
|
|
Capabilities: []unified.ResourceCapability{
|
|
{
|
|
Name: "restart",
|
|
Type: unified.CapabilityTypeCommon,
|
|
Description: "Restart the VM",
|
|
MinimumApprovalLevel: minimumApproval,
|
|
InternalHandler: "proxmox.vm.restart",
|
|
Params: []unified.CapabilityParam{
|
|
{Name: "mode", Type: "string", Required: true, Enum: []string{"graceful", "force"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func newServiceEnv(t *testing.T, resource unified.Resource) *serviceEnv {
|
|
t.Helper()
|
|
env := &serviceEnv{
|
|
store: unified.NewMemoryStore(),
|
|
executor: &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}},
|
|
}
|
|
env.registry = unified.NewRegistry(env.store)
|
|
env.registry.IngestResources([]unified.Resource{resource})
|
|
env.service = &Service{
|
|
Registry: func(orgID string) (*unified.ResourceRegistry, error) {
|
|
if orgID != "default" {
|
|
return nil, errors.New("unknown org")
|
|
}
|
|
return env.registry, nil
|
|
},
|
|
Store: func(orgID string) (Store, error) {
|
|
if orgID != "default" {
|
|
return nil, errors.New("unknown org")
|
|
}
|
|
return env.store, nil
|
|
},
|
|
Executor: env.executor,
|
|
DecisionAuthorizer: DecisionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionDecision) error { return nil }),
|
|
ExecutionAuthorizer: ExecutionAuthorizerFunc(func(context.Context, string, unified.ActionAuditRecord, unified.ActionActor) error { return nil }),
|
|
OnActionCompleted: func(record unified.ActionAuditRecord) {
|
|
env.completed = append(env.completed, record)
|
|
},
|
|
}
|
|
return env
|
|
}
|
|
|
|
func restartRequest() unified.ActionRequest {
|
|
return unified.ActionRequest{
|
|
RequestID: "req-1",
|
|
ResourceID: "vm:42",
|
|
CapabilityName: "restart",
|
|
Params: map[string]any{"mode": "graceful"},
|
|
Reason: "Recover after confirmed outage",
|
|
RequestedBy: "agent:test",
|
|
}
|
|
}
|
|
|
|
func TestPlanPersistsPendingAuditAndLifecycle(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if !plan.RequiresApproval {
|
|
t.Fatal("expected admin-gated capability to require approval")
|
|
}
|
|
|
|
record, ok, err := env.store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
|
|
}
|
|
if record.State != unified.ActionStatePending {
|
|
t.Fatalf("audit state = %q, want pending_approval", record.State)
|
|
}
|
|
events, err := env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionLifecycleEvents: %v", err)
|
|
}
|
|
if len(events) != 2 {
|
|
t.Fatalf("lifecycle events = %d, want planned + pending", len(events))
|
|
}
|
|
|
|
// Idempotent replan must not duplicate lifecycle events.
|
|
if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); err != nil {
|
|
t.Fatalf("replan: %v", err)
|
|
}
|
|
events, err = env.store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionLifecycleEvents after replan: %v", err)
|
|
}
|
|
if len(events) != 2 {
|
|
t.Fatalf("replan duplicated lifecycle events: %d", len(events))
|
|
}
|
|
}
|
|
|
|
func TestPlanFollowsAdvertisedLifecycleSynonym(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
// The fixture advertises only "restart"; a request for the lifecycle
|
|
// synonym "reboot" must plan the advertised verb.
|
|
req := restartRequest()
|
|
req.CapabilityName = "reboot"
|
|
plan, err := env.service.Plan(context.Background(), "default", req, testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan with lifecycle synonym: %v", err)
|
|
}
|
|
record, ok, err := env.store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
|
|
}
|
|
if record.Request.CapabilityName != "restart" {
|
|
t.Fatalf("persisted capability = %q, want the advertised verb %q", record.Request.CapabilityName, "restart")
|
|
}
|
|
|
|
// Verbs without an advertised synonym still fail closed.
|
|
unknown := restartRequest()
|
|
unknown.CapabilityName = "shutdown"
|
|
var capErr *CapabilityNotFoundError
|
|
if _, err := env.service.Plan(context.Background(), "default", unknown, testActionActor("requester", "default")); !errors.As(err, &capErr) {
|
|
t.Fatalf("non-synonym verb error = %v, want CapabilityNotFoundError", err)
|
|
}
|
|
}
|
|
|
|
func TestPlanFailsClosedOnUnknownResourceAndCapability(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
missing := restartRequest()
|
|
missing.ResourceID = "vm:404"
|
|
var notFound *ResourceNotFoundError
|
|
if _, err := env.service.Plan(context.Background(), "default", missing, testActionActor("requester", "default")); !errors.As(err, ¬Found) {
|
|
t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
|
|
}
|
|
|
|
unknownCap := restartRequest()
|
|
unknownCap.CapabilityName = "detonate"
|
|
_, err := env.service.Plan(context.Background(), "default", unknownCap, testActionActor("requester", "default"))
|
|
var capErr *CapabilityNotFoundError
|
|
if !errors.As(err, &capErr) || !errors.Is(err, actionplanner.ErrCapabilityNotFound) {
|
|
t.Fatalf("unknown capability error = %v, want CapabilityNotFoundError wrapping ErrCapabilityNotFound", err)
|
|
}
|
|
if capErr.ResourceID != "vm:42" || capErr.CapabilityName != "detonate" {
|
|
t.Fatalf("capability error detail = %#v", capErr)
|
|
}
|
|
|
|
empty := restartRequest()
|
|
empty.ResourceID = " "
|
|
var validation *actionplanner.ValidationError
|
|
if _, err := env.service.Plan(context.Background(), "default", empty, testActionActor("requester", "default")); !errors.As(err, &validation) {
|
|
t.Fatalf("empty resource id error = %v, want ValidationError", err)
|
|
}
|
|
|
|
audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionAudits: %v", err)
|
|
}
|
|
if len(audits) != 0 {
|
|
t.Fatalf("failed plans must not persist audits, got %d", len(audits))
|
|
}
|
|
}
|
|
|
|
func TestPlanAvailabilityRefusalPersistsNothing(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
env.executor.readiness = &unified.ResourceActionReadiness{
|
|
Name: "restart",
|
|
Available: false,
|
|
ReasonCode: "agent_disconnected",
|
|
Reason: "no connected command agent",
|
|
}
|
|
|
|
_, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
var refused *AvailabilityRefusedError
|
|
if !errors.As(err, &refused) {
|
|
t.Fatalf("error = %v, want AvailabilityRefusedError", err)
|
|
}
|
|
if refused.Readiness.ReasonCode != "agent_disconnected" {
|
|
t.Fatalf("readiness = %#v", refused.Readiness)
|
|
}
|
|
audits, err := env.store.GetActionAudits("vm:42", time.Time{}, 10)
|
|
if err != nil {
|
|
t.Fatalf("GetActionAudits: %v", err)
|
|
}
|
|
if len(audits) != 0 {
|
|
t.Fatalf("refused availability must not persist audits, got %d", len(audits))
|
|
}
|
|
}
|
|
|
|
func TestDecideApprovesPendingAction(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
|
|
updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
|
|
Actor: "operator@example.com",
|
|
Method: unified.MethodAPI,
|
|
Outcome: unified.OutcomeApproved,
|
|
Reason: "confirmed outage",
|
|
}))
|
|
|
|
if err != nil {
|
|
t.Fatalf("Decide: %v", err)
|
|
}
|
|
if updated.State != unified.ActionStateApproved {
|
|
t.Fatalf("state = %q, want approved", updated.State)
|
|
}
|
|
if len(updated.Approvals) != 1 || updated.Approvals[0].Actor != "operator@example.com" {
|
|
t.Fatalf("approvals = %#v", updated.Approvals)
|
|
}
|
|
|
|
var notFound *ActionNotFoundError
|
|
if _, err := env.service.Decide(context.Background(), "default", "act_missing", testActionDecision(t, env.service, "default", "act_missing", unified.ActionApprovalRecord{
|
|
Actor: "operator@example.com", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
|
|
})); !errors.As(err, ¬Found) {
|
|
t.Fatalf("unknown action error = %v, want ActionNotFoundError", err)
|
|
}
|
|
}
|
|
|
|
func TestExecuteRunsApprovedActionToTerminalAudit(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
|
|
Actor: "operator@example.com", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
|
|
})); err != nil {
|
|
t.Fatalf("Decide: %v", err)
|
|
}
|
|
|
|
completed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator@example.com", "default"), "approved restart")
|
|
if err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
if completed.State != unified.ActionStateCompleted {
|
|
t.Fatalf("state = %q, want completed", completed.State)
|
|
}
|
|
if env.executor.calls != 1 {
|
|
t.Fatalf("executor calls = %d, want 1", env.executor.calls)
|
|
}
|
|
if len(env.completed) != 1 || env.completed[0].ID != plan.ActionID {
|
|
t.Fatalf("completion publisher observed %#v", env.completed)
|
|
}
|
|
|
|
record, ok, err := env.store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
|
|
}
|
|
if record.State != unified.ActionStateCompleted || record.Result == nil || !record.Result.Success {
|
|
t.Fatalf("terminal audit = state %q result %#v", record.State, record.Result)
|
|
}
|
|
}
|
|
|
|
func TestExecuteRefusesLostReadinessBeforeAdmission(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
env.executor.readiness = &unified.ResourceActionReadiness{Name: "restart", Available: true}
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
env.executor.readiness = &unified.ResourceActionReadiness{
|
|
Name: "restart",
|
|
Available: false,
|
|
ReasonCode: "command_agent_disconnected",
|
|
Reason: "command agent disconnected",
|
|
}
|
|
|
|
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("requester", "default"), "")
|
|
var refused *AvailabilityRefusedError
|
|
if !errors.As(err, &refused) || !errors.Is(err, unified.ErrActionExecutionUnavailable) {
|
|
t.Fatalf("error = %v, want AvailabilityRefusedError wrapping ErrActionExecutionUnavailable", err)
|
|
}
|
|
if failed.State != unified.ActionStateFailed || failed.Result == nil ||
|
|
failed.Result.ActionResultV2 == nil ||
|
|
failed.Result.ActionResultV2.Execution.ReasonCode != "action_execution_unavailable" ||
|
|
!strings.HasPrefix(failed.Result.ErrorMessage, "action_execution_unavailable: command agent disconnected") {
|
|
t.Fatalf("failed audit = %#v", failed)
|
|
}
|
|
if env.executor.calls != 0 {
|
|
t.Fatalf("executor calls = %d, want no dispatch", env.executor.calls)
|
|
}
|
|
if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
|
|
t.Fatalf("completion publisher observed %#v", env.completed)
|
|
}
|
|
if _, found, getErr := env.store.GetActionDispatchAttempt(plan.ActionID); getErr != nil || found {
|
|
t.Fatalf("dispatch attempt found=%v err=%v, want none", found, getErr)
|
|
}
|
|
}
|
|
|
|
func TestExecuteRefusesUnapprovedActionWithoutDispatch(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator@example.com", "default"), ""); !errors.Is(err, unified.ErrActionNotApproved) {
|
|
t.Fatalf("error = %v, want ErrActionNotApproved", err)
|
|
}
|
|
if env.executor.calls != 0 {
|
|
t.Fatalf("executor must not run for unapproved actions, calls = %d", env.executor.calls)
|
|
}
|
|
}
|
|
|
|
func TestExecuteRefusesRemediationLockedResource(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
NeverAutoRemediate: true,
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
|
|
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
|
|
if !errors.Is(err, unified.ErrResourceRemediationLocked) {
|
|
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
|
|
}
|
|
if env.executor.calls != 0 {
|
|
t.Fatalf("executor must not run on locked resources, calls = %d", env.executor.calls)
|
|
}
|
|
if failed.State != unified.ActionStateFailed {
|
|
t.Fatalf("refusal must persist a terminal failed audit, state = %q", failed.State)
|
|
}
|
|
if len(env.completed) != 1 || env.completed[0].State != unified.ActionStateFailed {
|
|
t.Fatalf("refusal must publish the failed record, got %#v", env.completed)
|
|
}
|
|
}
|
|
|
|
func TestExecuteRefusesDriftedPlan(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
|
|
// The capability policy tightens between plan and dispatch: the
|
|
// replanned contract no longer matches the recorded plan hash.
|
|
drifted := testResource(now, unified.ApprovalAdmin)
|
|
env.registry = unified.NewRegistry(env.store)
|
|
env.registry.IngestResources([]unified.Resource{drifted})
|
|
|
|
failed, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), "")
|
|
if !errors.Is(err, unified.ErrActionPlanDrift) {
|
|
t.Fatalf("error = %v, want ErrActionPlanDrift", err)
|
|
}
|
|
if env.executor.calls != 0 {
|
|
t.Fatalf("executor must not run on drifted plans, calls = %d", env.executor.calls)
|
|
}
|
|
if failed.State != unified.ActionStateFailed {
|
|
t.Fatalf("drift refusal must persist a terminal failed audit, state = %q", failed.State)
|
|
}
|
|
}
|
|
|
|
func TestExecuteFailsClosedWithoutExecutor(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
env.service.Executor = nil
|
|
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrExecutorUnavailable) {
|
|
t.Fatalf("error = %v, want ErrExecutorUnavailable", err)
|
|
}
|
|
}
|
|
|
|
func TestLifecycleFailsClosedWithoutStore(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
env.service.Store = func(string) (Store, error) { return nil, errors.New("db offline") }
|
|
|
|
if _, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default")); !errors.Is(err, ErrStoreUnavailable) {
|
|
t.Fatalf("Plan error = %v, want ErrStoreUnavailable", err)
|
|
}
|
|
if _, err := env.service.Decide(context.Background(), "default", "act_x", testActionDecision(t, env.service, "default", "act_x", unified.ActionApprovalRecord{Outcome: unified.OutcomeApproved})); !errors.Is(err, ErrStoreUnavailable) {
|
|
t.Fatalf("Decide error = %v, want ErrStoreUnavailable", err)
|
|
}
|
|
if _, err := env.service.Execute(context.Background(), "default", "act_x", testActionActor("agent:test", "default"), ""); !errors.Is(err, ErrStoreUnavailable) {
|
|
t.Fatalf("Execute error = %v, want ErrStoreUnavailable", err)
|
|
}
|
|
}
|
|
|
|
func TestCapabilitiesUsesSameRegistryResolutionAsPlanning(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
capabilities, err := env.service.Capabilities(context.Background(), "default", "vm:42")
|
|
if err != nil {
|
|
t.Fatalf("Capabilities: %v", err)
|
|
}
|
|
if len(capabilities) != 1 || capabilities[0].Name != "restart" {
|
|
t.Fatalf("capabilities = %#v", capabilities)
|
|
}
|
|
|
|
var notFound *ResourceNotFoundError
|
|
if _, err := env.service.Capabilities(context.Background(), "default", "vm:404"); !errors.As(err, ¬Found) {
|
|
t.Fatalf("unknown resource error = %v, want ResourceNotFoundError", err)
|
|
}
|
|
}
|
|
|
|
func TestPlanWithOptionsPersistsOriginAcrossLifecycle(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
|
|
origin := &unified.ActionOrigin{
|
|
Surface: "patrol",
|
|
FindingID: "finding-9",
|
|
InvestigationID: "inv-9",
|
|
ProposalID: "prop-9",
|
|
}
|
|
plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), Origin: origin})
|
|
if err != nil {
|
|
t.Fatalf("PlanWithOptions: %v", err)
|
|
}
|
|
|
|
record, ok, err := env.store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit: ok=%v err=%v", ok, err)
|
|
}
|
|
if record.Origin == nil || record.Origin.FindingID != "finding-9" {
|
|
t.Fatalf("plan audit origin = %#v", record.Origin)
|
|
}
|
|
|
|
// Origin must survive the decision transition: the decision persists
|
|
// the record loaded from the store, so the broker-owned metadata
|
|
// stays reconcilable at approval time.
|
|
updated, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
|
|
Actor: "operator@example.com", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
|
|
}))
|
|
|
|
if err != nil {
|
|
t.Fatalf("Decide: %v", err)
|
|
}
|
|
if updated.Origin == nil || updated.Origin.ProposalID != "prop-9" {
|
|
t.Fatalf("decision record origin = %#v", updated.Origin)
|
|
}
|
|
|
|
// Plain Plan must never stamp an origin.
|
|
plain := restartRequest()
|
|
plain.RequestID = "req-plain"
|
|
plainPlan, err := env.service.Plan(context.Background(), "default", plain, testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
plainRecord, ok, err := env.store.GetActionAudit(plainPlan.ActionID)
|
|
if err != nil || !ok {
|
|
t.Fatalf("GetActionAudit(plain): ok=%v err=%v", ok, err)
|
|
}
|
|
if plainRecord.Origin != nil {
|
|
t.Fatalf("plain plan must not carry an origin, got %#v", plainRecord.Origin)
|
|
}
|
|
}
|
|
|
|
func TestOnActionTransitionFiresAfterEachPersistedState(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
var transitions []unified.ActionState
|
|
var transitionOrgs []string
|
|
env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
|
|
transitions = append(transitions, record.State)
|
|
transitionOrgs = append(transitionOrgs, orgID)
|
|
}
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if _, err := env.service.Decide(context.Background(), "default", plan.ActionID, testActionDecision(t, env.service, "default", plan.ActionID, unified.ActionApprovalRecord{
|
|
Actor: "operator@example.com", Method: unified.MethodAPI, Outcome: unified.OutcomeApproved,
|
|
})); err != nil {
|
|
t.Fatalf("Decide: %v", err)
|
|
}
|
|
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator@example.com", "default"), ""); err != nil {
|
|
t.Fatalf("Execute: %v", err)
|
|
}
|
|
|
|
want := []unified.ActionState{unified.ActionStatePending, unified.ActionStateApproved, unified.ActionStateExecuting, unified.ActionStateCompleted}
|
|
if len(transitions) != len(want) {
|
|
t.Fatalf("transitions = %v, want %v", transitions, want)
|
|
}
|
|
for i := range want {
|
|
if transitions[i] != want[i] {
|
|
t.Fatalf("transitions = %v, want %v", transitions, want)
|
|
}
|
|
if transitionOrgs[i] != "default" {
|
|
t.Fatalf("transition org = %q, want default", transitionOrgs[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestOnActionTransitionFiresForPersistedRefusals(t *testing.T) {
|
|
now := time.Now().UTC()
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalNone))
|
|
var transitions []unified.ActionState
|
|
env.service.OnActionTransition = func(orgID string, record unified.ActionAuditRecord) {
|
|
transitions = append(transitions, record.State)
|
|
}
|
|
|
|
plan, err := env.service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatalf("Plan: %v", err)
|
|
}
|
|
if err := env.store.SetResourceOperatorState(unified.ResourceOperatorState{
|
|
CanonicalID: "vm:42",
|
|
NeverAutoRemediate: true,
|
|
}); err != nil {
|
|
t.Fatalf("SetResourceOperatorState: %v", err)
|
|
}
|
|
if _, err := env.service.Execute(context.Background(), "default", plan.ActionID, testActionActor("agent:test", "default"), ""); !errors.Is(err, unified.ErrResourceRemediationLocked) {
|
|
t.Fatalf("error = %v, want ErrResourceRemediationLocked", err)
|
|
}
|
|
|
|
want := []unified.ActionState{unified.ActionStatePlanned, unified.ActionStateFailed}
|
|
if len(transitions) != len(want) || transitions[0] != want[0] || transitions[1] != want[1] {
|
|
t.Fatalf("transitions = %v, want %v", transitions, want)
|
|
}
|
|
}
|
|
|
|
func TestActionPolicyProvenanceParticipatesInFreshnessWithoutAuthorizingDispatch(t *testing.T) {
|
|
now := time.Date(2026, 7, 11, 21, 0, 0, 0, time.UTC)
|
|
env := newServiceEnv(t, testResource(now, unified.ApprovalAdmin))
|
|
factor := unified.ActionPolicyAuthorityFactor{
|
|
Kind: unified.ActionPolicyAuthorityTenant, SourceID: "patrol-tenant-policy",
|
|
Revision: "tenant-policy:sha256:0123456789abcdef01234567", Status: unified.ActionPolicyAuthorityConsulted,
|
|
ReasonCodes: []unified.ActionPolicyReasonCode{unified.PolicyReasonTenantModeAssisted, unified.PolicyReasonTenantFullLocked},
|
|
}
|
|
plan, err := env.service.PlanWithOptions(context.Background(), "default", restartRequest(), PlanOptions{Actor: testActionActor("requester", "default"), PolicyFactors: []unified.ActionPolicyAuthorityFactor{factor}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record, found, err := env.service.Get("default", plan.ActionID)
|
|
if err != nil || !found {
|
|
t.Fatalf("Get: found=%v err=%v", found, err)
|
|
}
|
|
if err := env.service.ValidatePlanFresh("default", record); err != nil {
|
|
t.Fatalf("captured descriptive snapshot should replan deterministically: %v", err)
|
|
}
|
|
record.Plan.PolicyDecision.Authorities[1].Revision = "tenant-policy:sha256:1123456789abcdef01234567"
|
|
record.Plan.PolicyDecision.DecisionID = unified.ActionPolicyDecisionDigest(record.Plan.PolicyDecision)
|
|
if err := env.service.ValidatePlanFresh("default", record); !errors.Is(err, unified.ErrActionPlanDrift) {
|
|
t.Fatalf("provenance drift error=%v", err)
|
|
}
|
|
if _, _, _, err := unified.BeginPolicyActionExecution(record, unified.ActionApprovalRecord{Actor: "pulse_patrol_policy", Method: unified.MethodPolicy, Outcome: unified.OutcomeApproved, Timestamp: now}, unified.ActionPolicyAuthorizationLease{}, now); !errors.Is(err, unified.ErrActionPolicyAuthorizationInvalid) {
|
|
t.Fatalf("descriptive provenance authorized dispatch: %v", err)
|
|
}
|
|
}
|
|
|
|
// Issue1649: an executing action whose agent never returns completion evidence
|
|
// used to have no exit at all. Expiry skips executing rows and only a terminal
|
|
// agent receipt could complete one, so a Docker update interrupted by an agent
|
|
// restart stayed "executing" forever with no operator route out. Reconciliation
|
|
// now self-heals the known stranded shapes; ForceFail is the manual override
|
|
// for anything left, and it must refuse to rewrite settled truth.
|
|
func TestIssue1649ForceFailTerminalizesWedgedExecutingActionWithAudit(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
executor := &reconcilingExecutor{executeErr: context.DeadlineExceeded, found: false}
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), executor)
|
|
var completed []unified.ActionAuditRecord
|
|
var transitions []unified.ActionAuditRecord
|
|
service.OnActionCompleted = func(record unified.ActionAuditRecord) { completed = append(completed, record) }
|
|
service.OnActionTransition = func(_ string, record unified.ActionAuditRecord) { transitions = append(transitions, record) }
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "wedge it"); !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("Execute error=%v", err)
|
|
}
|
|
if current, _, err := service.Get("default", plan.ActionID); err != nil || current.State != unified.ActionStateExecuting {
|
|
t.Fatalf("precondition current=%#v err=%v", current, err)
|
|
}
|
|
|
|
failed, err := service.ForceFail("default", plan.ActionID, "operator", "verified by hand, container is running the new image")
|
|
if err != nil || failed.State != unified.ActionStateFailed || failed.Result == nil || failed.Result.ActionResultV2 == nil {
|
|
t.Fatalf("ForceFail failed=%#v err=%v", failed, err)
|
|
}
|
|
truth := failed.Result.ActionResultV2
|
|
if truth.Execution.Status != unified.ActionExecutionInconclusive || truth.Execution.ReasonCode != ForceFailReasonCode {
|
|
t.Fatalf("execution truth=%#v", truth.Execution)
|
|
}
|
|
if truth.Verification.Status != unified.ActionVerificationInconclusive || truth.Verification.EvidenceClass != unified.ActionEvidenceNone {
|
|
t.Fatalf("verification truth=%#v", truth.Verification)
|
|
}
|
|
if !strings.Contains(failed.Result.ErrorMessage, "verified by hand") {
|
|
t.Fatalf("operator reason missing from audit result: %q", failed.Result.ErrorMessage)
|
|
}
|
|
if executor.executeCalls != 1 {
|
|
t.Fatalf("ForceFail touched the transport: execute=%d", executor.executeCalls)
|
|
}
|
|
persisted, found, err := store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !found || persisted.State != unified.ActionStateFailed {
|
|
t.Fatalf("persisted=%#v found=%v err=%v", persisted, found, err)
|
|
}
|
|
events, err := store.GetActionLifecycleEvents(plan.ActionID, time.Time{}, 100)
|
|
if err != nil || len(events) == 0 {
|
|
t.Fatalf("events=%#v err=%v", events, err)
|
|
}
|
|
terminal := events[0]
|
|
if terminal.State != unified.ActionStateFailed || terminal.Actor != "operator" || !strings.Contains(terminal.Message, "Operator force-failed") {
|
|
t.Fatalf("terminal audit event=%#v", terminal)
|
|
}
|
|
if len(completed) != 1 || completed[0].State != unified.ActionStateFailed {
|
|
t.Fatalf("completed=%#v", completed)
|
|
}
|
|
if len(transitions) == 0 || transitions[len(transitions)-1].State != unified.ActionStateFailed {
|
|
t.Fatalf("transitions=%#v", transitions)
|
|
}
|
|
// The row is settled, so the recovery loop stops reconciling it and the
|
|
// override cannot be replayed over the persisted outcome.
|
|
recovered, err := service.RecoverExecutingActions(context.Background(), "default", "system:restart-recovery", 100)
|
|
if err != nil || len(recovered) != 0 {
|
|
t.Fatalf("recovered=%#v err=%v", recovered, err)
|
|
}
|
|
if _, err := service.ForceFail("default", plan.ActionID, "operator", "again"); !errors.Is(err, unified.ErrActionExecutionFinal) {
|
|
t.Fatalf("replayed ForceFail error=%v", err)
|
|
}
|
|
}
|
|
|
|
func TestIssue1649ForceFailRefusesTerminalAndUndispatchedActions(t *testing.T) {
|
|
store := unified.NewMemoryStore()
|
|
service := serviceForStore(t, store, testResource(time.Now().UTC(), unified.ApprovalNone), &stubExecutor{result: &unified.ExecutionResult{Success: true, Output: "restarted"}})
|
|
plan, err := service.Plan(context.Background(), "default", restartRequest(), testActionActor("requester", "default"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Planned but never dispatched: expiry still owns this row, so the
|
|
// override must not manufacture an unknown-effect failure for it.
|
|
if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionNotExecuting) {
|
|
t.Fatalf("undispatched ForceFail error=%v", err)
|
|
}
|
|
current, err := service.Execute(context.Background(), "default", plan.ActionID, testActionActor("operator", "default"), "run it")
|
|
if err != nil || current.State != unified.ActionStateCompleted {
|
|
t.Fatalf("current=%#v err=%v", current, err)
|
|
}
|
|
if _, err := service.ForceFail("default", plan.ActionID, "operator", ""); !errors.Is(err, unified.ErrActionExecutionFinal) {
|
|
t.Fatalf("completed ForceFail error=%v", err)
|
|
}
|
|
settled, found, err := store.GetActionAudit(plan.ActionID)
|
|
if err != nil || !found || settled.State != unified.ActionStateCompleted || !settled.Result.Success {
|
|
t.Fatalf("refused override rewrote settled truth: %#v found=%v err=%v", settled, found, err)
|
|
}
|
|
var notFound *ActionNotFoundError
|
|
if _, err := service.ForceFail("default", "act_missing", "operator", ""); !errors.As(err, ¬Found) {
|
|
t.Fatalf("missing action ForceFail error=%v", err)
|
|
}
|
|
}
|