mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
rcourtman
2a7019b0fa
Proxmox planning and dispatch now require a unique credential-admitted typed runner with durable receipts. Development authentication preserves explicit bearer identity so runner activation keeps its tenant and credential scope. Preserve observer and receiver timestamps from their separate clocks instead of rejecting or rewriting valid evidence. Keep completed execution separate from stale or inconclusive verification, and label independent observations accurately in action reviews. Verified with targeted race suites, action-review tests and frontend build, plus a real Assistant start plan and approved VM110 start/stop with independent Proxmox confirmation. Final action reviews passed Playwright at 1440, 900 and 390 pixels, including retained completed, rejected and expired history.
311 lines
18 KiB
Go
311 lines
18 KiB
Go
package api
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
|
|
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
)
|
|
|
|
func dockerContainerExecutionResult(resourceID, agentID string, req agentexec.DockerContainerLifecyclePayload, facts agentexec.DockerContainerLifecycleResultPayload, independent *dockerContainerPostconditionObservation, receivedAt time.Time) (*unified.ExecutionResult, error) {
|
|
summary := dockerContainerFactSummary(facts)
|
|
execution := unified.ActionExecutionTruth{Status: unified.ActionExecutionNotRun, ReasonCode: actionPreflightReasonCode(facts.ReasonCode), Summary: summary}
|
|
if facts.MutationCompleted {
|
|
execution = unified.ActionExecutionTruth{Status: unified.ActionExecutionSucceeded, Summary: summary}
|
|
} else if facts.MutationStarted {
|
|
execution = unified.ActionExecutionTruth{Status: unified.ActionExecutionInconclusive, ReasonCode: "possible_partial_effect", Summary: summary}
|
|
}
|
|
|
|
verification := unified.ActionVerificationTruth{Status: unified.ActionVerificationInconclusive, EvidenceClass: unified.ActionEvidenceNone, ReasonCode: "agent_readback_unavailable", Summary: "The typed operation did not return a usable container readback."}
|
|
if facts.ReadbackRan {
|
|
if !freshDockerLifecycleObservation(facts.Before.ObservedAt, facts.After.ObservedAt, receivedAt) {
|
|
verification.ReasonCode = "stale_agent_readback"
|
|
verification.Summary = "The agent readback was stale, skewed, or had invalid mutation chronology."
|
|
} else {
|
|
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
|
|
Version: unified.ActionEvidenceVersion, ID: req.Operation + "-agent-readback", ObserverID: agentID,
|
|
ObserverKind: "unified_agent", ObserverTrustDomain: "agent:" + agentID, ExecutorTrustDomain: "agent:" + agentID,
|
|
Method: "typed_container_read_after_write", SubjectID: resourceID, ObservedAt: facts.After.ObservedAt.UTC(), ReceivedAt: receivedAt.UTC(), Summary: summary,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("normalize docker lifecycle evidence: %w", err)
|
|
}
|
|
status := unified.ActionVerificationContradicted
|
|
reason := "postcondition_contradicted"
|
|
if dockerLifecycleRunningHealthUnknown(req.Operation, facts.After) {
|
|
status = unified.ActionVerificationInconclusive
|
|
reason = "container_health_unknown"
|
|
} else if dockerLifecycleFactsMatch(req.Operation, facts.Before, facts.After) {
|
|
status = unified.ActionVerificationConfirmed
|
|
reason = ""
|
|
}
|
|
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceAgentAttested, ReasonCode: reason, Summary: summary, Evidence: []unified.ActionEvidence{evidence}}
|
|
}
|
|
}
|
|
if independent != nil && freshDockerLifecycleObservation(facts.Before.ObservedAt, independent.Snapshot.ObservedAt, independent.ReceivedAt) && strings.TrimSpace(independent.TrustDomain) != "" && strings.TrimSpace(independent.TrustDomain) != "agent:"+agentID && strings.EqualFold(independent.Snapshot.ContainerID, facts.ContainerID) {
|
|
observationDigest, err := operationreceipt.DigestCanonicalJSON(struct {
|
|
ActionID string `json:"action_id"`
|
|
SubjectID string `json:"subject_id"`
|
|
Before agentexec.DockerContainerLifecycleSnapshot `json:"before"`
|
|
After agentexec.DockerContainerObservationSnapshot `json:"after"`
|
|
}{req.ActionID, resourceID, facts.Before, independent.Snapshot})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
independentSummary := fmt.Sprintf("Direct daemon observation: before=%s; after=%s; running=%t; health=%s", facts.Before.State, independent.Snapshot.State, independent.Snapshot.Running, independent.Snapshot.Health)
|
|
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
|
|
Version: unified.ActionEvidenceVersion, ID: req.ActionID + "-direct-daemon-observation", ObserverID: independent.ObserverID,
|
|
ObserverKind: "docker_daemon_observer", ObserverTrustDomain: independent.TrustDomain, ExecutorTrustDomain: "agent:" + agentID,
|
|
Method: independent.Method, SubjectID: resourceID, ObservedAt: independent.Snapshot.ObservedAt.UTC(), ReceivedAt: independent.ReceivedAt.UTC(),
|
|
Summary: independentSummary, Refs: []unified.ActionEvidenceRef{{ID: req.ActionID, Kind: "docker_before_after", Digest: observationDigest}},
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("normalize independent docker observation: %w", err)
|
|
}
|
|
status := unified.ActionVerificationContradicted
|
|
reason := "postcondition_contradicted"
|
|
if dockerLifecycleObservationFactsMatch(req.Operation, facts.Before, independent.Snapshot) {
|
|
status = unified.ActionVerificationConfirmed
|
|
reason = ""
|
|
}
|
|
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceIndependent, ReasonCode: reason, Summary: independentSummary, Evidence: []unified.ActionEvidence{evidence}}
|
|
}
|
|
|
|
canonical := unified.ActionResultV2{
|
|
Version: unified.ActionResultV2Version, Execution: execution, Verification: verification,
|
|
Compensation: unified.ActionCompensationTruth{Support: unified.ActionCompensationUnavailable, Status: unified.ActionCompensationNotAvailable, Summary: "Container restart is non-rollbackable. Lab fixture cleanup is compensation for the disposable test environment, not product rollback."},
|
|
}
|
|
legacy := &unified.ExecutionResult{Output: summary}
|
|
projected, _, err := unified.ApplyActionResultV2(legacy, canonical)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("apply canonical docker lifecycle action truth: %w", err)
|
|
}
|
|
projected.Verification = unified.LegacyActionVerificationFromV2(*projected.ActionResultV2)
|
|
return projected, nil
|
|
}
|
|
|
|
func dockerContainerUpdateExecutionResult(resourceID, agentID string, facts agentexec.DockerContainerUpdateResultPayload, independent *dockerContainerPostconditionObservation, receivedAt time.Time) (*unified.ExecutionResult, error) {
|
|
summary := dockerContainerUpdateFactSummary(facts)
|
|
execution := unified.ActionExecutionTruth{Status: unified.ActionExecutionNotRun, ReasonCode: actionPreflightReasonCode(facts.ReasonCode), Summary: summary}
|
|
if facts.MutationCompleted {
|
|
execution = unified.ActionExecutionTruth{Status: unified.ActionExecutionSucceeded, Summary: summary}
|
|
} else if facts.MutationStarted {
|
|
execution = unified.ActionExecutionTruth{Status: unified.ActionExecutionInconclusive, ReasonCode: "possible_partial_effect", Summary: summary}
|
|
}
|
|
|
|
verification := unified.ActionVerificationTruth{Status: unified.ActionVerificationInconclusive, EvidenceClass: unified.ActionEvidenceNone, ReasonCode: "agent_readback_unavailable", Summary: "The typed operation did not return a usable replacement-container readback."}
|
|
if facts.ReadbackRan {
|
|
if !freshDockerUpdateObservation(facts.After.ObservedAt, receivedAt) {
|
|
verification.ReasonCode = "stale_agent_readback"
|
|
verification.Summary = "The agent readback was stale or skewed."
|
|
} else {
|
|
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
|
|
Version: unified.ActionEvidenceVersion, ID: facts.Operation + "-agent-readback", ObserverID: agentID,
|
|
ObserverKind: "unified_agent", ObserverTrustDomain: "agent:" + agentID, ExecutorTrustDomain: "agent:" + agentID,
|
|
Method: "typed_container_read_after_write", SubjectID: resourceID, ObservedAt: facts.After.ObservedAt.UTC(), ReceivedAt: receivedAt.UTC(), Summary: summary,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("normalize docker update evidence: %w", err)
|
|
}
|
|
status := unified.ActionVerificationContradicted
|
|
reason := "postcondition_contradicted"
|
|
// A running replacement must also have a usable running/health
|
|
// readback; identity alone must not confirm an unhealthy update.
|
|
// Preserve updates of containers intentionally left stopped.
|
|
if facts.After.Running && !agentexec.IsDockerContainerHealth(facts.After.Health) {
|
|
status = unified.ActionVerificationInconclusive
|
|
reason = "container_health_unknown"
|
|
} else if dockerUpdateFactsMatch(facts, facts.After.ContainerID) &&
|
|
(!facts.After.Running || (facts.After.State == "running" &&
|
|
agentexec.DockerContainerHealthAllowsVerifiedRunningState(facts.After.Health))) {
|
|
status = unified.ActionVerificationConfirmed
|
|
reason = ""
|
|
}
|
|
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceAgentAttested, ReasonCode: reason, Summary: summary, Evidence: []unified.ActionEvidence{evidence}}
|
|
}
|
|
}
|
|
if independent != nil && facts.NewContainerID != "" && freshDockerUpdateObservation(independent.Snapshot.ObservedAt, independent.ReceivedAt) && strings.TrimSpace(independent.TrustDomain) != "" && strings.TrimSpace(independent.TrustDomain) != "agent:"+agentID {
|
|
observationDigest, err := operationreceipt.DigestCanonicalJSON(struct {
|
|
ActionID string `json:"action_id"`
|
|
SubjectID string `json:"subject_id"`
|
|
NewContainerID string `json:"new_container_id"`
|
|
After agentexec.DockerContainerObservationSnapshot `json:"after"`
|
|
}{facts.ActionID, resourceID, facts.NewContainerID, independent.Snapshot})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
independentSummary := fmt.Sprintf("Direct daemon observation of replacement container: state=%s; running=%t", independent.Snapshot.State, independent.Snapshot.Running)
|
|
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
|
|
Version: unified.ActionEvidenceVersion, ID: facts.ActionID + "-direct-daemon-observation", ObserverID: independent.ObserverID,
|
|
ObserverKind: "docker_daemon_observer", ObserverTrustDomain: independent.TrustDomain, ExecutorTrustDomain: "agent:" + agentID,
|
|
Method: independent.Method, SubjectID: resourceID, ObservedAt: independent.Snapshot.ObservedAt.UTC(), ReceivedAt: independent.ReceivedAt.UTC(),
|
|
Summary: independentSummary, Refs: []unified.ActionEvidenceRef{{ID: facts.ActionID, Kind: "docker_update_replacement", Digest: observationDigest}},
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("normalize independent docker update observation: %w", err)
|
|
}
|
|
status := unified.ActionVerificationContradicted
|
|
reason := "postcondition_contradicted"
|
|
// Identity is necessary but not sufficient: a replacement can exit or
|
|
// become unhealthy after the agent reported success. Compare against
|
|
// its readback rather than requiring running unconditionally, because
|
|
// updates deliberately preserve stopped containers.
|
|
if !facts.ReadbackRan || facts.After.State == "" {
|
|
status = unified.ActionVerificationInconclusive
|
|
reason = "agent_readback_unavailable"
|
|
} else if dockerUpdateFactsMatch(facts, independent.Snapshot.ContainerID) &&
|
|
independent.Snapshot.State == facts.After.State &&
|
|
independent.Snapshot.Running == facts.After.Running &&
|
|
(!independent.Snapshot.Running || (independent.Snapshot.State == "running" &&
|
|
agentexec.DockerContainerHealthAllowsVerifiedRunningState(independent.Snapshot.Health))) {
|
|
status = unified.ActionVerificationConfirmed
|
|
reason = ""
|
|
}
|
|
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceIndependent, ReasonCode: reason, Summary: independentSummary, Evidence: []unified.ActionEvidence{evidence}}
|
|
}
|
|
|
|
canonical := unified.ActionResultV2{
|
|
Version: unified.ActionResultV2Version, Execution: execution, Verification: verification,
|
|
Compensation: dockerUpdateCompensationTruth(facts),
|
|
}
|
|
legacy := &unified.ExecutionResult{Output: summary}
|
|
projected, _, err := unified.ApplyActionResultV2(legacy, canonical)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("apply canonical docker update action truth: %w", err)
|
|
}
|
|
projected.Verification = unified.LegacyActionVerificationFromV2(*projected.ActionResultV2)
|
|
return projected, nil
|
|
}
|
|
|
|
// dockerUpdateCompensationTruth reports real compensation state: the agent
|
|
// creates a backup rename before recreating and restores it when the
|
|
// replacement fails, so unlike restarts this operation declares support.
|
|
func dockerUpdateCompensationTruth(facts agentexec.DockerContainerUpdateResultPayload) unified.ActionCompensationTruth {
|
|
truth := unified.ActionCompensationTruth{
|
|
Support: unified.ActionCompensationDeclared,
|
|
Strategy: "backup_rename_restore",
|
|
Trigger: "replacement_create_start_or_stability_failure",
|
|
}
|
|
switch {
|
|
case facts.MutationCompleted:
|
|
truth.Status = unified.ActionCompensationNotNeeded
|
|
truth.Summary = "Update succeeded; the renamed backup container is removed automatically after a stability window."
|
|
case !facts.MutationStarted:
|
|
truth.Status = unified.ActionCompensationNotNeeded
|
|
truth.Summary = "The update failed before mutating the container, so there was nothing to roll back."
|
|
case facts.RolledBack:
|
|
truth.Status = unified.ActionCompensationSucceeded
|
|
truth.Summary = "The update failed and the original container was restored from its backup."
|
|
case facts.RollbackAttempted:
|
|
truth.Status = unified.ActionCompensationFailed
|
|
truth.Summary = "The update failed and the automatic restore did not complete; the original container may still carry its backup name (" + strings.TrimSpace(facts.BackupContainer) + ")."
|
|
default:
|
|
truth.Status = unified.ActionCompensationNotAttempted
|
|
truth.Summary = "The update failed after mutation started and no automatic restore ran."
|
|
}
|
|
return truth
|
|
}
|
|
|
|
func dockerContainerUpdateFactSummary(facts agentexec.DockerContainerUpdateResultPayload) string {
|
|
summary := fmt.Sprintf(
|
|
"Container update: phase=%s; mutation started=%t; mutation completed=%t; readback ran=%t; container=%s; replacement=%s; old image=%s; new image=%s; backup=%s; rollback attempted=%t; rolled back=%t",
|
|
strings.TrimSpace(facts.ExecutionPhase), facts.MutationStarted, facts.MutationCompleted, facts.ReadbackRan,
|
|
strings.TrimSpace(facts.ContainerName), strings.TrimSpace(facts.NewContainerID),
|
|
shortDockerDigest(facts.OldImageDigest), shortDockerDigest(facts.NewImageDigest),
|
|
strings.TrimSpace(facts.BackupContainer), facts.RollbackAttempted, facts.RolledBack,
|
|
)
|
|
if errText := strings.TrimSpace(facts.Error); errText != "" {
|
|
summary += "; error=" + errText
|
|
}
|
|
return summary
|
|
}
|
|
|
|
// actionRefusalUncoded is the legacy aggregate recorded when a pre-mutation
|
|
// refusal arrives with no machine reason code. Agents older than the typed
|
|
// refusal contract report every refusal this way.
|
|
const actionRefusalUncoded = "preflight_refused"
|
|
|
|
// actionPreflightReasonCode preserves typed agent refusals while retaining the
|
|
// legacy aggregate for older agents that do not yet send a reason_code.
|
|
func actionPreflightReasonCode(reasonCode string) string {
|
|
reasonCode = strings.TrimSpace(reasonCode)
|
|
if agentexec.IsActionRefusalReasonCode(reasonCode) {
|
|
return reasonCode
|
|
}
|
|
return actionRefusalUncoded
|
|
}
|
|
|
|
func shortDockerDigest(digest string) string {
|
|
digest = strings.TrimSpace(digest)
|
|
const previewLen = 19 // "sha256:" + 12 hex characters
|
|
if len(digest) <= previewLen {
|
|
return digest
|
|
}
|
|
return digest[:previewLen]
|
|
}
|
|
|
|
func freshDockerUpdateObservation(observedAt, receivedAt time.Time) bool {
|
|
if observedAt.IsZero() || receivedAt.IsZero() {
|
|
return false
|
|
}
|
|
return !observedAt.Before(receivedAt.UTC().Add(-30*time.Minute)) && !observedAt.After(receivedAt.UTC().Add(5*time.Minute))
|
|
}
|
|
|
|
// dockerUpdateFactsMatch confirms the observed container is the replacement
|
|
// the agent claims to have created. Independent verification additionally
|
|
// compares its state with the agent readback, preserving stopped updates.
|
|
func dockerUpdateFactsMatch(facts agentexec.DockerContainerUpdateResultPayload, observedContainerID string) bool {
|
|
if facts.NewContainerID == "" || observedContainerID == "" {
|
|
return false
|
|
}
|
|
return strings.EqualFold(facts.NewContainerID, observedContainerID)
|
|
}
|
|
|
|
func dockerContainerFactSummary(facts agentexec.DockerContainerLifecycleResultPayload) string {
|
|
return fmt.Sprintf("Container lifecycle: phase=%s; mutation started=%t; mutation completed=%t; readback ran=%t; before=%s/%s; after=%s/%s", strings.TrimSpace(facts.ExecutionPhase), facts.MutationStarted, facts.MutationCompleted, facts.ReadbackRan, strings.TrimSpace(facts.Before.State), dockerLifecycleHealthSummary(facts.Before.Health), strings.TrimSpace(facts.After.State), dockerLifecycleHealthSummary(facts.After.Health))
|
|
}
|
|
|
|
func dockerLifecycleHealthSummary(health string) string {
|
|
if health = strings.TrimSpace(health); health != "" {
|
|
return health
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func freshDockerLifecycleObservation(before, after, receivedAt time.Time) bool {
|
|
if before.IsZero() || after.IsZero() || receivedAt.IsZero() || after.Before(before) {
|
|
return false
|
|
}
|
|
return !after.Before(receivedAt.UTC().Add(-15*time.Minute)) && !after.After(receivedAt.UTC().Add(5*time.Minute))
|
|
}
|
|
|
|
func dockerLifecycleFactsMatch(operation string, before, after agentexec.DockerContainerLifecycleSnapshot) bool {
|
|
if !strings.EqualFold(before.ContainerID, after.ContainerID) || after.ContainerID == "" {
|
|
return false
|
|
}
|
|
switch operation {
|
|
case agentexec.DockerContainerOperationStart:
|
|
return after.Running && after.State == "running" && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
|
|
case agentexec.DockerContainerOperationStop:
|
|
return !after.Running && after.State == "exited"
|
|
case agentexec.DockerContainerOperationRestart:
|
|
return after.Running && after.State == "running" && !after.StartedAt.IsZero() && after.StartedAt.After(before.StartedAt) && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func dockerLifecycleObservationFactsMatch(operation string, before agentexec.DockerContainerLifecycleSnapshot, after agentexec.DockerContainerObservationSnapshot) bool {
|
|
return dockerLifecycleFactsMatch(operation, before, agentexec.DockerContainerLifecycleSnapshot{
|
|
ContainerID: after.ContainerID, State: after.State, Running: after.Running, Health: after.Health,
|
|
StartedAt: after.StartedAt, RestartCount: after.RestartCount, ObservedAt: after.ObservedAt,
|
|
})
|
|
}
|
|
|
|
func dockerLifecycleRunningHealthUnknown(operation string, after agentexec.DockerContainerLifecycleSnapshot) bool {
|
|
return (operation == agentexec.DockerContainerOperationStart || operation == agentexec.DockerContainerOperationRestart) && !agentexec.IsDockerContainerHealth(after.Health)
|
|
}
|