Wire production Proxmox action verification

This commit is contained in:
rcourtman
2026-07-13 11:04:09 +01:00
parent 5cde383b98
commit f095da2fdb
20 changed files with 932 additions and 52 deletions
+16 -1
View File
@@ -8113,7 +8113,7 @@
},
{
"id": "action-governance-auditability-post-rc-hardening",
"summary": "Task 07 durable delivery and generic agent-operation receipts are accepted: transactional dispatch admission, one-shot send CAS, immutable attempt/action/operation/digest/agent binding, query-only restart reconciliation, strict sanitized terminal envelopes, and permanent replay-denial tombstones. Task 09 consumes that owner for both APT workflows and completes the detector-to-finding, exact empty-parameter proposal, shared policy/approval, typed dispatch, delayed/callback-loss/reopened-server receipt reconciliation, ActionResultV2 truth, terminal audit, and finding reconciliation floor without blind resend. Task 11 adds the desktop/browser tier-5 Product Trust consumer: bounded command/path/package-free Patrol evidence; exact empty parameter and elevated-versus-low-risk review; separate execution, evidence-sourced verification, and recovery cards; agent-attested confirmed updates; reboot fact without reboot authority; partial/unknown-health recovery; measured irreversible cleanup; and one durable receipt across reconnect, all exercised in current-build Chromium and a 390-pixel viewport. Malformed phases, counts, timestamps, usage, or cleanup arithmetic fail closed. The closed RG-01 through RG-12 matrix now supplies the disposable Debian/Ubuntu tier-6, Docker, browser, physical-device, live-Relay, revocation, cleanup, and independent Task 12 proof that previously kept claims 16 and 17 and both workflow scorecards operationally open. Task 10 remains sole owner of execution, verification, evidence, compensation, and rollback truth, and RG06/RG09 product outcomes remain agent-attested fix_verification_unknown. This follow-up now tracks only broader post-RC auditability beyond the proved bounded capability set, including additional operation and platform coverage, deeper compensation and rollback proof, and MSP or fleet aggregation; it does not reopen the accepted matrix. Raw model command, file write, arbitrary pod exec, legacy run_command, /api/ai/run-command, and enterprise command remediation remain retired with no replacement or executable historical authority.",
"summary": "Task 07 durable delivery and generic agent-operation receipts are accepted: transactional dispatch admission, one-shot send CAS, immutable attempt/action/operation/digest/agent binding, query-only restart reconciliation, strict sanitized terminal envelopes, and permanent replay-denial tombstones. Task 09 consumes that owner for both APT workflows and completes the detector-to-finding, exact empty-parameter proposal, shared policy/approval, typed dispatch, delayed/callback-loss/reopened-server receipt reconciliation, ActionResultV2 truth, terminal audit, and finding reconciliation floor without blind resend. Task 11 adds the desktop/browser tier-5 Product Trust consumer: bounded command/path/package-free Patrol evidence; exact empty parameter and elevated-versus-low-risk review; separate execution, evidence-sourced verification, and recovery cards; agent-attested confirmed updates; reboot fact without reboot authority; partial/unknown-health recovery; measured irreversible cleanup; and one durable receipt across reconnect, all exercised in current-build Chromium and a 390-pixel viewport. Malformed phases, counts, timestamps, usage, or cleanup arithmetic fail closed. The closed RG-01 through RG-12 matrix now supplies the disposable Debian/Ubuntu tier-6, Docker, browser, physical-device, live-Relay, revocation, cleanup, and independent Task 12 proof that previously kept claims 16 and 17 and both workflow scorecards operationally open. Task 10 remains sole owner of execution, verification, evidence, compensation, and rollback truth, and RG06/RG09 product outcomes remain agent-attested fix_verification_unknown. Proxmox VM/LXC lifecycle now supplies the first production distinct-trust-domain verification path: the node agent executes, the tenant-scoped server Proxmox client observes status and uptime, reboot requires an uptime reset, and ActionResultV2 keeps execution success independent from postcondition contradiction. This follow-up now tracks only broader post-RC auditability beyond the proved bounded capability set, including additional operation and platform coverage, deeper compensation and rollback proof, and MSP or fleet aggregation; it does not reopen the accepted matrix. Raw model command, file write, arbitrary pod exec, legacy run_command, /api/ai/run-command, and enterprise command remediation remain retired with no replacement or executable historical authority.",
"owner": "project-owner",
"status": "planned",
"recorded_at": "2026-04-25",
@@ -8486,6 +8486,21 @@
"repo": "pulse",
"path": "docs/release-control/v6/internal/SOURCE_OF_TRUTH.md",
"kind": "file"
},
{
"repo": "pulse",
"path": "internal/api/proxmox_guest_action_observer.go",
"kind": "file"
},
{
"repo": "pulse",
"path": "internal/api/proxmox_guest_action_result.go",
"kind": "file"
},
{
"repo": "pulse",
"path": "internal/monitoring/proxmox_action_observer.go",
"kind": "file"
}
]
},
@@ -31,6 +31,7 @@ that binary, not separate customer-facing agent products.
5. `internal/hostagent/agent.go`
5a. `internal/dockeragent/agent.go`
5b. `internal/kubernetesagent/agent.go`
5c. `internal/agentexec/verifier_postconditions.go`
6. `cmd/pulse-agent/main.go`
7. `scripts/install.sh`
8. `scripts/install.ps1`
@@ -481,6 +482,12 @@ connected Proxmox node command agent and records action audit plus verification.
Lifecycle surfaces must not run `qm` / `pct`, SSH to a node, call Proxmox
mutation APIs, or substitute a guest-local agent to perform VM/LXC lifecycle
control.
The executing node agent's status read remains agent-attested. The closed
postcondition registry in `internal/agentexec/verifier_postconditions.go`
defines start, shutdown, stop, and reboot checks for both `qm` and `pct`, while
the API-owned verifier supplies provider observations. Reboot is not confirmed
from status alone: it requires a distinct Proxmox API observation whose uptime
reset proves that the guest actually restarted.
Disconnected command-agent state is also API-owned readiness: lifecycle
surfaces may reflect missing backend-advertised capabilities, but must not
reconnect, substitute, or directly address an agent to make a stale container
@@ -4500,6 +4507,8 @@ RG-06 and RG-09 keep the executing agent's fresh typed readback classified as
fixture changed, but it does not enter the authenticated product action result
and therefore cannot upgrade the product finding to `fix_verified`. Until a
distinct-trust-domain observation is ingested into `ActionResultV2`, those
findings remain `fix_verification_unknown` and unresolved. The Docker restart
journey, which does ingest its direct daemon observation as independent
evidence, remains the positive `fix_verified` control.
findings remain `fix_verification_unknown` and unresolved. Proxmox VM/LXC
lifecycle is now the first production path to ingest a server-side provider
observation as independent evidence; the Docker restart lab journey remains a
positive control but does not imply that agent-reported Docker inventory is an
independent production observer.
@@ -37,7 +37,9 @@ product API routes free of maintainer commercial analytics.
5a. `internal/api/action_executor.go`
5b. `internal/api/docker_container_action_executor.go`
5c. `internal/api/proxmox_guest_action_executor.go`
5d. `internal/api/host_update_action_executor.go`
5d. `internal/api/proxmox_guest_action_observer.go`
5e. `internal/api/proxmox_guest_action_result.go`
5f. `internal/api/host_update_action_executor.go`
6a. `internal/actionlifecycle/service.go`
7. `internal/actionplanner/planner.go`
8. `pkg/pulsecli/api_client.go`
@@ -7423,6 +7425,19 @@ the action, subject, observation and receipt times, before/after facts, and
canonical evidence digest. It is independent operational evidence, not a
cryptographic attestation.
Proxmox VM and LXC lifecycle execution now consumes that same two-axis truth
contract in production. The node agent remains the executor, while the API
composition root injects the tenant-scoped monitoring client as a direct
Proxmox control-plane observer. Fresh, identity-matched control-plane reads use
an `agent:*`-distinct trust domain and are digest-bound into `ActionResultV2`;
missing clients, mismatched guest identity, stale timestamps, or same-domain
evidence fail closed to the agent-attested or inconclusive result. Start,
shutdown, stop, and reboot all evaluate the shared closed postcondition
registry. Reboot additionally requires the direct API uptime to reset from a
fresh pre-action observation, so a guest that merely stayed `running` cannot
be reported as verified. A contradicted postcondition never rewrites a
successful command dispatch into execution failure.
### Canonical mutation plane
`internal/mutationregistry/manifest.json` is the generated closed registry for
@@ -133,6 +133,7 @@ resource health.
55. `internal/monitoring/resource_stale_thresholds.go`
56. `internal/monitoring/recovery_ingest.go`
57. `internal/monitoring/multi_tenant_monitor.go`
58. `internal/monitoring/proxmox_action_observer.go`
## Shared Boundaries
@@ -1646,3 +1647,11 @@ Task 09 preserves two APT telemetry clocks at host-agent ingest: `CheckedAt`
is agent-observed time and `ObservedAt` is server-received time. Monitoring
must not overwrite the former with the latter; replay and skew safety consumes
both timestamps downstream.
Monitoring now also exposes a bounded direct Proxmox guest observation for the
governed action verifier. `ObserveProxmoxGuest` resolves the configured
instance client under the monitor lock, reads VM or LXC status and uptime from
the Proxmox API, validates the requested guest identity, and stamps server
observation time. It must not satisfy this contract from cached resource state
or node-agent telemetry: the action layer depends on this read remaining in a
trust domain distinct from the node agent that executes `qm` / `pct`.
@@ -421,7 +421,10 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant.
router setup may register the Proxmox executor alongside Docker / Podman,
but it must not resolve guests, probe node agents, call `qm` / `pct`, poll
verification state, or refresh inventory outside the route-local action
plan/execute path.
plan/execute path. Injecting the tenant-monitor resolver at setup remains
constant work; the direct Proxmox status/uptime reads are action-local and
poll at most once per second within the closed postcondition registry's
bounded verification window.
Retiring self-hosted trial acquisition follows that same rule: removing
`/auth/trial-activate` and `POST /api/license/trial/start` from public-path
and CSRF inventories must stay as constant-time route-table absence rather
@@ -371,6 +371,13 @@ the `white_label` branding entitlement.
command, package selector, or deletion rule. Cache entry names,
fingerprint, raw APT output, and agent error text remain out of model
context and terminal action output.
Proxmox VM/LXC lifecycle observer wiring is also not mutation authority.
It may resolve only the current tenant's existing monitor and use that
monitor's configured Proxmox client for bounded status/uptime reads after a
governed node-agent action. It must not expose provider credentials or raw
provider responses, cross tenant or guest identity, call a provider
mutation API, or classify the executing agent's own trust domain as
independent evidence.
Router glue may also pass monitor-owned source freshness thresholds into
unified-resource adapters, but those thresholds are operational cadence
metadata only. They must not disclose credentials, command output, raw
@@ -148,6 +148,14 @@ recovery surfaces may observe disconnected command readiness, but they must not
re-enable command config, reinterpret the suppression as backup or restore
state, or introduce a recovery-local command-token binding path.
Proxmox VM/LXC lifecycle verification in `internal/api/` is likewise adjacent
action-governance evidence, not recovery evidence. The node agent may execute a
governed `qm` / `pct` action while the tenant-scoped server Proxmox client
independently observes guest status and uptime, but a confirmed start, stop,
shutdown, or reboot proves only that action's registered postcondition. Storage
and recovery surfaces must not reinterpret that `ActionResultV2` evidence as a
recovery point, backup freshness, restore success, or protected-workload state.
Hosted tenant agent install commands in `internal/api/cloud_agent_install_command.go`
are adjacent API/lifecycle transport only. A provider-hosted MSP PVE/PBS install
token may allow agent reporting for the scoped tenant workspace, but it must not
@@ -662,6 +662,12 @@ second-click local confirmation. Proof is owned by the colocated action tests,
Unified-resource consumers must not infer Proxmox lifecycle affordances from
table row status alone or issue direct `qm` / `pct`, SSH, provider API, or
guest-agent calls outside the governed action contract.
The lifecycle executor must project terminal truth through `ActionResultV2`
without coupling execution success to verification. Same-agent status reads
remain `agent_attested`; only a fresh, identity-matched read through the
tenant's server-side Proxmox API client may be `independent`. Reboot
confirmation additionally requires an uptime reset relative to the direct
pre-action observation, not merely a post-action `running` status.
Agent-managed Linux hosts may advertise `install_os_updates` only when the
report carries supported APT package posture and typed command operations
are enabled. The capability is admin-floor, `elevated` auto-authorization
@@ -1781,6 +1787,13 @@ Workflow, API, AI, agent, Docker, host-agent, and relay packages must not
declare competing truth enums. Generated wire mirrors remain a bounded later
presentation concern, not a second source of semantics.
The first production distinct-trust-domain consumer is the Proxmox VM/LXC
lifecycle executor. Its node-agent dispatch and Proxmox API observation remain
separate evidence domains, and the API-owned projection binds the exact action,
resource, before/after snapshots, observation times, and canonical digest.
This does not authorize provider-local mutation or relax the false-independence
guard for Docker or agent-attested APT readback.
The canonical action resource contract now owns immutable `ActionActor` and
versioned `ApprovalRequirement` bindings. Actor subject/kind/credential/org and
requirement floor/quorum/separation are part of deterministic action identity
@@ -2,6 +2,7 @@ package agentexec
import (
"sort"
"strconv"
"strings"
"time"
)
@@ -19,6 +20,7 @@ const (
// VM and container status as reported by Proxmox.
FieldVMStatus PostconditionField = "status"
FieldContainerStatus PostconditionField = "status"
FieldGuestUptime PostconditionField = "uptime"
// Systemd unit fields read via DBus or systemctl show.
FieldUnitActiveState PostconditionField = "ActiveState"
@@ -49,6 +51,10 @@ const (
// EqualsField: observed value must equal another observed field on the
// same read (e.g. readyReplicas == desiredReplicas).
CompareEqualsField PostconditionComparator = "equals_field"
// LessThanBefore: the numeric observed value must be lower than the
// corresponding pre-action observation. This proves that a running guest
// actually restarted instead of merely remaining online.
CompareLessThanBefore PostconditionComparator = "less_than_before"
)
// PostconditionCheck is one assertion the verifier evaluates against the
@@ -76,6 +82,16 @@ type CapabilityPostcondition struct {
Checks []PostconditionCheck `json:"checks"`
}
// PostconditionEvaluation is the provider-neutral result of evaluating one
// registered postcondition against bounded before/after observations.
// Conclusive=false means the observation did not contain enough valid data to
// claim either confirmation or contradiction.
type PostconditionEvaluation struct {
Conclusive bool
Matched bool
ReasonCode string
}
// defaultVerifyWindow is the per-capability fallback window. The agentexec
// policy carries an operator-tunable verify_window that overrides this; the
// per-capability values here are the substrate's "what is reasonable for
@@ -111,6 +127,72 @@ func CapabilityPostconditionNames() []string {
return names
}
// EvaluateCapabilityPostcondition applies the closed registry definition to
// provider-normalized string observations. Providers only map their typed read
// into the closed field vocabulary; they do not reimplement the comparisons.
func EvaluateCapabilityPostcondition(capability string, before, after map[PostconditionField]string, actionStartedAt time.Time) (PostconditionEvaluation, bool) {
postcondition, ok := LookupCapabilityPostcondition(capability)
if !ok {
return PostconditionEvaluation{}, false
}
for _, check := range postcondition.Checks {
observed, exists := normalizedPostconditionValue(after, check.Field)
if !exists {
return PostconditionEvaluation{ReasonCode: "observed_field_missing"}, true
}
switch check.Comparator {
case CompareEquals:
if !strings.EqualFold(observed, strings.TrimSpace(check.Expected)) {
return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true
}
case CompareEqualsField:
peer, exists := normalizedPostconditionValue(after, PostconditionField(check.Expected))
if !exists {
return PostconditionEvaluation{ReasonCode: "comparison_field_missing"}, true
}
if !strings.EqualFold(observed, peer) {
return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true
}
case CompareAfterOrEqualActionStart:
if actionStartedAt.IsZero() {
return PostconditionEvaluation{ReasonCode: "action_start_missing"}, true
}
observedAt, err := time.Parse(time.RFC3339Nano, observed)
if err != nil {
return PostconditionEvaluation{ReasonCode: "observed_timestamp_invalid"}, true
}
if observedAt.Before(actionStartedAt) {
return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true
}
case CompareLessThanBefore:
previous, exists := normalizedPostconditionValue(before, check.Field)
if !exists {
return PostconditionEvaluation{ReasonCode: "before_field_missing"}, true
}
previousValue, previousErr := strconv.ParseUint(previous, 10, 64)
observedValue, observedErr := strconv.ParseUint(observed, 10, 64)
if previousErr != nil || observedErr != nil {
return PostconditionEvaluation{ReasonCode: "observed_number_invalid"}, true
}
if previousValue <= 1 {
return PostconditionEvaluation{ReasonCode: "before_value_insufficient"}, true
}
if observedValue >= previousValue {
return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true
}
default:
return PostconditionEvaluation{ReasonCode: "comparator_unsupported"}, true
}
}
return PostconditionEvaluation{Conclusive: true, Matched: true}, true
}
func normalizedPostconditionValue(values map[PostconditionField]string, field PostconditionField) (string, bool) {
value, ok := values[field]
value = strings.TrimSpace(value)
return value, ok && value != ""
}
// capabilityPostconditions is the closed registry of postconditions the
// verifier substrate knows how to evaluate. New tool capabilities must add
// an entry here AND a corresponding test in verifier_postconditions_test.go
@@ -125,6 +207,34 @@ var capabilityPostconditions = map[string]CapabilityPostcondition{
{Field: FieldVMStatus, Comparator: CompareEquals, Expected: "running"},
},
},
"qm.shutdown": {
Capability: "qm.shutdown",
VerifyRead: "qm status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox VM transitioned to stopped after graceful shutdown",
Checks: []PostconditionCheck{
{Field: FieldVMStatus, Comparator: CompareEquals, Expected: "stopped"},
},
},
"qm.stop": {
Capability: "qm.stop",
VerifyRead: "qm status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox VM transitioned to stopped after hard stop",
Checks: []PostconditionCheck{
{Field: FieldVMStatus, Comparator: CompareEquals, Expected: "stopped"},
},
},
"qm.reboot": {
Capability: "qm.reboot",
VerifyRead: "Proxmox API guest status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox VM is running with uptime reset after reboot",
Checks: []PostconditionCheck{
{Field: FieldVMStatus, Comparator: CompareEquals, Expected: "running"},
{Field: FieldGuestUptime, Comparator: CompareLessThanBefore},
},
},
"pct.start": {
Capability: "pct.start",
VerifyRead: "pct status <vmid>",
@@ -134,6 +244,34 @@ var capabilityPostconditions = map[string]CapabilityPostcondition{
{Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "running"},
},
},
"pct.shutdown": {
Capability: "pct.shutdown",
VerifyRead: "pct status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox CT transitioned to stopped after graceful shutdown",
Checks: []PostconditionCheck{
{Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "stopped"},
},
},
"pct.stop": {
Capability: "pct.stop",
VerifyRead: "pct status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox CT transitioned to stopped after hard stop",
Checks: []PostconditionCheck{
{Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "stopped"},
},
},
"pct.reboot": {
Capability: "pct.reboot",
VerifyRead: "Proxmox API guest status <vmid>",
Window: defaultVerifyWindow,
Description: "Proxmox CT is running with uptime reset after reboot",
Checks: []PostconditionCheck{
{Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "running"},
{Field: FieldGuestUptime, Comparator: CompareLessThanBefore},
},
},
"docker.restart": {
Capability: "docker.restart",
VerifyRead: "docker inspect <container>",
@@ -12,7 +12,12 @@ import (
// of capabilities the verifier substrate must understand. Drift either way
// (missing entry, drift in field/comparator) breaks the verifier contract.
func TestCapabilityPostconditionsCoversRequiredCapabilities(t *testing.T) {
required := []string{"qm.start", "pct.start", "docker.restart", "systemctl.restart", "kubectl.rollout"}
required := []string{
"docker.restart", "kubectl.rollout",
"pct.reboot", "pct.shutdown", "pct.start", "pct.stop",
"qm.reboot", "qm.shutdown", "qm.start", "qm.stop",
"systemctl.restart",
}
sort.Strings(required)
got := CapabilityPostconditionNames()
if !reflect.DeepEqual(got, required) {
@@ -54,6 +59,30 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) {
expectWindow: 2 * time.Minute,
mustReferenceCmd: "qm status",
},
{
capability: "qm.shutdown",
minChecks: 1,
expectFields: []PostconditionField{FieldVMStatus},
expectCompare: []PostconditionComparator{CompareEquals},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "qm status",
},
{
capability: "qm.stop",
minChecks: 1,
expectFields: []PostconditionField{FieldVMStatus},
expectCompare: []PostconditionComparator{CompareEquals},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "qm status",
},
{
capability: "qm.reboot",
minChecks: 2,
expectFields: []PostconditionField{FieldVMStatus, FieldGuestUptime},
expectCompare: []PostconditionComparator{CompareEquals, CompareLessThanBefore},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "Proxmox API",
},
{
capability: "pct.start",
minChecks: 1,
@@ -62,6 +91,30 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) {
expectWindow: 2 * time.Minute,
mustReferenceCmd: "pct status",
},
{
capability: "pct.shutdown",
minChecks: 1,
expectFields: []PostconditionField{FieldContainerStatus},
expectCompare: []PostconditionComparator{CompareEquals},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "pct status",
},
{
capability: "pct.stop",
minChecks: 1,
expectFields: []PostconditionField{FieldContainerStatus},
expectCompare: []PostconditionComparator{CompareEquals},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "pct status",
},
{
capability: "pct.reboot",
minChecks: 2,
expectFields: []PostconditionField{FieldContainerStatus, FieldGuestUptime},
expectCompare: []PostconditionComparator{CompareEquals, CompareLessThanBefore},
expectWindow: 2 * time.Minute,
mustReferenceCmd: "Proxmox API",
},
{
capability: "docker.restart",
minChecks: 2,
@@ -123,6 +176,38 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) {
}
}
func TestEvaluateCapabilityPostcondition(t *testing.T) {
t.Run("reboot requires uptime reset", func(t *testing.T) {
before := map[PostconditionField]string{FieldVMStatus: "running", FieldGuestUptime: "7200"}
after := map[PostconditionField]string{FieldVMStatus: "running", FieldGuestUptime: "0"}
evaluation, ok := EvaluateCapabilityPostcondition("qm.reboot", before, after, time.Now())
if !ok || !evaluation.Conclusive || !evaluation.Matched {
t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok)
}
after[FieldGuestUptime] = "7201"
evaluation, ok = EvaluateCapabilityPostcondition("qm.reboot", before, after, time.Now())
if !ok || !evaluation.Conclusive || evaluation.Matched || evaluation.ReasonCode != "postcondition_contradicted" {
t.Fatalf("non-reset evaluation = %#v, ok=%v", evaluation, ok)
}
})
t.Run("missing reboot baseline is inconclusive", func(t *testing.T) {
after := map[PostconditionField]string{FieldContainerStatus: "running", FieldGuestUptime: "4"}
evaluation, ok := EvaluateCapabilityPostcondition("pct.reboot", nil, after, time.Now())
if !ok || evaluation.Conclusive || evaluation.Matched || evaluation.ReasonCode != "before_field_missing" {
t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok)
}
})
t.Run("status mismatch is contradicted", func(t *testing.T) {
evaluation, ok := EvaluateCapabilityPostcondition("qm.shutdown", nil, map[PostconditionField]string{FieldVMStatus: "running"}, time.Now())
if !ok || !evaluation.Conclusive || evaluation.Matched {
t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok)
}
})
}
// TestLookupCapabilityPostconditionUnknown verifies the false return path so
// callers can branch into VerificationUnknown rather than seeing a zero-value
// entry that looks valid.
+3 -2
View File
@@ -19995,7 +19995,8 @@ func TestContract_ProxmoxLifecycleActionsResolveNodeCommandAgentAndVerifyState(t
"Trusted: true",
"func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(",
"proxmoxGuestStatusCommand(kind, vmid)",
"parseProxmoxGuestStatus(lastOutput) == expected",
"agentexec.EvaluateCapabilityPostcondition",
"func (e proxmoxGuestActionExecutor) observeProxmoxGuestPostcondition(",
} {
if !strings.Contains(src, snippet) {
t.Fatalf("proxmox lifecycle executor must pin command-agent/trusted verification snippet %q", snippet)
@@ -20014,7 +20015,7 @@ func TestContract_ProxmoxLifecycleActionsResolveNodeCommandAgentAndVerifyState(t
for _, snippet := range []string{
"newRoutedActionExecutor(",
"newDockerContainerActionExecutor(r.resourceHandlers, r.agentExecServer)",
"newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer)",
"newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer, newProxmoxGuestMonitoringObserver(r.resolveMonitorForOrg))",
"newHostStorageCleanupActionExecutor(r.resourceHandlers, r.agentExecServer)",
"newHostUpdateActionExecutor(r.resourceHandlers, r.agentExecServer)",
} {
+117 -29
View File
@@ -27,13 +27,36 @@ const (
type proxmoxGuestActionExecutor struct {
resources *ResourceHandlers
agents actionAgentCommander
observer proxmoxGuestPostconditionObserver
}
func newProxmoxGuestActionExecutor(resources *ResourceHandlers, agents actionAgentCommander) ActionExecutor {
type proxmoxGuestLifecycleSnapshot struct {
Instance string
Node string
VMID int
Kind proxmoxGuestKind
Status string
Uptime uint64
ObservedAt time.Time
}
type proxmoxGuestPostconditionObservation struct {
ObserverID string
TrustDomain string
Method string
Snapshot proxmoxGuestLifecycleSnapshot
ReceivedAt time.Time
}
type proxmoxGuestPostconditionObserver interface {
ObserveProxmoxGuest(context.Context, string, string, string, int, proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error)
}
func newProxmoxGuestActionExecutor(resources *ResourceHandlers, agents actionAgentCommander, observer proxmoxGuestPostconditionObserver) ActionExecutor {
if resources == nil || agents == nil {
return nil
}
return proxmoxGuestActionExecutor{resources: resources, agents: agents}
return proxmoxGuestActionExecutor{resources: resources, agents: agents, observer: observer}
}
func (e proxmoxGuestActionExecutor) ActionHandlerNames() []string {
@@ -63,8 +86,15 @@ func (e proxmoxGuestActionExecutor) ExecuteAction(ctx context.Context, record un
if err != nil {
return nil, err
}
var independentBefore *proxmoxGuestPostconditionObservation
if e.observer != nil {
if observation, observeErr := e.observer.ObserveProxmoxGuest(ctx, record.Request.ResourceID, resource.Proxmox.Instance, resource.Proxmox.NodeName, vmid, kind); observeErr == nil && validProxmoxGuestObservation(resource, kind, observation) {
independentBefore = &observation
}
}
command := proxmoxGuestLifecycleCommand(kind, operation, vmid)
actionStartedAt := time.Now().UTC()
result, err := e.agents.ExecuteCommand(ctx, agentID, agentexec.ExecuteCommandPayload{
RequestID: attempt.ID,
Command: command,
@@ -78,22 +108,13 @@ func (e proxmoxGuestActionExecutor) ExecuteAction(ctx context.Context, record un
}
output := redactActionOutput(commandOutput(result))
execution := &unified.ExecutionResult{
Success: result.ExitCode == 0,
Output: output,
}
if result.ExitCode != 0 {
execution.ErrorMessage = strings.TrimSpace(firstNonEmpty(result.Error, output, fmt.Sprintf("proxmox guest %s exited with status %d", operation, result.ExitCode)))
return execution, nil
return proxmoxGuestExecutionResult(record.ID, record.Request.ResourceID, agentID, kind, operation, result.ExitCode, output, result.Error, nil, independentBefore, nil, agentexec.PostconditionEvaluation{}, actionStartedAt)
}
verification := e.verifyProxmoxGuestState(ctx, agentID, record.ID, kind, vmid, operation)
execution.Verification = verification
if verification != nil && verification.Ran && !verification.Success {
execution.Success = false
execution.ErrorMessage = "proxmox guest lifecycle action completed but verification did not confirm the expected state"
}
return execution, nil
agentVerification := e.verifyProxmoxGuestState(ctx, agentID, record.ID, kind, vmid, operation, actionStartedAt)
independentAfter, independentEvaluation := e.observeProxmoxGuestPostcondition(ctx, record.Request.ResourceID, resource, kind, operation, independentBefore, actionStartedAt)
return proxmoxGuestExecutionResult(record.ID, record.Request.ResourceID, agentID, kind, operation, result.ExitCode, output, result.Error, agentVerification, independentBefore, independentAfter, independentEvaluation, actionStartedAt)
}
func (e proxmoxGuestActionExecutor) CheckActionAvailable(ctx context.Context, req unified.ActionRequest, resource unified.Resource) unified.ResourceActionReadiness {
@@ -300,14 +321,18 @@ func proxmoxGuestLifecycleTimeout(operation string) int {
}
}
func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, agentID, actionID string, kind proxmoxGuestKind, vmid int, operation string) *unified.ActionVerificationResult {
expected := proxmoxExpectedGuestStatus(operation)
if expected == "" {
return &unified.ActionVerificationResult{Ran: false}
func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, agentID, actionID string, kind proxmoxGuestKind, vmid int, operation string, actionStartedAt time.Time) *unified.ActionVerificationResult {
capability := proxmoxPostconditionCapability(kind, operation)
if _, ok := agentexec.LookupCapabilityPostcondition(capability); !ok {
return &unified.ActionVerificationResult{Ran: false, Note: "No registered postcondition is available for this Proxmox action."}
}
if strings.EqualFold(strings.TrimSpace(operation), "reboot") {
return &unified.ActionVerificationResult{Ran: false, Note: "The executing agent's status-only read cannot prove that the guest restarted."}
}
command := proxmoxGuestStatusCommand(kind, vmid)
var lastOutput string
var lastEvaluation agentexec.PostconditionEvaluation
for attempt := 0; attempt < 5; attempt++ {
if attempt > 0 {
timer := time.NewTimer(1 * time.Second)
@@ -331,7 +356,11 @@ func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context,
return &unified.ActionVerificationResult{Ran: false}
}
lastOutput = redactActionOutput(commandOutput(result))
if result.ExitCode == 0 && parseProxmoxGuestStatus(lastOutput) == expected {
if result.ExitCode != 0 {
continue
}
lastEvaluation, _ = agentexec.EvaluateCapabilityPostcondition(capability, nil, proxmoxGuestPostconditionValues(kind, parseProxmoxGuestStatus(lastOutput), 0), actionStartedAt)
if lastEvaluation.Conclusive && lastEvaluation.Matched {
return &unified.ActionVerificationResult{
Ran: true,
Command: command,
@@ -348,19 +377,78 @@ func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context,
Output: lastOutput,
Success: false,
RanAt: time.Now().UTC(),
Note: "expected status " + expected,
Note: firstNonEmpty(lastEvaluation.ReasonCode, "postcondition was not confirmed"),
}
}
func proxmoxExpectedGuestStatus(operation string) string {
switch strings.TrimSpace(operation) {
case "start", "reboot":
return "running"
case "shutdown", "stop":
return "stopped"
default:
return ""
func (e proxmoxGuestActionExecutor) observeProxmoxGuestPostcondition(ctx context.Context, resourceID string, resource unified.Resource, kind proxmoxGuestKind, operation string, before *proxmoxGuestPostconditionObservation, actionStartedAt time.Time) (*proxmoxGuestPostconditionObservation, agentexec.PostconditionEvaluation) {
if e.observer == nil || resource.Proxmox == nil {
return nil, agentexec.PostconditionEvaluation{}
}
capability := proxmoxPostconditionCapability(kind, operation)
postcondition, ok := agentexec.LookupCapabilityPostcondition(capability)
if !ok {
return nil, agentexec.PostconditionEvaluation{ReasonCode: "postcondition_unregistered"}
}
window := postcondition.Window
if window <= 0 {
window = 2 * time.Minute
}
verifyCtx, cancel := context.WithTimeout(ctx, window)
defer cancel()
var beforeValues map[agentexec.PostconditionField]string
if before != nil {
beforeValues = proxmoxGuestPostconditionValues(kind, before.Snapshot.Status, before.Snapshot.Uptime)
}
var last *proxmoxGuestPostconditionObservation
lastEvaluation := agentexec.PostconditionEvaluation{ReasonCode: "independent_observation_unavailable"}
for {
observation, err := e.observer.ObserveProxmoxGuest(verifyCtx, resourceID, resource.Proxmox.Instance, resource.Proxmox.NodeName, resource.Proxmox.VMID, kind)
if err == nil && validProxmoxGuestObservation(resource, kind, observation) {
last = &observation
lastEvaluation, _ = agentexec.EvaluateCapabilityPostcondition(capability, beforeValues, proxmoxGuestPostconditionValues(kind, observation.Snapshot.Status, observation.Snapshot.Uptime), actionStartedAt)
if lastEvaluation.Conclusive && lastEvaluation.Matched {
return last, lastEvaluation
}
}
timer := time.NewTimer(time.Second)
select {
case <-verifyCtx.Done():
timer.Stop()
return last, lastEvaluation
case <-timer.C:
}
}
}
func proxmoxPostconditionCapability(kind proxmoxGuestKind, operation string) string {
tool := "qm"
if kind == proxmoxGuestCT {
tool = "pct"
}
return tool + "." + strings.ToLower(strings.TrimSpace(operation))
}
func proxmoxGuestPostconditionValues(kind proxmoxGuestKind, status string, uptime uint64) map[agentexec.PostconditionField]string {
statusField := agentexec.FieldVMStatus
if kind == proxmoxGuestCT {
statusField = agentexec.FieldContainerStatus
}
return map[agentexec.PostconditionField]string{
statusField: strings.ToLower(strings.TrimSpace(status)),
agentexec.FieldGuestUptime: strconv.FormatUint(uptime, 10),
}
}
func validProxmoxGuestObservation(resource unified.Resource, kind proxmoxGuestKind, observation proxmoxGuestPostconditionObservation) bool {
if resource.Proxmox == nil || observation.Snapshot.VMID != resource.Proxmox.VMID || observation.Snapshot.Kind != kind {
return false
}
return strings.EqualFold(strings.TrimSpace(observation.Snapshot.Instance), strings.TrimSpace(resource.Proxmox.Instance)) &&
strings.EqualFold(strings.TrimSpace(observation.Snapshot.Node), strings.TrimSpace(resource.Proxmox.NodeName)) &&
strings.TrimSpace(observation.Snapshot.Status) != "" && !observation.Snapshot.ObservedAt.IsZero() && !observation.ReceivedAt.IsZero()
}
func parseProxmoxGuestStatus(output string) string {
@@ -29,7 +29,7 @@ func TestProxmoxGuestActionExecutorDispatchesVMShutdownAndVerification(t *testin
{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"},
{RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"},
}}
executor := newProxmoxGuestActionExecutor(h, agents)
executor := newProxmoxGuestActionExecutor(h, agents, nil)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown"))
if err != nil {
@@ -73,7 +73,7 @@ func TestProxmoxGuestActionExecutorDispatchesLXCStartAndVerification(t *testing.
{RequestID: "act_ct", Success: true, ExitCode: 0, Stdout: "start requested"},
{RequestID: "act_ct-verify-1", Success: true, ExitCode: 0, Stdout: "status: running"},
}}
executor := newProxmoxGuestActionExecutor(h, agents)
executor := newProxmoxGuestActionExecutor(h, agents, nil)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_ct"), proxmoxGuestActionRecord("act_ct", "system-container:101", "start"))
if err != nil {
@@ -110,7 +110,7 @@ func TestProxmoxGuestActionExecutorResolvesCommandAgentByNodeHostname(t *testing
},
agentByHost: map[string]string{"delly": "command-agent-1"},
}
executor := newProxmoxGuestActionExecutor(h, agents).(proxmoxGuestActionExecutor)
executor := newProxmoxGuestActionExecutor(h, agents, nil).(proxmoxGuestActionExecutor)
readiness := executor.CheckActionAvailable(context.Background(), unified.ActionRequest{
RequestID: "req-availability",
@@ -137,7 +137,7 @@ func TestProxmoxGuestActionExecutorResolvesCommandAgentByNodeHostname(t *testing
}
}
func TestProxmoxGuestActionExecutorVerificationFailureFailsAction(t *testing.T) {
func TestProxmoxGuestActionExecutorVerificationContradictionDoesNotRewriteExecution(t *testing.T) {
now := time.Now().UTC()
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
h.SetStateProvider(resourceUnifiedSeedProvider{
@@ -154,17 +154,163 @@ func TestProxmoxGuestActionExecutorVerificationFailureFailsAction(t *testing.T)
{RequestID: "act_vm-verify-4", Success: true, ExitCode: 0, Stdout: "status: running"},
{RequestID: "act_vm-verify-5", Success: true, ExitCode: 0, Stdout: "status: running"},
}}
executor := newProxmoxGuestActionExecutor(h, agents)
executor := newProxmoxGuestActionExecutor(h, agents, nil)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown"))
if err != nil {
t.Fatalf("ExecuteAction: %v", err)
}
if result == nil || result.Success || result.Verification == nil || result.Verification.Success {
t.Fatalf("result = %#v, want failed verification to fail action", result)
if result == nil || !result.Success || result.Verification == nil || result.Verification.Success {
t.Fatalf("result = %#v, want succeeded execution with contradicted verification", result)
}
if !strings.Contains(result.ErrorMessage, "verification did not confirm") {
t.Fatalf("error = %q, want verification failure", result.ErrorMessage)
if result.ActionResultV2 == nil || result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded || result.ActionResultV2.Verification.Status != unified.ActionVerificationContradicted || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceAgentAttested {
t.Fatalf("canonical truth = %#v, want independent execution and verification axes", result.ActionResultV2)
}
if result.ErrorMessage != "" {
t.Fatalf("error = %q, want no execution error from verification contradiction", result.ErrorMessage)
}
}
func TestProxmoxGuestActionExecutorUsesIndependentControlPlaneVerification(t *testing.T) {
now := time.Now().UTC()
resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now)
resource.Proxmox.Uptime = 3600
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
h.SetStateProvider(resourceUnifiedSeedProvider{
snapshot: models.StateSnapshot{LastUpdate: now},
resources: []unified.Resource{resource},
})
agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{
{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"},
{RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"},
}}
observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{
proxmoxGuestActionObservation(now.Add(-time.Second), "running", 3600, "proxmox-control-plane:default:homelab"),
proxmoxGuestActionObservation(now.Add(time.Second), "stopped", 0, "proxmox-control-plane:default:homelab"),
}}
executor := newProxmoxGuestActionExecutor(h, agents, observer)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown"))
if err != nil {
t.Fatalf("ExecuteAction: %v", err)
}
if result == nil || result.ActionResultV2 == nil {
t.Fatalf("result = %#v, want canonical truth", result)
}
truth := result.ActionResultV2.Verification
if truth.Status != unified.ActionVerificationConfirmed || truth.EvidenceClass != unified.ActionEvidenceIndependent || len(truth.Evidence) != 1 {
t.Fatalf("verification truth = %#v, want independent confirmation", truth)
}
if truth.Evidence[0].ObserverKind != "proxmox_control_plane" || truth.Evidence[0].ObserverTrustDomain == truth.Evidence[0].ExecutorTrustDomain {
t.Fatalf("independent evidence = %#v", truth.Evidence[0])
}
}
func TestProxmoxGuestActionExecutorRequiresUptimeResetToVerifyReboot(t *testing.T) {
now := time.Now().UTC()
resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now)
resource.Proxmox.Uptime = 7200
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}})
agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "reboot requested"}}}
observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{
proxmoxGuestActionObservation(now.Add(-time.Second), "running", 7200, "proxmox-control-plane:default:homelab"),
proxmoxGuestActionObservation(now.Add(time.Second), "running", 4, "proxmox-control-plane:default:homelab"),
}}
executor := newProxmoxGuestActionExecutor(h, agents, observer)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "reboot"))
if err != nil {
t.Fatalf("ExecuteAction: %v", err)
}
if result == nil || result.ActionResultV2 == nil || result.ActionResultV2.Verification.Status != unified.ActionVerificationConfirmed || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceIndependent {
t.Fatalf("result = %#v, want independently verified reboot", result)
}
if len(agents.calls) != 1 {
t.Fatalf("agent calls = %d, want mutation only because status-only agent read cannot prove reboot", len(agents.calls))
}
}
func TestProxmoxGuestActionExecutorKeepsIndependentContradictionSeparateFromExecution(t *testing.T) {
now := time.Now().UTC()
resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now)
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}})
agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "reboot requested"}}}
observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{
proxmoxGuestActionObservation(now.Add(-time.Second), "running", 7200, "proxmox-control-plane:default:homelab"),
proxmoxGuestActionObservation(now.Add(time.Second), "running", 7201, "proxmox-control-plane:default:homelab"),
}}
executor := newProxmoxGuestActionExecutor(h, agents, observer)
ctx, cancel := context.WithTimeout(actionDispatchTestContext(t, "act_vm"), 100*time.Millisecond)
defer cancel()
result, err := executor.ExecuteAction(ctx, proxmoxGuestActionRecord("act_vm", "vm:160", "reboot"))
if err != nil {
t.Fatalf("ExecuteAction: %v", err)
}
if result == nil || result.ActionResultV2 == nil || !result.Success {
t.Fatalf("result = %#v, want successful command execution", result)
}
truth := result.ActionResultV2.Verification
if truth.Status != unified.ActionVerificationContradicted || truth.EvidenceClass != unified.ActionEvidenceIndependent {
t.Fatalf("verification truth = %#v, want independent contradiction", truth)
}
}
func TestProxmoxGuestActionExecutorRejectsSameDomainIndependentEvidence(t *testing.T) {
now := time.Now().UTC()
resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now)
h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()})
h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}})
agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{
{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"},
{RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"},
}}
observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{
proxmoxGuestActionObservation(now.Add(-time.Second), "running", 100, "agent:node-agent-1"),
proxmoxGuestActionObservation(now.Add(time.Second), "stopped", 0, "agent:node-agent-1"),
}}
executor := newProxmoxGuestActionExecutor(h, agents, observer)
result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown"))
if err != nil {
t.Fatalf("ExecuteAction: %v", err)
}
if result == nil || result.ActionResultV2 == nil || result.ActionResultV2.Verification.Status != unified.ActionVerificationConfirmed || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceAgentAttested {
t.Fatalf("result = %#v, want same-domain observation rejected in favor of agent-attested truth", result)
}
}
type fakeProxmoxGuestPostconditionObserver struct {
observations []proxmoxGuestPostconditionObservation
next int
}
func (o *fakeProxmoxGuestPostconditionObserver) ObserveProxmoxGuest(context.Context, string, string, string, int, proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error) {
if o.next >= len(o.observations) {
return proxmoxGuestPostconditionObservation{}, context.DeadlineExceeded
}
observation := o.observations[o.next]
o.next++
return observation, nil
}
func proxmoxGuestActionObservation(observedAt time.Time, status string, uptime uint64, trustDomain string) proxmoxGuestPostconditionObservation {
return proxmoxGuestPostconditionObservation{
ObserverID: "proxmox-api:default:homelab",
TrustDomain: trustDomain,
Method: "proxmox_api_guest_status_current",
Snapshot: proxmoxGuestLifecycleSnapshot{
Instance: "homelab",
Node: "delly",
VMID: 160,
Kind: proxmoxGuestVM,
Status: status,
Uptime: uptime,
ObservedAt: observedAt,
},
ReceivedAt: observedAt,
}
}
@@ -181,7 +327,7 @@ func TestHandlePlanActionRejectsDisconnectedProxmoxNodeCommandAgent(t *testing.T
h,
newProxmoxGuestActionExecutor(h, &fakeDockerActionAgentCommander{
connected: map[string]bool{"node-agent-1": false},
}),
}, nil),
))
rec := httptest.NewRecorder()
@@ -228,7 +374,7 @@ func TestResourceResponsesFilterDisconnectedProxmoxLifecycleCapabilities(t *test
h,
newProxmoxGuestActionExecutor(h, &fakeDockerActionAgentCommander{
connected: map[string]bool{"node-agent-1": false},
}),
}, nil),
))
listRec := httptest.NewRecorder()
@@ -0,0 +1,54 @@
package api
import (
"context"
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
)
type proxmoxGuestMonitorResolver func(string) *monitoring.Monitor
type proxmoxGuestMonitoringObserver struct {
resolveMonitor proxmoxGuestMonitorResolver
}
func newProxmoxGuestMonitoringObserver(resolveMonitor proxmoxGuestMonitorResolver) proxmoxGuestPostconditionObserver {
if resolveMonitor == nil {
return nil
}
return proxmoxGuestMonitoringObserver{resolveMonitor: resolveMonitor}
}
func (o proxmoxGuestMonitoringObserver) ObserveProxmoxGuest(ctx context.Context, resourceID, instance, node string, vmid int, kind proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error) {
orgID := strings.TrimSpace(GetOrgID(ctx))
if orgID == "" {
orgID = "default"
}
monitor := o.resolveMonitor(orgID)
if monitor == nil {
return proxmoxGuestPostconditionObservation{}, fmt.Errorf("monitor unavailable for organization %q", orgID)
}
observation, err := monitor.ObserveProxmoxGuest(ctx, instance, node, vmid, string(kind))
receivedAt := time.Now().UTC()
if err != nil {
return proxmoxGuestPostconditionObservation{}, err
}
return proxmoxGuestPostconditionObservation{
ObserverID: "proxmox-api:" + orgID + ":" + strings.TrimSpace(instance),
TrustDomain: "proxmox-control-plane:" + orgID + ":" + strings.TrimSpace(instance),
Method: "proxmox_api_guest_status_current",
Snapshot: proxmoxGuestLifecycleSnapshot{
Instance: observation.Instance,
Node: observation.Node,
VMID: observation.VMID,
Kind: proxmoxGuestKind(observation.Kind),
Status: observation.Status,
Uptime: observation.Uptime,
ObservedAt: observation.ObservedAt,
},
ReceivedAt: receivedAt,
}, nil
}
+127
View File
@@ -0,0 +1,127 @@
package api
import (
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
"github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt"
unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func proxmoxGuestExecutionResult(actionID, resourceID, agentID string, kind proxmoxGuestKind, operation string, exitCode int, output, commandError string, agentVerification *unified.ActionVerificationResult, independentBefore, independentAfter *proxmoxGuestPostconditionObservation, independentEvaluation agentexec.PostconditionEvaluation, actionStartedAt time.Time) (*unified.ExecutionResult, error) {
executionSummary := fmt.Sprintf("Proxmox guest %s command exited with status %d.", strings.TrimSpace(operation), exitCode)
execution := unified.ActionExecutionTruth{Status: unified.ActionExecutionSucceeded, Summary: executionSummary}
verification := unified.ActionVerificationTruth{Status: unified.ActionVerificationInconclusive, EvidenceClass: unified.ActionEvidenceNone, ReasonCode: "agent_readback_inconclusive", Summary: "The executing agent did not return a conclusive postcondition read."}
legacy := &unified.ExecutionResult{Output: strings.TrimSpace(output)}
if exitCode != 0 {
execution.Status = unified.ActionExecutionFailed
execution.ReasonCode = "proxmox_command_failed"
execution.Summary = strings.TrimSpace(firstNonEmpty(commandError, output, executionSummary))
verification = unified.ActionVerificationTruth{Status: unified.ActionVerificationNotAttempted, EvidenceClass: unified.ActionEvidenceNone}
legacy.ErrorMessage = execution.Summary
} else if agentVerification != nil && agentVerification.Ran {
agentSummary := strings.TrimSpace(firstNonEmpty(agentVerification.Note, agentVerification.Output, "The executing agent read the Proxmox guest postcondition."))
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
Version: unified.ActionEvidenceVersion,
ID: actionID + "-agent-proxmox-readback",
ObserverID: agentID,
ObserverKind: "unified_agent",
ObserverTrustDomain: "agent:" + agentID,
ExecutorTrustDomain: "agent:" + agentID,
Method: "server_owned_proxmox_cli_status",
SubjectID: resourceID,
ObservedAt: agentVerification.RanAt.UTC(),
ReceivedAt: time.Now().UTC(),
Summary: agentSummary,
})
if err != nil {
return nil, fmt.Errorf("normalize Proxmox agent readback evidence: %w", err)
}
status := unified.ActionVerificationContradicted
reason := "postcondition_contradicted"
if agentVerification.Success {
status = unified.ActionVerificationConfirmed
reason = ""
}
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceAgentAttested, ReasonCode: reason, Summary: agentSummary, Evidence: []unified.ActionEvidence{evidence}}
}
if exitCode == 0 && usableIndependentProxmoxObservation(agentID, actionStartedAt, independentAfter) {
beforeSnapshot := (*proxmoxGuestLifecycleSnapshot)(nil)
if independentBefore != nil && usableIndependentProxmoxBeforeObservation(actionStartedAt, independentBefore) {
beforeCopy := independentBefore.Snapshot
beforeSnapshot = &beforeCopy
}
observationDigest, err := operationreceipt.DigestCanonicalJSON(struct {
ActionID string `json:"action_id"`
SubjectID string `json:"subject_id"`
Before *proxmoxGuestLifecycleSnapshot `json:"before,omitempty"`
After proxmoxGuestLifecycleSnapshot `json:"after"`
}{ActionID: actionID, SubjectID: resourceID, Before: beforeSnapshot, After: independentAfter.Snapshot})
if err != nil {
return nil, fmt.Errorf("digest Proxmox control-plane observation: %w", err)
}
independentSummary := fmt.Sprintf("Proxmox API observation: status=%s; uptime=%ds", independentAfter.Snapshot.Status, independentAfter.Snapshot.Uptime)
evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{
Version: unified.ActionEvidenceVersion,
ID: actionID + "-proxmox-api-observation",
ObserverID: independentAfter.ObserverID,
ObserverKind: "proxmox_control_plane",
ObserverTrustDomain: independentAfter.TrustDomain,
ExecutorTrustDomain: "agent:" + agentID,
Method: independentAfter.Method,
SubjectID: resourceID,
ObservedAt: independentAfter.Snapshot.ObservedAt.UTC(),
ReceivedAt: independentAfter.ReceivedAt.UTC(),
Summary: independentSummary,
Refs: []unified.ActionEvidenceRef{{ID: actionID, Kind: "proxmox_before_after", Digest: observationDigest}},
})
if err != nil {
return nil, fmt.Errorf("normalize Proxmox control-plane evidence: %w", err)
}
status := unified.ActionVerificationInconclusive
reason := strings.TrimSpace(firstNonEmpty(independentEvaluation.ReasonCode, "postcondition_inconclusive"))
if independentEvaluation.Conclusive {
status = unified.ActionVerificationContradicted
reason = "postcondition_contradicted"
if independentEvaluation.Matched {
status = unified.ActionVerificationConfirmed
reason = ""
}
}
verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceIndependent, ReasonCode: reason, Summary: independentSummary, Evidence: []unified.ActionEvidence{evidence}}
}
canonical := unified.ActionResultV2{
Version: unified.ActionResultV2Version,
Execution: execution,
Verification: verification,
Compensation: unified.ActionCompensationTruth{Support: unified.ActionCompensationUnavailable, Status: unified.ActionCompensationNotAvailable, Summary: "Proxmox guest lifecycle actions do not provide automatic rollback."},
}
projected, _, err := unified.ApplyActionResultV2(legacy, canonical)
if err != nil {
return nil, fmt.Errorf("apply canonical Proxmox lifecycle action truth: %w", err)
}
projected.Verification = unified.LegacyActionVerificationFromV2(*projected.ActionResultV2)
return projected, nil
}
func usableIndependentProxmoxObservation(agentID string, actionStartedAt time.Time, observation *proxmoxGuestPostconditionObservation) bool {
if observation == nil || strings.TrimSpace(observation.ObserverID) == "" || strings.TrimSpace(observation.TrustDomain) == "" || strings.TrimSpace(observation.Method) == "" || strings.TrimSpace(observation.TrustDomain) == "agent:"+strings.TrimSpace(agentID) {
return false
}
observedAt := observation.Snapshot.ObservedAt.UTC()
receivedAt := observation.ReceivedAt.UTC()
return !actionStartedAt.IsZero() && !observedAt.Before(actionStartedAt.UTC()) && !observedAt.After(receivedAt.Add(5*time.Minute)) && !observedAt.Before(receivedAt.Add(-15*time.Minute))
}
func usableIndependentProxmoxBeforeObservation(actionStartedAt time.Time, observation *proxmoxGuestPostconditionObservation) bool {
if observation == nil || actionStartedAt.IsZero() || observation.Snapshot.ObservedAt.IsZero() || observation.ReceivedAt.IsZero() {
return false
}
observedAt := observation.Snapshot.ObservedAt.UTC()
receivedAt := observation.ReceivedAt.UTC()
return !observedAt.After(actionStartedAt.UTC()) && !observedAt.After(receivedAt.Add(5*time.Minute)) && !observedAt.Before(receivedAt.Add(-15*time.Minute))
}
+1 -1
View File
@@ -639,7 +639,7 @@ func (r *Router) setupRoutes() {
r.resourceHandlers.SetActionExecutor(newRoutedActionExecutor(
r.resourceHandlers,
newDockerContainerActionExecutor(r.resourceHandlers, r.agentExecServer),
newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer),
newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer, newProxmoxGuestMonitoringObserver(r.resolveMonitorForOrg)),
newHostStorageCleanupActionExecutor(r.resourceHandlers, r.agentExecServer),
newHostUpdateActionExecutor(r.resourceHandlers, r.agentExecServer),
))
@@ -123,6 +123,24 @@ func TestPVETagStyleRefreshStaysPerInstance(t *testing.T) {
}
}
func TestProxmoxActionObserverUsesDirectControlPlaneClient(t *testing.T) {
data, err := os.ReadFile("proxmox_action_observer.go")
if err != nil {
t.Fatalf("read proxmox_action_observer.go: %v", err)
}
source := string(data)
for _, snippet := range []string{"m.getPVEClient(instance)", "client.GetVMStatus(ctx, node, vmid)", "client.GetContainerStatus(ctx, node, vmid)"} {
if !strings.Contains(source, snippet) {
t.Fatalf("Proxmox action observer must use direct control-plane read %q", snippet)
}
}
for _, forbidden := range []string{"m.GetState(", "m.ReadState(", "m.resourceStore", "m.state."} {
if strings.Contains(source, forbidden) {
t.Fatalf("Proxmox action observer must not derive independent evidence through cached monitoring state %q", forbidden)
}
}
}
func TestBroadcastResourceDiskIOUsesUnifiedResourceMetrics(t *testing.T) {
hasDiskIO, readRate, writeRate := monitorDiskIOMetricInput(&unifiedresources.ResourceMetrics{
DiskRead: &unifiedresources.MetricValue{Value: 4096.4, Unit: "bytes/s", Source: unifiedresources.SourceAgent},
@@ -0,0 +1,82 @@
package monitoring
import (
"context"
"fmt"
"strings"
"time"
)
// ProxmoxGuestObservation is a direct server-side Proxmox API read used by
// the governed action layer as an observer distinct from the node agent that
// executes guest lifecycle commands.
type ProxmoxGuestObservation struct {
Instance string
Node string
VMID int
Kind string
Status string
Uptime uint64
ObservedAt time.Time
}
// ObserveProxmoxGuest reads current guest state through the configured
// Proxmox control-plane client. It does not use cached monitor state or the
// mutating node agent, so callers may classify a valid observation under the
// Proxmox control-plane trust domain.
func (m *Monitor) ObserveProxmoxGuest(ctx context.Context, instance, node string, vmid int, kind string) (ProxmoxGuestObservation, error) {
if m == nil {
return ProxmoxGuestObservation{}, fmt.Errorf("monitor unavailable")
}
instance = strings.TrimSpace(instance)
node = strings.TrimSpace(node)
kind = strings.ToLower(strings.TrimSpace(kind))
if instance == "" || node == "" || vmid <= 0 {
return ProxmoxGuestObservation{}, fmt.Errorf("proxmox guest observer requires instance, node, and vmid")
}
client, ok := m.getPVEClient(instance)
if !ok || client == nil {
return ProxmoxGuestObservation{}, fmt.Errorf("proxmox client for instance %q unavailable", instance)
}
observation := ProxmoxGuestObservation{
Instance: instance,
Node: node,
VMID: vmid,
Kind: kind,
}
switch kind {
case "vm", "qemu":
status, err := client.GetVMStatus(ctx, node, vmid)
if err != nil {
return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox VM status: %w", err)
}
if status == nil {
return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox VM status: empty response")
}
observation.Kind = "vm"
observation.Status = strings.ToLower(strings.TrimSpace(status.Status))
observation.Uptime = status.Uptime
case "ct", "lxc", "container":
status, err := client.GetContainerStatus(ctx, node, vmid)
if err != nil {
return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: %w", err)
}
if status == nil {
return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: empty response")
}
if observedVMID := int(status.VMID); observedVMID != 0 && observedVMID != vmid {
return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: vmid mismatch")
}
observation.Kind = "ct"
observation.Status = strings.ToLower(strings.TrimSpace(status.Status))
observation.Uptime = status.Uptime
default:
return ProxmoxGuestObservation{}, fmt.Errorf("unsupported Proxmox guest kind %q", kind)
}
if observation.Status == "" {
return ProxmoxGuestObservation{}, fmt.Errorf("Proxmox guest observation omitted status")
}
observation.ObservedAt = time.Now().UTC()
return observation, nil
}
@@ -0,0 +1,62 @@
package monitoring
import (
"context"
"testing"
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
)
func TestObserveProxmoxGuestReadsDirectControlPlaneState(t *testing.T) {
client := &actionObserverPVEClient{
stubPVEClient: &stubPVEClient{},
vmStatus: &proxmox.VMStatus{Status: "running", Uptime: 42},
ctStatus: &proxmox.Container{VMID: proxmox.FlexInt(101), Status: "stopped", Uptime: 9},
}
monitor := &Monitor{pveClients: map[string]PVEClientInterface{"homelab": client}}
vm, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 160, "vm")
if err != nil {
t.Fatalf("ObserveProxmoxGuest VM: %v", err)
}
if vm.Kind != "vm" || vm.Status != "running" || vm.Uptime != 42 || vm.Instance != "homelab" || vm.Node != "node-a" || vm.VMID != 160 || vm.ObservedAt.IsZero() {
t.Fatalf("VM observation = %#v", vm)
}
ct, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 101, "lxc")
if err != nil {
t.Fatalf("ObserveProxmoxGuest CT: %v", err)
}
if ct.Kind != "ct" || ct.Status != "stopped" || ct.Uptime != 9 || ct.VMID != 101 {
t.Fatalf("CT observation = %#v", ct)
}
}
func TestObserveProxmoxGuestFailsClosedForMissingClientAndIdentityMismatch(t *testing.T) {
monitor := &Monitor{pveClients: map[string]PVEClientInterface{}}
if _, err := monitor.ObserveProxmoxGuest(context.Background(), "missing", "node-a", 160, "vm"); err == nil {
t.Fatal("missing Proxmox client unexpectedly produced an observation")
}
monitor.pveClients["homelab"] = &actionObserverPVEClient{
stubPVEClient: &stubPVEClient{},
ctStatus: &proxmox.Container{VMID: proxmox.FlexInt(999), Status: "running"},
}
if _, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 101, "ct"); err == nil {
t.Fatal("mismatched Proxmox CT identity unexpectedly produced an observation")
}
}
type actionObserverPVEClient struct {
*stubPVEClient
vmStatus *proxmox.VMStatus
ctStatus *proxmox.Container
}
func (c *actionObserverPVEClient) GetVMStatus(context.Context, string, int) (*proxmox.VMStatus, error) {
return c.vmStatus, nil
}
func (c *actionObserverPVEClient) GetContainerStatus(context.Context, string, int) (*proxmox.Container, error) {
return c.ctStatus, nil
}
@@ -2970,8 +2970,8 @@ class SubsystemLookupTest(unittest.TestCase):
{
"heading": "## Shared Boundaries",
"path": "internal/api/access_control_handlers.go",
"line": 1212,
"heading_line": 142,
"line": 1214,
"heading_line": 144,
}
],
)