From f095da2fdbde3f44522ef9afae148bb98358b983 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Mon, 13 Jul 2026 11:04:09 +0100 Subject: [PATCH] Wire production Proxmox action verification --- docs/release-control/v6/internal/status.json | 17 +- .../v6/internal/subsystems/agent-lifecycle.md | 15 +- .../v6/internal/subsystems/api-contracts.md | 17 +- .../v6/internal/subsystems/monitoring.md | 9 + .../subsystems/performance-and-scalability.md | 5 +- .../internal/subsystems/security-privacy.md | 7 + .../internal/subsystems/storage-recovery.md | 8 + .../internal/subsystems/unified-resources.md | 13 ++ internal/agentexec/verifier_postconditions.go | 138 ++++++++++++++ .../agentexec/verifier_postconditions_test.go | 87 ++++++++- internal/api/contract_test.go | 5 +- internal/api/proxmox_guest_action_executor.go | 146 ++++++++++++--- .../api/proxmox_guest_action_executor_test.go | 168 ++++++++++++++++-- internal/api/proxmox_guest_action_observer.go | 54 ++++++ internal/api/proxmox_guest_action_result.go | 127 +++++++++++++ internal/api/router.go | 2 +- .../monitoring/canonical_guardrails_test.go | 18 ++ .../monitoring/proxmox_action_observer.go | 82 +++++++++ .../proxmox_action_observer_test.go | 62 +++++++ .../release_control/subsystem_lookup_test.py | 4 +- 20 files changed, 932 insertions(+), 52 deletions(-) create mode 100644 internal/api/proxmox_guest_action_observer.go create mode 100644 internal/api/proxmox_guest_action_result.go create mode 100644 internal/monitoring/proxmox_action_observer.go create mode 100644 internal/monitoring/proxmox_action_observer_test.go diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 874f77e44..114dfdc2d 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -8113,7 +8113,7 @@ }, { "id": "action-governance-auditability-post-rc-hardening", - "summary": "Task 07 durable delivery and generic agent-operation receipts are accepted: transactional dispatch admission, one-shot send CAS, immutable attempt/action/operation/digest/agent binding, query-only restart reconciliation, strict sanitized terminal envelopes, and permanent replay-denial tombstones. Task 09 consumes that owner for both APT workflows and completes the detector-to-finding, exact empty-parameter proposal, shared policy/approval, typed dispatch, delayed/callback-loss/reopened-server receipt reconciliation, ActionResultV2 truth, terminal audit, and finding reconciliation floor without blind resend. Task 11 adds the desktop/browser tier-5 Product Trust consumer: bounded command/path/package-free Patrol evidence; exact empty parameter and elevated-versus-low-risk review; separate execution, evidence-sourced verification, and recovery cards; agent-attested confirmed updates; reboot fact without reboot authority; partial/unknown-health recovery; measured irreversible cleanup; and one durable receipt across reconnect, all exercised in current-build Chromium and a 390-pixel viewport. Malformed phases, counts, timestamps, usage, or cleanup arithmetic fail closed. The closed RG-01 through RG-12 matrix now supplies the disposable Debian/Ubuntu tier-6, Docker, browser, physical-device, live-Relay, revocation, cleanup, and independent Task 12 proof that previously kept claims 16 and 17 and both workflow scorecards operationally open. Task 10 remains sole owner of execution, verification, evidence, compensation, and rollback truth, and RG06/RG09 product outcomes remain agent-attested fix_verification_unknown. This follow-up now tracks only broader post-RC auditability beyond the proved bounded capability set, including additional operation and platform coverage, deeper compensation and rollback proof, and MSP or fleet aggregation; it does not reopen the accepted matrix. Raw model command, file write, arbitrary pod exec, legacy run_command, /api/ai/run-command, and enterprise command remediation remain retired with no replacement or executable historical authority.", + "summary": "Task 07 durable delivery and generic agent-operation receipts are accepted: transactional dispatch admission, one-shot send CAS, immutable attempt/action/operation/digest/agent binding, query-only restart reconciliation, strict sanitized terminal envelopes, and permanent replay-denial tombstones. Task 09 consumes that owner for both APT workflows and completes the detector-to-finding, exact empty-parameter proposal, shared policy/approval, typed dispatch, delayed/callback-loss/reopened-server receipt reconciliation, ActionResultV2 truth, terminal audit, and finding reconciliation floor without blind resend. Task 11 adds the desktop/browser tier-5 Product Trust consumer: bounded command/path/package-free Patrol evidence; exact empty parameter and elevated-versus-low-risk review; separate execution, evidence-sourced verification, and recovery cards; agent-attested confirmed updates; reboot fact without reboot authority; partial/unknown-health recovery; measured irreversible cleanup; and one durable receipt across reconnect, all exercised in current-build Chromium and a 390-pixel viewport. Malformed phases, counts, timestamps, usage, or cleanup arithmetic fail closed. The closed RG-01 through RG-12 matrix now supplies the disposable Debian/Ubuntu tier-6, Docker, browser, physical-device, live-Relay, revocation, cleanup, and independent Task 12 proof that previously kept claims 16 and 17 and both workflow scorecards operationally open. Task 10 remains sole owner of execution, verification, evidence, compensation, and rollback truth, and RG06/RG09 product outcomes remain agent-attested fix_verification_unknown. Proxmox VM/LXC lifecycle now supplies the first production distinct-trust-domain verification path: the node agent executes, the tenant-scoped server Proxmox client observes status and uptime, reboot requires an uptime reset, and ActionResultV2 keeps execution success independent from postcondition contradiction. This follow-up now tracks only broader post-RC auditability beyond the proved bounded capability set, including additional operation and platform coverage, deeper compensation and rollback proof, and MSP or fleet aggregation; it does not reopen the accepted matrix. Raw model command, file write, arbitrary pod exec, legacy run_command, /api/ai/run-command, and enterprise command remediation remain retired with no replacement or executable historical authority.", "owner": "project-owner", "status": "planned", "recorded_at": "2026-04-25", @@ -8486,6 +8486,21 @@ "repo": "pulse", "path": "docs/release-control/v6/internal/SOURCE_OF_TRUTH.md", "kind": "file" + }, + { + "repo": "pulse", + "path": "internal/api/proxmox_guest_action_observer.go", + "kind": "file" + }, + { + "repo": "pulse", + "path": "internal/api/proxmox_guest_action_result.go", + "kind": "file" + }, + { + "repo": "pulse", + "path": "internal/monitoring/proxmox_action_observer.go", + "kind": "file" } ] }, diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 506e09ecb..b9c73eaad 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -31,6 +31,7 @@ that binary, not separate customer-facing agent products. 5. `internal/hostagent/agent.go` 5a. `internal/dockeragent/agent.go` 5b. `internal/kubernetesagent/agent.go` + 5c. `internal/agentexec/verifier_postconditions.go` 6. `cmd/pulse-agent/main.go` 7. `scripts/install.sh` 8. `scripts/install.ps1` @@ -481,6 +482,12 @@ connected Proxmox node command agent and records action audit plus verification. Lifecycle surfaces must not run `qm` / `pct`, SSH to a node, call Proxmox mutation APIs, or substitute a guest-local agent to perform VM/LXC lifecycle control. +The executing node agent's status read remains agent-attested. The closed +postcondition registry in `internal/agentexec/verifier_postconditions.go` +defines start, shutdown, stop, and reboot checks for both `qm` and `pct`, while +the API-owned verifier supplies provider observations. Reboot is not confirmed +from status alone: it requires a distinct Proxmox API observation whose uptime +reset proves that the guest actually restarted. Disconnected command-agent state is also API-owned readiness: lifecycle surfaces may reflect missing backend-advertised capabilities, but must not reconnect, substitute, or directly address an agent to make a stale container @@ -4500,6 +4507,8 @@ RG-06 and RG-09 keep the executing agent's fresh typed readback classified as fixture changed, but it does not enter the authenticated product action result and therefore cannot upgrade the product finding to `fix_verified`. Until a distinct-trust-domain observation is ingested into `ActionResultV2`, those -findings remain `fix_verification_unknown` and unresolved. The Docker restart -journey, which does ingest its direct daemon observation as independent -evidence, remains the positive `fix_verified` control. +findings remain `fix_verification_unknown` and unresolved. Proxmox VM/LXC +lifecycle is now the first production path to ingest a server-side provider +observation as independent evidence; the Docker restart lab journey remains a +positive control but does not imply that agent-reported Docker inventory is an +independent production observer. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 85246dbb0..abf66d893 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -37,7 +37,9 @@ product API routes free of maintainer commercial analytics. 5a. `internal/api/action_executor.go` 5b. `internal/api/docker_container_action_executor.go` 5c. `internal/api/proxmox_guest_action_executor.go` - 5d. `internal/api/host_update_action_executor.go` + 5d. `internal/api/proxmox_guest_action_observer.go` + 5e. `internal/api/proxmox_guest_action_result.go` + 5f. `internal/api/host_update_action_executor.go` 6a. `internal/actionlifecycle/service.go` 7. `internal/actionplanner/planner.go` 8. `pkg/pulsecli/api_client.go` @@ -7423,6 +7425,19 @@ the action, subject, observation and receipt times, before/after facts, and canonical evidence digest. It is independent operational evidence, not a cryptographic attestation. +Proxmox VM and LXC lifecycle execution now consumes that same two-axis truth +contract in production. The node agent remains the executor, while the API +composition root injects the tenant-scoped monitoring client as a direct +Proxmox control-plane observer. Fresh, identity-matched control-plane reads use +an `agent:*`-distinct trust domain and are digest-bound into `ActionResultV2`; +missing clients, mismatched guest identity, stale timestamps, or same-domain +evidence fail closed to the agent-attested or inconclusive result. Start, +shutdown, stop, and reboot all evaluate the shared closed postcondition +registry. Reboot additionally requires the direct API uptime to reset from a +fresh pre-action observation, so a guest that merely stayed `running` cannot +be reported as verified. A contradicted postcondition never rewrites a +successful command dispatch into execution failure. + ### Canonical mutation plane `internal/mutationregistry/manifest.json` is the generated closed registry for diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index d3621a701..4c8c0219a 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -133,6 +133,7 @@ resource health. 55. `internal/monitoring/resource_stale_thresholds.go` 56. `internal/monitoring/recovery_ingest.go` 57. `internal/monitoring/multi_tenant_monitor.go` +58. `internal/monitoring/proxmox_action_observer.go` ## Shared Boundaries @@ -1646,3 +1647,11 @@ Task 09 preserves two APT telemetry clocks at host-agent ingest: `CheckedAt` is agent-observed time and `ObservedAt` is server-received time. Monitoring must not overwrite the former with the latter; replay and skew safety consumes both timestamps downstream. + +Monitoring now also exposes a bounded direct Proxmox guest observation for the +governed action verifier. `ObserveProxmoxGuest` resolves the configured +instance client under the monitor lock, reads VM or LXC status and uptime from +the Proxmox API, validates the requested guest identity, and stamps server +observation time. It must not satisfy this contract from cached resource state +or node-agent telemetry: the action layer depends on this read remaining in a +trust domain distinct from the node agent that executes `qm` / `pct`. diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md index 879a1ba52..0b57ef56a 100644 --- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md +++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md @@ -421,7 +421,10 @@ change may globally weaken the Task 03 lifecycle-state idempotency invariant. router setup may register the Proxmox executor alongside Docker / Podman, but it must not resolve guests, probe node agents, call `qm` / `pct`, poll verification state, or refresh inventory outside the route-local action - plan/execute path. + plan/execute path. Injecting the tenant-monitor resolver at setup remains + constant work; the direct Proxmox status/uptime reads are action-local and + poll at most once per second within the closed postcondition registry's + bounded verification window. Retiring self-hosted trial acquisition follows that same rule: removing `/auth/trial-activate` and `POST /api/license/trial/start` from public-path and CSRF inventories must stay as constant-time route-table absence rather diff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.md index 119b877f7..cf7d278d8 100644 --- a/docs/release-control/v6/internal/subsystems/security-privacy.md +++ b/docs/release-control/v6/internal/subsystems/security-privacy.md @@ -371,6 +371,13 @@ the `white_label` branding entitlement. command, package selector, or deletion rule. Cache entry names, fingerprint, raw APT output, and agent error text remain out of model context and terminal action output. + Proxmox VM/LXC lifecycle observer wiring is also not mutation authority. + It may resolve only the current tenant's existing monitor and use that + monitor's configured Proxmox client for bounded status/uptime reads after a + governed node-agent action. It must not expose provider credentials or raw + provider responses, cross tenant or guest identity, call a provider + mutation API, or classify the executing agent's own trust domain as + independent evidence. Router glue may also pass monitor-owned source freshness thresholds into unified-resource adapters, but those thresholds are operational cadence metadata only. They must not disclose credentials, command output, raw diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 1a60e7c50..b4df4c4e6 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -148,6 +148,14 @@ recovery surfaces may observe disconnected command readiness, but they must not re-enable command config, reinterpret the suppression as backup or restore state, or introduce a recovery-local command-token binding path. +Proxmox VM/LXC lifecycle verification in `internal/api/` is likewise adjacent +action-governance evidence, not recovery evidence. The node agent may execute a +governed `qm` / `pct` action while the tenant-scoped server Proxmox client +independently observes guest status and uptime, but a confirmed start, stop, +shutdown, or reboot proves only that action's registered postcondition. Storage +and recovery surfaces must not reinterpret that `ActionResultV2` evidence as a +recovery point, backup freshness, restore success, or protected-workload state. + Hosted tenant agent install commands in `internal/api/cloud_agent_install_command.go` are adjacent API/lifecycle transport only. A provider-hosted MSP PVE/PBS install token may allow agent reporting for the scoped tenant workspace, but it must not diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md index 43a9fd7ea..578f72775 100644 --- a/docs/release-control/v6/internal/subsystems/unified-resources.md +++ b/docs/release-control/v6/internal/subsystems/unified-resources.md @@ -662,6 +662,12 @@ second-click local confirmation. Proof is owned by the colocated action tests, Unified-resource consumers must not infer Proxmox lifecycle affordances from table row status alone or issue direct `qm` / `pct`, SSH, provider API, or guest-agent calls outside the governed action contract. + The lifecycle executor must project terminal truth through `ActionResultV2` + without coupling execution success to verification. Same-agent status reads + remain `agent_attested`; only a fresh, identity-matched read through the + tenant's server-side Proxmox API client may be `independent`. Reboot + confirmation additionally requires an uptime reset relative to the direct + pre-action observation, not merely a post-action `running` status. Agent-managed Linux hosts may advertise `install_os_updates` only when the report carries supported APT package posture and typed command operations are enabled. The capability is admin-floor, `elevated` auto-authorization @@ -1781,6 +1787,13 @@ Workflow, API, AI, agent, Docker, host-agent, and relay packages must not declare competing truth enums. Generated wire mirrors remain a bounded later presentation concern, not a second source of semantics. +The first production distinct-trust-domain consumer is the Proxmox VM/LXC +lifecycle executor. Its node-agent dispatch and Proxmox API observation remain +separate evidence domains, and the API-owned projection binds the exact action, +resource, before/after snapshots, observation times, and canonical digest. +This does not authorize provider-local mutation or relax the false-independence +guard for Docker or agent-attested APT readback. + The canonical action resource contract now owns immutable `ActionActor` and versioned `ApprovalRequirement` bindings. Actor subject/kind/credential/org and requirement floor/quorum/separation are part of deterministic action identity diff --git a/internal/agentexec/verifier_postconditions.go b/internal/agentexec/verifier_postconditions.go index 7cd15df4c..4bc1efb30 100644 --- a/internal/agentexec/verifier_postconditions.go +++ b/internal/agentexec/verifier_postconditions.go @@ -2,6 +2,7 @@ package agentexec import ( "sort" + "strconv" "strings" "time" ) @@ -19,6 +20,7 @@ const ( // VM and container status as reported by Proxmox. FieldVMStatus PostconditionField = "status" FieldContainerStatus PostconditionField = "status" + FieldGuestUptime PostconditionField = "uptime" // Systemd unit fields read via DBus or systemctl show. FieldUnitActiveState PostconditionField = "ActiveState" @@ -49,6 +51,10 @@ const ( // EqualsField: observed value must equal another observed field on the // same read (e.g. readyReplicas == desiredReplicas). CompareEqualsField PostconditionComparator = "equals_field" + // LessThanBefore: the numeric observed value must be lower than the + // corresponding pre-action observation. This proves that a running guest + // actually restarted instead of merely remaining online. + CompareLessThanBefore PostconditionComparator = "less_than_before" ) // PostconditionCheck is one assertion the verifier evaluates against the @@ -76,6 +82,16 @@ type CapabilityPostcondition struct { Checks []PostconditionCheck `json:"checks"` } +// PostconditionEvaluation is the provider-neutral result of evaluating one +// registered postcondition against bounded before/after observations. +// Conclusive=false means the observation did not contain enough valid data to +// claim either confirmation or contradiction. +type PostconditionEvaluation struct { + Conclusive bool + Matched bool + ReasonCode string +} + // defaultVerifyWindow is the per-capability fallback window. The agentexec // policy carries an operator-tunable verify_window that overrides this; the // per-capability values here are the substrate's "what is reasonable for @@ -111,6 +127,72 @@ func CapabilityPostconditionNames() []string { return names } +// EvaluateCapabilityPostcondition applies the closed registry definition to +// provider-normalized string observations. Providers only map their typed read +// into the closed field vocabulary; they do not reimplement the comparisons. +func EvaluateCapabilityPostcondition(capability string, before, after map[PostconditionField]string, actionStartedAt time.Time) (PostconditionEvaluation, bool) { + postcondition, ok := LookupCapabilityPostcondition(capability) + if !ok { + return PostconditionEvaluation{}, false + } + for _, check := range postcondition.Checks { + observed, exists := normalizedPostconditionValue(after, check.Field) + if !exists { + return PostconditionEvaluation{ReasonCode: "observed_field_missing"}, true + } + switch check.Comparator { + case CompareEquals: + if !strings.EqualFold(observed, strings.TrimSpace(check.Expected)) { + return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true + } + case CompareEqualsField: + peer, exists := normalizedPostconditionValue(after, PostconditionField(check.Expected)) + if !exists { + return PostconditionEvaluation{ReasonCode: "comparison_field_missing"}, true + } + if !strings.EqualFold(observed, peer) { + return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true + } + case CompareAfterOrEqualActionStart: + if actionStartedAt.IsZero() { + return PostconditionEvaluation{ReasonCode: "action_start_missing"}, true + } + observedAt, err := time.Parse(time.RFC3339Nano, observed) + if err != nil { + return PostconditionEvaluation{ReasonCode: "observed_timestamp_invalid"}, true + } + if observedAt.Before(actionStartedAt) { + return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true + } + case CompareLessThanBefore: + previous, exists := normalizedPostconditionValue(before, check.Field) + if !exists { + return PostconditionEvaluation{ReasonCode: "before_field_missing"}, true + } + previousValue, previousErr := strconv.ParseUint(previous, 10, 64) + observedValue, observedErr := strconv.ParseUint(observed, 10, 64) + if previousErr != nil || observedErr != nil { + return PostconditionEvaluation{ReasonCode: "observed_number_invalid"}, true + } + if previousValue <= 1 { + return PostconditionEvaluation{ReasonCode: "before_value_insufficient"}, true + } + if observedValue >= previousValue { + return PostconditionEvaluation{Conclusive: true, ReasonCode: "postcondition_contradicted"}, true + } + default: + return PostconditionEvaluation{ReasonCode: "comparator_unsupported"}, true + } + } + return PostconditionEvaluation{Conclusive: true, Matched: true}, true +} + +func normalizedPostconditionValue(values map[PostconditionField]string, field PostconditionField) (string, bool) { + value, ok := values[field] + value = strings.TrimSpace(value) + return value, ok && value != "" +} + // capabilityPostconditions is the closed registry of postconditions the // verifier substrate knows how to evaluate. New tool capabilities must add // an entry here AND a corresponding test in verifier_postconditions_test.go @@ -125,6 +207,34 @@ var capabilityPostconditions = map[string]CapabilityPostcondition{ {Field: FieldVMStatus, Comparator: CompareEquals, Expected: "running"}, }, }, + "qm.shutdown": { + Capability: "qm.shutdown", + VerifyRead: "qm status ", + Window: defaultVerifyWindow, + Description: "Proxmox VM transitioned to stopped after graceful shutdown", + Checks: []PostconditionCheck{ + {Field: FieldVMStatus, Comparator: CompareEquals, Expected: "stopped"}, + }, + }, + "qm.stop": { + Capability: "qm.stop", + VerifyRead: "qm status ", + Window: defaultVerifyWindow, + Description: "Proxmox VM transitioned to stopped after hard stop", + Checks: []PostconditionCheck{ + {Field: FieldVMStatus, Comparator: CompareEquals, Expected: "stopped"}, + }, + }, + "qm.reboot": { + Capability: "qm.reboot", + VerifyRead: "Proxmox API guest status ", + Window: defaultVerifyWindow, + Description: "Proxmox VM is running with uptime reset after reboot", + Checks: []PostconditionCheck{ + {Field: FieldVMStatus, Comparator: CompareEquals, Expected: "running"}, + {Field: FieldGuestUptime, Comparator: CompareLessThanBefore}, + }, + }, "pct.start": { Capability: "pct.start", VerifyRead: "pct status ", @@ -134,6 +244,34 @@ var capabilityPostconditions = map[string]CapabilityPostcondition{ {Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "running"}, }, }, + "pct.shutdown": { + Capability: "pct.shutdown", + VerifyRead: "pct status ", + Window: defaultVerifyWindow, + Description: "Proxmox CT transitioned to stopped after graceful shutdown", + Checks: []PostconditionCheck{ + {Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "stopped"}, + }, + }, + "pct.stop": { + Capability: "pct.stop", + VerifyRead: "pct status ", + Window: defaultVerifyWindow, + Description: "Proxmox CT transitioned to stopped after hard stop", + Checks: []PostconditionCheck{ + {Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "stopped"}, + }, + }, + "pct.reboot": { + Capability: "pct.reboot", + VerifyRead: "Proxmox API guest status ", + Window: defaultVerifyWindow, + Description: "Proxmox CT is running with uptime reset after reboot", + Checks: []PostconditionCheck{ + {Field: FieldContainerStatus, Comparator: CompareEquals, Expected: "running"}, + {Field: FieldGuestUptime, Comparator: CompareLessThanBefore}, + }, + }, "docker.restart": { Capability: "docker.restart", VerifyRead: "docker inspect ", diff --git a/internal/agentexec/verifier_postconditions_test.go b/internal/agentexec/verifier_postconditions_test.go index 24e64ab27..161c216b6 100644 --- a/internal/agentexec/verifier_postconditions_test.go +++ b/internal/agentexec/verifier_postconditions_test.go @@ -12,7 +12,12 @@ import ( // of capabilities the verifier substrate must understand. Drift either way // (missing entry, drift in field/comparator) breaks the verifier contract. func TestCapabilityPostconditionsCoversRequiredCapabilities(t *testing.T) { - required := []string{"qm.start", "pct.start", "docker.restart", "systemctl.restart", "kubectl.rollout"} + required := []string{ + "docker.restart", "kubectl.rollout", + "pct.reboot", "pct.shutdown", "pct.start", "pct.stop", + "qm.reboot", "qm.shutdown", "qm.start", "qm.stop", + "systemctl.restart", + } sort.Strings(required) got := CapabilityPostconditionNames() if !reflect.DeepEqual(got, required) { @@ -54,6 +59,30 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) { expectWindow: 2 * time.Minute, mustReferenceCmd: "qm status", }, + { + capability: "qm.shutdown", + minChecks: 1, + expectFields: []PostconditionField{FieldVMStatus}, + expectCompare: []PostconditionComparator{CompareEquals}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "qm status", + }, + { + capability: "qm.stop", + minChecks: 1, + expectFields: []PostconditionField{FieldVMStatus}, + expectCompare: []PostconditionComparator{CompareEquals}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "qm status", + }, + { + capability: "qm.reboot", + minChecks: 2, + expectFields: []PostconditionField{FieldVMStatus, FieldGuestUptime}, + expectCompare: []PostconditionComparator{CompareEquals, CompareLessThanBefore}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "Proxmox API", + }, { capability: "pct.start", minChecks: 1, @@ -62,6 +91,30 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) { expectWindow: 2 * time.Minute, mustReferenceCmd: "pct status", }, + { + capability: "pct.shutdown", + minChecks: 1, + expectFields: []PostconditionField{FieldContainerStatus}, + expectCompare: []PostconditionComparator{CompareEquals}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "pct status", + }, + { + capability: "pct.stop", + minChecks: 1, + expectFields: []PostconditionField{FieldContainerStatus}, + expectCompare: []PostconditionComparator{CompareEquals}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "pct status", + }, + { + capability: "pct.reboot", + minChecks: 2, + expectFields: []PostconditionField{FieldContainerStatus, FieldGuestUptime}, + expectCompare: []PostconditionComparator{CompareEquals, CompareLessThanBefore}, + expectWindow: 2 * time.Minute, + mustReferenceCmd: "Proxmox API", + }, { capability: "docker.restart", minChecks: 2, @@ -123,6 +176,38 @@ func TestCapabilityPostconditionEntriesParse(t *testing.T) { } } +func TestEvaluateCapabilityPostcondition(t *testing.T) { + t.Run("reboot requires uptime reset", func(t *testing.T) { + before := map[PostconditionField]string{FieldVMStatus: "running", FieldGuestUptime: "7200"} + after := map[PostconditionField]string{FieldVMStatus: "running", FieldGuestUptime: "0"} + evaluation, ok := EvaluateCapabilityPostcondition("qm.reboot", before, after, time.Now()) + if !ok || !evaluation.Conclusive || !evaluation.Matched { + t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok) + } + + after[FieldGuestUptime] = "7201" + evaluation, ok = EvaluateCapabilityPostcondition("qm.reboot", before, after, time.Now()) + if !ok || !evaluation.Conclusive || evaluation.Matched || evaluation.ReasonCode != "postcondition_contradicted" { + t.Fatalf("non-reset evaluation = %#v, ok=%v", evaluation, ok) + } + }) + + t.Run("missing reboot baseline is inconclusive", func(t *testing.T) { + after := map[PostconditionField]string{FieldContainerStatus: "running", FieldGuestUptime: "4"} + evaluation, ok := EvaluateCapabilityPostcondition("pct.reboot", nil, after, time.Now()) + if !ok || evaluation.Conclusive || evaluation.Matched || evaluation.ReasonCode != "before_field_missing" { + t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok) + } + }) + + t.Run("status mismatch is contradicted", func(t *testing.T) { + evaluation, ok := EvaluateCapabilityPostcondition("qm.shutdown", nil, map[PostconditionField]string{FieldVMStatus: "running"}, time.Now()) + if !ok || !evaluation.Conclusive || evaluation.Matched { + t.Fatalf("evaluation = %#v, ok=%v", evaluation, ok) + } + }) +} + // TestLookupCapabilityPostconditionUnknown verifies the false return path so // callers can branch into VerificationUnknown rather than seeing a zero-value // entry that looks valid. diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go index bbf129f85..a01b76cbe 100644 --- a/internal/api/contract_test.go +++ b/internal/api/contract_test.go @@ -19995,7 +19995,8 @@ func TestContract_ProxmoxLifecycleActionsResolveNodeCommandAgentAndVerifyState(t "Trusted: true", "func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(", "proxmoxGuestStatusCommand(kind, vmid)", - "parseProxmoxGuestStatus(lastOutput) == expected", + "agentexec.EvaluateCapabilityPostcondition", + "func (e proxmoxGuestActionExecutor) observeProxmoxGuestPostcondition(", } { if !strings.Contains(src, snippet) { t.Fatalf("proxmox lifecycle executor must pin command-agent/trusted verification snippet %q", snippet) @@ -20014,7 +20015,7 @@ func TestContract_ProxmoxLifecycleActionsResolveNodeCommandAgentAndVerifyState(t for _, snippet := range []string{ "newRoutedActionExecutor(", "newDockerContainerActionExecutor(r.resourceHandlers, r.agentExecServer)", - "newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer)", + "newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer, newProxmoxGuestMonitoringObserver(r.resolveMonitorForOrg))", "newHostStorageCleanupActionExecutor(r.resourceHandlers, r.agentExecServer)", "newHostUpdateActionExecutor(r.resourceHandlers, r.agentExecServer)", } { diff --git a/internal/api/proxmox_guest_action_executor.go b/internal/api/proxmox_guest_action_executor.go index 13a8093f2..362ed008c 100644 --- a/internal/api/proxmox_guest_action_executor.go +++ b/internal/api/proxmox_guest_action_executor.go @@ -27,13 +27,36 @@ const ( type proxmoxGuestActionExecutor struct { resources *ResourceHandlers agents actionAgentCommander + observer proxmoxGuestPostconditionObserver } -func newProxmoxGuestActionExecutor(resources *ResourceHandlers, agents actionAgentCommander) ActionExecutor { +type proxmoxGuestLifecycleSnapshot struct { + Instance string + Node string + VMID int + Kind proxmoxGuestKind + Status string + Uptime uint64 + ObservedAt time.Time +} + +type proxmoxGuestPostconditionObservation struct { + ObserverID string + TrustDomain string + Method string + Snapshot proxmoxGuestLifecycleSnapshot + ReceivedAt time.Time +} + +type proxmoxGuestPostconditionObserver interface { + ObserveProxmoxGuest(context.Context, string, string, string, int, proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error) +} + +func newProxmoxGuestActionExecutor(resources *ResourceHandlers, agents actionAgentCommander, observer proxmoxGuestPostconditionObserver) ActionExecutor { if resources == nil || agents == nil { return nil } - return proxmoxGuestActionExecutor{resources: resources, agents: agents} + return proxmoxGuestActionExecutor{resources: resources, agents: agents, observer: observer} } func (e proxmoxGuestActionExecutor) ActionHandlerNames() []string { @@ -63,8 +86,15 @@ func (e proxmoxGuestActionExecutor) ExecuteAction(ctx context.Context, record un if err != nil { return nil, err } + var independentBefore *proxmoxGuestPostconditionObservation + if e.observer != nil { + if observation, observeErr := e.observer.ObserveProxmoxGuest(ctx, record.Request.ResourceID, resource.Proxmox.Instance, resource.Proxmox.NodeName, vmid, kind); observeErr == nil && validProxmoxGuestObservation(resource, kind, observation) { + independentBefore = &observation + } + } command := proxmoxGuestLifecycleCommand(kind, operation, vmid) + actionStartedAt := time.Now().UTC() result, err := e.agents.ExecuteCommand(ctx, agentID, agentexec.ExecuteCommandPayload{ RequestID: attempt.ID, Command: command, @@ -78,22 +108,13 @@ func (e proxmoxGuestActionExecutor) ExecuteAction(ctx context.Context, record un } output := redactActionOutput(commandOutput(result)) - execution := &unified.ExecutionResult{ - Success: result.ExitCode == 0, - Output: output, - } if result.ExitCode != 0 { - execution.ErrorMessage = strings.TrimSpace(firstNonEmpty(result.Error, output, fmt.Sprintf("proxmox guest %s exited with status %d", operation, result.ExitCode))) - return execution, nil + return proxmoxGuestExecutionResult(record.ID, record.Request.ResourceID, agentID, kind, operation, result.ExitCode, output, result.Error, nil, independentBefore, nil, agentexec.PostconditionEvaluation{}, actionStartedAt) } - verification := e.verifyProxmoxGuestState(ctx, agentID, record.ID, kind, vmid, operation) - execution.Verification = verification - if verification != nil && verification.Ran && !verification.Success { - execution.Success = false - execution.ErrorMessage = "proxmox guest lifecycle action completed but verification did not confirm the expected state" - } - return execution, nil + agentVerification := e.verifyProxmoxGuestState(ctx, agentID, record.ID, kind, vmid, operation, actionStartedAt) + independentAfter, independentEvaluation := e.observeProxmoxGuestPostcondition(ctx, record.Request.ResourceID, resource, kind, operation, independentBefore, actionStartedAt) + return proxmoxGuestExecutionResult(record.ID, record.Request.ResourceID, agentID, kind, operation, result.ExitCode, output, result.Error, agentVerification, independentBefore, independentAfter, independentEvaluation, actionStartedAt) } func (e proxmoxGuestActionExecutor) CheckActionAvailable(ctx context.Context, req unified.ActionRequest, resource unified.Resource) unified.ResourceActionReadiness { @@ -300,14 +321,18 @@ func proxmoxGuestLifecycleTimeout(operation string) int { } } -func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, agentID, actionID string, kind proxmoxGuestKind, vmid int, operation string) *unified.ActionVerificationResult { - expected := proxmoxExpectedGuestStatus(operation) - if expected == "" { - return &unified.ActionVerificationResult{Ran: false} +func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, agentID, actionID string, kind proxmoxGuestKind, vmid int, operation string, actionStartedAt time.Time) *unified.ActionVerificationResult { + capability := proxmoxPostconditionCapability(kind, operation) + if _, ok := agentexec.LookupCapabilityPostcondition(capability); !ok { + return &unified.ActionVerificationResult{Ran: false, Note: "No registered postcondition is available for this Proxmox action."} + } + if strings.EqualFold(strings.TrimSpace(operation), "reboot") { + return &unified.ActionVerificationResult{Ran: false, Note: "The executing agent's status-only read cannot prove that the guest restarted."} } command := proxmoxGuestStatusCommand(kind, vmid) var lastOutput string + var lastEvaluation agentexec.PostconditionEvaluation for attempt := 0; attempt < 5; attempt++ { if attempt > 0 { timer := time.NewTimer(1 * time.Second) @@ -331,7 +356,11 @@ func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, return &unified.ActionVerificationResult{Ran: false} } lastOutput = redactActionOutput(commandOutput(result)) - if result.ExitCode == 0 && parseProxmoxGuestStatus(lastOutput) == expected { + if result.ExitCode != 0 { + continue + } + lastEvaluation, _ = agentexec.EvaluateCapabilityPostcondition(capability, nil, proxmoxGuestPostconditionValues(kind, parseProxmoxGuestStatus(lastOutput), 0), actionStartedAt) + if lastEvaluation.Conclusive && lastEvaluation.Matched { return &unified.ActionVerificationResult{ Ran: true, Command: command, @@ -348,19 +377,78 @@ func (e proxmoxGuestActionExecutor) verifyProxmoxGuestState(ctx context.Context, Output: lastOutput, Success: false, RanAt: time.Now().UTC(), - Note: "expected status " + expected, + Note: firstNonEmpty(lastEvaluation.ReasonCode, "postcondition was not confirmed"), } } -func proxmoxExpectedGuestStatus(operation string) string { - switch strings.TrimSpace(operation) { - case "start", "reboot": - return "running" - case "shutdown", "stop": - return "stopped" - default: - return "" +func (e proxmoxGuestActionExecutor) observeProxmoxGuestPostcondition(ctx context.Context, resourceID string, resource unified.Resource, kind proxmoxGuestKind, operation string, before *proxmoxGuestPostconditionObservation, actionStartedAt time.Time) (*proxmoxGuestPostconditionObservation, agentexec.PostconditionEvaluation) { + if e.observer == nil || resource.Proxmox == nil { + return nil, agentexec.PostconditionEvaluation{} } + capability := proxmoxPostconditionCapability(kind, operation) + postcondition, ok := agentexec.LookupCapabilityPostcondition(capability) + if !ok { + return nil, agentexec.PostconditionEvaluation{ReasonCode: "postcondition_unregistered"} + } + window := postcondition.Window + if window <= 0 { + window = 2 * time.Minute + } + verifyCtx, cancel := context.WithTimeout(ctx, window) + defer cancel() + + var beforeValues map[agentexec.PostconditionField]string + if before != nil { + beforeValues = proxmoxGuestPostconditionValues(kind, before.Snapshot.Status, before.Snapshot.Uptime) + } + var last *proxmoxGuestPostconditionObservation + lastEvaluation := agentexec.PostconditionEvaluation{ReasonCode: "independent_observation_unavailable"} + for { + observation, err := e.observer.ObserveProxmoxGuest(verifyCtx, resourceID, resource.Proxmox.Instance, resource.Proxmox.NodeName, resource.Proxmox.VMID, kind) + if err == nil && validProxmoxGuestObservation(resource, kind, observation) { + last = &observation + lastEvaluation, _ = agentexec.EvaluateCapabilityPostcondition(capability, beforeValues, proxmoxGuestPostconditionValues(kind, observation.Snapshot.Status, observation.Snapshot.Uptime), actionStartedAt) + if lastEvaluation.Conclusive && lastEvaluation.Matched { + return last, lastEvaluation + } + } + + timer := time.NewTimer(time.Second) + select { + case <-verifyCtx.Done(): + timer.Stop() + return last, lastEvaluation + case <-timer.C: + } + } +} + +func proxmoxPostconditionCapability(kind proxmoxGuestKind, operation string) string { + tool := "qm" + if kind == proxmoxGuestCT { + tool = "pct" + } + return tool + "." + strings.ToLower(strings.TrimSpace(operation)) +} + +func proxmoxGuestPostconditionValues(kind proxmoxGuestKind, status string, uptime uint64) map[agentexec.PostconditionField]string { + statusField := agentexec.FieldVMStatus + if kind == proxmoxGuestCT { + statusField = agentexec.FieldContainerStatus + } + return map[agentexec.PostconditionField]string{ + statusField: strings.ToLower(strings.TrimSpace(status)), + agentexec.FieldGuestUptime: strconv.FormatUint(uptime, 10), + } +} + +func validProxmoxGuestObservation(resource unified.Resource, kind proxmoxGuestKind, observation proxmoxGuestPostconditionObservation) bool { + if resource.Proxmox == nil || observation.Snapshot.VMID != resource.Proxmox.VMID || observation.Snapshot.Kind != kind { + return false + } + return strings.EqualFold(strings.TrimSpace(observation.Snapshot.Instance), strings.TrimSpace(resource.Proxmox.Instance)) && + strings.EqualFold(strings.TrimSpace(observation.Snapshot.Node), strings.TrimSpace(resource.Proxmox.NodeName)) && + strings.TrimSpace(observation.Snapshot.Status) != "" && !observation.Snapshot.ObservedAt.IsZero() && !observation.ReceivedAt.IsZero() } func parseProxmoxGuestStatus(output string) string { diff --git a/internal/api/proxmox_guest_action_executor_test.go b/internal/api/proxmox_guest_action_executor_test.go index 15a668695..9c8a3e88c 100644 --- a/internal/api/proxmox_guest_action_executor_test.go +++ b/internal/api/proxmox_guest_action_executor_test.go @@ -29,7 +29,7 @@ func TestProxmoxGuestActionExecutorDispatchesVMShutdownAndVerification(t *testin {RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"}, {RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"}, }} - executor := newProxmoxGuestActionExecutor(h, agents) + executor := newProxmoxGuestActionExecutor(h, agents, nil) result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown")) if err != nil { @@ -73,7 +73,7 @@ func TestProxmoxGuestActionExecutorDispatchesLXCStartAndVerification(t *testing. {RequestID: "act_ct", Success: true, ExitCode: 0, Stdout: "start requested"}, {RequestID: "act_ct-verify-1", Success: true, ExitCode: 0, Stdout: "status: running"}, }} - executor := newProxmoxGuestActionExecutor(h, agents) + executor := newProxmoxGuestActionExecutor(h, agents, nil) result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_ct"), proxmoxGuestActionRecord("act_ct", "system-container:101", "start")) if err != nil { @@ -110,7 +110,7 @@ func TestProxmoxGuestActionExecutorResolvesCommandAgentByNodeHostname(t *testing }, agentByHost: map[string]string{"delly": "command-agent-1"}, } - executor := newProxmoxGuestActionExecutor(h, agents).(proxmoxGuestActionExecutor) + executor := newProxmoxGuestActionExecutor(h, agents, nil).(proxmoxGuestActionExecutor) readiness := executor.CheckActionAvailable(context.Background(), unified.ActionRequest{ RequestID: "req-availability", @@ -137,7 +137,7 @@ func TestProxmoxGuestActionExecutorResolvesCommandAgentByNodeHostname(t *testing } } -func TestProxmoxGuestActionExecutorVerificationFailureFailsAction(t *testing.T) { +func TestProxmoxGuestActionExecutorVerificationContradictionDoesNotRewriteExecution(t *testing.T) { now := time.Now().UTC() h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) h.SetStateProvider(resourceUnifiedSeedProvider{ @@ -154,17 +154,163 @@ func TestProxmoxGuestActionExecutorVerificationFailureFailsAction(t *testing.T) {RequestID: "act_vm-verify-4", Success: true, ExitCode: 0, Stdout: "status: running"}, {RequestID: "act_vm-verify-5", Success: true, ExitCode: 0, Stdout: "status: running"}, }} - executor := newProxmoxGuestActionExecutor(h, agents) + executor := newProxmoxGuestActionExecutor(h, agents, nil) result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown")) if err != nil { t.Fatalf("ExecuteAction: %v", err) } - if result == nil || result.Success || result.Verification == nil || result.Verification.Success { - t.Fatalf("result = %#v, want failed verification to fail action", result) + if result == nil || !result.Success || result.Verification == nil || result.Verification.Success { + t.Fatalf("result = %#v, want succeeded execution with contradicted verification", result) } - if !strings.Contains(result.ErrorMessage, "verification did not confirm") { - t.Fatalf("error = %q, want verification failure", result.ErrorMessage) + if result.ActionResultV2 == nil || result.ActionResultV2.Execution.Status != unified.ActionExecutionSucceeded || result.ActionResultV2.Verification.Status != unified.ActionVerificationContradicted || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceAgentAttested { + t.Fatalf("canonical truth = %#v, want independent execution and verification axes", result.ActionResultV2) + } + if result.ErrorMessage != "" { + t.Fatalf("error = %q, want no execution error from verification contradiction", result.ErrorMessage) + } +} + +func TestProxmoxGuestActionExecutorUsesIndependentControlPlaneVerification(t *testing.T) { + now := time.Now().UTC() + resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now) + resource.Proxmox.Uptime = 3600 + h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h.SetStateProvider(resourceUnifiedSeedProvider{ + snapshot: models.StateSnapshot{LastUpdate: now}, + resources: []unified.Resource{resource}, + }) + agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{ + {RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"}, + {RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"}, + }} + observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{ + proxmoxGuestActionObservation(now.Add(-time.Second), "running", 3600, "proxmox-control-plane:default:homelab"), + proxmoxGuestActionObservation(now.Add(time.Second), "stopped", 0, "proxmox-control-plane:default:homelab"), + }} + executor := newProxmoxGuestActionExecutor(h, agents, observer) + + result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown")) + if err != nil { + t.Fatalf("ExecuteAction: %v", err) + } + if result == nil || result.ActionResultV2 == nil { + t.Fatalf("result = %#v, want canonical truth", result) + } + truth := result.ActionResultV2.Verification + if truth.Status != unified.ActionVerificationConfirmed || truth.EvidenceClass != unified.ActionEvidenceIndependent || len(truth.Evidence) != 1 { + t.Fatalf("verification truth = %#v, want independent confirmation", truth) + } + if truth.Evidence[0].ObserverKind != "proxmox_control_plane" || truth.Evidence[0].ObserverTrustDomain == truth.Evidence[0].ExecutorTrustDomain { + t.Fatalf("independent evidence = %#v", truth.Evidence[0]) + } +} + +func TestProxmoxGuestActionExecutorRequiresUptimeResetToVerifyReboot(t *testing.T) { + now := time.Now().UTC() + resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now) + resource.Proxmox.Uptime = 7200 + h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}}) + agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "reboot requested"}}} + observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{ + proxmoxGuestActionObservation(now.Add(-time.Second), "running", 7200, "proxmox-control-plane:default:homelab"), + proxmoxGuestActionObservation(now.Add(time.Second), "running", 4, "proxmox-control-plane:default:homelab"), + }} + executor := newProxmoxGuestActionExecutor(h, agents, observer) + + result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "reboot")) + if err != nil { + t.Fatalf("ExecuteAction: %v", err) + } + if result == nil || result.ActionResultV2 == nil || result.ActionResultV2.Verification.Status != unified.ActionVerificationConfirmed || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceIndependent { + t.Fatalf("result = %#v, want independently verified reboot", result) + } + if len(agents.calls) != 1 { + t.Fatalf("agent calls = %d, want mutation only because status-only agent read cannot prove reboot", len(agents.calls)) + } +} + +func TestProxmoxGuestActionExecutorKeepsIndependentContradictionSeparateFromExecution(t *testing.T) { + now := time.Now().UTC() + resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now) + h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}}) + agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{{RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "reboot requested"}}} + observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{ + proxmoxGuestActionObservation(now.Add(-time.Second), "running", 7200, "proxmox-control-plane:default:homelab"), + proxmoxGuestActionObservation(now.Add(time.Second), "running", 7201, "proxmox-control-plane:default:homelab"), + }} + executor := newProxmoxGuestActionExecutor(h, agents, observer) + ctx, cancel := context.WithTimeout(actionDispatchTestContext(t, "act_vm"), 100*time.Millisecond) + defer cancel() + + result, err := executor.ExecuteAction(ctx, proxmoxGuestActionRecord("act_vm", "vm:160", "reboot")) + if err != nil { + t.Fatalf("ExecuteAction: %v", err) + } + if result == nil || result.ActionResultV2 == nil || !result.Success { + t.Fatalf("result = %#v, want successful command execution", result) + } + truth := result.ActionResultV2.Verification + if truth.Status != unified.ActionVerificationContradicted || truth.EvidenceClass != unified.ActionEvidenceIndependent { + t.Fatalf("verification truth = %#v, want independent contradiction", truth) + } +} + +func TestProxmoxGuestActionExecutorRejectsSameDomainIndependentEvidence(t *testing.T) { + now := time.Now().UTC() + resource := proxmoxGuestActionResource("vm:160", unified.ResourceTypeVM, "running", now) + h := newActionTestResourceHandlers(t, &config.Config{DataPath: t.TempDir()}) + h.SetStateProvider(resourceUnifiedSeedProvider{snapshot: models.StateSnapshot{LastUpdate: now}, resources: []unified.Resource{resource}}) + agents := &fakeDockerActionAgentCommander{results: []*agentexec.CommandResultPayload{ + {RequestID: "act_vm", Success: true, ExitCode: 0, Stdout: "shutdown requested"}, + {RequestID: "act_vm-verify-1", Success: true, ExitCode: 0, Stdout: "status: stopped"}, + }} + observer := &fakeProxmoxGuestPostconditionObserver{observations: []proxmoxGuestPostconditionObservation{ + proxmoxGuestActionObservation(now.Add(-time.Second), "running", 100, "agent:node-agent-1"), + proxmoxGuestActionObservation(now.Add(time.Second), "stopped", 0, "agent:node-agent-1"), + }} + executor := newProxmoxGuestActionExecutor(h, agents, observer) + + result, err := executor.ExecuteAction(actionDispatchTestContext(t, "act_vm"), proxmoxGuestActionRecord("act_vm", "vm:160", "shutdown")) + if err != nil { + t.Fatalf("ExecuteAction: %v", err) + } + if result == nil || result.ActionResultV2 == nil || result.ActionResultV2.Verification.Status != unified.ActionVerificationConfirmed || result.ActionResultV2.Verification.EvidenceClass != unified.ActionEvidenceAgentAttested { + t.Fatalf("result = %#v, want same-domain observation rejected in favor of agent-attested truth", result) + } +} + +type fakeProxmoxGuestPostconditionObserver struct { + observations []proxmoxGuestPostconditionObservation + next int +} + +func (o *fakeProxmoxGuestPostconditionObserver) ObserveProxmoxGuest(context.Context, string, string, string, int, proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error) { + if o.next >= len(o.observations) { + return proxmoxGuestPostconditionObservation{}, context.DeadlineExceeded + } + observation := o.observations[o.next] + o.next++ + return observation, nil +} + +func proxmoxGuestActionObservation(observedAt time.Time, status string, uptime uint64, trustDomain string) proxmoxGuestPostconditionObservation { + return proxmoxGuestPostconditionObservation{ + ObserverID: "proxmox-api:default:homelab", + TrustDomain: trustDomain, + Method: "proxmox_api_guest_status_current", + Snapshot: proxmoxGuestLifecycleSnapshot{ + Instance: "homelab", + Node: "delly", + VMID: 160, + Kind: proxmoxGuestVM, + Status: status, + Uptime: uptime, + ObservedAt: observedAt, + }, + ReceivedAt: observedAt, } } @@ -181,7 +327,7 @@ func TestHandlePlanActionRejectsDisconnectedProxmoxNodeCommandAgent(t *testing.T h, newProxmoxGuestActionExecutor(h, &fakeDockerActionAgentCommander{ connected: map[string]bool{"node-agent-1": false}, - }), + }, nil), )) rec := httptest.NewRecorder() @@ -228,7 +374,7 @@ func TestResourceResponsesFilterDisconnectedProxmoxLifecycleCapabilities(t *test h, newProxmoxGuestActionExecutor(h, &fakeDockerActionAgentCommander{ connected: map[string]bool{"node-agent-1": false}, - }), + }, nil), )) listRec := httptest.NewRecorder() diff --git a/internal/api/proxmox_guest_action_observer.go b/internal/api/proxmox_guest_action_observer.go new file mode 100644 index 000000000..5fec6d29b --- /dev/null +++ b/internal/api/proxmox_guest_action_observer.go @@ -0,0 +1,54 @@ +package api + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/monitoring" +) + +type proxmoxGuestMonitorResolver func(string) *monitoring.Monitor + +type proxmoxGuestMonitoringObserver struct { + resolveMonitor proxmoxGuestMonitorResolver +} + +func newProxmoxGuestMonitoringObserver(resolveMonitor proxmoxGuestMonitorResolver) proxmoxGuestPostconditionObserver { + if resolveMonitor == nil { + return nil + } + return proxmoxGuestMonitoringObserver{resolveMonitor: resolveMonitor} +} + +func (o proxmoxGuestMonitoringObserver) ObserveProxmoxGuest(ctx context.Context, resourceID, instance, node string, vmid int, kind proxmoxGuestKind) (proxmoxGuestPostconditionObservation, error) { + orgID := strings.TrimSpace(GetOrgID(ctx)) + if orgID == "" { + orgID = "default" + } + monitor := o.resolveMonitor(orgID) + if monitor == nil { + return proxmoxGuestPostconditionObservation{}, fmt.Errorf("monitor unavailable for organization %q", orgID) + } + observation, err := monitor.ObserveProxmoxGuest(ctx, instance, node, vmid, string(kind)) + receivedAt := time.Now().UTC() + if err != nil { + return proxmoxGuestPostconditionObservation{}, err + } + return proxmoxGuestPostconditionObservation{ + ObserverID: "proxmox-api:" + orgID + ":" + strings.TrimSpace(instance), + TrustDomain: "proxmox-control-plane:" + orgID + ":" + strings.TrimSpace(instance), + Method: "proxmox_api_guest_status_current", + Snapshot: proxmoxGuestLifecycleSnapshot{ + Instance: observation.Instance, + Node: observation.Node, + VMID: observation.VMID, + Kind: proxmoxGuestKind(observation.Kind), + Status: observation.Status, + Uptime: observation.Uptime, + ObservedAt: observation.ObservedAt, + }, + ReceivedAt: receivedAt, + }, nil +} diff --git a/internal/api/proxmox_guest_action_result.go b/internal/api/proxmox_guest_action_result.go new file mode 100644 index 000000000..d9861cc0c --- /dev/null +++ b/internal/api/proxmox_guest_action_result.go @@ -0,0 +1,127 @@ +package api + +import ( + "fmt" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/agentexec" + "github.com/rcourtman/pulse-go-rewrite/internal/operationreceipt" + unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func proxmoxGuestExecutionResult(actionID, resourceID, agentID string, kind proxmoxGuestKind, operation string, exitCode int, output, commandError string, agentVerification *unified.ActionVerificationResult, independentBefore, independentAfter *proxmoxGuestPostconditionObservation, independentEvaluation agentexec.PostconditionEvaluation, actionStartedAt time.Time) (*unified.ExecutionResult, error) { + executionSummary := fmt.Sprintf("Proxmox guest %s command exited with status %d.", strings.TrimSpace(operation), exitCode) + execution := unified.ActionExecutionTruth{Status: unified.ActionExecutionSucceeded, Summary: executionSummary} + verification := unified.ActionVerificationTruth{Status: unified.ActionVerificationInconclusive, EvidenceClass: unified.ActionEvidenceNone, ReasonCode: "agent_readback_inconclusive", Summary: "The executing agent did not return a conclusive postcondition read."} + legacy := &unified.ExecutionResult{Output: strings.TrimSpace(output)} + if exitCode != 0 { + execution.Status = unified.ActionExecutionFailed + execution.ReasonCode = "proxmox_command_failed" + execution.Summary = strings.TrimSpace(firstNonEmpty(commandError, output, executionSummary)) + verification = unified.ActionVerificationTruth{Status: unified.ActionVerificationNotAttempted, EvidenceClass: unified.ActionEvidenceNone} + legacy.ErrorMessage = execution.Summary + } else if agentVerification != nil && agentVerification.Ran { + agentSummary := strings.TrimSpace(firstNonEmpty(agentVerification.Note, agentVerification.Output, "The executing agent read the Proxmox guest postcondition.")) + evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{ + Version: unified.ActionEvidenceVersion, + ID: actionID + "-agent-proxmox-readback", + ObserverID: agentID, + ObserverKind: "unified_agent", + ObserverTrustDomain: "agent:" + agentID, + ExecutorTrustDomain: "agent:" + agentID, + Method: "server_owned_proxmox_cli_status", + SubjectID: resourceID, + ObservedAt: agentVerification.RanAt.UTC(), + ReceivedAt: time.Now().UTC(), + Summary: agentSummary, + }) + if err != nil { + return nil, fmt.Errorf("normalize Proxmox agent readback evidence: %w", err) + } + status := unified.ActionVerificationContradicted + reason := "postcondition_contradicted" + if agentVerification.Success { + status = unified.ActionVerificationConfirmed + reason = "" + } + verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceAgentAttested, ReasonCode: reason, Summary: agentSummary, Evidence: []unified.ActionEvidence{evidence}} + } + + if exitCode == 0 && usableIndependentProxmoxObservation(agentID, actionStartedAt, independentAfter) { + beforeSnapshot := (*proxmoxGuestLifecycleSnapshot)(nil) + if independentBefore != nil && usableIndependentProxmoxBeforeObservation(actionStartedAt, independentBefore) { + beforeCopy := independentBefore.Snapshot + beforeSnapshot = &beforeCopy + } + observationDigest, err := operationreceipt.DigestCanonicalJSON(struct { + ActionID string `json:"action_id"` + SubjectID string `json:"subject_id"` + Before *proxmoxGuestLifecycleSnapshot `json:"before,omitempty"` + After proxmoxGuestLifecycleSnapshot `json:"after"` + }{ActionID: actionID, SubjectID: resourceID, Before: beforeSnapshot, After: independentAfter.Snapshot}) + if err != nil { + return nil, fmt.Errorf("digest Proxmox control-plane observation: %w", err) + } + independentSummary := fmt.Sprintf("Proxmox API observation: status=%s; uptime=%ds", independentAfter.Snapshot.Status, independentAfter.Snapshot.Uptime) + evidence, err := unified.NormalizeActionEvidence(unified.ActionEvidence{ + Version: unified.ActionEvidenceVersion, + ID: actionID + "-proxmox-api-observation", + ObserverID: independentAfter.ObserverID, + ObserverKind: "proxmox_control_plane", + ObserverTrustDomain: independentAfter.TrustDomain, + ExecutorTrustDomain: "agent:" + agentID, + Method: independentAfter.Method, + SubjectID: resourceID, + ObservedAt: independentAfter.Snapshot.ObservedAt.UTC(), + ReceivedAt: independentAfter.ReceivedAt.UTC(), + Summary: independentSummary, + Refs: []unified.ActionEvidenceRef{{ID: actionID, Kind: "proxmox_before_after", Digest: observationDigest}}, + }) + if err != nil { + return nil, fmt.Errorf("normalize Proxmox control-plane evidence: %w", err) + } + status := unified.ActionVerificationInconclusive + reason := strings.TrimSpace(firstNonEmpty(independentEvaluation.ReasonCode, "postcondition_inconclusive")) + if independentEvaluation.Conclusive { + status = unified.ActionVerificationContradicted + reason = "postcondition_contradicted" + if independentEvaluation.Matched { + status = unified.ActionVerificationConfirmed + reason = "" + } + } + verification = unified.ActionVerificationTruth{Status: status, EvidenceClass: unified.ActionEvidenceIndependent, ReasonCode: reason, Summary: independentSummary, Evidence: []unified.ActionEvidence{evidence}} + } + + canonical := unified.ActionResultV2{ + Version: unified.ActionResultV2Version, + Execution: execution, + Verification: verification, + Compensation: unified.ActionCompensationTruth{Support: unified.ActionCompensationUnavailable, Status: unified.ActionCompensationNotAvailable, Summary: "Proxmox guest lifecycle actions do not provide automatic rollback."}, + } + projected, _, err := unified.ApplyActionResultV2(legacy, canonical) + if err != nil { + return nil, fmt.Errorf("apply canonical Proxmox lifecycle action truth: %w", err) + } + projected.Verification = unified.LegacyActionVerificationFromV2(*projected.ActionResultV2) + return projected, nil +} + +func usableIndependentProxmoxObservation(agentID string, actionStartedAt time.Time, observation *proxmoxGuestPostconditionObservation) bool { + if observation == nil || strings.TrimSpace(observation.ObserverID) == "" || strings.TrimSpace(observation.TrustDomain) == "" || strings.TrimSpace(observation.Method) == "" || strings.TrimSpace(observation.TrustDomain) == "agent:"+strings.TrimSpace(agentID) { + return false + } + observedAt := observation.Snapshot.ObservedAt.UTC() + receivedAt := observation.ReceivedAt.UTC() + return !actionStartedAt.IsZero() && !observedAt.Before(actionStartedAt.UTC()) && !observedAt.After(receivedAt.Add(5*time.Minute)) && !observedAt.Before(receivedAt.Add(-15*time.Minute)) +} + +func usableIndependentProxmoxBeforeObservation(actionStartedAt time.Time, observation *proxmoxGuestPostconditionObservation) bool { + if observation == nil || actionStartedAt.IsZero() || observation.Snapshot.ObservedAt.IsZero() || observation.ReceivedAt.IsZero() { + return false + } + observedAt := observation.Snapshot.ObservedAt.UTC() + receivedAt := observation.ReceivedAt.UTC() + return !observedAt.After(actionStartedAt.UTC()) && !observedAt.After(receivedAt.Add(5*time.Minute)) && !observedAt.Before(receivedAt.Add(-15*time.Minute)) +} diff --git a/internal/api/router.go b/internal/api/router.go index 5452551da..6f955ede3 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -639,7 +639,7 @@ func (r *Router) setupRoutes() { r.resourceHandlers.SetActionExecutor(newRoutedActionExecutor( r.resourceHandlers, newDockerContainerActionExecutor(r.resourceHandlers, r.agentExecServer), - newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer), + newProxmoxGuestActionExecutor(r.resourceHandlers, r.agentExecServer, newProxmoxGuestMonitoringObserver(r.resolveMonitorForOrg)), newHostStorageCleanupActionExecutor(r.resourceHandlers, r.agentExecServer), newHostUpdateActionExecutor(r.resourceHandlers, r.agentExecServer), )) diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go index 1d3650f27..5d0df0e14 100644 --- a/internal/monitoring/canonical_guardrails_test.go +++ b/internal/monitoring/canonical_guardrails_test.go @@ -123,6 +123,24 @@ func TestPVETagStyleRefreshStaysPerInstance(t *testing.T) { } } +func TestProxmoxActionObserverUsesDirectControlPlaneClient(t *testing.T) { + data, err := os.ReadFile("proxmox_action_observer.go") + if err != nil { + t.Fatalf("read proxmox_action_observer.go: %v", err) + } + source := string(data) + for _, snippet := range []string{"m.getPVEClient(instance)", "client.GetVMStatus(ctx, node, vmid)", "client.GetContainerStatus(ctx, node, vmid)"} { + if !strings.Contains(source, snippet) { + t.Fatalf("Proxmox action observer must use direct control-plane read %q", snippet) + } + } + for _, forbidden := range []string{"m.GetState(", "m.ReadState(", "m.resourceStore", "m.state."} { + if strings.Contains(source, forbidden) { + t.Fatalf("Proxmox action observer must not derive independent evidence through cached monitoring state %q", forbidden) + } + } +} + func TestBroadcastResourceDiskIOUsesUnifiedResourceMetrics(t *testing.T) { hasDiskIO, readRate, writeRate := monitorDiskIOMetricInput(&unifiedresources.ResourceMetrics{ DiskRead: &unifiedresources.MetricValue{Value: 4096.4, Unit: "bytes/s", Source: unifiedresources.SourceAgent}, diff --git a/internal/monitoring/proxmox_action_observer.go b/internal/monitoring/proxmox_action_observer.go new file mode 100644 index 000000000..ef63e289c --- /dev/null +++ b/internal/monitoring/proxmox_action_observer.go @@ -0,0 +1,82 @@ +package monitoring + +import ( + "context" + "fmt" + "strings" + "time" +) + +// ProxmoxGuestObservation is a direct server-side Proxmox API read used by +// the governed action layer as an observer distinct from the node agent that +// executes guest lifecycle commands. +type ProxmoxGuestObservation struct { + Instance string + Node string + VMID int + Kind string + Status string + Uptime uint64 + ObservedAt time.Time +} + +// ObserveProxmoxGuest reads current guest state through the configured +// Proxmox control-plane client. It does not use cached monitor state or the +// mutating node agent, so callers may classify a valid observation under the +// Proxmox control-plane trust domain. +func (m *Monitor) ObserveProxmoxGuest(ctx context.Context, instance, node string, vmid int, kind string) (ProxmoxGuestObservation, error) { + if m == nil { + return ProxmoxGuestObservation{}, fmt.Errorf("monitor unavailable") + } + instance = strings.TrimSpace(instance) + node = strings.TrimSpace(node) + kind = strings.ToLower(strings.TrimSpace(kind)) + if instance == "" || node == "" || vmid <= 0 { + return ProxmoxGuestObservation{}, fmt.Errorf("proxmox guest observer requires instance, node, and vmid") + } + client, ok := m.getPVEClient(instance) + if !ok || client == nil { + return ProxmoxGuestObservation{}, fmt.Errorf("proxmox client for instance %q unavailable", instance) + } + + observation := ProxmoxGuestObservation{ + Instance: instance, + Node: node, + VMID: vmid, + Kind: kind, + } + switch kind { + case "vm", "qemu": + status, err := client.GetVMStatus(ctx, node, vmid) + if err != nil { + return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox VM status: %w", err) + } + if status == nil { + return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox VM status: empty response") + } + observation.Kind = "vm" + observation.Status = strings.ToLower(strings.TrimSpace(status.Status)) + observation.Uptime = status.Uptime + case "ct", "lxc", "container": + status, err := client.GetContainerStatus(ctx, node, vmid) + if err != nil { + return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: %w", err) + } + if status == nil { + return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: empty response") + } + if observedVMID := int(status.VMID); observedVMID != 0 && observedVMID != vmid { + return ProxmoxGuestObservation{}, fmt.Errorf("observe Proxmox CT status: vmid mismatch") + } + observation.Kind = "ct" + observation.Status = strings.ToLower(strings.TrimSpace(status.Status)) + observation.Uptime = status.Uptime + default: + return ProxmoxGuestObservation{}, fmt.Errorf("unsupported Proxmox guest kind %q", kind) + } + if observation.Status == "" { + return ProxmoxGuestObservation{}, fmt.Errorf("Proxmox guest observation omitted status") + } + observation.ObservedAt = time.Now().UTC() + return observation, nil +} diff --git a/internal/monitoring/proxmox_action_observer_test.go b/internal/monitoring/proxmox_action_observer_test.go new file mode 100644 index 000000000..4806e1450 --- /dev/null +++ b/internal/monitoring/proxmox_action_observer_test.go @@ -0,0 +1,62 @@ +package monitoring + +import ( + "context" + "testing" + + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +func TestObserveProxmoxGuestReadsDirectControlPlaneState(t *testing.T) { + client := &actionObserverPVEClient{ + stubPVEClient: &stubPVEClient{}, + vmStatus: &proxmox.VMStatus{Status: "running", Uptime: 42}, + ctStatus: &proxmox.Container{VMID: proxmox.FlexInt(101), Status: "stopped", Uptime: 9}, + } + monitor := &Monitor{pveClients: map[string]PVEClientInterface{"homelab": client}} + + vm, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 160, "vm") + if err != nil { + t.Fatalf("ObserveProxmoxGuest VM: %v", err) + } + if vm.Kind != "vm" || vm.Status != "running" || vm.Uptime != 42 || vm.Instance != "homelab" || vm.Node != "node-a" || vm.VMID != 160 || vm.ObservedAt.IsZero() { + t.Fatalf("VM observation = %#v", vm) + } + + ct, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 101, "lxc") + if err != nil { + t.Fatalf("ObserveProxmoxGuest CT: %v", err) + } + if ct.Kind != "ct" || ct.Status != "stopped" || ct.Uptime != 9 || ct.VMID != 101 { + t.Fatalf("CT observation = %#v", ct) + } +} + +func TestObserveProxmoxGuestFailsClosedForMissingClientAndIdentityMismatch(t *testing.T) { + monitor := &Monitor{pveClients: map[string]PVEClientInterface{}} + if _, err := monitor.ObserveProxmoxGuest(context.Background(), "missing", "node-a", 160, "vm"); err == nil { + t.Fatal("missing Proxmox client unexpectedly produced an observation") + } + + monitor.pveClients["homelab"] = &actionObserverPVEClient{ + stubPVEClient: &stubPVEClient{}, + ctStatus: &proxmox.Container{VMID: proxmox.FlexInt(999), Status: "running"}, + } + if _, err := monitor.ObserveProxmoxGuest(context.Background(), "homelab", "node-a", 101, "ct"); err == nil { + t.Fatal("mismatched Proxmox CT identity unexpectedly produced an observation") + } +} + +type actionObserverPVEClient struct { + *stubPVEClient + vmStatus *proxmox.VMStatus + ctStatus *proxmox.Container +} + +func (c *actionObserverPVEClient) GetVMStatus(context.Context, string, int) (*proxmox.VMStatus, error) { + return c.vmStatus, nil +} + +func (c *actionObserverPVEClient) GetContainerStatus(context.Context, string, int) (*proxmox.Container, error) { + return c.ctStatus, nil +} diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index c69a15b6a..7a56ac6bf 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -2970,8 +2970,8 @@ class SubsystemLookupTest(unittest.TestCase): { "heading": "## Shared Boundaries", "path": "internal/api/access_control_handlers.go", - "line": 1212, - "heading_line": 142, + "line": 1214, + "heading_line": 144, } ], )