mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
Add native PVE action qualification harness
This commit is contained in:
rcourtman
@@ -1320,6 +1320,32 @@ requires recovery inspection. Package and containment-probe units never gain
|
||||
`AF_NETLINK` or the handoff relaxation. This provider-aware path remains
|
||||
unqualified until disposable-PVE proof establishes the native scope handoff,
|
||||
bridged networking, postcondition, and cancellation behavior.
|
||||
`scripts/run-native-pve-action-qualification.sh` is the fail-closed entrypoint
|
||||
for the live provider slice. It requires a clean exact source revision, pinned
|
||||
SSH host identity, a dedicated PVE node with no installed or loaded Pulse
|
||||
runner, distinct VM and container IDs, and a confirmation bound to the machine
|
||||
ID, PVE node, source revision, and both guest IDs. The Linux test then rejects
|
||||
templates, HA-managed guests, non-local owners, running initial state, missing
|
||||
`pulse-disposable` tags, absent bridged interfaces, and config-identity drift.
|
||||
It runs the candidate runner as the real hidden launcher behind a temporary
|
||||
runtime-only anchor and executes the test inside a root-owned systemd
|
||||
supervisor. The supervisor's independent `ExecStopPost` cleanup revalidates the
|
||||
durable original identities, stops exact qualification action units and any
|
||||
still-running original guests, and removes only the matching anchor invocation
|
||||
after normal exit, timeout, or process termination. The harness revalidates the
|
||||
exact PVE config digest before every mutation and cleanup and records artifact
|
||||
hashes/build metadata, bound production results, per-operation transient-unit
|
||||
and sideband disappearance, VMID-specific native PVE cgroups, per-NIC live
|
||||
bridge attachment paths, final states, and any emergency cleanup in
|
||||
root-private structured receipts. Emergency cleanup, missing cleanup evidence,
|
||||
or identity ambiguity fails the run; an identity mismatch never authorizes a
|
||||
cleanup mutation against a replacement guest. The reboot observation proves
|
||||
only that the fixed provider command returned and the production running
|
||||
postcondition passed, not an independent in-guest reboot. A checked-in harness,
|
||||
skipped local test, or unattested receipt is not provider qualification.
|
||||
Cancellation after provider handoff, hook inheritance, WSS/API authorization,
|
||||
durable reconnect/replay, and exact release-artifact reproduction remain
|
||||
separate required evidence and must not be inferred from this lifecycle pass.
|
||||
Docker/Podman lifecycle remains on the already-connected daemon API operator;
|
||||
the action runner does not fall back to a killable external runtime CLI.
|
||||
Host storage-pressure cleanup is a second closed agent operation. The report
|
||||
|
||||
@@ -1388,6 +1388,7 @@
|
||||
"scripts/install.sh",
|
||||
"scripts/installtests/windows_agent_lifecycle.ps1",
|
||||
"scripts/installtests/windowslifecycleserver/main.go",
|
||||
"scripts/run-native-pve-action-qualification.sh",
|
||||
"tests/integration/tests/84-docker-restart-real-lab-artifact.spec.ts"
|
||||
],
|
||||
"verification": {
|
||||
@@ -1457,6 +1458,20 @@
|
||||
"internal/agenthelper/update_activation_test.go"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "native-pve-action-qualification",
|
||||
"label": "native disposable-PVE typed action qualification proof",
|
||||
"match_prefixes": [],
|
||||
"match_files": [
|
||||
"scripts/run-native-pve-action-qualification.sh"
|
||||
],
|
||||
"allow_same_subsystem_tests": false,
|
||||
"test_prefixes": [],
|
||||
"exact_files": [
|
||||
"internal/hostagent/proxmox_guest_lifecycle_pve_linux_test.go",
|
||||
"scripts/installtests/native_pve_action_qualification_test.go"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "patrol-autonomy-colima-real-lab",
|
||||
"label": "RG-06 server-owned limited-unattended-autonomy Colima proof",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,164 @@
|
||||
package installtests
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNativePVEQualificationEvidenceVerifierFailsClosedWithPythonOptimize(t *testing.T) {
|
||||
scriptBytes, err := os.ReadFile(repoFile("scripts", "run-native-pve-action-qualification.sh"))
|
||||
if err != nil {
|
||||
t.Fatalf("read native PVE qualification wrapper: %v", err)
|
||||
}
|
||||
startMarker := "import hashlib\n"
|
||||
endMarker := "\nPY\nthen\n"
|
||||
start := strings.Index(string(scriptBytes), startMarker)
|
||||
end := strings.Index(string(scriptBytes), endMarker)
|
||||
if start < 0 || end <= start {
|
||||
t.Fatal("native PVE evidence verifier heredoc was not found")
|
||||
}
|
||||
verifier := string(scriptBytes)[start:end]
|
||||
if strings.Contains(verifier, "assert ") {
|
||||
t.Fatal("native PVE evidence verifier must not depend on optimizable Python assertions")
|
||||
}
|
||||
|
||||
paths, args := writeNativePVEEvidenceFixtures(t)
|
||||
if output, err := runNativePVEEvidenceVerifier(verifier, args); err != nil {
|
||||
t.Fatalf("valid evidence was rejected with PYTHONOPTIMIZE=1: %v: %s", err, output)
|
||||
}
|
||||
|
||||
receiptBytes, err := os.ReadFile(paths[2])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var receipt map[string]any
|
||||
if err := json.Unmarshal(receiptBytes, &receipt); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
receipt["guests"].([]any)[0].(map[string]any)["operations"].([]any)[0].(map[string]any)["result"].(map[string]any)["execution_phase"] = "mutate"
|
||||
writeNativePVEEvidenceJSON(t, paths[2], receipt)
|
||||
if output, err := runNativePVEEvidenceVerifier(verifier, args); err == nil {
|
||||
t.Fatalf("malformed production result was accepted with PYTHONOPTIMIZE=1: %s", output)
|
||||
}
|
||||
}
|
||||
|
||||
func writeNativePVEEvidenceFixtures(t *testing.T) ([3]string, []string) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
paths := [3]string{filepath.Join(dir, "manifest.json"), filepath.Join(dir, "cleanup.json"), filepath.Join(dir, "receipt.json")}
|
||||
sourceCommit := strings.Repeat("a", 40)
|
||||
machineID := strings.Repeat("b", 32)
|
||||
node := "pve-a"
|
||||
clusterID := "standalone:pve-a"
|
||||
runnerHash := strings.Repeat("c", 64)
|
||||
testHash := strings.Repeat("d", 64)
|
||||
supervisorUnit := "pulse-pve-qualification-aaaaaaaa-101-102.service"
|
||||
supervisorInvocationID := strings.Repeat("e", 32)
|
||||
runnerInvocationID := strings.Repeat("f", 32)
|
||||
guests := []any{
|
||||
map[string]any{"kind": "vm", "vmid": 101, "node": node, "config_digest": strings.Repeat("1", 40), "bridges": []string{"vmbr0"}, "networks": map[string]string{"net0": "vmbr0"}},
|
||||
map[string]any{"kind": "ct", "vmid": 102, "node": node, "config_digest": strings.Repeat("2", 40), "bridges": []string{"vmbr0"}, "networks": map[string]string{"net0": "vmbr0"}},
|
||||
}
|
||||
common := map[string]any{
|
||||
"source_commit": sourceCommit, "machine_id": machineID, "node": node, "cluster_id": clusterID,
|
||||
"supervisor_unit": supervisorUnit, "supervisor_invocation_id": supervisorInvocationID,
|
||||
"runner_sha256": runnerHash, "test_sha256": testHash,
|
||||
}
|
||||
manifest := cloneNativePVEEvidenceMap(common)
|
||||
manifest["schema_version"] = 1
|
||||
manifest["runner_anchor_invocation_id"] = runnerInvocationID
|
||||
manifest["guests"] = guests
|
||||
writeNativePVEEvidenceJSON(t, paths[0], manifest)
|
||||
manifestBytes, err := os.ReadFile(paths[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
manifestHash := fmt.Sprintf("%x", sha256.Sum256(manifestBytes))
|
||||
|
||||
cleanup := cloneNativePVEEvidenceMap(common)
|
||||
cleanup["schema_version"] = 1
|
||||
cleanup["result"] = "passed"
|
||||
cleanup["manifest_sha256"] = manifestHash
|
||||
cleanup["runner_anchor_invocation_id"] = runnerInvocationID
|
||||
cleanup["anchor_stopped"] = true
|
||||
cleanup["action_units_gone"] = true
|
||||
cleanup["sideband_empty"] = true
|
||||
cleanup["guests"] = []any{
|
||||
map[string]any{"identity": guests[0], "stopped": true},
|
||||
map[string]any{"identity": guests[1], "stopped": true},
|
||||
}
|
||||
writeNativePVEEvidenceJSON(t, paths[1], cleanup)
|
||||
|
||||
receipt := cloneNativePVEEvidenceMap(common)
|
||||
receipt["schema_version"] = 1
|
||||
receipt["result"] = "passed"
|
||||
receipt["manifest_sha256"] = manifestHash
|
||||
receipt["runner_anchor_invocation_id"] = runnerInvocationID
|
||||
receipt["guests"] = []any{
|
||||
nativePVEFixtureGuest(guests[0].(map[string]any)),
|
||||
nativePVEFixtureGuest(guests[1].(map[string]any)),
|
||||
}
|
||||
writeNativePVEEvidenceJSON(t, paths[2], receipt)
|
||||
args := []string{paths[0], paths[1], paths[2], sourceCommit, machineID, node, clusterID, runnerHash, testHash, supervisorUnit, supervisorInvocationID, "101", "102"}
|
||||
return paths, args
|
||||
}
|
||||
|
||||
func nativePVEFixtureGuest(identity map[string]any) map[string]any {
|
||||
operations := []string{"start", "reboot", "shutdown", "start", "stop"}
|
||||
before := []string{"stopped", "running", "running", "stopped", "running"}
|
||||
after := []string{"running", "running", "stopped", "running", "stopped"}
|
||||
observations := make([]any, 0, len(operations))
|
||||
for index, operation := range operations {
|
||||
result := map[string]any{
|
||||
"operation": operation, "guest_kind": identity["kind"], "vmid": identity["vmid"],
|
||||
"execution_phase": "complete", "mutation_started": true, "mutation_completed": true, "readback_ran": true,
|
||||
"before": map[string]any{"status": before[index]}, "after": map[string]any{"status": after[index]},
|
||||
}
|
||||
observation := map[string]any{"operation": operation, "result": result, "action_units_gone": true, "sideband_empty": true}
|
||||
if after[index] == "running" {
|
||||
vmid := int(identity["vmid"].(int))
|
||||
if identity["kind"] == "vm" {
|
||||
observation["cgroup"] = fmt.Sprintf("0::/qemu.slice/qemu-%d.scope", vmid)
|
||||
} else {
|
||||
observation["cgroup"] = fmt.Sprintf("0::/lxc.payload.%d", vmid)
|
||||
}
|
||||
observation["link_paths"] = map[string]any{"net0": []string{fmt.Sprintf("guest-%d-net0", vmid), "vmbr0"}}
|
||||
}
|
||||
observations = append(observations, observation)
|
||||
}
|
||||
return map[string]any{"identity": identity, "operations": observations, "final_state": "stopped", "emergency_cleanup": false}
|
||||
}
|
||||
|
||||
func runNativePVEEvidenceVerifier(verifier string, args []string) (string, error) {
|
||||
commandArgs := append([]string{"-I", "-"}, args...)
|
||||
command := exec.Command("python3", commandArgs...)
|
||||
command.Env = append(os.Environ(), "PYTHONOPTIMIZE=1")
|
||||
command.Stdin = strings.NewReader(verifier)
|
||||
output, err := command.CombinedOutput()
|
||||
return strings.TrimSpace(string(output)), err
|
||||
}
|
||||
|
||||
func writeNativePVEEvidenceJSON(t *testing.T, path string, value any) {
|
||||
t.Helper()
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(data, '\n'), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func cloneNativePVEEvidenceMap(source map[string]any) map[string]any {
|
||||
result := make(map[string]any, len(source))
|
||||
for key, value := range source {
|
||||
result[key] = value
|
||||
}
|
||||
return result
|
||||
}
|
||||
+330
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PVE_HOST="${PVE_HOST:?set PVE_HOST to the dedicated disposable PVE node}"
|
||||
PVE_USER="${PVE_USER:-root}"
|
||||
PVE_VMID="${PVE_VMID:?set PVE_VMID to the disposable VM ID}"
|
||||
PVE_CTID="${PVE_CTID:?set PVE_CTID to the disposable container ID}"
|
||||
PVE_KNOWN_HOSTS_FILE="${PVE_KNOWN_HOSTS_FILE:?set PVE_KNOWN_HOSTS_FILE to an absolute pinned known_hosts file}"
|
||||
PULSE_PVE_QUALIFICATION_CONFIRM="${PULSE_PVE_QUALIFICATION_CONFIRM:-}"
|
||||
PULSE_PVE_QUALIFICATION_OUTPUT_DIR="${PULSE_PVE_QUALIFICATION_OUTPUT_DIR:?set PULSE_PVE_QUALIFICATION_OUTPUT_DIR to an absolute local evidence directory}"
|
||||
|
||||
if [[ "${PVE_USER}" != "root" ]]; then
|
||||
echo "ERROR: native PVE qualification requires PVE_USER=root" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! "${PVE_HOST}" =~ ^[A-Za-z0-9._:-]+$ || "${PVE_HOST}" == -* ]]; then
|
||||
echo "ERROR: PVE_HOST contains unsupported characters" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ ! "${PVE_VMID}" =~ ^[1-9][0-9]{0,8}$ || ! "${PVE_CTID}" =~ ^[1-9][0-9]{0,8}$ || "${PVE_VMID}" == "${PVE_CTID}" ]]; then
|
||||
echo "ERROR: PVE_VMID and PVE_CTID must be distinct positive decimal guest IDs" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "${PVE_KNOWN_HOSTS_FILE}" != /* || ! -f "${PVE_KNOWN_HOSTS_FILE}" ]]; then
|
||||
echo "ERROR: PVE_KNOWN_HOSTS_FILE must be an existing absolute path" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "${PULSE_PVE_QUALIFICATION_OUTPUT_DIR}" != /* || ! -d "${PULSE_PVE_QUALIFICATION_OUTPUT_DIR}" ]]; then
|
||||
echo "ERROR: PULSE_PVE_QUALIFICATION_OUTPUT_DIR must be absolute" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ -n "$(git -C "${REPO_ROOT}" status --porcelain)" ]]; then
|
||||
echo "ERROR: qualification builds require a clean exact source checkout" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
readonly PVE_TARGET="${PVE_USER}@${PVE_HOST}"
|
||||
readonly SOURCE_COMMIT="$(git -C "${REPO_ROOT}" rev-parse HEAD)"
|
||||
readonly RUN_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
readonly OUTPUT_DIR="${PULSE_PVE_QUALIFICATION_OUTPUT_DIR}/${RUN_STAMP}-${SOURCE_COMMIT:0:12}"
|
||||
LOCAL_BUILD_DIR=""
|
||||
REMOTE_DIR=""
|
||||
REMOVE_REMOTE_DIR=false
|
||||
|
||||
ssh_cmd() {
|
||||
ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \
|
||||
-o "UserKnownHostsFile=${PVE_KNOWN_HOSTS_FILE}" -- "${PVE_TARGET}" "$@"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ "${REMOVE_REMOTE_DIR}" == "true" && -n "${REMOTE_DIR}" && "${REMOTE_DIR}" == /run/pulse-pve-qualification.* ]]; then
|
||||
ssh_cmd "rm -f '${REMOTE_DIR}/pulse-agent-runner' '${REMOTE_DIR}/hostagent.test' '${REMOTE_DIR}/state/manifest.json' '${REMOTE_DIR}/state/receipt.json' '${REMOTE_DIR}/state/cleanup.json'; rmdir '${REMOTE_DIR}/state/typed-actions' >/dev/null 2>&1 || true; rmdir '${REMOTE_DIR}/state' '${REMOTE_DIR}'" >/dev/null 2>&1 || true
|
||||
elif [[ -n "${REMOTE_DIR}" && "${REMOTE_DIR}" == /run/pulse-pve-qualification.* ]]; then
|
||||
echo "WARNING: incomplete qualification state retained for inspection at ${PVE_TARGET}:${REMOTE_DIR}" >&2
|
||||
fi
|
||||
if [[ -n "${LOCAL_BUILD_DIR}" && "${LOCAL_BUILD_DIR}" == /tmp/pulse-pve-qualification.* ]]; then
|
||||
rm -f "${LOCAL_BUILD_DIR}/pulse-agent-runner" "${LOCAL_BUILD_DIR}/hostagent.test" >/dev/null 2>&1 || true
|
||||
rmdir "${LOCAL_BUILD_DIR}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
sha256_files() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$@"
|
||||
else
|
||||
shasum -a 256 "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
if ! ssh-keygen -F "${PVE_HOST}" -f "${PVE_KNOWN_HOSTS_FILE}" >/dev/null; then
|
||||
echo "ERROR: PVE_HOST is not pinned in PVE_KNOWN_HOSTS_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "$(ssh_cmd "id -u")" != "0" ]]; then
|
||||
echo "ERROR: the PVE SSH session must run as root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
remote_machine_id="$(ssh_cmd "tr -d '\\r\\n' </etc/machine-id")"
|
||||
remote_node="$(ssh_cmd "basename \"\$(readlink -f /etc/pve/local)\"")"
|
||||
if [[ ! "${remote_machine_id}" =~ ^[0-9a-f]{32}$ || ! "${remote_node}" =~ ^[A-Za-z0-9][A-Za-z0-9.-]*$ ]]; then
|
||||
echo "ERROR: target did not return a valid PVE machine/node identity" >&2
|
||||
exit 2
|
||||
fi
|
||||
remote_cluster_id="$(ssh_cmd "if test -f /etc/pve/corosync.conf; then printf 'corosync-sha256:'; sha256sum /etc/pve/corosync.conf | cut -d ' ' -f 1; else printf 'standalone:${remote_node}'; fi")"
|
||||
cluster_confirmation_hash="$(printf '%s' "${remote_cluster_id}" | sha256_files | awk '{print substr($1,1,12)}')"
|
||||
expected_confirmation="I_HAVE_VERIFIED_THIS_DISPOSABLE_PVE_TARGET_MACHINE_${remote_machine_id:0:12}_NODE_${remote_node}_CLUSTER_${cluster_confirmation_hash}_COMMIT_${SOURCE_COMMIT}_VM_${PVE_VMID}_CT_${PVE_CTID}"
|
||||
if [[ "${PULSE_PVE_QUALIFICATION_CONFIRM}" != "${expected_confirmation}" ]]; then
|
||||
echo "ERROR: exact destructive confirmation required:" >&2
|
||||
echo "export PULSE_PVE_QUALIFICATION_CONFIRM=${expected_confirmation}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
remote_runner_load_state="$(ssh_cmd "/usr/bin/systemctl show --property=LoadState --value pulse-agent-runner.service")"
|
||||
if [[ "${remote_runner_load_state}" != "not-found" ]]; then
|
||||
echo "ERROR: target already has a loaded pulse-agent-runner.service; use a dedicated disposable PVE node" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ -n "$(ssh_cmd "/usr/bin/systemctl list-units --all --full --plain --no-legend 'pulse-agent-action-*.service'")" ]]; then
|
||||
echo "ERROR: target already has Pulse typed-action units" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
remote_arch="$(ssh_cmd "uname -m")"
|
||||
case "${remote_arch}" in
|
||||
x86_64) go_arch=amd64 ;;
|
||||
aarch64|arm64) go_arch=arm64 ;;
|
||||
*)
|
||||
echo "ERROR: unsupported PVE architecture: ${remote_arch}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
mkdir "${OUTPUT_DIR}"
|
||||
chmod 0700 "${OUTPUT_DIR}"
|
||||
LOCAL_BUILD_DIR="$(mktemp -d /tmp/pulse-pve-qualification.XXXXXX)"
|
||||
echo "Building exact ${SOURCE_COMMIT} qualification binaries for linux/${go_arch}"
|
||||
(
|
||||
cd "${REPO_ROOT}"
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH="${go_arch}" GOFLAGS= GOWORK=off go build -trimpath -o "${LOCAL_BUILD_DIR}/pulse-agent-runner" ./cmd/pulse-agent-runner
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH="${go_arch}" GOFLAGS= GOWORK=off go test -c -trimpath -o "${LOCAL_BUILD_DIR}/hostagent.test" ./internal/hostagent
|
||||
)
|
||||
chmod 0700 "${LOCAL_BUILD_DIR}/pulse-agent-runner" "${LOCAL_BUILD_DIR}/hostagent.test"
|
||||
sha256_files "${LOCAL_BUILD_DIR}/pulse-agent-runner" "${LOCAL_BUILD_DIR}/hostagent.test" >"${OUTPUT_DIR}/artifact-sha256.txt"
|
||||
|
||||
REMOTE_DIR="$(ssh_cmd "mktemp -d /run/pulse-pve-qualification.XXXXXX")"
|
||||
if [[ "${REMOTE_DIR}" != /run/pulse-pve-qualification.* ]]; then
|
||||
echo "ERROR: remote qualification directory was outside the expected boundary" >&2
|
||||
exit 2
|
||||
fi
|
||||
scp -q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \
|
||||
-o "UserKnownHostsFile=${PVE_KNOWN_HOSTS_FILE}" -- \
|
||||
"${LOCAL_BUILD_DIR}/pulse-agent-runner" "${LOCAL_BUILD_DIR}/hostagent.test" \
|
||||
"${PVE_TARGET}:${REMOTE_DIR}/"
|
||||
ssh_cmd "chown root:root '${REMOTE_DIR}/pulse-agent-runner' '${REMOTE_DIR}/hostagent.test' && chmod 0700 '${REMOTE_DIR}/pulse-agent-runner' '${REMOTE_DIR}/hostagent.test' && mkdir -m 0700 '${REMOTE_DIR}/state'"
|
||||
|
||||
remote_hashes="$(ssh_cmd "sha256sum '${REMOTE_DIR}/pulse-agent-runner' '${REMOTE_DIR}/hostagent.test'")"
|
||||
local_runner_hash="$(sha256_files "${LOCAL_BUILD_DIR}/pulse-agent-runner" | awk '{print $1}')"
|
||||
local_test_hash="$(sha256_files "${LOCAL_BUILD_DIR}/hostagent.test" | awk '{print $1}')"
|
||||
if ! grep -q "^${local_runner_hash} " <<<"${remote_hashes}" || ! grep -q "^${local_test_hash} " <<<"${remote_hashes}"; then
|
||||
echo "ERROR: copied qualification artifact hashes did not match" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "Running native PVE qualification against disposable VM ${PVE_VMID} and CT ${PVE_CTID}"
|
||||
qualification_unit="pulse-pve-qualification-${SOURCE_COMMIT:0:8}-${PVE_VMID}-${PVE_CTID}.service"
|
||||
evidence_unit="${qualification_unit%.service}-evidence.service"
|
||||
if [[ "$(ssh_cmd "/usr/bin/systemctl show --property=LoadState --value '${qualification_unit}'")" != "not-found" ]]; then
|
||||
echo "ERROR: qualification supervisor unit already exists: ${qualification_unit}" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "$(ssh_cmd "/usr/bin/systemctl show --property=LoadState --value '${evidence_unit}'")" != "not-found" ]]; then
|
||||
echo "ERROR: qualification evidence-retention unit already exists: ${evidence_unit}" >&2
|
||||
exit 2
|
||||
fi
|
||||
cleanup_exec="/usr/bin/env PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C PULSE_AGENT_RUNNER_STATE_DIR=${REMOTE_DIR}/state PULSE_TEST_PVE_CLEANUP_MANIFEST=${REMOTE_DIR}/state/manifest.json PULSE_TEST_PVE_CLEANUP_RECEIPT=${REMOTE_DIR}/state/cleanup.json ${REMOTE_DIR}/hostagent.test -test.v -test.run TestNativePVEQualificationSupervisorCleanup -test.timeout=15m"
|
||||
ssh_cmd "/usr/bin/systemd-run --no-ask-password --quiet --remain-after-exit --service-type=exec --unit='${qualification_unit}' --property=User=root --property=Group=root --property=UMask=0077 --property=WorkingDirectory=/ --property=RuntimeMaxSec=20m --property=TimeoutStopSec=20m --property='ExecStopPost=${cleanup_exec}' -- /usr/bin/env PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin LC_ALL=C PULSE_AGENT_RUNNER_STATE_DIR='${REMOTE_DIR}/state' PULSE_TEST_PVE_SUPERVISOR_UNIT='${qualification_unit}' PULSE_TEST_PVE_CONFIRM='${expected_confirmation}' PULSE_TEST_PVE_VM_ID='${PVE_VMID}' PULSE_TEST_PVE_CT_ID='${PVE_CTID}' PULSE_TEST_ACTION_RUNNER_BINARY='${REMOTE_DIR}/pulse-agent-runner' PULSE_TEST_ACTION_RUNNER_SHA256='${local_runner_hash}' PULSE_TEST_HOSTAGENT_SHA256='${local_test_hash}' PULSE_TEST_PVE_RECEIPT_PATH='${REMOTE_DIR}/state/receipt.json' PULSE_TEST_SOURCE_COMMIT='${SOURCE_COMMIT}' '${REMOTE_DIR}/hostagent.test' -test.v -test.run TestNativePVEProxmoxGuestLifecycleQualification -test.timeout=0"
|
||||
|
||||
supervisor_properties="$(ssh_cmd "/usr/bin/systemctl show --property=InvocationID --property=ActiveState --property=ExecStopPost '${qualification_unit}'")"
|
||||
supervisor_invocation_id="$(sed -n 's/^InvocationID=//p' <<<"${supervisor_properties}")"
|
||||
if [[ ! "${supervisor_invocation_id}" =~ ^[0-9a-f]{32}$ ]] ||
|
||||
! grep -qx 'ActiveState=active' <<<"${supervisor_properties}" ||
|
||||
! grep '^ExecStopPost=' <<<"${supervisor_properties}" | grep -Fq "PULSE_AGENT_RUNNER_STATE_DIR=${REMOTE_DIR}/state"; then
|
||||
echo "ERROR: qualification supervisor did not start with an exact active InvocationID" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_cmd "/usr/bin/systemd-run --no-ask-password --quiet --collect --service-type=exec --unit='${evidence_unit}' --property='Wants=${qualification_unit}' --property='After=${qualification_unit}' --property=RuntimeMaxSec=45m -- /usr/bin/sleep infinity"
|
||||
evidence_unit_properties="$(ssh_cmd "/usr/bin/systemctl show --property=InvocationID --property=ActiveState --property=Wants --property=After '${evidence_unit}'")"
|
||||
evidence_unit_invocation_id="$(sed -n 's/^InvocationID=//p' <<<"${evidence_unit_properties}")"
|
||||
if [[ ! "${evidence_unit_invocation_id}" =~ ^[0-9a-f]{32}$ ]] ||
|
||||
! grep -qx 'ActiveState=active' <<<"${evidence_unit_properties}" ||
|
||||
! sed -n 's/^Wants=//p' <<<"${evidence_unit_properties}" | tr ' ' '\n' | grep -Fxq "${qualification_unit}" ||
|
||||
! sed -n 's/^After=//p' <<<"${evidence_unit_properties}" | tr ' ' '\n' | grep -Fxq "${qualification_unit}"; then
|
||||
echo "ERROR: supervisor evidence-retention unit was not bound exactly" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
deadline=$((SECONDS + 2400))
|
||||
stop_requested=false
|
||||
supervisor_stop_status=not-requested
|
||||
pre_stop_supervisor_properties=""
|
||||
while (( SECONDS < deadline )); do
|
||||
supervisor_active_state="$(ssh_cmd "/usr/bin/systemctl show --property=ActiveState --value '${qualification_unit}'" 2>/dev/null || true)"
|
||||
if [[ "${stop_requested}" == "false" && "${supervisor_active_state}" == "active" ]] && ssh_cmd "test -s '${REMOTE_DIR}/state/receipt.json'" >/dev/null 2>&1; then
|
||||
stop_requested=true
|
||||
pre_stop_supervisor_properties="$(ssh_cmd "/usr/bin/systemctl show --property=InvocationID --property=ActiveState --property=Result --property=ExecMainStatus '${qualification_unit}'")"
|
||||
if ! grep -qx "InvocationID=${supervisor_invocation_id}" <<<"${pre_stop_supervisor_properties}" ||
|
||||
! grep -qx 'ActiveState=active' <<<"${pre_stop_supervisor_properties}" ||
|
||||
! grep -qx 'Result=success' <<<"${pre_stop_supervisor_properties}" ||
|
||||
! grep -qx 'ExecMainStatus=0' <<<"${pre_stop_supervisor_properties}"; then
|
||||
echo "ERROR: qualification main process did not reach the exact successful terminal state" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ssh_cmd "/usr/bin/timeout 21m /usr/bin/systemctl --no-ask-password stop '${qualification_unit}'"; then
|
||||
supervisor_stop_status=0
|
||||
else
|
||||
supervisor_stop_status=failed
|
||||
echo "ERROR: qualification supervisor stop/ExecStopPost job failed" >&2
|
||||
exit 1
|
||||
fi
|
||||
supervisor_active_state="$(ssh_cmd "/usr/bin/systemctl show --property=ActiveState --value '${qualification_unit}'" 2>/dev/null || true)"
|
||||
fi
|
||||
if ssh_cmd "test -s '${REMOTE_DIR}/state/cleanup.json'" >/dev/null 2>&1 && [[ -z "${supervisor_active_state}" || "${supervisor_active_state}" == "inactive" || "${supervisor_active_state}" == "failed" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
if ! ssh_cmd "test -s '${REMOTE_DIR}/state/cleanup.json'" >/dev/null 2>&1; then
|
||||
ssh_cmd "/usr/bin/timeout 21m /usr/bin/systemctl stop '${qualification_unit}'" >/dev/null 2>&1 || true
|
||||
echo "ERROR: qualification supervisor did not produce bounded cleanup evidence" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
final_supervisor_properties="$(ssh_cmd "/usr/bin/systemctl show --property=InvocationID --property=ActiveState --property=Result --property=ExecMainStatus --property=ExecStopPost '${qualification_unit}'" 2>/dev/null || true)"
|
||||
if [[ "${supervisor_stop_status}" != "0" ]] ||
|
||||
! grep -qx "InvocationID=${supervisor_invocation_id}" <<<"${final_supervisor_properties}" ||
|
||||
! grep -qx 'ActiveState=inactive' <<<"${final_supervisor_properties}" ||
|
||||
! grep -qx 'Result=success' <<<"${final_supervisor_properties}" ||
|
||||
! grep -qx 'ExecMainStatus=0' <<<"${final_supervisor_properties}" ||
|
||||
! grep '^ExecStopPost=' <<<"${final_supervisor_properties}" | grep -Eq 'code=exited.*status=0([ /;}]|$)'; then
|
||||
echo "ERROR: qualification supervisor did not complete with the exact successful main and ExecStopPost result" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! ssh_cmd "/usr/bin/timeout 30s /usr/bin/systemctl --no-ask-password stop '${evidence_unit}'"; then
|
||||
echo "ERROR: qualification evidence-retention unit did not stop" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ssh_cmd "/usr/bin/journalctl --no-pager --output=short-iso-precise --unit='${qualification_unit}'" >"${OUTPUT_DIR}/transcript.txt"
|
||||
printf '%s\n' "started:" "${supervisor_properties}" "before_stop:" "${pre_stop_supervisor_properties}" "stop_status=${supervisor_stop_status}" "after_stop:" "${final_supervisor_properties}" >"${OUTPUT_DIR}/supervisor-properties.txt"
|
||||
for name in manifest.json cleanup.json; do
|
||||
scp -q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \
|
||||
-o "UserKnownHostsFile=${PVE_KNOWN_HOSTS_FILE}" -- \
|
||||
"${PVE_TARGET}:${REMOTE_DIR}/state/${name}" "${OUTPUT_DIR}/${name}"
|
||||
done
|
||||
scp -q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes \
|
||||
-o "UserKnownHostsFile=${PVE_KNOWN_HOSTS_FILE}" -- \
|
||||
"${PVE_TARGET}:${REMOTE_DIR}/state/receipt.json" "${OUTPUT_DIR}/receipt.json"
|
||||
sha256_files "${OUTPUT_DIR}/transcript.txt" >"${OUTPUT_DIR}/transcript-sha256.txt"
|
||||
printf '%s\n' "source_commit=${SOURCE_COMMIT}" "pve_node=${remote_node}" "pve_machine_id=${remote_machine_id}" "pve_cluster_id=${remote_cluster_id}" "pve_vmid=${PVE_VMID}" "pve_ctid=${PVE_CTID}" "supervisor_unit=${qualification_unit}" "supervisor_invocation_id=${supervisor_invocation_id}" "evidence_unit=${evidence_unit}" "evidence_unit_invocation_id=${evidence_unit_invocation_id}" "completed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"${OUTPUT_DIR}/run-metadata.txt"
|
||||
chmod 0600 "${OUTPUT_DIR}"/*
|
||||
|
||||
if ! python3 -I - "${OUTPUT_DIR}/manifest.json" "${OUTPUT_DIR}/cleanup.json" "${OUTPUT_DIR}/receipt.json" "${SOURCE_COMMIT}" "${remote_machine_id}" "${remote_node}" "${remote_cluster_id}" "${local_runner_hash}" "${local_test_hash}" "${qualification_unit}" "${supervisor_invocation_id}" "${PVE_VMID}" "${PVE_CTID}" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
manifest_path, cleanup_path, receipt_path = map(pathlib.Path, sys.argv[1:4])
|
||||
source_commit, machine_id, node, cluster_id, runner_hash, test_hash, supervisor_unit, supervisor_invocation_id, vmid, ctid = sys.argv[4:]
|
||||
manifest_bytes = manifest_path.read_bytes()
|
||||
manifest = json.loads(manifest_bytes)
|
||||
cleanup = json.loads(cleanup_path.read_text())
|
||||
receipt = json.loads(receipt_path.read_text())
|
||||
manifest_hash = hashlib.sha256(manifest_bytes).hexdigest()
|
||||
|
||||
def require(condition, message):
|
||||
if not condition:
|
||||
raise SystemExit(f"invalid native PVE qualification evidence: {message}")
|
||||
|
||||
expected_common = {
|
||||
"source_commit": source_commit,
|
||||
"machine_id": machine_id,
|
||||
"node": node,
|
||||
"cluster_id": cluster_id,
|
||||
"supervisor_unit": supervisor_unit,
|
||||
"supervisor_invocation_id": supervisor_invocation_id,
|
||||
"runner_sha256": runner_hash,
|
||||
"test_sha256": test_hash,
|
||||
}
|
||||
require(manifest.get("schema_version") == 1, "manifest schema")
|
||||
require(all(manifest.get(key) == value for key, value in expected_common.items()), "manifest run binding")
|
||||
require(len(manifest.get("guests", [])) == 2, "manifest guest count")
|
||||
require({(item.get("kind"), item.get("vmid")) for item in manifest["guests"]} == {("vm", int(vmid)), ("ct", int(ctid))}, "manifest guest identities")
|
||||
require(all(item.get("node") == node and item.get("config_digest") and item.get("bridges") and item.get("networks") for item in manifest["guests"]), "manifest guest evidence")
|
||||
require(cleanup.get("schema_version") == 1 and cleanup.get("result") == "passed", "cleanup result")
|
||||
require(all(cleanup.get(key) == value for key, value in expected_common.items()), "cleanup run binding")
|
||||
require(cleanup.get("manifest_sha256") == manifest_hash, "cleanup manifest hash")
|
||||
require(cleanup.get("runner_anchor_invocation_id") == manifest.get("runner_anchor_invocation_id"), "cleanup runner anchor")
|
||||
require(cleanup.get("anchor_stopped") is True, "cleanup anchor state")
|
||||
require(cleanup.get("action_units_gone") is True and cleanup.get("sideband_empty") is True, "cleanup containment")
|
||||
require(len(cleanup.get("guests", [])) == 2 and all(item.get("stopped") is True and not item.get("error") for item in cleanup["guests"]), "cleanup guest states")
|
||||
require([item.get("identity") for item in cleanup["guests"]] == manifest["guests"], "cleanup guest binding")
|
||||
require(receipt.get("schema_version") == 1 and receipt.get("result") == "passed", "qualification result")
|
||||
require(all(receipt.get(key) == value for key, value in expected_common.items()), "qualification run binding")
|
||||
require(receipt.get("manifest_sha256") == manifest_hash, "qualification manifest hash")
|
||||
require(receipt.get("runner_anchor_invocation_id") == manifest.get("runner_anchor_invocation_id"), "qualification runner anchor")
|
||||
require(len(receipt.get("guests", [])) == 2 and all(item.get("final_state") == "stopped" and item.get("emergency_cleanup") is False for item in receipt["guests"]), "qualification guest states")
|
||||
require([item.get("identity") for item in receipt["guests"]] == manifest["guests"], "qualification guest binding")
|
||||
expected_operations = ["start", "reboot", "shutdown", "start", "stop"]
|
||||
expected_before = ["stopped", "running", "running", "stopped", "running"]
|
||||
expected_after = ["running", "running", "stopped", "running", "stopped"]
|
||||
for guest in receipt["guests"]:
|
||||
operations = guest.get("operations", [])
|
||||
identity = guest["identity"]
|
||||
require([item.get("operation") for item in operations] == expected_operations, "operation sequence")
|
||||
require([item.get("result", {}).get("before", {}).get("status") for item in operations] == expected_before, "operation before states")
|
||||
require([item.get("result", {}).get("after", {}).get("status") for item in operations] == expected_after, "operation after states")
|
||||
require(all(item.get("action_units_gone") is True and item.get("sideband_empty") is True for item in operations), "per-operation containment")
|
||||
for index, item in enumerate(operations):
|
||||
result = item.get("result", {})
|
||||
require(result.get("operation") == expected_operations[index] and result.get("guest_kind") == identity["kind"] and result.get("vmid") == identity["vmid"], "production result identity")
|
||||
require(result.get("execution_phase") == "complete" and result.get("mutation_started") is True and result.get("mutation_completed") is True and result.get("readback_ran") is True, "production result completion")
|
||||
require(not result.get("error") and not result.get("reason_code"), "production result error fields")
|
||||
if expected_after[index] == "running":
|
||||
cgroup = item.get("cgroup", "")
|
||||
vmid_text = str(identity["vmid"])
|
||||
cgroup_markers = [f"qemu-{vmid_text}.scope", f"/{vmid_text}.scope"] if identity["kind"] == "vm" else [f"lxc.payload.{vmid_text}", f"/lxc/{vmid_text}/", f"machine-lxc\\x2d{vmid_text}.scope"]
|
||||
require(any(marker in cgroup for marker in cgroup_markers), "VMID-specific cgroup")
|
||||
link_paths = item.get("link_paths", {})
|
||||
require(set(link_paths) == set(identity["networks"]), "per-NIC link-path keys")
|
||||
require(all(isinstance(link_paths[network], list) and len(link_paths[network]) >= 2 and link_paths[network][-1] == bridge for network, bridge in identity["networks"].items()), "per-NIC live bridge paths")
|
||||
PY
|
||||
then
|
||||
echo "ERROR: qualification or supervisor cleanup receipt was not structurally passing" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$(ssh_cmd "/usr/bin/systemctl show --property=LoadState --value pulse-agent-runner.service")" != "not-found" || -n "$(ssh_cmd "/usr/bin/systemctl list-units --all --full --plain --no-legend 'pulse-agent-action-*.service'")" ]]; then
|
||||
echo "ERROR: qualification cleanup left Pulse runtime units behind" >&2
|
||||
exit 1
|
||||
fi
|
||||
for evidence_file in "${OUTPUT_DIR}"/*; do
|
||||
[[ "${evidence_file}" == "${OUTPUT_DIR}/evidence-sha256.txt" ]] && continue
|
||||
sha256_files "${evidence_file}"
|
||||
done >"${OUTPUT_DIR}/evidence-sha256.txt"
|
||||
chmod 0600 "${OUTPUT_DIR}/evidence-sha256.txt"
|
||||
ssh_cmd "/usr/bin/systemctl reset-failed '${qualification_unit}'" >/dev/null 2>&1 || true
|
||||
REMOVE_REMOTE_DIR=true
|
||||
echo "Native PVE qualification passed; evidence retained at ${OUTPUT_DIR}"
|
||||
Reference in New Issue
Block a user