ci: opt-in dist-upgrade and reboot for the currency lane

This commit is contained in:
goodolclint-claude[bot]
2026-09-01 14:02:15 -05:00
parent c52d2e40e8
commit 7e66dfc051
4 changed files with 242 additions and 3 deletions
@@ -2,16 +2,22 @@
# Prepares the test environment on the nested PVE node.
# Only performs operations that have no PVE API equivalent.
#
# Usage: prepare-test-environment.sh <nested-pve-ip> <root-password>
# Usage: prepare-test-environment.sh <nested-pve-ip> <root-password> [dist-upgrade] [pkg-out]
#
# Operations:
# - Optionally dist-upgrade the node, record its package set, and reboot
# (currency lane only; off unless <dist-upgrade> is 1)
# - Enable snippets+import content types on local storage (pvesm set)
# - Upload cloud-init user-data snippet (SCP — no snippet upload API)
set -euo pipefail
NESTED_IP="${1:?Usage: prepare-test-environment.sh <ip> <password>}"
NESTED_IP="${1:?Usage: prepare-test-environment.sh <ip> <password> [dist-upgrade] [pkg-out]}"
ROOT_PASS="$2"
DIST_UPGRADE="${3:-0}"
PKG_OUT="${4:-}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SSH_OPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
SSH_CMD="sshpass -p ${ROOT_PASS} ssh ${SSH_OPTS} root@${NESTED_IP}"
@@ -19,6 +25,32 @@ SCP_CMD="sshpass -p ${ROOT_PASS} scp ${SSH_OPTS}"
echo "=== Preparing test environment on ${NESTED_IP} ==="
if [[ "${DIST_UPGRADE}" == "1" ]]; then
echo "Running dist-upgrade (currency lane)..."
${SSH_CMD} "DEBIAN_FRONTEND=noninteractive apt-get update -qq && \
DEBIAN_FRONTEND=noninteractive apt-get -y \
-o Dpkg::Options::=--force-confold \
-o Dpkg::Options::=--force-confdef \
dist-upgrade"
if [[ -n "${PKG_OUT}" ]]; then
echo "Recording package set to ${PKG_OUT}..."
${SSH_CMD} "dpkg-query -W -f='\${binary:Package}\t\${Version}\n' | sort" > "${PKG_OUT}"
fi
# A PVE dist-upgrade pulls proxmox-kernel-*; without a reboot the node runs
# new userspace on the old kernel. Reboot unconditionally rather than
# testing /var/run/reboot-required — that file comes from
# update-notifier-common, which is not guaranteed on a PVE node.
echo "Rebooting after dist-upgrade..."
${SSH_CMD} "systemctl reboot" || true
# The API stays up for a few seconds after the reboot is issued, so polling
# immediately would match the pre-reboot node and return at once.
sleep 30
bash "${SCRIPT_DIR}/wait-for-api.sh" "${NESTED_IP}" 8006 600
fi
# Enable snippets and import content types on local storage
echo "Configuring local storage content types..."
${SSH_CMD} "mkdir -p /var/lib/vz/snippets && pvesm set local --content images,iso,vztmpl,snippets,import"
@@ -0,0 +1,95 @@
#!/usr/bin/env bash
# Self-check for prepare-test-environment.sh's opt-in dist-upgrade branch.
#
# Stubs sshpass/curl/sleep on PATH so both paths run offline in ~0s, then
# asserts on the commands the script actually issued.
#
# Run: bash tests/infrastructure/scripts/prepare-test-environment.test.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TARGET="$SCRIPT_DIR/prepare-test-environment.sh"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT
mkdir -p "$TMP/bin"
# Fake sshpass: log every invocation, emit a plausible dpkg-query result.
cat > "$TMP/bin/sshpass" <<'STUB'
#!/usr/bin/env bash
echo "$*" >> "$STUB_LOG"
case "$*" in
*dpkg-query*) echo -e "proxmox-kernel-6.14\t6.14.11-1\npve-manager\t9.2.1" ;;
esac
exit 0
STUB
# wait-for-api.sh greps curl output for "version"; sleep must not really sleep.
cat > "$TMP/bin/curl" <<'STUB'
#!/usr/bin/env bash
echo '{"data":{"version":"9.2.1"}}'
STUB
cat > "$TMP/bin/sleep" <<'STUB'
#!/usr/bin/env bash
exit 0
STUB
chmod +x "$TMP/bin/"*
export PATH="$TMP/bin:$PATH"
fail=0
check() {
local desc="$1" haystack="$2" needle="$3" want="$4"
if grep -q -- "$needle" "$haystack"; then found=yes; else found=no; fi
if [[ "$found" == "$want" ]]; then
echo " ok: $desc"
else
echo " FAIL: $desc (expected present=$want, got present=$found)"
fail=1
fi
}
echo "case 1: no dist-upgrade argument — lane 1 path must be untouched"
export STUB_LOG="$TMP/log1"
: > "$STUB_LOG"
bash "$TARGET" 10.0.0.1 secret > "$TMP/out1" 2>&1
check "no dist-upgrade issued" "$STUB_LOG" "dist-upgrade" no
check "no reboot issued" "$STUB_LOG" "systemctl reboot" no
check "no package set recorded" "$STUB_LOG" "dpkg-query" no
check "storage still configured" "$STUB_LOG" "pvesm set local" yes
echo "case 2: dist-upgrade requested"
export STUB_LOG="$TMP/log2"
: > "$STUB_LOG"
bash "$TARGET" 10.0.0.1 secret 1 "$TMP/packages.txt" > "$TMP/out2" 2>&1
check "dist-upgrade issued" "$STUB_LOG" "dist-upgrade" yes
check "reboot issued" "$STUB_LOG" "systemctl reboot" yes
check "package set recorded" "$STUB_LOG" "dpkg-query" yes
check "storage still configured" "$STUB_LOG" "pvesm set local" yes
# The reboot must be issued after the upgrade, or the node records a package
# set it never booted.
upgrade_line=$(grep -n "dist-upgrade" "$STUB_LOG" | head -1 | cut -d: -f1)
reboot_line=$(grep -n "systemctl reboot" "$STUB_LOG" | head -1 | cut -d: -f1)
if [[ "$reboot_line" -gt "$upgrade_line" ]]; then
echo " ok: reboot ordered after dist-upgrade"
else
echo " FAIL: reboot ordered before dist-upgrade"
fail=1
fi
if [[ -s "$TMP/packages.txt" ]]; then
echo " ok: package file non-empty"
else
echo " FAIL: package file empty or missing"
fail=1
fi
if [[ "$fail" -eq 0 ]]; then
echo "PASS"
else
echo "FAILED"
exit 1
fi
@@ -65,6 +65,10 @@ CONFIG_FILE="${CONFIG_FILE:-$WORK_DIR/config.json}"
MODULE_ARTIFACT="${MODULE_ARTIFACT:-$REPO_ROOT/publish/netstandard2.0}"
PVE_VERSIONS="${PVE_VERSIONS:-9}"
SKIP_PROVISION="${SKIP_PROVISION:-false}"
# Currency lane only: dist-upgrade each nested node, record its package set,
# and reboot before the suite runs. Off for the pinned gating lane — the ISO is
# the pin, and upgrading is what produced the pve-cluster version mismatch.
PVE_DIST_UPGRADE="${PVE_DIST_UPGRADE:-0}"
STORAGE_ISCSI_IQN="${STORAGE_ISCSI_IQN:-iqn.2024-01.local.test:storage}"
STORAGE_VM_FQDN="${STORAGE_VM_FQDN:-pvetest-storage.test.local}"
STORAGE_VMID="${STORAGE_VMID:-5080}"
@@ -397,7 +401,8 @@ cmd_provision() {
local ip
ip=$(jq -r .host "$WORK_DIR/${node}.json")
log "Preparing test environment on $node ($ip)..."
bash "$SCRIPT_DIR/prepare-test-environment.sh" "$ip" "$PVE_PASSWORD"
bash "$SCRIPT_DIR/prepare-test-environment.sh" "$ip" "$PVE_PASSWORD" \
"$PVE_DIST_UPGRADE" "$WORK_DIR/${node}-packages.txt"
done
# Write test config — merge with existing config to preserve entries