mirror of
https://github.com/GoodOlClint/PSProxmoxVE.git
synced 2026-09-03 18:55:33 +00:00
goodolclint-claude[bot]
5c6cd7f40f
Run 173 left node B with healthy corosync (2-member primary component, both links connected) but no /etc/pve/corosync.conf, no dcdb/status journal lines, and pvecm status reporting it is not part of a cluster. That file is database-backed: pmxcfs creates it only when it starts with no config.db and imports /etc/corosync/corosync.conf, so a surviving standalone config.db would mean silent local mode. Capture the package versions, pmxcfs command line, /etc/pve mount, .members, the config.db and its backup dir, whether the database holds a corosync.conf row, and the CPG group membership. Read-only; the sqlite3 CLI is not guaranteed on a PVE node, so fall back to strings. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
107 lines
3.8 KiB
Bash
107 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
|
# Dump corosync state from both nested PVE nodes after a cluster test failure.
|
|
#
|
|
# Usage: diagnose-cluster.sh [8|9]
|
|
#
|
|
# The PVE API reports a joined-but-offline node as online=0 with no further
|
|
# detail; corosync's own view lives only on the nodes, which the cleanup job
|
|
# destroys minutes later. Best-effort: never fails the caller.
|
|
#
|
|
# Required env vars:
|
|
# PVE_PASSWORD Root password for the nested PVE instances
|
|
#
|
|
# Optional env vars:
|
|
# CONFIG_FILE Test config JSON (default: $CACHE_DIR/work/config.json)
|
|
# CACHE_DIR Shared cache mount (default: /opt/pve-integration)
|
|
|
|
VERSION="${1:-9}"
|
|
CACHE_DIR="${CACHE_DIR:-/opt/pve-integration}"
|
|
CONFIG_FILE="${CONFIG_FILE:-$CACHE_DIR/work/config.json}"
|
|
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
echo "diagnose-cluster: no config at $CONFIG_FILE — nothing to inspect"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -z "${PVE_PASSWORD:-}" ]]; then
|
|
echo "diagnose-cluster: PVE_PASSWORD unset — cannot reach the nodes"
|
|
exit 0
|
|
fi
|
|
|
|
SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10)
|
|
|
|
dump_node() {
|
|
local label="$1" ip="$2"
|
|
echo
|
|
echo "══════════ $label ($ip) ══════════"
|
|
if [[ -z "$ip" || "$ip" == "null" ]]; then
|
|
echo " no address in $CONFIG_FILE"
|
|
return
|
|
fi
|
|
|
|
sshpass -p "$PVE_PASSWORD" ssh "${SSH_OPTS[@]}" "root@${ip}" bash -s <<'REMOTE' 2>&1 || echo " ssh to $ip failed (rc=$?)"
|
|
set +e
|
|
echo "--- hostname / resolution ---"
|
|
hostname -f
|
|
echo "hostname -i: $(hostname -i 2>&1)"
|
|
grep -vE '^\s*#' /etc/hosts | grep -vE '^\s*$'
|
|
echo
|
|
echo "--- addresses ---"
|
|
ip -4 -o addr show scope global
|
|
echo
|
|
echo "--- pmxcfs mode: cluster or local? ---"
|
|
# /etc/pve/corosync.conf is database-backed; pmxcfs only creates it when it
|
|
# starts with no config.db and imports /etc/corosync/corosync.conf. A surviving
|
|
# standalone config.db means silent local mode with corosync otherwise healthy.
|
|
dpkg-query -W pve-cluster corosync 2>&1
|
|
tr '\0' ' ' < "/proc/$(systemctl show pve-cluster -p MainPID --value)/cmdline" 2>&1; echo
|
|
findmnt --target /etc/pve --output TARGET,SOURCE,FSTYPE 2>&1
|
|
echo ".members: $(cat /etc/pve/.members 2>&1 | tr -d '\n')"
|
|
ls -la /var/lib/pve-cluster/ 2>&1
|
|
ls -la /var/lib/pve-cluster/backup/ 2>&1
|
|
if command -v sqlite3 >/dev/null 2>&1; then
|
|
sqlite3 -readonly /var/lib/pve-cluster/config.db \
|
|
"PRAGMA quick_check; SELECT name,version,writer,mtime,length(data) FROM tree WHERE name='corosync.conf';" 2>&1
|
|
else
|
|
echo "sqlite3 absent; corosync.conf occurrences in config.db: $(strings /var/lib/pve-cluster/config.db 2>/dev/null | grep -c '^corosync\.conf$')"
|
|
fi
|
|
echo
|
|
echo "--- corosync-cpgtool (pmxcfs joins dcdb/status CPG groups when clustered) ---"
|
|
corosync-cpgtool 2>&1
|
|
echo
|
|
echo "--- corosync.conf ---"
|
|
cat /etc/pve/corosync.conf 2>&1 || cat /etc/corosync/corosync.conf 2>&1
|
|
echo
|
|
echo "--- corosync-cfgtool -s ---"
|
|
corosync-cfgtool -s 2>&1
|
|
echo
|
|
echo "--- pvecm status ---"
|
|
pvecm status 2>&1
|
|
echo
|
|
echo "--- corosync service ---"
|
|
systemctl is-active corosync pve-cluster 2>&1
|
|
echo
|
|
echo "--- journalctl -u corosync (last 60) ---"
|
|
journalctl -u corosync -n 60 --no-pager 2>&1
|
|
echo
|
|
echo "--- journalctl -u pve-cluster (last 30) ---"
|
|
journalctl -u pve-cluster -n 30 --no-pager 2>&1
|
|
echo
|
|
echo "--- cluster task logs ---"
|
|
# "Cluster join aborted!" is generic; the reason is only in the task log.
|
|
find /var/log/pve/tasks -type f \( -name '*clusterjoin*' -o -name '*clustercreate*' \) \
|
|
-exec echo "== {} ==" \; -exec cat {} \; 2>&1 | tail -80
|
|
REMOTE
|
|
}
|
|
|
|
echo "=== Cluster diagnostics for PVE $VERSION ==="
|
|
node_a="$(jq -r ".pve${VERSION}.nodes.a.host // empty" "$CONFIG_FILE")"
|
|
node_b="$(jq -r ".pve${VERSION}.nodes.b.host // empty" "$CONFIG_FILE")"
|
|
|
|
dump_node "node A" "$node_a"
|
|
dump_node "node B" "$node_b"
|
|
|
|
echo
|
|
echo "=== End cluster diagnostics ==="
|
|
exit 0
|