mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
fix(e2e): kill the node-failure phase abruptly instead of draining it (#785)
* fix(e2e): kill the node-failure phase abruptly instead of draining it Phase 8 used `systemctl stop spinifex.target`, which now drains guests gracefully since the target-stop drain fix landed, so it tested an orderly shutdown rather than a failure. Since a target unit has no process of its own and cannot be signalled, and PartOf= only propagates stop/restart jobs rather than kill, SIGKILL each core service's main process directly instead, leaving orphaned guest qemu/nbdkit processes for recovery to reattach. Also fix a stale comment on spinifex-predastore.service's TimeoutStopSec: it claimed to match viperblock's drain budget, which is now 150s rather than 30s. The value was already correct; only the rationale was outdated. * fix(e2e): hold node2's outage open through Phase 8's assertions Restart=on-failure was respawning each killed unit within ~5s, so every Phase 8 assertion ran against an already-recovered node2 and DEGRADED_PEERS never actually dropped. Stop the same units right after the kill to cancel the queued restart and keep them down for the rest of the phase, and make the NATS degraded-peer count a real assertion (sized off NODE_COUNT) now that it can genuinely fail.
This commit is contained in:
Josh Siviour
@@ -1,4 +1,4 @@
|
||||
# spinifex-unit-version: 1
|
||||
# spinifex-unit-version: 2
|
||||
[Unit]
|
||||
Description=Spinifex Predastore (S3-compatible Object Storage)
|
||||
PartOf=spinifex.target
|
||||
@@ -18,9 +18,9 @@ ExecStartPre=/var/lib/spinifex/wait-for-nats.sh
|
||||
ExecStart=/usr/local/bin/spx service predastore start
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
# Matches viperblock's drain budget: viperblock's WAL flush writes into
|
||||
# predastore, and After= already stops viperblock first, but this is the
|
||||
# backstop if that ordering is ever bypassed (direct stop, future edit).
|
||||
# Bounds predastore's own graceful stop. viperblock's WAL flush writes into
|
||||
# predastore, but After= already stops viperblock first (within its own 150s
|
||||
# budget); this is the backstop if that ordering is ever bypassed.
|
||||
TimeoutStopSec=30
|
||||
OOMScoreAdjust=-500
|
||||
# Legitimately the most memory-hungry infra service (S3 buffering/caching);
|
||||
|
||||
@@ -15,7 +15,7 @@ var Units = map[string]string{
|
||||
"spinifex-nats-watchdog.timer": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS JetStream liveness watchdog timer\nPartOf=spinifex.target\n\n[Timer]\n# Delay the first probe so NATS startup + JetStream cluster quorum settle (a\n# forming cluster reports JS not-current transiently, which is not a fault).\nOnBootSec=60s\nOnUnitActiveSec=30s\nAccuracySec=5s\n\n[Install]\nWantedBy=timers.target\n",
|
||||
"spinifex-nats.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex NATS Message Bus\nPartOf=spinifex.target\nBefore=spinifex-predastore.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-nats\nGroup=spinifex\nExecStart=/usr/local/bin/spx service nats start\nRestart=on-failure\nRestartSec=5\n# nats-server installs its own SIGTERM handler (JetStream store shutdown, raft\n# stepdown, client drain) with no internal deadline; every dependent service\n# stops before nats does, so size this for nats's own graceful-stop needs.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/nats/nats.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/nats /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/nats /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-northstar.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Northstar (Authoritative DNS)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-predastore.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-northstar\nGroup=spinifex\nExecStart=/usr/local/bin/spx service northstar start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() is called with an internal 5s context; margin covers the\n# NATS unsubscribe/close and process exit that follow it.\nTimeoutStopSec=10\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/northstar/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/northstar /etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/northstar /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\n# RG-9: exactly CAP_NET_BIND_SERVICE so the unprivileged user binds :53 (no root).\nAmbientCapabilities=CAP_NET_BIND_SERVICE\nCapabilityBoundingSet=CAP_NET_BIND_SERVICE\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-predastore.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Matches viperblock's drain budget: viperblock's WAL flush writes into\n# predastore, and After= already stops viperblock first, but this is the\n# backstop if that ordering is ever bypassed (direct stop, future edit).\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-predastore.service": "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# Guest block I/O arrives with no caller trace; sample those roots at 5% so\n# they don't drown request traces. Request-linked spans are always kept.\nEnvironment=MULGA_ROOT_TRACE_RATIO=0.05\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Bounds predastore's own graceful stop. viperblock's WAL flush writes into\n# predastore, but After= already stops viperblock first (within its own 150s\n# budget); this is the backstop if that ordering is ever bypassed.\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-qmp-collector.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex QMP Telemetry Collector (guest metrics)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\n# spinifex-daemon user: the collector reads the daemon's telemetry QMP\n# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service qmp-collector start\nRestart=on-failure\nRestartSec=5\n# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown\n# is near-instant, so this only bounds an unexpectedly wedged process.\nTimeoutStopSec=10\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\n# Same runtime dir as the daemon: telemetry sockets + metadata live there\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: read-mostly telemetry tier — no capabilities, no device access\nProtectSystem=full\nPrivateDevices=yes\nNoNewPrivileges=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads\n# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=512\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-shutdown.service": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex graceful guest drain\nPartOf=spinifex.target\n# After= all services so it stops FIRST on target/host shutdown: the drain runs\n# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).\nAfter=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n# ExecStop fires on ANY target stop, including the stop half of `systemctl\n# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless\n# of whether the host is actually going down. --unless-restarting drains on a\n# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it\n# skips only when a restart of the stack is already queued, so QEMU/nbdkit\n# survive (KillMode=process) and the daemon reconnects on the way back up.\nExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting\nTimeoutStopSec=180\n\n[Install]\nWantedBy=spinifex.target\n",
|
||||
"spinifex-system.slice": "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n",
|
||||
|
||||
@@ -426,7 +426,10 @@ var unitBodyHashes = map[string]map[int]string{
|
||||
"spinifex-nats-watchdog.timer": {1: "9d43a8d2aa4fd80ab5f944d7620c666fd2a255850de2e6494479b8746922c979"},
|
||||
"spinifex-nats.service": {1: "f7f9900b95e364dc2575684a9bf16fa1f96a903a3a4aa773a34c04d761106838"},
|
||||
"spinifex-northstar.service": {1: "030c382aed20f4383acdb7df7a830d48dd76dc29be5965b595cc3842ae43a3f1"},
|
||||
"spinifex-predastore.service": {1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8"},
|
||||
"spinifex-predastore.service": {
|
||||
1: "d0f6415b8f0e6ff3c045f2d3ce2794c347bf141066d7e0bd85fcec48797854d8",
|
||||
2: "157e9a7683ac58760ad96679cad9f94121294f34d5cb668d1e586ba0686b4968",
|
||||
},
|
||||
"spinifex-qmp-collector.service": {1: "beb18e6dd9351901f19d992cb2f757fb0e0e4a4d986402ccdb0ebb0a449f225c"},
|
||||
"spinifex-shutdown.service": {1: "bcdc455916f35aa7494b2fe25e691339e8f1e22f031dfd9fd95203a9aa4bdaa4"},
|
||||
"spinifex-system.slice": {1: "ca450c2b28a8b13dd767957fa9469bd74bd222d7abed79945e83d564d5ce16dd"},
|
||||
|
||||
@@ -979,11 +979,20 @@ echo ""
|
||||
# ==========================================================================
|
||||
echo "Phase 8: Node Failure"
|
||||
echo "========================================"
|
||||
echo "Stopping services on node2 ($NODE2_IP) to simulate node failure..."
|
||||
echo "Abruptly killing spinifex services on node2 ($NODE2_IP) to simulate an unclean node failure..."
|
||||
|
||||
# Stop services on node2 only — SPINIFEX_FORCE_LOCAL_STOP prevents coordinated
|
||||
# cluster shutdown which would kill all nodes via NATS
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl stop spinifex.target" || {
|
||||
# spinifex.target has no process of its own and PartOf= only propagates
|
||||
# stop/restart jobs (not kill), so SIGKILL each unit directly. --kill-whom=main
|
||||
# hits only the service's own process, leaving qemu/nbdkit orphaned for reattach.
|
||||
NODE2_KILL_UNITS="spinifex-daemon.service spinifex-viperblock.service spinifex-predastore.service spinifex-nats.service spinifex-awsgw.service spinifex-vpcd.service spinifex-northstar.service spinifex-qmp-collector.service spinifex-ui.service"
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl kill -s SIGKILL --kill-whom=main $NODE2_KILL_UNITS" || {
|
||||
echo " WARNING: systemctl kill returned non-zero (may be expected)"
|
||||
}
|
||||
|
||||
# Cancels each unit's queued Restart=on-failure job so the outage holds rather
|
||||
# than self-healing within RestartSec=5. The main process is already dead, so
|
||||
# this settles the unit instead of draining it — spinifex.target stays untouched.
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl stop $NODE2_KILL_UNITS" || {
|
||||
echo " WARNING: systemctl stop returned non-zero (may be expected)"
|
||||
}
|
||||
|
||||
@@ -1017,15 +1026,21 @@ else
|
||||
echo " Node3 not present in this cluster — skipping node3 survival check"
|
||||
fi
|
||||
|
||||
# Check NATS degraded state (should have 1 route instead of 2)
|
||||
# Check NATS degraded state: node1 should see every peer except itself and
|
||||
# the now-dead node2. The stop above holds node2 down for the rest of this
|
||||
# phase, so this is a real assertion now, not a best-effort guess.
|
||||
EXPECTED_DEGRADED_PEERS=$((NODE_COUNT - 2))
|
||||
if [ "$EXPECTED_DEGRADED_PEERS" -lt 0 ]; then
|
||||
EXPECTED_DEGRADED_PEERS=0
|
||||
fi
|
||||
NATS_DEGRADED=$(curl -s "http://127.0.0.1:${NATS_MONITOR_PORT}/routez" 2>/dev/null)
|
||||
DEGRADED_PEERS=$(echo "$NATS_DEGRADED" | jq -r '[.routes[].remote_name] | unique | length' 2>/dev/null || echo "0")
|
||||
echo " NATS peers during failure: $DEGRADED_PEERS (expected: 1)"
|
||||
if [ "$DEGRADED_PEERS" -eq 1 ]; then
|
||||
echo " NATS peers during failure: $DEGRADED_PEERS (expected: $EXPECTED_DEGRADED_PEERS)"
|
||||
if [ "$DEGRADED_PEERS" -eq "$EXPECTED_DEGRADED_PEERS" ]; then
|
||||
pass_test "NATS degraded mode"
|
||||
else
|
||||
echo " WARNING: Expected 1 NATS peer during node2 failure, got $DEGRADED_PEERS"
|
||||
# Not fatal — NATS might take a moment to detect
|
||||
echo " ERROR: Expected $EXPECTED_DEGRADED_PEERS NATS peer(s) during node2 failure, got $DEGRADED_PEERS"
|
||||
fail_test "NATS degraded mode"
|
||||
fi
|
||||
|
||||
# Verify describe-instances still works from surviving nodes
|
||||
@@ -1048,6 +1063,8 @@ echo "Phase 9: Node Recovery"
|
||||
echo "========================================"
|
||||
echo "Restarting services on node2 ($NODE2_IP)..."
|
||||
|
||||
# Phase 8 stopped every unit it killed, so node2 has been genuinely down
|
||||
# since then; this start is the actual recovery, not a backstop.
|
||||
peer_ssh "$NODE2_IP" "sudo systemctl start spinifex.target" || {
|
||||
echo " ERROR: Failed to restart services on node2"
|
||||
fail_test "Node2 restart"
|
||||
|
||||
Reference in New Issue
Block a user