mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
fix: extra logging
This commit is contained in:
@@ -120,7 +120,7 @@ func (m *Manager) classifyRestoredInstances() []*VM {
|
||||
// when MarkRecoveryFailed fired on the previous daemon run.
|
||||
if instance.Status == StateError {
|
||||
slog.Warn("Instance in error state; skipping recovery relaunch (operator must retry or terminate)",
|
||||
"instance", instance.ID)
|
||||
"instance", instance.ID, "managedBy", instance.ManagedBy, "instanceType", instance.InstanceType)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -300,9 +300,11 @@ func (m *Manager) relaunchAll(toLaunch []*VM) {
|
||||
"instanceId", inst.ID, "status", string(status))
|
||||
return
|
||||
}
|
||||
slog.Info("Launching instance (recovery)", "instance", inst.ID)
|
||||
slog.Info("Launching instance (recovery)",
|
||||
"instance", inst.ID, "managedBy", inst.ManagedBy, "instanceType", inst.InstanceType)
|
||||
if err := m.Run(inst); err != nil {
|
||||
slog.Error("Failed to launch instance during recovery", "instanceId", inst.ID, "err", err)
|
||||
slog.Error("Failed to launch instance during recovery",
|
||||
"instanceId", inst.ID, "managedBy", inst.ManagedBy, "instanceType", inst.InstanceType, "err", err)
|
||||
m.MarkRecoveryFailed(inst, "recovery_launch_failed")
|
||||
}
|
||||
}(instance)
|
||||
|
||||
@@ -129,12 +129,22 @@ cleanup() {
|
||||
trap cleanup EXIT
|
||||
|
||||
dump_diagnostics() {
|
||||
log "--- spinifex service journals (last 100 lines each) ---"
|
||||
# Window from reboot onwards so recovery failure logs aren't scrolled out
|
||||
# by NATS warm-up / per-instance state-transition spam.
|
||||
local mode label
|
||||
if [ -n "${REBOOT_START:-}" ]; then
|
||||
mode="--since=@${REBOOT_START}"
|
||||
label="since reboot (@${REBOOT_START})"
|
||||
else
|
||||
mode="-n 200"
|
||||
label="last 200 lines each"
|
||||
fi
|
||||
log "--- spinifex service journals (${label}) ---"
|
||||
for svc in spinifex-nats spinifex-predastore spinifex-viperblock \
|
||||
spinifex-daemon spinifex-awsgw spinifex-vpcd \
|
||||
ovn-controller ovs-vswitchd; do
|
||||
echo "=== ${svc} ==="
|
||||
node_ssh "sudo journalctl -u ${svc} --no-pager -n 100 2>/dev/null || true"
|
||||
node_ssh "sudo journalctl -u ${svc} --no-pager ${mode} 2>/dev/null || true"
|
||||
done
|
||||
}
|
||||
|
||||
@@ -537,7 +547,12 @@ wait_for_targets_healthy "$ALB_TG_ARN" 2 "ALB post-reboot" "$LB_RECOVER_SECS" ||
|
||||
# 8.6: re-run the traffic burst
|
||||
echo ""
|
||||
log "Re-running traffic burst against same ALB..."
|
||||
PRE_BURST_FAILED=$FAILED
|
||||
run_http_burst "http://${ALB_PUBLIC_IP}:80" "ALB post-reboot"
|
||||
if [ "$FAILED" -gt "$PRE_BURST_FAILED" ]; then
|
||||
log "Post-reboot traffic failed — dumping journals from reboot onwards before cleanup overwrites them."
|
||||
dump_diagnostics
|
||||
fi
|
||||
|
||||
# 8.7: ovn-nbctl diff (this is where the known persistence bug lands)
|
||||
log "Diffing ovn-nbctl show against pre-reboot snapshot..."
|
||||
|
||||
Reference in New Issue
Block a user