gsadmin/haproxy-openmanager
1
0
Fork 0
mirror of https://github.com/taylanbakircioglu/haproxy-openmanager.git synced 2026-09-21 10:03:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: prevent daemon crash-loop on exit 1 with retry-backoff mechanism

Browse Source 
- Replace immediate exit 1 with retry-with-backoff in all daemon mode paths
  (dependency checks: 5 retries, 30/60/90/120/150s; config file: 5 retries, 15/30/45/60/75s)
- Make socat missing non-fatal in daemon mode (agent continues without stats)
- Add SystemD StartLimitBurst=5/StartLimitIntervalSec=120 for new installs
- Add macOS launchd ThrottleInterval=30 for new installs
- Fix Linux daemon fallback to installer mode (would hang on interactive read)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
taylanbakircioglu
2026-02-16 19:03:30 +03:00
parent 93602fa65d
commit dc978f847d
2 changed files with 182 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/utils/agent_scripts/linux_install.sh
+92 -45
View File
@@ -260,38 +260,58 @@ fi
if [[ "$DAEMON_MODE" == "true" ]]; then
log "INFO" "DAEMON: Starting agent daemon with existing configuration..."
# Verify jq is available (required for config parsing in daemon mode)
if ! command -v jq &> /dev/null; then
log "ERROR" "DAEMON: jq is required but not found. Cannot parse agent configuration."
echo "FATAL: jq is not installed. Install jq and restart the agent service." >&2
exit 1
fi
# Verify critical dependencies with retry (prevents rapid restart loops)
# SystemD Restart=always would cause fast crash-loop if we exit immediately
_dep_retry=0
_dep_max=5
while ! command -v jq &> /dev/null || ! command -v curl &> /dev/null; do
_dep_retry=$((_dep_retry + 1))
_missing=""
command -v jq &> /dev/null || _missing="jq"
command -v curl &> /dev/null || _missing="${_missing:+$_missing }curl"
if [[ $_dep_retry -ge $_dep_max ]]; then
log "ERROR" "DAEMON: Required tools not found after $_dep_max retries: $_missing"
log "ERROR" "DAEMON: Install missing tools and restart: sudo systemctl restart haproxy-agent"
exit 1
fi
_wait=$((_dep_retry * 30))
log "WARN" "DAEMON: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..."
sleep $_wait
done
# Load existing configuration
CONFIG_FILE="/etc/haproxy-agent/config.json"
LOG_FILE="/var/log/haproxy-agent/agent.log"
PID_FILE="/var/run/haproxy-agent.pid"
# Load configuration from file
if [[ -f "$CONFIG_FILE" ]]; then
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE" 2>/dev/null)
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null)
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE" 2>/dev/null)
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE" 2>/dev/null)
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE" 2>/dev/null)
log "INFO" "DAEMON: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON (MacOS style)
log "INFO" "DAEMON: Going directly to daemon mode..."
# Jump to the daemon section at the end of the script
SKIP_TO_DAEMON=true
else
log "ERROR" "DAEMON: Configuration file not found: $CONFIG_FILE"
log "INFO" "DAEMON: Falling back to installer mode..."
DAEMON_MODE=false
fi
# Load configuration from file (with retry - config may be temporarily unavailable)
_cfg_retry=0
_cfg_max=5
while [[ ! -f "$CONFIG_FILE" ]]; do
_cfg_retry=$((_cfg_retry + 1))
if [[ $_cfg_retry -ge $_cfg_max ]]; then
log "ERROR" "DAEMON: Configuration file not found after $_cfg_max retries: $CONFIG_FILE"
log "ERROR" "DAEMON: Re-install the agent to recreate the configuration."
exit 1
fi
_wait=$((_cfg_retry * 15))
log "WARN" "DAEMON: Config not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..."
sleep $_wait
done
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE" 2>/dev/null)
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null)
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE" 2>/dev/null)
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE" 2>/dev/null)
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE" 2>/dev/null)
log "INFO" "DAEMON: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON (MacOS style)
log "INFO" "DAEMON: Going directly to daemon mode..."
# Jump to the daemon section at the end of the script
SKIP_TO_DAEMON=true
fi
# Configuration variables - MODIFY THESE IF NEEDED
@@ -922,24 +942,48 @@ find_binary() {
return 1
}
# Initialize global binary paths for daemon
CURL_BIN=$(find_binary curl)
if [[ -z "$CURL_BIN" ]]; then
echo "ERROR: curl not found in system" >&2
exit 1
fi
# Initialize global binary paths for daemon (with retry to prevent crash loops)
_dep_retry=0
_dep_max=5
while true; do
CURL_BIN=$(find_binary curl)
JQ_AVAILABLE=false
command -v jq &> /dev/null && JQ_AVAILABLE=true
if [[ -n "$CURL_BIN" ]] && [[ "$JQ_AVAILABLE" == "true" ]]; then
break
fi
_dep_retry=$((_dep_retry + 1))
_missing=""
[[ -z "$CURL_BIN" ]] && _missing="curl"
[[ "$JQ_AVAILABLE" != "true" ]] && _missing="${_missing:+$_missing }jq"
if [[ $_dep_retry -ge $_dep_max ]]; then
echo "ERROR: Required tools not found after $_dep_max retries: $_missing" >&2
echo "Install missing tools and restart the agent service." >&2
exit 1
fi
_wait=$((_dep_retry * 30))
echo "WARN: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..." >&2
sleep $_wait
done
# Verify jq is available (critical dependency for JSON config parsing)
if ! command -v jq &> /dev/null; then
echo "ERROR: jq not found. Install jq and restart the agent service." >&2
exit 1
fi
# Load configuration
if [[ ! -f "$CONFIG_FILE" ]]; then
echo "ERROR: Configuration not found: $CONFIG_FILE" >&2
exit 1
fi
# Load configuration (with retry - config may be temporarily unavailable after upgrade)
_cfg_retry=0
_cfg_max=5
while [[ ! -f "$CONFIG_FILE" ]]; do
_cfg_retry=$((_cfg_retry + 1))
if [[ $_cfg_retry -ge $_cfg_max ]]; then
echo "ERROR: Configuration not found after $_cfg_max retries: $CONFIG_FILE" >&2
echo "Re-install the agent to recreate the configuration." >&2
exit 1
fi
_wait=$((_cfg_retry * 15))
echo "WARN: Config file not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..." >&2
sleep $_wait
done
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
@@ -2089,8 +2133,7 @@ run_daemon() {
if [[ -n "$SOCAT_BIN" ]]; then
log "INFO" "Found socat: $SOCAT_BIN"
else
log "ERROR" "socat not found in system"
exit 1
log "WARN" "socat not found - HAProxy stats will be unavailable (agent will still function)"
fi
if [[ -n "$JQ_BIN" ]]; then
@@ -2300,6 +2343,10 @@ KillSignal=SIGTERM
Restart=always
RestartSec=10
# Prevent rapid restart loops: max 5 restarts within 2 minutes
# Allows multiple legitimate upgrades (exit 0 + restart) while catching crash loops
StartLimitBurst=5
StartLimitIntervalSec=120
StandardOutput=append:$LOG_DIR/agent.log
StandardError=append:$LOG_DIR/agent.log
Environment=SKIP_TO_DAEMON=true
backend/utils/agent_scripts/macos_install.sh
+90 -44
View File
@@ -109,37 +109,58 @@ log "DEBUG" "Config file exists: $(test -f "/etc/haproxy-agent/config.json" && e
if [[ "$1" == "daemon" && -f "/etc/haproxy-agent/config.json" ]] || [[ "$HAPROXY_AGENT_SKIP_SETUP" == "true" ]]; then
log "INFO" "Starting agent daemon with existing configuration..."
# Verify jq is available (required for config parsing in daemon mode)
if ! command -v jq &> /dev/null; then
log "ERROR" "DAEMON: jq is required but not found. Cannot parse agent configuration."
echo "FATAL: jq is not installed. Install jq (brew install jq) and restart the agent service." >&2
exit 1
fi
# Verify critical dependencies with retry (prevents rapid restart loops)
# launchd KeepAlive=true would cause fast crash-loop if we exit immediately
_dep_retry=0
_dep_max=5
while ! command -v jq &> /dev/null || ! command -v curl &> /dev/null; do
_dep_retry=$((_dep_retry + 1))
_missing=""
command -v jq &> /dev/null || _missing="jq"
command -v curl &> /dev/null || _missing="${_missing:+$_missing }curl"
if [[ $_dep_retry -ge $_dep_max ]]; then
log "ERROR" "DAEMON: Required tools not found after $_dep_max retries: $_missing"
log "ERROR" "DAEMON: Install missing tools: brew install $_missing"
exit 1
fi
_wait=$((_dep_retry * 30))
log "WARN" "DAEMON: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..."
sleep $_wait
done
# Use provided config file or default
CONFIG_FILE="${HAPROXY_AGENT_CONFIG_FILE:-/etc/haproxy-agent/config.json}"
LOG_FILE="/var/log/haproxy-agent/agent.log"
PID_FILE="/var/run/haproxy-agent.pid"
# Load configuration from file
if [[ -f "$CONFIG_FILE" ]]; then
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE")
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE")
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE")
log "INFO" "AGENT UPGRADE: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON
log "INFO" "AGENT UPGRADE: Going directly to daemon mode..."
# Jump to the daemon section at the end of the script
SKIP_TO_DAEMON=true
else
log "ERROR" "Configuration file not found: $CONFIG_FILE"
exit 1
fi
# Load configuration from file (with retry - config may be temporarily unavailable)
_cfg_retry=0
_cfg_max=5
while [[ ! -f "$CONFIG_FILE" ]]; do
_cfg_retry=$((_cfg_retry + 1))
if [[ $_cfg_retry -ge $_cfg_max ]]; then
log "ERROR" "DAEMON: Configuration file not found after $_cfg_max retries: $CONFIG_FILE"
log "ERROR" "DAEMON: Re-install the agent to recreate the configuration."
exit 1
fi
_wait=$((_cfg_retry * 15))
log "WARN" "DAEMON: Config not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..."
sleep $_wait
done
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE")
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE")
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE")
log "INFO" "AGENT UPGRADE: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON
log "INFO" "AGENT UPGRADE: Going directly to daemon mode..."
# Jump to the daemon section at the end of the script
SKIP_TO_DAEMON=true
fi
# Configuration variables - MODIFY THESE IF NEEDED
@@ -784,24 +805,48 @@ find_binary() {
return 1
}
# Initialize global binary paths for daemon
CURL_BIN=$(find_binary curl)
if [[ -z "$CURL_BIN" ]]; then
echo "ERROR: curl not found in system" >&2
exit 1
fi
# Initialize global binary paths for daemon (with retry to prevent crash loops)
_dep_retry=0
_dep_max=5
while true; do
CURL_BIN=$(find_binary curl)
JQ_AVAILABLE=false
command -v jq &> /dev/null && JQ_AVAILABLE=true
if [[ -n "$CURL_BIN" ]] && [[ "$JQ_AVAILABLE" == "true" ]]; then
break
fi
_dep_retry=$((_dep_retry + 1))
_missing=""
[[ -z "$CURL_BIN" ]] && _missing="curl"
[[ "$JQ_AVAILABLE" != "true" ]] && _missing="${_missing:+$_missing }jq"
if [[ $_dep_retry -ge $_dep_max ]]; then
echo "ERROR: Required tools not found after $_dep_max retries: $_missing" >&2
echo "Install missing tools (brew install $_missing) and restart the agent service." >&2
exit 1
fi
_wait=$((_dep_retry * 30))
echo "WARN: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..." >&2
sleep $_wait
done
# Verify jq is available (critical dependency for JSON config parsing)
if ! command -v jq &> /dev/null; then
echo "ERROR: jq not found. Install jq (brew install jq) and restart the agent service." >&2
exit 1
fi
# Load configuration
if [[ ! -f "$CONFIG_FILE" ]]; then
echo "ERROR: Configuration not found: $CONFIG_FILE" >&2
exit 1
fi
# Load configuration (with retry - config may be temporarily unavailable after upgrade)
_cfg_retry=0
_cfg_max=5
while [[ ! -f "$CONFIG_FILE" ]]; do
_cfg_retry=$((_cfg_retry + 1))
if [[ $_cfg_retry -ge $_cfg_max ]]; then
echo "ERROR: Configuration not found after $_cfg_max retries: $CONFIG_FILE" >&2
echo "Re-install the agent to recreate the configuration." >&2
exit 1
fi
_wait=$((_cfg_retry * 15))
echo "WARN: Config file not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..." >&2
sleep $_wait
done
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
@@ -1981,8 +2026,7 @@ run_daemon() {
if [[ -n "$SOCAT_BIN" ]]; then
log "INFO" "Found socat: $SOCAT_BIN"
else
log "ERROR" "socat not found in system"
exit 1
log "WARN" "socat not found - HAProxy stats will be unavailable (agent will still function)"
fi
if [[ -n "$JQ_BIN" ]]; then
@@ -2181,6 +2225,8 @@ cat > "/Library/LaunchDaemons/com.haproxy.agent.plist" << PLIST_EOF
<true/>
<key>KeepAlive</key>
<true/>
<key>ThrottleInterval</key>
<integer>30</integer>
<key>StandardOutPath</key>
<string>$LOG_DIR/agent.log</string>
<key>StandardErrorPath</key>