mirror of
https://github.com/taylanbakircioglu/haproxy-openmanager.git
synced 2026-09-12 05:48:58 +00:00
taylanbakircioglu
64d42663cd
The Linux/macOS agent installer could abort during "pre-installation cleanup"
(terminal showed `Killing processes matching: haproxy-agent` then `Killed`,
returning to the prompt) when the install script's own command line contained
"haproxy-agent". The cleanup killed processes via `pgrep -f "$pattern"` starting
with the bare string "haproxy-agent", which also matched the running installer's
own command line and a sudo/PAM ancestor that the $$/$PPID self-exclusion did not
cover, so the installer terminated itself before installing.
- linux_install.sh / macos_install.sh: the cleanup kill loop now targets ONLY
the installed agent - "$INSTALL_DIR/haproxy-agent" (the daemon binary path) and
the agent service/label ("haproxy-agent.service" / "com.haproxy.agent") - never
the bare "haproxy-agent" substring. Neither pattern can match the installer's
own command line. The redundant bare pattern is dropped (the service is stopped
separately, and the binary-path pattern still catches a running daemon).
- frontend (AgentManagement.js): name the downloaded scripts
install-agent-<platform>.sh / uninstall-agent-<platform>.sh (matching the
backend's suggested filename) - defense in depth so this cannot resurface.
Installer-only change. The running agent and its privilege model are unchanged
(it runs as root for HAProxy reload, config writes, keepalived, and self-upgrade).
The cleanup runs only on a full interactive install (gated by SKIP_TO_DAEMON), so
daemon mode, self-upgrade, and config/version apply are unaffected. Both agent
scripts kept in sync. Scripts parse on bash 4.2-5.2; full backend suite green.
Addresses #31.
4006 lines
188 KiB
Bash
4006 lines
188 KiB
Bash
#!/bin/bash
|
|
# PRODUCTION FIX: Removed 'set -e' for stability
|
|
# 'set -e' causes agent to exit on any command failure
|
|
# In production, we prefer logging errors over agent crashes
|
|
|
|
# HAProxy Management Agent Installation Script for Linux
|
|
# Generated by HAProxy Management UI on: $(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
|
# Target Cluster ID: $CLUSTER_ID
|
|
# Script Build ID: 20250925-v1.0.2-ssl-deployment-ready
|
|
|
|
# Early variable definition for backup (before interactive mode)
|
|
HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}"
|
|
|
|
# Quick daemon mode detection (before installer UI)
|
|
# Suppress installer messages if running in daemon mode
|
|
# CRITICAL FIX: Only check config.json for daemon mode when NOT running interactively
|
|
# Previous bug: config.json from old install caused script to silently enter daemon mode
|
|
# when user ran ./install-agent.sh manually, producing no output at all
|
|
QUIET_MODE=false
|
|
if [[ "$1" == "daemon" || "$SKIP_TO_DAEMON" == "true" ]]; then
|
|
QUIET_MODE=true
|
|
elif [[ -f "/etc/haproxy-agent/config.json" ]] && [[ ! -t 0 ]]; then
|
|
# Config exists AND not running from terminal (systemd/non-interactive)
|
|
QUIET_MODE=true
|
|
fi
|
|
|
|
# Show installer header only in interactive installation mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo "=========================================="
|
|
echo "HAProxy Management Agent Installer v{{AGENT_VERSION}}"
|
|
echo "Linux Edition"
|
|
echo "With Version Management & Auto-Upgrade"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "IMPORTANT WARNING"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "After agent installation, ALL your existing frontend and backend"
|
|
echo "configurations will be removed to enable full management through"
|
|
echo "HAProxy OpenManager."
|
|
echo ""
|
|
echo "An automatic backup will be created: haproxy.cfg.initial-backup"
|
|
echo ""
|
|
echo "REQUIRED STEPS AFTER INSTALLATION:"
|
|
echo " 1. Login to HAProxy OpenManager interface"
|
|
echo " 2. Use the BULK IMPORT feature to restore your configuration"
|
|
echo ""
|
|
echo "IMPORTANT: After installation, ALL configuration changes MUST be"
|
|
echo " made through the HAProxy OpenManager interface only."
|
|
echo " Manual changes will be overwritten on the next update."
|
|
echo ""
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
# Create automatic backup BEFORE asking for confirmation
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
BACKUP_PATH="${HAPROXY_CONFIG_PATH}.initial-backup"
|
|
if [[ ! -f "$BACKUP_PATH" ]]; then
|
|
echo "Creating backup: $BACKUP_PATH"
|
|
cp "$HAPROXY_CONFIG_PATH" "$BACKUP_PATH" && echo "Backup created successfully" || echo "WARNING: Backup failed"
|
|
echo ""
|
|
fi
|
|
fi
|
|
|
|
read -p "Do you understand and wish to continue? (Y/N): " -n 1 -r
|
|
echo ""
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
echo "Installation cancelled."
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
echo "Proceeding with installation..."
|
|
echo ""
|
|
fi
|
|
|
|
# ============================================
|
|
# CRITICAL: Function Definitions (Must be at top for daemon mode)
|
|
# ============================================
|
|
|
|
# Dynamically locate required binaries
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Logging function - JSON format, single line with timestamp
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
local log_file="${LOG_FILE:-/var/log/haproxy-agent/agent.log}"
|
|
local agent_name="${AGENT_NAME:-unknown}"
|
|
|
|
# Ensure log directory exists
|
|
mkdir -p "$(dirname "$log_file")" 2>/dev/null || true
|
|
|
|
# Escape message for JSON (replace backslash, quotes, newlines, tabs)
|
|
local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ')
|
|
|
|
# Remove any control characters and emojis (keep only printable ASCII + basic UTF-8)
|
|
escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g')
|
|
|
|
# Write JSON log entry (single line) - ONLY to log file
|
|
echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$agent_name\",\"message\":\"$escaped_message\"}" >> "$log_file" 2>/dev/null || true
|
|
|
|
# DO NOT print to stdout in daemon mode (causes duplicate logs)
|
|
# Installer messages are shown via direct echo commands
|
|
}
|
|
|
|
# Get HAProxy service status
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
# Old get_server_statuses() removed - using new version with lazy init (see line ~900)
|
|
|
|
# Collect system information
|
|
collect_system_info() {
|
|
local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux")
|
|
local kernel_version=$(uname -r 2>/dev/null || echo "Unknown")
|
|
local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0")
|
|
local cpu_count=$(nproc 2>/dev/null || echo "1")
|
|
local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0")
|
|
local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0")
|
|
local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "")
|
|
local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "")
|
|
|
|
cat <<SYSTEM_INFO_EOF
|
|
"operating_system": "$os_info",
|
|
"kernel_version": "$kernel_version",
|
|
"uptime": $uptime_seconds,
|
|
"cpu_count": $cpu_count,
|
|
"memory_total": $memory_bytes,
|
|
"disk_space": $disk_bytes,
|
|
"ip_address": "$ip_address",
|
|
"network_interfaces": ["${network_interfaces//,/\",\"}"],
|
|
"capabilities": ["haproxy_management", "ssl_deployment", "config_reload", "systemd_service", "keepalived_management"]
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Send heartbeat
|
|
# Old send_heartbeat() removed - using new version with haproxy_stats_csv support (see line ~1174)
|
|
|
|
# Check for pending configuration requests
|
|
check_config_requests() {
|
|
local curl_bin="${CURL_BIN:-$(find_binary curl)}"
|
|
|
|
# Get pending config requests from backend
|
|
local response=$("$curl_bin" -k -s -X GET "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}")
|
|
|
|
# Check if there are pending requests
|
|
local pending_count=$(echo "$response" | jq -r '.pending_requests | length' 2>/dev/null)
|
|
|
|
if [[ "$pending_count" -gt 0 ]]; then
|
|
log "INFO" "CONFIG: Found $pending_count pending config request(s)"
|
|
|
|
# Process each request
|
|
echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do
|
|
local request_id=$(echo "$request" | jq -r '.request_id')
|
|
local request_type=$(echo "$request" | jq -r '.request_type')
|
|
|
|
log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)"
|
|
|
|
# Read haproxy.cfg content
|
|
local config_path="${HAPROXY_CONFIG_PATH:-{{HAPROXY_CONFIG_PATH}}}"
|
|
if [[ -f "$config_path" ]]; then
|
|
local config_content=$(cat "$config_path")
|
|
local file_size=$(wc -c < "$config_path")
|
|
|
|
# Escape config content for JSON
|
|
local escaped_content=$(echo "$config_content" | jq -Rs .)
|
|
|
|
# Submit config response
|
|
local response_payload=$(cat <<CONFIG_RESPONSE_EOF
|
|
{
|
|
"request_id": $request_id,
|
|
"config_content": $escaped_content,
|
|
"config_path": "$config_path"
|
|
}
|
|
CONFIG_RESPONSE_EOF
|
|
)
|
|
|
|
"$curl_bin" -k -s -X POST "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/config-response" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}" \
|
|
-d "$response_payload" > /dev/null 2>&1
|
|
|
|
log "INFO" "Config response sent for request #$request_id (size: $file_size bytes)"
|
|
else
|
|
log "ERROR" "Config file not found: $config_path"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
# ============================================
|
|
# End of Function Definitions
|
|
# ============================================
|
|
|
|
# CRITICAL FIX: Enhanced daemon mode detection for Linux
|
|
# Check multiple conditions for daemon mode
|
|
DAEMON_MODE=false
|
|
|
|
# Method 1: Explicit daemon argument
|
|
if [[ "$1" == "daemon" ]]; then
|
|
log "INFO" "DAEMON: Explicit daemon argument detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
# Method 2: Environment variable from systemd
|
|
if [[ "$SKIP_TO_DAEMON" == "true" ]]; then
|
|
log "INFO" "DAEMON: Environment variable SKIP_TO_DAEMON=true detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
# Method 3: Config file exists (upgrade scenario)
|
|
# CRITICAL FIX: Only enter daemon mode from config.json when NOT running interactively
|
|
# If user runs ./install-agent.sh from terminal with old config.json present,
|
|
# we should show the installer, not silently enter daemon mode
|
|
if [[ -f "/etc/haproxy-agent/config.json" ]] && [[ ! -t 0 ]]; then
|
|
log "INFO" "DAEMON: Existing config file detected (non-interactive mode)"
|
|
DAEMON_MODE=true
|
|
elif [[ -f "/etc/haproxy-agent/config.json" ]] && [[ -t 0 ]]; then
|
|
log "INFO" "INSTALLER: Old config.json found but running interactively - showing installer"
|
|
echo ""
|
|
echo "WARNING: Previous agent installation detected!"
|
|
echo " Config file found: /etc/haproxy-agent/config.json"
|
|
echo ""
|
|
echo " The old agent configuration will be cleaned up during installation."
|
|
echo ""
|
|
fi
|
|
|
|
# Method 4: Legacy environment variable
|
|
if [[ "$HAPROXY_AGENT_SKIP_SETUP" == "true" ]]; then
|
|
log "INFO" "DAEMON: Legacy skip setup detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
if [[ "$DAEMON_MODE" == "true" ]]; then
|
|
log "INFO" "DAEMON: Starting agent daemon with existing configuration..."
|
|
|
|
# Verify critical dependencies with retry (prevents rapid restart loops)
|
|
# SystemD Restart=always would cause fast crash-loop if we exit immediately
|
|
_dep_retry=0
|
|
_dep_max=5
|
|
while ! command -v jq &> /dev/null || ! command -v curl &> /dev/null; do
|
|
_dep_retry=$((_dep_retry + 1))
|
|
_missing=""
|
|
command -v jq &> /dev/null || _missing="jq"
|
|
command -v curl &> /dev/null || _missing="${_missing:+$_missing }curl"
|
|
if [[ $_dep_retry -ge $_dep_max ]]; then
|
|
log "ERROR" "DAEMON: Required tools not found after $_dep_max retries: $_missing"
|
|
log "ERROR" "DAEMON: Install missing tools and restart: sudo systemctl restart haproxy-agent"
|
|
exit 1
|
|
fi
|
|
_wait=$((_dep_retry * 30))
|
|
log "WARN" "DAEMON: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..."
|
|
sleep $_wait
|
|
done
|
|
|
|
# Load existing configuration
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/var/run/haproxy-agent.pid"
|
|
|
|
# Load configuration from file (with retry - config may be temporarily unavailable)
|
|
_cfg_retry=0
|
|
_cfg_max=5
|
|
while [[ ! -f "$CONFIG_FILE" ]]; do
|
|
_cfg_retry=$((_cfg_retry + 1))
|
|
if [[ $_cfg_retry -ge $_cfg_max ]]; then
|
|
log "ERROR" "DAEMON: Configuration file not found after $_cfg_max retries: $CONFIG_FILE"
|
|
log "ERROR" "DAEMON: Re-install the agent to recreate the configuration."
|
|
exit 1
|
|
fi
|
|
_wait=$((_cfg_retry * 15))
|
|
log "WARN" "DAEMON: Config not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..."
|
|
sleep $_wait
|
|
done
|
|
|
|
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE" 2>/dev/null)
|
|
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null)
|
|
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE" 2>/dev/null)
|
|
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE" 2>/dev/null)
|
|
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE" 2>/dev/null)
|
|
|
|
log "INFO" "DAEMON: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
|
|
|
|
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON (MacOS style)
|
|
log "INFO" "DAEMON: Going directly to daemon mode..."
|
|
|
|
# Jump to the daemon section at the end of the script
|
|
SKIP_TO_DAEMON=true
|
|
fi
|
|
|
|
# Configuration variables - MODIFY THESE IF NEEDED
|
|
MANAGEMENT_URL="{{MANAGEMENT_URL}}"
|
|
CLUSTER_ID="{{CLUSTER_ID}}"
|
|
AGENT_NAME="{{AGENT_NAME}}"
|
|
HOSTNAME_PREFIX="{{HOSTNAME_PREFIX}}"
|
|
HOSTNAME="${HOSTNAME_PREFIX}-$(hostname -s)"
|
|
HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}"
|
|
HAPROXY_BIN_PATH="{{HAPROXY_BIN_PATH}}"
|
|
STATS_SOCKET_PATH="{{STATS_SOCKET_PATH}}"
|
|
HAPROXY_SERVICE_NAME="haproxy"
|
|
|
|
# Verify curl is available before any network operations
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
if ! command -v curl &> /dev/null; then
|
|
log "ERROR" "curl is not installed. Attempting to install..."
|
|
CURL_INSTALLED=false
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get update -qq && apt-get install -y -qq curl && CURL_INSTALLED=true
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y -q curl && CURL_INSTALLED=true
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y -q curl && CURL_INSTALLED=true
|
|
elif command -v apk &> /dev/null; then
|
|
apk add --no-cache curl && CURL_INSTALLED=true
|
|
fi
|
|
if [[ "$CURL_INSTALLED" != "true" ]] || ! command -v curl &> /dev/null; then
|
|
log "ERROR" "Failed to install curl automatically."
|
|
echo ""
|
|
echo " curl is required for this agent to communicate with the management server."
|
|
echo " Please install it manually and re-run this script."
|
|
exit 1
|
|
fi
|
|
log "INFO" "curl installed successfully"
|
|
fi
|
|
fi
|
|
|
|
# Skip authentication if we already have token from config (daemon mode)
|
|
if [[ "$SKIP_TO_DAEMON" == "true" && -n "$AGENT_TOKEN" ]]; then
|
|
log "DEBUG" "AGENT UPGRADE: Skipping authentication - using token from config"
|
|
else
|
|
# SECURE AGENT AUTHENTICATION
|
|
echo ""
|
|
log "INFO" "Agent Authentication Setup"
|
|
echo "============================================"
|
|
echo "This agent requires a secure API token for authentication."
|
|
echo "Please obtain the API token from the Security page in the"
|
|
echo "HAProxy Management UI before proceeding."
|
|
echo ""
|
|
echo "Steps to get your API token:"
|
|
echo "1. Open HAProxy Management UI"
|
|
echo "2. Go to Security page"
|
|
echo "3. Create a new agent token for: $AGENT_NAME"
|
|
echo "4. Copy the generated token"
|
|
echo ""
|
|
|
|
# Prompt for API token with validation
|
|
while true; do
|
|
echo -n "Enter Agent API Token: "
|
|
read -r AGENT_TOKEN
|
|
|
|
if [[ -z "$AGENT_TOKEN" ]]; then
|
|
log "ERROR" "API token cannot be empty"
|
|
continue
|
|
fi
|
|
|
|
# Validate token format (should start with 'hap_')
|
|
if [[ ! "$AGENT_TOKEN" =~ ^hap_.+ ]]; then
|
|
log "ERROR" "Invalid token format. Token should start with 'hap_'"
|
|
echo " Please ensure you copied the complete token from the Security page."
|
|
continue
|
|
fi
|
|
|
|
# Test token validity by making a test API call
|
|
log "DEBUG" "Validating API token..."
|
|
TEST_RESPONSE=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-H "User-Agent: haproxy-agent-installer" 2>/dev/null || echo "FAILED")
|
|
|
|
if [[ "$TEST_RESPONSE" == "FAILED" ]] || [[ ! "$TEST_RESPONSE" =~ clusters ]]; then
|
|
log "ERROR" "API token validation failed"
|
|
echo " Please check:"
|
|
echo " - Token is correct and complete"
|
|
echo " - Token has not been revoked"
|
|
echo " - Management URL is accessible: $MANAGEMENT_URL"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "API token validated successfully"
|
|
break
|
|
done
|
|
|
|
fi
|
|
|
|
# Skip installer output in quiet mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo ""
|
|
fi
|
|
|
|
# Dynamically locate required binaries (global scope)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
CURL_BIN=$(find_binary curl)
|
|
if [[ -z "$CURL_BIN" ]]; then
|
|
log "ERROR" "curl not found in system"
|
|
exit 1
|
|
fi
|
|
|
|
# Installation paths for Linux
|
|
INSTALL_DIR="/usr/local/bin"
|
|
CONFIG_DIR="/etc/haproxy-agent"
|
|
LOG_DIR="/var/log/haproxy-agent"
|
|
|
|
# Skip cleanup if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# CRITICAL: Clean installation - Remove existing agent components
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo ""
|
|
echo "Performing pre-installation cleanup..."
|
|
fi
|
|
|
|
# Function to safely remove files/directories
|
|
safe_remove() {
|
|
local path="$1"
|
|
local desc="$2"
|
|
|
|
if [[ -e "$path" ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Removing $desc..."
|
|
rm -rf "$path" 2>/dev/null || true
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " $desc removed"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# 1. Stop and kill all existing agent processes
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "Terminating existing HAProxy Agent processes..."
|
|
KILLED_COUNT=0
|
|
INSTALLER_PID=$$
|
|
# issue #31: match ONLY the installed agent (binary path + service), never the bare string
|
|
# "haproxy-agent". With pgrep -f, that bare string can also match the installer's OWN command line
|
|
# or a sudo/PAM ancestor (which the $$/$PPID guard does not fully cover), making the cleanup kill
|
|
# the installer itself ("Killed", install aborts). The systemd service is also stopped below; the
|
|
# "$INSTALL_DIR/haproxy-agent" path still catches any running daemon.
|
|
for pattern in "$INSTALL_DIR/haproxy-agent" "haproxy-agent.service"; do
|
|
PIDS=$(pgrep -f "$pattern" 2>/dev/null || true)
|
|
if [[ -n "$PIDS" ]]; then
|
|
FILTERED=""
|
|
for p in $PIDS; do
|
|
[[ "$p" == "$INSTALLER_PID" || "$p" == "$PPID" ]] && continue
|
|
FILTERED="$FILTERED $p"
|
|
done
|
|
FILTERED=$(echo "$FILTERED" | xargs)
|
|
if [[ -n "$FILTERED" ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Killing processes matching: $pattern"
|
|
kill -TERM $FILTERED 2>/dev/null || true
|
|
sleep 2
|
|
kill -KILL $FILTERED 2>/dev/null || true
|
|
KILLED_COUNT=$((KILLED_COUNT + $(echo "$FILTERED" | wc -w)))
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [[ $KILLED_COUNT -gt 0 ]]; then
|
|
log "INFO" "Terminated $KILLED_COUNT existing agent processes"
|
|
else
|
|
log "INFO" "No existing agent processes found"
|
|
fi
|
|
|
|
# 2. Stop and disable existing systemd services
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "Cleaning up existing systemd services..."
|
|
if systemctl is-enabled haproxy-agent >/dev/null 2>&1; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Stopping and disabling haproxy-agent service"
|
|
systemctl stop haproxy-agent 2>/dev/null || true
|
|
systemctl disable haproxy-agent 2>/dev/null || true
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Service haproxy-agent cleaned up"
|
|
fi
|
|
|
|
# 3. Remove existing files and directories
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "Removing existing agent files..."
|
|
safe_remove "/etc/systemd/system/haproxy-agent.service" "systemd service file"
|
|
safe_remove "/usr/local/bin/haproxy-agent" "agent binary"
|
|
safe_remove "/etc/haproxy-agent" "configuration directory"
|
|
safe_remove "/var/log/haproxy-agent" "log directory"
|
|
safe_remove "/var/run/haproxy-agent.pid" "PID file"
|
|
|
|
# Clean up temporary config files
|
|
TEMP_COUNT=0
|
|
for temp_file in /tmp/haproxy_new_*.cfg; do
|
|
if [[ -f "$temp_file" ]]; then
|
|
rm -f "$temp_file"
|
|
TEMP_COUNT=$((TEMP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $TEMP_COUNT -gt 0 ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Removed $TEMP_COUNT temporary config files"
|
|
fi
|
|
|
|
log "INFO" "Existing agent components cleaned up"
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "Pre-installation cleanup completed"
|
|
[[ "$QUIET_MODE" != "true" ]] && echo ""
|
|
|
|
fi # End of cleanup section
|
|
|
|
# Detect architecture
|
|
ARCH=$(uname -m)
|
|
case "$ARCH" in
|
|
"x86_64") ARCH_SUFFIX="linux-amd64"; [[ "$QUIET_MODE" != "true" ]] && echo "Detected: Linux AMD64" ;;
|
|
"aarch64"|"arm64") ARCH_SUFFIX="linux-arm64"; [[ "$QUIET_MODE" != "true" ]] && echo "Detected: Linux ARM64" ;;
|
|
*) ARCH_SUFFIX="linux-unknown"; log "WARN" "Unknown architecture: $ARCH" ;;
|
|
esac
|
|
|
|
# Display configuration only in interactive mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo "Agent Configuration:"
|
|
echo " Management URL: $MANAGEMENT_URL"
|
|
echo " Cluster ID: $CLUSTER_ID"
|
|
echo " Agent Name: $AGENT_NAME"
|
|
echo " Hostname: $HOSTNAME"
|
|
echo " Architecture: $ARCH ($ARCH_SUFFIX)"
|
|
echo ""
|
|
echo "HAProxy Integration:"
|
|
echo " Config Path: $HAPROXY_CONFIG_PATH"
|
|
echo " Stats Socket: $STATS_SOCKET_PATH"
|
|
echo " Service Name: $HAPROXY_SERVICE_NAME"
|
|
echo ""
|
|
fi
|
|
|
|
# Only show agent capabilities during initial installation, not on daemon restart
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
log "INFO" "This agent will:"
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo " • Monitor and report HAProxy service status"
|
|
echo " • Receive configuration updates from management UI"
|
|
echo " • Apply changes to HAProxy config file: $HAPROXY_CONFIG_PATH"
|
|
echo " • Manage HAProxy service lifecycle (restart/reload)"
|
|
echo " • Send heartbeats every 10 seconds"
|
|
echo ""
|
|
fi
|
|
else
|
|
# Daemon mode - reduced logging for restart scenarios
|
|
log "DEBUG" "AGENT UPGRADE: Skipping entire installation - going to daemon mode"
|
|
fi
|
|
|
|
# Check root permissions
|
|
if [[ "$SKIP_TO_DAEMON" != "true" && $EUID -ne 0 ]]; then
|
|
log "WARN" "This script needs root privileges for agent service installation."
|
|
echo " Please run with: sudo $0"
|
|
exit 1
|
|
fi
|
|
|
|
# Skip validation if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
# CRITICAL: Ensure jq is available before cluster validation (which requires it)
|
|
if ! command -v jq &> /dev/null; then
|
|
log "WARN" "jq is not installed. Attempting to install..."
|
|
JQ_INSTALLED=false
|
|
|
|
# Detect package manager and install jq
|
|
if command -v apt-get &> /dev/null; then
|
|
log "INFO" "Detected apt-get (Debian/Ubuntu)"
|
|
if apt-get update -qq && apt-get install -y -qq jq; then
|
|
JQ_INSTALLED=true
|
|
fi
|
|
elif command -v dnf &> /dev/null; then
|
|
log "INFO" "Detected dnf (RHEL/Fedora/CentOS 8+)"
|
|
if dnf install -y -q jq; then
|
|
JQ_INSTALLED=true
|
|
fi
|
|
elif command -v yum &> /dev/null; then
|
|
log "INFO" "Detected yum (RHEL/CentOS 7)"
|
|
if yum install -y -q jq; then
|
|
JQ_INSTALLED=true
|
|
fi
|
|
elif command -v zypper &> /dev/null; then
|
|
log "INFO" "Detected zypper (SUSE/openSUSE)"
|
|
if zypper --non-interactive install -y jq; then
|
|
JQ_INSTALLED=true
|
|
fi
|
|
elif command -v apk &> /dev/null; then
|
|
log "INFO" "Detected apk (Alpine Linux)"
|
|
if apk add --no-cache jq; then
|
|
JQ_INSTALLED=true
|
|
fi
|
|
fi
|
|
|
|
if [[ "$JQ_INSTALLED" != "true" ]] || ! command -v jq &> /dev/null; then
|
|
log "ERROR" "Failed to install jq automatically."
|
|
echo ""
|
|
echo " jq is required for this agent to function."
|
|
echo " Please install it manually using one of the following commands:"
|
|
echo ""
|
|
echo " Debian/Ubuntu: sudo apt-get install -y jq"
|
|
echo " RHEL/CentOS 8+: sudo dnf install -y jq"
|
|
echo " RHEL/CentOS 7: sudo yum install -y jq"
|
|
echo " SUSE/openSUSE: sudo zypper install -y jq"
|
|
echo " Alpine Linux: sudo apk add jq"
|
|
echo ""
|
|
echo " After installing jq, re-run this script."
|
|
exit 1
|
|
fi
|
|
log "INFO" "jq installed successfully: $(jq --version)"
|
|
else
|
|
log "INFO" "jq found: $(jq --version)"
|
|
fi
|
|
|
|
# Validate cluster exists by checking management API
|
|
log "DEBUG" "Validating HAProxy Cluster..."
|
|
CLUSTER_CHECK=$("$CURL_BIN" -k -s -f "$MANAGEMENT_URL/api/clusters" -H "X-API-Key: $AGENT_TOKEN" -H "User-Agent: haproxy-agent-installer" || echo "FAILED")
|
|
if [[ "$CLUSTER_CHECK" == "FAILED" ]]; then
|
|
log "ERROR" "Failed to connect to management API!"
|
|
echo " Please check your network connection and management URL."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if our specific cluster ID exists in the response
|
|
if ! echo "$CLUSTER_CHECK" | jq -e ".clusters[]? | select(.id==$CLUSTER_ID)" >/dev/null; then
|
|
log "ERROR" "HAProxy Cluster (ID: $CLUSTER_ID) not found!"
|
|
echo " Available clusters:"
|
|
echo "$CLUSTER_CHECK" | jq -r '.clusters[] | " - \(.id): \(.name)"'
|
|
echo " Please ensure the cluster exists in the management UI before running this script."
|
|
exit 1
|
|
fi
|
|
log "INFO" "Cluster validation successful"
|
|
fi
|
|
|
|
# Check dependencies
|
|
check_dependencies() {
|
|
log "DEBUG" "Checking system requirements..."
|
|
|
|
# Check if HAProxy is installed
|
|
if ! command -v haproxy &> /dev/null; then
|
|
log "WARN" "HAProxy not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get update && apt-get install -y haproxy
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y haproxy
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y haproxy
|
|
else
|
|
log "ERROR" "Cannot install HAProxy automatically. Please install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "HAProxy found: $(haproxy -v | head -1)"
|
|
|
|
# CRITICAL: Validate existing HAProxy config
|
|
# Agent is designed to manage an EXISTING HAProxy installation with global+defaults sections
|
|
log "DEBUG" "Validating existing HAProxy configuration..."
|
|
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Check if config has global section
|
|
if ! grep -q "^global" "$HAPROXY_CONFIG_PATH"; then
|
|
log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'global' section!"
|
|
echo ""
|
|
echo "INSTALLATION REQUIREMENT NOT MET"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Your HAProxy config must contain:"
|
|
echo " • global section (with logging, stats socket, etc.)"
|
|
echo " • defaults section (with timeouts, mode, etc.)"
|
|
echo ""
|
|
echo "The agent will:"
|
|
echo " PRESERVE your global and defaults sections"
|
|
echo " MANAGE only frontends and backends"
|
|
echo ""
|
|
echo "Current config: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "Please ensure HAProxy is properly configured before installing agent."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if config has defaults section
|
|
if ! grep -q "^defaults" "$HAPROXY_CONFIG_PATH"; then
|
|
log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'defaults' section!"
|
|
echo ""
|
|
echo "INSTALLATION REQUIREMENT NOT MET"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Your HAProxy config must contain:"
|
|
echo " • global section (with logging, stats socket, etc.)"
|
|
echo " • defaults section (with timeouts, mode, etc.)"
|
|
echo ""
|
|
echo "The agent will:"
|
|
echo " PRESERVE your global and defaults sections"
|
|
echo " MANAGE only frontends and backends"
|
|
echo ""
|
|
echo "Current config: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "Please ensure HAProxy is properly configured before installing agent."
|
|
exit 1
|
|
fi
|
|
|
|
log "INFO" "Existing HAProxy config validated (global+defaults sections present)"
|
|
else
|
|
log "WARN" "HAProxy config not found at $HAPROXY_CONFIG_PATH"
|
|
log "WARN" "Agent expects to manage an existing HAProxy installation"
|
|
echo ""
|
|
echo " WARNING: HAProxy config not found"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Config file expected at: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "The agent will create a new config if you continue, but this is NOT recommended."
|
|
echo "It's better to set up HAProxy first with proper global/defaults sections."
|
|
echo ""
|
|
read -p "Continue anyway? (yes/no): " -r CONTINUE_WITHOUT_CONFIG
|
|
if [[ "$CONTINUE_WITHOUT_CONFIG" != "yes" ]]; then
|
|
log "INFO" "Installation cancelled by user"
|
|
exit 0
|
|
fi
|
|
log "WARN" "User chose to continue without existing HAProxy config"
|
|
fi
|
|
|
|
# Check if config directory exists
|
|
HAPROXY_DIR=$(dirname "$HAPROXY_CONFIG_PATH")
|
|
if [[ ! -d "$HAPROXY_DIR" ]]; then
|
|
echo "Creating HAProxy config directory: $HAPROXY_DIR"
|
|
mkdir -p "$HAPROXY_DIR"
|
|
fi
|
|
|
|
# Check if stats socket directory exists
|
|
STATS_DIR=$(dirname "$STATS_SOCKET_PATH")
|
|
if [[ ! -d "$STATS_DIR" ]]; then
|
|
echo "Creating HAProxy stats socket directory: $STATS_DIR"
|
|
mkdir -p "$STATS_DIR"
|
|
fi
|
|
|
|
# Verify jq is available (should already be installed from early check)
|
|
if ! command -v jq &> /dev/null; then
|
|
log "ERROR" "jq is required but not installed. Please install jq and re-run this script."
|
|
exit 1
|
|
fi
|
|
log "INFO" "jq found: $(jq --version)"
|
|
|
|
# Check if socat is installed (required for HAProxy stats socket)
|
|
if ! command -v socat &> /dev/null; then
|
|
log "WARN" "socat not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get install -y socat
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y socat
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y socat
|
|
else
|
|
log "ERROR" "Cannot install socat automatically. Please install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "socat found: $(socat -V | head -1)"
|
|
|
|
# Check if openssl is installed (required for SSL certificate deployment)
|
|
if ! command -v openssl &> /dev/null; then
|
|
log "WARN" "openssl not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get install -y -qq openssl
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y -q openssl
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y -q openssl
|
|
elif command -v apk &> /dev/null; then
|
|
apk add --no-cache openssl
|
|
else
|
|
log "WARN" "Cannot install openssl automatically. SSL certificate features may not work."
|
|
fi
|
|
fi
|
|
if command -v openssl &> /dev/null; then
|
|
log "INFO" "openssl found: $(openssl version)"
|
|
fi
|
|
}
|
|
|
|
# Skip dependency check if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
check_dependencies
|
|
fi
|
|
|
|
# Skip final cleanup if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# CRITICAL: Final cleanup as root (clean any remaining files)
|
|
echo ""
|
|
echo "Final cleanup as root (removing any remaining files)..."
|
|
safe_remove "/etc/systemd/system/haproxy-agent.service" "existing systemd service file"
|
|
safe_remove "/usr/local/bin/haproxy-agent" "existing agent binary"
|
|
safe_remove "/etc/haproxy-agent" "existing configuration directory"
|
|
safe_remove "/var/log/haproxy-agent" "existing log directory"
|
|
safe_remove "/var/run/haproxy-agent.pid" "existing PID file"
|
|
safe_remove "/tmp/haproxy-agent.pid" "temp PID file"
|
|
|
|
# Clean up HAProxy config backups
|
|
BACKUP_COUNT=0
|
|
for backup_file in /etc/haproxy/haproxy.cfg.backup.*; do
|
|
if [[ -f "$backup_file" ]]; then
|
|
rm -f "$backup_file"
|
|
BACKUP_COUNT=$((BACKUP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $BACKUP_COUNT -gt 0 ]]; then
|
|
echo " Removed $BACKUP_COUNT HAProxy config backups"
|
|
fi
|
|
|
|
# Clean up any remaining temp files
|
|
TEMP_COUNT=0
|
|
for temp_file in /tmp/haproxy_new_*.cfg /tmp/*agent*.log; do
|
|
if [[ -f "$temp_file" ]]; then
|
|
rm -f "$temp_file"
|
|
TEMP_COUNT=$((TEMP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $TEMP_COUNT -gt 0 ]]; then
|
|
echo " Removed $TEMP_COUNT temporary files"
|
|
fi
|
|
|
|
log "INFO" "Final cleanup completed - Ready for fresh installation"
|
|
echo ""
|
|
|
|
fi # End of final cleanup section
|
|
|
|
# Skip directory creation if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# Create directories with proper permissions for upgrades
|
|
echo "Creating directories..."
|
|
mkdir -p "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR"
|
|
|
|
# Set comprehensive permissions for upgrade operations
|
|
log "INFO" "Setting permissions for upgrade operations..."
|
|
|
|
# Ensure agent binary directory is writable (for in-place upgrades)
|
|
chmod 755 "$INSTALL_DIR"
|
|
chown root:root "$INSTALL_DIR" 2>/dev/null || true
|
|
|
|
# Ensure config directory is accessible
|
|
chmod 755 "$CONFIG_DIR"
|
|
chown root:root "$CONFIG_DIR" 2>/dev/null || true
|
|
|
|
# Ensure log directory is writable
|
|
chmod 755 "$LOG_DIR"
|
|
chown root:root "$LOG_DIR" 2>/dev/null || true
|
|
|
|
# Create and set permissions for temp directory (upgrade markers)
|
|
# Agent will run as root, so ensure /tmp is accessible
|
|
mkdir -p /tmp
|
|
chmod 1777 /tmp 2>/dev/null || true # Sticky bit for shared temp usage
|
|
|
|
# Create agent-specific temp directory for upgrade markers
|
|
AGENT_TEMP_DIR="/tmp/haproxy-agent"
|
|
mkdir -p "$AGENT_TEMP_DIR"
|
|
chmod 755 "$AGENT_TEMP_DIR"
|
|
chown root:root "$AGENT_TEMP_DIR" 2>/dev/null || true
|
|
|
|
# Test upgrade marker creation/removal capabilities
|
|
touch "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "Upgrade marker permissions verified"
|
|
else
|
|
log "WARN" "Warning: Upgrade marker permissions may be restricted"
|
|
fi
|
|
|
|
# Ensure agent binary will have execution permissions for all operations
|
|
log "INFO" "Configuring comprehensive agent permissions..."
|
|
|
|
log "INFO" "Upgrade permissions configured"
|
|
|
|
# ╔═══════════════════════════════════════════════════════════════════════════╗
|
|
# ║ WARNING: EMBEDDED AGENT DAEMON SCRIPT STARTS HERE (Line ~755) ║
|
|
# ║ ║
|
|
# ║ This heredoc contains the COMPLETE agent daemon that will be installed ║
|
|
# ║ to /usr/local/bin/haproxy-agent and run as a systemd service. ║
|
|
# ║ ║
|
|
# ║ IMPORTANT: When updating agent functionality (heartbeat, config, ║
|
|
# ║ SSL, upgrade, etc.), you MUST update BOTH: ║
|
|
# ║ 1. This embedded daemon (Line 755-1999) ║
|
|
# ║ 2. The daemon functions below (Line 2000+) ║
|
|
# ║ ║
|
|
# ║ Common functions to update in BOTH places: ║
|
|
# ║ • send_heartbeat() - Line ~1099 (embedded) & ~2367 (installer) ║
|
|
# ║ • check_config_updates() - Multiple locations ║
|
|
# ║ • check_agent_upgrade() - Multiple locations ║
|
|
# ║ • SSL deployment functions ║
|
|
# ║ ║
|
|
# ║ TIP: Search for function name to find all occurrences! ║
|
|
# ╚═══════════════════════════════════════════════════════════════════════════╝
|
|
|
|
# Create agent service script (similar to macOS but with Linux-specific adjustments)
|
|
echo "Installing HAProxy Agent..."
|
|
cat > "$INSTALL_DIR/haproxy-agent" << 'AGENT_SCRIPT'
|
|
#!/bin/bash
|
|
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/var/run/haproxy-agent.pid"
|
|
|
|
# Dynamic binary finder function (embedded in daemon)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Initialize global binary paths for daemon (with retry to prevent crash loops)
|
|
_dep_retry=0
|
|
_dep_max=5
|
|
while true; do
|
|
CURL_BIN=$(find_binary curl)
|
|
JQ_AVAILABLE=false
|
|
command -v jq &> /dev/null && JQ_AVAILABLE=true
|
|
|
|
if [[ -n "$CURL_BIN" ]] && [[ "$JQ_AVAILABLE" == "true" ]]; then
|
|
break
|
|
fi
|
|
|
|
_dep_retry=$((_dep_retry + 1))
|
|
_missing=""
|
|
[[ -z "$CURL_BIN" ]] && _missing="curl"
|
|
[[ "$JQ_AVAILABLE" != "true" ]] && _missing="${_missing:+$_missing }jq"
|
|
|
|
if [[ $_dep_retry -ge $_dep_max ]]; then
|
|
echo "ERROR: Required tools not found after $_dep_max retries: $_missing" >&2
|
|
echo "Install missing tools and restart the agent service." >&2
|
|
exit 1
|
|
fi
|
|
|
|
_wait=$((_dep_retry * 30))
|
|
echo "WARN: Required tools missing ($_missing). Retry $_dep_retry/$_dep_max in ${_wait}s..." >&2
|
|
sleep $_wait
|
|
done
|
|
|
|
# Load configuration (with retry - config may be temporarily unavailable after upgrade)
|
|
_cfg_retry=0
|
|
_cfg_max=5
|
|
while [[ ! -f "$CONFIG_FILE" ]]; do
|
|
_cfg_retry=$((_cfg_retry + 1))
|
|
if [[ $_cfg_retry -ge $_cfg_max ]]; then
|
|
echo "ERROR: Configuration not found after $_cfg_max retries: $CONFIG_FILE" >&2
|
|
echo "Re-install the agent to recreate the configuration." >&2
|
|
exit 1
|
|
fi
|
|
_wait=$((_cfg_retry * 15))
|
|
echo "WARN: Config file not found: $CONFIG_FILE. Retry $_cfg_retry/$_cfg_max in ${_wait}s..." >&2
|
|
sleep $_wait
|
|
done
|
|
|
|
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
|
|
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
|
|
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE")
|
|
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE")
|
|
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE")
|
|
|
|
# Load HAProxy paths from config.json as baseline (written during installation)
|
|
HAPROXY_CONFIG_PATH=$(jq -r '.haproxy.config_path // empty' "$CONFIG_FILE" 2>/dev/null)
|
|
HAPROXY_BIN_PATH=$(jq -r '.haproxy.bin_path // empty' "$CONFIG_FILE" 2>/dev/null)
|
|
STATS_SOCKET_PATH=$(jq -r '.haproxy.stats_socket_path // empty' "$CONFIG_FILE" 2>/dev/null)
|
|
|
|
# SSL incremental update timestamp file (agent-specific to avoid race conditions)
|
|
# Each agent maintains its own SSL sync timestamp to prevent conflicts
|
|
SSL_SYNC_TIMESTAMP_FILE="/tmp/haproxy-agent-ssl-sync-${AGENT_NAME}"
|
|
|
|
# Get dynamic HAProxy paths from cluster configuration (overrides config.json when available)
|
|
get_cluster_paths() {
|
|
local cluster_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/clusters" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
local _cfg _bin _sock
|
|
_cfg=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_config_path // empty' 2>/dev/null)
|
|
_bin=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_bin_path // empty' 2>/dev/null)
|
|
_sock=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .stats_socket_path // empty' 2>/dev/null)
|
|
# Only override if API returned non-empty values
|
|
[[ -n "$_cfg" ]] && HAPROXY_CONFIG_PATH="$_cfg"
|
|
[[ -n "$_bin" ]] && HAPROXY_BIN_PATH="$_bin"
|
|
[[ -n "$_sock" ]] && STATS_SOCKET_PATH="$_sock"
|
|
fi
|
|
}
|
|
|
|
# Initialize cluster paths (enhances config.json values with live cluster configuration)
|
|
get_cluster_paths
|
|
|
|
# Enable debug mode for troubleshooting (0=off, 1=on)
|
|
DEBUG_MODE=0
|
|
|
|
# Logging - JSON format, single line with timestamp
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
|
|
if [[ "$level" == "DEBUG" && "${DEBUG_MODE:-0}" != "1" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
# Escape message for JSON (replace backslash, quotes, newlines, tabs)
|
|
local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ')
|
|
|
|
# Remove any control characters and emojis (keep only printable ASCII + basic UTF-8)
|
|
escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g')
|
|
|
|
echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$AGENT_NAME\",\"message\":\"$escaped_message\"}" >> "$LOG_FILE"
|
|
}
|
|
|
|
# Register agent on first run
|
|
register_agent() {
|
|
log "INFO" "Registering agent with management server..."
|
|
|
|
local haproxy_version="unknown"
|
|
if command -v haproxy &> /dev/null; then
|
|
haproxy_version=$(haproxy -v | head -1 | awk '{print $3}')
|
|
fi
|
|
|
|
local arch=$(uname -m)
|
|
platform=$(uname -s | tr '[:upper:]' '[:lower:]') # Remove local to make it global
|
|
local system_info=$(collect_system_info)
|
|
# issue #31: if collect_system_info produced no JSON content (empty on an unusual host), the
|
|
# '$system_info,' line below would collapse to a bare comma and break the heartbeat JSON. A
|
|
# valid fragment always contains a quoted key; if none is present, fall back to one. The glob
|
|
# '*"*' is the most portable bash test (no POSIX class / pattern-substitution), safe on bash 3.x+.
|
|
[[ "$system_info" != *'"'* ]] && system_info='"operating_system": "unknown"'
|
|
|
|
local json_payload=$(cat <<SIMPLE_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"platform": "$platform",
|
|
"architecture": "$arch",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_version": "$haproxy_version",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"status": "online",
|
|
"haproxy_status": "initializing",
|
|
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
}
|
|
SIMPLE_EOF
|
|
)
|
|
|
|
log "DEBUG" "Sending registration JSON: $json_payload"
|
|
|
|
REGISTER_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$json_payload")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "Agent registered successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "Agent registration failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check HAProxy status (Linux specific)
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
elif pgrep -f "haproxy" > /dev/null 2>&1; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get full HAProxy stats CSV from socket (for dashboard metrics)
|
|
get_haproxy_stats_csv() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "STATS: socat not available, cannot fetch stats CSV"
|
|
echo ""
|
|
return
|
|
fi
|
|
log "INFO" "STATS: Initialized socat: $SOCAT_BIN"
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH"
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
# Return CSV data (base64 encoded to avoid JSON escaping issues)
|
|
if [[ -n "$stats_output" ]]; then
|
|
local csv_bytes=$(echo "$stats_output" | wc -c | tr -d ' ')
|
|
local csv_lines=$(echo "$stats_output" | wc -l | tr -d ' ')
|
|
local csv_base64=$(echo "$stats_output" | base64 | tr -d '\n')
|
|
local base64_size=${#csv_base64}
|
|
log "INFO" "STATS: Fetched CSV: $csv_bytes bytes, $csv_lines lines, base64: $base64_size chars"
|
|
echo "$csv_base64"
|
|
else
|
|
log "WARN" "STATS: Failed to fetch stats from socket: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
get_server_statuses() {
|
|
local server_statuses_json="{}"
|
|
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "DEBUG" "STATS: socat not available for server statuses"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "DEBUG" "Stats socket not found: $STATS_SOCKET_PATH"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
|
|
log "DEBUG" "Querying HAProxy stats socket: $STATS_SOCKET_PATH"
|
|
|
|
# Query HAProxy stats via socket with permission fix attempt
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "DEBUG" "Stats socket query failed, attempting permission fix"
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
log "DEBUG" "Applied chmod 666 to stats socket"
|
|
|
|
# Retry after permission fix
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "DEBUG" "Stats socket query failed after permission fix"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
else
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
log "DEBUG" "Stats socket query successful, parsing output"
|
|
|
|
# Parse stats output and build JSON
|
|
local backend_servers=()
|
|
while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do
|
|
# Skip header and empty lines
|
|
[[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue
|
|
|
|
# Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0)
|
|
[[ "$type" != "2" ]] && continue
|
|
|
|
# Extract backend and server names
|
|
local backend_name="$pxname"
|
|
local server_name="$svname"
|
|
|
|
# Map HAProxy status to our format
|
|
local server_status
|
|
case "$status" in
|
|
"UP") server_status="UP" ;;
|
|
"DOWN") server_status="DOWN" ;;
|
|
"MAINT") server_status="MAINT" ;;
|
|
"DRAIN") server_status="DRAIN" ;;
|
|
"NOLB") server_status="NOLB" ;;
|
|
*) server_status="UNKNOWN" ;;
|
|
esac
|
|
|
|
backend_servers+=("$backend_name:$server_name:$server_status")
|
|
done <<< "$stats_output"
|
|
|
|
# Build JSON structure
|
|
if [[ ${#backend_servers[@]} -gt 0 ]]; then
|
|
local json_parts=()
|
|
local current_backend=""
|
|
local backend_json=""
|
|
|
|
for entry in "${backend_servers[@]}"; do
|
|
IFS=':' read -r backend_name server_name server_status <<< "$entry"
|
|
|
|
if [[ "$current_backend" != "$backend_name" ]]; then
|
|
# Save previous backend if exists
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
# Start new backend
|
|
current_backend="$backend_name"
|
|
backend_json=""
|
|
fi
|
|
|
|
backend_json+="\"$server_name\": \"$server_status\","
|
|
done
|
|
|
|
# Save last backend
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
|
|
# Combine all backends
|
|
if [[ ${#json_parts[@]} -gt 0 ]]; then
|
|
local IFS=','
|
|
server_statuses_json="{${json_parts[*]}}"
|
|
fi
|
|
fi
|
|
|
|
echo "$server_statuses_json"
|
|
}
|
|
|
|
# Collect system information (Linux specific)
|
|
collect_system_info() {
|
|
# Get system information
|
|
local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux")
|
|
local kernel_version=$(uname -r 2>/dev/null || echo "Unknown")
|
|
local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0")
|
|
local cpu_count=$(nproc 2>/dev/null || echo "1")
|
|
local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0")
|
|
local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0")
|
|
local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "")
|
|
local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "")
|
|
|
|
# Build system info JSON
|
|
cat <<SYSTEM_INFO_EOF
|
|
"operating_system": "$os_info",
|
|
"kernel_version": "$kernel_version",
|
|
"uptime": $uptime_seconds,
|
|
"cpu_count": $cpu_count,
|
|
"memory_total": $memory_bytes,
|
|
"disk_space": $disk_bytes,
|
|
"ip_address": "$ip_address",
|
|
"network_interfaces": ["${network_interfaces//,/\",\"}"],
|
|
"capabilities": ["haproxy_management", "ssl_deployment", "config_reload", "systemd_service", "keepalived_management"]
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Detect keepalived VRRP state (MASTER/BACKUP) and virtual IP
|
|
get_keepalive_state() {
|
|
local state=""
|
|
local vip=""
|
|
|
|
# Fast exit: Check if keepalived is running (instant check, <5ms)
|
|
if ! systemctl is-active --quiet keepalived 2>/dev/null; then
|
|
if ! pgrep -x keepalived >/dev/null 2>&1; then
|
|
echo "|"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Method 1: journalctl (most reliable on RHEL/CentOS/Ubuntu with systemd)
|
|
if command -v journalctl &>/dev/null; then
|
|
state=$(journalctl -u keepalived -n 50 --no-pager 2>/dev/null \
|
|
| grep -oE "(MASTER|BACKUP|FAULT)" | tail -1)
|
|
fi
|
|
|
|
# Method 2: Fallback to log files (for non-systemd or restricted journalctl)
|
|
if [[ -z "$state" ]]; then
|
|
for logfile in /var/log/messages /var/log/syslog /var/log/keepalived.log; do
|
|
if [[ -r "$logfile" ]]; then
|
|
state=$(tail -200 "$logfile" 2>/dev/null \
|
|
| grep -i keepalived | grep -oE "(MASTER|BACKUP|FAULT)" | tail -1)
|
|
[[ -n "$state" ]] && break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Method 3: VIP presence on local interfaces — the most portable signal, independent
|
|
# of logging/journald (works on any distro / init system / keepalived install type).
|
|
# MASTER iff ANY configured VIP is actually held locally; keepalived up but holding no
|
|
# VIP => BACKUP. Exact whole-line IP match (grep -Fxq) so 10.0.0.1 can't falsely match
|
|
# 10.0.0.10/100, and ALL configured VIPs are checked (not just the first).
|
|
if [[ -z "$state" ]] && [[ -r /etc/keepalived/keepalived.conf ]]; then
|
|
local conf_vips=$(grep -A20 'virtual_ipaddress' /etc/keepalived/keepalived.conf 2>/dev/null \
|
|
| grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}')
|
|
if [[ -n "$conf_vips" ]]; then
|
|
local local_ips=$(ip -4 -o addr show 2>/dev/null | grep -oE 'inet ([0-9]{1,3}\.){3}[0-9]{1,3}' | awk '{print $2}')
|
|
state="BACKUP"
|
|
local _cvip
|
|
for _cvip in $conf_vips; do
|
|
if printf '%s\n' "$local_ips" | grep -Fxq "$_cvip"; then state="MASTER"; break; fi
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# Extract VIP from keepalived.conf
|
|
if [[ -r /etc/keepalived/keepalived.conf ]]; then
|
|
vip=$(grep -A10 'virtual_ipaddress' /etc/keepalived/keepalived.conf 2>/dev/null \
|
|
| grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
|
fi
|
|
|
|
echo "${state}|${vip}"
|
|
return 0
|
|
}
|
|
|
|
# Send heartbeat
|
|
send_heartbeat() {
|
|
local haproxy_status=$(get_haproxy_status)
|
|
local server_statuses=$(get_server_statuses)
|
|
local haproxy_stats_csv=$(get_haproxy_stats_csv)
|
|
local system_info=$(collect_system_info)
|
|
# issue #31: if collect_system_info produced no JSON content (empty on an unusual host), the
|
|
# '$system_info,' line below would collapse to a bare comma and break the heartbeat JSON. A
|
|
# valid fragment always contains a quoted key; if none is present, fall back to one. The glob
|
|
# '*"*' is the most portable bash test (no POSIX class / pattern-substitution), safe on bash 3.x+.
|
|
[[ "$system_info" != *'"'* ]] && system_info='"operating_system": "unknown"'
|
|
|
|
# Get HAProxy version for heartbeat (safe extraction, fallback to "unknown")
|
|
local haproxy_version="unknown"
|
|
if command -v haproxy &> /dev/null; then
|
|
haproxy_version=$(haproxy -v 2>/dev/null | head -1 | awk '{print $3}' || echo "unknown")
|
|
fi
|
|
|
|
# Ensure platform is available (global variable set during registration)
|
|
[[ -z "$platform" ]] && platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
|
|
# Detect keepalived VRRP state
|
|
local keepalive_info=$(get_keepalive_state)
|
|
local keepalive_state=$(echo "$keepalive_info" | cut -d'|' -f1)
|
|
local keepalive_ip=$(echo "$keepalive_info" | cut -d'|' -f2)
|
|
|
|
local heartbeat_payload
|
|
# Include stats CSV if available
|
|
if [[ -n "$haproxy_stats_csv" && "$server_statuses" != "{}" ]]; then
|
|
log "INFO" "HEARTBEAT: Including stats CSV (${#haproxy_stats_csv} chars) with server statuses"
|
|
heartbeat_payload=$(cat <<STATS_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"haproxy_version": "$haproxy_version",
|
|
"keepalive_state": "$keepalive_state",
|
|
"keepalive_ip": "$keepalive_ip",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}",
|
|
"server_statuses": $server_statuses,
|
|
"haproxy_stats_csv": "$haproxy_stats_csv"
|
|
}
|
|
STATS_HEARTBEAT_EOF
|
|
)
|
|
elif [[ "$server_statuses" != "{}" ]]; then
|
|
log "WARN" "HEARTBEAT: Stats CSV empty, sending heartbeat WITHOUT stats (only server statuses)"
|
|
heartbeat_payload=$(cat <<SIMPLE_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"haproxy_version": "$haproxy_version",
|
|
"keepalive_state": "$keepalive_state",
|
|
"keepalive_ip": "$keepalive_ip",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}",
|
|
"server_statuses": $server_statuses
|
|
}
|
|
SIMPLE_HEARTBEAT_EOF
|
|
)
|
|
else
|
|
log "WARN" "HEARTBEAT: No server statuses and no stats CSV - minimal heartbeat"
|
|
heartbeat_payload=$(cat <<SIMPLE_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"haproxy_version": "$haproxy_version",
|
|
"keepalive_state": "$keepalive_state",
|
|
"keepalive_ip": "$keepalive_ip",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}"
|
|
}
|
|
SIMPLE_HEARTBEAT_EOF
|
|
)
|
|
fi
|
|
|
|
log "DEBUG" "Sending heartbeat"
|
|
|
|
# Send heartbeat to backend with HTTP status code check
|
|
# Use temp files for both payload and response to handle large stats CSV (>200KB, production can be >1MB)
|
|
local temp_payload="/tmp/heartbeat_payload_$$.json"
|
|
local temp_response="/tmp/heartbeat_response_$$.txt"
|
|
|
|
# Write payload to temp file to avoid any size limits
|
|
printf '%s' "$heartbeat_payload" > "$temp_payload"
|
|
|
|
# Send request using temp file
|
|
"$CURL_BIN" -k -s -w "\n%{http_code}" -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
--data-binary @"$temp_payload" > "$temp_response" 2>&1
|
|
|
|
local http_code=$(tail -n1 "$temp_response" 2>/dev/null || echo "000")
|
|
local response_body=$(head -n-1 "$temp_response" 2>/dev/null || echo "")
|
|
|
|
# Cleanup temp files
|
|
rm -f "$temp_payload" "$temp_response"
|
|
|
|
# Check HTTP status code
|
|
if [[ "$http_code" == "200" || "$http_code" == "201" ]]; then
|
|
# Success - only log in debug mode to reduce spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "HEARTBEAT: Sent successfully (HTTP $http_code)"
|
|
else
|
|
# Failed - log error with details
|
|
log "ERROR" "Heartbeat failed (HTTP $http_code)"
|
|
if [[ -n "$response_body" && "$response_body" != "" ]]; then
|
|
# Log first 200 chars of error response
|
|
local error_preview="${response_body:0:200}"
|
|
log "ERROR" "Backend response: $error_preview"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Deploy SSL certificates to local filesystem (Linux)
|
|
deploy_ssl_certificates() {
|
|
local ssl_certs_json="$1"
|
|
|
|
# Global counter: tracks how many cert files were ACTUALLY changed (for reload decisions)
|
|
SSL_CERTS_CHANGED_COUNT=0
|
|
|
|
log "INFO" "SSL DEPLOYMENT: Processing SSL certificates..."
|
|
|
|
# Create SSL directory if it doesn't exist
|
|
local ssl_dir="/etc/ssl/haproxy"
|
|
if [[ ! -d "$ssl_dir" ]]; then
|
|
log "INFO" "Creating SSL directory: $ssl_dir"
|
|
mkdir -p "$ssl_dir"
|
|
chmod 755 "$ssl_dir"
|
|
fi
|
|
|
|
# Parse and deploy each certificate
|
|
local cert_count=$(echo "$ssl_certs_json" | jq length 2>/dev/null || echo "0")
|
|
log "INFO" "Found $cert_count SSL certificates to deploy"
|
|
|
|
for ((i=0; i<cert_count; i++)); do
|
|
local cert_data=$(echo "$ssl_certs_json" | jq -r ".[$i]")
|
|
local cert_name=$(echo "$cert_data" | jq -r '.name // "unknown"')
|
|
local cert_domain=$(echo "$cert_data" | jq -r '.domain // "unknown"')
|
|
local cert_file_path=$(echo "$cert_data" | jq -r '.file_path // "/etc/ssl/haproxy/unknown.pem"')
|
|
local cert_content=$(echo "$cert_data" | jq -r '.certificate_content // ""')
|
|
local key_content=$(echo "$cert_data" | jq -r '.private_key_content // ""')
|
|
local chain_content=$(echo "$cert_data" | jq -r '.chain_content // ""')
|
|
local cert_status=$(echo "$cert_data" | jq -r '.status // "unknown"')
|
|
local usage_type=$(echo "$cert_data" | jq -r '.usage_type // "frontend"')
|
|
|
|
log "INFO" "Deploying SSL certificate: $cert_name ($cert_domain) - Usage: $usage_type - Status: $cert_status"
|
|
|
|
# Validate certificate content
|
|
if [[ -z "$cert_content" || "$cert_content" == "null" ]]; then
|
|
log "ERROR" "Certificate content is empty for $cert_name, skipping"
|
|
continue
|
|
fi
|
|
|
|
# CRITICAL: Private key validation depends on usage_type
|
|
# - Frontend SSL: private key REQUIRED (for HAProxy bind)
|
|
# - Server SSL: private key OPTIONAL (CA certificate only for backend verification)
|
|
if [[ "$usage_type" == "frontend" ]]; then
|
|
if [[ -z "$key_content" || "$key_content" == "null" ]]; then
|
|
log "ERROR" "Private key is required for Frontend SSL: $cert_name, skipping"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Build combined PEM content (same format as embedded daemon for consistent checksums)
|
|
local new_content=""
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s\n\n%s" "$cert_content" "$key_content" "$chain_content")
|
|
else
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$key_content")
|
|
fi
|
|
else
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$chain_content")
|
|
else
|
|
new_content="$cert_content"
|
|
fi
|
|
fi
|
|
|
|
# Compare checksums to detect actual changes (avoid unnecessary writes + reloads)
|
|
local new_checksum=$(echo "$new_content" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
local existing_checksum=""
|
|
if [[ -f "$cert_file_path" ]]; then
|
|
existing_checksum=$(cat "$cert_file_path" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
fi
|
|
|
|
if [[ "$existing_checksum" == "$new_checksum" ]]; then
|
|
log "DEBUG" "SSL certificate unchanged: $cert_name (checksum match), skipping deploy"
|
|
continue
|
|
fi
|
|
|
|
# Certificate is new or changed - write to temp file and validate
|
|
local temp_cert_file="/tmp/ssl_cert_$$.pem"
|
|
echo "$new_content" > "$temp_cert_file"
|
|
|
|
# Validate the combined certificate file
|
|
if openssl x509 -in "$temp_cert_file" -noout -text >/dev/null 2>&1; then
|
|
# Move to final location
|
|
mv "$temp_cert_file" "$cert_file_path"
|
|
chmod 600 "$cert_file_path"
|
|
chown root:root "$cert_file_path" 2>/dev/null || true
|
|
|
|
# OPTIMIZATION: Only count toward reload if cert is actually referenced in HAProxy config
|
|
# This prevents unnecessary HAProxy reloads for SSL certificates that are deployed
|
|
# to disk but not used by any frontend/backend in this cluster's HAProxy configuration.
|
|
# File is ALWAYS deployed (for future use), but reload is only triggered when needed.
|
|
# Use -F for fixed-string matching (avoids regex interpretation of dots in filenames)
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]] && grep -qF "$cert_file_path" "$HAPROXY_CONFIG_PATH" 2>/dev/null; then
|
|
SSL_CERTS_CHANGED_COUNT=$((SSL_CERTS_CHANGED_COUNT + 1))
|
|
log "INFO" "SSL certificate deployed (CHANGED & IN USE): $cert_file_path"
|
|
else
|
|
log "INFO" "SSL certificate deployed (CHANGED but NOT in HAProxy config): $cert_file_path - no reload needed"
|
|
fi
|
|
|
|
# Log certificate details
|
|
local cert_subject=$(openssl x509 -in "$cert_file_path" -noout -subject 2>/dev/null | sed 's/subject=//')
|
|
local cert_expiry=$(openssl x509 -in "$cert_file_path" -noout -dates 2>/dev/null | grep notAfter | sed 's/notAfter=//')
|
|
log "DEBUG" "Certificate subject: $cert_subject"
|
|
log "DEBUG" "Certificate expires: $cert_expiry"
|
|
else
|
|
log "ERROR" "Invalid certificate format for $cert_name, removing temp file"
|
|
rm -f "$temp_cert_file"
|
|
fi
|
|
done
|
|
|
|
if [[ $SSL_CERTS_CHANGED_COUNT -gt 0 ]]; then
|
|
log "INFO" "SSL DEPLOYMENT: Completed - $SSL_CERTS_CHANGED_COUNT certificate(s) changed"
|
|
else
|
|
log "INFO" "SSL DEPLOYMENT: Completed - all certificates up to date"
|
|
fi
|
|
}
|
|
|
|
# Fetch and deploy SSL certificates from separate endpoint (Linux)
|
|
# Args: $1 = "full" to skip incremental sync and fetch ALL certs (used by standalone SSL check)
|
|
fetch_and_deploy_ssl_certificates() {
|
|
local fetch_mode="${1:-incremental}"
|
|
|
|
# Reset global counter before fetch
|
|
SSL_CERTS_CHANGED_COUNT=0
|
|
|
|
# Only log SSL fetch in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "SSL FETCH: Getting SSL certificates (mode: $fetch_mode)..."
|
|
|
|
# Build SSL endpoint URL with incremental update support
|
|
local ssl_endpoint="$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates"
|
|
|
|
# Add timestamp parameter for incremental updates (skip for full sync)
|
|
if [[ "$fetch_mode" != "full" && -f "$SSL_SYNC_TIMESTAMP_FILE" ]]; then
|
|
local last_ssl_sync=$(cat "$SSL_SYNC_TIMESTAMP_FILE" 2>/dev/null || echo "")
|
|
if [[ -n "$last_ssl_sync" ]]; then
|
|
ssl_endpoint="${ssl_endpoint}?since=${last_ssl_sync}"
|
|
log "DEBUG" "SSL INCREMENTAL: Requesting SSL certificates since $last_ssl_sync"
|
|
fi
|
|
fi
|
|
|
|
# Fetch SSL certificates from dedicated endpoint
|
|
local ssl_response=$("$CURL_BIN" -k -s -X GET "$ssl_endpoint" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "WARN" "Failed to fetch SSL certificates, skipping SSL deployment"
|
|
return 0
|
|
fi
|
|
|
|
local ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"')
|
|
local ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"')
|
|
|
|
if [[ "$ssl_status" != "available" ]]; then
|
|
log "DEBUG" "SSL certificates not available (status: $ssl_status)"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then
|
|
log "INFO" "Deploying SSL certificates..."
|
|
deploy_ssl_certificates "$ssl_certificates"
|
|
|
|
# Save current timestamp for next incremental update
|
|
date -u +"%Y-%m-%dT%H:%M:%SZ" > "$SSL_SYNC_TIMESTAMP_FILE"
|
|
log "DEBUG" "SSL INCREMENTAL: Saved SSL sync timestamp for next incremental update"
|
|
else
|
|
log "DEBUG" "No SSL certificates to deploy"
|
|
fi
|
|
}
|
|
|
|
# Independent SSL certificate sync check (mirrors embedded daemon behavior)
|
|
# Runs periodically in daemon loop, independent of config version changes
|
|
# If any cert files actually changed on disk, triggers HAProxy reload
|
|
check_ssl_updates() {
|
|
# Fetch ALL certificates (full sync, not incremental) so checksum comparison catches everything
|
|
fetch_and_deploy_ssl_certificates "full"
|
|
|
|
if [[ ${SSL_CERTS_CHANGED_COUNT:-0} -gt 0 ]]; then
|
|
log "INFO" "SSL SYNC: $SSL_CERTS_CHANGED_COUNT certificate(s) changed, triggering HAProxy reload..."
|
|
|
|
# Validate HAProxy config before reload (cert files are already on disk)
|
|
if "$HAPROXY_BIN_PATH" -c -f "$HAPROXY_CONFIG_PATH" >/dev/null 2>&1; then
|
|
reload_haproxy_service
|
|
log "INFO" "SSL SYNC: HAProxy reloaded successfully after SSL certificate update"
|
|
else
|
|
# Config validation may fail if config references a cert that was just added
|
|
# but config hasn't been updated yet - this is expected, config update will handle it
|
|
log "WARN" "SSL SYNC: Certificates deployed but HAProxy config validation failed, reload will happen on next config update"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Reload HAProxy service (Linux specific)
|
|
# Issue #27 (v1.7.0) — HA/VIP (Keepalived) convergence. OPT-IN and inert by default:
|
|
# a node with no APPLIED VIP gets status=not_configured and (lacking our ownership
|
|
# marker) does NOTHING. Never clobbers a hand-managed keepalived. Uses $AGENT_TOKEN
|
|
# (kept in sync with the live token in both daemon loops).
|
|
fetch_and_deploy_keepalived_config() {
|
|
local marker="# Managed by HAProxy OpenManager"
|
|
local resp status http_code we_own="false" conf chk
|
|
|
|
# Timeouts so a hung management server can never stall the daemon loop.
|
|
resp=$(curl -k -s --connect-timeout 10 --max-time 30 -w '\n%{http_code}' -X GET \
|
|
"$MANAGEMENT_URL/api/agents/$AGENT_NAME/keepalived-config" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null) || return 0
|
|
http_code="${resp##*$'\n'}" # last line = HTTP status
|
|
resp="${resp%$'\n'*}" # everything before = JSON body
|
|
[[ -z "$resp" ]] && return 0
|
|
status=$(echo "$resp" | jq -r '.status // "unknown"' 2>/dev/null)
|
|
# A 404 means this agent was deleted server-side (node decommissioned / pool removed):
|
|
# self-heal teardown OUR keepalived so a removed node stops advertising the VIP. The
|
|
# teardown branch below is marker-guarded, so a node we don't manage stays untouched.
|
|
[[ "$http_code" == "404" ]] && status="teardown"
|
|
# Cluster-driven keepalived.conf path (delivered in every response); default is universal.
|
|
conf=$(echo "$resp" | jq -r '.config_path // empty' 2>/dev/null)
|
|
[[ -z "$conf" || "$conf" == "null" ]] && conf="/etc/keepalived/keepalived.conf"
|
|
chk="$(dirname "$conf")/check_haproxy.sh"
|
|
if [[ -f "$conf" ]] && grep -q "$marker" "$conf" 2>/dev/null; then we_own="true"; fi
|
|
|
|
_kp_report() { # $1=state $2=vip_id(or empty) $3=hash $4=message
|
|
local vid="${2:-null}"; [[ -z "$2" ]] && vid="null"
|
|
curl -k -s --connect-timeout 10 --max-time 30 -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/keepalived-status" \
|
|
-H "X-API-Key: $AGENT_TOKEN" -H "Content-Type: application/json" \
|
|
-d "{\"vip_id\":${vid},\"state\":\"$1\",\"config_hash\":\"${3:-}\",\"message\":\"${4:-}\"}" \
|
|
>/dev/null 2>&1 || true
|
|
}
|
|
_kp_teardown() {
|
|
# $1=purge ("true" only on an explicit operator opt-in delete). Default: stop+disable
|
|
# keepalived and remove OUR config (the VIP is released) but KEEP the package — the safe
|
|
# enterprise default. Purge only when opted in AND we are the ones who installed it
|
|
# (the .hom_installed marker); never remove a package the admin pre-installed.
|
|
local purge="${1:-false}" marker_inst; marker_inst="$(dirname "$conf")/.hom_installed"
|
|
log "INFO" "KEEPALIVED: tearing down our managed VIP config (purge=$purge)"
|
|
systemctl stop keepalived >/dev/null 2>&1
|
|
systemctl disable keepalived >/dev/null 2>&1
|
|
rm -f "$conf" "$chk"
|
|
if [[ "$purge" == "true" && -f "$marker_inst" ]]; then
|
|
log "INFO" "KEEPALIVED: uninstalling keepalived package (operator opt-in)"
|
|
if command -v apt-get >/dev/null 2>&1; then timeout 300 apt-get purge -y -qq keepalived >/dev/null 2>&1
|
|
elif command -v dnf >/dev/null 2>&1; then timeout 300 dnf remove -y -q keepalived >/dev/null 2>&1
|
|
elif command -v yum >/dev/null 2>&1; then timeout 300 yum remove -y -q keepalived >/dev/null 2>&1
|
|
elif command -v zypper >/dev/null 2>&1; then timeout 300 zypper --non-interactive remove -y keepalived >/dev/null 2>&1
|
|
elif command -v apk >/dev/null 2>&1; then timeout 300 apk del keepalived >/dev/null 2>&1
|
|
fi
|
|
rm -f "$marker_inst"
|
|
if command -v keepalived >/dev/null 2>&1; then
|
|
_kp_report "disabled" "" "" "config removed; package uninstall attempted (still present)"
|
|
else
|
|
_kp_report "disabled" "" "" "torn down + keepalived package uninstalled"
|
|
fi
|
|
elif [[ "$purge" == "true" ]]; then
|
|
log "INFO" "KEEPALIVED: package was pre-existing (not installed by us) — left in place; our config removed"
|
|
_kp_report "disabled" "" "" "torn down (package left: pre-existing, not installed by us)"
|
|
else
|
|
_kp_report "disabled" "" "" "torn down"
|
|
fi
|
|
}
|
|
|
|
case "$status" in
|
|
available) : ;; # fall through to converge
|
|
teardown)
|
|
# Explicit server-side delete. Honor the operator's opt-in package purge (.purge);
|
|
# marker-guarded inside _kp_teardown, and a node we don't own (no marker conf) is a no-op.
|
|
local _kp_purge; _kp_purge=$(echo "$resp" | jq -r '.purge // false' 2>/dev/null)
|
|
[[ "$we_own" == "true" ]] && _kp_teardown "$_kp_purge"
|
|
return 0 ;;
|
|
not_configured)
|
|
[[ "$we_own" == "true" ]] && _kp_teardown "false" # T-2 orphan self-heal: graceful only, never purge
|
|
return 0 ;;
|
|
*) return 0 ;; # unknown / auth error → no-op
|
|
esac
|
|
|
|
# --- status == available: converge ---
|
|
local vip_id new_conf check_script install_if new_hash
|
|
vip_id=$(echo "$resp" | jq -r '.keepalived.vip_id // empty' 2>/dev/null)
|
|
new_conf=$(echo "$resp" | jq -r '.keepalived.config_content // empty' 2>/dev/null)
|
|
new_hash=$(echo "$resp" | jq -r '.keepalived.config_hash // empty' 2>/dev/null)
|
|
check_script=$(echo "$resp" | jq -r '.keepalived.check_script // empty' 2>/dev/null)
|
|
install_if=$(echo "$resp" | jq -r '.keepalived.install_if_missing // false' 2>/dev/null)
|
|
[[ -z "$new_conf" ]] && return 0
|
|
|
|
# Ownership guard: never overwrite a keepalived.conf we don't own.
|
|
if [[ -f "$conf" && "$we_own" != "true" ]]; then
|
|
log "WARN" "KEEPALIVED: $conf is externally managed — refusing to overwrite"
|
|
_kp_report "externally_managed" "$vip_id" "" "pre-existing unmanaged keepalived.conf"
|
|
return 0
|
|
fi
|
|
|
|
# Hybrid install: install keepalived only if missing.
|
|
if ! command -v keepalived >/dev/null 2>&1; then
|
|
if [[ "$install_if" == "true" ]]; then
|
|
log "INFO" "KEEPALIVED: installing package..."
|
|
if command -v apt-get >/dev/null 2>&1; then timeout 300 apt-get install -y -qq keepalived >/dev/null 2>&1
|
|
elif command -v dnf >/dev/null 2>&1; then timeout 300 dnf install -y -q keepalived >/dev/null 2>&1
|
|
elif command -v yum >/dev/null 2>&1; then timeout 300 yum install -y -q keepalived >/dev/null 2>&1
|
|
elif command -v zypper >/dev/null 2>&1; then timeout 300 zypper --non-interactive install -y keepalived >/dev/null 2>&1
|
|
elif command -v apk >/dev/null 2>&1; then timeout 300 apk add --no-cache keepalived >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
if ! command -v keepalived >/dev/null 2>&1; then
|
|
log "ERROR" "KEEPALIVED: not installed/available"
|
|
_kp_report "error" "$vip_id" "" "keepalived not installed"
|
|
return 0
|
|
fi
|
|
# We installed keepalived (it was missing) — drop a marker so an opt-in uninstall on
|
|
# delete removes only OUR install, never an admin's pre-existing keepalived package.
|
|
mkdir -p "$(dirname "$conf")" 2>/dev/null; : > "$(dirname "$conf")/.hom_installed" 2>/dev/null
|
|
fi
|
|
|
|
# Idempotency: skip write+reload when the on-disk content already matches.
|
|
if [[ -f "$conf" ]]; then
|
|
local cur_hash would_hash
|
|
cur_hash=$(md5sum "$conf" 2>/dev/null | awk '{print $1}')
|
|
would_hash=$(printf '%s' "$new_conf" | md5sum 2>/dev/null | awk '{print $1}')
|
|
if [[ -n "$cur_hash" && "$cur_hash" == "$would_hash" ]]; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$conf")"
|
|
if [[ -n "$check_script" ]]; then
|
|
printf '%s' "$check_script" > "$chk"
|
|
chown root:root "$chk" 2>/dev/null
|
|
chmod 0755 "$chk" # root-owned, not world-writable — required by enable_script_security
|
|
fi
|
|
# Validate on a TEMP file and swap in only on success: a bad render must never land on
|
|
# $conf (it would also make the md5 idempotency guard above suppress retries forever).
|
|
local tmp_conf="${conf}.hom.tmp"
|
|
printf '%s' "$new_conf" > "$tmp_conf"
|
|
chmod 0644 "$tmp_conf"
|
|
if ! keepalived -t -f "$tmp_conf" >/dev/null 2>&1; then
|
|
rm -f "$tmp_conf"
|
|
log "ERROR" "KEEPALIVED: config validation failed (keepalived -t) — keeping current config, not (re)starting"
|
|
_kp_report "error" "$vip_id" "$new_hash" "keepalived -t failed"
|
|
return 0
|
|
fi
|
|
mv -f "$tmp_conf" "$conf"
|
|
chown root:root "$conf" 2>/dev/null
|
|
chmod 0644 "$conf"
|
|
systemctl enable keepalived >/dev/null 2>&1
|
|
if systemctl reload keepalived >/dev/null 2>&1 || systemctl restart keepalived >/dev/null 2>&1; then
|
|
log "INFO" "KEEPALIVED: applied config for VIP ${vip_id}"
|
|
_kp_report "enabled" "$vip_id" "$new_hash" "applied"
|
|
else
|
|
log "ERROR" "KEEPALIVED: reload/restart failed"
|
|
_kp_report "error" "$vip_id" "$new_hash" "reload/restart failed"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
reload_haproxy_service() {
|
|
log "INFO" "Performing zero-downtime HAProxy configuration reload..."
|
|
|
|
# IMPROVED STRATEGY: Use HAProxy's seamless reload with verification
|
|
# This ensures new config is applied without traffic interruption
|
|
|
|
# Step 1: Validate new config before attempting reload
|
|
if ! "$HAPROXY_BIN_PATH" -c -f "$HAPROXY_CONFIG_PATH" >/dev/null 2>&1; then
|
|
log "ERROR" "New configuration validation failed - aborting reload"
|
|
return 1
|
|
fi
|
|
|
|
# Step 2: Try systemd reload first (most reliable on Linux)
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
log "INFO" "Using systemctl reload for zero-downtime update"
|
|
if systemctl reload haproxy 2>/dev/null; then
|
|
log "INFO" "HAProxy reloaded successfully via systemctl"
|
|
|
|
# Verify reload was successful
|
|
sleep 2
|
|
if systemctl is-active --quiet haproxy; then
|
|
log "INFO" "HAProxy service is active after systemctl reload"
|
|
return 0
|
|
else
|
|
log "WARN" "HAProxy service not active after systemctl reload, trying manual method"
|
|
fi
|
|
else
|
|
log "WARN" "systemctl reload failed, trying manual seamless reload"
|
|
fi
|
|
fi
|
|
|
|
# Step 3: Manual seamless reload using HAProxy's -sf option
|
|
local haproxy_master_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1)
|
|
|
|
if [[ -n "$haproxy_master_pid" ]]; then
|
|
log "INFO" "Found HAProxy master process (PID: $haproxy_master_pid)"
|
|
|
|
# Use HAProxy's seamless reload with new process
|
|
log "INFO" "Starting seamless reload with new configuration..."
|
|
|
|
if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D -sf "$haproxy_master_pid"; then
|
|
log "INFO" "Seamless reload initiated successfully"
|
|
|
|
# Verify new process is running and old one is gone
|
|
sleep 3
|
|
local new_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1)
|
|
|
|
if [[ -n "$new_pid" && "$new_pid" != "$haproxy_master_pid" ]]; then
|
|
log "INFO" "HAProxy seamlessly reloaded (New PID: $new_pid, Old PID: $haproxy_master_pid)"
|
|
return 0
|
|
else
|
|
log "WARN" "Seamless reload may have failed, verifying process status..."
|
|
|
|
# Additional verification
|
|
if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "HAProxy is still running after reload attempt"
|
|
return 0
|
|
else
|
|
log "ERROR" "HAProxy process not found after reload"
|
|
fi
|
|
fi
|
|
else
|
|
log "ERROR" "Seamless reload command failed"
|
|
fi
|
|
else
|
|
log "WARN" "No HAProxy master process found, starting fresh instance"
|
|
fi
|
|
|
|
# Step 4: Fallback - start new HAProxy instance (only if no process running)
|
|
if ! pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "Starting new HAProxy instance..."
|
|
|
|
if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D; then
|
|
sleep 2
|
|
if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "HAProxy started successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "HAProxy failed to start"
|
|
return 1
|
|
fi
|
|
else
|
|
log "ERROR" "Failed to start HAProxy: $HAPROXY_BIN_PATH"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "HAProxy configuration reload completed"
|
|
return 0
|
|
}
|
|
|
|
# Check for agent upgrade and perform if requested
|
|
check_agent_upgrade() {
|
|
log "DEBUG" "Checking for agent upgrade requests..."
|
|
|
|
# Check if upgrade was recently completed to prevent loops (agent-specific marker)
|
|
local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME"
|
|
if [[ -f "$agent_marker" ]]; then
|
|
local upgrade_time=$(head -n1 "$agent_marker" | cut -d' ' -f1-2)
|
|
local current_time=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
# Convert times to epoch for comparison
|
|
local upgrade_epoch=$(date -d "$upgrade_time" '+%s' 2>/dev/null || echo "0")
|
|
local current_epoch=$(date '+%s')
|
|
local time_diff=$((current_epoch - upgrade_epoch))
|
|
|
|
# Remove marker if older than 5 minutes (300 seconds)
|
|
if [[ $time_diff -gt 300 ]]; then
|
|
log "DEBUG" "Upgrade marker expired (${time_diff}s old), removing and allowing new upgrades"
|
|
rm -f "$agent_marker" 2>/dev/null || true
|
|
else
|
|
log "DEBUG" "Recent upgrade completed at: $upgrade_time - skipping upgrade check (${time_diff}s ago)"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Check if agent is marked for upgrade by querying agent-specific status
|
|
local agent_status_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "DEBUG" "Failed to fetch agent upgrade status"
|
|
return 0
|
|
fi
|
|
|
|
local should_upgrade=$(echo "$agent_status_response" | jq -r '.should_upgrade // false')
|
|
local target_version=$(echo "$agent_status_response" | jq -r '.target_version // ""')
|
|
local current_version="{{AGENT_VERSION}}" # This agent's version
|
|
|
|
# Only upgrade if explicitly requested via UI and target version is different
|
|
if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "$current_version" ]]; then
|
|
log "INFO" "AGENT UPGRADE: Upgrade requested to version $target_version (current: $current_version)"
|
|
perform_agent_upgrade "$target_version"
|
|
fi
|
|
}
|
|
|
|
# Perform agent upgrade
|
|
perform_agent_upgrade() {
|
|
local new_version="$1"
|
|
log "INFO" "AGENT UPGRADE: Starting upgrade to version $new_version"
|
|
|
|
# Download new agent script
|
|
local temp_script="/tmp/haproxy-agent-new"
|
|
local upgrade_url="$MANAGEMENT_URL/api/agents/generate-install-script"
|
|
|
|
# Get pool_id from agent's current cluster configuration
|
|
local POOL_ID=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null)
|
|
if [[ -z "$POOL_ID" || "$POOL_ID" == "null" ]]; then
|
|
# Fallback: get pool_id from cluster_id via API
|
|
local cluster_info=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
POOL_ID=$(echo "$cluster_info" | jq -r '.pool_id // 1' 2>/dev/null)
|
|
[[ -z "$POOL_ID" || "$POOL_ID" == "null" ]] && POOL_ID=1
|
|
fi
|
|
|
|
# Create upgrade request payload with correct pool_id
|
|
local upgrade_payload=$(cat <<UPGRADE_EOF
|
|
{
|
|
"platform": "$(uname -s | tr '[:upper:]' '[:lower:]')",
|
|
"architecture": "$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')",
|
|
"pool_id": $POOL_ID,
|
|
"cluster_id": $CLUSTER_ID,
|
|
"agent_name": "$AGENT_NAME",
|
|
"hostname_prefix": "$(echo $HOSTNAME | cut -d'-' -f1)",
|
|
"haproxy_bin_path": "$HAPROXY_BIN_PATH",
|
|
"haproxy_config_path": "$HAPROXY_CONFIG_PATH",
|
|
"stats_socket_path": "$STATS_SOCKET_PATH"
|
|
}
|
|
UPGRADE_EOF
|
|
)
|
|
|
|
# Download new script
|
|
log "INFO" "AGENT UPGRADE: Downloading new agent script"
|
|
local script_response=$("$CURL_BIN" -k -s -X POST "$upgrade_url" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$upgrade_payload")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
local new_script=$(echo "$script_response" | jq -r '.script // ""')
|
|
|
|
if [[ -n "$new_script" && "$new_script" != "null" ]]; then
|
|
echo "$new_script" > "$temp_script"
|
|
chmod +x "$temp_script"
|
|
|
|
log "INFO" "AGENT UPGRADE: New script downloaded, performing in-place upgrade"
|
|
|
|
# Get current script path
|
|
local current_script="/usr/local/bin/haproxy-agent"
|
|
|
|
# Create backup of current script in dedicated backup directory
|
|
local backup_dir="/usr/local/share/haproxy-agent-backups"
|
|
local backup_file="$backup_dir/haproxy-agent-$(date '+%Y%m%d-%H%M%S').backup"
|
|
mkdir -p "$backup_dir" 2>/dev/null || true
|
|
cp "$current_script" "$backup_file" 2>/dev/null || true
|
|
if [[ -f "$backup_file" ]]; then
|
|
log "INFO" "AGENT UPGRADE: Backup created at $backup_file"
|
|
else
|
|
log "WARN" "AGENT UPGRADE: Could not create backup, proceeding anyway"
|
|
fi
|
|
|
|
# Replace current script with new version (in-place update)
|
|
cp "$temp_script" "$current_script"
|
|
chmod +x "$current_script"
|
|
|
|
log "INFO" "AGENT UPGRADE: Script updated in-place, preparing clean restart"
|
|
|
|
# Mark upgrade as complete
|
|
log "INFO" "AGENT UPGRADE: Notifying backend of upgrade completion..."
|
|
local completion_response=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"completed\"}" 2>&1)
|
|
local completion_status=$?
|
|
|
|
if [[ $completion_status -eq 0 ]]; then
|
|
log "INFO" "AGENT UPGRADE: Backend notified successfully: $completion_response"
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to notify backend: $completion_response"
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Agent upgrade completed successfully"
|
|
log "INFO" "DAEMON: Exiting current process - SystemD will auto-restart with new version"
|
|
|
|
# Clean up temporary files
|
|
rm -f "$temp_script"
|
|
|
|
# Create agent-specific marker file to prevent upgrade loop on restart
|
|
local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME"
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') - Upgrade to $new_version completed" > "$agent_marker" 2>/dev/null || true
|
|
chmod 666 "$agent_marker" 2>/dev/null || true
|
|
|
|
# Exit cleanly - SystemD will automatically restart with new script
|
|
log "INFO" "AGENT UPGRADE: Clean exit - new process will start with version $new_version"
|
|
exit 0
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to download new script"
|
|
# Mark upgrade as failed
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1
|
|
fi
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to fetch new script from server"
|
|
# Mark upgrade as failed
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1
|
|
fi
|
|
}
|
|
|
|
# Check and apply configuration updates (Linux)
|
|
check_config_updates() {
|
|
# Only log config check in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "Checking for configuration updates..."
|
|
|
|
# Use agent-specific config endpoint
|
|
CONFIG_RESPONSE=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "ERROR" "Failed to fetch configuration"
|
|
return 1
|
|
fi
|
|
|
|
log "DEBUG" "Config response received"
|
|
|
|
# Parse JSON response using jq
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
log "ERROR" "jq is required for config parsing but not found"
|
|
return 1
|
|
fi
|
|
|
|
# Extract config details from response
|
|
CONFIG_STATUS=$(echo "$CONFIG_RESPONSE" | jq -r '.status // "unknown"')
|
|
CONFIG_VERSION=$(echo "$CONFIG_RESPONSE" | jq -r '.version // "unknown"')
|
|
CONFIG_CONTENT=$(echo "$CONFIG_RESPONSE" | jq -r '.config_content // ""')
|
|
CONFIG_CHECKSUM=$(echo "$CONFIG_RESPONSE" | jq -r '.checksum // ""')
|
|
|
|
# Update paths dynamically from config API response (cluster settings may change without reinstall)
|
|
local _api_cfg _api_bin _api_sock
|
|
_api_cfg=$(echo "$CONFIG_RESPONSE" | jq -r '.haproxy_config_path // empty' 2>/dev/null)
|
|
_api_bin=$(echo "$CONFIG_RESPONSE" | jq -r '.haproxy_bin_path // empty' 2>/dev/null)
|
|
_api_sock=$(echo "$CONFIG_RESPONSE" | jq -r '.stats_socket_path // empty' 2>/dev/null)
|
|
[[ -n "$_api_cfg" && "$_api_cfg" != "null" ]] && HAPROXY_CONFIG_PATH="$_api_cfg"
|
|
[[ -n "$_api_bin" && "$_api_bin" != "null" ]] && HAPROXY_BIN_PATH="$_api_bin"
|
|
[[ -n "$_api_sock" && "$_api_sock" != "null" ]] && STATS_SOCKET_PATH="$_api_sock"
|
|
|
|
log "DEBUG" "Config status: $CONFIG_STATUS, version: $CONFIG_VERSION, config_path: $HAPROXY_CONFIG_PATH"
|
|
|
|
# Check if config is available
|
|
if [[ "$CONFIG_STATUS" != "available" ]]; then
|
|
log "WARN" "No configuration available (status: $CONFIG_STATUS)"
|
|
return 0
|
|
fi
|
|
|
|
# Check if config content is empty or invalid
|
|
if [[ -z "$CONFIG_CONTENT" || "$CONFIG_CONTENT" == "null" ]]; then
|
|
log "WARN" "Configuration content is empty"
|
|
return 0
|
|
fi
|
|
|
|
# CRITICAL: Prevent overwriting valid config with invalid content
|
|
if [[ "$CONFIG_CONTENT" == "# No configuration available"* ]] || [[ "$CONFIG_CONTENT" == "# Agent is disabled"* ]] || [[ "$CONFIG_CONTENT" == "# No cluster assigned"* ]]; then
|
|
log "WARN" "Received invalid configuration content, preserving existing HAProxy config"
|
|
log "DEBUG" "Invalid config content: ${CONFIG_CONTENT:0:100}..."
|
|
return 0
|
|
fi
|
|
|
|
# Additional validation: Check if config content looks like a valid HAProxy config
|
|
# Note: Partial configs may not have global/defaults, and may also have NO frontend/backend/listen (all deleted)
|
|
if ! echo "$CONFIG_CONTENT" | grep -q -E "(frontend|backend|listen)"; then
|
|
log "INFO" "Partial config has no frontend/backend/listen sections (all entities may have been deleted - this is OK)"
|
|
log "DEBUG" "Config content preview: ${CONFIG_CONTENT:0:200}..."
|
|
# Don't return - continue with merge, will result in global+defaults only config
|
|
fi
|
|
|
|
# Check if this version is already applied (CRITICAL: Prevent reapplying same config)
|
|
if [[ "$CONFIG_VERSION" == "$last_applied_version" ]]; then
|
|
log "DEBUG" "Configuration version $CONFIG_VERSION already applied, skipping"
|
|
# OPTIMIZATION: SSL certificates are fetched separately and independently
|
|
# Only fetch SSL if config hasn't changed to reduce unnecessary API calls
|
|
return 0
|
|
fi
|
|
|
|
# Fetch and deploy SSL certificates when config version changes
|
|
log "INFO" "Config version changed ($last_applied_version → $CONFIG_VERSION), fetching SSL certificates..."
|
|
fetch_and_deploy_ssl_certificates
|
|
local _ssl_changed_in_config_update=${SSL_CERTS_CHANGED_COUNT:-0}
|
|
|
|
# Create backup of current config (CRITICAL: Always backup before changes)
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
local backup_file="$HAPROXY_CONFIG_PATH.backup.$(date +%Y%m%d_%H%M%S)"
|
|
cp "$HAPROXY_CONFIG_PATH" "$backup_file"
|
|
if [[ -f "$backup_file" ]]; then
|
|
log "INFO" "Current config backed up to: $backup_file"
|
|
else
|
|
log "ERROR" "Failed to create config backup, aborting config update"
|
|
return 1
|
|
fi
|
|
else
|
|
log "WARN" "No existing HAProxy config found at: $HAPROXY_CONFIG_PATH"
|
|
fi
|
|
|
|
# Write new configuration to temp file first
|
|
TEMP_CONFIG="/tmp/haproxy_new_$$.cfg"
|
|
echo "$CONFIG_CONTENT" > "$TEMP_CONFIG"
|
|
|
|
# CRITICAL: Check if this is a PARTIAL config (only frontends/backends without global/defaults)
|
|
# Backend generates partial configs to preserve agent-specific global/defaults settings
|
|
if head -20 "$TEMP_CONFIG" | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then
|
|
log "INFO" "CONFIG UPDATE: Detected PARTIAL config - preserving global/defaults from existing haproxy.cfg"
|
|
|
|
# Extract global and defaults sections from current haproxy.cfg
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Extract global section only (from start until any section: defaults/listen/frontend/backend)
|
|
# CRITICAL FIX: Stop at ANY section header, not just 'defaults'
|
|
# This prevents including 'listen stats' in global when it comes before defaults
|
|
# CRITICAL FIX: Match keywords followed by whitespace OR end of line
|
|
# 'defaults' without a name has no trailing whitespace, just newline
|
|
awk '/^(defaults|listen|frontend|backend)([[:space:]]|$)/{exit} {print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-global-$$.cfg 2>/dev/null
|
|
|
|
# Extract defaults section (ONLY the FIRST defaults section)
|
|
# CRITICAL FIX: Exit at ANY section header including another 'defaults'
|
|
# This prevents capturing multiple defaults sections if config is malformed
|
|
# Also match keywords at end of line (no trailing whitespace)
|
|
awk 'BEGIN{started=0} /^defaults([[:space:]]|$)/{if(started) exit; started=1} /^(frontend|backend|listen)([[:space:]]|$)/{if(started) exit} started{print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-defaults-$$.cfg 2>/dev/null
|
|
|
|
# Extract all listen sections (preserve existing listen blocks like stats monitoring)
|
|
# This extracts ALL listen blocks from existing config to preserve them
|
|
awk '
|
|
/^listen[[:space:]]/ {
|
|
if (listen_content != "") {
|
|
# Print previous listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
in_listen=1
|
|
listen_content=$0 "\n"
|
|
next
|
|
}
|
|
in_listen {
|
|
# Check if we hit another section (frontend, backend, listen, global, defaults)
|
|
# CRITICAL FIX: Match keywords followed by whitespace OR end of line
|
|
# This handles "defaults" without trailing whitespace
|
|
if (/^(frontend|backend|listen|global|defaults)([[:space:]]|$)/) {
|
|
# Print current listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
listen_content=""
|
|
in_listen=0
|
|
} else {
|
|
listen_content = listen_content $0 "\n"
|
|
}
|
|
}
|
|
END {
|
|
# Print final listen block if any
|
|
if (listen_content != "") {
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
}
|
|
' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-listen-$$.cfg 2>/dev/null
|
|
|
|
# Check if extraction was successful
|
|
if [[ -s /tmp/haproxy-global-$$.cfg && -s /tmp/haproxy-defaults-$$.cfg ]]; then
|
|
listen_lines=0
|
|
[[ -s /tmp/haproxy-listen-$$.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen-$$.cfg)
|
|
log "INFO" "CONFIG UPDATE: Extracted global ($(wc -l < /tmp/haproxy-global-$$.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults-$$.cfg) lines), listen blocks ($listen_lines lines)"
|
|
|
|
# Merge: existing global + existing defaults + existing listen blocks + new frontends/backends
|
|
merged_config="/tmp/haproxy-merged-$$.cfg"
|
|
|
|
# Copy global section as-is, normalize trailing blanks to exactly 1 line
|
|
# Remove excessive trailing blanks (keep max 1), preserve content
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global-$$.cfg > "$merged_config"
|
|
|
|
# Copy defaults section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults-$$.cfg >> "$merged_config"
|
|
|
|
# Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring)
|
|
if [[ -s /tmp/haproxy-listen-$$.cfg ]]; then
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen-$$.cfg >> "$merged_config"
|
|
log "INFO" "CONFIG UPDATE: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen-$$.cfg) listen blocks from existing config"
|
|
fi
|
|
|
|
# Extract only frontend/backend sections from partial config (skip comments)
|
|
awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' "$TEMP_CONFIG" >> "$merged_config"
|
|
|
|
# Replace temp config with merged version
|
|
mv "$merged_config" "$TEMP_CONFIG"
|
|
log "INFO" "CONFIG UPDATE: Successfully merged - preserved local global/defaults/listen blocks, applied new frontends/backends"
|
|
|
|
# Cleanup extraction temp files
|
|
rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg
|
|
else
|
|
log "ERROR" "CONFIG UPDATE: Failed to extract global/defaults sections from existing config"
|
|
log "ERROR" "CONFIG UPDATE: This will cause HAProxy to fail - aborting config update"
|
|
rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg "$TEMP_CONFIG"
|
|
return 1
|
|
fi
|
|
else
|
|
log "ERROR" "CONFIG UPDATE: No existing haproxy.cfg found - cannot merge partial config"
|
|
log "ERROR" "CONFIG UPDATE: Partial config requires existing global/defaults - aborting"
|
|
rm -f "$TEMP_CONFIG"
|
|
return 1
|
|
fi
|
|
else
|
|
log "INFO" "CONFIG UPDATE: Full config detected - using as-is (includes global/defaults)"
|
|
fi
|
|
|
|
# Log config content for debugging
|
|
log "DEBUG" "Generated config content (first 20 lines):"
|
|
log "DEBUG" "$(head -20 "$TEMP_CONFIG")"
|
|
log "DEBUG" "Config file size: $(wc -l < "$TEMP_CONFIG") lines"
|
|
|
|
# Validate new configuration with retry logic
|
|
log "DEBUG" "Validating config with: $HAPROXY_BIN_PATH -c -f $TEMP_CONFIG"
|
|
VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1)
|
|
VALIDATION_RESULT=$?
|
|
|
|
if [[ $VALIDATION_RESULT -eq 0 ]]; then
|
|
log "INFO" "New configuration is valid"
|
|
|
|
# Apply new configuration
|
|
mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH"
|
|
log "INFO" "Configuration updated to version: $CONFIG_VERSION"
|
|
|
|
# Reload HAProxy service
|
|
reload_haproxy_service
|
|
|
|
# Update last applied version to prevent reapplying same config
|
|
last_applied_version="$CONFIG_VERSION"
|
|
|
|
# Send instant notification to backend for real-time UI sync
|
|
log "INFO" "Sending instant notification for config version: $CONFIG_VERSION"
|
|
NOTIFICATION_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" 2>&1)
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "INSTANT SYNC: Notification sent successfully"
|
|
else
|
|
log "ERROR" "INSTANT SYNC: Notification failed - $NOTIFICATION_RESPONSE"
|
|
fi
|
|
|
|
# Send config sync to update database entities based on actual config
|
|
log "INFO" "Syncing database with current config content"
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Properly escape JSON content
|
|
CONFIG_JSON=$(cat "$HAPROXY_CONFIG_PATH" | jq -Rs .)
|
|
SYNC_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-sync" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"config_content\":$CONFIG_JSON}" 2>&1)
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "CONFIG SYNC: Database synced with current config"
|
|
else
|
|
log "ERROR" "CONFIG SYNC: Database sync failed - $SYNC_RESPONSE"
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "Configuration applied and HAProxy reloaded successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "New configuration validation failed (attempt 1/3)"
|
|
log "ERROR" "Validation error output: $VALIDATION_OUTPUT"
|
|
|
|
# CRITICAL: Retry validation with delay (race condition fix)
|
|
for retry in 2 3; do
|
|
log "INFO" "Retrying validation in 3 seconds... (attempt $retry/3)"
|
|
sleep 3
|
|
|
|
RETRY_VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1)
|
|
RETRY_VALIDATION_RESULT=$?
|
|
|
|
if [[ $RETRY_VALIDATION_RESULT -eq 0 ]]; then
|
|
log "INFO" "Configuration validation successful on retry $retry"
|
|
|
|
# Apply new configuration
|
|
mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH"
|
|
log "INFO" "Configuration updated to version: $CONFIG_VERSION"
|
|
|
|
# Reload HAProxy service
|
|
reload_haproxy_service
|
|
|
|
# Update last applied version to prevent reapplying same config
|
|
last_applied_version="$CONFIG_VERSION"
|
|
|
|
# Send instant notification to backend for real-time UI sync
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" > /dev/null 2>&1
|
|
|
|
log "INFO" "Configuration applied and HAProxy reloaded successfully"
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
log "ERROR" "Configuration validation failed after 3 attempts"
|
|
log "ERROR" "Final validation error: $RETRY_VALIDATION_OUTPUT"
|
|
|
|
# CRITICAL DEBUG: Keep failed config for inspection instead of deleting
|
|
# Move to a debug location so admin can inspect what went wrong
|
|
DEBUG_CONFIG="/tmp/haproxy-failed-${CONFIG_VERSION}-$(date +%Y%m%d-%H%M%S).cfg"
|
|
mv "$TEMP_CONFIG" "$DEBUG_CONFIG" 2>/dev/null || true
|
|
log "ERROR" "Failed config saved to: $DEBUG_CONFIG (for debugging - delete manually after inspection)"
|
|
log "ERROR" "To validate manually: $HAPROXY_BIN_PATH -c -f $DEBUG_CONFIG"
|
|
|
|
# Clean up old failed configs (keep last 5) to prevent disk space issues
|
|
ls -t /tmp/haproxy-failed-*.cfg 2>/dev/null | tail -n +6 | xargs rm -f 2>/dev/null || true
|
|
|
|
# Send validation failure notification to backend
|
|
VALIDATION_ERROR_JSON=$(echo "$RETRY_VALIDATION_OUTPUT" | jq -Rs .)
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-validation-failed" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$CONFIG_VERSION\",\"validation_error\":$VALIDATION_ERROR_JSON}" > /dev/null 2>&1 || true
|
|
|
|
# CRITICAL: If SSL certificates were changed but config validation failed,
|
|
# still reload HAProxy with the EXISTING (known-good) config so it picks up new certs
|
|
if [[ ${_ssl_changed_in_config_update:-0} -gt 0 ]]; then
|
|
log "WARN" "Config validation failed but $_ssl_changed_in_config_update SSL cert(s) changed - reloading HAProxy with current config"
|
|
if "$HAPROXY_BIN_PATH" -c -f "$HAPROXY_CONFIG_PATH" >/dev/null 2>&1; then
|
|
reload_haproxy_service
|
|
log "INFO" "HAProxy reloaded with existing config to apply SSL certificate changes"
|
|
else
|
|
log "ERROR" "Cannot reload HAProxy - even existing config fails validation"
|
|
fi
|
|
fi
|
|
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Main daemon
|
|
run_daemon() {
|
|
# Check for existing running instance (CRITICAL: Prevent multiple agents)
|
|
if [[ -f "$PID_FILE" ]]; then
|
|
local existing_pid=$(cat "$PID_FILE" 2>/dev/null)
|
|
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then
|
|
log "ERROR" "Agent already running with PID: $existing_pid"
|
|
log "ERROR" "Terminating existing instance to prevent conflicts..."
|
|
kill -TERM "$existing_pid" 2>/dev/null || true
|
|
sleep 3
|
|
kill -KILL "$existing_pid" 2>/dev/null || true
|
|
rm -f "$PID_FILE"
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "Starting HAProxy Agent: $AGENT_NAME (Linux)"
|
|
echo $$ > "$PID_FILE"
|
|
|
|
# CRITICAL: Setup graceful shutdown signal handling
|
|
# This allows agent to stop quickly (<2s) instead of waiting for systemd timeout (90s)
|
|
SHUTDOWN_REQUESTED=false
|
|
|
|
trap_handler() {
|
|
log "INFO" "Shutdown signal received (SIGTERM/SIGINT), stopping agent gracefully..."
|
|
SHUTDOWN_REQUESTED=true
|
|
}
|
|
|
|
# Register signal handlers for graceful shutdown
|
|
trap trap_handler SIGTERM SIGINT SIGQUIT
|
|
log "INFO" "Signal handlers registered for graceful shutdown"
|
|
|
|
# Dynamically locate required binaries for daemon operations
|
|
SOCAT_BIN=$(find_binary socat)
|
|
JQ_BIN=$(find_binary jq)
|
|
|
|
if [[ -n "$SOCAT_BIN" ]]; then
|
|
log "INFO" "Found socat: $SOCAT_BIN"
|
|
else
|
|
log "WARN" "socat not found - HAProxy stats will be unavailable (agent will still function)"
|
|
fi
|
|
|
|
if [[ -n "$JQ_BIN" ]]; then
|
|
log "INFO" "Found jq: $JQ_BIN"
|
|
fi
|
|
|
|
log "INFO" "Using curl: $CURL_BIN"
|
|
|
|
# CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart)
|
|
# Agent restart should NOT trigger config reapplication - fetch last applied version from backend
|
|
log "INFO" "Fetching last applied config version from backend..."
|
|
agent_info_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ $? -eq 0 && -n "$agent_info_response" ]]; then
|
|
# Extract applied_config_version for this agent
|
|
last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null)
|
|
|
|
if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then
|
|
last_applied_version="none"
|
|
log "WARN" "Could not fetch applied_config_version from backend, starting with 'none'"
|
|
else
|
|
log "INFO" "Loaded last applied version from database: $last_applied_version"
|
|
fi
|
|
else
|
|
last_applied_version="none"
|
|
log "WARN" "Failed to fetch agent info from backend, starting with version 'none'"
|
|
fi
|
|
|
|
# CRITICAL: Startup delay to avoid race conditions
|
|
log "INFO" "Startup delay for system stabilization..."
|
|
sleep 5
|
|
|
|
# Register agent on startup
|
|
if ! register_agent; then
|
|
log "ERROR" "Failed to register agent, continuing with heartbeats..."
|
|
fi
|
|
|
|
# Send initial heartbeat
|
|
sleep 1
|
|
send_heartbeat
|
|
|
|
# Main agent loop with interruptible sleep for fast shutdown
|
|
# Sleep is broken into 1-second intervals to check shutdown flag every second
|
|
# This allows agent to stop in 1-2 seconds instead of waiting up to 30 seconds
|
|
local _loop_count=0
|
|
|
|
while [[ "$SHUTDOWN_REQUESTED" == "false" ]]; do
|
|
# Interruptible sleep: break 30s into 30x 1s intervals
|
|
for i in {1..30}; do
|
|
# Check shutdown flag every second
|
|
[[ "$SHUTDOWN_REQUESTED" == "true" ]] && break
|
|
sleep 1
|
|
done
|
|
|
|
# Exit loop if shutdown requested
|
|
[[ "$SHUTDOWN_REQUESTED" == "true" ]] && break
|
|
|
|
# Normal agent operations
|
|
send_heartbeat
|
|
|
|
# Check for pending config requests (if function exists)
|
|
if type check_config_requests &>/dev/null; then
|
|
check_config_requests
|
|
fi
|
|
|
|
check_config_updates
|
|
|
|
# Independent SSL certificate sync every 5th iteration (~2.5 minutes)
|
|
# This mirrors the embedded daemon behavior: SSL changes trigger HAProxy reload
|
|
# even when config version hasn't changed (e.g., certificate content renewal)
|
|
_loop_count=$((_loop_count + 1))
|
|
if (( _loop_count % 5 == 0 )); then
|
|
check_ssl_updates
|
|
# Issue #27 — HA/VIP convergence at the SSL cadence (inert for non-VIP nodes)
|
|
if type fetch_and_deploy_keepalived_config &>/dev/null; then
|
|
fetch_and_deploy_keepalived_config
|
|
fi
|
|
fi
|
|
|
|
check_agent_upgrade
|
|
done
|
|
|
|
# Graceful shutdown cleanup
|
|
log "INFO" "Agent stopped gracefully"
|
|
rm -f "$PID_FILE"
|
|
exit 0
|
|
}
|
|
|
|
case "${1:-daemon}" in
|
|
"daemon") run_daemon ;;
|
|
"stop")
|
|
# Improved stop command with graceful → force kill escalation
|
|
if [[ -f "$PID_FILE" ]]; then
|
|
local pid=$(cat "$PID_FILE" 2>/dev/null)
|
|
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
|
echo "Stopping HAProxy Agent (PID: $pid)..."
|
|
|
|
# Try graceful SIGTERM first
|
|
kill -TERM "$pid" 2>/dev/null
|
|
|
|
# Wait up to 5 seconds for graceful shutdown
|
|
local count=0
|
|
while kill -0 "$pid" 2>/dev/null && [[ $count -lt 5 ]]; do
|
|
sleep 1
|
|
count=$((count + 1))
|
|
done
|
|
|
|
# Force kill if still running after 5 seconds
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
echo "Graceful stop timeout, force stopping agent..."
|
|
kill -KILL "$pid" 2>/dev/null
|
|
sleep 1
|
|
fi
|
|
|
|
echo "Agent stopped successfully"
|
|
else
|
|
echo "Agent is not running (stale PID file)"
|
|
fi
|
|
else
|
|
echo "Agent is not running (no PID file)"
|
|
fi
|
|
rm -f "$PID_FILE"
|
|
;;
|
|
*) echo "Usage: $0 {daemon|stop}"; exit 1 ;;
|
|
esac
|
|
AGENT_SCRIPT
|
|
|
|
chmod +x "$INSTALL_DIR/haproxy-agent"
|
|
|
|
# Set comprehensive permissions for self-upgrade capability
|
|
log "INFO" "Configuring agent self-upgrade permissions..."
|
|
|
|
# Agent binary permissions - must be writable by root for in-place upgrades
|
|
chmod 755 "$INSTALL_DIR/haproxy-agent"
|
|
chown root:root "$INSTALL_DIR/haproxy-agent" 2>/dev/null || true
|
|
|
|
# Create backup directory for agent upgrades
|
|
AGENT_BACKUP_DIR="/usr/local/share/haproxy-agent-backups"
|
|
mkdir -p "$AGENT_BACKUP_DIR"
|
|
chmod 755 "$AGENT_BACKUP_DIR"
|
|
chown root:root "$AGENT_BACKUP_DIR" 2>/dev/null || true
|
|
|
|
# Ensure all required directories have proper ownership and permissions
|
|
# This eliminates the need for sudo during upgrades
|
|
log "INFO" "Final permission validation..."
|
|
|
|
# Validate all critical paths are accessible
|
|
for dir_path in "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" "$AGENT_BACKUP_DIR"; do
|
|
if [[ -d "$dir_path" ]] && [[ -w "$dir_path" ]]; then
|
|
log "INFO" "Directory writable: $dir_path"
|
|
else
|
|
log "WARN" "Directory access issue: $dir_path"
|
|
fi
|
|
done
|
|
|
|
# Validate temp directory access
|
|
if touch "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null; then
|
|
log "INFO" "Temp directory access confirmed"
|
|
else
|
|
log "WARN" "Temp directory access issue"
|
|
fi
|
|
|
|
log "INFO" "All agent self-upgrade permissions configured - no sudo required for future operations"
|
|
|
|
# Get pool_id for this cluster
|
|
CLUSTER_POOL_ID=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \
|
|
-H "X-API-Key: $AGENT_TOKEN" | jq -r '.pool_id // 1' 2>/dev/null)
|
|
[[ -z "$CLUSTER_POOL_ID" || "$CLUSTER_POOL_ID" == "null" ]] && CLUSTER_POOL_ID=1
|
|
|
|
# Create configuration
|
|
echo "Creating configuration..."
|
|
cat > "$CONFIG_DIR/config.json" << CONFIG_EOF
|
|
{
|
|
"management": {
|
|
"url": "$MANAGEMENT_URL",
|
|
"token": "$AGENT_TOKEN",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"pool_id": $CLUSTER_POOL_ID
|
|
},
|
|
"agent": {
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"platform": "$platform",
|
|
"architecture": "$ARCH"
|
|
},
|
|
"haproxy": {
|
|
"config_path": "$HAPROXY_CONFIG_PATH",
|
|
"bin_path": "$HAPROXY_BIN_PATH",
|
|
"stats_socket_path": "$STATS_SOCKET_PATH",
|
|
"service_name": "$HAPROXY_SERVICE_NAME"
|
|
}
|
|
}
|
|
CONFIG_EOF
|
|
|
|
# Set config file permissions for upgrade access
|
|
chmod 644 "$CONFIG_DIR/config.json"
|
|
chown root:root "$CONFIG_DIR/config.json" 2>/dev/null || true
|
|
log "INFO" "Configuration file permissions set"
|
|
|
|
# Create systemd service for Linux
|
|
log "DEBUG" "Creating systemd service for Linux..."
|
|
cat > "/etc/systemd/system/haproxy-agent.service" << SERVICE_EOF
|
|
[Unit]
|
|
Description=HAProxy Management Agent
|
|
After=network.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=root
|
|
ExecStart=$INSTALL_DIR/haproxy-agent daemon
|
|
ExecStop=$INSTALL_DIR/haproxy-agent stop
|
|
|
|
# Graceful shutdown configuration
|
|
# Timeout after 10 seconds instead of default 90 seconds
|
|
TimeoutStopSec=10
|
|
# Mixed mode: send SIGTERM to main process, SIGKILL to remaining
|
|
KillMode=mixed
|
|
# Explicit signal for graceful shutdown
|
|
KillSignal=SIGTERM
|
|
|
|
Restart=always
|
|
RestartSec=10
|
|
# Prevent rapid restart loops: max 5 restarts within 2 minutes
|
|
# Allows multiple legitimate upgrades (exit 0 + restart) while catching crash loops
|
|
StartLimitBurst=5
|
|
StartLimitIntervalSec=120
|
|
StandardOutput=append:$LOG_DIR/agent.log
|
|
StandardError=append:$LOG_DIR/agent.log
|
|
Environment=SKIP_TO_DAEMON=true
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
SERVICE_EOF
|
|
|
|
echo "Starting systemd service..."
|
|
systemctl daemon-reload
|
|
systemctl enable haproxy-agent
|
|
systemctl start haproxy-agent
|
|
|
|
sleep 3
|
|
SERVICE_STATUS=$(systemctl is-active haproxy-agent 2>/dev/null || echo "inactive")
|
|
|
|
if [[ "$SERVICE_STATUS" == "active" ]]; then
|
|
echo ""
|
|
log "INFO" "HAProxy Management Agent installed successfully on Linux!"
|
|
echo ""
|
|
echo "Service Information:"
|
|
echo " Status: active"
|
|
echo " Architecture: $ARCH ($ARCH_SUFFIX)"
|
|
echo " Cluster ID: $CLUSTER_ID"
|
|
echo " Agent Name: $AGENT_NAME"
|
|
echo " Config: $CONFIG_DIR/config.json"
|
|
echo " Logs: tail -f $LOG_DIR/agent.log"
|
|
echo ""
|
|
log "DEBUG" "Linux Management Commands:"
|
|
echo " Start: sudo systemctl start haproxy-agent"
|
|
echo " Stop: sudo systemctl stop haproxy-agent"
|
|
echo " Status: sudo systemctl status haproxy-agent"
|
|
echo " Logs: sudo journalctl -u haproxy-agent -f"
|
|
echo ""
|
|
echo "Agent is now running and will:"
|
|
echo " • Send heartbeats every 10 seconds to: $MANAGEMENT_URL"
|
|
echo " • Monitor HAProxy service status and report to management UI"
|
|
echo " • Wait for configuration update tasks from management UI"
|
|
echo " • Apply HAProxy config changes to: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "Your Linux HAProxy instance is now centrally manageable!"
|
|
echo ""
|
|
echo "Next Steps:"
|
|
echo " 1. Check agent logs: tail -f $LOG_DIR/agent.log"
|
|
echo " 2. Verify agent registration in management UI"
|
|
echo " 3. Test configuration deployment from UI"
|
|
echo ""
|
|
else
|
|
log "ERROR" "Agent service failed to start"
|
|
echo "Check logs: tail -n 20 $LOG_DIR/agent.log"
|
|
echo "Check service: systemctl status haproxy-agent"
|
|
exit 1
|
|
fi
|
|
|
|
fi # End of installer section - skip if in daemon mode
|
|
|
|
# Check if script should skip to daemon mode (from upgrade) - EXACT MacOS COPY
|
|
if [[ "$SKIP_TO_DAEMON" == "true" ]]; then
|
|
# Reduced logging on daemon restart to prevent log spam
|
|
log "DEBUG" "AGENT UPGRADE: Daemon mode detected, starting agent daemon..."
|
|
|
|
log "DEBUG" "DAEMON: Starting HAProxy Agent: $AGENT_NAME (Linux)"
|
|
log "DEBUG" "DAEMON: Loading essential functions for daemon mode..."
|
|
|
|
# CRITICAL: Define essential functions within SKIP_TO_DAEMON block
|
|
# These functions are needed by daemon loop but aren't accessible from main script scope
|
|
|
|
# Find binary utility (needed by stats functions)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=""
|
|
|
|
# Common binary paths
|
|
for path in "/usr/bin" "/usr/local/bin" "/bin" "/usr/sbin" "/usr/local/sbin" "/sbin" "/opt/homebrew/bin"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
binary_path="$path/$binary_name"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
else
|
|
command -v "$binary_name" 2>/dev/null || echo ""
|
|
fi
|
|
}
|
|
|
|
# Get HAProxy status (Linux specific)
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
elif pgrep -f "haproxy" > /dev/null 2>&1; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get full HAProxy stats CSV from socket (for dashboard metrics)
|
|
get_haproxy_stats_csv() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "STATS: socat not available, cannot fetch stats CSV"
|
|
echo ""
|
|
return
|
|
fi
|
|
log "INFO" "STATS: Initialized socat: $SOCAT_BIN"
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH"
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
# Return CSV data (base64 encoded to avoid JSON escaping issues)
|
|
if [[ -n "$stats_output" ]]; then
|
|
log "INFO" "STATS: Fetched CSV: ${#stats_output} bytes, $(echo "$stats_output" | wc -l) lines, base64: $(echo "$stats_output" | base64 -w 0 | wc -c) chars"
|
|
echo "$stats_output" | base64 -w 0
|
|
else
|
|
log "WARN" "STATS: No stats data retrieved from socket: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
get_server_statuses() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "SOCAT not available for server status checks"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "Stats socket not found: $STATS_SOCKET_PATH"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket with permission fix attempt
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "WARN" "Failed to query server stats via socket after permission fix"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
else
|
|
log "WARN" "Failed to query server stats via socket"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Parse stats output and build JSON - EXACT COPY from main script
|
|
local backend_servers=()
|
|
while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do
|
|
# Skip header and empty lines
|
|
[[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue
|
|
|
|
# Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0)
|
|
[[ "$type" != "2" ]] && continue
|
|
|
|
# Extract backend and server names
|
|
local backend_name="$pxname"
|
|
local server_name="$svname"
|
|
|
|
# Map HAProxy status to our format
|
|
local server_status
|
|
case "$status" in
|
|
"UP") server_status="UP" ;;
|
|
"DOWN") server_status="DOWN" ;;
|
|
"MAINT") server_status="MAINT" ;;
|
|
"DRAIN") server_status="DRAIN" ;;
|
|
"NOLB") server_status="NOLB" ;;
|
|
*) server_status="UNKNOWN" ;;
|
|
esac
|
|
|
|
backend_servers+=("$backend_name:$server_name:$server_status")
|
|
done <<< "$stats_output"
|
|
|
|
# Build JSON structure - EXACT COPY from main script
|
|
if [[ ${#backend_servers[@]} -gt 0 ]]; then
|
|
local json_parts=()
|
|
local current_backend=""
|
|
local backend_json=""
|
|
|
|
for entry in "${backend_servers[@]}"; do
|
|
IFS=':' read -r backend_name server_name server_status <<< "$entry"
|
|
|
|
if [[ "$current_backend" != "$backend_name" ]]; then
|
|
# Save previous backend if exists
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
# Start new backend
|
|
current_backend="$backend_name"
|
|
backend_json=""
|
|
fi
|
|
|
|
backend_json+="\"$server_name\": \"$server_status\","
|
|
done
|
|
|
|
# Save last backend
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
|
|
# Combine all backends
|
|
if [[ ${#json_parts[@]} -gt 0 ]]; then
|
|
local IFS=','
|
|
server_statuses_json="{${json_parts[*]}}"
|
|
fi
|
|
fi
|
|
|
|
echo "$server_statuses_json"
|
|
}
|
|
|
|
# Collect basic system information for heartbeat
|
|
collect_system_info() {
|
|
local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | tr -d ',')
|
|
local disk_usage=$(df -h / 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
|
local memory_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}')
|
|
|
|
cat <<SYSTEM_INFO_EOF
|
|
{
|
|
"timestamp": "$(date -u '+%Y-%m-%dT%H:%M:%SZ')",
|
|
"load_average": "${load_avg}",
|
|
"disk_usage": "${disk_usage}",
|
|
"memory_usage": "${memory_usage}"
|
|
}
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Detect keepalived VRRP state (MASTER/BACKUP) and virtual IP (DAEMON version)
|
|
get_keepalive_state() {
|
|
local state=""
|
|
local vip=""
|
|
|
|
if ! systemctl is-active --quiet keepalived 2>/dev/null; then
|
|
if ! pgrep -x keepalived >/dev/null 2>&1; then
|
|
echo "|"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
if command -v journalctl &>/dev/null; then
|
|
state=$(journalctl -u keepalived -n 50 --no-pager 2>/dev/null \
|
|
| grep -oE "(MASTER|BACKUP|FAULT)" | tail -1)
|
|
fi
|
|
|
|
if [[ -z "$state" ]]; then
|
|
for logfile in /var/log/messages /var/log/syslog /var/log/keepalived.log; do
|
|
if [[ -r "$logfile" ]]; then
|
|
state=$(tail -200 "$logfile" 2>/dev/null \
|
|
| grep -i keepalived | grep -oE "(MASTER|BACKUP|FAULT)" | tail -1)
|
|
[[ -n "$state" ]] && break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Method 3: VIP presence on local interfaces — portable, logging-independent.
|
|
# MASTER iff ANY configured VIP is held locally (exact whole-line match, all VIPs).
|
|
if [[ -z "$state" ]] && [[ -r /etc/keepalived/keepalived.conf ]]; then
|
|
local conf_vips=$(grep -A20 'virtual_ipaddress' /etc/keepalived/keepalived.conf 2>/dev/null \
|
|
| grep -oE '([0-9]{1,3}\.){3}[0-9]{1,3}')
|
|
if [[ -n "$conf_vips" ]]; then
|
|
local local_ips=$(ip -4 -o addr show 2>/dev/null | grep -oE 'inet ([0-9]{1,3}\.){3}[0-9]{1,3}' | awk '{print $2}')
|
|
state="BACKUP"
|
|
local _cvip
|
|
for _cvip in $conf_vips; do
|
|
if printf '%s\n' "$local_ips" | grep -Fxq "$_cvip"; then state="MASTER"; break; fi
|
|
done
|
|
fi
|
|
fi
|
|
|
|
if [[ -r /etc/keepalived/keepalived.conf ]]; then
|
|
vip=$(grep -A10 'virtual_ipaddress' /etc/keepalived/keepalived.conf 2>/dev/null \
|
|
| grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -1)
|
|
fi
|
|
|
|
echo "${state}|${vip}"
|
|
return 0
|
|
}
|
|
|
|
# Send heartbeat (DAEMON version - includes all stats)
|
|
send_heartbeat() {
|
|
local haproxy_status=$(get_haproxy_status)
|
|
local server_statuses=$(get_server_statuses)
|
|
local haproxy_stats_csv=$(get_haproxy_stats_csv)
|
|
local system_info=$(collect_system_info)
|
|
|
|
# Detect primary IP for backend update
|
|
local agent_ip=""
|
|
agent_ip=$(ip route get 8.8.8.8 2>/dev/null | awk '{print $7; exit}' || hostname -I 2>/dev/null | awk '{print $1}' || echo "")
|
|
|
|
# Get HAProxy version for heartbeat (safe extraction, fallback to "unknown")
|
|
local haproxy_version="unknown"
|
|
if command -v haproxy &> /dev/null; then
|
|
haproxy_version=$(haproxy -v 2>/dev/null | head -1 | awk '{print $3}' || echo "unknown")
|
|
fi
|
|
|
|
# Ensure platform is available (global variable set during registration)
|
|
if [[ -z "$platform" ]]; then
|
|
platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
fi
|
|
|
|
# Detect keepalived VRRP state
|
|
local keepalive_info=$(get_keepalive_state)
|
|
local keepalive_state=$(echo "$keepalive_info" | cut -d'|' -f1)
|
|
local keepalive_ip=$(echo "$keepalive_info" | cut -d'|' -f2)
|
|
|
|
# Network interfaces (DAEMON) — reported so the HA/VIP create form can offer this
|
|
# node's real NICs. /sys/class/net is universal across distros (no iproute2 needed);
|
|
# exclude loopback here, and the UI further hides docker/veth/bridge interfaces. Empty
|
|
# -> "[]" (never "[\"\"]") so a node with no NICs doesn't surface a blank option.
|
|
local net_ifaces ni_json
|
|
net_ifaces=$(ls /sys/class/net 2>/dev/null | grep -vE '^lo$' | tr '\n' ',' | sed 's/,$//')
|
|
if [ -n "$net_ifaces" ]; then ni_json="[\"${net_ifaces//,/\",\"}\"]"; else ni_json="[]"; fi
|
|
|
|
# Prepare heartbeat payload with all data
|
|
local heartbeat_payload="{
|
|
\"name\": \"$AGENT_NAME\",
|
|
\"hostname\": \"$(hostname)\",
|
|
\"ip_address\": \"$agent_ip\",
|
|
\"status\": \"online\",
|
|
\"platform\": \"$platform\",
|
|
\"architecture\": \"$(uname -m)\",
|
|
\"version\": \"{{AGENT_VERSION}}\",
|
|
\"capabilities\": [\"haproxy_management\", \"ssl_deployment\", \"config_reload\", \"systemd_service\", \"keepalived_management\"],
|
|
\"network_interfaces\": $ni_json,
|
|
\"haproxy_status\": \"$haproxy_status\",
|
|
\"haproxy_version\": \"$haproxy_version\",
|
|
\"cluster_id\": $CLUSTER_ID,
|
|
\"server_statuses\": $server_statuses,
|
|
\"system_info\": $system_info"
|
|
|
|
# Always send keepalive state (even empty) so backend can clear stale data
|
|
heartbeat_payload+=",\"keepalive_state\": \"$keepalive_state\",\"keepalive_ip\": \"$keepalive_ip\""
|
|
|
|
# Include applied config version for backend tracking
|
|
heartbeat_payload+=",\"applied_config_version\": \"${last_applied_version:-none}\""
|
|
|
|
# CRITICAL: Add haproxy_stats_csv only if available (for dashboard metrics)
|
|
if [[ -n "$haproxy_stats_csv" && "$haproxy_stats_csv" != "" ]]; then
|
|
heartbeat_payload+=",\"haproxy_stats_csv\": \"$haproxy_stats_csv\""
|
|
log "INFO" "HEARTBEAT: Including stats CSV (${#haproxy_stats_csv} chars) with server statuses"
|
|
else
|
|
log "WARN" "HEARTBEAT: No server statuses and no stats CSV - minimal heartbeat"
|
|
fi
|
|
|
|
heartbeat_payload+="}"
|
|
|
|
# Send heartbeat to backend with HTTP status code check
|
|
# Use temp files for both payload and response to handle large stats CSV (>200KB, production can be >1MB)
|
|
local temp_payload="/tmp/heartbeat_payload_$$.json"
|
|
local temp_response="/tmp/heartbeat_response_$$.txt"
|
|
|
|
# Write payload to temp file to avoid any size limits
|
|
printf '%s' "$heartbeat_payload" > "$temp_payload"
|
|
|
|
# Send request using temp file
|
|
curl -k -s -w "\n%{http_code}" -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
--data-binary @"$temp_payload" > "$temp_response" 2>&1
|
|
|
|
local http_code=$(tail -n1 "$temp_response" 2>/dev/null || echo "000")
|
|
local response_body=$(head -n-1 "$temp_response" 2>/dev/null || echo "")
|
|
|
|
# Cleanup temp files
|
|
rm -f "$temp_payload" "$temp_response"
|
|
|
|
# Check HTTP status code
|
|
if [[ "$http_code" == "200" || "$http_code" == "201" ]]; then
|
|
# Success - only log in debug mode to reduce spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "HEARTBEAT: Sent successfully (HTTP $http_code)"
|
|
else
|
|
# Failed - log error with details
|
|
log "ERROR" "Heartbeat failed (HTTP $http_code)"
|
|
if [[ -n "$response_body" && "$response_body" != "" ]]; then
|
|
# Log first 200 chars of error response
|
|
local error_preview="${response_body:0:200}"
|
|
log "ERROR" "Backend response: $error_preview"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Check for pending configuration requests (DAEMON MODE)
|
|
check_config_requests() {
|
|
local curl_bin="${CURL_BIN:-$(find_binary curl)}"
|
|
|
|
# Get pending config requests from backend
|
|
log "DEBUG" "CONFIG: Checking for pending config requests from: ${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests"
|
|
|
|
local response=$("$curl_bin" -k -s -X GET "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}")
|
|
|
|
log "DEBUG" "CONFIG: Response: ${response:0:200}..."
|
|
|
|
# Check if there are pending requests
|
|
local pending_count=$(echo "$response" | jq -r '.pending_requests | length' 2>/dev/null)
|
|
|
|
log "DEBUG" "CONFIG: Pending count: $pending_count"
|
|
|
|
if [[ "$pending_count" -gt 0 ]]; then
|
|
log "INFO" "CONFIG: Found $pending_count pending config request(s)"
|
|
|
|
# Process each request
|
|
echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do
|
|
local request_id=$(echo "$request" | jq -r '.request_id')
|
|
local request_type=$(echo "$request" | jq -r '.request_type')
|
|
|
|
log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)"
|
|
|
|
# Read haproxy.cfg content
|
|
local config_path="${HAPROXY_CONFIG_PATH}"
|
|
if [[ -f "$config_path" ]]; then
|
|
local file_size=$(wc -c < "$config_path")
|
|
log "INFO" "CONFIG: Reading config from: $config_path (size: $file_size bytes)"
|
|
|
|
local config_content=$(cat "$config_path")
|
|
|
|
# Escape config content for JSON
|
|
log "DEBUG" "CONFIG: Escaping config content for JSON..."
|
|
local escaped_content=$(echo "$config_content" | jq -Rs .)
|
|
|
|
if [[ -z "$escaped_content" ]]; then
|
|
log "ERROR" "CONFIG: JSON escaping failed for config content!"
|
|
continue
|
|
fi
|
|
|
|
local escaped_size=${#escaped_content}
|
|
log "DEBUG" "CONFIG: JSON escaped content size: $escaped_size bytes"
|
|
|
|
# Submit config response
|
|
local response_payload=$(cat <<CONFIG_RESPONSE_EOF
|
|
{
|
|
"request_id": $request_id,
|
|
"config_content": $escaped_content,
|
|
"config_path": "$config_path"
|
|
}
|
|
CONFIG_RESPONSE_EOF
|
|
)
|
|
|
|
local payload_size=${#response_payload}
|
|
log "INFO" "CONFIG: Sending config response (payload size: $payload_size bytes)..."
|
|
|
|
local http_response=$("$curl_bin" -k -s -w "\n%{http_code}" -X POST "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/config-response" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}" \
|
|
-d "$response_payload")
|
|
|
|
local http_code=$(echo "$http_response" | tail -n1)
|
|
local response_body=$(echo "$http_response" | head -n-1)
|
|
|
|
if [[ "$http_code" == "200" || "$http_code" == "201" ]]; then
|
|
log "INFO" "CONFIG: Config response sent for request #$request_id (size: $file_size bytes, HTTP $http_code)"
|
|
else
|
|
log "ERROR" "CONFIG: Config response failed for request #$request_id (HTTP $http_code): $response_body"
|
|
fi
|
|
else
|
|
log "ERROR" "CONFIG: Config file not found: $config_path"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
log "DEBUG" "DAEMON: Essential functions loaded for daemon mode (including check_config_requests)"
|
|
|
|
# CRITICAL FIX: Remove misleading "upgrade completion" heartbeat from daemon startup
|
|
# Agent will send normal heartbeat in daemon loop - no need for special startup heartbeat
|
|
# This was causing confusion in logs and potential upgrade loop issues
|
|
log "DEBUG" "DAEMON: Starting continuous daemon mode..."
|
|
|
|
# Initialize daemon environment
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/tmp/haproxy-agent-$AGENT_NAME.pid"
|
|
|
|
# Write PID (use /tmp to avoid permission issues)
|
|
echo $$ > "$PID_FILE" 2>/dev/null || true
|
|
|
|
# CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart)
|
|
# Agent restart/upgrade should NOT trigger config reapplication - fetch last applied version
|
|
log "DEBUG" "DAEMON: Fetching last applied config version from backend..."
|
|
agent_info_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ $? -eq 0 && -n "$agent_info_response" ]]; then
|
|
# Extract applied_config_version for this agent using jq
|
|
last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null)
|
|
|
|
if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then
|
|
last_applied_version="none"
|
|
log "WARN" "DAEMON: Could not fetch applied_config_version, starting with 'none'"
|
|
else
|
|
log "INFO" "DAEMON: Loaded last applied version from database: $last_applied_version"
|
|
fi
|
|
else
|
|
last_applied_version="none"
|
|
log "WARN" "DAEMON: Failed to fetch agent info, starting with version 'none'"
|
|
fi
|
|
|
|
last_validation_failed_version="none"
|
|
|
|
# CRITICAL: Initialize HAProxy paths BEFORE daemon loop so SSL reload works from first iteration
|
|
# Without this, HAPROXY_BIN/HAPROXY_CONFIG are empty until first config version change,
|
|
# causing SSL-triggered reloads to silently fail with "config validation failed"
|
|
HAPROXY_BIN="${HAPROXY_BIN_PATH}"
|
|
HAPROXY_CONFIG="${HAPROXY_CONFIG_PATH}"
|
|
log "INFO" "DAEMON: Initialized HAProxy paths - bin: $HAPROXY_BIN, config: $HAPROXY_CONFIG"
|
|
|
|
# Issue #27 (v1.7.0) — HA/VIP (Keepalived) convergence (DAEMON copy). Inert by
|
|
# default: a node with no APPLIED VIP gets not_configured and (lacking our marker)
|
|
# does nothing. Never clobbers a hand-managed keepalived. $AGENT_TOKEN is synced to
|
|
# the live token at the top of each loop iteration below.
|
|
fetch_and_deploy_keepalived_config() {
|
|
local marker="# Managed by HAProxy OpenManager"
|
|
local resp status http_code we_own="false" conf chk
|
|
|
|
# Timeouts so a hung management server can never stall the daemon loop.
|
|
resp=$(curl -k -s --connect-timeout 10 --max-time 30 -w '\n%{http_code}' -X GET \
|
|
"$MANAGEMENT_URL/api/agents/$AGENT_NAME/keepalived-config" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null) || return 0
|
|
http_code="${resp##*$'\n'}" # last line = HTTP status
|
|
resp="${resp%$'\n'*}" # everything before = JSON body
|
|
[[ -z "$resp" ]] && return 0
|
|
status=$(echo "$resp" | jq -r '.status // "unknown"' 2>/dev/null)
|
|
# A 404 means this agent was deleted server-side (node decommissioned / pool removed):
|
|
# self-heal teardown OUR keepalived so a removed node stops advertising the VIP. The
|
|
# teardown branch below is marker-guarded, so a node we don't manage stays untouched.
|
|
[[ "$http_code" == "404" ]] && status="teardown"
|
|
# Cluster-driven keepalived.conf path (delivered in every response); default is universal.
|
|
conf=$(echo "$resp" | jq -r '.config_path // empty' 2>/dev/null)
|
|
[[ -z "$conf" || "$conf" == "null" ]] && conf="/etc/keepalived/keepalived.conf"
|
|
chk="$(dirname "$conf")/check_haproxy.sh"
|
|
if [[ -f "$conf" ]] && grep -q "$marker" "$conf" 2>/dev/null; then we_own="true"; fi
|
|
|
|
_kp_report() {
|
|
local vid="${2:-null}"; [[ -z "$2" ]] && vid="null"
|
|
curl -k -s --connect-timeout 10 --max-time 30 -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/keepalived-status" \
|
|
-H "X-API-Key: $AGENT_TOKEN" -H "Content-Type: application/json" \
|
|
-d "{\"vip_id\":${vid},\"state\":\"$1\",\"config_hash\":\"${3:-}\",\"message\":\"${4:-}\"}" \
|
|
>/dev/null 2>&1 || true
|
|
}
|
|
_kp_teardown() {
|
|
# $1=purge ("true" only on an explicit operator opt-in delete). Default: stop+disable
|
|
# keepalived and remove OUR config (VIP released) but KEEP the package. Purge only when
|
|
# opted in AND we installed it (.hom_installed marker) — never an admin's package.
|
|
local purge="${1:-false}" marker_inst; marker_inst="$(dirname "$conf")/.hom_installed"
|
|
log "INFO" "KEEPALIVED: tearing down our managed VIP config (purge=$purge)"
|
|
systemctl stop keepalived >/dev/null 2>&1
|
|
systemctl disable keepalived >/dev/null 2>&1
|
|
rm -f "$conf" "$chk"
|
|
if [[ "$purge" == "true" && -f "$marker_inst" ]]; then
|
|
log "INFO" "KEEPALIVED: uninstalling keepalived package (operator opt-in)"
|
|
if command -v apt-get >/dev/null 2>&1; then timeout 300 apt-get purge -y -qq keepalived >/dev/null 2>&1
|
|
elif command -v dnf >/dev/null 2>&1; then timeout 300 dnf remove -y -q keepalived >/dev/null 2>&1
|
|
elif command -v yum >/dev/null 2>&1; then timeout 300 yum remove -y -q keepalived >/dev/null 2>&1
|
|
elif command -v zypper >/dev/null 2>&1; then timeout 300 zypper --non-interactive remove -y keepalived >/dev/null 2>&1
|
|
elif command -v apk >/dev/null 2>&1; then timeout 300 apk del keepalived >/dev/null 2>&1
|
|
fi
|
|
rm -f "$marker_inst"
|
|
if command -v keepalived >/dev/null 2>&1; then
|
|
_kp_report "disabled" "" "" "config removed; package uninstall attempted (still present)"
|
|
else
|
|
_kp_report "disabled" "" "" "torn down + keepalived package uninstalled"
|
|
fi
|
|
elif [[ "$purge" == "true" ]]; then
|
|
log "INFO" "KEEPALIVED: package was pre-existing (not installed by us) — left in place; our config removed"
|
|
_kp_report "disabled" "" "" "torn down (package left: pre-existing, not installed by us)"
|
|
else
|
|
_kp_report "disabled" "" "" "torn down"
|
|
fi
|
|
}
|
|
|
|
case "$status" in
|
|
available) : ;;
|
|
teardown)
|
|
local _kp_purge; _kp_purge=$(echo "$resp" | jq -r '.purge // false' 2>/dev/null)
|
|
[[ "$we_own" == "true" ]] && _kp_teardown "$_kp_purge" # explicit delete: honor opt-in purge
|
|
return 0 ;;
|
|
not_configured)
|
|
[[ "$we_own" == "true" ]] && _kp_teardown "false" # T-2 orphan self-heal: graceful only
|
|
return 0 ;;
|
|
*) return 0 ;;
|
|
esac
|
|
|
|
local vip_id new_conf check_script install_if new_hash
|
|
vip_id=$(echo "$resp" | jq -r '.keepalived.vip_id // empty' 2>/dev/null)
|
|
new_conf=$(echo "$resp" | jq -r '.keepalived.config_content // empty' 2>/dev/null)
|
|
new_hash=$(echo "$resp" | jq -r '.keepalived.config_hash // empty' 2>/dev/null)
|
|
check_script=$(echo "$resp" | jq -r '.keepalived.check_script // empty' 2>/dev/null)
|
|
install_if=$(echo "$resp" | jq -r '.keepalived.install_if_missing // false' 2>/dev/null)
|
|
[[ -z "$new_conf" ]] && return 0
|
|
|
|
if [[ -f "$conf" && "$we_own" != "true" ]]; then
|
|
log "WARN" "KEEPALIVED: $conf is externally managed — refusing to overwrite"
|
|
_kp_report "externally_managed" "$vip_id" "" "pre-existing unmanaged keepalived.conf"
|
|
return 0
|
|
fi
|
|
|
|
if ! command -v keepalived >/dev/null 2>&1; then
|
|
if [[ "$install_if" == "true" ]]; then
|
|
log "INFO" "KEEPALIVED: installing package..."
|
|
if command -v apt-get >/dev/null 2>&1; then timeout 300 apt-get install -y -qq keepalived >/dev/null 2>&1
|
|
elif command -v dnf >/dev/null 2>&1; then timeout 300 dnf install -y -q keepalived >/dev/null 2>&1
|
|
elif command -v yum >/dev/null 2>&1; then timeout 300 yum install -y -q keepalived >/dev/null 2>&1
|
|
elif command -v zypper >/dev/null 2>&1; then timeout 300 zypper --non-interactive install -y keepalived >/dev/null 2>&1
|
|
elif command -v apk >/dev/null 2>&1; then timeout 300 apk add --no-cache keepalived >/dev/null 2>&1
|
|
fi
|
|
fi
|
|
if ! command -v keepalived >/dev/null 2>&1; then
|
|
log "ERROR" "KEEPALIVED: not installed/available"
|
|
_kp_report "error" "$vip_id" "" "keepalived not installed"
|
|
return 0
|
|
fi
|
|
# We installed keepalived (it was missing) — drop a marker so an opt-in uninstall on
|
|
# delete removes only OUR install, never an admin's pre-existing keepalived package.
|
|
mkdir -p "$(dirname "$conf")" 2>/dev/null; : > "$(dirname "$conf")/.hom_installed" 2>/dev/null
|
|
fi
|
|
|
|
if [[ -f "$conf" ]]; then
|
|
local cur_hash would_hash
|
|
cur_hash=$(md5sum "$conf" 2>/dev/null | awk '{print $1}')
|
|
would_hash=$(printf '%s' "$new_conf" | md5sum 2>/dev/null | awk '{print $1}')
|
|
if [[ -n "$cur_hash" && "$cur_hash" == "$would_hash" ]]; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$conf")"
|
|
if [[ -n "$check_script" ]]; then
|
|
printf '%s' "$check_script" > "$chk"
|
|
chown root:root "$chk" 2>/dev/null
|
|
chmod 0755 "$chk"
|
|
fi
|
|
# Validate on a TEMP file and swap in only on success: a bad render must never land on
|
|
# $conf (it would also make the md5 idempotency guard above suppress retries forever).
|
|
local tmp_conf="${conf}.hom.tmp"
|
|
printf '%s' "$new_conf" > "$tmp_conf"
|
|
chmod 0644 "$tmp_conf"
|
|
if ! keepalived -t -f "$tmp_conf" >/dev/null 2>&1; then
|
|
rm -f "$tmp_conf"
|
|
log "ERROR" "KEEPALIVED: config validation failed (keepalived -t) — keeping current config, not (re)starting"
|
|
_kp_report "error" "$vip_id" "$new_hash" "keepalived -t failed"
|
|
return 0
|
|
fi
|
|
mv -f "$tmp_conf" "$conf"
|
|
chown root:root "$conf" 2>/dev/null
|
|
chmod 0644 "$conf"
|
|
systemctl enable keepalived >/dev/null 2>&1
|
|
if systemctl reload keepalived >/dev/null 2>&1 || systemctl restart keepalived >/dev/null 2>&1; then
|
|
log "INFO" "KEEPALIVED: applied config for VIP ${vip_id}"
|
|
_kp_report "enabled" "$vip_id" "$new_hash" "applied"
|
|
else
|
|
log "ERROR" "KEEPALIVED: reload/restart failed"
|
|
_kp_report "error" "$vip_id" "$new_hash" "reload/restart failed"
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
log "DEBUG" "DAEMON: Starting agent monitoring loop for $AGENT_NAME"
|
|
|
|
# Enhanced daemon loop with upgrade capability - EXACT MacOS COPY
|
|
while true; do
|
|
sleep 30
|
|
|
|
# Read current token from config (fix for API key mismatch)
|
|
CURRENT_AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null || echo "$AGENT_TOKEN")
|
|
AGENT_TOKEN="$CURRENT_AGENT_TOKEN" # Update AGENT_TOKEN for send_heartbeat function
|
|
|
|
# Ensure platform is available for heartbeat
|
|
[[ -z "$platform" ]] && platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
|
|
# Call send_heartbeat function (includes server_statuses)
|
|
send_heartbeat
|
|
|
|
# Only log heartbeat in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "DAEMON: Heartbeat sent for agent $AGENT_NAME"
|
|
|
|
# Check for pending configuration requests (if function exists)
|
|
if type check_config_requests &>/dev/null; then
|
|
check_config_requests
|
|
fi
|
|
|
|
# Check for SSL certificates and deploy them
|
|
# Silent check - only log if changes are detected (reduce log spam)
|
|
ssl_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ -n "$ssl_response" ]] && ! echo "$ssl_response" | grep -q '"error":'; then
|
|
ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"' 2>/dev/null)
|
|
|
|
if [[ "$ssl_status" == "available" ]]; then
|
|
ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"' 2>/dev/null)
|
|
|
|
if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then
|
|
# Create SSL directory if it doesn't exist
|
|
SSL_DIR="/etc/ssl/haproxy"
|
|
mkdir -p "$SSL_DIR" 2>/dev/null || true
|
|
|
|
# Track if any certificate was actually deployed
|
|
ssl_deployed_count=0
|
|
|
|
# Process each certificate
|
|
# IMPORTANT: Use process substitution to avoid subshell (pipe would lose ssl_deployed_count)
|
|
while IFS= read -r cert_data; do
|
|
if [[ -n "$cert_data" ]]; then
|
|
cert_name=$(echo "$cert_data" | jq -r '.name // "unknown"' 2>/dev/null)
|
|
cert_domain=$(echo "$cert_data" | jq -r '.domain // "unknown"' 2>/dev/null)
|
|
cert_content=$(echo "$cert_data" | jq -r '.certificate_content // ""' 2>/dev/null)
|
|
key_content=$(echo "$cert_data" | jq -r '.private_key_content // ""' 2>/dev/null)
|
|
chain_content=$(echo "$cert_data" | jq -r '.chain_content // ""' 2>/dev/null)
|
|
cert_file_path=$(echo "$cert_data" | jq -r '.file_path // ""' 2>/dev/null)
|
|
usage_type=$(echo "$cert_data" | jq -r '.usage_type // "frontend"' 2>/dev/null)
|
|
|
|
# CRITICAL: Certificate content is always required, but private key depends on usage_type
|
|
# - Frontend SSL: requires both cert AND private key (for bind ssl crt)
|
|
# - Server SSL: requires only cert (CA file for backend verification)
|
|
if [[ -n "$cert_content" && "$cert_content" != "null" ]]; then
|
|
# For frontend SSL, private key is required
|
|
if [[ "$usage_type" == "frontend" && ( -z "$key_content" || "$key_content" == "null" ) ]]; then
|
|
log "WARN" "Skipping Frontend SSL $cert_name: private key missing"
|
|
continue
|
|
fi
|
|
|
|
# Use original path or fallback
|
|
if [[ -n "$cert_file_path" ]]; then
|
|
ssl_file="$cert_file_path"
|
|
else
|
|
ssl_file="$SSL_DIR/${cert_name}.pem"
|
|
fi
|
|
|
|
# Check if certificate already exists and is unchanged
|
|
# IMPORTANT: Calculate checksum with EXACT same format as file will be written
|
|
# Format depends on what content is available:
|
|
# - Frontend SSL: cert + key [+ chain]
|
|
# - Server SSL: cert [+ chain] (no private key)
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
# Has private key - frontend SSL
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s\n\n%s" "$cert_content" "$key_content" "$chain_content")
|
|
else
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$key_content")
|
|
fi
|
|
else
|
|
# No private key - server SSL (CA cert only)
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$chain_content")
|
|
else
|
|
new_content="$cert_content"
|
|
fi
|
|
fi
|
|
|
|
existing_checksum=""
|
|
new_checksum=$(echo "$new_content" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
|
|
if [[ -f "$ssl_file" ]]; then
|
|
existing_checksum=$(cat "$ssl_file" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
fi
|
|
|
|
# Only deploy if certificate is new or changed
|
|
if [[ "$existing_checksum" != "$new_checksum" ]]; then
|
|
# Create combined PEM file
|
|
# Format depends on usage_type:
|
|
# - Frontend SSL: cert + key + [chain]
|
|
# - Server SSL: cert + [chain] (CA cert only, no private key)
|
|
{
|
|
echo "$cert_content"
|
|
# Only add private key if provided (server SSL may not have it)
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
echo ""
|
|
echo "$key_content"
|
|
fi
|
|
# Add chain if present
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
echo ""
|
|
echo "$chain_content"
|
|
fi
|
|
} > "$ssl_file" 2>/dev/null
|
|
|
|
if [[ -f "$ssl_file" ]]; then
|
|
chmod 600 "$ssl_file" 2>/dev/null
|
|
# OPTIMIZATION: Only count toward reload if cert is referenced in HAProxy config
|
|
# Prevents unnecessary reloads for certs not used by this cluster
|
|
if [[ -f "$HAPROXY_CONFIG" ]] && grep -qF "$ssl_file" "$HAPROXY_CONFIG" 2>/dev/null; then
|
|
log "INFO" "DAEMON: SSL certificate deployed (CHANGED & IN USE): $ssl_file ($cert_domain)"
|
|
ssl_deployed_count=$((ssl_deployed_count + 1))
|
|
else
|
|
log "DEBUG" "DAEMON: SSL certificate deployed (CHANGED but NOT in config): $ssl_file ($cert_domain) - no reload needed"
|
|
fi
|
|
else
|
|
log "ERROR" "DAEMON: Failed to deploy SSL certificate: $cert_name"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
done < <(echo "$ssl_certificates" | jq -c '.[]' 2>/dev/null)
|
|
|
|
# CRITICAL: Reload HAProxy if any SSL certificates were updated AND referenced in config
|
|
# This ensures HAProxy picks up the new SSL certificates only when needed
|
|
if [[ $ssl_deployed_count -gt 0 ]]; then
|
|
log "INFO" "DAEMON: SSL certificates updated ($ssl_deployed_count files in use), triggering HAProxy reload..."
|
|
|
|
# Validate HAProxy config before reload
|
|
if "$HAPROXY_BIN" -c -f "$HAPROXY_CONFIG" >/dev/null 2>&1; then
|
|
# Reload HAProxy using systemctl (Linux best practice)
|
|
if systemctl reload haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy reloaded successfully after SSL update"
|
|
else
|
|
log "WARN" "DAEMON: systemctl reload failed after SSL update, attempting restart..."
|
|
if systemctl restart haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy restarted successfully after SSL update"
|
|
else
|
|
log "ERROR" "DAEMON: Failed to reload/restart HAProxy after SSL update"
|
|
fi
|
|
fi
|
|
else
|
|
# IMPORTANT: Config validation may fail if SSL was updated but config not yet synced
|
|
# This is normal - config update will trigger reload with both new SSL and config
|
|
log "WARN" "DAEMON: SSL deployed but HAProxy config not yet synced (validation failed), reload will happen on next config update"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Issue #27 — HA/VIP (Keepalived) convergence at the SSL cadence (~2.5 min);
|
|
# inert for non-VIP nodes (status=not_configured + no ownership marker).
|
|
_kp_loop_count=$(( ${_kp_loop_count:-0} + 1 ))
|
|
if (( _kp_loop_count % 5 == 0 )) && type fetch_and_deploy_keepalived_config &>/dev/null; then
|
|
fetch_and_deploy_keepalived_config
|
|
fi
|
|
|
|
# Check for configuration updates with proper error handling
|
|
config_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
# Validate API response before processing
|
|
if [[ -n "$config_response" ]] && ! echo "$config_response" | grep -q '"error":'; then
|
|
if echo "$config_response" | grep -q '"config_content":'; then
|
|
# Extract config version for tracking
|
|
config_version=$(echo "$config_response" | jq -r '.version // "unknown"' 2>/dev/null)
|
|
|
|
# CRITICAL: Check if this version is already applied (prevent infinite loop)
|
|
if [[ "$config_version" == "$last_applied_version" ]]; then
|
|
# Only log in debug mode to avoid spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration version $config_version already applied, skipping"
|
|
else
|
|
log "INFO" "DAEMON: Configuration update detected (version: $config_version)"
|
|
|
|
# Extract config content for validation
|
|
daemon_config_content=$(echo "$config_response" | jq -r '.config_content' 2>/dev/null)
|
|
|
|
# CRITICAL: Apply same validation as main function
|
|
# IMPORTANT: Don't use 'continue' here - it prevents upgrade check from running!
|
|
config_is_valid=true
|
|
|
|
if [[ -z "$daemon_config_content" || "$daemon_config_content" == "null" ]]; then
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration content is empty, skipping config update"
|
|
config_is_valid=false
|
|
elif [[ "$daemon_config_content" == "# No configuration available"* ]] || [[ "$daemon_config_content" == "# Agent is disabled"* ]] || [[ "$daemon_config_content" == "# No cluster assigned"* ]]; then
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Received placeholder config (no-config state), skipping update"
|
|
config_is_valid=false
|
|
elif ! echo "$daemon_config_content" | grep -q -E "(global|defaults|frontend|backend|listen)"; then
|
|
log "ERROR" "DAEMON: Configuration content does not appear to be valid HAProxy config"
|
|
log "DEBUG" "DAEMON: Config preview: ${daemon_config_content:0:200}..."
|
|
config_is_valid=false
|
|
|
|
# CRITICAL: Report this error to backend for UI display
|
|
# This catches backend-side config generation errors (Python exceptions, etc.)
|
|
config_error_preview="${daemon_config_content:0:500}"
|
|
if [[ -n "$config_version" && "$config_version" != "$last_validation_failed_version" ]]; then
|
|
log "INFO" "DAEMON: Reporting invalid config format to backend..."
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-validation-failed" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"status\": \"invalid_config_format\", \"version\": \"$config_version\", \"validation_error\": $(echo "Configuration does not appear to be valid HAProxy config. Content preview: $config_error_preview" | jq -Rs .)}" \
|
|
>/dev/null 2>&1
|
|
|
|
if [ $? -eq 0 ]; then
|
|
log "INFO" "DAEMON: Invalid config notification sent to backend"
|
|
else
|
|
log "WARN" "DAEMON: Failed to send invalid config notification to backend"
|
|
fi
|
|
|
|
# Mark this version as validation failed to prevent repeated error logs
|
|
# (same pattern as existing HAProxy validation failure at line ~3022)
|
|
last_validation_failed_version="$config_version"
|
|
fi
|
|
fi
|
|
|
|
# Only process config if it's valid
|
|
if [[ "$config_is_valid" == "true" ]]; then
|
|
|
|
# Write validated config to temp file
|
|
echo "$daemon_config_content" > /tmp/haproxy-new-config.cfg 2>/dev/null
|
|
|
|
if [[ -f "/tmp/haproxy-new-config.cfg" && -s "/tmp/haproxy-new-config.cfg" ]]; then
|
|
# Get paths DYNAMICALLY from API response first (cluster config can be changed without reinstall)
|
|
# Fallback to local config file if not present in API response
|
|
api_bin_path=$(echo "$config_response" | jq -r '.haproxy_bin_path // empty' 2>/dev/null)
|
|
api_config_path=$(echo "$config_response" | jq -r '.haproxy_config_path // empty' 2>/dev/null)
|
|
api_socket_path=$(echo "$config_response" | jq -r '.stats_socket_path // empty' 2>/dev/null)
|
|
|
|
# Use API values if present, otherwise fallback to local config
|
|
if [[ -n "$api_bin_path" && "$api_bin_path" != "null" ]]; then
|
|
HAPROXY_BIN="$api_bin_path"
|
|
log "DEBUG" "DAEMON: Using dynamic haproxy_bin_path from cluster: $HAPROXY_BIN"
|
|
else
|
|
HAPROXY_BIN=$(jq -r '.haproxy.bin_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_BIN_PATH}}")
|
|
fi
|
|
|
|
if [[ -n "$api_config_path" && "$api_config_path" != "null" ]]; then
|
|
HAPROXY_CONFIG="$api_config_path"
|
|
log "DEBUG" "DAEMON: Using dynamic haproxy_config_path from cluster: $HAPROXY_CONFIG"
|
|
else
|
|
HAPROXY_CONFIG=$(jq -r '.haproxy.config_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_CONFIG_PATH}}")
|
|
fi
|
|
|
|
if [[ -n "$api_socket_path" && "$api_socket_path" != "null" ]]; then
|
|
STATS_SOCKET="$api_socket_path"
|
|
log "DEBUG" "DAEMON: Using dynamic stats_socket_path from cluster: $STATS_SOCKET"
|
|
else
|
|
STATS_SOCKET=$(jq -r '.haproxy.stats_socket_path' "$CONFIG_FILE" 2>/dev/null || echo "{{STATS_SOCKET_PATH}}")
|
|
fi
|
|
|
|
# CRITICAL NEW FEATURE: Check if this is a partial config (only frontends/backends)
|
|
# Backend now generates partial configs to preserve global/defaults sections
|
|
if head -20 /tmp/haproxy-new-config.cfg | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then
|
|
log "INFO" "DAEMON: Detected PARTIAL config - preserving global/defaults from local haproxy.cfg"
|
|
|
|
# Extract global section from current haproxy.cfg
|
|
if [[ -f "$HAPROXY_CONFIG" ]]; then
|
|
# Extract global section only (from 'global' until any other section: defaults/listen/frontend/backend)
|
|
# CRITICAL FIX: Stop at ANY section header, not just 'defaults'
|
|
# This prevents including 'listen stats' in global when it comes before defaults
|
|
# CRITICAL FIX: Match keywords followed by whitespace OR end of line
|
|
# 'defaults' without a name has no trailing whitespace, just newline
|
|
awk '/^(defaults|listen|frontend|backend)([[:space:]]|$)/{exit} {print}' "$HAPROXY_CONFIG" > /tmp/haproxy-global.cfg 2>/dev/null
|
|
|
|
# Extract defaults section (ONLY the FIRST defaults section)
|
|
# CRITICAL FIX: Exit at ANY section header including another 'defaults'
|
|
# This prevents capturing multiple defaults sections if config is malformed
|
|
# Also match keywords at end of line (no trailing whitespace)
|
|
awk 'BEGIN{started=0} /^defaults([[:space:]]|$)/{if(started) exit; started=1} /^(frontend|backend|listen)([[:space:]]|$)/{if(started) exit} started{print}' "$HAPROXY_CONFIG" > /tmp/haproxy-defaults.cfg 2>/dev/null
|
|
|
|
# Extract all listen sections (preserve existing listen blocks like stats monitoring)
|
|
# CRITICAL FIX: Extract ALL listen blocks from existing config to preserve them
|
|
awk '
|
|
/^listen[[:space:]]/ {
|
|
if (listen_content != "") {
|
|
# Print previous listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
in_listen=1
|
|
listen_content=$0 "\n"
|
|
next
|
|
}
|
|
in_listen {
|
|
# Check if we hit another section (frontend, backend, listen, global, defaults)
|
|
# CRITICAL FIX: Match keywords followed by whitespace OR end of line
|
|
# This handles "defaults" without trailing whitespace
|
|
if (/^(frontend|backend|listen|global|defaults)([[:space:]]|$)/) {
|
|
# Print current listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
listen_content=""
|
|
in_listen=0
|
|
} else {
|
|
listen_content = listen_content $0 "\n"
|
|
}
|
|
}
|
|
END {
|
|
# Print final listen block if any
|
|
if (listen_content != "") {
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
}
|
|
' "$HAPROXY_CONFIG" > /tmp/haproxy-listen.cfg 2>/dev/null
|
|
|
|
# Check if we successfully extracted sections
|
|
if [[ -s /tmp/haproxy-global.cfg && -s /tmp/haproxy-defaults.cfg ]]; then
|
|
listen_lines=0
|
|
[[ -s /tmp/haproxy-listen.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen.cfg)
|
|
log "INFO" "DAEMON: Extracted global ($(wc -l < /tmp/haproxy-global.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults.cfg) lines), listen blocks ($listen_lines lines)"
|
|
|
|
# Merge: global + defaults + listen blocks + new frontends/backends
|
|
# Copy global section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global.cfg > /tmp/haproxy-merged.cfg
|
|
|
|
# Copy defaults section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults.cfg >> /tmp/haproxy-merged.cfg
|
|
|
|
# Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring)
|
|
if [[ -s /tmp/haproxy-listen.cfg ]]; then
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen.cfg >> /tmp/haproxy-merged.cfg
|
|
log "INFO" "DAEMON: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen.cfg) listen blocks from existing config"
|
|
fi
|
|
|
|
# Skip comment header from partial config, add only frontends/backends
|
|
awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' /tmp/haproxy-new-config.cfg >> /tmp/haproxy-merged.cfg
|
|
|
|
# Replace new config with merged version
|
|
mv /tmp/haproxy-merged.cfg /tmp/haproxy-new-config.cfg
|
|
log "INFO" "DAEMON: Config merged - preserved local global/defaults/listen blocks, applied new frontends/backends"
|
|
|
|
# Cleanup temp files
|
|
rm -f /tmp/haproxy-global.cfg /tmp/haproxy-defaults.cfg /tmp/haproxy-listen.cfg
|
|
else
|
|
log "WARN" "DAEMON: Failed to extract global/defaults, using partial config as-is (may fail validation)"
|
|
fi
|
|
else
|
|
log "WARN" "DAEMON: No existing haproxy.cfg found, using partial config as-is"
|
|
fi
|
|
else
|
|
log "INFO" "DAEMON: Full config detected - using as-is"
|
|
fi
|
|
|
|
# CRITICAL: Create backup before applying new config
|
|
daemon_backup_file="${HAPROXY_CONFIG}.backup-$(date +%Y%m%d-%H%M%S)"
|
|
if [[ -f "$HAPROXY_CONFIG" ]]; then
|
|
cp "$HAPROXY_CONFIG" "$daemon_backup_file" 2>/dev/null
|
|
if [[ -f "$daemon_backup_file" ]]; then
|
|
log "INFO" "DAEMON: Config backed up to: $daemon_backup_file"
|
|
else
|
|
log "ERROR" "DAEMON: Failed to create config backup, skipping config update"
|
|
rm -f /tmp/haproxy-new-config.cfg
|
|
fi
|
|
fi
|
|
|
|
# CRITICAL FIX: Validate BEFORE applying (was: apply then validate)
|
|
# Test new config from temp file FIRST
|
|
if "$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c >/dev/null 2>&1; then
|
|
# Validation successful - NOW apply new config
|
|
cp /tmp/haproxy-new-config.cfg "$HAPROXY_CONFIG"
|
|
log "INFO" "DAEMON: Configuration validation successful"
|
|
|
|
# Reload HAProxy using systemctl (Linux best practice)
|
|
if systemctl reload haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy reloaded successfully via systemctl"
|
|
else
|
|
log "WARN" "DAEMON: systemctl reload failed, attempting restart..."
|
|
systemctl restart haproxy >/dev/null 2>&1
|
|
fi
|
|
log "INFO" "DAEMON: Configuration applied and HAProxy reloaded"
|
|
|
|
# CRITICAL: Notify backend of successful config application with version
|
|
log "DEBUG" "Sending config-applied notification for version: $config_version"
|
|
log "DEBUG" "Using token: ${CURRENT_AGENT_TOKEN:0:10}..."
|
|
log "DEBUG" "Agent name: $AGENT_NAME"
|
|
|
|
config_applied_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"status\": \"applied\", \"version\": \"$config_version\"}" 2>&1)
|
|
|
|
log "DEBUG" "Config-applied response: $config_applied_response"
|
|
|
|
if echo "$config_applied_response" | grep -q '"status":"ok"'; then
|
|
log "INFO" "DAEMON: Config-applied notification sent successfully"
|
|
else
|
|
log "ERROR" "DAEMON: Config-applied notification failed: $config_applied_response"
|
|
fi
|
|
|
|
# CRITICAL: Update last applied version to prevent reapplying same config
|
|
last_applied_version="$config_version"
|
|
log "INFO" "DAEMON: Configuration applied successfully (version: $config_version)"
|
|
else
|
|
# CRITICAL: Validation failed - config was NEVER applied to haproxy.cfg
|
|
# No restore needed - original config is still intact
|
|
|
|
# Check if we already reported this validation failure to avoid spam
|
|
if [[ "$config_version" != "$last_validation_failed_version" ]]; then
|
|
log "ERROR" "DAEMON: Configuration validation failed - rejected before applying"
|
|
log "ERROR" "DAEMON: Original HAProxy config remains unchanged"
|
|
# Get validation error details (without 'local' to avoid scope issues)
|
|
validation_error=$("$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c 2>&1)
|
|
log "ERROR" "DAEMON: Validation error: $validation_error"
|
|
|
|
# CRITICAL NEW FEATURE: Notify backend of validation failure for UI display
|
|
log "INFO" "DAEMON: Sending validation error notification to backend..."
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-validation-failed" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"status\": \"validation_failed\", \"version\": \"$config_version\", \"validation_error\": $(echo "$validation_error" | jq -Rs .)}" \
|
|
>/dev/null 2>&1
|
|
|
|
if [ $? -eq 0 ]; then
|
|
log "INFO" "DAEMON: Validation error notification sent to backend successfully"
|
|
else
|
|
log "WARN" "DAEMON: Failed to send validation error notification to backend"
|
|
fi
|
|
|
|
# Mark this version as validation failed to prevent repeated error logs
|
|
last_validation_failed_version="$config_version"
|
|
|
|
# CRITICAL DEBUG FEATURE: Save failed config for troubleshooting
|
|
# Instead of deleting, rename with timestamp for later inspection
|
|
failed_config_timestamp=$(date +%Y%m%d-%H%M%S)
|
|
failed_config_path="/tmp/haproxy-failed-${config_version}-${failed_config_timestamp}.cfg"
|
|
if [[ -f /tmp/haproxy-new-config.cfg ]]; then
|
|
mv /tmp/haproxy-new-config.cfg "$failed_config_path" 2>/dev/null
|
|
if [[ -f "$failed_config_path" ]]; then
|
|
log "INFO" "DAEMON: Failed config saved to: $failed_config_path"
|
|
log "INFO" "DAEMON: Debug: cat $failed_config_path"
|
|
log "INFO" "DAEMON: Debug: haproxy -c -f $failed_config_path"
|
|
# Clean up old failed configs (keep last 5)
|
|
ls -t /tmp/haproxy-failed-*.cfg 2>/dev/null | tail -n +6 | xargs rm -f 2>/dev/null
|
|
fi
|
|
fi
|
|
else
|
|
# Already reported this validation failure, just log in DEBUG to avoid spam
|
|
log "DEBUG" "DAEMON: Config version $config_version still failing validation (already reported)"
|
|
fi
|
|
fi
|
|
|
|
# Clean up temp file (if validation was successful, it's already copied to haproxy.cfg)
|
|
# (if validation failed, it's already moved to haproxy-failed-*.cfg above)
|
|
rm -f /tmp/haproxy-new-config.cfg 2>/dev/null
|
|
fi
|
|
fi # End of config_is_valid check
|
|
fi # End of version already applied check
|
|
fi
|
|
else
|
|
# Log API error for debugging
|
|
if echo "$config_response" | grep -q '"error":'; then
|
|
error_msg=$(echo "$config_response" | jq -r '.error.message' 2>/dev/null || echo "Unknown API error")
|
|
log "DEBUG" "DAEMON: API Error: $error_msg (skipping config check)"
|
|
fi
|
|
fi
|
|
|
|
# Check for upgrade requests - backend controls upgrade via agent status
|
|
# CRITICAL: Always log upgrade check for debugging
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Checking upgrade status (current: {{AGENT_VERSION}})"
|
|
|
|
upgrade_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ -n "$upgrade_response" ]]; then
|
|
should_upgrade=$(echo "$upgrade_response" | jq -r '.should_upgrade // false' 2>/dev/null)
|
|
target_version=$(echo "$upgrade_response" | jq -r '.target_version // ""' 2>/dev/null)
|
|
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade status response: should_upgrade=$should_upgrade, target=$target_version, current={{AGENT_VERSION}}"
|
|
|
|
# Only upgrade if backend says we should AND target version is different from current
|
|
if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "{{AGENT_VERSION}}" ]]; then
|
|
log "INFO" "DAEMON: Starting upgrade from {{AGENT_VERSION}} to $target_version (backend requested)"
|
|
|
|
# Download new script with validation
|
|
temp_script="/tmp/haproxy-agent-upgrade-$AGENT_NAME.sh"
|
|
log "INFO" "DAEMON: Downloading new agent script..."
|
|
|
|
# Get correct pool_id from config
|
|
daemon_pool_id=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null)
|
|
[[ -z "$daemon_pool_id" || "$daemon_pool_id" == "null" ]] && daemon_pool_id=1
|
|
|
|
# CRITICAL FIX: Normalize architecture format (x86_64 -> amd64, aarch64 -> arm64)
|
|
daemon_arch=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')
|
|
|
|
script_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/generate-install-script" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"platform\":\"linux\",\"architecture\":\"$daemon_arch\",\"pool_id\":$daemon_pool_id,\"cluster_id\":$CLUSTER_ID,\"agent_name\":\"$AGENT_NAME\",\"hostname_prefix\":\"$(hostname | cut -d'.' -f1)\",\"haproxy_config_path\":\"/etc/haproxy/haproxy.cfg\",\"haproxy_bin_path\":\"/usr/sbin/haproxy\",\"stats_socket_path\":\"/var/run/haproxy/admin.sock\"}" 2>/dev/null)
|
|
|
|
# Validate API response
|
|
if [[ -z "$script_response" ]]; then
|
|
log "ERROR" "DAEMON: Empty API response, aborting upgrade"
|
|
continue
|
|
fi
|
|
|
|
if ! echo "$script_response" | grep -q '"script"'; then
|
|
log "ERROR" "DAEMON: Invalid API response format, aborting upgrade"
|
|
echo "Response: ${script_response:0:200}..."
|
|
continue
|
|
fi
|
|
|
|
# Extract script content with validation
|
|
echo "$script_response" | jq -r '.script' > "$temp_script" 2>/dev/null
|
|
|
|
# Validate downloaded script
|
|
if [[ ! -f "$temp_script" ]]; then
|
|
log "ERROR" "DAEMON: Failed to create temp script file, aborting upgrade"
|
|
continue
|
|
fi
|
|
|
|
if [[ ! -s "$temp_script" ]]; then
|
|
log "ERROR" "DAEMON: Downloaded script is empty, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
# Check if script is valid bash script
|
|
if ! head -1 "$temp_script" | grep -q "#!/bin/bash"; then
|
|
log "ERROR" "DAEMON: Downloaded script is not a valid bash script, aborting upgrade"
|
|
echo "First line: $(head -1 "$temp_script")"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
script_size=$(wc -c < "$temp_script")
|
|
if [[ $script_size -lt 10000 ]]; then
|
|
log "ERROR" "DAEMON: Downloaded script too small ($script_size bytes), aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Script validation passed ($script_size bytes)"
|
|
|
|
# Create backup before upgrade (ensure backup directory exists)
|
|
mkdir -p "/usr/local/share/haproxy-agent-backups" 2>/dev/null || true
|
|
backup_file="/usr/local/share/haproxy-agent-backups/haproxy-agent-backup-$(date +%Y%m%d-%H%M%S).sh"
|
|
if ! cp /usr/local/bin/haproxy-agent "$backup_file" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to create backup, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Backup created: $backup_file"
|
|
|
|
# Atomic script replacement
|
|
temp_target="/usr/local/bin/haproxy-agent.new"
|
|
if ! cp "$temp_script" "$temp_target" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to copy new script, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
if ! chmod 755 "$temp_target" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to set permissions, aborting upgrade"
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
# Final validation of target script
|
|
if ! head -1 "$temp_target" | grep -q "#!/bin/bash"; then
|
|
log "ERROR" "DAEMON: Target script corrupted, restoring backup"
|
|
cp "$backup_file" /usr/local/bin/haproxy-agent
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
# Atomic move
|
|
if ! mv "$temp_target" /usr/local/bin/haproxy-agent 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to replace script, restoring backup"
|
|
cp "$backup_file" /usr/local/bin/haproxy-agent 2>/dev/null
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Script updated successfully"
|
|
|
|
# Notify completion
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"version\":\"$target_version\",\"status\":\"completed\"}" >/dev/null 2>&1
|
|
|
|
log "INFO" "DAEMON: Upgrade to $target_version completed - restarting..."
|
|
|
|
# Clean up
|
|
rm -f "$temp_script"
|
|
|
|
# Exit to allow service restart with new script
|
|
exit 0
|
|
else
|
|
# Debug: Log why upgrade was not performed
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade not needed - should_upgrade: $should_upgrade, target_version: $target_version, current: {{AGENT_VERSION}}"
|
|
fi
|
|
fi
|
|
done
|
|
# CRITICAL FIX: Daemon loop should never exit
|
|
log "ERROR" "DAEMON: Daemon loop exited unexpectedly, restarting..."
|
|
exit 1
|
|
fi
|
|
|