#!/bin/bash set -e # HAProxy Management Agent Installation Script for Linux # Generated by HAProxy Management UI on: $(date -u '+%Y-%m-%d %H:%M:%S UTC') # Target Cluster ID: $CLUSTER_ID # Script Build ID: 20250925-v1.0.2-ssl-deployment-ready # Early variable definition for backup (before interactive mode) HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}" # Quick daemon mode detection (before installer UI) # Suppress installer messages if running in daemon mode QUIET_MODE=false if [[ "$1" == "daemon" || "$SKIP_TO_DAEMON" == "true" || -f "/etc/haproxy-agent/config.json" ]]; then QUIET_MODE=true fi # Show installer header only in interactive installation mode if [[ "$QUIET_MODE" != "true" ]]; then echo "==========================================" echo "HAProxy Management Agent Installer v{{AGENT_VERSION}}" echo "Linux Edition" echo "With Version Management & Auto-Upgrade" echo "==========================================" echo "" echo "⚠️ IMPORTANT WARNING ⚠️" echo "==========================================" echo "" echo "After agent installation, ALL your existing frontend and backend" echo "configurations will be removed to enable full management through" echo "HAProxy OpenManager." echo "" echo "An automatic backup will be created: haproxy.cfg.initial-backup" echo "" echo "📋 REQUIRED STEPS AFTER INSTALLATION:" echo " 1. Login to HAProxy OpenManager interface" echo " 2. Use the BULK IMPORT feature to restore your configuration" echo "" echo "⚠️ IMPORTANT: After installation, ALL configuration changes MUST be" echo " made through the HAProxy OpenManager interface only." echo " Manual changes will be overwritten on the next update." echo "" echo "==========================================" echo "" # Create automatic backup BEFORE asking for confirmation if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then BACKUP_PATH="${HAPROXY_CONFIG_PATH}.initial-backup" if [[ ! -f "$BACKUP_PATH" ]]; then echo "📦 Creating backup: $BACKUP_PATH" cp "$HAPROXY_CONFIG_PATH" "$BACKUP_PATH" && echo "✅ Backup created successfully" || echo "⚠️ Backup failed" echo "" fi fi read -p "Do you understand and wish to continue? (Y/N): " -n 1 -r echo "" if [[ ! $REPLY =~ ^[Yy]$ ]]; then echo "Installation cancelled." exit 1 fi echo "" echo "Proceeding with installation..." echo "" fi # ============================================ # CRITICAL: Function Definitions (Must be at top for daemon mode) # ============================================ # Dynamically locate required binaries find_binary() { local binary_name="$1" local binary_path=$(command -v "$binary_name" 2>/dev/null) if [[ -n "$binary_path" ]]; then echo "$binary_path" return 0 fi # Fallback: search in common locations local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin") for path in "${search_paths[@]}"; do if [[ -x "$path/$binary_name" ]]; then echo "$path/$binary_name" return 0 fi done return 1 } # Logging function - JSON format, single line with timestamp log() { local level="$1" local message="$2" local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') local log_file="${LOG_FILE:-/var/log/haproxy-agent/agent.log}" local agent_name="${AGENT_NAME:-unknown}" # Ensure log directory exists mkdir -p "$(dirname "$log_file")" 2>/dev/null || true # Escape message for JSON (replace backslash, quotes, newlines, tabs) local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ') # Remove any control characters and emojis (keep only printable ASCII + basic UTF-8) escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g') # Write JSON log entry (single line) - ONLY to log file echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$agent_name\",\"message\":\"$escaped_message\"}" >> "$log_file" 2>/dev/null || true # DO NOT print to stdout in daemon mode (causes duplicate logs) # Installer messages are shown via direct echo commands } # Get HAProxy service status get_haproxy_status() { if systemctl is-active --quiet haproxy 2>/dev/null; then echo "running" else echo "stopped" fi } # Get server statuses from HAProxy stats socket # Old get_server_statuses() removed - using new version with lazy init (see line ~900) # Collect system information collect_system_info() { local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux") local kernel_version=$(uname -r 2>/dev/null || echo "Unknown") local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0") local cpu_count=$(nproc 2>/dev/null || echo "1") local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0") local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0") local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "") local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "") cat </dev/null) if [[ "$pending_count" -gt 0 ]]; then log "INFO" "CONFIG: Found $pending_count pending config request(s)" # Process each request echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do local request_id=$(echo "$request" | jq -r '.request_id') local request_type=$(echo "$request" | jq -r '.request_type') log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)" # Read haproxy.cfg content local config_path="${HAPROXY_CONFIG_PATH:-{{HAPROXY_CONFIG_PATH}}}" if [[ -f "$config_path" ]]; then local config_content=$(cat "$config_path") local file_size=$(wc -c < "$config_path") # Escape config content for JSON local escaped_content=$(echo "$config_content" | jq -Rs .) # Submit config response local response_payload=$(cat < /dev/null 2>&1 log "INFO" "Config response sent for request #$request_id (size: $file_size bytes)" else log "ERROR" "Config file not found: $config_path" fi done fi } # ============================================ # End of Function Definitions # ============================================ # CRITICAL FIX: Enhanced daemon mode detection for Linux # Check multiple conditions for daemon mode DAEMON_MODE=false # Method 1: Explicit daemon argument if [[ "$1" == "daemon" ]]; then log "INFO" "DAEMON: Explicit daemon argument detected" DAEMON_MODE=true fi # Method 2: Environment variable from systemd if [[ "$SKIP_TO_DAEMON" == "true" ]]; then log "INFO" "DAEMON: Environment variable SKIP_TO_DAEMON=true detected" DAEMON_MODE=true fi # Method 3: Config file exists (upgrade scenario) if [[ -f "/etc/haproxy-agent/config.json" ]]; then log "INFO" "DAEMON: Existing config file detected" DAEMON_MODE=true fi # Method 4: Legacy environment variable if [[ "$HAPROXY_AGENT_SKIP_SETUP" == "true" ]]; then log "INFO" "DAEMON: Legacy skip setup detected" DAEMON_MODE=true fi if [[ "$DAEMON_MODE" == "true" ]]; then log "INFO" "DAEMON: Starting agent daemon with existing configuration..." # Load existing configuration CONFIG_FILE="/etc/haproxy-agent/config.json" LOG_FILE="/var/log/haproxy-agent/agent.log" PID_FILE="/var/run/haproxy-agent.pid" # Load configuration from file if [[ -f "$CONFIG_FILE" ]]; then MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE" 2>/dev/null) AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null) CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE" 2>/dev/null) AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE" 2>/dev/null) HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE" 2>/dev/null) log "INFO" "DAEMON: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID" # FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON (MacOS style) log "INFO" "DAEMON: Going directly to daemon mode..." # Jump to the daemon section at the end of the script SKIP_TO_DAEMON=true else log "ERROR" "DAEMON: Configuration file not found: $CONFIG_FILE" log "INFO" "DAEMON: Falling back to installer mode..." DAEMON_MODE=false fi fi # Configuration variables - MODIFY THESE IF NEEDED MANAGEMENT_URL="{{MANAGEMENT_URL}}" CLUSTER_ID="{{CLUSTER_ID}}" AGENT_NAME="{{AGENT_NAME}}" HOSTNAME_PREFIX="{{HOSTNAME_PREFIX}}" HOSTNAME="${HOSTNAME_PREFIX}-$(hostname -s)" HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}" HAPROXY_BIN_PATH="{{HAPROXY_BIN_PATH}}" STATS_SOCKET_PATH="{{STATS_SOCKET_PATH}}" HAPROXY_SERVICE_NAME="haproxy" # Skip authentication if we already have token from config (daemon mode) if [[ "$SKIP_TO_DAEMON" == "true" && -n "$AGENT_TOKEN" ]]; then log "DEBUG" "AGENT UPGRADE: Skipping authentication - using token from config" else # SECURE AGENT AUTHENTICATION echo "" log "INFO" "Agent Authentication Setup" echo "============================================" echo "This agent requires a secure API token for authentication." echo "Please obtain the API token from the Security page in the" echo "HAProxy Management UI before proceeding." echo "" echo "Steps to get your API token:" echo "1. Open HAProxy Management UI" echo "2. Go to Security page" echo "3. Create a new agent token for: $AGENT_NAME" echo "4. Copy the generated token" echo "" # Prompt for API token with validation while true; do echo -n "Enter Agent API Token: " read -r AGENT_TOKEN if [[ -z "$AGENT_TOKEN" ]]; then log "ERROR" "API token cannot be empty" continue fi # Validate token format (should start with 'hap_') if [[ ! "$AGENT_TOKEN" =~ ^hap_.+ ]]; then log "ERROR" "Invalid token format. Token should start with 'hap_'" echo " Please ensure you copied the complete token from the Security page." continue fi # Test token validity by making a test API call log "DEBUG" "Validating API token..." TEST_RESPONSE=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters" \ -H "X-API-Key: $AGENT_TOKEN" \ -H "User-Agent: haproxy-agent-installer" 2>/dev/null || echo "FAILED") if [[ "$TEST_RESPONSE" == "FAILED" ]] || [[ ! "$TEST_RESPONSE" =~ clusters ]]; then log "ERROR" "API token validation failed" echo " Please check:" echo " - Token is correct and complete" echo " - Token has not been revoked" echo " - Management URL is accessible: $MANAGEMENT_URL" echo "" continue fi log "INFO" "API token validated successfully" break done fi # Skip installer output in quiet mode if [[ "$QUIET_MODE" != "true" ]]; then echo "" fi # Dynamically locate required binaries (global scope) find_binary() { local binary_name="$1" local binary_path=$(command -v "$binary_name" 2>/dev/null) if [[ -n "$binary_path" ]]; then echo "$binary_path" return 0 fi # Fallback: search in common locations local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin") for path in "${search_paths[@]}"; do if [[ -x "$path/$binary_name" ]]; then echo "$path/$binary_name" return 0 fi done return 1 } CURL_BIN=$(find_binary curl) if [[ -z "$CURL_BIN" ]]; then log "ERROR" "curl not found in system" exit 1 fi # Installation paths for Linux INSTALL_DIR="/usr/local/bin" CONFIG_DIR="/etc/haproxy-agent" LOG_DIR="/var/log/haproxy-agent" # Skip cleanup if in daemon mode if [[ "$SKIP_TO_DAEMON" != "true" ]]; then # CRITICAL: Clean installation - Remove existing agent components if [[ "$QUIET_MODE" != "true" ]]; then echo "" echo "🧹 Performing pre-installation cleanup..." fi # Function to safely remove files/directories safe_remove() { local path="$1" local desc="$2" if [[ -e "$path" ]]; then [[ "$QUIET_MODE" != "true" ]] && echo " 🗑️ Removing $desc..." rm -rf "$path" 2>/dev/null || true [[ "$QUIET_MODE" != "true" ]] && echo " $desc removed" return 0 fi } # 1. Stop and kill all existing agent processes [[ "$QUIET_MODE" != "true" ]] && echo "🔪 Terminating existing HAProxy Agent processes..." KILLED_COUNT=0 for pattern in "haproxy-agent" "/usr/local/bin/haproxy-agent" "haproxy-agent.service"; do PIDS=$(pgrep -f "$pattern" 2>/dev/null || true) if [[ -n "$PIDS" ]]; then [[ "$QUIET_MODE" != "true" ]] && echo " Killing processes matching: $pattern" kill -TERM $PIDS 2>/dev/null || true sleep 2 kill -KILL $PIDS 2>/dev/null || true KILLED_COUNT=$((KILLED_COUNT + $(echo "$PIDS" | wc -w))) fi done if [[ $KILLED_COUNT -gt 0 ]]; then log "INFO" "Terminated $KILLED_COUNT existing agent processes" else log "INFO" "No existing agent processes found" fi # 2. Stop and disable existing systemd services [[ "$QUIET_MODE" != "true" ]] && echo "🛑 Cleaning up existing systemd services..." if systemctl is-enabled haproxy-agent >/dev/null 2>&1; then [[ "$QUIET_MODE" != "true" ]] && echo " Stopping and disabling haproxy-agent service" systemctl stop haproxy-agent 2>/dev/null || true systemctl disable haproxy-agent 2>/dev/null || true [[ "$QUIET_MODE" != "true" ]] && echo " Service haproxy-agent cleaned up" fi # 3. Remove existing files and directories [[ "$QUIET_MODE" != "true" ]] && echo "🗑️ Removing existing agent files..." safe_remove "/etc/systemd/system/haproxy-agent.service" "systemd service file" safe_remove "/usr/local/bin/haproxy-agent" "agent binary" safe_remove "/etc/haproxy-agent" "configuration directory" safe_remove "/var/log/haproxy-agent" "log directory" safe_remove "/var/run/haproxy-agent.pid" "PID file" # Clean up temporary config files TEMP_COUNT=0 for temp_file in /tmp/haproxy_new_*.cfg; do if [[ -f "$temp_file" ]]; then rm -f "$temp_file" TEMP_COUNT=$((TEMP_COUNT + 1)) fi done if [[ $TEMP_COUNT -gt 0 ]]; then [[ "$QUIET_MODE" != "true" ]] && echo " Removed $TEMP_COUNT temporary config files" fi log "INFO" "Existing agent components cleaned up" [[ "$QUIET_MODE" != "true" ]] && echo "Pre-installation cleanup completed" [[ "$QUIET_MODE" != "true" ]] && echo "" fi # End of cleanup section # Detect architecture ARCH=$(uname -m) case "$ARCH" in "x86_64") ARCH_SUFFIX="linux-amd64"; [[ "$QUIET_MODE" != "true" ]] && echo "🖥️ Detected: Linux AMD64" ;; "aarch64"|"arm64") ARCH_SUFFIX="linux-arm64"; [[ "$QUIET_MODE" != "true" ]] && echo "🦾 Detected: Linux ARM64" ;; *) ARCH_SUFFIX="linux-unknown"; log "WARN" "Unknown architecture: $ARCH" ;; esac # Display configuration only in interactive mode if [[ "$QUIET_MODE" != "true" ]]; then echo "📋 Agent Configuration:" echo " Management URL: $MANAGEMENT_URL" echo " Cluster ID: $CLUSTER_ID" echo " Agent Name: $AGENT_NAME" echo " Hostname: $HOSTNAME" echo " Architecture: $ARCH ($ARCH_SUFFIX)" echo "" echo "📋 HAProxy Integration:" echo " Config Path: $HAPROXY_CONFIG_PATH" echo " Stats Socket: $STATS_SOCKET_PATH" echo " Service Name: $HAPROXY_SERVICE_NAME" echo "" fi # Only show agent capabilities during initial installation, not on daemon restart if [[ "$SKIP_TO_DAEMON" != "true" ]]; then log "INFO" "This agent will:" if [[ "$QUIET_MODE" != "true" ]]; then echo " • Monitor and report HAProxy service status" echo " • Receive configuration updates from management UI" echo " • Apply changes to HAProxy config file: $HAPROXY_CONFIG_PATH" echo " • Manage HAProxy service lifecycle (restart/reload)" echo " • Send heartbeats every 10 seconds" echo "" fi else # Daemon mode - reduced logging for restart scenarios log "DEBUG" "AGENT UPGRADE: Skipping entire installation - going to daemon mode" fi # Check root permissions if [[ "$SKIP_TO_DAEMON" != "true" && $EUID -ne 0 ]]; then log "WARN" "This script needs root privileges for agent service installation." echo " Please run with: sudo $0" exit 1 fi # Skip validation if in daemon mode if [[ "$SKIP_TO_DAEMON" != "true" ]]; then # Validate cluster exists by checking management API log "DEBUG" "Validating HAProxy Cluster..." CLUSTER_CHECK=$("$CURL_BIN" -k -s -f "$MANAGEMENT_URL/api/clusters" -H "User-Agent: haproxy-agent-installer" || echo "FAILED") if [[ "$CLUSTER_CHECK" == "FAILED" ]]; then log "ERROR" "Failed to connect to management API!" echo " Please check your network connection and management URL." exit 1 fi # Check if our specific cluster ID exists in the response if ! echo "$CLUSTER_CHECK" | jq -e ".clusters[]? | select(.id==$CLUSTER_ID)" >/dev/null; then log "ERROR" "HAProxy Cluster (ID: $CLUSTER_ID) not found!" echo " Available clusters:" echo "$CLUSTER_CHECK" | jq -r '.clusters[] | " - \(.id): \(.name)"' echo " Please ensure the cluster exists in the management UI before running this script." exit 1 fi log "INFO" "Cluster validation successful" fi # Check dependencies check_dependencies() { log "DEBUG" "Checking system requirements..." # Check if HAProxy is installed if ! command -v haproxy &> /dev/null; then log "WARN" "HAProxy not found. Installing..." if command -v apt-get &> /dev/null; then apt-get update && apt-get install -y haproxy elif command -v yum &> /dev/null; then yum install -y haproxy elif command -v dnf &> /dev/null; then dnf install -y haproxy else log "ERROR" "Cannot install HAProxy automatically. Please install manually." exit 1 fi fi log "INFO" "HAProxy found: $(haproxy -v | head -1)" # CRITICAL: Validate existing HAProxy config # Agent is designed to manage an EXISTING HAProxy installation with global+defaults sections log "DEBUG" "Validating existing HAProxy configuration..." if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then # Check if config has global section if ! grep -q "^global" "$HAPROXY_CONFIG_PATH"; then log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'global' section!" echo "" echo "INSTALLATION REQUIREMENT NOT MET" echo "" echo "This agent is designed to manage an EXISTING HAProxy installation." echo "Your HAProxy config must contain:" echo " • global section (with logging, stats socket, etc.)" echo " • defaults section (with timeouts, mode, etc.)" echo "" echo "The agent will:" echo " PRESERVE your global and defaults sections" echo " MANAGE only frontends and backends" echo "" echo "Current config: $HAPROXY_CONFIG_PATH" echo "" echo "Please ensure HAProxy is properly configured before installing agent." exit 1 fi # Check if config has defaults section if ! grep -q "^defaults" "$HAPROXY_CONFIG_PATH"; then log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'defaults' section!" echo "" echo "INSTALLATION REQUIREMENT NOT MET" echo "" echo "This agent is designed to manage an EXISTING HAProxy installation." echo "Your HAProxy config must contain:" echo " • global section (with logging, stats socket, etc.)" echo " • defaults section (with timeouts, mode, etc.)" echo "" echo "The agent will:" echo " PRESERVE your global and defaults sections" echo " MANAGE only frontends and backends" echo "" echo "Current config: $HAPROXY_CONFIG_PATH" echo "" echo "Please ensure HAProxy is properly configured before installing agent." exit 1 fi log "INFO" "Existing HAProxy config validated (global+defaults sections present)" else log "WARN" "HAProxy config not found at $HAPROXY_CONFIG_PATH" log "WARN" "Agent expects to manage an existing HAProxy installation" echo "" echo " WARNING: HAProxy config not found" echo "" echo "This agent is designed to manage an EXISTING HAProxy installation." echo "Config file expected at: $HAPROXY_CONFIG_PATH" echo "" echo "The agent will create a new config if you continue, but this is NOT recommended." echo "It's better to set up HAProxy first with proper global/defaults sections." echo "" read -p "Continue anyway? (yes/no): " -r CONTINUE_WITHOUT_CONFIG if [[ "$CONTINUE_WITHOUT_CONFIG" != "yes" ]]; then log "INFO" "Installation cancelled by user" exit 0 fi log "WARN" "User chose to continue without existing HAProxy config" fi # Check if config directory exists HAPROXY_DIR=$(dirname "$HAPROXY_CONFIG_PATH") if [[ ! -d "$HAPROXY_DIR" ]]; then echo "Creating HAProxy config directory: $HAPROXY_DIR" mkdir -p "$HAPROXY_DIR" fi # Check if stats socket directory exists STATS_DIR=$(dirname "$STATS_SOCKET_PATH") if [[ ! -d "$STATS_DIR" ]]; then echo "Creating HAProxy stats socket directory: $STATS_DIR" mkdir -p "$STATS_DIR" fi # Check if jq is installed if ! command -v jq &> /dev/null; then log "WARN" "jq not found. Installing..." if command -v apt-get &> /dev/null; then apt-get install -y jq elif command -v yum &> /dev/null; then yum install -y jq elif command -v dnf &> /dev/null; then dnf install -y jq else log "ERROR" "Cannot install jq automatically. Please install manually." exit 1 fi fi log "INFO" "jq found: $(jq --version)" # Check if socat is installed (required for HAProxy stats socket) if ! command -v socat &> /dev/null; then log "WARN" "socat not found. Installing..." if command -v apt-get &> /dev/null; then apt-get install -y socat elif command -v yum &> /dev/null; then yum install -y socat elif command -v dnf &> /dev/null; then dnf install -y socat else log "ERROR" "Cannot install socat automatically. Please install manually." exit 1 fi fi log "INFO" "socat found: $(socat -V | head -1)" } # Skip dependency check if in daemon mode if [[ "$SKIP_TO_DAEMON" != "true" ]]; then check_dependencies fi # Skip final cleanup if in daemon mode if [[ "$SKIP_TO_DAEMON" != "true" ]]; then # CRITICAL: Final cleanup as root (clean any remaining files) echo "" echo "🧹 Final cleanup as root (removing any remaining files)..." safe_remove "/etc/systemd/system/haproxy-agent.service" "existing systemd service file" safe_remove "/usr/local/bin/haproxy-agent" "existing agent binary" safe_remove "/etc/haproxy-agent" "existing configuration directory" safe_remove "/var/log/haproxy-agent" "existing log directory" safe_remove "/var/run/haproxy-agent.pid" "existing PID file" safe_remove "/tmp/haproxy-agent.pid" "temp PID file" # Clean up HAProxy config backups BACKUP_COUNT=0 for backup_file in /etc/haproxy/haproxy.cfg.backup.*; do if [[ -f "$backup_file" ]]; then rm -f "$backup_file" BACKUP_COUNT=$((BACKUP_COUNT + 1)) fi done if [[ $BACKUP_COUNT -gt 0 ]]; then echo " Removed $BACKUP_COUNT HAProxy config backups" fi # Clean up any remaining temp files TEMP_COUNT=0 for temp_file in /tmp/haproxy_new_*.cfg /tmp/*agent*.log; do if [[ -f "$temp_file" ]]; then rm -f "$temp_file" TEMP_COUNT=$((TEMP_COUNT + 1)) fi done if [[ $TEMP_COUNT -gt 0 ]]; then echo " Removed $TEMP_COUNT temporary files" fi log "INFO" "Final cleanup completed - Ready for fresh installation" echo "" fi # End of final cleanup section # Skip directory creation if in daemon mode if [[ "$SKIP_TO_DAEMON" != "true" ]]; then # Create directories with proper permissions for upgrades echo "Creating directories..." mkdir -p "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" # Set comprehensive permissions for upgrade operations log "INFO" "Setting permissions for upgrade operations..." # Ensure agent binary directory is writable (for in-place upgrades) chmod 755 "$INSTALL_DIR" chown root:root "$INSTALL_DIR" 2>/dev/null || true # Ensure config directory is accessible chmod 755 "$CONFIG_DIR" chown root:root "$CONFIG_DIR" 2>/dev/null || true # Ensure log directory is writable chmod 755 "$LOG_DIR" chown root:root "$LOG_DIR" 2>/dev/null || true # Create and set permissions for temp directory (upgrade markers) # Agent will run as root, so ensure /tmp is accessible mkdir -p /tmp chmod 1777 /tmp 2>/dev/null || true # Sticky bit for shared temp usage # Create agent-specific temp directory for upgrade markers AGENT_TEMP_DIR="/tmp/haproxy-agent" mkdir -p "$AGENT_TEMP_DIR" chmod 755 "$AGENT_TEMP_DIR" chown root:root "$AGENT_TEMP_DIR" 2>/dev/null || true # Test upgrade marker creation/removal capabilities touch "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null if [[ $? -eq 0 ]]; then log "INFO" "Upgrade marker permissions verified" else log "WARN" "Warning: Upgrade marker permissions may be restricted" fi # Ensure agent binary will have execution permissions for all operations log "INFO" "Configuring comprehensive agent permissions..." log "INFO" "Upgrade permissions configured" # Create agent service script (similar to macOS but with Linux-specific adjustments) echo "�� Installing HAProxy Agent..." cat > "$INSTALL_DIR/haproxy-agent" << 'AGENT_SCRIPT' #!/bin/bash CONFIG_FILE="/etc/haproxy-agent/config.json" LOG_FILE="/var/log/haproxy-agent/agent.log" PID_FILE="/var/run/haproxy-agent.pid" # Dynamic binary finder function (embedded in daemon) find_binary() { local binary_name="$1" local binary_path=$(command -v "$binary_name" 2>/dev/null) if [[ -n "$binary_path" ]]; then echo "$binary_path" return 0 fi # Fallback: search in common locations local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin") for path in "${search_paths[@]}"; do if [[ -x "$path/$binary_name" ]]; then echo "$path/$binary_name" return 0 fi done return 1 } # Initialize global binary paths for daemon CURL_BIN=$(find_binary curl) if [[ -z "$CURL_BIN" ]]; then echo "ERROR: curl not found in system" >&2 exit 1 fi # Load configuration if [[ ! -f "$CONFIG_FILE" ]]; then echo "ERROR: Configuration not found: $CONFIG_FILE" >&2 exit 1 fi MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE") AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE") CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE") AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE") HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE") # SSL incremental update timestamp file (agent-specific to avoid race conditions) # Each agent maintains its own SSL sync timestamp to prevent conflicts SSL_SYNC_TIMESTAMP_FILE="/tmp/haproxy-agent-ssl-sync-${AGENT_NAME}" # Get dynamic HAProxy paths from cluster configuration get_cluster_paths() { local cluster_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/clusters" \ -H "X-API-Key: $AGENT_TOKEN") if [[ $? -eq 0 ]]; then HAPROXY_CONFIG_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_config_path // "/etc/haproxy/haproxy.cfg"') HAPROXY_BIN_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_bin_path // "/usr/sbin/haproxy"') STATS_SOCKET_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .stats_socket_path // "/var/run/haproxy/admin.sock"') else # Fallback to config file values if cluster fetch fails HAPROXY_CONFIG_PATH=$(jq -r '.haproxy.config_path // "/etc/haproxy/haproxy.cfg"' "$CONFIG_FILE") HAPROXY_BIN_PATH=$(jq -r '.haproxy.bin_path // "/usr/sbin/haproxy"' "$CONFIG_FILE") STATS_SOCKET_PATH=$(jq -r '.haproxy.stats_socket_path // "/var/run/haproxy/admin.sock"' "$CONFIG_FILE") fi } # Initialize cluster paths get_cluster_paths # Enable debug mode for troubleshooting (0=off, 1=on) DEBUG_MODE=0 # Logging - JSON format, single line with timestamp log() { local level="$1" local message="$2" if [[ "$level" == "DEBUG" && "${DEBUG_MODE:-0}" != "1" ]]; then return 0 fi local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ') # Escape message for JSON (replace backslash, quotes, newlines, tabs) local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ') # Remove any control characters and emojis (keep only printable ASCII + basic UTF-8) escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g') echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$AGENT_NAME\",\"message\":\"$escaped_message\"}" >> "$LOG_FILE" } # Register agent on first run register_agent() { log "INFO" "Registering agent with management server..." local haproxy_version="unknown" if command -v haproxy &> /dev/null; then haproxy_version=$(haproxy -v | head -1 | awk '{print $3}') fi local arch=$(uname -m) platform=$(uname -s | tr '[:upper:]' '[:lower:]') # Remove local to make it global local system_info=$(collect_system_info) local json_payload=$(cat </dev/null; then echo "running" elif pgrep -f "haproxy" > /dev/null 2>&1; then echo "running" else echo "stopped" fi } # Get full HAProxy stats CSV from socket (for dashboard metrics) get_haproxy_stats_csv() { # Lazy initialize SOCAT_BIN if not set (for upgrade scenarios) if [[ -z "$SOCAT_BIN" ]]; then SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null) if [[ -z "$SOCAT_BIN" ]]; then log "WARN" "STATS: socat not available, cannot fetch stats CSV" echo "" return fi log "INFO" "STATS: Initialized socat: $SOCAT_BIN" fi # Lazy initialize STATS_SOCKET_PATH if not set if [[ -z "$STATS_SOCKET_PATH" ]]; then STATS_SOCKET_PATH="/var/run/haproxy/admin.sock" log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH" fi # Check if stats socket exists and is accessible if [[ ! -S "$STATS_SOCKET_PATH" ]]; then log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH" echo "" return fi # Query HAProxy stats via socket local stats_output if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then # Try to fix socket permission if we have root access if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH" chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null # Retry after permission fix stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null) fi fi # Return CSV data (base64 encoded to avoid JSON escaping issues) if [[ -n "$stats_output" ]]; then local csv_bytes=$(echo "$stats_output" | wc -c | tr -d ' ') local csv_lines=$(echo "$stats_output" | wc -l | tr -d ' ') local csv_base64=$(echo "$stats_output" | base64 | tr -d '\n') local base64_size=${#csv_base64} log "INFO" "STATS: Fetched CSV: $csv_bytes bytes, $csv_lines lines, base64: $base64_size chars" echo "$csv_base64" else log "WARN" "STATS: Failed to fetch stats from socket: $STATS_SOCKET_PATH" echo "" fi } # Get server statuses from HAProxy stats socket get_server_statuses() { local server_statuses_json="{}" # Lazy initialize SOCAT_BIN if not set (for upgrade scenarios) if [[ -z "$SOCAT_BIN" ]]; then SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null) if [[ -z "$SOCAT_BIN" ]]; then log "DEBUG" "STATS: socat not available for server statuses" echo "$server_statuses_json" return fi fi # Lazy initialize STATS_SOCKET_PATH if not set if [[ -z "$STATS_SOCKET_PATH" ]]; then STATS_SOCKET_PATH="/var/run/haproxy/admin.sock" fi # Check if stats socket exists and is accessible if [[ ! -S "$STATS_SOCKET_PATH" ]]; then log "DEBUG" "Stats socket not found: $STATS_SOCKET_PATH" echo "$server_statuses_json" return fi log "DEBUG" "Querying HAProxy stats socket: $STATS_SOCKET_PATH" # Query HAProxy stats via socket with permission fix attempt local stats_output if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then log "DEBUG" "Stats socket query failed, attempting permission fix" # Try to fix socket permission if we have root access if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null log "DEBUG" "Applied chmod 666 to stats socket" # Retry after permission fix if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then log "DEBUG" "Stats socket query failed after permission fix" echo "$server_statuses_json" return fi else echo "$server_statuses_json" return fi fi log "DEBUG" "Stats socket query successful, parsing output" # Parse stats output and build JSON local backend_servers=() while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do # Skip header and empty lines [[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue # Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0) [[ "$type" != "2" ]] && continue # Extract backend and server names local backend_name="$pxname" local server_name="$svname" # Map HAProxy status to our format local server_status case "$status" in "UP") server_status="UP" ;; "DOWN") server_status="DOWN" ;; "MAINT") server_status="MAINT" ;; "DRAIN") server_status="DRAIN" ;; "NOLB") server_status="NOLB" ;; *) server_status="UNKNOWN" ;; esac backend_servers+=("$backend_name:$server_name:$server_status") done <<< "$stats_output" # Build JSON structure if [[ ${#backend_servers[@]} -gt 0 ]]; then local json_parts=() local current_backend="" local backend_json="" for entry in "${backend_servers[@]}"; do IFS=':' read -r backend_name server_name server_status <<< "$entry" if [[ "$current_backend" != "$backend_name" ]]; then # Save previous backend if exists if [[ -n "$current_backend" && -n "$backend_json" ]]; then json_parts+=("\"$current_backend\": {${backend_json%,}}") fi # Start new backend current_backend="$backend_name" backend_json="" fi backend_json+="\"$server_name\": \"$server_status\"," done # Save last backend if [[ -n "$current_backend" && -n "$backend_json" ]]; then json_parts+=("\"$current_backend\": {${backend_json%,}}") fi # Combine all backends if [[ ${#json_parts[@]} -gt 0 ]]; then local IFS=',' server_statuses_json="{${json_parts[*]}}" fi fi echo "$server_statuses_json" } # Collect system information (Linux specific) collect_system_info() { # Get system information local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux") local kernel_version=$(uname -r 2>/dev/null || echo "Unknown") local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0") local cpu_count=$(nproc 2>/dev/null || echo "1") local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0") local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0") local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "") local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "") # Build system info JSON cat < /dev/null 2>&1 if [[ $? -eq 0 ]]; then # Only log heartbeat success in debug mode to reduce log spam [[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "Heartbeat sent successfully (HAProxy: $haproxy_status)" else log "ERROR" "Heartbeat failed" fi } # Deploy SSL certificates to local filesystem (Linux) deploy_ssl_certificates() { local ssl_certs_json="$1" log "INFO" "SSL DEPLOYMENT: Processing SSL certificates..." # Create SSL directory if it doesn't exist local ssl_dir="/etc/ssl/haproxy" if [[ ! -d "$ssl_dir" ]]; then log "INFO" "Creating SSL directory: $ssl_dir" mkdir -p "$ssl_dir" chmod 755 "$ssl_dir" fi # Parse and deploy each certificate local cert_count=$(echo "$ssl_certs_json" | jq length 2>/dev/null || echo "0") log "INFO" "Found $cert_count SSL certificates to deploy" for ((i=0; i "$temp_cert_file" # Append private key (only if provided - server SSL may not have it) if [[ -n "$key_content" && "$key_content" != "null" ]]; then echo "" >> "$temp_cert_file" echo "$key_content" >> "$temp_cert_file" fi # Append certificate chain if present if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then echo "" >> "$temp_cert_file" echo "$chain_content" >> "$temp_cert_file" log "DEBUG" "Added certificate chain for $cert_name" fi # Validate the combined certificate file if openssl x509 -in "$temp_cert_file" -noout -text >/dev/null 2>&1; then # Move to final location mv "$temp_cert_file" "$cert_file_path" chmod 600 "$cert_file_path" chown root:root "$cert_file_path" 2>/dev/null || true log "INFO" "SSL certificate deployed: $cert_file_path" # Log certificate details local cert_subject=$(openssl x509 -in "$cert_file_path" -noout -subject 2>/dev/null | sed 's/subject=//') local cert_expiry=$(openssl x509 -in "$cert_file_path" -noout -dates 2>/dev/null | grep notAfter | sed 's/notAfter=//') log "DEBUG" "Certificate subject: $cert_subject" log "DEBUG" "Certificate expires: $cert_expiry" else log "ERROR" "Invalid certificate format for $cert_name, removing temp file" rm -f "$temp_cert_file" fi done log "INFO" "SSL DEPLOYMENT: Completed" } # Fetch and deploy SSL certificates from separate endpoint (Linux) fetch_and_deploy_ssl_certificates() { # Only log SSL fetch in debug mode to reduce log spam [[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "SSL FETCH: Getting SSL certificates..." # Build SSL endpoint URL with incremental update support local ssl_endpoint="$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates" # Add timestamp parameter for incremental updates if we have a last SSL sync time if [[ -f "$SSL_SYNC_TIMESTAMP_FILE" ]]; then local last_ssl_sync=$(cat "$SSL_SYNC_TIMESTAMP_FILE" 2>/dev/null || echo "") if [[ -n "$last_ssl_sync" ]]; then ssl_endpoint="${ssl_endpoint}?since=${last_ssl_sync}" log "DEBUG" "SSL INCREMENTAL: Requesting SSL certificates since $last_ssl_sync" fi fi # Fetch SSL certificates from dedicated endpoint local ssl_response=$("$CURL_BIN" -k -s -X GET "$ssl_endpoint" \ -H "X-API-Key: $AGENT_TOKEN") if [[ $? -ne 0 ]]; then log "WARN" "Failed to fetch SSL certificates, skipping SSL deployment" return 0 fi local ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"') local ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"') if [[ "$ssl_status" != "available" ]]; then log "DEBUG" "SSL certificates not available (status: $ssl_status)" return 0 fi if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then log "INFO" "Deploying SSL certificates..." deploy_ssl_certificates "$ssl_certificates" # Save current timestamp for next incremental update date -u +"%Y-%m-%dT%H:%M:%SZ" > "$SSL_SYNC_TIMESTAMP_FILE" log "DEBUG" "SSL INCREMENTAL: Saved SSL sync timestamp for next incremental update" else log "DEBUG" "No SSL certificates to deploy" fi } # Reload HAProxy service (Linux specific) reload_haproxy_service() { log "INFO" "Performing zero-downtime HAProxy configuration reload..." # IMPROVED STRATEGY: Use HAProxy's seamless reload with verification # This ensures new config is applied without traffic interruption # Step 1: Validate new config before attempting reload if ! "$HAPROXY_BIN_PATH" -c -f "$HAPROXY_CONFIG_PATH" >/dev/null 2>&1; then log "ERROR" "New configuration validation failed - aborting reload" return 1 fi # Step 2: Try systemd reload first (most reliable on Linux) if systemctl is-active --quiet haproxy 2>/dev/null; then log "INFO" "Using systemctl reload for zero-downtime update" if systemctl reload haproxy 2>/dev/null; then log "INFO" "HAProxy reloaded successfully via systemctl" # Verify reload was successful sleep 2 if systemctl is-active --quiet haproxy; then log "INFO" "HAProxy service is active after systemctl reload" return 0 else log "WARN" "HAProxy service not active after systemctl reload, trying manual method" fi else log "WARN" "systemctl reload failed, trying manual seamless reload" fi fi # Step 3: Manual seamless reload using HAProxy's -sf option local haproxy_master_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1) if [[ -n "$haproxy_master_pid" ]]; then log "INFO" "Found HAProxy master process (PID: $haproxy_master_pid)" # Use HAProxy's seamless reload with new process log "INFO" "Starting seamless reload with new configuration..." if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D -sf "$haproxy_master_pid"; then log "INFO" "Seamless reload initiated successfully" # Verify new process is running and old one is gone sleep 3 local new_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1) if [[ -n "$new_pid" && "$new_pid" != "$haproxy_master_pid" ]]; then log "INFO" "HAProxy seamlessly reloaded (New PID: $new_pid, Old PID: $haproxy_master_pid)" return 0 else log "WARN" "Seamless reload may have failed, verifying process status..." # Additional verification if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then log "INFO" "HAProxy is still running after reload attempt" return 0 else log "ERROR" "HAProxy process not found after reload" fi fi else log "ERROR" "Seamless reload command failed" fi else log "WARN" "No HAProxy master process found, starting fresh instance" fi # Step 4: Fallback - start new HAProxy instance (only if no process running) if ! pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then log "INFO" "Starting new HAProxy instance..." if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D; then sleep 2 if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then log "INFO" "HAProxy started successfully" return 0 else log "ERROR" "HAProxy failed to start" return 1 fi else log "ERROR" "Failed to start HAProxy: $HAPROXY_BIN_PATH" return 1 fi fi log "INFO" "HAProxy configuration reload completed" return 0 } # Check for agent upgrade and perform if requested check_agent_upgrade() { log "DEBUG" "Checking for agent upgrade requests..." # Check if upgrade was recently completed to prevent loops (agent-specific marker) local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME" if [[ -f "$agent_marker" ]]; then local upgrade_time=$(head -n1 "$agent_marker" | cut -d' ' -f1-2) local current_time=$(date '+%Y-%m-%d %H:%M:%S') # Convert times to epoch for comparison local upgrade_epoch=$(date -d "$upgrade_time" '+%s' 2>/dev/null || echo "0") local current_epoch=$(date '+%s') local time_diff=$((current_epoch - upgrade_epoch)) # Remove marker if older than 5 minutes (300 seconds) if [[ $time_diff -gt 300 ]]; then log "DEBUG" "Upgrade marker expired (${time_diff}s old), removing and allowing new upgrades" rm -f "$agent_marker" 2>/dev/null || true else log "DEBUG" "Recent upgrade completed at: $upgrade_time - skipping upgrade check (${time_diff}s ago)" return 0 fi fi # Check if agent is marked for upgrade by querying agent-specific status local agent_status_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \ -H "X-API-Key: $AGENT_TOKEN") if [[ $? -ne 0 ]]; then log "DEBUG" "Failed to fetch agent upgrade status" return 0 fi local should_upgrade=$(echo "$agent_status_response" | jq -r '.should_upgrade // false') local target_version=$(echo "$agent_status_response" | jq -r '.target_version // ""') local current_version="{{AGENT_VERSION}}" # This agent's version # Only upgrade if explicitly requested via UI and target version is different if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "$current_version" ]]; then log "INFO" "AGENT UPGRADE: Upgrade requested to version $target_version (current: $current_version)" perform_agent_upgrade "$target_version" fi } # Perform agent upgrade perform_agent_upgrade() { local new_version="$1" log "INFO" "AGENT UPGRADE: Starting upgrade to version $new_version" # Download new agent script local temp_script="/tmp/haproxy-agent-new" local upgrade_url="$MANAGEMENT_URL/api/agents/generate-install-script" # Get pool_id from agent's current cluster configuration local POOL_ID=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null) if [[ -z "$POOL_ID" || "$POOL_ID" == "null" ]]; then # Fallback: get pool_id from cluster_id via API local cluster_info=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \ -H "X-API-Key: $AGENT_TOKEN" 2>/dev/null) POOL_ID=$(echo "$cluster_info" | jq -r '.pool_id // 1' 2>/dev/null) [[ -z "$POOL_ID" || "$POOL_ID" == "null" ]] && POOL_ID=1 fi # Create upgrade request payload with correct pool_id local upgrade_payload=$(cat < "$temp_script" chmod +x "$temp_script" log "INFO" "AGENT UPGRADE: New script downloaded, performing in-place upgrade" # Get current script path local current_script="/usr/local/bin/haproxy-agent" # Create backup of current script in dedicated backup directory local backup_dir="/usr/local/share/haproxy-agent-backups" local backup_file="$backup_dir/haproxy-agent-$(date '+%Y%m%d-%H%M%S').backup" mkdir -p "$backup_dir" 2>/dev/null || true cp "$current_script" "$backup_file" 2>/dev/null || true if [[ -f "$backup_file" ]]; then log "INFO" "AGENT UPGRADE: Backup created at $backup_file" else log "WARN" "AGENT UPGRADE: Could not create backup, proceeding anyway" fi # Replace current script with new version (in-place update) cp "$temp_script" "$current_script" chmod +x "$current_script" log "INFO" "AGENT UPGRADE: Script updated in-place, preparing clean restart" # Mark upgrade as complete log "INFO" "AGENT UPGRADE: Notifying backend of upgrade completion..." local completion_response=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"version\":\"$new_version\",\"status\":\"completed\"}" 2>&1) local completion_status=$? if [[ $completion_status -eq 0 ]]; then log "INFO" "AGENT UPGRADE: Backend notified successfully: $completion_response" else log "ERROR" "AGENT UPGRADE: Failed to notify backend: $completion_response" fi log "INFO" "DAEMON: Agent upgrade completed successfully" log "INFO" "DAEMON: Exiting current process - SystemD will auto-restart with new version" # Clean up temporary files rm -f "$temp_script" # Create agent-specific marker file to prevent upgrade loop on restart local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME" echo "$(date '+%Y-%m-%d %H:%M:%S') - Upgrade to $new_version completed" > "$agent_marker" 2>/dev/null || true chmod 666 "$agent_marker" 2>/dev/null || true # Exit cleanly - SystemD will automatically restart with new script log "INFO" "AGENT UPGRADE: Clean exit - new process will start with version $new_version" exit 0 else log "ERROR" "AGENT UPGRADE: Failed to download new script" # Mark upgrade as failed "$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1 fi else log "ERROR" "AGENT UPGRADE: Failed to fetch new script from server" # Mark upgrade as failed "$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1 fi } # Check and apply configuration updates (Linux) check_config_updates() { # Only log config check in debug mode to reduce log spam [[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "Checking for configuration updates..." # Use agent-specific config endpoint CONFIG_RESPONSE=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \ -H "X-API-Key: $AGENT_TOKEN") if [[ $? -ne 0 ]]; then log "ERROR" "Failed to fetch configuration" return 1 fi log "DEBUG" "Config response received" # Parse JSON response using jq if ! command -v jq >/dev/null 2>&1; then log "ERROR" "jq is required for config parsing but not found" return 1 fi # Extract config details from response CONFIG_STATUS=$(echo "$CONFIG_RESPONSE" | jq -r '.status // "unknown"') CONFIG_VERSION=$(echo "$CONFIG_RESPONSE" | jq -r '.version // "unknown"') CONFIG_CONTENT=$(echo "$CONFIG_RESPONSE" | jq -r '.config_content // ""') CONFIG_CHECKSUM=$(echo "$CONFIG_RESPONSE" | jq -r '.checksum // ""') log "DEBUG" "Config status: $CONFIG_STATUS, version: $CONFIG_VERSION" # Check if config is available if [[ "$CONFIG_STATUS" != "available" ]]; then log "WARN" "No configuration available (status: $CONFIG_STATUS)" return 0 fi # Check if config content is empty or invalid if [[ -z "$CONFIG_CONTENT" || "$CONFIG_CONTENT" == "null" ]]; then log "WARN" "Configuration content is empty" return 0 fi # CRITICAL: Prevent overwriting valid config with invalid content if [[ "$CONFIG_CONTENT" == "# No configuration available"* ]] || [[ "$CONFIG_CONTENT" == "# Agent is disabled"* ]] || [[ "$CONFIG_CONTENT" == "# No cluster assigned"* ]]; then log "WARN" "Received invalid configuration content, preserving existing HAProxy config" log "DEBUG" "Invalid config content: ${CONFIG_CONTENT:0:100}..." return 0 fi # Additional validation: Check if config content looks like a valid HAProxy config # Note: Partial configs may not have global/defaults, and may also have NO frontend/backend/listen (all deleted) if ! echo "$CONFIG_CONTENT" | grep -q -E "(frontend|backend|listen)"; then log "INFO" "Partial config has no frontend/backend/listen sections (all entities may have been deleted - this is OK)" log "DEBUG" "Config content preview: ${CONFIG_CONTENT:0:200}..." # Don't return - continue with merge, will result in global+defaults only config fi # Check if this version is already applied (CRITICAL: Prevent reapplying same config) if [[ "$CONFIG_VERSION" == "$last_applied_version" ]]; then log "DEBUG" "Configuration version $CONFIG_VERSION already applied, skipping" # OPTIMIZATION: SSL certificates are fetched separately and independently # Only fetch SSL if config hasn't changed to reduce unnecessary API calls return 0 fi # OPTIMIZATION: Fetch and deploy SSL certificates ONLY when config changes # This prevents unnecessary SSL API calls every 30 seconds log "INFO" "Config version changed ($last_applied_version → $CONFIG_VERSION), fetching SSL certificates..." fetch_and_deploy_ssl_certificates # Create backup of current config (CRITICAL: Always backup before changes) if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then local backup_file="$HAPROXY_CONFIG_PATH.backup.$(date +%Y%m%d_%H%M%S)" cp "$HAPROXY_CONFIG_PATH" "$backup_file" if [[ -f "$backup_file" ]]; then log "INFO" "Current config backed up to: $backup_file" else log "ERROR" "Failed to create config backup, aborting config update" return 1 fi else log "WARN" "No existing HAProxy config found at: $HAPROXY_CONFIG_PATH" fi # Write new configuration to temp file first TEMP_CONFIG="/tmp/haproxy_new_$$.cfg" echo "$CONFIG_CONTENT" > "$TEMP_CONFIG" # CRITICAL: Check if this is a PARTIAL config (only frontends/backends without global/defaults) # Backend generates partial configs to preserve agent-specific global/defaults settings if head -20 "$TEMP_CONFIG" | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then log "INFO" "CONFIG UPDATE: Detected PARTIAL config - preserving global/defaults from existing haproxy.cfg" # Extract global and defaults sections from current haproxy.cfg if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then # Extract global section (everything from start until 'defaults' keyword) awk '/^defaults[[:space:]]*$/{exit} {print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-global-$$.cfg 2>/dev/null # Extract defaults section (from 'defaults' until first 'frontend/backend/listen') awk '/^defaults[[:space:]]*$/{flag=1} /^(frontend|backend|listen)[[:space:]]/{if(flag && !/^defaults/) exit} flag{print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-defaults-$$.cfg 2>/dev/null # Extract all listen sections (preserve existing listen blocks like stats monitoring) # This extracts ALL listen blocks from existing config to preserve them awk ' /^listen[[:space:]]/ { if (listen_content != "") { # Print previous listen block gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content } in_listen=1 listen_content=$0 "\n" next } in_listen { # Check if we hit another section (frontend, backend, listen, global, defaults) if (/^(frontend|backend|listen|global|defaults)[[:space:]]/) { # Print current listen block gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content listen_content="" in_listen=0 } else { listen_content = listen_content $0 "\n" } } END { # Print final listen block if any if (listen_content != "") { gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content } } ' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-listen-$$.cfg 2>/dev/null # Check if extraction was successful if [[ -s /tmp/haproxy-global-$$.cfg && -s /tmp/haproxy-defaults-$$.cfg ]]; then listen_lines=0 [[ -s /tmp/haproxy-listen-$$.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen-$$.cfg) log "INFO" "CONFIG UPDATE: Extracted global ($(wc -l < /tmp/haproxy-global-$$.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults-$$.cfg) lines), listen blocks ($listen_lines lines)" # Merge: existing global + existing defaults + existing listen blocks + new frontends/backends merged_config="/tmp/haproxy-merged-$$.cfg" # Copy global section as-is, normalize trailing blanks to exactly 1 line # Remove excessive trailing blanks (keep max 1), preserve content awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global-$$.cfg > "$merged_config" # Copy defaults section as-is, normalize trailing blanks to exactly 1 line awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults-$$.cfg >> "$merged_config" # Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring) if [[ -s /tmp/haproxy-listen-$$.cfg ]]; then awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen-$$.cfg >> "$merged_config" log "INFO" "CONFIG UPDATE: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen-$$.cfg) listen blocks from existing config" fi # Extract only frontend/backend sections from partial config (skip comments) awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' "$TEMP_CONFIG" >> "$merged_config" # Replace temp config with merged version mv "$merged_config" "$TEMP_CONFIG" log "INFO" "CONFIG UPDATE: Successfully merged - preserved local global/defaults/listen blocks, applied new frontends/backends" # Cleanup extraction temp files rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg else log "ERROR" "CONFIG UPDATE: Failed to extract global/defaults sections from existing config" log "ERROR" "CONFIG UPDATE: This will cause HAProxy to fail - aborting config update" rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg "$TEMP_CONFIG" return 1 fi else log "ERROR" "CONFIG UPDATE: No existing haproxy.cfg found - cannot merge partial config" log "ERROR" "CONFIG UPDATE: Partial config requires existing global/defaults - aborting" rm -f "$TEMP_CONFIG" return 1 fi else log "INFO" "CONFIG UPDATE: Full config detected - using as-is (includes global/defaults)" fi # Log config content for debugging log "DEBUG" "Generated config content (first 20 lines):" log "DEBUG" "$(head -20 "$TEMP_CONFIG")" log "DEBUG" "Config file size: $(wc -l < "$TEMP_CONFIG") lines" # Validate new configuration with retry logic log "DEBUG" "Validating config with: $HAPROXY_BIN_PATH -c -f $TEMP_CONFIG" VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1) VALIDATION_RESULT=$? if [[ $VALIDATION_RESULT -eq 0 ]]; then log "INFO" "New configuration is valid" # Apply new configuration mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH" log "INFO" "Configuration updated to version: $CONFIG_VERSION" # Reload HAProxy service reload_haproxy_service # Update last applied version to prevent reapplying same config last_applied_version="$CONFIG_VERSION" # Send instant notification to backend for real-time UI sync log "INFO" "Sending instant notification for config version: $CONFIG_VERSION" NOTIFICATION_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" 2>&1) if [[ $? -eq 0 ]]; then log "INFO" "INSTANT SYNC: Notification sent successfully" else log "ERROR" "INSTANT SYNC: Notification failed - $NOTIFICATION_RESPONSE" fi # Send config sync to update database entities based on actual config log "INFO" "Syncing database with current config content" if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then # Properly escape JSON content CONFIG_JSON=$(cat "$HAPROXY_CONFIG_PATH" | jq -Rs .) SYNC_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-sync" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"config_content\":$CONFIG_JSON}" 2>&1) if [[ $? -eq 0 ]]; then log "INFO" "CONFIG SYNC: Database synced with current config" else log "ERROR" "CONFIG SYNC: Database sync failed - $SYNC_RESPONSE" fi fi log "INFO" "Configuration applied and HAProxy reloaded successfully" return 0 else log "ERROR" "New configuration validation failed (attempt 1/3)" log "ERROR" "Validation error output: $VALIDATION_OUTPUT" # CRITICAL: Retry validation with delay (race condition fix) for retry in 2 3; do log "INFO" "Retrying validation in 3 seconds... (attempt $retry/3)" sleep 3 RETRY_VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1) RETRY_VALIDATION_RESULT=$? if [[ $RETRY_VALIDATION_RESULT -eq 0 ]]; then log "INFO" "Configuration validation successful on retry $retry" # Apply new configuration mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH" log "INFO" "Configuration updated to version: $CONFIG_VERSION" # Reload HAProxy service reload_haproxy_service # Update last applied version to prevent reapplying same config last_applied_version="$CONFIG_VERSION" # Send instant notification to backend for real-time UI sync "$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \ -H "Content-Type: application/json" \ -H "X-API-Key: $AGENT_TOKEN" \ -d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" > /dev/null 2>&1 log "INFO" "Configuration applied and HAProxy reloaded successfully" return 0 fi done log "ERROR" "Configuration validation failed after 3 attempts" log "ERROR" "Final validation error: $RETRY_VALIDATION_OUTPUT" rm -f "$TEMP_CONFIG" return 1 fi } # Main daemon run_daemon() { # Check for existing running instance (CRITICAL: Prevent multiple agents) if [[ -f "$PID_FILE" ]]; then local existing_pid=$(cat "$PID_FILE" 2>/dev/null) if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then log "ERROR" "Agent already running with PID: $existing_pid" log "ERROR" "Terminating existing instance to prevent conflicts..." kill -TERM "$existing_pid" 2>/dev/null || true sleep 3 kill -KILL "$existing_pid" 2>/dev/null || true rm -f "$PID_FILE" fi fi log "INFO" "Starting HAProxy Agent: $AGENT_NAME (Linux)" echo $$ > "$PID_FILE" # CRITICAL: Setup graceful shutdown signal handling # This allows agent to stop quickly (<2s) instead of waiting for systemd timeout (90s) SHUTDOWN_REQUESTED=false trap_handler() { log "INFO" "Shutdown signal received (SIGTERM/SIGINT), stopping agent gracefully..." SHUTDOWN_REQUESTED=true } # Register signal handlers for graceful shutdown trap trap_handler SIGTERM SIGINT SIGQUIT log "INFO" "Signal handlers registered for graceful shutdown" # Dynamically locate required binaries for daemon operations SOCAT_BIN=$(find_binary socat) JQ_BIN=$(find_binary jq) if [[ -n "$SOCAT_BIN" ]]; then log "INFO" "Found socat: $SOCAT_BIN" else log "ERROR" "socat not found in system" exit 1 fi if [[ -n "$JQ_BIN" ]]; then log "INFO" "Found jq: $JQ_BIN" fi log "INFO" "Using curl: $CURL_BIN" # CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart) # Agent restart should NOT trigger config reapplication - fetch last applied version from backend log "INFO" "Fetching last applied config version from backend..." agent_info_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents" \ -H "X-API-Key: $AGENT_TOKEN" 2>/dev/null) if [[ $? -eq 0 && -n "$agent_info_response" ]]; then # Extract applied_config_version for this agent last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null) if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then last_applied_version="none" log "WARN" "Could not fetch applied_config_version from backend, starting with 'none'" else log "INFO" "Loaded last applied version from database: $last_applied_version" fi else last_applied_version="none" log "WARN" "Failed to fetch agent info from backend, starting with version 'none'" fi # CRITICAL: Startup delay to avoid race conditions log "INFO" "Startup delay for system stabilization..." sleep 5 # Register agent on startup if ! register_agent; then log "ERROR" "Failed to register agent, continuing with heartbeats..." fi # Send initial heartbeat sleep 1 send_heartbeat # Main agent loop with interruptible sleep for fast shutdown # Sleep is broken into 1-second intervals to check shutdown flag every second # This allows agent to stop in 1-2 seconds instead of waiting up to 30 seconds while [[ "$SHUTDOWN_REQUESTED" == "false" ]]; do # Interruptible sleep: break 30s into 30x 1s intervals for i in {1..30}; do # Check shutdown flag every second [[ "$SHUTDOWN_REQUESTED" == "true" ]] && break sleep 1 done # Exit loop if shutdown requested [[ "$SHUTDOWN_REQUESTED" == "true" ]] && break # Normal agent operations send_heartbeat # Check for pending config requests (if function exists) if type check_config_requests &>/dev/null; then check_config_requests fi check_config_updates check_agent_upgrade done # Graceful shutdown cleanup log "INFO" "Agent stopped gracefully" rm -f "$PID_FILE" exit 0 } case "${1:-daemon}" in "daemon") run_daemon ;; "stop") # Improved stop command with graceful → force kill escalation if [[ -f "$PID_FILE" ]]; then local pid=$(cat "$PID_FILE" 2>/dev/null) if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then echo "Stopping HAProxy Agent (PID: $pid)..." # Try graceful SIGTERM first kill -TERM "$pid" 2>/dev/null # Wait up to 5 seconds for graceful shutdown local count=0 while kill -0 "$pid" 2>/dev/null && [[ $count -lt 5 ]]; do sleep 1 count=$((count + 1)) done # Force kill if still running after 5 seconds if kill -0 "$pid" 2>/dev/null; then echo "Graceful stop timeout, force stopping agent..." kill -KILL "$pid" 2>/dev/null sleep 1 fi echo "Agent stopped successfully" else echo "Agent is not running (stale PID file)" fi else echo "Agent is not running (no PID file)" fi rm -f "$PID_FILE" ;; *) echo "Usage: $0 {daemon|stop}"; exit 1 ;; esac AGENT_SCRIPT chmod +x "$INSTALL_DIR/haproxy-agent" # Set comprehensive permissions for self-upgrade capability log "INFO" "Configuring agent self-upgrade permissions..." # Agent binary permissions - must be writable by root for in-place upgrades chmod 755 "$INSTALL_DIR/haproxy-agent" chown root:root "$INSTALL_DIR/haproxy-agent" 2>/dev/null || true # Create backup directory for agent upgrades AGENT_BACKUP_DIR="/usr/local/share/haproxy-agent-backups" mkdir -p "$AGENT_BACKUP_DIR" chmod 755 "$AGENT_BACKUP_DIR" chown root:root "$AGENT_BACKUP_DIR" 2>/dev/null || true # Ensure all required directories have proper ownership and permissions # This eliminates the need for sudo during upgrades log "INFO" "Final permission validation..." # Validate all critical paths are accessible for dir_path in "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" "$AGENT_BACKUP_DIR"; do if [[ -d "$dir_path" ]] && [[ -w "$dir_path" ]]; then log "INFO" "Directory writable: $dir_path" else log "WARN" "Directory access issue: $dir_path" fi done # Validate temp directory access if touch "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null; then log "INFO" "Temp directory access confirmed" else log "WARN" "Temp directory access issue" fi log "INFO" "All agent self-upgrade permissions configured - no sudo required for future operations" # Get pool_id for this cluster CLUSTER_POOL_ID=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \ -H "X-API-Key: $AGENT_TOKEN" | jq -r '.pool_id // 1' 2>/dev/null) [[ -z "$CLUSTER_POOL_ID" || "$CLUSTER_POOL_ID" == "null" ]] && CLUSTER_POOL_ID=1 # Create configuration echo "⚙️ Creating configuration..." cat > "$CONFIG_DIR/config.json" << CONFIG_EOF { "management": { "url": "$MANAGEMENT_URL", "token": "$AGENT_TOKEN", "cluster_id": $CLUSTER_ID, "pool_id": $CLUSTER_POOL_ID }, "agent": { "name": "$AGENT_NAME", "hostname": "$HOSTNAME", "platform": "$platform", "architecture": "$ARCH" }, "haproxy": { "config_path": "$HAPROXY_CONFIG_PATH", "bin_path": "$HAPROXY_BIN_PATH", "stats_socket_path": "$STATS_SOCKET_PATH", "service_name": "$HAPROXY_SERVICE_NAME" } } CONFIG_EOF # Set config file permissions for upgrade access chmod 644 "$CONFIG_DIR/config.json" chown root:root "$CONFIG_DIR/config.json" 2>/dev/null || true log "INFO" "Configuration file permissions set" # Create systemd service for Linux log "DEBUG" "Creating systemd service for Linux..." cat > "/etc/systemd/system/haproxy-agent.service" << SERVICE_EOF [Unit] Description=HAProxy Management Agent After=network.target Wants=network-online.target [Service] Type=simple User=root ExecStart=$INSTALL_DIR/haproxy-agent daemon ExecStop=$INSTALL_DIR/haproxy-agent stop # Graceful shutdown configuration # Timeout after 10 seconds instead of default 90 seconds TimeoutStopSec=10 # Mixed mode: send SIGTERM to main process, SIGKILL to remaining KillMode=mixed # Explicit signal for graceful shutdown KillSignal=SIGTERM Restart=always RestartSec=10 StandardOutput=append:$LOG_DIR/agent.log StandardError=append:$LOG_DIR/agent.log Environment=SKIP_TO_DAEMON=true [Install] WantedBy=multi-user.target SERVICE_EOF echo "Starting systemd service..." systemctl daemon-reload systemctl enable haproxy-agent systemctl start haproxy-agent sleep 3 SERVICE_STATUS=$(systemctl is-active haproxy-agent 2>/dev/null || echo "inactive") if [[ "$SERVICE_STATUS" == "active" ]]; then echo "" log "INFO" "HAProxy Management Agent installed successfully on Linux!" echo "" echo "Service Information:" echo " Status: active" echo " Architecture: $ARCH ($ARCH_SUFFIX)" echo " Cluster ID: $CLUSTER_ID" echo " Agent Name: $AGENT_NAME" echo " Config: $CONFIG_DIR/config.json" echo " Logs: tail -f $LOG_DIR/agent.log" echo "" log "DEBUG" "Linux Management Commands:" echo " Start: sudo systemctl start haproxy-agent" echo " Stop: sudo systemctl stop haproxy-agent" echo " Status: sudo systemctl status haproxy-agent" echo " Logs: sudo journalctl -u haproxy-agent -f" echo "" echo "Agent is now running and will:" echo " • Send heartbeats every 10 seconds to: $MANAGEMENT_URL" echo " • Monitor HAProxy service status and report to management UI" echo " • Wait for configuration update tasks from management UI" echo " • Apply HAProxy config changes to: $HAPROXY_CONFIG_PATH" echo "" echo "✨ Your Linux HAProxy instance is now centrally manageable!" echo "" echo "Next Steps:" echo " 1. Check agent logs: tail -f $LOG_DIR/agent.log" echo " 2. Verify agent registration in management UI" echo " 3. Test configuration deployment from UI" echo "" else log "ERROR" "Agent service failed to start" echo "Check logs: tail -n 20 $LOG_DIR/agent.log" echo "Check service: systemctl status haproxy-agent" exit 1 fi fi # End of installer section - skip if in daemon mode # Check if script should skip to daemon mode (from upgrade) - EXACT MacOS COPY if [[ "$SKIP_TO_DAEMON" == "true" ]]; then # Reduced logging on daemon restart to prevent log spam log "DEBUG" "AGENT UPGRADE: Daemon mode detected, starting agent daemon..." log "DEBUG" "DAEMON: Starting HAProxy Agent: $AGENT_NAME (Linux)" log "DEBUG" "DAEMON: Loading essential functions for daemon mode..." # CRITICAL: Define essential functions within SKIP_TO_DAEMON block # These functions are needed by daemon loop but aren't accessible from main script scope # Find binary utility (needed by stats functions) find_binary() { local binary_name="$1" local binary_path="" # Common binary paths for path in "/usr/bin" "/usr/local/bin" "/bin" "/usr/sbin" "/usr/local/sbin" "/sbin" "/opt/homebrew/bin"; do if [[ -x "$path/$binary_name" ]]; then binary_path="$path/$binary_name" break fi done if [[ -n "$binary_path" ]]; then echo "$binary_path" else command -v "$binary_name" 2>/dev/null || echo "" fi } # Get HAProxy status (Linux specific) get_haproxy_status() { if systemctl is-active --quiet haproxy 2>/dev/null; then echo "running" elif pgrep -f "haproxy" > /dev/null 2>&1; then echo "running" else echo "stopped" fi } # Get full HAProxy stats CSV from socket (for dashboard metrics) get_haproxy_stats_csv() { # Lazy initialize SOCAT_BIN if not set (for upgrade scenarios) if [[ -z "$SOCAT_BIN" ]]; then SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null) if [[ -z "$SOCAT_BIN" ]]; then log "WARN" "STATS: socat not available, cannot fetch stats CSV" echo "" return fi log "INFO" "STATS: Initialized socat: $SOCAT_BIN" fi # Lazy initialize STATS_SOCKET_PATH if not set if [[ -z "$STATS_SOCKET_PATH" ]]; then STATS_SOCKET_PATH="/var/run/haproxy/admin.sock" log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH" fi # Check if stats socket exists and is accessible if [[ ! -S "$STATS_SOCKET_PATH" ]]; then log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH" echo "" return fi # Query HAProxy stats via socket local stats_output if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then # Try to fix socket permission if we have root access if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH" chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null # Retry after permission fix stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null) fi fi # Return CSV data (base64 encoded to avoid JSON escaping issues) if [[ -n "$stats_output" ]]; then log "INFO" "STATS: Fetched CSV: ${#stats_output} bytes, $(echo "$stats_output" | wc -l) lines, base64: $(echo "$stats_output" | base64 -w 0 | wc -c) chars" echo "$stats_output" | base64 -w 0 else log "WARN" "STATS: No stats data retrieved from socket: $STATS_SOCKET_PATH" echo "" fi } # Get server statuses from HAProxy stats socket get_server_statuses() { # Lazy initialize SOCAT_BIN if not set (for upgrade scenarios) if [[ -z "$SOCAT_BIN" ]]; then SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null) if [[ -z "$SOCAT_BIN" ]]; then log "WARN" "SOCAT not available for server status checks" echo "{}" return fi fi # Lazy initialize STATS_SOCKET_PATH if not set if [[ -z "$STATS_SOCKET_PATH" ]]; then STATS_SOCKET_PATH="/var/run/haproxy/admin.sock" fi # Check if stats socket exists and is accessible if [[ ! -S "$STATS_SOCKET_PATH" ]]; then log "WARN" "Stats socket not found: $STATS_SOCKET_PATH" echo "{}" return fi # Query HAProxy stats via socket with permission fix attempt local stats_output if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then # Try to fix socket permission if we have root access if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null # Retry after permission fix if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then log "WARN" "Failed to query server stats via socket after permission fix" echo "{}" return fi else log "WARN" "Failed to query server stats via socket" echo "{}" return fi fi # Parse stats output and build JSON - EXACT COPY from main script local backend_servers=() while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do # Skip header and empty lines [[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue # Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0) [[ "$type" != "2" ]] && continue # Extract backend and server names local backend_name="$pxname" local server_name="$svname" # Map HAProxy status to our format local server_status case "$status" in "UP") server_status="UP" ;; "DOWN") server_status="DOWN" ;; "MAINT") server_status="MAINT" ;; "DRAIN") server_status="DRAIN" ;; "NOLB") server_status="NOLB" ;; *) server_status="UNKNOWN" ;; esac backend_servers+=("$backend_name:$server_name:$server_status") done <<< "$stats_output" # Build JSON structure - EXACT COPY from main script if [[ ${#backend_servers[@]} -gt 0 ]]; then local json_parts=() local current_backend="" local backend_json="" for entry in "${backend_servers[@]}"; do IFS=':' read -r backend_name server_name server_status <<< "$entry" if [[ "$current_backend" != "$backend_name" ]]; then # Save previous backend if exists if [[ -n "$current_backend" && -n "$backend_json" ]]; then json_parts+=("\"$current_backend\": {${backend_json%,}}") fi # Start new backend current_backend="$backend_name" backend_json="" fi backend_json+="\"$server_name\": \"$server_status\"," done # Save last backend if [[ -n "$current_backend" && -n "$backend_json" ]]; then json_parts+=("\"$current_backend\": {${backend_json%,}}") fi # Combine all backends if [[ ${#json_parts[@]} -gt 0 ]]; then local IFS=',' server_statuses_json="{${json_parts[*]}}" fi fi echo "$server_statuses_json" } # Collect basic system information for heartbeat collect_system_info() { local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | tr -d ',') local disk_usage=$(df -h / 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%') local memory_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}') cat </dev/null 2>&1 } # Check for pending configuration requests (DAEMON MODE) check_config_requests() { local curl_bin="${CURL_BIN:-$(find_binary curl)}" # Get pending config requests from backend log "DEBUG" "CONFIG: Checking for pending config requests from: ${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" local response=$("$curl_bin" -k -s -X GET "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" \ -H "Content-Type: application/json" \ -H "X-API-Key: ${AGENT_TOKEN}") log "DEBUG" "CONFIG: Response: ${response:0:200}..." # Check if there are pending requests local pending_count=$(echo "$response" | jq -r '.pending_requests | length' 2>/dev/null) log "DEBUG" "CONFIG: Pending count: $pending_count" if [[ "$pending_count" -gt 0 ]]; then log "INFO" "CONFIG: Found $pending_count pending config request(s)" # Process each request echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do local request_id=$(echo "$request" | jq -r '.request_id') local request_type=$(echo "$request" | jq -r '.request_type') log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)" # Read haproxy.cfg content local config_path="${HAPROXY_CONFIG_PATH}" if [[ -f "$config_path" ]]; then local file_size=$(wc -c < "$config_path") log "INFO" "CONFIG: Reading config from: $config_path (size: $file_size bytes)" local config_content=$(cat "$config_path") # Escape config content for JSON log "DEBUG" "CONFIG: Escaping config content for JSON..." local escaped_content=$(echo "$config_content" | jq -Rs .) if [[ -z "$escaped_content" ]]; then log "ERROR" "CONFIG: JSON escaping failed for config content!" continue fi local escaped_size=${#escaped_content} log "DEBUG" "CONFIG: JSON escaped content size: $escaped_size bytes" # Submit config response local response_payload=$(cat </dev/null 2>&1 log "DEBUG" "DAEMON: Agent upgrade completed successfully" log "DEBUG" "DAEMON: Agent is now online with version {{AGENT_VERSION}}" # FINAL FIX: Implement daemon loop directly (no exec to avoid infinite recursion) - EXACT MacOS COPY log "DEBUG" "DAEMON: Starting continuous daemon mode..." # Initialize daemon environment CONFIG_FILE="/etc/haproxy-agent/config.json" LOG_FILE="/var/log/haproxy-agent/agent.log" PID_FILE="/tmp/haproxy-agent-$AGENT_NAME.pid" # Write PID (use /tmp to avoid permission issues) echo $$ > "$PID_FILE" 2>/dev/null || true # CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart) # Agent restart/upgrade should NOT trigger config reapplication - fetch last applied version log "DEBUG" "DAEMON: Fetching last applied config version from backend..." agent_info_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents" \ -H "X-API-Key: $AGENT_TOKEN" 2>/dev/null) if [[ $? -eq 0 && -n "$agent_info_response" ]]; then # Extract applied_config_version for this agent using jq last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null) if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then last_applied_version="none" log "WARN" "DAEMON: Could not fetch applied_config_version, starting with 'none'" else log "INFO" "DAEMON: Loaded last applied version from database: $last_applied_version" fi else last_applied_version="none" log "WARN" "DAEMON: Failed to fetch agent info, starting with version 'none'" fi last_validation_failed_version="none" log "DEBUG" "DAEMON: Starting agent monitoring loop for $AGENT_NAME" # Enhanced daemon loop with upgrade capability - EXACT MacOS COPY while true; do sleep 30 # Read current token from config (fix for API key mismatch) CURRENT_AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null || echo "$AGENT_TOKEN") AGENT_TOKEN="$CURRENT_AGENT_TOKEN" # Update AGENT_TOKEN for send_heartbeat function # Ensure platform is available for heartbeat [[ -z "$platform" ]] && platform=$(uname -s | tr '[:upper:]' '[:lower:]') # Call send_heartbeat function (includes server_statuses) send_heartbeat # Only log heartbeat in debug mode to reduce log spam [[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "DAEMON: Heartbeat sent for agent $AGENT_NAME" # Check for pending configuration requests (if function exists) if type check_config_requests &>/dev/null; then check_config_requests fi # Check for SSL certificates and deploy them # Silent check - only log if changes are detected (reduce log spam) ssl_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null) if [[ -n "$ssl_response" ]] && ! echo "$ssl_response" | grep -q '"error":'; then ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"' 2>/dev/null) if [[ "$ssl_status" == "available" ]]; then ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"' 2>/dev/null) if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then # Create SSL directory if it doesn't exist SSL_DIR="/etc/ssl/haproxy" mkdir -p "$SSL_DIR" 2>/dev/null || true # Track if any certificate was actually deployed ssl_deployed_count=0 # Process each certificate # IMPORTANT: Use process substitution to avoid subshell (pipe would lose ssl_deployed_count) while IFS= read -r cert_data; do if [[ -n "$cert_data" ]]; then cert_name=$(echo "$cert_data" | jq -r '.name // "unknown"' 2>/dev/null) cert_domain=$(echo "$cert_data" | jq -r '.domain // "unknown"' 2>/dev/null) cert_content=$(echo "$cert_data" | jq -r '.certificate_content // ""' 2>/dev/null) key_content=$(echo "$cert_data" | jq -r '.private_key_content // ""' 2>/dev/null) chain_content=$(echo "$cert_data" | jq -r '.chain_content // ""' 2>/dev/null) cert_file_path=$(echo "$cert_data" | jq -r '.file_path // ""' 2>/dev/null) usage_type=$(echo "$cert_data" | jq -r '.usage_type // "frontend"' 2>/dev/null) # CRITICAL: Certificate content is always required, but private key depends on usage_type # - Frontend SSL: requires both cert AND private key (for bind ssl crt) # - Server SSL: requires only cert (CA file for backend verification) if [[ -n "$cert_content" && "$cert_content" != "null" ]]; then # For frontend SSL, private key is required if [[ "$usage_type" == "frontend" && ( -z "$key_content" || "$key_content" == "null" ) ]]; then log "WARN" "Skipping Frontend SSL $cert_name: private key missing" continue fi # Use original path or fallback if [[ -n "$cert_file_path" ]]; then ssl_file="$cert_file_path" else ssl_file="$SSL_DIR/${cert_name}.pem" fi # Check if certificate already exists and is unchanged # IMPORTANT: Calculate checksum with EXACT same format as file will be written # Format depends on what content is available: # - Frontend SSL: cert + key [+ chain] # - Server SSL: cert [+ chain] (no private key) if [[ -n "$key_content" && "$key_content" != "null" ]]; then # Has private key - frontend SSL if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then new_content=$(printf "%s\n\n%s\n\n%s" "$cert_content" "$key_content" "$chain_content") else new_content=$(printf "%s\n\n%s" "$cert_content" "$key_content") fi else # No private key - server SSL (CA cert only) if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then new_content=$(printf "%s\n\n%s" "$cert_content" "$chain_content") else new_content="$cert_content" fi fi existing_checksum="" new_checksum=$(echo "$new_content" | md5sum | awk '{print $1}' 2>/dev/null) if [[ -f "$ssl_file" ]]; then existing_checksum=$(cat "$ssl_file" | md5sum | awk '{print $1}' 2>/dev/null) fi # Only deploy if certificate is new or changed if [[ "$existing_checksum" != "$new_checksum" ]]; then # Create combined PEM file # Format depends on usage_type: # - Frontend SSL: cert + key + [chain] # - Server SSL: cert + [chain] (CA cert only, no private key) { echo "$cert_content" # Only add private key if provided (server SSL may not have it) if [[ -n "$key_content" && "$key_content" != "null" ]]; then echo "" echo "$key_content" fi # Add chain if present if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then echo "" echo "$chain_content" fi } > "$ssl_file" 2>/dev/null if [[ -f "$ssl_file" ]]; then chmod 600 "$ssl_file" 2>/dev/null # Reduced logging to DEBUG to prevent spam on every check log "DEBUG" "DAEMON: SSL certificate deployed: $ssl_file ($cert_domain)" ssl_deployed_count=$((ssl_deployed_count + 1)) else log "ERROR" "DAEMON: Failed to deploy SSL certificate: $cert_name" fi fi fi fi done < <(echo "$ssl_certificates" | jq -c '.[]' 2>/dev/null) # CRITICAL: Reload HAProxy if any SSL certificates were updated # This ensures HAProxy picks up the new SSL certificates if [[ $ssl_deployed_count -gt 0 ]]; then log "INFO" "DAEMON: SSL certificates updated ($ssl_deployed_count files), triggering HAProxy reload..." # Validate HAProxy config before reload if "$HAPROXY_BIN" -c -f "$HAPROXY_CONFIG" >/dev/null 2>&1; then # Reload HAProxy using systemctl (Linux best practice) if systemctl reload haproxy >/dev/null 2>&1; then log "INFO" "DAEMON: HAProxy reloaded successfully after SSL update" else log "WARN" "DAEMON: systemctl reload failed after SSL update, attempting restart..." if systemctl restart haproxy >/dev/null 2>&1; then log "INFO" "DAEMON: HAProxy restarted successfully after SSL update" else log "ERROR" "DAEMON: Failed to reload/restart HAProxy after SSL update" fi fi else # IMPORTANT: Config validation may fail if SSL was updated but config not yet synced # This is normal - config update will trigger reload with both new SSL and config log "WARN" "DAEMON: SSL deployed but HAProxy config not yet synced (validation failed), reload will happen on next config update" fi fi fi fi fi # Check for configuration updates with proper error handling config_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null) # Validate API response before processing if [[ -n "$config_response" ]] && ! echo "$config_response" | grep -q '"error":'; then if echo "$config_response" | grep -q '"config_content":'; then # Extract config version for tracking config_version=$(echo "$config_response" | jq -r '.version // "unknown"' 2>/dev/null) # CRITICAL: Check if this version is already applied (prevent infinite loop) if [[ "$config_version" == "$last_applied_version" ]]; then # Only log in debug mode to avoid spam [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration version $config_version already applied, skipping" else log "INFO" "DAEMON: Configuration update detected (version: $config_version)" # Extract config content for validation daemon_config_content=$(echo "$config_response" | jq -r '.config_content' 2>/dev/null) # CRITICAL: Apply same validation as main function # IMPORTANT: Don't use 'continue' here - it prevents upgrade check from running! config_is_valid=true if [[ -z "$daemon_config_content" || "$daemon_config_content" == "null" ]]; then [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration content is empty, skipping config update" config_is_valid=false elif [[ "$daemon_config_content" == "# No configuration available"* ]] || [[ "$daemon_config_content" == "# Agent is disabled"* ]] || [[ "$daemon_config_content" == "# No cluster assigned"* ]]; then [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Received placeholder config (no-config state), skipping update" config_is_valid=false elif ! echo "$daemon_config_content" | grep -q -E "(global|defaults|frontend|backend|listen)"; then log "ERROR" "DAEMON: Configuration content does not appear to be valid HAProxy config" log "DEBUG" "DAEMON: Config preview: ${daemon_config_content:0:200}..." config_is_valid=false fi # Only process config if it's valid if [[ "$config_is_valid" == "true" ]]; then # Write validated config to temp file echo "$daemon_config_content" > /tmp/haproxy-new-config.cfg 2>/dev/null if [[ -f "/tmp/haproxy-new-config.cfg" && -s "/tmp/haproxy-new-config.cfg" ]]; then # Get paths from config HAPROXY_BIN=$(jq -r '.haproxy.bin_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_BIN_PATH}}") HAPROXY_CONFIG=$(jq -r '.haproxy.config_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_CONFIG_PATH}}") STATS_SOCKET=$(jq -r '.haproxy.stats_socket_path' "$CONFIG_FILE" 2>/dev/null || echo "{{STATS_SOCKET_PATH}}") # CRITICAL NEW FEATURE: Check if this is a partial config (only frontends/backends) # Backend now generates partial configs to preserve global/defaults sections if head -20 /tmp/haproxy-new-config.cfg | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then log "INFO" "DAEMON: Detected PARTIAL config - preserving global/defaults from local haproxy.cfg" # Extract global section from current haproxy.cfg if [[ -f "$HAPROXY_CONFIG" ]]; then # Extract everything from start until 'defaults' section awk '/^defaults[[:space:]]*$/{exit} {print}' "$HAPROXY_CONFIG" > /tmp/haproxy-global.cfg 2>/dev/null # Extract defaults section (from 'defaults' to first 'frontend/backend/listen') # CRITICAL FIX: Exit BEFORE printing the frontend/backend/listen line awk '/^defaults[[:space:]]*$/{flag=1} /^(frontend|backend|listen)[[:space:]]/{if(flag && !/^defaults/) exit} flag{print}' "$HAPROXY_CONFIG" > /tmp/haproxy-defaults.cfg 2>/dev/null # Extract all listen sections (preserve existing listen blocks like stats monitoring) # CRITICAL FIX: Extract ALL listen blocks from existing config to preserve them awk ' /^listen[[:space:]]/ { if (listen_content != "") { # Print previous listen block gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content } in_listen=1 listen_content=$0 "\n" next } in_listen { # Check if we hit another section (frontend, backend, listen, global, defaults) if (/^(frontend|backend|listen|global|defaults)[[:space:]]/) { # Print current listen block gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content listen_content="" in_listen=0 } else { listen_content = listen_content $0 "\n" } } END { # Print final listen block if any if (listen_content != "") { gsub(/\n[[:space:]]*$/, "", listen_content) print listen_content } } ' "$HAPROXY_CONFIG" > /tmp/haproxy-listen.cfg 2>/dev/null # Check if we successfully extracted sections if [[ -s /tmp/haproxy-global.cfg && -s /tmp/haproxy-defaults.cfg ]]; then listen_lines=0 [[ -s /tmp/haproxy-listen.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen.cfg) log "INFO" "DAEMON: Extracted global ($(wc -l < /tmp/haproxy-global.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults.cfg) lines), listen blocks ($listen_lines lines)" # Merge: global + defaults + listen blocks + new frontends/backends # Copy global section as-is, normalize trailing blanks to exactly 1 line awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global.cfg > /tmp/haproxy-merged.cfg # Copy defaults section as-is, normalize trailing blanks to exactly 1 line awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults.cfg >> /tmp/haproxy-merged.cfg # Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring) if [[ -s /tmp/haproxy-listen.cfg ]]; then awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen.cfg >> /tmp/haproxy-merged.cfg log "INFO" "DAEMON: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen.cfg) listen blocks from existing config" fi # Skip comment header from partial config, add only frontends/backends awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' /tmp/haproxy-new-config.cfg >> /tmp/haproxy-merged.cfg # Replace new config with merged version mv /tmp/haproxy-merged.cfg /tmp/haproxy-new-config.cfg log "INFO" "DAEMON: Config merged - preserved local global/defaults/listen blocks, applied new frontends/backends" # Cleanup temp files rm -f /tmp/haproxy-global.cfg /tmp/haproxy-defaults.cfg /tmp/haproxy-listen.cfg else log "WARN" "DAEMON: Failed to extract global/defaults, using partial config as-is (may fail validation)" fi else log "WARN" "DAEMON: No existing haproxy.cfg found, using partial config as-is" fi else log "INFO" "DAEMON: Full config detected - using as-is" fi # CRITICAL: Create backup before applying new config daemon_backup_file="${HAPROXY_CONFIG}.backup-$(date +%Y%m%d-%H%M%S)" if [[ -f "$HAPROXY_CONFIG" ]]; then cp "$HAPROXY_CONFIG" "$daemon_backup_file" 2>/dev/null if [[ -f "$daemon_backup_file" ]]; then log "INFO" "DAEMON: Config backed up to: $daemon_backup_file" else log "ERROR" "DAEMON: Failed to create config backup, skipping config update" rm -f /tmp/haproxy-new-config.cfg fi fi # CRITICAL FIX: Validate BEFORE applying (was: apply then validate) # Test new config from temp file FIRST if "$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c >/dev/null 2>&1; then # Validation successful - NOW apply new config cp /tmp/haproxy-new-config.cfg "$HAPROXY_CONFIG" log "INFO" "DAEMON: Configuration validation successful" # Reload HAProxy using systemctl (Linux best practice) if systemctl reload haproxy >/dev/null 2>&1; then log "INFO" "DAEMON: HAProxy reloaded successfully via systemctl" else log "WARN" "DAEMON: systemctl reload failed, attempting restart..." systemctl restart haproxy >/dev/null 2>&1 fi log "INFO" "DAEMON: Configuration applied and HAProxy reloaded" # CRITICAL: Notify backend of successful config application with version log "DEBUG" "Sending config-applied notification for version: $config_version" log "DEBUG" "Using token: ${CURRENT_AGENT_TOKEN:0:10}..." log "DEBUG" "Agent name: $AGENT_NAME" config_applied_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \ -H "Content-Type: application/json" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" \ -d "{\"status\": \"applied\", \"version\": \"$config_version\"}" 2>&1) log "DEBUG" "Config-applied response: $config_applied_response" if echo "$config_applied_response" | grep -q '"status":"ok"'; then log "INFO" "DAEMON: Config-applied notification sent successfully" else log "ERROR" "DAEMON: Config-applied notification failed: $config_applied_response" fi # CRITICAL: Update last applied version to prevent reapplying same config last_applied_version="$config_version" log "INFO" "DAEMON: Configuration applied successfully (version: $config_version)" else # CRITICAL: Validation failed - config was NEVER applied to haproxy.cfg # No restore needed - original config is still intact # Check if we already reported this validation failure to avoid spam if [[ "$config_version" != "$last_validation_failed_version" ]]; then log "ERROR" "DAEMON: Configuration validation failed - rejected before applying" log "ERROR" "DAEMON: Original HAProxy config remains unchanged" # Get validation error details (without 'local' to avoid scope issues) validation_error=$("$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c 2>&1) log "ERROR" "DAEMON: Validation error: $validation_error" # CRITICAL NEW FEATURE: Notify backend of validation failure for UI display log "INFO" "DAEMON: Sending validation error notification to backend..." curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-validation-failed" \ -H "Content-Type: application/json" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" \ -d "{\"status\": \"validation_failed\", \"version\": \"$config_version\", \"validation_error\": $(echo "$validation_error" | jq -Rs .)}" \ >/dev/null 2>&1 if [ $? -eq 0 ]; then log "INFO" "DAEMON: Validation error notification sent to backend successfully" else log "WARN" "DAEMON: Failed to send validation error notification to backend" fi # Mark this version as validation failed to prevent repeated error logs last_validation_failed_version="$config_version" else # Already reported this validation failure, just log in DEBUG to avoid spam log "DEBUG" "DAEMON: Config version $config_version still failing validation (already reported)" fi fi # Clean up rm -f /tmp/haproxy-new-config.cfg fi fi # End of config_is_valid check fi # End of version already applied check fi else # Log API error for debugging if echo "$config_response" | grep -q '"error":'; then error_msg=$(echo "$config_response" | jq -r '.error.message' 2>/dev/null || echo "Unknown API error") log "DEBUG" "DAEMON: API Error: $error_msg (skipping config check)" fi fi # Check for upgrade requests - backend controls upgrade via agent status # CRITICAL: Always log upgrade check for debugging [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Checking upgrade status (current: {{AGENT_VERSION}})" upgrade_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null) if [[ -n "$upgrade_response" ]]; then should_upgrade=$(echo "$upgrade_response" | jq -r '.should_upgrade // false' 2>/dev/null) target_version=$(echo "$upgrade_response" | jq -r '.target_version // ""' 2>/dev/null) [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade status response: should_upgrade=$should_upgrade, target=$target_version, current={{AGENT_VERSION}}" # Only upgrade if backend says we should AND target version is different from current if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "{{AGENT_VERSION}}" ]]; then log "INFO" "DAEMON: Starting upgrade from {{AGENT_VERSION}} to $target_version (backend requested)" # Download new script with validation temp_script="/tmp/haproxy-agent-upgrade-$AGENT_NAME.sh" log "INFO" "DAEMON: Downloading new agent script..." # Get correct pool_id from config daemon_pool_id=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null) [[ -z "$daemon_pool_id" || "$daemon_pool_id" == "null" ]] && daemon_pool_id=1 # CRITICAL FIX: Normalize architecture format (x86_64 -> amd64, aarch64 -> arm64) daemon_arch=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/') script_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/generate-install-script" \ -H "Content-Type: application/json" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" \ -d "{\"platform\":\"linux\",\"architecture\":\"$daemon_arch\",\"pool_id\":$daemon_pool_id,\"cluster_id\":$CLUSTER_ID,\"agent_name\":\"$AGENT_NAME\",\"hostname_prefix\":\"$(hostname | cut -d'.' -f1)\",\"haproxy_config_path\":\"/etc/haproxy/haproxy.cfg\",\"haproxy_bin_path\":\"/usr/sbin/haproxy\",\"stats_socket_path\":\"/var/run/haproxy/admin.sock\"}" 2>/dev/null) # Validate API response if [[ -z "$script_response" ]]; then log "ERROR" "DAEMON: Empty API response, aborting upgrade" continue fi if ! echo "$script_response" | grep -q '"script"'; then log "ERROR" "DAEMON: Invalid API response format, aborting upgrade" echo "Response: ${script_response:0:200}..." continue fi # Extract script content with validation echo "$script_response" | jq -r '.script' > "$temp_script" 2>/dev/null # Validate downloaded script if [[ ! -f "$temp_script" ]]; then log "ERROR" "DAEMON: Failed to create temp script file, aborting upgrade" continue fi if [[ ! -s "$temp_script" ]]; then log "ERROR" "DAEMON: Downloaded script is empty, aborting upgrade" rm -f "$temp_script" continue fi # Check if script is valid bash script if ! head -1 "$temp_script" | grep -q "#!/bin/bash"; then log "ERROR" "DAEMON: Downloaded script is not a valid bash script, aborting upgrade" echo "First line: $(head -1 "$temp_script")" rm -f "$temp_script" continue fi script_size=$(wc -c < "$temp_script") if [[ $script_size -lt 10000 ]]; then log "ERROR" "DAEMON: Downloaded script too small ($script_size bytes), aborting upgrade" rm -f "$temp_script" continue fi log "INFO" "DAEMON: Script validation passed ($script_size bytes)" # Create backup before upgrade (ensure backup directory exists) mkdir -p "/usr/local/share/haproxy-agent-backups" 2>/dev/null || true backup_file="/usr/local/share/haproxy-agent-backups/haproxy-agent-backup-$(date +%Y%m%d-%H%M%S).sh" if ! cp /usr/local/bin/haproxy-agent "$backup_file" 2>/dev/null; then log "ERROR" "DAEMON: Failed to create backup, aborting upgrade" rm -f "$temp_script" continue fi log "INFO" "DAEMON: Backup created: $backup_file" # Atomic script replacement temp_target="/usr/local/bin/haproxy-agent.new" if ! cp "$temp_script" "$temp_target" 2>/dev/null; then log "ERROR" "DAEMON: Failed to copy new script, aborting upgrade" rm -f "$temp_script" continue fi if ! chmod 755 "$temp_target" 2>/dev/null; then log "ERROR" "DAEMON: Failed to set permissions, aborting upgrade" rm -f "$temp_script" "$temp_target" continue fi # Final validation of target script if ! head -1 "$temp_target" | grep -q "#!/bin/bash"; then log "ERROR" "DAEMON: Target script corrupted, restoring backup" cp "$backup_file" /usr/local/bin/haproxy-agent rm -f "$temp_script" "$temp_target" continue fi # Atomic move if ! mv "$temp_target" /usr/local/bin/haproxy-agent 2>/dev/null; then log "ERROR" "DAEMON: Failed to replace script, restoring backup" cp "$backup_file" /usr/local/bin/haproxy-agent 2>/dev/null rm -f "$temp_script" "$temp_target" continue fi log "INFO" "DAEMON: Script updated successfully" # Notify completion curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \ -H "Content-Type: application/json" \ -H "X-API-Key: $CURRENT_AGENT_TOKEN" \ -d "{\"version\":\"$target_version\",\"status\":\"completed\"}" >/dev/null 2>&1 log "INFO" "DAEMON: Upgrade to $target_version completed - restarting..." # Clean up rm -f "$temp_script" # Exit to allow service restart with new script exit 0 else # Debug: Log why upgrade was not performed [[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade not needed - should_upgrade: $should_upgrade, target_version: $target_version, current: {{AGENT_VERSION}}" fi fi done # CRITICAL FIX: Daemon loop should never exit log "ERROR" "DAEMON: Daemon loop exited unexpectedly, restarting..." exit 1 fi