mirror of
https://github.com/taylanbakircioglu/haproxy-openmanager.git
synced 2026-09-17 08:05:09 +00:00
taylanbakircioglu
b3b0544b11
PROBLEM: - Agent stop took ~90 seconds (systemd default timeout) - No signal handling (trap) - SIGTERM ignored during sleep 30 - Uninterruptible sleep blocked graceful shutdown SOLUTION: 1. Signal Handler - Added trap handler for SIGTERM/SIGINT/SIGQUIT - SHUTDOWN_REQUESTED flag for graceful exit 2. Interruptible Sleep - Changed: sleep 30 → 30x sleep 1 - Check shutdown flag every second - Agent stops in 1-2 seconds instead of 90 3. Improved Stop Command - Graceful SIGTERM → wait 5s → force SIGKILL - Verification that process actually stopped - User feedback during stop operation 4. Systemd Timeout Configuration (Linux) - TimeoutStopSec=10 (instead of default 90s) - KillMode=mixed (SIGTERM main, SIGKILL others) - KillSignal=SIGTERM (explicit) IMPACT: - Stop time: 90s → 1-2s (98% improvement) - Config apply: ✅ SAFE - completes before shutdown - HAProxy reload: ✅ SAFE - subprocess not affected - Stats sending: ✅ SAFE - heartbeat completes - Agent upgrade: ✅ SAFE - upgrade completes - Backward compatible: ✅ YES TECHNICAL DETAILS: - Bash signal handling: functions complete atomically - Subprocess isolation: systemctl/curl not interrupted - Loop control timing: check only between functions FILES CHANGED: - backend/utils/agent_scripts/linux_install.sh (+79 lines) - backend/utils/agent_scripts/macos_install.sh (+68 lines)
3071 lines
136 KiB
Bash
3071 lines
136 KiB
Bash
#!/bin/bash
|
|
set -e
|
|
|
|
# HAProxy Management Agent Installation Script for Linux
|
|
# Generated by HAProxy Management UI on: $(date -u '+%Y-%m-%d %H:%M:%S UTC')
|
|
# Target Cluster ID: $CLUSTER_ID
|
|
# Script Build ID: 20250925-v1.0.2-ssl-deployment-ready
|
|
|
|
# Early variable definition for backup (before interactive mode)
|
|
HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}"
|
|
|
|
# Quick daemon mode detection (before installer UI)
|
|
# Suppress installer messages if running in daemon mode
|
|
QUIET_MODE=false
|
|
if [[ "$1" == "daemon" || "$SKIP_TO_DAEMON" == "true" || -f "/etc/haproxy-agent/config.json" ]]; then
|
|
QUIET_MODE=true
|
|
fi
|
|
|
|
# Show installer header only in interactive installation mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo "=========================================="
|
|
echo "HAProxy Management Agent Installer v{{AGENT_VERSION}}"
|
|
echo "Linux Edition"
|
|
echo "With Version Management & Auto-Upgrade"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "⚠️ IMPORTANT WARNING ⚠️"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "After agent installation, ALL your existing frontend and backend"
|
|
echo "configurations will be removed to enable full management through"
|
|
echo "HAProxy OpenManager."
|
|
echo ""
|
|
echo "An automatic backup will be created: haproxy.cfg.initial-backup"
|
|
echo ""
|
|
echo "📋 REQUIRED STEPS AFTER INSTALLATION:"
|
|
echo " 1. Login to HAProxy OpenManager interface"
|
|
echo " 2. Use the BULK IMPORT feature to restore your configuration"
|
|
echo ""
|
|
echo "⚠️ IMPORTANT: After installation, ALL configuration changes MUST be"
|
|
echo " made through the HAProxy OpenManager interface only."
|
|
echo " Manual changes will be overwritten on the next update."
|
|
echo ""
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
# Create automatic backup BEFORE asking for confirmation
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
BACKUP_PATH="${HAPROXY_CONFIG_PATH}.initial-backup"
|
|
if [[ ! -f "$BACKUP_PATH" ]]; then
|
|
echo "📦 Creating backup: $BACKUP_PATH"
|
|
cp "$HAPROXY_CONFIG_PATH" "$BACKUP_PATH" && echo "✅ Backup created successfully" || echo "⚠️ Backup failed"
|
|
echo ""
|
|
fi
|
|
fi
|
|
|
|
read -p "Do you understand and wish to continue? (Y/N): " -n 1 -r
|
|
echo ""
|
|
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
|
|
echo "Installation cancelled."
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
echo "Proceeding with installation..."
|
|
echo ""
|
|
fi
|
|
|
|
# ============================================
|
|
# CRITICAL: Function Definitions (Must be at top for daemon mode)
|
|
# ============================================
|
|
|
|
# Dynamically locate required binaries
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Logging function - JSON format, single line with timestamp
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
local log_file="${LOG_FILE:-/var/log/haproxy-agent/agent.log}"
|
|
local agent_name="${AGENT_NAME:-unknown}"
|
|
|
|
# Ensure log directory exists
|
|
mkdir -p "$(dirname "$log_file")" 2>/dev/null || true
|
|
|
|
# Escape message for JSON (replace backslash, quotes, newlines, tabs)
|
|
local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ')
|
|
|
|
# Remove any control characters and emojis (keep only printable ASCII + basic UTF-8)
|
|
escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g')
|
|
|
|
# Write JSON log entry (single line) - ONLY to log file
|
|
echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$agent_name\",\"message\":\"$escaped_message\"}" >> "$log_file" 2>/dev/null || true
|
|
|
|
# DO NOT print to stdout in daemon mode (causes duplicate logs)
|
|
# Installer messages are shown via direct echo commands
|
|
}
|
|
|
|
# Get HAProxy service status
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
# Old get_server_statuses() removed - using new version with lazy init (see line ~900)
|
|
|
|
# Collect system information
|
|
collect_system_info() {
|
|
local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux")
|
|
local kernel_version=$(uname -r 2>/dev/null || echo "Unknown")
|
|
local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0")
|
|
local cpu_count=$(nproc 2>/dev/null || echo "1")
|
|
local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0")
|
|
local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0")
|
|
local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "")
|
|
local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "")
|
|
|
|
cat <<SYSTEM_INFO_EOF
|
|
"operating_system": "$os_info",
|
|
"kernel_version": "$kernel_version",
|
|
"uptime": $uptime_seconds,
|
|
"cpu_count": $cpu_count,
|
|
"memory_total": $memory_bytes,
|
|
"disk_space": $disk_bytes,
|
|
"ip_address": "$ip_address",
|
|
"network_interfaces": ["${network_interfaces//,/\",\"}"],
|
|
"capabilities": ["haproxy_management", "ssl_deployment", "config_reload", "systemd_service"]
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Send heartbeat
|
|
# Old send_heartbeat() removed - using new version with haproxy_stats_csv support (see line ~1174)
|
|
|
|
# Check for pending configuration requests
|
|
check_config_requests() {
|
|
local curl_bin="${CURL_BIN:-$(find_binary curl)}"
|
|
|
|
# Get pending config requests from backend
|
|
local response=$("$curl_bin" -k -s -X GET "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}")
|
|
|
|
# Check if there are pending requests
|
|
local pending_count=$(echo "$response" | jq -r '.pending_requests | length' 2>/dev/null)
|
|
|
|
if [[ "$pending_count" -gt 0 ]]; then
|
|
log "INFO" "CONFIG: Found $pending_count pending config request(s)"
|
|
|
|
# Process each request
|
|
echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do
|
|
local request_id=$(echo "$request" | jq -r '.request_id')
|
|
local request_type=$(echo "$request" | jq -r '.request_type')
|
|
|
|
log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)"
|
|
|
|
# Read haproxy.cfg content
|
|
local config_path="${HAPROXY_CONFIG_PATH:-{{HAPROXY_CONFIG_PATH}}}"
|
|
if [[ -f "$config_path" ]]; then
|
|
local config_content=$(cat "$config_path")
|
|
local file_size=$(wc -c < "$config_path")
|
|
|
|
# Escape config content for JSON
|
|
local escaped_content=$(echo "$config_content" | jq -Rs .)
|
|
|
|
# Submit config response
|
|
local response_payload=$(cat <<CONFIG_RESPONSE_EOF
|
|
{
|
|
"request_id": $request_id,
|
|
"config_content": $escaped_content,
|
|
"config_path": "$config_path"
|
|
}
|
|
CONFIG_RESPONSE_EOF
|
|
)
|
|
|
|
"$curl_bin" -k -s -X POST "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/config-response" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}" \
|
|
-d "$response_payload" > /dev/null 2>&1
|
|
|
|
log "INFO" "Config response sent for request #$request_id (size: $file_size bytes)"
|
|
else
|
|
log "ERROR" "Config file not found: $config_path"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
# ============================================
|
|
# End of Function Definitions
|
|
# ============================================
|
|
|
|
# CRITICAL FIX: Enhanced daemon mode detection for Linux
|
|
# Check multiple conditions for daemon mode
|
|
DAEMON_MODE=false
|
|
|
|
# Method 1: Explicit daemon argument
|
|
if [[ "$1" == "daemon" ]]; then
|
|
log "INFO" "DAEMON: Explicit daemon argument detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
# Method 2: Environment variable from systemd
|
|
if [[ "$SKIP_TO_DAEMON" == "true" ]]; then
|
|
log "INFO" "DAEMON: Environment variable SKIP_TO_DAEMON=true detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
# Method 3: Config file exists (upgrade scenario)
|
|
if [[ -f "/etc/haproxy-agent/config.json" ]]; then
|
|
log "INFO" "DAEMON: Existing config file detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
# Method 4: Legacy environment variable
|
|
if [[ "$HAPROXY_AGENT_SKIP_SETUP" == "true" ]]; then
|
|
log "INFO" "DAEMON: Legacy skip setup detected"
|
|
DAEMON_MODE=true
|
|
fi
|
|
|
|
if [[ "$DAEMON_MODE" == "true" ]]; then
|
|
log "INFO" "DAEMON: Starting agent daemon with existing configuration..."
|
|
|
|
# Load existing configuration
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/var/run/haproxy-agent.pid"
|
|
|
|
# Load configuration from file
|
|
if [[ -f "$CONFIG_FILE" ]]; then
|
|
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE" 2>/dev/null)
|
|
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null)
|
|
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE" 2>/dev/null)
|
|
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE" 2>/dev/null)
|
|
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE" 2>/dev/null)
|
|
|
|
log "INFO" "DAEMON: Configuration loaded - Agent: $AGENT_NAME, Cluster: $CLUSTER_ID"
|
|
|
|
# FIXED: Jump directly to daemon mode instead of setting SKIP_TO_DAEMON (MacOS style)
|
|
log "INFO" "DAEMON: Going directly to daemon mode..."
|
|
|
|
# Jump to the daemon section at the end of the script
|
|
SKIP_TO_DAEMON=true
|
|
else
|
|
log "ERROR" "DAEMON: Configuration file not found: $CONFIG_FILE"
|
|
log "INFO" "DAEMON: Falling back to installer mode..."
|
|
DAEMON_MODE=false
|
|
fi
|
|
fi
|
|
|
|
# Configuration variables - MODIFY THESE IF NEEDED
|
|
MANAGEMENT_URL="{{MANAGEMENT_URL}}"
|
|
CLUSTER_ID="{{CLUSTER_ID}}"
|
|
AGENT_NAME="{{AGENT_NAME}}"
|
|
HOSTNAME_PREFIX="{{HOSTNAME_PREFIX}}"
|
|
HOSTNAME="${HOSTNAME_PREFIX}-$(hostname -s)"
|
|
HAPROXY_CONFIG_PATH="{{HAPROXY_CONFIG_PATH}}"
|
|
HAPROXY_BIN_PATH="{{HAPROXY_BIN_PATH}}"
|
|
STATS_SOCKET_PATH="{{STATS_SOCKET_PATH}}"
|
|
HAPROXY_SERVICE_NAME="haproxy"
|
|
|
|
# Skip authentication if we already have token from config (daemon mode)
|
|
if [[ "$SKIP_TO_DAEMON" == "true" && -n "$AGENT_TOKEN" ]]; then
|
|
log "DEBUG" "AGENT UPGRADE: Skipping authentication - using token from config"
|
|
else
|
|
# SECURE AGENT AUTHENTICATION
|
|
echo ""
|
|
log "INFO" "Agent Authentication Setup"
|
|
echo "============================================"
|
|
echo "This agent requires a secure API token for authentication."
|
|
echo "Please obtain the API token from the Security page in the"
|
|
echo "HAProxy Management UI before proceeding."
|
|
echo ""
|
|
echo "Steps to get your API token:"
|
|
echo "1. Open HAProxy Management UI"
|
|
echo "2. Go to Security page"
|
|
echo "3. Create a new agent token for: $AGENT_NAME"
|
|
echo "4. Copy the generated token"
|
|
echo ""
|
|
|
|
# Prompt for API token with validation
|
|
while true; do
|
|
echo -n "Enter Agent API Token: "
|
|
read -r AGENT_TOKEN
|
|
|
|
if [[ -z "$AGENT_TOKEN" ]]; then
|
|
log "ERROR" "API token cannot be empty"
|
|
continue
|
|
fi
|
|
|
|
# Validate token format (should start with 'hap_')
|
|
if [[ ! "$AGENT_TOKEN" =~ ^hap_.+ ]]; then
|
|
log "ERROR" "Invalid token format. Token should start with 'hap_'"
|
|
echo " Please ensure you copied the complete token from the Security page."
|
|
continue
|
|
fi
|
|
|
|
# Test token validity by making a test API call
|
|
log "DEBUG" "Validating API token..."
|
|
TEST_RESPONSE=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-H "User-Agent: haproxy-agent-installer" 2>/dev/null || echo "FAILED")
|
|
|
|
if [[ "$TEST_RESPONSE" == "FAILED" ]] || [[ ! "$TEST_RESPONSE" =~ clusters ]]; then
|
|
log "ERROR" "API token validation failed"
|
|
echo " Please check:"
|
|
echo " - Token is correct and complete"
|
|
echo " - Token has not been revoked"
|
|
echo " - Management URL is accessible: $MANAGEMENT_URL"
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "API token validated successfully"
|
|
break
|
|
done
|
|
|
|
fi
|
|
|
|
# Skip installer output in quiet mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo ""
|
|
fi
|
|
|
|
# Dynamically locate required binaries (global scope)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
CURL_BIN=$(find_binary curl)
|
|
if [[ -z "$CURL_BIN" ]]; then
|
|
log "ERROR" "curl not found in system"
|
|
exit 1
|
|
fi
|
|
|
|
# Installation paths for Linux
|
|
INSTALL_DIR="/usr/local/bin"
|
|
CONFIG_DIR="/etc/haproxy-agent"
|
|
LOG_DIR="/var/log/haproxy-agent"
|
|
|
|
# Skip cleanup if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# CRITICAL: Clean installation - Remove existing agent components
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo ""
|
|
echo "🧹 Performing pre-installation cleanup..."
|
|
fi
|
|
|
|
# Function to safely remove files/directories
|
|
safe_remove() {
|
|
local path="$1"
|
|
local desc="$2"
|
|
|
|
if [[ -e "$path" ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " 🗑️ Removing $desc..."
|
|
rm -rf "$path" 2>/dev/null || true
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " $desc removed"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# 1. Stop and kill all existing agent processes
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "🔪 Terminating existing HAProxy Agent processes..."
|
|
KILLED_COUNT=0
|
|
for pattern in "haproxy-agent" "/usr/local/bin/haproxy-agent" "haproxy-agent.service"; do
|
|
PIDS=$(pgrep -f "$pattern" 2>/dev/null || true)
|
|
if [[ -n "$PIDS" ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Killing processes matching: $pattern"
|
|
kill -TERM $PIDS 2>/dev/null || true
|
|
sleep 2
|
|
kill -KILL $PIDS 2>/dev/null || true
|
|
KILLED_COUNT=$((KILLED_COUNT + $(echo "$PIDS" | wc -w)))
|
|
fi
|
|
done
|
|
|
|
if [[ $KILLED_COUNT -gt 0 ]]; then
|
|
log "INFO" "Terminated $KILLED_COUNT existing agent processes"
|
|
else
|
|
log "INFO" "No existing agent processes found"
|
|
fi
|
|
|
|
# 2. Stop and disable existing systemd services
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "🛑 Cleaning up existing systemd services..."
|
|
if systemctl is-enabled haproxy-agent >/dev/null 2>&1; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Stopping and disabling haproxy-agent service"
|
|
systemctl stop haproxy-agent 2>/dev/null || true
|
|
systemctl disable haproxy-agent 2>/dev/null || true
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Service haproxy-agent cleaned up"
|
|
fi
|
|
|
|
# 3. Remove existing files and directories
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "🗑️ Removing existing agent files..."
|
|
safe_remove "/etc/systemd/system/haproxy-agent.service" "systemd service file"
|
|
safe_remove "/usr/local/bin/haproxy-agent" "agent binary"
|
|
safe_remove "/etc/haproxy-agent" "configuration directory"
|
|
safe_remove "/var/log/haproxy-agent" "log directory"
|
|
safe_remove "/var/run/haproxy-agent.pid" "PID file"
|
|
|
|
# Clean up temporary config files
|
|
TEMP_COUNT=0
|
|
for temp_file in /tmp/haproxy_new_*.cfg; do
|
|
if [[ -f "$temp_file" ]]; then
|
|
rm -f "$temp_file"
|
|
TEMP_COUNT=$((TEMP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $TEMP_COUNT -gt 0 ]]; then
|
|
[[ "$QUIET_MODE" != "true" ]] && echo " Removed $TEMP_COUNT temporary config files"
|
|
fi
|
|
|
|
log "INFO" "Existing agent components cleaned up"
|
|
[[ "$QUIET_MODE" != "true" ]] && echo "Pre-installation cleanup completed"
|
|
[[ "$QUIET_MODE" != "true" ]] && echo ""
|
|
|
|
fi # End of cleanup section
|
|
|
|
# Detect architecture
|
|
ARCH=$(uname -m)
|
|
case "$ARCH" in
|
|
"x86_64") ARCH_SUFFIX="linux-amd64"; [[ "$QUIET_MODE" != "true" ]] && echo "🖥️ Detected: Linux AMD64" ;;
|
|
"aarch64"|"arm64") ARCH_SUFFIX="linux-arm64"; [[ "$QUIET_MODE" != "true" ]] && echo "🦾 Detected: Linux ARM64" ;;
|
|
*) ARCH_SUFFIX="linux-unknown"; log "WARN" "Unknown architecture: $ARCH" ;;
|
|
esac
|
|
|
|
# Display configuration only in interactive mode
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo "📋 Agent Configuration:"
|
|
echo " Management URL: $MANAGEMENT_URL"
|
|
echo " Cluster ID: $CLUSTER_ID"
|
|
echo " Agent Name: $AGENT_NAME"
|
|
echo " Hostname: $HOSTNAME"
|
|
echo " Architecture: $ARCH ($ARCH_SUFFIX)"
|
|
echo ""
|
|
echo "📋 HAProxy Integration:"
|
|
echo " Config Path: $HAPROXY_CONFIG_PATH"
|
|
echo " Stats Socket: $STATS_SOCKET_PATH"
|
|
echo " Service Name: $HAPROXY_SERVICE_NAME"
|
|
echo ""
|
|
fi
|
|
|
|
# Only show agent capabilities during initial installation, not on daemon restart
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
log "INFO" "This agent will:"
|
|
if [[ "$QUIET_MODE" != "true" ]]; then
|
|
echo " • Monitor and report HAProxy service status"
|
|
echo " • Receive configuration updates from management UI"
|
|
echo " • Apply changes to HAProxy config file: $HAPROXY_CONFIG_PATH"
|
|
echo " • Manage HAProxy service lifecycle (restart/reload)"
|
|
echo " • Send heartbeats every 10 seconds"
|
|
echo ""
|
|
fi
|
|
else
|
|
# Daemon mode - reduced logging for restart scenarios
|
|
log "DEBUG" "AGENT UPGRADE: Skipping entire installation - going to daemon mode"
|
|
fi
|
|
|
|
# Check root permissions
|
|
if [[ "$SKIP_TO_DAEMON" != "true" && $EUID -ne 0 ]]; then
|
|
log "WARN" "This script needs root privileges for agent service installation."
|
|
echo " Please run with: sudo $0"
|
|
exit 1
|
|
fi
|
|
|
|
# Skip validation if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
# Validate cluster exists by checking management API
|
|
log "DEBUG" "Validating HAProxy Cluster..."
|
|
CLUSTER_CHECK=$("$CURL_BIN" -k -s -f "$MANAGEMENT_URL/api/clusters" -H "User-Agent: haproxy-agent-installer" || echo "FAILED")
|
|
if [[ "$CLUSTER_CHECK" == "FAILED" ]]; then
|
|
log "ERROR" "Failed to connect to management API!"
|
|
echo " Please check your network connection and management URL."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if our specific cluster ID exists in the response
|
|
if ! echo "$CLUSTER_CHECK" | jq -e ".clusters[]? | select(.id==$CLUSTER_ID)" >/dev/null; then
|
|
log "ERROR" "HAProxy Cluster (ID: $CLUSTER_ID) not found!"
|
|
echo " Available clusters:"
|
|
echo "$CLUSTER_CHECK" | jq -r '.clusters[] | " - \(.id): \(.name)"'
|
|
echo " Please ensure the cluster exists in the management UI before running this script."
|
|
exit 1
|
|
fi
|
|
log "INFO" "Cluster validation successful"
|
|
fi
|
|
|
|
# Check dependencies
|
|
check_dependencies() {
|
|
log "DEBUG" "Checking system requirements..."
|
|
|
|
# Check if HAProxy is installed
|
|
if ! command -v haproxy &> /dev/null; then
|
|
log "WARN" "HAProxy not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get update && apt-get install -y haproxy
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y haproxy
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y haproxy
|
|
else
|
|
log "ERROR" "Cannot install HAProxy automatically. Please install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "HAProxy found: $(haproxy -v | head -1)"
|
|
|
|
# CRITICAL: Validate existing HAProxy config
|
|
# Agent is designed to manage an EXISTING HAProxy installation with global+defaults sections
|
|
log "DEBUG" "Validating existing HAProxy configuration..."
|
|
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Check if config has global section
|
|
if ! grep -q "^global" "$HAPROXY_CONFIG_PATH"; then
|
|
log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'global' section!"
|
|
echo ""
|
|
echo "INSTALLATION REQUIREMENT NOT MET"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Your HAProxy config must contain:"
|
|
echo " • global section (with logging, stats socket, etc.)"
|
|
echo " • defaults section (with timeouts, mode, etc.)"
|
|
echo ""
|
|
echo "The agent will:"
|
|
echo " PRESERVE your global and defaults sections"
|
|
echo " MANAGE only frontends and backends"
|
|
echo ""
|
|
echo "Current config: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "Please ensure HAProxy is properly configured before installing agent."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if config has defaults section
|
|
if ! grep -q "^defaults" "$HAPROXY_CONFIG_PATH"; then
|
|
log "ERROR" "Existing HAProxy config ($HAPROXY_CONFIG_PATH) is missing 'defaults' section!"
|
|
echo ""
|
|
echo "INSTALLATION REQUIREMENT NOT MET"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Your HAProxy config must contain:"
|
|
echo " • global section (with logging, stats socket, etc.)"
|
|
echo " • defaults section (with timeouts, mode, etc.)"
|
|
echo ""
|
|
echo "The agent will:"
|
|
echo " PRESERVE your global and defaults sections"
|
|
echo " MANAGE only frontends and backends"
|
|
echo ""
|
|
echo "Current config: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "Please ensure HAProxy is properly configured before installing agent."
|
|
exit 1
|
|
fi
|
|
|
|
log "INFO" "Existing HAProxy config validated (global+defaults sections present)"
|
|
else
|
|
log "WARN" "HAProxy config not found at $HAPROXY_CONFIG_PATH"
|
|
log "WARN" "Agent expects to manage an existing HAProxy installation"
|
|
echo ""
|
|
echo " WARNING: HAProxy config not found"
|
|
echo ""
|
|
echo "This agent is designed to manage an EXISTING HAProxy installation."
|
|
echo "Config file expected at: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "The agent will create a new config if you continue, but this is NOT recommended."
|
|
echo "It's better to set up HAProxy first with proper global/defaults sections."
|
|
echo ""
|
|
read -p "Continue anyway? (yes/no): " -r CONTINUE_WITHOUT_CONFIG
|
|
if [[ "$CONTINUE_WITHOUT_CONFIG" != "yes" ]]; then
|
|
log "INFO" "Installation cancelled by user"
|
|
exit 0
|
|
fi
|
|
log "WARN" "User chose to continue without existing HAProxy config"
|
|
fi
|
|
|
|
# Check if config directory exists
|
|
HAPROXY_DIR=$(dirname "$HAPROXY_CONFIG_PATH")
|
|
if [[ ! -d "$HAPROXY_DIR" ]]; then
|
|
echo "Creating HAProxy config directory: $HAPROXY_DIR"
|
|
mkdir -p "$HAPROXY_DIR"
|
|
fi
|
|
|
|
# Check if stats socket directory exists
|
|
STATS_DIR=$(dirname "$STATS_SOCKET_PATH")
|
|
if [[ ! -d "$STATS_DIR" ]]; then
|
|
echo "Creating HAProxy stats socket directory: $STATS_DIR"
|
|
mkdir -p "$STATS_DIR"
|
|
fi
|
|
|
|
# Check if jq is installed
|
|
if ! command -v jq &> /dev/null; then
|
|
log "WARN" "jq not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get install -y jq
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y jq
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y jq
|
|
else
|
|
log "ERROR" "Cannot install jq automatically. Please install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "jq found: $(jq --version)"
|
|
|
|
# Check if socat is installed (required for HAProxy stats socket)
|
|
if ! command -v socat &> /dev/null; then
|
|
log "WARN" "socat not found. Installing..."
|
|
if command -v apt-get &> /dev/null; then
|
|
apt-get install -y socat
|
|
elif command -v yum &> /dev/null; then
|
|
yum install -y socat
|
|
elif command -v dnf &> /dev/null; then
|
|
dnf install -y socat
|
|
else
|
|
log "ERROR" "Cannot install socat automatically. Please install manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "socat found: $(socat -V | head -1)"
|
|
}
|
|
|
|
# Skip dependency check if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
check_dependencies
|
|
fi
|
|
|
|
# Skip final cleanup if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# CRITICAL: Final cleanup as root (clean any remaining files)
|
|
echo ""
|
|
echo "🧹 Final cleanup as root (removing any remaining files)..."
|
|
safe_remove "/etc/systemd/system/haproxy-agent.service" "existing systemd service file"
|
|
safe_remove "/usr/local/bin/haproxy-agent" "existing agent binary"
|
|
safe_remove "/etc/haproxy-agent" "existing configuration directory"
|
|
safe_remove "/var/log/haproxy-agent" "existing log directory"
|
|
safe_remove "/var/run/haproxy-agent.pid" "existing PID file"
|
|
safe_remove "/tmp/haproxy-agent.pid" "temp PID file"
|
|
|
|
# Clean up HAProxy config backups
|
|
BACKUP_COUNT=0
|
|
for backup_file in /etc/haproxy/haproxy.cfg.backup.*; do
|
|
if [[ -f "$backup_file" ]]; then
|
|
rm -f "$backup_file"
|
|
BACKUP_COUNT=$((BACKUP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $BACKUP_COUNT -gt 0 ]]; then
|
|
echo " Removed $BACKUP_COUNT HAProxy config backups"
|
|
fi
|
|
|
|
# Clean up any remaining temp files
|
|
TEMP_COUNT=0
|
|
for temp_file in /tmp/haproxy_new_*.cfg /tmp/*agent*.log; do
|
|
if [[ -f "$temp_file" ]]; then
|
|
rm -f "$temp_file"
|
|
TEMP_COUNT=$((TEMP_COUNT + 1))
|
|
fi
|
|
done
|
|
|
|
if [[ $TEMP_COUNT -gt 0 ]]; then
|
|
echo " Removed $TEMP_COUNT temporary files"
|
|
fi
|
|
|
|
log "INFO" "Final cleanup completed - Ready for fresh installation"
|
|
echo ""
|
|
|
|
fi # End of final cleanup section
|
|
|
|
# Skip directory creation if in daemon mode
|
|
if [[ "$SKIP_TO_DAEMON" != "true" ]]; then
|
|
|
|
# Create directories with proper permissions for upgrades
|
|
echo "Creating directories..."
|
|
mkdir -p "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR"
|
|
|
|
# Set comprehensive permissions for upgrade operations
|
|
log "INFO" "Setting permissions for upgrade operations..."
|
|
|
|
# Ensure agent binary directory is writable (for in-place upgrades)
|
|
chmod 755 "$INSTALL_DIR"
|
|
chown root:root "$INSTALL_DIR" 2>/dev/null || true
|
|
|
|
# Ensure config directory is accessible
|
|
chmod 755 "$CONFIG_DIR"
|
|
chown root:root "$CONFIG_DIR" 2>/dev/null || true
|
|
|
|
# Ensure log directory is writable
|
|
chmod 755 "$LOG_DIR"
|
|
chown root:root "$LOG_DIR" 2>/dev/null || true
|
|
|
|
# Create and set permissions for temp directory (upgrade markers)
|
|
# Agent will run as root, so ensure /tmp is accessible
|
|
mkdir -p /tmp
|
|
chmod 1777 /tmp 2>/dev/null || true # Sticky bit for shared temp usage
|
|
|
|
# Create agent-specific temp directory for upgrade markers
|
|
AGENT_TEMP_DIR="/tmp/haproxy-agent"
|
|
mkdir -p "$AGENT_TEMP_DIR"
|
|
chmod 755 "$AGENT_TEMP_DIR"
|
|
chown root:root "$AGENT_TEMP_DIR" 2>/dev/null || true
|
|
|
|
# Test upgrade marker creation/removal capabilities
|
|
touch "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-upgrade-test-$AGENT_NAME" 2>/dev/null
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "Upgrade marker permissions verified"
|
|
else
|
|
log "WARN" "Warning: Upgrade marker permissions may be restricted"
|
|
fi
|
|
|
|
# Ensure agent binary will have execution permissions for all operations
|
|
log "INFO" "Configuring comprehensive agent permissions..."
|
|
|
|
log "INFO" "Upgrade permissions configured"
|
|
|
|
# Create agent service script (similar to macOS but with Linux-specific adjustments)
|
|
echo "�� Installing HAProxy Agent..."
|
|
cat > "$INSTALL_DIR/haproxy-agent" << 'AGENT_SCRIPT'
|
|
#!/bin/bash
|
|
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/var/run/haproxy-agent.pid"
|
|
|
|
# Dynamic binary finder function (embedded in daemon)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=$(command -v "$binary_name" 2>/dev/null)
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
return 0
|
|
fi
|
|
|
|
# Fallback: search in common locations
|
|
local search_paths=("/usr/local/bin" "/usr/bin" "/bin" "/sbin" "/usr/sbin")
|
|
for path in "${search_paths[@]}"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
echo "$path/$binary_name"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Initialize global binary paths for daemon
|
|
CURL_BIN=$(find_binary curl)
|
|
if [[ -z "$CURL_BIN" ]]; then
|
|
echo "ERROR: curl not found in system" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Load configuration
|
|
if [[ ! -f "$CONFIG_FILE" ]]; then
|
|
echo "ERROR: Configuration not found: $CONFIG_FILE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
MANAGEMENT_URL=$(jq -r '.management.url' "$CONFIG_FILE")
|
|
AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE")
|
|
CLUSTER_ID=$(jq -r '.management.cluster_id' "$CONFIG_FILE")
|
|
AGENT_NAME=$(jq -r '.agent.name' "$CONFIG_FILE")
|
|
HOSTNAME=$(jq -r '.agent.hostname' "$CONFIG_FILE")
|
|
|
|
# SSL incremental update timestamp file (agent-specific to avoid race conditions)
|
|
# Each agent maintains its own SSL sync timestamp to prevent conflicts
|
|
SSL_SYNC_TIMESTAMP_FILE="/tmp/haproxy-agent-ssl-sync-${AGENT_NAME}"
|
|
|
|
# Get dynamic HAProxy paths from cluster configuration
|
|
get_cluster_paths() {
|
|
local cluster_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/clusters" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
HAPROXY_CONFIG_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_config_path // "/etc/haproxy/haproxy.cfg"')
|
|
HAPROXY_BIN_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .haproxy_bin_path // "/usr/sbin/haproxy"')
|
|
STATS_SOCKET_PATH=$(echo "$cluster_response" | jq -r '.clusters[] | select(.id == '$CLUSTER_ID') | .stats_socket_path // "/var/run/haproxy/admin.sock"')
|
|
else
|
|
# Fallback to config file values if cluster fetch fails
|
|
HAPROXY_CONFIG_PATH=$(jq -r '.haproxy.config_path // "/etc/haproxy/haproxy.cfg"' "$CONFIG_FILE")
|
|
HAPROXY_BIN_PATH=$(jq -r '.haproxy.bin_path // "/usr/sbin/haproxy"' "$CONFIG_FILE")
|
|
STATS_SOCKET_PATH=$(jq -r '.haproxy.stats_socket_path // "/var/run/haproxy/admin.sock"' "$CONFIG_FILE")
|
|
fi
|
|
}
|
|
|
|
# Initialize cluster paths
|
|
get_cluster_paths
|
|
|
|
# Enable debug mode for troubleshooting (0=off, 1=on)
|
|
DEBUG_MODE=0
|
|
|
|
# Logging - JSON format, single line with timestamp
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
|
|
if [[ "$level" == "DEBUG" && "${DEBUG_MODE:-0}" != "1" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
|
|
|
|
# Escape message for JSON (replace backslash, quotes, newlines, tabs)
|
|
local escaped_message=$(echo "$message" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/ /g' | tr '\n' ' ' | tr '\r' ' ')
|
|
|
|
# Remove any control characters and emojis (keep only printable ASCII + basic UTF-8)
|
|
escaped_message=$(echo "$escaped_message" | LC_ALL=C sed 's/[^[:print:] ]//g')
|
|
|
|
echo "{\"timestamp\":\"$timestamp\",\"level\":\"$level\",\"agent\":\"$AGENT_NAME\",\"message\":\"$escaped_message\"}" >> "$LOG_FILE"
|
|
}
|
|
|
|
# Register agent on first run
|
|
register_agent() {
|
|
log "INFO" "Registering agent with management server..."
|
|
|
|
local haproxy_version="unknown"
|
|
if command -v haproxy &> /dev/null; then
|
|
haproxy_version=$(haproxy -v | head -1 | awk '{print $3}')
|
|
fi
|
|
|
|
local arch=$(uname -m)
|
|
platform=$(uname -s | tr '[:upper:]' '[:lower:]') # Remove local to make it global
|
|
local system_info=$(collect_system_info)
|
|
|
|
local json_payload=$(cat <<SIMPLE_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"platform": "$platform",
|
|
"architecture": "$arch",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_version": "$haproxy_version",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"status": "online",
|
|
"haproxy_status": "initializing",
|
|
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
}
|
|
SIMPLE_EOF
|
|
)
|
|
|
|
log "DEBUG" "Sending registration JSON: $json_payload"
|
|
|
|
REGISTER_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$json_payload")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "Agent registered successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "Agent registration failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check HAProxy status (Linux specific)
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
elif pgrep -f "haproxy" > /dev/null 2>&1; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get full HAProxy stats CSV from socket (for dashboard metrics)
|
|
get_haproxy_stats_csv() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "STATS: socat not available, cannot fetch stats CSV"
|
|
echo ""
|
|
return
|
|
fi
|
|
log "INFO" "STATS: Initialized socat: $SOCAT_BIN"
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH"
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
# Return CSV data (base64 encoded to avoid JSON escaping issues)
|
|
if [[ -n "$stats_output" ]]; then
|
|
local csv_bytes=$(echo "$stats_output" | wc -c | tr -d ' ')
|
|
local csv_lines=$(echo "$stats_output" | wc -l | tr -d ' ')
|
|
local csv_base64=$(echo "$stats_output" | base64 | tr -d '\n')
|
|
local base64_size=${#csv_base64}
|
|
log "INFO" "STATS: Fetched CSV: $csv_bytes bytes, $csv_lines lines, base64: $base64_size chars"
|
|
echo "$csv_base64"
|
|
else
|
|
log "WARN" "STATS: Failed to fetch stats from socket: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
get_server_statuses() {
|
|
local server_statuses_json="{}"
|
|
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "DEBUG" "STATS: socat not available for server statuses"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "DEBUG" "Stats socket not found: $STATS_SOCKET_PATH"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
|
|
log "DEBUG" "Querying HAProxy stats socket: $STATS_SOCKET_PATH"
|
|
|
|
# Query HAProxy stats via socket with permission fix attempt
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "DEBUG" "Stats socket query failed, attempting permission fix"
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
log "DEBUG" "Applied chmod 666 to stats socket"
|
|
|
|
# Retry after permission fix
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "DEBUG" "Stats socket query failed after permission fix"
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
else
|
|
echo "$server_statuses_json"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
log "DEBUG" "Stats socket query successful, parsing output"
|
|
|
|
# Parse stats output and build JSON
|
|
local backend_servers=()
|
|
while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do
|
|
# Skip header and empty lines
|
|
[[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue
|
|
|
|
# Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0)
|
|
[[ "$type" != "2" ]] && continue
|
|
|
|
# Extract backend and server names
|
|
local backend_name="$pxname"
|
|
local server_name="$svname"
|
|
|
|
# Map HAProxy status to our format
|
|
local server_status
|
|
case "$status" in
|
|
"UP") server_status="UP" ;;
|
|
"DOWN") server_status="DOWN" ;;
|
|
"MAINT") server_status="MAINT" ;;
|
|
"DRAIN") server_status="DRAIN" ;;
|
|
"NOLB") server_status="NOLB" ;;
|
|
*) server_status="UNKNOWN" ;;
|
|
esac
|
|
|
|
backend_servers+=("$backend_name:$server_name:$server_status")
|
|
done <<< "$stats_output"
|
|
|
|
# Build JSON structure
|
|
if [[ ${#backend_servers[@]} -gt 0 ]]; then
|
|
local json_parts=()
|
|
local current_backend=""
|
|
local backend_json=""
|
|
|
|
for entry in "${backend_servers[@]}"; do
|
|
IFS=':' read -r backend_name server_name server_status <<< "$entry"
|
|
|
|
if [[ "$current_backend" != "$backend_name" ]]; then
|
|
# Save previous backend if exists
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
# Start new backend
|
|
current_backend="$backend_name"
|
|
backend_json=""
|
|
fi
|
|
|
|
backend_json+="\"$server_name\": \"$server_status\","
|
|
done
|
|
|
|
# Save last backend
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
|
|
# Combine all backends
|
|
if [[ ${#json_parts[@]} -gt 0 ]]; then
|
|
local IFS=','
|
|
server_statuses_json="{${json_parts[*]}}"
|
|
fi
|
|
fi
|
|
|
|
echo "$server_statuses_json"
|
|
}
|
|
|
|
# Collect system information (Linux specific)
|
|
collect_system_info() {
|
|
# Get system information
|
|
local os_info=$(lsb_release -d 2>/dev/null | cut -f2 || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2 2>/dev/null || echo "Linux")
|
|
local kernel_version=$(uname -r 2>/dev/null || echo "Unknown")
|
|
local uptime_seconds=$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo "0")
|
|
local cpu_count=$(nproc 2>/dev/null || echo "1")
|
|
local memory_bytes=$(awk '/MemTotal/ {print $2*1024}' /proc/meminfo 2>/dev/null || echo "0")
|
|
local disk_bytes=$(df / | awk 'NR==2 {print $2*1024}' 2>/dev/null || echo "0")
|
|
local ip_address=$(ip route get 8.8.8.8 | awk '{print $7; exit}' 2>/dev/null || hostname -I | awk '{print $1}' 2>/dev/null || echo "")
|
|
local network_interfaces=$(ip link show | grep '^[0-9]' | awk -F': ' '{print $2}' | tr '\n' ',' | sed 's/,$//' 2>/dev/null || echo "")
|
|
|
|
# Build system info JSON
|
|
cat <<SYSTEM_INFO_EOF
|
|
"operating_system": "$os_info",
|
|
"kernel_version": "$kernel_version",
|
|
"uptime": $uptime_seconds,
|
|
"cpu_count": $cpu_count,
|
|
"memory_total": $memory_bytes,
|
|
"disk_space": $disk_bytes,
|
|
"ip_address": "$ip_address",
|
|
"network_interfaces": ["${network_interfaces//,/\",\"}"],
|
|
"capabilities": ["haproxy_management", "ssl_deployment", "config_reload", "systemd_service"]
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Send heartbeat
|
|
send_heartbeat() {
|
|
local haproxy_status=$(get_haproxy_status)
|
|
local server_statuses=$(get_server_statuses)
|
|
local haproxy_stats_csv=$(get_haproxy_stats_csv)
|
|
local system_info=$(collect_system_info)
|
|
|
|
# Ensure platform is available (global variable set during registration)
|
|
[[ -z "$platform" ]] && platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
|
|
local heartbeat_payload
|
|
# Include stats CSV if available
|
|
if [[ -n "$haproxy_stats_csv" && "$server_statuses" != "{}" ]]; then
|
|
log "INFO" "HEARTBEAT: Including stats CSV (${#haproxy_stats_csv} chars) with server statuses"
|
|
heartbeat_payload=$(cat <<STATS_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}",
|
|
"server_statuses": $server_statuses,
|
|
"haproxy_stats_csv": "$haproxy_stats_csv"
|
|
}
|
|
STATS_HEARTBEAT_EOF
|
|
)
|
|
elif [[ "$server_statuses" != "{}" ]]; then
|
|
log "WARN" "HEARTBEAT: Stats CSV empty, sending heartbeat WITHOUT stats (only server statuses)"
|
|
heartbeat_payload=$(cat <<SIMPLE_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}",
|
|
"server_statuses": $server_statuses
|
|
}
|
|
SIMPLE_HEARTBEAT_EOF
|
|
)
|
|
else
|
|
log "WARN" "HEARTBEAT: No server statuses and no stats CSV - minimal heartbeat"
|
|
heartbeat_payload=$(cat <<SIMPLE_HEARTBEAT_EOF
|
|
{
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"status": "online",
|
|
"platform": "$platform",
|
|
"architecture": "$(uname -m)",
|
|
"version": "{{AGENT_VERSION}}",
|
|
$system_info,
|
|
"haproxy_status": "$haproxy_status",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"applied_config_version": "${last_applied_version}"
|
|
}
|
|
SIMPLE_HEARTBEAT_EOF
|
|
)
|
|
fi
|
|
|
|
log "DEBUG" "Sending heartbeat"
|
|
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$heartbeat_payload" > /dev/null 2>&1
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
# Only log heartbeat success in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "Heartbeat sent successfully (HAProxy: $haproxy_status)"
|
|
else
|
|
log "ERROR" "Heartbeat failed"
|
|
fi
|
|
}
|
|
|
|
# Deploy SSL certificates to local filesystem (Linux)
|
|
deploy_ssl_certificates() {
|
|
local ssl_certs_json="$1"
|
|
|
|
log "INFO" "SSL DEPLOYMENT: Processing SSL certificates..."
|
|
|
|
# Create SSL directory if it doesn't exist
|
|
local ssl_dir="/etc/ssl/haproxy"
|
|
if [[ ! -d "$ssl_dir" ]]; then
|
|
log "INFO" "Creating SSL directory: $ssl_dir"
|
|
mkdir -p "$ssl_dir"
|
|
chmod 755 "$ssl_dir"
|
|
fi
|
|
|
|
# Parse and deploy each certificate
|
|
local cert_count=$(echo "$ssl_certs_json" | jq length 2>/dev/null || echo "0")
|
|
log "INFO" "Found $cert_count SSL certificates to deploy"
|
|
|
|
for ((i=0; i<cert_count; i++)); do
|
|
local cert_data=$(echo "$ssl_certs_json" | jq -r ".[$i]")
|
|
local cert_name=$(echo "$cert_data" | jq -r '.name // "unknown"')
|
|
local cert_domain=$(echo "$cert_data" | jq -r '.domain // "unknown"')
|
|
local cert_file_path=$(echo "$cert_data" | jq -r '.file_path // "/etc/ssl/haproxy/unknown.pem"')
|
|
local cert_content=$(echo "$cert_data" | jq -r '.certificate_content // ""')
|
|
local key_content=$(echo "$cert_data" | jq -r '.private_key_content // ""')
|
|
local chain_content=$(echo "$cert_data" | jq -r '.chain_content // ""')
|
|
local cert_status=$(echo "$cert_data" | jq -r '.status // "unknown"')
|
|
local usage_type=$(echo "$cert_data" | jq -r '.usage_type // "frontend"')
|
|
|
|
log "INFO" "Deploying SSL certificate: $cert_name ($cert_domain) - Usage: $usage_type - Status: $cert_status"
|
|
|
|
# Validate certificate content
|
|
if [[ -z "$cert_content" || "$cert_content" == "null" ]]; then
|
|
log "ERROR" "Certificate content is empty for $cert_name, skipping"
|
|
continue
|
|
fi
|
|
|
|
# CRITICAL: Private key validation depends on usage_type
|
|
# - Frontend SSL: private key REQUIRED (for HAProxy bind)
|
|
# - Server SSL: private key OPTIONAL (CA certificate only for backend verification)
|
|
if [[ "$usage_type" == "frontend" ]]; then
|
|
if [[ -z "$key_content" || "$key_content" == "null" ]]; then
|
|
log "ERROR" "Private key is required for Frontend SSL: $cert_name, skipping"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Create combined PEM file (HAProxy format: cert + key + chain)
|
|
local temp_cert_file="/tmp/ssl_cert_$$.pem"
|
|
|
|
# Write certificate
|
|
echo "$cert_content" > "$temp_cert_file"
|
|
|
|
# Append private key (only if provided - server SSL may not have it)
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
echo "" >> "$temp_cert_file"
|
|
echo "$key_content" >> "$temp_cert_file"
|
|
fi
|
|
|
|
# Append certificate chain if present
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
echo "" >> "$temp_cert_file"
|
|
echo "$chain_content" >> "$temp_cert_file"
|
|
log "DEBUG" "Added certificate chain for $cert_name"
|
|
fi
|
|
|
|
# Validate the combined certificate file
|
|
if openssl x509 -in "$temp_cert_file" -noout -text >/dev/null 2>&1; then
|
|
# Move to final location
|
|
mv "$temp_cert_file" "$cert_file_path"
|
|
chmod 600 "$cert_file_path"
|
|
chown root:root "$cert_file_path" 2>/dev/null || true
|
|
|
|
log "INFO" "SSL certificate deployed: $cert_file_path"
|
|
|
|
# Log certificate details
|
|
local cert_subject=$(openssl x509 -in "$cert_file_path" -noout -subject 2>/dev/null | sed 's/subject=//')
|
|
local cert_expiry=$(openssl x509 -in "$cert_file_path" -noout -dates 2>/dev/null | grep notAfter | sed 's/notAfter=//')
|
|
log "DEBUG" "Certificate subject: $cert_subject"
|
|
log "DEBUG" "Certificate expires: $cert_expiry"
|
|
else
|
|
log "ERROR" "Invalid certificate format for $cert_name, removing temp file"
|
|
rm -f "$temp_cert_file"
|
|
fi
|
|
done
|
|
|
|
log "INFO" "SSL DEPLOYMENT: Completed"
|
|
}
|
|
|
|
# Fetch and deploy SSL certificates from separate endpoint (Linux)
|
|
fetch_and_deploy_ssl_certificates() {
|
|
# Only log SSL fetch in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "SSL FETCH: Getting SSL certificates..."
|
|
|
|
# Build SSL endpoint URL with incremental update support
|
|
local ssl_endpoint="$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates"
|
|
|
|
# Add timestamp parameter for incremental updates if we have a last SSL sync time
|
|
if [[ -f "$SSL_SYNC_TIMESTAMP_FILE" ]]; then
|
|
local last_ssl_sync=$(cat "$SSL_SYNC_TIMESTAMP_FILE" 2>/dev/null || echo "")
|
|
if [[ -n "$last_ssl_sync" ]]; then
|
|
ssl_endpoint="${ssl_endpoint}?since=${last_ssl_sync}"
|
|
log "DEBUG" "SSL INCREMENTAL: Requesting SSL certificates since $last_ssl_sync"
|
|
fi
|
|
fi
|
|
|
|
# Fetch SSL certificates from dedicated endpoint
|
|
local ssl_response=$("$CURL_BIN" -k -s -X GET "$ssl_endpoint" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "WARN" "Failed to fetch SSL certificates, skipping SSL deployment"
|
|
return 0
|
|
fi
|
|
|
|
local ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"')
|
|
local ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"')
|
|
|
|
if [[ "$ssl_status" != "available" ]]; then
|
|
log "DEBUG" "SSL certificates not available (status: $ssl_status)"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then
|
|
log "INFO" "Deploying SSL certificates..."
|
|
deploy_ssl_certificates "$ssl_certificates"
|
|
|
|
# Save current timestamp for next incremental update
|
|
date -u +"%Y-%m-%dT%H:%M:%SZ" > "$SSL_SYNC_TIMESTAMP_FILE"
|
|
log "DEBUG" "SSL INCREMENTAL: Saved SSL sync timestamp for next incremental update"
|
|
else
|
|
log "DEBUG" "No SSL certificates to deploy"
|
|
fi
|
|
}
|
|
|
|
# Reload HAProxy service (Linux specific)
|
|
reload_haproxy_service() {
|
|
log "INFO" "Performing zero-downtime HAProxy configuration reload..."
|
|
|
|
# IMPROVED STRATEGY: Use HAProxy's seamless reload with verification
|
|
# This ensures new config is applied without traffic interruption
|
|
|
|
# Step 1: Validate new config before attempting reload
|
|
if ! "$HAPROXY_BIN_PATH" -c -f "$HAPROXY_CONFIG_PATH" >/dev/null 2>&1; then
|
|
log "ERROR" "New configuration validation failed - aborting reload"
|
|
return 1
|
|
fi
|
|
|
|
# Step 2: Try systemd reload first (most reliable on Linux)
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
log "INFO" "Using systemctl reload for zero-downtime update"
|
|
if systemctl reload haproxy 2>/dev/null; then
|
|
log "INFO" "HAProxy reloaded successfully via systemctl"
|
|
|
|
# Verify reload was successful
|
|
sleep 2
|
|
if systemctl is-active --quiet haproxy; then
|
|
log "INFO" "HAProxy service is active after systemctl reload"
|
|
return 0
|
|
else
|
|
log "WARN" "HAProxy service not active after systemctl reload, trying manual method"
|
|
fi
|
|
else
|
|
log "WARN" "systemctl reload failed, trying manual seamless reload"
|
|
fi
|
|
fi
|
|
|
|
# Step 3: Manual seamless reload using HAProxy's -sf option
|
|
local haproxy_master_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1)
|
|
|
|
if [[ -n "$haproxy_master_pid" ]]; then
|
|
log "INFO" "Found HAProxy master process (PID: $haproxy_master_pid)"
|
|
|
|
# Use HAProxy's seamless reload with new process
|
|
log "INFO" "Starting seamless reload with new configuration..."
|
|
|
|
if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D -sf "$haproxy_master_pid"; then
|
|
log "INFO" "Seamless reload initiated successfully"
|
|
|
|
# Verify new process is running and old one is gone
|
|
sleep 3
|
|
local new_pid=$(pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH.*-D" | head -1)
|
|
|
|
if [[ -n "$new_pid" && "$new_pid" != "$haproxy_master_pid" ]]; then
|
|
log "INFO" "HAProxy seamlessly reloaded (New PID: $new_pid, Old PID: $haproxy_master_pid)"
|
|
return 0
|
|
else
|
|
log "WARN" "Seamless reload may have failed, verifying process status..."
|
|
|
|
# Additional verification
|
|
if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "HAProxy is still running after reload attempt"
|
|
return 0
|
|
else
|
|
log "ERROR" "HAProxy process not found after reload"
|
|
fi
|
|
fi
|
|
else
|
|
log "ERROR" "Seamless reload command failed"
|
|
fi
|
|
else
|
|
log "WARN" "No HAProxy master process found, starting fresh instance"
|
|
fi
|
|
|
|
# Step 4: Fallback - start new HAProxy instance (only if no process running)
|
|
if ! pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "Starting new HAProxy instance..."
|
|
|
|
if "$HAPROXY_BIN_PATH" -f "$HAPROXY_CONFIG_PATH" -D; then
|
|
sleep 2
|
|
if pgrep -f "haproxy.*$HAPROXY_CONFIG_PATH" > /dev/null; then
|
|
log "INFO" "HAProxy started successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "HAProxy failed to start"
|
|
return 1
|
|
fi
|
|
else
|
|
log "ERROR" "Failed to start HAProxy: $HAPROXY_BIN_PATH"
|
|
return 1
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "HAProxy configuration reload completed"
|
|
return 0
|
|
}
|
|
|
|
# Check for agent upgrade and perform if requested
|
|
check_agent_upgrade() {
|
|
log "DEBUG" "Checking for agent upgrade requests..."
|
|
|
|
# Check if upgrade was recently completed to prevent loops (agent-specific marker)
|
|
local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME"
|
|
if [[ -f "$agent_marker" ]]; then
|
|
local upgrade_time=$(head -n1 "$agent_marker" | cut -d' ' -f1-2)
|
|
local current_time=$(date '+%Y-%m-%d %H:%M:%S')
|
|
|
|
# Convert times to epoch for comparison
|
|
local upgrade_epoch=$(date -d "$upgrade_time" '+%s' 2>/dev/null || echo "0")
|
|
local current_epoch=$(date '+%s')
|
|
local time_diff=$((current_epoch - upgrade_epoch))
|
|
|
|
# Remove marker if older than 5 minutes (300 seconds)
|
|
if [[ $time_diff -gt 300 ]]; then
|
|
log "DEBUG" "Upgrade marker expired (${time_diff}s old), removing and allowing new upgrades"
|
|
rm -f "$agent_marker" 2>/dev/null || true
|
|
else
|
|
log "DEBUG" "Recent upgrade completed at: $upgrade_time - skipping upgrade check (${time_diff}s ago)"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Check if agent is marked for upgrade by querying agent-specific status
|
|
local agent_status_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "DEBUG" "Failed to fetch agent upgrade status"
|
|
return 0
|
|
fi
|
|
|
|
local should_upgrade=$(echo "$agent_status_response" | jq -r '.should_upgrade // false')
|
|
local target_version=$(echo "$agent_status_response" | jq -r '.target_version // ""')
|
|
local current_version="{{AGENT_VERSION}}" # This agent's version
|
|
|
|
# Only upgrade if explicitly requested via UI and target version is different
|
|
if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "$current_version" ]]; then
|
|
log "INFO" "AGENT UPGRADE: Upgrade requested to version $target_version (current: $current_version)"
|
|
perform_agent_upgrade "$target_version"
|
|
fi
|
|
}
|
|
|
|
# Perform agent upgrade
|
|
perform_agent_upgrade() {
|
|
local new_version="$1"
|
|
log "INFO" "AGENT UPGRADE: Starting upgrade to version $new_version"
|
|
|
|
# Download new agent script
|
|
local temp_script="/tmp/haproxy-agent-new"
|
|
local upgrade_url="$MANAGEMENT_URL/api/agents/generate-install-script"
|
|
|
|
# Get pool_id from agent's current cluster configuration
|
|
local POOL_ID=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null)
|
|
if [[ -z "$POOL_ID" || "$POOL_ID" == "null" ]]; then
|
|
# Fallback: get pool_id from cluster_id via API
|
|
local cluster_info=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
POOL_ID=$(echo "$cluster_info" | jq -r '.pool_id // 1' 2>/dev/null)
|
|
[[ -z "$POOL_ID" || "$POOL_ID" == "null" ]] && POOL_ID=1
|
|
fi
|
|
|
|
# Create upgrade request payload with correct pool_id
|
|
local upgrade_payload=$(cat <<UPGRADE_EOF
|
|
{
|
|
"platform": "$(uname -s | tr '[:upper:]' '[:lower:]')",
|
|
"architecture": "$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')",
|
|
"pool_id": $POOL_ID,
|
|
"cluster_id": $CLUSTER_ID,
|
|
"agent_name": "$AGENT_NAME",
|
|
"hostname_prefix": "$(echo $HOSTNAME | cut -d'-' -f1)",
|
|
"haproxy_bin_path": "$HAPROXY_BIN_PATH",
|
|
"haproxy_config_path": "$HAPROXY_CONFIG_PATH",
|
|
"stats_socket_path": "$STATS_SOCKET_PATH"
|
|
}
|
|
UPGRADE_EOF
|
|
)
|
|
|
|
# Download new script
|
|
log "INFO" "AGENT UPGRADE: Downloading new agent script"
|
|
local script_response=$("$CURL_BIN" -k -s -X POST "$upgrade_url" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$upgrade_payload")
|
|
|
|
if [[ $? -eq 0 ]]; then
|
|
local new_script=$(echo "$script_response" | jq -r '.script // ""')
|
|
|
|
if [[ -n "$new_script" && "$new_script" != "null" ]]; then
|
|
echo "$new_script" > "$temp_script"
|
|
chmod +x "$temp_script"
|
|
|
|
log "INFO" "AGENT UPGRADE: New script downloaded, performing in-place upgrade"
|
|
|
|
# Get current script path
|
|
local current_script="/usr/local/bin/haproxy-agent"
|
|
|
|
# Create backup of current script in dedicated backup directory
|
|
local backup_dir="/usr/local/share/haproxy-agent-backups"
|
|
local backup_file="$backup_dir/haproxy-agent-$(date '+%Y%m%d-%H%M%S').backup"
|
|
mkdir -p "$backup_dir" 2>/dev/null || true
|
|
cp "$current_script" "$backup_file" 2>/dev/null || true
|
|
if [[ -f "$backup_file" ]]; then
|
|
log "INFO" "AGENT UPGRADE: Backup created at $backup_file"
|
|
else
|
|
log "WARN" "AGENT UPGRADE: Could not create backup, proceeding anyway"
|
|
fi
|
|
|
|
# Replace current script with new version (in-place update)
|
|
cp "$temp_script" "$current_script"
|
|
chmod +x "$current_script"
|
|
|
|
log "INFO" "AGENT UPGRADE: Script updated in-place, preparing clean restart"
|
|
|
|
# Mark upgrade as complete
|
|
log "INFO" "AGENT UPGRADE: Notifying backend of upgrade completion..."
|
|
local completion_response=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"completed\"}" 2>&1)
|
|
local completion_status=$?
|
|
|
|
if [[ $completion_status -eq 0 ]]; then
|
|
log "INFO" "AGENT UPGRADE: Backend notified successfully: $completion_response"
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to notify backend: $completion_response"
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Agent upgrade completed successfully"
|
|
log "INFO" "DAEMON: Exiting current process - SystemD will auto-restart with new version"
|
|
|
|
# Clean up temporary files
|
|
rm -f "$temp_script"
|
|
|
|
# Create agent-specific marker file to prevent upgrade loop on restart
|
|
local agent_marker="/tmp/haproxy-agent-upgrade-complete-$AGENT_NAME"
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') - Upgrade to $new_version completed" > "$agent_marker" 2>/dev/null || true
|
|
chmod 666 "$agent_marker" 2>/dev/null || true
|
|
|
|
# Exit cleanly - SystemD will automatically restart with new script
|
|
log "INFO" "AGENT UPGRADE: Clean exit - new process will start with version $new_version"
|
|
exit 0
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to download new script"
|
|
# Mark upgrade as failed
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1
|
|
fi
|
|
else
|
|
log "ERROR" "AGENT UPGRADE: Failed to fetch new script from server"
|
|
# Mark upgrade as failed
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$new_version\",\"status\":\"failed\"}" >/dev/null 2>&1
|
|
fi
|
|
}
|
|
|
|
# Check and apply configuration updates (Linux)
|
|
check_config_updates() {
|
|
# Only log config check in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "Checking for configuration updates..."
|
|
|
|
# Use agent-specific config endpoint
|
|
CONFIG_RESPONSE=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \
|
|
-H "X-API-Key: $AGENT_TOKEN")
|
|
|
|
if [[ $? -ne 0 ]]; then
|
|
log "ERROR" "Failed to fetch configuration"
|
|
return 1
|
|
fi
|
|
|
|
log "DEBUG" "Config response received"
|
|
|
|
# Parse JSON response using jq
|
|
if ! command -v jq >/dev/null 2>&1; then
|
|
log "ERROR" "jq is required for config parsing but not found"
|
|
return 1
|
|
fi
|
|
|
|
# Extract config details from response
|
|
CONFIG_STATUS=$(echo "$CONFIG_RESPONSE" | jq -r '.status // "unknown"')
|
|
CONFIG_VERSION=$(echo "$CONFIG_RESPONSE" | jq -r '.version // "unknown"')
|
|
CONFIG_CONTENT=$(echo "$CONFIG_RESPONSE" | jq -r '.config_content // ""')
|
|
CONFIG_CHECKSUM=$(echo "$CONFIG_RESPONSE" | jq -r '.checksum // ""')
|
|
|
|
log "DEBUG" "Config status: $CONFIG_STATUS, version: $CONFIG_VERSION"
|
|
|
|
# Check if config is available
|
|
if [[ "$CONFIG_STATUS" != "available" ]]; then
|
|
log "WARN" "No configuration available (status: $CONFIG_STATUS)"
|
|
return 0
|
|
fi
|
|
|
|
# Check if config content is empty or invalid
|
|
if [[ -z "$CONFIG_CONTENT" || "$CONFIG_CONTENT" == "null" ]]; then
|
|
log "WARN" "Configuration content is empty"
|
|
return 0
|
|
fi
|
|
|
|
# CRITICAL: Prevent overwriting valid config with invalid content
|
|
if [[ "$CONFIG_CONTENT" == "# No configuration available"* ]] || [[ "$CONFIG_CONTENT" == "# Agent is disabled"* ]] || [[ "$CONFIG_CONTENT" == "# No cluster assigned"* ]]; then
|
|
log "WARN" "Received invalid configuration content, preserving existing HAProxy config"
|
|
log "DEBUG" "Invalid config content: ${CONFIG_CONTENT:0:100}..."
|
|
return 0
|
|
fi
|
|
|
|
# Additional validation: Check if config content looks like a valid HAProxy config
|
|
# Note: Partial configs may not have global/defaults, and may also have NO frontend/backend/listen (all deleted)
|
|
if ! echo "$CONFIG_CONTENT" | grep -q -E "(frontend|backend|listen)"; then
|
|
log "INFO" "Partial config has no frontend/backend/listen sections (all entities may have been deleted - this is OK)"
|
|
log "DEBUG" "Config content preview: ${CONFIG_CONTENT:0:200}..."
|
|
# Don't return - continue with merge, will result in global+defaults only config
|
|
fi
|
|
|
|
# Check if this version is already applied (CRITICAL: Prevent reapplying same config)
|
|
if [[ "$CONFIG_VERSION" == "$last_applied_version" ]]; then
|
|
log "DEBUG" "Configuration version $CONFIG_VERSION already applied, skipping"
|
|
# OPTIMIZATION: SSL certificates are fetched separately and independently
|
|
# Only fetch SSL if config hasn't changed to reduce unnecessary API calls
|
|
return 0
|
|
fi
|
|
|
|
# OPTIMIZATION: Fetch and deploy SSL certificates ONLY when config changes
|
|
# This prevents unnecessary SSL API calls every 30 seconds
|
|
log "INFO" "Config version changed ($last_applied_version → $CONFIG_VERSION), fetching SSL certificates..."
|
|
fetch_and_deploy_ssl_certificates
|
|
|
|
# Create backup of current config (CRITICAL: Always backup before changes)
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
local backup_file="$HAPROXY_CONFIG_PATH.backup.$(date +%Y%m%d_%H%M%S)"
|
|
cp "$HAPROXY_CONFIG_PATH" "$backup_file"
|
|
if [[ -f "$backup_file" ]]; then
|
|
log "INFO" "Current config backed up to: $backup_file"
|
|
else
|
|
log "ERROR" "Failed to create config backup, aborting config update"
|
|
return 1
|
|
fi
|
|
else
|
|
log "WARN" "No existing HAProxy config found at: $HAPROXY_CONFIG_PATH"
|
|
fi
|
|
|
|
# Write new configuration to temp file first
|
|
TEMP_CONFIG="/tmp/haproxy_new_$$.cfg"
|
|
echo "$CONFIG_CONTENT" > "$TEMP_CONFIG"
|
|
|
|
# CRITICAL: Check if this is a PARTIAL config (only frontends/backends without global/defaults)
|
|
# Backend generates partial configs to preserve agent-specific global/defaults settings
|
|
if head -20 "$TEMP_CONFIG" | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then
|
|
log "INFO" "CONFIG UPDATE: Detected PARTIAL config - preserving global/defaults from existing haproxy.cfg"
|
|
|
|
# Extract global and defaults sections from current haproxy.cfg
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Extract global section (everything from start until 'defaults' keyword)
|
|
awk '/^defaults[[:space:]]*$/{exit} {print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-global-$$.cfg 2>/dev/null
|
|
|
|
# Extract defaults section (from 'defaults' until first 'frontend/backend/listen')
|
|
awk '/^defaults[[:space:]]*$/{flag=1} /^(frontend|backend|listen)[[:space:]]/{if(flag && !/^defaults/) exit} flag{print}' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-defaults-$$.cfg 2>/dev/null
|
|
|
|
# Extract all listen sections (preserve existing listen blocks like stats monitoring)
|
|
# This extracts ALL listen blocks from existing config to preserve them
|
|
awk '
|
|
/^listen[[:space:]]/ {
|
|
if (listen_content != "") {
|
|
# Print previous listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
in_listen=1
|
|
listen_content=$0 "\n"
|
|
next
|
|
}
|
|
in_listen {
|
|
# Check if we hit another section (frontend, backend, listen, global, defaults)
|
|
if (/^(frontend|backend|listen|global|defaults)[[:space:]]/) {
|
|
# Print current listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
listen_content=""
|
|
in_listen=0
|
|
} else {
|
|
listen_content = listen_content $0 "\n"
|
|
}
|
|
}
|
|
END {
|
|
# Print final listen block if any
|
|
if (listen_content != "") {
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
}
|
|
' "$HAPROXY_CONFIG_PATH" > /tmp/haproxy-listen-$$.cfg 2>/dev/null
|
|
|
|
# Check if extraction was successful
|
|
if [[ -s /tmp/haproxy-global-$$.cfg && -s /tmp/haproxy-defaults-$$.cfg ]]; then
|
|
listen_lines=0
|
|
[[ -s /tmp/haproxy-listen-$$.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen-$$.cfg)
|
|
log "INFO" "CONFIG UPDATE: Extracted global ($(wc -l < /tmp/haproxy-global-$$.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults-$$.cfg) lines), listen blocks ($listen_lines lines)"
|
|
|
|
# Merge: existing global + existing defaults + existing listen blocks + new frontends/backends
|
|
merged_config="/tmp/haproxy-merged-$$.cfg"
|
|
|
|
# Copy global section as-is, normalize trailing blanks to exactly 1 line
|
|
# Remove excessive trailing blanks (keep max 1), preserve content
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global-$$.cfg > "$merged_config"
|
|
|
|
# Copy defaults section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults-$$.cfg >> "$merged_config"
|
|
|
|
# Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring)
|
|
if [[ -s /tmp/haproxy-listen-$$.cfg ]]; then
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen-$$.cfg >> "$merged_config"
|
|
log "INFO" "CONFIG UPDATE: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen-$$.cfg) listen blocks from existing config"
|
|
fi
|
|
|
|
# Extract only frontend/backend sections from partial config (skip comments)
|
|
awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' "$TEMP_CONFIG" >> "$merged_config"
|
|
|
|
# Replace temp config with merged version
|
|
mv "$merged_config" "$TEMP_CONFIG"
|
|
log "INFO" "CONFIG UPDATE: Successfully merged - preserved local global/defaults/listen blocks, applied new frontends/backends"
|
|
|
|
# Cleanup extraction temp files
|
|
rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg
|
|
else
|
|
log "ERROR" "CONFIG UPDATE: Failed to extract global/defaults sections from existing config"
|
|
log "ERROR" "CONFIG UPDATE: This will cause HAProxy to fail - aborting config update"
|
|
rm -f /tmp/haproxy-global-$$.cfg /tmp/haproxy-defaults-$$.cfg /tmp/haproxy-listen-$$.cfg "$TEMP_CONFIG"
|
|
return 1
|
|
fi
|
|
else
|
|
log "ERROR" "CONFIG UPDATE: No existing haproxy.cfg found - cannot merge partial config"
|
|
log "ERROR" "CONFIG UPDATE: Partial config requires existing global/defaults - aborting"
|
|
rm -f "$TEMP_CONFIG"
|
|
return 1
|
|
fi
|
|
else
|
|
log "INFO" "CONFIG UPDATE: Full config detected - using as-is (includes global/defaults)"
|
|
fi
|
|
|
|
# Log config content for debugging
|
|
log "DEBUG" "Generated config content (first 20 lines):"
|
|
log "DEBUG" "$(head -20 "$TEMP_CONFIG")"
|
|
log "DEBUG" "Config file size: $(wc -l < "$TEMP_CONFIG") lines"
|
|
|
|
# Validate new configuration with retry logic
|
|
log "DEBUG" "Validating config with: $HAPROXY_BIN_PATH -c -f $TEMP_CONFIG"
|
|
VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1)
|
|
VALIDATION_RESULT=$?
|
|
|
|
if [[ $VALIDATION_RESULT -eq 0 ]]; then
|
|
log "INFO" "New configuration is valid"
|
|
|
|
# Apply new configuration
|
|
mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH"
|
|
log "INFO" "Configuration updated to version: $CONFIG_VERSION"
|
|
|
|
# Reload HAProxy service
|
|
reload_haproxy_service
|
|
|
|
# Update last applied version to prevent reapplying same config
|
|
last_applied_version="$CONFIG_VERSION"
|
|
|
|
# Send instant notification to backend for real-time UI sync
|
|
log "INFO" "Sending instant notification for config version: $CONFIG_VERSION"
|
|
NOTIFICATION_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" 2>&1)
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "INSTANT SYNC: Notification sent successfully"
|
|
else
|
|
log "ERROR" "INSTANT SYNC: Notification failed - $NOTIFICATION_RESPONSE"
|
|
fi
|
|
|
|
# Send config sync to update database entities based on actual config
|
|
log "INFO" "Syncing database with current config content"
|
|
if [[ -f "$HAPROXY_CONFIG_PATH" ]]; then
|
|
# Properly escape JSON content
|
|
CONFIG_JSON=$(cat "$HAPROXY_CONFIG_PATH" | jq -Rs .)
|
|
SYNC_RESPONSE=$("$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-sync" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"config_content\":$CONFIG_JSON}" 2>&1)
|
|
if [[ $? -eq 0 ]]; then
|
|
log "INFO" "CONFIG SYNC: Database synced with current config"
|
|
else
|
|
log "ERROR" "CONFIG SYNC: Database sync failed - $SYNC_RESPONSE"
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "Configuration applied and HAProxy reloaded successfully"
|
|
return 0
|
|
else
|
|
log "ERROR" "New configuration validation failed (attempt 1/3)"
|
|
log "ERROR" "Validation error output: $VALIDATION_OUTPUT"
|
|
|
|
# CRITICAL: Retry validation with delay (race condition fix)
|
|
for retry in 2 3; do
|
|
log "INFO" "Retrying validation in 3 seconds... (attempt $retry/3)"
|
|
sleep 3
|
|
|
|
RETRY_VALIDATION_OUTPUT=$("$HAPROXY_BIN_PATH" -c -f "$TEMP_CONFIG" 2>&1)
|
|
RETRY_VALIDATION_RESULT=$?
|
|
|
|
if [[ $RETRY_VALIDATION_RESULT -eq 0 ]]; then
|
|
log "INFO" "Configuration validation successful on retry $retry"
|
|
|
|
# Apply new configuration
|
|
mv "$TEMP_CONFIG" "$HAPROXY_CONFIG_PATH"
|
|
log "INFO" "Configuration updated to version: $CONFIG_VERSION"
|
|
|
|
# Reload HAProxy service
|
|
reload_haproxy_service
|
|
|
|
# Update last applied version to prevent reapplying same config
|
|
last_applied_version="$CONFIG_VERSION"
|
|
|
|
# Send instant notification to backend for real-time UI sync
|
|
"$CURL_BIN" -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"version\":\"$CONFIG_VERSION\",\"status\":\"applied\"}" > /dev/null 2>&1
|
|
|
|
log "INFO" "Configuration applied and HAProxy reloaded successfully"
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
log "ERROR" "Configuration validation failed after 3 attempts"
|
|
log "ERROR" "Final validation error: $RETRY_VALIDATION_OUTPUT"
|
|
rm -f "$TEMP_CONFIG"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Main daemon
|
|
run_daemon() {
|
|
# Check for existing running instance (CRITICAL: Prevent multiple agents)
|
|
if [[ -f "$PID_FILE" ]]; then
|
|
local existing_pid=$(cat "$PID_FILE" 2>/dev/null)
|
|
if [[ -n "$existing_pid" ]] && kill -0 "$existing_pid" 2>/dev/null; then
|
|
log "ERROR" "Agent already running with PID: $existing_pid"
|
|
log "ERROR" "Terminating existing instance to prevent conflicts..."
|
|
kill -TERM "$existing_pid" 2>/dev/null || true
|
|
sleep 3
|
|
kill -KILL "$existing_pid" 2>/dev/null || true
|
|
rm -f "$PID_FILE"
|
|
fi
|
|
fi
|
|
|
|
log "INFO" "Starting HAProxy Agent: $AGENT_NAME (Linux)"
|
|
echo $$ > "$PID_FILE"
|
|
|
|
# CRITICAL: Setup graceful shutdown signal handling
|
|
# This allows agent to stop quickly (<2s) instead of waiting for systemd timeout (90s)
|
|
SHUTDOWN_REQUESTED=false
|
|
|
|
trap_handler() {
|
|
log "INFO" "Shutdown signal received (SIGTERM/SIGINT), stopping agent gracefully..."
|
|
SHUTDOWN_REQUESTED=true
|
|
}
|
|
|
|
# Register signal handlers for graceful shutdown
|
|
trap trap_handler SIGTERM SIGINT SIGQUIT
|
|
log "INFO" "Signal handlers registered for graceful shutdown"
|
|
|
|
# Dynamically locate required binaries for daemon operations
|
|
SOCAT_BIN=$(find_binary socat)
|
|
JQ_BIN=$(find_binary jq)
|
|
|
|
if [[ -n "$SOCAT_BIN" ]]; then
|
|
log "INFO" "Found socat: $SOCAT_BIN"
|
|
else
|
|
log "ERROR" "socat not found in system"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -n "$JQ_BIN" ]]; then
|
|
log "INFO" "Found jq: $JQ_BIN"
|
|
fi
|
|
|
|
log "INFO" "Using curl: $CURL_BIN"
|
|
|
|
# CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart)
|
|
# Agent restart should NOT trigger config reapplication - fetch last applied version from backend
|
|
log "INFO" "Fetching last applied config version from backend..."
|
|
agent_info_response=$("$CURL_BIN" -k -s -X GET "$MANAGEMENT_URL/api/agents" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ $? -eq 0 && -n "$agent_info_response" ]]; then
|
|
# Extract applied_config_version for this agent
|
|
last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null)
|
|
|
|
if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then
|
|
last_applied_version="none"
|
|
log "WARN" "Could not fetch applied_config_version from backend, starting with 'none'"
|
|
else
|
|
log "INFO" "Loaded last applied version from database: $last_applied_version"
|
|
fi
|
|
else
|
|
last_applied_version="none"
|
|
log "WARN" "Failed to fetch agent info from backend, starting with version 'none'"
|
|
fi
|
|
|
|
# CRITICAL: Startup delay to avoid race conditions
|
|
log "INFO" "Startup delay for system stabilization..."
|
|
sleep 5
|
|
|
|
# Register agent on startup
|
|
if ! register_agent; then
|
|
log "ERROR" "Failed to register agent, continuing with heartbeats..."
|
|
fi
|
|
|
|
# Send initial heartbeat
|
|
sleep 1
|
|
send_heartbeat
|
|
|
|
# Main agent loop with interruptible sleep for fast shutdown
|
|
# Sleep is broken into 1-second intervals to check shutdown flag every second
|
|
# This allows agent to stop in 1-2 seconds instead of waiting up to 30 seconds
|
|
while [[ "$SHUTDOWN_REQUESTED" == "false" ]]; do
|
|
# Interruptible sleep: break 30s into 30x 1s intervals
|
|
for i in {1..30}; do
|
|
# Check shutdown flag every second
|
|
[[ "$SHUTDOWN_REQUESTED" == "true" ]] && break
|
|
sleep 1
|
|
done
|
|
|
|
# Exit loop if shutdown requested
|
|
[[ "$SHUTDOWN_REQUESTED" == "true" ]] && break
|
|
|
|
# Normal agent operations
|
|
send_heartbeat
|
|
|
|
# Check for pending config requests (if function exists)
|
|
if type check_config_requests &>/dev/null; then
|
|
check_config_requests
|
|
fi
|
|
|
|
check_config_updates
|
|
check_agent_upgrade
|
|
done
|
|
|
|
# Graceful shutdown cleanup
|
|
log "INFO" "Agent stopped gracefully"
|
|
rm -f "$PID_FILE"
|
|
exit 0
|
|
}
|
|
|
|
case "${1:-daemon}" in
|
|
"daemon") run_daemon ;;
|
|
"stop")
|
|
# Improved stop command with graceful → force kill escalation
|
|
if [[ -f "$PID_FILE" ]]; then
|
|
local pid=$(cat "$PID_FILE" 2>/dev/null)
|
|
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
|
echo "Stopping HAProxy Agent (PID: $pid)..."
|
|
|
|
# Try graceful SIGTERM first
|
|
kill -TERM "$pid" 2>/dev/null
|
|
|
|
# Wait up to 5 seconds for graceful shutdown
|
|
local count=0
|
|
while kill -0 "$pid" 2>/dev/null && [[ $count -lt 5 ]]; do
|
|
sleep 1
|
|
count=$((count + 1))
|
|
done
|
|
|
|
# Force kill if still running after 5 seconds
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
echo "Graceful stop timeout, force stopping agent..."
|
|
kill -KILL "$pid" 2>/dev/null
|
|
sleep 1
|
|
fi
|
|
|
|
echo "Agent stopped successfully"
|
|
else
|
|
echo "Agent is not running (stale PID file)"
|
|
fi
|
|
else
|
|
echo "Agent is not running (no PID file)"
|
|
fi
|
|
rm -f "$PID_FILE"
|
|
;;
|
|
*) echo "Usage: $0 {daemon|stop}"; exit 1 ;;
|
|
esac
|
|
AGENT_SCRIPT
|
|
|
|
chmod +x "$INSTALL_DIR/haproxy-agent"
|
|
|
|
# Set comprehensive permissions for self-upgrade capability
|
|
log "INFO" "Configuring agent self-upgrade permissions..."
|
|
|
|
# Agent binary permissions - must be writable by root for in-place upgrades
|
|
chmod 755 "$INSTALL_DIR/haproxy-agent"
|
|
chown root:root "$INSTALL_DIR/haproxy-agent" 2>/dev/null || true
|
|
|
|
# Create backup directory for agent upgrades
|
|
AGENT_BACKUP_DIR="/usr/local/share/haproxy-agent-backups"
|
|
mkdir -p "$AGENT_BACKUP_DIR"
|
|
chmod 755 "$AGENT_BACKUP_DIR"
|
|
chown root:root "$AGENT_BACKUP_DIR" 2>/dev/null || true
|
|
|
|
# Ensure all required directories have proper ownership and permissions
|
|
# This eliminates the need for sudo during upgrades
|
|
log "INFO" "Final permission validation..."
|
|
|
|
# Validate all critical paths are accessible
|
|
for dir_path in "$INSTALL_DIR" "$CONFIG_DIR" "$LOG_DIR" "$AGENT_BACKUP_DIR"; do
|
|
if [[ -d "$dir_path" ]] && [[ -w "$dir_path" ]]; then
|
|
log "INFO" "Directory writable: $dir_path"
|
|
else
|
|
log "WARN" "Directory access issue: $dir_path"
|
|
fi
|
|
done
|
|
|
|
# Validate temp directory access
|
|
if touch "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null && rm -f "/tmp/haproxy-agent-final-test-$AGENT_NAME" 2>/dev/null; then
|
|
log "INFO" "Temp directory access confirmed"
|
|
else
|
|
log "WARN" "Temp directory access issue"
|
|
fi
|
|
|
|
log "INFO" "All agent self-upgrade permissions configured - no sudo required for future operations"
|
|
|
|
# Get pool_id for this cluster
|
|
CLUSTER_POOL_ID=$(curl -k -s -X GET "$MANAGEMENT_URL/api/clusters/$CLUSTER_ID" \
|
|
-H "X-API-Key: $AGENT_TOKEN" | jq -r '.pool_id // 1' 2>/dev/null)
|
|
[[ -z "$CLUSTER_POOL_ID" || "$CLUSTER_POOL_ID" == "null" ]] && CLUSTER_POOL_ID=1
|
|
|
|
# Create configuration
|
|
echo "⚙️ Creating configuration..."
|
|
cat > "$CONFIG_DIR/config.json" << CONFIG_EOF
|
|
{
|
|
"management": {
|
|
"url": "$MANAGEMENT_URL",
|
|
"token": "$AGENT_TOKEN",
|
|
"cluster_id": $CLUSTER_ID,
|
|
"pool_id": $CLUSTER_POOL_ID
|
|
},
|
|
"agent": {
|
|
"name": "$AGENT_NAME",
|
|
"hostname": "$HOSTNAME",
|
|
"platform": "$platform",
|
|
"architecture": "$ARCH"
|
|
},
|
|
"haproxy": {
|
|
"config_path": "$HAPROXY_CONFIG_PATH",
|
|
"bin_path": "$HAPROXY_BIN_PATH",
|
|
"stats_socket_path": "$STATS_SOCKET_PATH",
|
|
"service_name": "$HAPROXY_SERVICE_NAME"
|
|
}
|
|
}
|
|
CONFIG_EOF
|
|
|
|
# Set config file permissions for upgrade access
|
|
chmod 644 "$CONFIG_DIR/config.json"
|
|
chown root:root "$CONFIG_DIR/config.json" 2>/dev/null || true
|
|
log "INFO" "Configuration file permissions set"
|
|
|
|
# Create systemd service for Linux
|
|
log "DEBUG" "Creating systemd service for Linux..."
|
|
cat > "/etc/systemd/system/haproxy-agent.service" << SERVICE_EOF
|
|
[Unit]
|
|
Description=HAProxy Management Agent
|
|
After=network.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=root
|
|
ExecStart=$INSTALL_DIR/haproxy-agent daemon
|
|
ExecStop=$INSTALL_DIR/haproxy-agent stop
|
|
|
|
# Graceful shutdown configuration
|
|
# Timeout after 10 seconds instead of default 90 seconds
|
|
TimeoutStopSec=10
|
|
# Mixed mode: send SIGTERM to main process, SIGKILL to remaining
|
|
KillMode=mixed
|
|
# Explicit signal for graceful shutdown
|
|
KillSignal=SIGTERM
|
|
|
|
Restart=always
|
|
RestartSec=10
|
|
StandardOutput=append:$LOG_DIR/agent.log
|
|
StandardError=append:$LOG_DIR/agent.log
|
|
Environment=SKIP_TO_DAEMON=true
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
SERVICE_EOF
|
|
|
|
echo "Starting systemd service..."
|
|
systemctl daemon-reload
|
|
systemctl enable haproxy-agent
|
|
systemctl start haproxy-agent
|
|
|
|
sleep 3
|
|
SERVICE_STATUS=$(systemctl is-active haproxy-agent 2>/dev/null || echo "inactive")
|
|
|
|
if [[ "$SERVICE_STATUS" == "active" ]]; then
|
|
echo ""
|
|
log "INFO" "HAProxy Management Agent installed successfully on Linux!"
|
|
echo ""
|
|
echo "Service Information:"
|
|
echo " Status: active"
|
|
echo " Architecture: $ARCH ($ARCH_SUFFIX)"
|
|
echo " Cluster ID: $CLUSTER_ID"
|
|
echo " Agent Name: $AGENT_NAME"
|
|
echo " Config: $CONFIG_DIR/config.json"
|
|
echo " Logs: tail -f $LOG_DIR/agent.log"
|
|
echo ""
|
|
log "DEBUG" "Linux Management Commands:"
|
|
echo " Start: sudo systemctl start haproxy-agent"
|
|
echo " Stop: sudo systemctl stop haproxy-agent"
|
|
echo " Status: sudo systemctl status haproxy-agent"
|
|
echo " Logs: sudo journalctl -u haproxy-agent -f"
|
|
echo ""
|
|
echo "Agent is now running and will:"
|
|
echo " • Send heartbeats every 10 seconds to: $MANAGEMENT_URL"
|
|
echo " • Monitor HAProxy service status and report to management UI"
|
|
echo " • Wait for configuration update tasks from management UI"
|
|
echo " • Apply HAProxy config changes to: $HAPROXY_CONFIG_PATH"
|
|
echo ""
|
|
echo "✨ Your Linux HAProxy instance is now centrally manageable!"
|
|
echo ""
|
|
echo "Next Steps:"
|
|
echo " 1. Check agent logs: tail -f $LOG_DIR/agent.log"
|
|
echo " 2. Verify agent registration in management UI"
|
|
echo " 3. Test configuration deployment from UI"
|
|
echo ""
|
|
else
|
|
log "ERROR" "Agent service failed to start"
|
|
echo "Check logs: tail -n 20 $LOG_DIR/agent.log"
|
|
echo "Check service: systemctl status haproxy-agent"
|
|
exit 1
|
|
fi
|
|
|
|
fi # End of installer section - skip if in daemon mode
|
|
|
|
# Check if script should skip to daemon mode (from upgrade) - EXACT MacOS COPY
|
|
if [[ "$SKIP_TO_DAEMON" == "true" ]]; then
|
|
# Reduced logging on daemon restart to prevent log spam
|
|
log "DEBUG" "AGENT UPGRADE: Daemon mode detected, starting agent daemon..."
|
|
|
|
log "DEBUG" "DAEMON: Starting HAProxy Agent: $AGENT_NAME (Linux)"
|
|
log "DEBUG" "DAEMON: Loading essential functions for daemon mode..."
|
|
|
|
# CRITICAL: Define essential functions within SKIP_TO_DAEMON block
|
|
# These functions are needed by daemon loop but aren't accessible from main script scope
|
|
|
|
# Find binary utility (needed by stats functions)
|
|
find_binary() {
|
|
local binary_name="$1"
|
|
local binary_path=""
|
|
|
|
# Common binary paths
|
|
for path in "/usr/bin" "/usr/local/bin" "/bin" "/usr/sbin" "/usr/local/sbin" "/sbin" "/opt/homebrew/bin"; do
|
|
if [[ -x "$path/$binary_name" ]]; then
|
|
binary_path="$path/$binary_name"
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [[ -n "$binary_path" ]]; then
|
|
echo "$binary_path"
|
|
else
|
|
command -v "$binary_name" 2>/dev/null || echo ""
|
|
fi
|
|
}
|
|
|
|
# Get HAProxy status (Linux specific)
|
|
get_haproxy_status() {
|
|
if systemctl is-active --quiet haproxy 2>/dev/null; then
|
|
echo "running"
|
|
elif pgrep -f "haproxy" > /dev/null 2>&1; then
|
|
echo "running"
|
|
else
|
|
echo "stopped"
|
|
fi
|
|
}
|
|
|
|
# Get full HAProxy stats CSV from socket (for dashboard metrics)
|
|
get_haproxy_stats_csv() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "STATS: socat not available, cannot fetch stats CSV"
|
|
echo ""
|
|
return
|
|
fi
|
|
log "INFO" "STATS: Initialized socat: $SOCAT_BIN"
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
log "INFO" "STATS: Using default socket path: $STATS_SOCKET_PATH"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "STATS: Socket not found: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "INFO" "STATS: Fixing socket permissions: $STATS_SOCKET_PATH"
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
# Return CSV data (base64 encoded to avoid JSON escaping issues)
|
|
if [[ -n "$stats_output" ]]; then
|
|
log "INFO" "STATS: Fetched CSV: ${#stats_output} bytes, $(echo "$stats_output" | wc -l) lines, base64: $(echo "$stats_output" | base64 -w 0 | wc -c) chars"
|
|
echo "$stats_output" | base64 -w 0
|
|
else
|
|
log "WARN" "STATS: No stats data retrieved from socket: $STATS_SOCKET_PATH"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
# Get server statuses from HAProxy stats socket
|
|
get_server_statuses() {
|
|
# Lazy initialize SOCAT_BIN if not set (for upgrade scenarios)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
SOCAT_BIN=$(command -v socat 2>/dev/null || find_binary socat 2>/dev/null)
|
|
if [[ -z "$SOCAT_BIN" ]]; then
|
|
log "WARN" "SOCAT not available for server status checks"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Lazy initialize STATS_SOCKET_PATH if not set
|
|
if [[ -z "$STATS_SOCKET_PATH" ]]; then
|
|
STATS_SOCKET_PATH="/var/run/haproxy/admin.sock"
|
|
fi
|
|
|
|
# Check if stats socket exists and is accessible
|
|
if [[ ! -S "$STATS_SOCKET_PATH" ]]; then
|
|
log "WARN" "Stats socket not found: $STATS_SOCKET_PATH"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
|
|
# Query HAProxy stats via socket with permission fix attempt
|
|
local stats_output
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
# Try to fix socket permission if we have root access
|
|
if [[ $EUID -eq 0 ]] && [[ -S "$STATS_SOCKET_PATH" ]]; then
|
|
chmod 666 "$STATS_SOCKET_PATH" 2>/dev/null
|
|
# Retry after permission fix
|
|
if ! stats_output=$(echo "show stat" | "$SOCAT_BIN" stdio "$STATS_SOCKET_PATH" 2>/dev/null); then
|
|
log "WARN" "Failed to query server stats via socket after permission fix"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
else
|
|
log "WARN" "Failed to query server stats via socket"
|
|
echo "{}"
|
|
return
|
|
fi
|
|
fi
|
|
|
|
# Parse stats output and build JSON - EXACT COPY from main script
|
|
local backend_servers=()
|
|
while IFS=',' read -r pxname svname qcur qmax scur smax slim stot bin bout dreq dresp ereq econ eresp wretr wredis status weight act bck chkfail chkdown lastchg downtime qlimit pid iid sid throttle lbtot tracked type rate rate_lim rate_max check_status check_code check_duration hrsp_1xx hrsp_2xx hrsp_3xx hrsp_4xx hrsp_5xx hrsp_other hanafail req_rate req_rate_max req_tot cli_abrt srv_abrt comp_in comp_out comp_byp comp_rsp lastsess last_chk last_agt qtime ctime rtime ttime agent_status agent_code agent_duration check_desc agent_desc check_rise check_fall check_health agent_rise agent_fall agent_health addr cookie mode algo conn_rate conn_rate_max conn_tot intercepted dcon dses rest; do
|
|
# Skip header and empty lines
|
|
[[ "$pxname" == "# pxname" || -z "$pxname" ]] && continue
|
|
|
|
# Only process server entries (type = 2), skip backends (type = 1) and frontends (type = 0)
|
|
[[ "$type" != "2" ]] && continue
|
|
|
|
# Extract backend and server names
|
|
local backend_name="$pxname"
|
|
local server_name="$svname"
|
|
|
|
# Map HAProxy status to our format
|
|
local server_status
|
|
case "$status" in
|
|
"UP") server_status="UP" ;;
|
|
"DOWN") server_status="DOWN" ;;
|
|
"MAINT") server_status="MAINT" ;;
|
|
"DRAIN") server_status="DRAIN" ;;
|
|
"NOLB") server_status="NOLB" ;;
|
|
*) server_status="UNKNOWN" ;;
|
|
esac
|
|
|
|
backend_servers+=("$backend_name:$server_name:$server_status")
|
|
done <<< "$stats_output"
|
|
|
|
# Build JSON structure - EXACT COPY from main script
|
|
if [[ ${#backend_servers[@]} -gt 0 ]]; then
|
|
local json_parts=()
|
|
local current_backend=""
|
|
local backend_json=""
|
|
|
|
for entry in "${backend_servers[@]}"; do
|
|
IFS=':' read -r backend_name server_name server_status <<< "$entry"
|
|
|
|
if [[ "$current_backend" != "$backend_name" ]]; then
|
|
# Save previous backend if exists
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
# Start new backend
|
|
current_backend="$backend_name"
|
|
backend_json=""
|
|
fi
|
|
|
|
backend_json+="\"$server_name\": \"$server_status\","
|
|
done
|
|
|
|
# Save last backend
|
|
if [[ -n "$current_backend" && -n "$backend_json" ]]; then
|
|
json_parts+=("\"$current_backend\": {${backend_json%,}}")
|
|
fi
|
|
|
|
# Combine all backends
|
|
if [[ ${#json_parts[@]} -gt 0 ]]; then
|
|
local IFS=','
|
|
server_statuses_json="{${json_parts[*]}}"
|
|
fi
|
|
fi
|
|
|
|
echo "$server_statuses_json"
|
|
}
|
|
|
|
# Collect basic system information for heartbeat
|
|
collect_system_info() {
|
|
local load_avg=$(uptime | awk -F'load average:' '{print $2}' | awk '{print $1}' | tr -d ',')
|
|
local disk_usage=$(df -h / 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
|
local memory_usage=$(free | awk 'NR==2{printf "%.1f", $3*100/$2}')
|
|
|
|
cat <<SYSTEM_INFO_EOF
|
|
{
|
|
"timestamp": "$(date -u '+%Y-%m-%dT%H:%M:%SZ')",
|
|
"load_average": "${load_avg}",
|
|
"disk_usage": "${disk_usage}",
|
|
"memory_usage": "${memory_usage}"
|
|
}
|
|
SYSTEM_INFO_EOF
|
|
}
|
|
|
|
# Send heartbeat (DAEMON version - includes all stats)
|
|
send_heartbeat() {
|
|
local haproxy_status=$(get_haproxy_status)
|
|
local server_statuses=$(get_server_statuses)
|
|
local haproxy_stats_csv=$(get_haproxy_stats_csv)
|
|
local system_info=$(collect_system_info)
|
|
|
|
# Ensure platform is available (global variable set during registration)
|
|
if [[ -z "$platform" ]]; then
|
|
platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
fi
|
|
|
|
# Prepare heartbeat payload with all data
|
|
local heartbeat_payload="{
|
|
\"name\": \"$AGENT_NAME\",
|
|
\"hostname\": \"$(hostname)\",
|
|
\"status\": \"online\",
|
|
\"platform\": \"$platform\",
|
|
\"architecture\": \"$(uname -m)\",
|
|
\"version\": \"{{AGENT_VERSION}}\",
|
|
\"haproxy_status\": \"$haproxy_status\",
|
|
\"cluster_id\": $CLUSTER_ID,
|
|
\"server_statuses\": $server_statuses,
|
|
\"system_info\": $system_info"
|
|
|
|
# CRITICAL: Add haproxy_stats_csv only if available (for dashboard metrics)
|
|
if [[ -n "$haproxy_stats_csv" && "$haproxy_stats_csv" != "" ]]; then
|
|
heartbeat_payload+=",\"haproxy_stats_csv\": \"$haproxy_stats_csv\""
|
|
log "INFO" "HEARTBEAT: Including stats CSV (${#haproxy_stats_csv} chars) with server statuses"
|
|
else
|
|
log "WARN" "HEARTBEAT: No server statuses and no stats CSV - minimal heartbeat"
|
|
fi
|
|
|
|
heartbeat_payload+="}"
|
|
|
|
# Send heartbeat to backend
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "$heartbeat_payload" >/dev/null 2>&1
|
|
}
|
|
|
|
# Check for pending configuration requests (DAEMON MODE)
|
|
check_config_requests() {
|
|
local curl_bin="${CURL_BIN:-$(find_binary curl)}"
|
|
|
|
# Get pending config requests from backend
|
|
log "DEBUG" "CONFIG: Checking for pending config requests from: ${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests"
|
|
|
|
local response=$("$curl_bin" -k -s -X GET "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/pending-requests" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}")
|
|
|
|
log "DEBUG" "CONFIG: Response: ${response:0:200}..."
|
|
|
|
# Check if there are pending requests
|
|
local pending_count=$(echo "$response" | jq -r '.pending_requests | length' 2>/dev/null)
|
|
|
|
log "DEBUG" "CONFIG: Pending count: $pending_count"
|
|
|
|
if [[ "$pending_count" -gt 0 ]]; then
|
|
log "INFO" "CONFIG: Found $pending_count pending config request(s)"
|
|
|
|
# Process each request
|
|
echo "$response" | jq -c '.pending_requests[]' 2>/dev/null | while read -r request; do
|
|
local request_id=$(echo "$request" | jq -r '.request_id')
|
|
local request_type=$(echo "$request" | jq -r '.request_type')
|
|
|
|
log "INFO" "CONFIG: Processing config request #$request_id (type: $request_type)"
|
|
|
|
# Read haproxy.cfg content
|
|
local config_path="${HAPROXY_CONFIG_PATH}"
|
|
if [[ -f "$config_path" ]]; then
|
|
local file_size=$(wc -c < "$config_path")
|
|
log "INFO" "CONFIG: Reading config from: $config_path (size: $file_size bytes)"
|
|
|
|
local config_content=$(cat "$config_path")
|
|
|
|
# Escape config content for JSON
|
|
log "DEBUG" "CONFIG: Escaping config content for JSON..."
|
|
local escaped_content=$(echo "$config_content" | jq -Rs .)
|
|
|
|
if [[ -z "$escaped_content" ]]; then
|
|
log "ERROR" "CONFIG: JSON escaping failed for config content!"
|
|
continue
|
|
fi
|
|
|
|
local escaped_size=${#escaped_content}
|
|
log "DEBUG" "CONFIG: JSON escaped content size: $escaped_size bytes"
|
|
|
|
# Submit config response
|
|
local response_payload=$(cat <<CONFIG_RESPONSE_EOF
|
|
{
|
|
"request_id": $request_id,
|
|
"config_content": $escaped_content,
|
|
"config_path": "$config_path"
|
|
}
|
|
CONFIG_RESPONSE_EOF
|
|
)
|
|
|
|
local payload_size=${#response_payload}
|
|
log "INFO" "CONFIG: Sending config response (payload size: $payload_size bytes)..."
|
|
|
|
local http_response=$("$curl_bin" -k -s -w "\n%{http_code}" -X POST "${MANAGEMENT_URL}/api/configuration/agents/${AGENT_NAME}/config-response" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: ${AGENT_TOKEN}" \
|
|
-d "$response_payload")
|
|
|
|
local http_code=$(echo "$http_response" | tail -n1)
|
|
local response_body=$(echo "$http_response" | head -n-1)
|
|
|
|
if [[ "$http_code" == "200" || "$http_code" == "201" ]]; then
|
|
log "INFO" "CONFIG: Config response sent for request #$request_id (size: $file_size bytes, HTTP $http_code)"
|
|
else
|
|
log "ERROR" "CONFIG: Config response failed for request #$request_id (HTTP $http_code): $response_body"
|
|
fi
|
|
else
|
|
log "ERROR" "CONFIG: Config file not found: $config_path"
|
|
fi
|
|
done
|
|
fi
|
|
}
|
|
|
|
log "DEBUG" "DAEMON: Essential functions loaded for daemon mode (including check_config_requests)"
|
|
|
|
# Send simple completion heartbeat
|
|
log "DEBUG" "DAEMON: Sending upgrade completion heartbeat..."
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/heartbeat" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $AGENT_TOKEN" \
|
|
-d "{\"name\":\"$AGENT_NAME\",\"hostname\":\"$(hostname)\",\"status\":\"online\",\"platform\":\"linux\",\"architecture\":\"$(uname -m)\",\"version\":\"{{AGENT_VERSION}}\",\"haproxy_status\":\"running\",\"cluster_id\":$CLUSTER_ID}" >/dev/null 2>&1
|
|
|
|
log "DEBUG" "DAEMON: Agent upgrade completed successfully"
|
|
log "DEBUG" "DAEMON: Agent is now online with version {{AGENT_VERSION}}"
|
|
|
|
# FINAL FIX: Implement daemon loop directly (no exec to avoid infinite recursion) - EXACT MacOS COPY
|
|
log "DEBUG" "DAEMON: Starting continuous daemon mode..."
|
|
|
|
# Initialize daemon environment
|
|
CONFIG_FILE="/etc/haproxy-agent/config.json"
|
|
LOG_FILE="/var/log/haproxy-agent/agent.log"
|
|
PID_FILE="/tmp/haproxy-agent-$AGENT_NAME.pid"
|
|
|
|
# Write PID (use /tmp to avoid permission issues)
|
|
echo $$ > "$PID_FILE" 2>/dev/null || true
|
|
|
|
# CRITICAL FIX: Initialize version tracking from database (prevent config reapply on restart)
|
|
# Agent restart/upgrade should NOT trigger config reapplication - fetch last applied version
|
|
log "DEBUG" "DAEMON: Fetching last applied config version from backend..."
|
|
agent_info_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents" \
|
|
-H "X-API-Key: $AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ $? -eq 0 && -n "$agent_info_response" ]]; then
|
|
# Extract applied_config_version for this agent using jq
|
|
last_applied_version=$(echo "$agent_info_response" | jq -r ".agents[] | select(.name==\"$AGENT_NAME\") | .applied_config_version // \"none\"" 2>/dev/null)
|
|
|
|
if [[ -z "$last_applied_version" || "$last_applied_version" == "null" ]]; then
|
|
last_applied_version="none"
|
|
log "WARN" "DAEMON: Could not fetch applied_config_version, starting with 'none'"
|
|
else
|
|
log "INFO" "DAEMON: Loaded last applied version from database: $last_applied_version"
|
|
fi
|
|
else
|
|
last_applied_version="none"
|
|
log "WARN" "DAEMON: Failed to fetch agent info, starting with version 'none'"
|
|
fi
|
|
|
|
last_validation_failed_version="none"
|
|
|
|
log "DEBUG" "DAEMON: Starting agent monitoring loop for $AGENT_NAME"
|
|
|
|
# Enhanced daemon loop with upgrade capability - EXACT MacOS COPY
|
|
while true; do
|
|
sleep 30
|
|
|
|
# Read current token from config (fix for API key mismatch)
|
|
CURRENT_AGENT_TOKEN=$(jq -r '.management.token' "$CONFIG_FILE" 2>/dev/null || echo "$AGENT_TOKEN")
|
|
AGENT_TOKEN="$CURRENT_AGENT_TOKEN" # Update AGENT_TOKEN for send_heartbeat function
|
|
|
|
# Ensure platform is available for heartbeat
|
|
[[ -z "$platform" ]] && platform=$(uname -s | tr '[:upper:]' '[:lower:]')
|
|
|
|
# Call send_heartbeat function (includes server_statuses)
|
|
send_heartbeat
|
|
|
|
# Only log heartbeat in debug mode to reduce log spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "INFO" "DAEMON: Heartbeat sent for agent $AGENT_NAME"
|
|
|
|
# Check for pending configuration requests (if function exists)
|
|
if type check_config_requests &>/dev/null; then
|
|
check_config_requests
|
|
fi
|
|
|
|
# Check for SSL certificates and deploy them
|
|
# Silent check - only log if changes are detected (reduce log spam)
|
|
ssl_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/ssl-certificates" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ -n "$ssl_response" ]] && ! echo "$ssl_response" | grep -q '"error":'; then
|
|
ssl_status=$(echo "$ssl_response" | jq -r '.status // "unknown"' 2>/dev/null)
|
|
|
|
if [[ "$ssl_status" == "available" ]]; then
|
|
ssl_certificates=$(echo "$ssl_response" | jq -r '.ssl_certificates // "[]"' 2>/dev/null)
|
|
|
|
if [[ "$ssl_certificates" != "[]" && "$ssl_certificates" != "null" ]]; then
|
|
# Create SSL directory if it doesn't exist
|
|
SSL_DIR="/etc/ssl/haproxy"
|
|
mkdir -p "$SSL_DIR" 2>/dev/null || true
|
|
|
|
# Track if any certificate was actually deployed
|
|
ssl_deployed_count=0
|
|
|
|
# Process each certificate
|
|
# IMPORTANT: Use process substitution to avoid subshell (pipe would lose ssl_deployed_count)
|
|
while IFS= read -r cert_data; do
|
|
if [[ -n "$cert_data" ]]; then
|
|
cert_name=$(echo "$cert_data" | jq -r '.name // "unknown"' 2>/dev/null)
|
|
cert_domain=$(echo "$cert_data" | jq -r '.domain // "unknown"' 2>/dev/null)
|
|
cert_content=$(echo "$cert_data" | jq -r '.certificate_content // ""' 2>/dev/null)
|
|
key_content=$(echo "$cert_data" | jq -r '.private_key_content // ""' 2>/dev/null)
|
|
chain_content=$(echo "$cert_data" | jq -r '.chain_content // ""' 2>/dev/null)
|
|
cert_file_path=$(echo "$cert_data" | jq -r '.file_path // ""' 2>/dev/null)
|
|
usage_type=$(echo "$cert_data" | jq -r '.usage_type // "frontend"' 2>/dev/null)
|
|
|
|
# CRITICAL: Certificate content is always required, but private key depends on usage_type
|
|
# - Frontend SSL: requires both cert AND private key (for bind ssl crt)
|
|
# - Server SSL: requires only cert (CA file for backend verification)
|
|
if [[ -n "$cert_content" && "$cert_content" != "null" ]]; then
|
|
# For frontend SSL, private key is required
|
|
if [[ "$usage_type" == "frontend" && ( -z "$key_content" || "$key_content" == "null" ) ]]; then
|
|
log "WARN" "Skipping Frontend SSL $cert_name: private key missing"
|
|
continue
|
|
fi
|
|
|
|
# Use original path or fallback
|
|
if [[ -n "$cert_file_path" ]]; then
|
|
ssl_file="$cert_file_path"
|
|
else
|
|
ssl_file="$SSL_DIR/${cert_name}.pem"
|
|
fi
|
|
|
|
# Check if certificate already exists and is unchanged
|
|
# IMPORTANT: Calculate checksum with EXACT same format as file will be written
|
|
# Format depends on what content is available:
|
|
# - Frontend SSL: cert + key [+ chain]
|
|
# - Server SSL: cert [+ chain] (no private key)
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
# Has private key - frontend SSL
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s\n\n%s" "$cert_content" "$key_content" "$chain_content")
|
|
else
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$key_content")
|
|
fi
|
|
else
|
|
# No private key - server SSL (CA cert only)
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
new_content=$(printf "%s\n\n%s" "$cert_content" "$chain_content")
|
|
else
|
|
new_content="$cert_content"
|
|
fi
|
|
fi
|
|
|
|
existing_checksum=""
|
|
new_checksum=$(echo "$new_content" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
|
|
if [[ -f "$ssl_file" ]]; then
|
|
existing_checksum=$(cat "$ssl_file" | md5sum | awk '{print $1}' 2>/dev/null)
|
|
fi
|
|
|
|
# Only deploy if certificate is new or changed
|
|
if [[ "$existing_checksum" != "$new_checksum" ]]; then
|
|
# Create combined PEM file
|
|
# Format depends on usage_type:
|
|
# - Frontend SSL: cert + key + [chain]
|
|
# - Server SSL: cert + [chain] (CA cert only, no private key)
|
|
{
|
|
echo "$cert_content"
|
|
# Only add private key if provided (server SSL may not have it)
|
|
if [[ -n "$key_content" && "$key_content" != "null" ]]; then
|
|
echo ""
|
|
echo "$key_content"
|
|
fi
|
|
# Add chain if present
|
|
if [[ -n "$chain_content" && "$chain_content" != "null" ]]; then
|
|
echo ""
|
|
echo "$chain_content"
|
|
fi
|
|
} > "$ssl_file" 2>/dev/null
|
|
|
|
if [[ -f "$ssl_file" ]]; then
|
|
chmod 600 "$ssl_file" 2>/dev/null
|
|
# Reduced logging to DEBUG to prevent spam on every check
|
|
log "DEBUG" "DAEMON: SSL certificate deployed: $ssl_file ($cert_domain)"
|
|
ssl_deployed_count=$((ssl_deployed_count + 1))
|
|
else
|
|
log "ERROR" "DAEMON: Failed to deploy SSL certificate: $cert_name"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
done < <(echo "$ssl_certificates" | jq -c '.[]' 2>/dev/null)
|
|
|
|
# CRITICAL: Reload HAProxy if any SSL certificates were updated
|
|
# This ensures HAProxy picks up the new SSL certificates
|
|
if [[ $ssl_deployed_count -gt 0 ]]; then
|
|
log "INFO" "DAEMON: SSL certificates updated ($ssl_deployed_count files), triggering HAProxy reload..."
|
|
|
|
# Validate HAProxy config before reload
|
|
if "$HAPROXY_BIN" -c -f "$HAPROXY_CONFIG" >/dev/null 2>&1; then
|
|
# Reload HAProxy using systemctl (Linux best practice)
|
|
if systemctl reload haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy reloaded successfully after SSL update"
|
|
else
|
|
log "WARN" "DAEMON: systemctl reload failed after SSL update, attempting restart..."
|
|
if systemctl restart haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy restarted successfully after SSL update"
|
|
else
|
|
log "ERROR" "DAEMON: Failed to reload/restart HAProxy after SSL update"
|
|
fi
|
|
fi
|
|
else
|
|
# IMPORTANT: Config validation may fail if SSL was updated but config not yet synced
|
|
# This is normal - config update will trigger reload with both new SSL and config
|
|
log "WARN" "DAEMON: SSL deployed but HAProxy config not yet synced (validation failed), reload will happen on next config update"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Check for configuration updates with proper error handling
|
|
config_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
# Validate API response before processing
|
|
if [[ -n "$config_response" ]] && ! echo "$config_response" | grep -q '"error":'; then
|
|
if echo "$config_response" | grep -q '"config_content":'; then
|
|
# Extract config version for tracking
|
|
config_version=$(echo "$config_response" | jq -r '.version // "unknown"' 2>/dev/null)
|
|
|
|
# CRITICAL: Check if this version is already applied (prevent infinite loop)
|
|
if [[ "$config_version" == "$last_applied_version" ]]; then
|
|
# Only log in debug mode to avoid spam
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration version $config_version already applied, skipping"
|
|
else
|
|
log "INFO" "DAEMON: Configuration update detected (version: $config_version)"
|
|
|
|
# Extract config content for validation
|
|
daemon_config_content=$(echo "$config_response" | jq -r '.config_content' 2>/dev/null)
|
|
|
|
# CRITICAL: Apply same validation as main function
|
|
# IMPORTANT: Don't use 'continue' here - it prevents upgrade check from running!
|
|
config_is_valid=true
|
|
|
|
if [[ -z "$daemon_config_content" || "$daemon_config_content" == "null" ]]; then
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Configuration content is empty, skipping config update"
|
|
config_is_valid=false
|
|
elif [[ "$daemon_config_content" == "# No configuration available"* ]] || [[ "$daemon_config_content" == "# Agent is disabled"* ]] || [[ "$daemon_config_content" == "# No cluster assigned"* ]]; then
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Received placeholder config (no-config state), skipping update"
|
|
config_is_valid=false
|
|
elif ! echo "$daemon_config_content" | grep -q -E "(global|defaults|frontend|backend|listen)"; then
|
|
log "ERROR" "DAEMON: Configuration content does not appear to be valid HAProxy config"
|
|
log "DEBUG" "DAEMON: Config preview: ${daemon_config_content:0:200}..."
|
|
config_is_valid=false
|
|
fi
|
|
|
|
# Only process config if it's valid
|
|
if [[ "$config_is_valid" == "true" ]]; then
|
|
|
|
# Write validated config to temp file
|
|
echo "$daemon_config_content" > /tmp/haproxy-new-config.cfg 2>/dev/null
|
|
|
|
if [[ -f "/tmp/haproxy-new-config.cfg" && -s "/tmp/haproxy-new-config.cfg" ]]; then
|
|
# Get paths from config
|
|
HAPROXY_BIN=$(jq -r '.haproxy.bin_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_BIN_PATH}}")
|
|
HAPROXY_CONFIG=$(jq -r '.haproxy.config_path' "$CONFIG_FILE" 2>/dev/null || echo "{{HAPROXY_CONFIG_PATH}}")
|
|
STATS_SOCKET=$(jq -r '.haproxy.stats_socket_path' "$CONFIG_FILE" 2>/dev/null || echo "{{STATS_SOCKET_PATH}}")
|
|
|
|
# CRITICAL NEW FEATURE: Check if this is a partial config (only frontends/backends)
|
|
# Backend now generates partial configs to preserve global/defaults sections
|
|
if head -20 /tmp/haproxy-new-config.cfg | grep -q "# Generated by HAProxy Open Manager - Partial Configuration"; then
|
|
log "INFO" "DAEMON: Detected PARTIAL config - preserving global/defaults from local haproxy.cfg"
|
|
|
|
# Extract global section from current haproxy.cfg
|
|
if [[ -f "$HAPROXY_CONFIG" ]]; then
|
|
# Extract everything from start until 'defaults' section
|
|
awk '/^defaults[[:space:]]*$/{exit} {print}' "$HAPROXY_CONFIG" > /tmp/haproxy-global.cfg 2>/dev/null
|
|
|
|
# Extract defaults section (from 'defaults' to first 'frontend/backend/listen')
|
|
# CRITICAL FIX: Exit BEFORE printing the frontend/backend/listen line
|
|
awk '/^defaults[[:space:]]*$/{flag=1} /^(frontend|backend|listen)[[:space:]]/{if(flag && !/^defaults/) exit} flag{print}' "$HAPROXY_CONFIG" > /tmp/haproxy-defaults.cfg 2>/dev/null
|
|
|
|
# Extract all listen sections (preserve existing listen blocks like stats monitoring)
|
|
# CRITICAL FIX: Extract ALL listen blocks from existing config to preserve them
|
|
awk '
|
|
/^listen[[:space:]]/ {
|
|
if (listen_content != "") {
|
|
# Print previous listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
in_listen=1
|
|
listen_content=$0 "\n"
|
|
next
|
|
}
|
|
in_listen {
|
|
# Check if we hit another section (frontend, backend, listen, global, defaults)
|
|
if (/^(frontend|backend|listen|global|defaults)[[:space:]]/) {
|
|
# Print current listen block
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
listen_content=""
|
|
in_listen=0
|
|
} else {
|
|
listen_content = listen_content $0 "\n"
|
|
}
|
|
}
|
|
END {
|
|
# Print final listen block if any
|
|
if (listen_content != "") {
|
|
gsub(/\n[[:space:]]*$/, "", listen_content)
|
|
print listen_content
|
|
}
|
|
}
|
|
' "$HAPROXY_CONFIG" > /tmp/haproxy-listen.cfg 2>/dev/null
|
|
|
|
# Check if we successfully extracted sections
|
|
if [[ -s /tmp/haproxy-global.cfg && -s /tmp/haproxy-defaults.cfg ]]; then
|
|
listen_lines=0
|
|
[[ -s /tmp/haproxy-listen.cfg ]] && listen_lines=$(wc -l < /tmp/haproxy-listen.cfg)
|
|
log "INFO" "DAEMON: Extracted global ($(wc -l < /tmp/haproxy-global.cfg) lines), defaults ($(wc -l < /tmp/haproxy-defaults.cfg) lines), listen blocks ($listen_lines lines)"
|
|
|
|
# Merge: global + defaults + listen blocks + new frontends/backends
|
|
# Copy global section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-global.cfg > /tmp/haproxy-merged.cfg
|
|
|
|
# Copy defaults section as-is, normalize trailing blanks to exactly 1 line
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-defaults.cfg >> /tmp/haproxy-merged.cfg
|
|
|
|
# Copy listen sections (if any) - these are preserved from existing config (e.g. stats monitoring)
|
|
if [[ -s /tmp/haproxy-listen.cfg ]]; then
|
|
awk '{lines[NR]=$0} END {n=NR; while(n>0 && lines[n]~/^[[:space:]]*$/){n--} for(i=1;i<=n;i++){print lines[i]}; print ""}' /tmp/haproxy-listen.cfg >> /tmp/haproxy-merged.cfg
|
|
log "INFO" "DAEMON: Preserved $(grep -c '^listen[[:space:]]' /tmp/haproxy-listen.cfg) listen blocks from existing config"
|
|
fi
|
|
|
|
# Skip comment header from partial config, add only frontends/backends
|
|
awk '/^(frontend|backend)[[:space:]]/{flag=1} flag{print}' /tmp/haproxy-new-config.cfg >> /tmp/haproxy-merged.cfg
|
|
|
|
# Replace new config with merged version
|
|
mv /tmp/haproxy-merged.cfg /tmp/haproxy-new-config.cfg
|
|
log "INFO" "DAEMON: Config merged - preserved local global/defaults/listen blocks, applied new frontends/backends"
|
|
|
|
# Cleanup temp files
|
|
rm -f /tmp/haproxy-global.cfg /tmp/haproxy-defaults.cfg /tmp/haproxy-listen.cfg
|
|
else
|
|
log "WARN" "DAEMON: Failed to extract global/defaults, using partial config as-is (may fail validation)"
|
|
fi
|
|
else
|
|
log "WARN" "DAEMON: No existing haproxy.cfg found, using partial config as-is"
|
|
fi
|
|
else
|
|
log "INFO" "DAEMON: Full config detected - using as-is"
|
|
fi
|
|
|
|
# CRITICAL: Create backup before applying new config
|
|
daemon_backup_file="${HAPROXY_CONFIG}.backup-$(date +%Y%m%d-%H%M%S)"
|
|
if [[ -f "$HAPROXY_CONFIG" ]]; then
|
|
cp "$HAPROXY_CONFIG" "$daemon_backup_file" 2>/dev/null
|
|
if [[ -f "$daemon_backup_file" ]]; then
|
|
log "INFO" "DAEMON: Config backed up to: $daemon_backup_file"
|
|
else
|
|
log "ERROR" "DAEMON: Failed to create config backup, skipping config update"
|
|
rm -f /tmp/haproxy-new-config.cfg
|
|
fi
|
|
fi
|
|
|
|
# CRITICAL FIX: Validate BEFORE applying (was: apply then validate)
|
|
# Test new config from temp file FIRST
|
|
if "$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c >/dev/null 2>&1; then
|
|
# Validation successful - NOW apply new config
|
|
cp /tmp/haproxy-new-config.cfg "$HAPROXY_CONFIG"
|
|
log "INFO" "DAEMON: Configuration validation successful"
|
|
|
|
# Reload HAProxy using systemctl (Linux best practice)
|
|
if systemctl reload haproxy >/dev/null 2>&1; then
|
|
log "INFO" "DAEMON: HAProxy reloaded successfully via systemctl"
|
|
else
|
|
log "WARN" "DAEMON: systemctl reload failed, attempting restart..."
|
|
systemctl restart haproxy >/dev/null 2>&1
|
|
fi
|
|
log "INFO" "DAEMON: Configuration applied and HAProxy reloaded"
|
|
|
|
# CRITICAL: Notify backend of successful config application with version
|
|
log "DEBUG" "Sending config-applied notification for version: $config_version"
|
|
log "DEBUG" "Using token: ${CURRENT_AGENT_TOKEN:0:10}..."
|
|
log "DEBUG" "Agent name: $AGENT_NAME"
|
|
|
|
config_applied_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-applied" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"status\": \"applied\", \"version\": \"$config_version\"}" 2>&1)
|
|
|
|
log "DEBUG" "Config-applied response: $config_applied_response"
|
|
|
|
if echo "$config_applied_response" | grep -q '"status":"ok"'; then
|
|
log "INFO" "DAEMON: Config-applied notification sent successfully"
|
|
else
|
|
log "ERROR" "DAEMON: Config-applied notification failed: $config_applied_response"
|
|
fi
|
|
|
|
# CRITICAL: Update last applied version to prevent reapplying same config
|
|
last_applied_version="$config_version"
|
|
log "INFO" "DAEMON: Configuration applied successfully (version: $config_version)"
|
|
else
|
|
# CRITICAL: Validation failed - config was NEVER applied to haproxy.cfg
|
|
# No restore needed - original config is still intact
|
|
|
|
# Check if we already reported this validation failure to avoid spam
|
|
if [[ "$config_version" != "$last_validation_failed_version" ]]; then
|
|
log "ERROR" "DAEMON: Configuration validation failed - rejected before applying"
|
|
log "ERROR" "DAEMON: Original HAProxy config remains unchanged"
|
|
# Get validation error details (without 'local' to avoid scope issues)
|
|
validation_error=$("$HAPROXY_BIN" -f /tmp/haproxy-new-config.cfg -c 2>&1)
|
|
log "ERROR" "DAEMON: Validation error: $validation_error"
|
|
|
|
# CRITICAL NEW FEATURE: Notify backend of validation failure for UI display
|
|
log "INFO" "DAEMON: Sending validation error notification to backend..."
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/config-validation-failed" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"status\": \"validation_failed\", \"version\": \"$config_version\", \"validation_error\": $(echo "$validation_error" | jq -Rs .)}" \
|
|
>/dev/null 2>&1
|
|
|
|
if [ $? -eq 0 ]; then
|
|
log "INFO" "DAEMON: Validation error notification sent to backend successfully"
|
|
else
|
|
log "WARN" "DAEMON: Failed to send validation error notification to backend"
|
|
fi
|
|
|
|
# Mark this version as validation failed to prevent repeated error logs
|
|
last_validation_failed_version="$config_version"
|
|
else
|
|
# Already reported this validation failure, just log in DEBUG to avoid spam
|
|
log "DEBUG" "DAEMON: Config version $config_version still failing validation (already reported)"
|
|
fi
|
|
fi
|
|
|
|
# Clean up
|
|
rm -f /tmp/haproxy-new-config.cfg
|
|
fi
|
|
fi # End of config_is_valid check
|
|
fi # End of version already applied check
|
|
fi
|
|
else
|
|
# Log API error for debugging
|
|
if echo "$config_response" | grep -q '"error":'; then
|
|
error_msg=$(echo "$config_response" | jq -r '.error.message' 2>/dev/null || echo "Unknown API error")
|
|
log "DEBUG" "DAEMON: API Error: $error_msg (skipping config check)"
|
|
fi
|
|
fi
|
|
|
|
# Check for upgrade requests - backend controls upgrade via agent status
|
|
# CRITICAL: Always log upgrade check for debugging
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Checking upgrade status (current: {{AGENT_VERSION}})"
|
|
|
|
upgrade_response=$(curl -k -s -X GET "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-status" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" 2>/dev/null)
|
|
|
|
if [[ -n "$upgrade_response" ]]; then
|
|
should_upgrade=$(echo "$upgrade_response" | jq -r '.should_upgrade // false' 2>/dev/null)
|
|
target_version=$(echo "$upgrade_response" | jq -r '.target_version // ""' 2>/dev/null)
|
|
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade status response: should_upgrade=$should_upgrade, target=$target_version, current={{AGENT_VERSION}}"
|
|
|
|
# Only upgrade if backend says we should AND target version is different from current
|
|
if [[ "$should_upgrade" == "true" && -n "$target_version" && "$target_version" != "{{AGENT_VERSION}}" ]]; then
|
|
log "INFO" "DAEMON: Starting upgrade from {{AGENT_VERSION}} to $target_version (backend requested)"
|
|
|
|
# Download new script with validation
|
|
temp_script="/tmp/haproxy-agent-upgrade-$AGENT_NAME.sh"
|
|
log "INFO" "DAEMON: Downloading new agent script..."
|
|
|
|
# Get correct pool_id from config
|
|
daemon_pool_id=$(jq -r '.management.pool_id // 1' "$CONFIG_FILE" 2>/dev/null)
|
|
[[ -z "$daemon_pool_id" || "$daemon_pool_id" == "null" ]] && daemon_pool_id=1
|
|
|
|
# CRITICAL FIX: Normalize architecture format (x86_64 -> amd64, aarch64 -> arm64)
|
|
daemon_arch=$(uname -m | sed 's/x86_64/amd64/; s/aarch64/arm64/')
|
|
|
|
script_response=$(curl -k -s -X POST "$MANAGEMENT_URL/api/agents/generate-install-script" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"platform\":\"linux\",\"architecture\":\"$daemon_arch\",\"pool_id\":$daemon_pool_id,\"cluster_id\":$CLUSTER_ID,\"agent_name\":\"$AGENT_NAME\",\"hostname_prefix\":\"$(hostname | cut -d'.' -f1)\",\"haproxy_config_path\":\"/etc/haproxy/haproxy.cfg\",\"haproxy_bin_path\":\"/usr/sbin/haproxy\",\"stats_socket_path\":\"/var/run/haproxy/admin.sock\"}" 2>/dev/null)
|
|
|
|
# Validate API response
|
|
if [[ -z "$script_response" ]]; then
|
|
log "ERROR" "DAEMON: Empty API response, aborting upgrade"
|
|
continue
|
|
fi
|
|
|
|
if ! echo "$script_response" | grep -q '"script"'; then
|
|
log "ERROR" "DAEMON: Invalid API response format, aborting upgrade"
|
|
echo "Response: ${script_response:0:200}..."
|
|
continue
|
|
fi
|
|
|
|
# Extract script content with validation
|
|
echo "$script_response" | jq -r '.script' > "$temp_script" 2>/dev/null
|
|
|
|
# Validate downloaded script
|
|
if [[ ! -f "$temp_script" ]]; then
|
|
log "ERROR" "DAEMON: Failed to create temp script file, aborting upgrade"
|
|
continue
|
|
fi
|
|
|
|
if [[ ! -s "$temp_script" ]]; then
|
|
log "ERROR" "DAEMON: Downloaded script is empty, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
# Check if script is valid bash script
|
|
if ! head -1 "$temp_script" | grep -q "#!/bin/bash"; then
|
|
log "ERROR" "DAEMON: Downloaded script is not a valid bash script, aborting upgrade"
|
|
echo "First line: $(head -1 "$temp_script")"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
script_size=$(wc -c < "$temp_script")
|
|
if [[ $script_size -lt 10000 ]]; then
|
|
log "ERROR" "DAEMON: Downloaded script too small ($script_size bytes), aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Script validation passed ($script_size bytes)"
|
|
|
|
# Create backup before upgrade (ensure backup directory exists)
|
|
mkdir -p "/usr/local/share/haproxy-agent-backups" 2>/dev/null || true
|
|
backup_file="/usr/local/share/haproxy-agent-backups/haproxy-agent-backup-$(date +%Y%m%d-%H%M%S).sh"
|
|
if ! cp /usr/local/bin/haproxy-agent "$backup_file" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to create backup, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Backup created: $backup_file"
|
|
|
|
# Atomic script replacement
|
|
temp_target="/usr/local/bin/haproxy-agent.new"
|
|
if ! cp "$temp_script" "$temp_target" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to copy new script, aborting upgrade"
|
|
rm -f "$temp_script"
|
|
continue
|
|
fi
|
|
|
|
if ! chmod 755 "$temp_target" 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to set permissions, aborting upgrade"
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
# Final validation of target script
|
|
if ! head -1 "$temp_target" | grep -q "#!/bin/bash"; then
|
|
log "ERROR" "DAEMON: Target script corrupted, restoring backup"
|
|
cp "$backup_file" /usr/local/bin/haproxy-agent
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
# Atomic move
|
|
if ! mv "$temp_target" /usr/local/bin/haproxy-agent 2>/dev/null; then
|
|
log "ERROR" "DAEMON: Failed to replace script, restoring backup"
|
|
cp "$backup_file" /usr/local/bin/haproxy-agent 2>/dev/null
|
|
rm -f "$temp_script" "$temp_target"
|
|
continue
|
|
fi
|
|
|
|
log "INFO" "DAEMON: Script updated successfully"
|
|
|
|
# Notify completion
|
|
curl -k -s -X POST "$MANAGEMENT_URL/api/agents/$AGENT_NAME/upgrade-complete" \
|
|
-H "Content-Type: application/json" \
|
|
-H "X-API-Key: $CURRENT_AGENT_TOKEN" \
|
|
-d "{\"version\":\"$target_version\",\"status\":\"completed\"}" >/dev/null 2>&1
|
|
|
|
log "INFO" "DAEMON: Upgrade to $target_version completed - restarting..."
|
|
|
|
# Clean up
|
|
rm -f "$temp_script"
|
|
|
|
# Exit to allow service restart with new script
|
|
exit 0
|
|
else
|
|
# Debug: Log why upgrade was not performed
|
|
[[ "${DEBUG_MODE:-0}" == "1" ]] && log "DEBUG" "DAEMON: Upgrade not needed - should_upgrade: $should_upgrade, target_version: $target_version, current: {{AGENT_VERSION}}"
|
|
fi
|
|
fi
|
|
done
|
|
# CRITICAL FIX: Daemon loop should never exit
|
|
log "ERROR" "DAEMON: Daemon loop exited unexpectedly, restarting..."
|
|
exit 1
|
|
fi
|
|
|