mirror of
https://github.com/taylanbakircioglu/haproxy-openmanager.git
synced 2026-09-12 05:48:58 +00:00
ef26860df9
Until now the only record of what happened was `user_activity_logs`, which stores non-GET 2xx operations with no bodies. When something failed you could see that a counter went up, never what was sent or what came back. This adds one queryable timeline covering both directions: - inbound: every API call, including GETs and including 4xx/5xx, with the user, client IP, status, duration and — redacted, size-capped — the request and response bodies. - outbound: every HTTP call the backend makes, tagged with who it went to (ACME/Let's Encrypt, Cloudflare, GoDaddy, HAProxy stats, agents, the ACME diagnostics probe). Outbound rows inherit the inbound request's id, so one operator action and the CA/DNS calls it triggered read as a single trace: opening a failed "Request Certificate" shows the exact POST /acme/new-order and the CA's 429 underneath. Implementation notes: - Capture is a pure-ASGI middleware that TEES the request and response streams rather than draining them. `await request.body()` inside a BaseHTTPMiddleware would consume the receive channel and break the raw-body agent heartbeat handler. Registered last so it is outermost: it then sees the final client-visible response and seeds correlation_id_context before the error handler reads it. - Rows are written by a batching background writer with a bounded queue, so the request path never awaits the database and a saturated logger drops rows visibly (surfaced on the page) instead of blocking. Redaction runs on the writer, off the request coroutine. - Secrets never land: headers are an allowlist with Authorization/Cookie kept only as a presence marker; body keys and value shapes are redacted (passwords, tokens, api_token, API keys, private-key PEMs, JWTs); the ACME JWS request body is never stored, because a stored protected+signature pair is a replayable credential — a summary is logged instead; DNS-provider errors record only the exception type; the ACME HTTP-01 challenge endpoint is excluded so key_authorization is never captured. - Retention is operator-configurable in Settings -> Request Log: separate day counts for successful and failed rows (7 / 30) plus a hard row cap (500k), whichever is reached first. Pruned in batches under a Postgres advisory lock, with the day counts bound as parameters, never interpolated. - New permissions requestlog.read / requestlog.manage. super_admin and security_admin get both, operator gets read, viewer gets neither. Schema: one new table (request_logs) plus its settings seed, SCHEMA_VERSION 10 -> 11, auto-migrated. No existing table altered, no agent or rendered-config change. Kill switches: REQUEST_LOG_ENABLED=false (middleware never registered) or the `enabled` toggle in Settings. Tests: 245 new (7 backend files + 1 frontend), full suite 1655 backend + 17 frontend passing.
123 lines
5.4 KiB
Python
123 lines
5.4 KiB
Python
import logging
|
|
import aiohttp
|
|
import asyncio
|
|
from typing import List, Dict, Any
|
|
from database.connection import get_database_connection, close_database_connection
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
async def notify_agents_config_change(cluster_id: int, version_name: str) -> List[Dict[str, Any]]:
|
|
"""Notify agents in cluster about configuration changes"""
|
|
results = []
|
|
conn = None
|
|
|
|
try:
|
|
conn = await get_database_connection()
|
|
|
|
# DISABLED: Agent-pull architecture - agents poll backend, not vice versa
|
|
# This notification system is incorrect for our pull-based architecture
|
|
logger.info(f"🔔 AGENT NOTIFICATION: Skipping agent notification for cluster {cluster_id} (pull-based architecture)")
|
|
|
|
# Return success - agents will pull changes on their own
|
|
return [{'node': 'cluster', 'success': True, 'message': 'Agent pull architecture - no push notification needed', 'version': version_name}]
|
|
|
|
# Notify each agent
|
|
for agent in agents:
|
|
try:
|
|
agent_url = f"http://{agent['ip_address']}:8081" # Agent default port
|
|
|
|
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session:
|
|
payload = {
|
|
"cluster_id": cluster_id,
|
|
"version_name": version_name,
|
|
"action": "config_update"
|
|
}
|
|
|
|
# v1.11.0: instrumented so the code stays correct if the push
|
|
# architecture is ever reverted. Unreachable today — see the
|
|
# unconditional early return above.
|
|
from utils.http_instrumentation import outbound_span, TARGET_AGENT
|
|
|
|
push_url = f"{agent_url}/api/config/update"
|
|
async with outbound_span(
|
|
target=TARGET_AGENT, method="POST", url=push_url, request_body=payload
|
|
) as span:
|
|
async with session.post(push_url, json=payload) as response:
|
|
if response.status == 200:
|
|
span.set_response(response.status, getattr(response, "headers", None))
|
|
results.append({
|
|
'node': agent['name'],
|
|
'success': True,
|
|
'message': f'Configuration updated successfully',
|
|
'version': version_name
|
|
})
|
|
logger.info(f"✅ Agent {agent['name']} notified successfully")
|
|
else:
|
|
error_text = await response.text()
|
|
span.set_response(response.status, getattr(response, "headers", None), error_text)
|
|
results.append({
|
|
'node': agent['name'],
|
|
'success': False,
|
|
'error': f'HTTP {response.status}: {error_text}'
|
|
})
|
|
logger.error(f"❌ Agent {agent['name']} notification failed: {response.status}")
|
|
|
|
except asyncio.TimeoutError:
|
|
results.append({
|
|
'node': agent['name'],
|
|
'success': False,
|
|
'error': 'Connection timeout'
|
|
})
|
|
logger.error(f"❌ Agent {agent['name']} timeout")
|
|
|
|
except Exception as e:
|
|
results.append({
|
|
'node': agent['name'],
|
|
'success': False,
|
|
'error': str(e)
|
|
})
|
|
logger.error(f"❌ Agent {agent['name']} error: {e}")
|
|
|
|
return results
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to notify agents for cluster {cluster_id}: {e}")
|
|
return [{'node': 'cluster', 'success': False, 'error': str(e)}]
|
|
|
|
finally:
|
|
if conn:
|
|
await close_database_connection(conn)
|
|
|
|
async def get_cluster_agents_status(cluster_id: int) -> Dict[str, Any]:
|
|
"""Get status of all agents in a cluster"""
|
|
conn = None
|
|
try:
|
|
conn = await get_database_connection()
|
|
|
|
# Get agent count by status for this cluster
|
|
stats = await conn.fetchrow("""
|
|
SELECT
|
|
COUNT(*) as total_agents,
|
|
COUNT(CASE WHEN a.status = 'online' THEN 1 END) as online_agents,
|
|
COUNT(CASE WHEN a.status = 'offline' THEN 1 END) as offline_agents,
|
|
COUNT(CASE WHEN a.status = 'warning' THEN 1 END) as warning_agents
|
|
FROM agents a
|
|
JOIN haproxy_cluster_pools p ON a.pool_id = p.id
|
|
JOIN haproxy_clusters c ON c.pool_id = p.id
|
|
WHERE c.id = $1
|
|
""", cluster_id)
|
|
|
|
return {
|
|
"total": stats["total_agents"] or 0,
|
|
"online": stats["online_agents"] or 0,
|
|
"offline": stats["offline_agents"] or 0,
|
|
"warning": stats["warning_agents"] or 0
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get cluster agents status: {e}")
|
|
return {"total": 0, "online": 0, "offline": 0, "warning": 0}
|
|
|
|
finally:
|
|
if conn:
|
|
await close_database_connection(conn) |