From 9e2ea047771c48ad979435c0901d79aab52e3979 Mon Sep 17 00:00:00 2001 From: taylanbakircioglu Date: Fri, 10 Jul 2026 18:34:36 +0300 Subject: [PATCH] feat(haproxy): preserve SPOE filter + frontend log-format on import/edit (v1.8.8, Issue #38) Bulk import / manual edit silently dropped `filter spoe engine ...` (Coraza WAF) and frontend `log-format` because the parser recognised only a fixed directive set. The regenerated config then missed the SPOE engine, so HAProxy failed with "unable to find SPOE engine 'coraza' used by the send-spoe-group". - parser: capture `filter` + `log-format`/`log-format-sd` into new ParsedFrontend fields - db: additive nullable `log_format` + `filters` TEXT columns on frontends (SCHEMA_VERSION 8->9) - generator: new `filter` bucket flushed before http-request rules so `filter` precedes `send-spoe-group`; `log-format` kept in prelude - bulk import: preview dict, change-detection, persist (create + merge-update); cluster-aware SPOE pre-flight advisories (missing-filter + host-prerequisite) surfaced in the UI - manual CRUD: full round-trip (get/create/update) incl. React form fields (no null-wipe) - reject/rollback: restore the new columns; restore path + wizard helper kept in parity - backend `option spop-check` recognised (suppresses spurious warning for coraza-spoa) - tests: test_spoe_filter_import.py; full suite green (1079 passed) --- backend/database/migrations.py | 16 +- backend/models/frontend.py | 5 + backend/routers/cluster.py | 21 +- backend/routers/config.py | 84 +++++++- backend/routers/frontend.py | 29 ++- backend/services/frontend_service.py | 6 +- backend/services/haproxy_config.py | 21 +- backend/tests/test_site_wizard_round11.py | 6 +- backend/tests/test_spoe_filter_import.py | 203 ++++++++++++++++++ backend/utils/entity_snapshot.py | 7 +- backend/utils/haproxy_config_parser.py | 36 +++- backend/version.json | 6 +- frontend/package.json | 2 +- frontend/src/components/BulkConfigImport.js | 49 ++++- frontend/src/components/FrontendManagement.js | 40 +++- 15 files changed, 487 insertions(+), 44 deletions(-) create mode 100644 backend/tests/test_spoe_filter_import.py diff --git a/backend/database/migrations.py b/backend/database/migrations.py index ba66a94..c37dfaf 100644 --- a/backend/database/migrations.py +++ b/backend/database/migrations.py @@ -194,7 +194,14 @@ async def ensure_agents_table(): 'use_backend_rules': "ALTER TABLE frontends ADD COLUMN use_backend_rules JSONB DEFAULT '[]'::jsonb;", 'request_headers': "ALTER TABLE frontends ADD COLUMN request_headers TEXT;", 'response_headers': "ALTER TABLE frontends ADD COLUMN response_headers TEXT;", - 'maxconn': "ALTER TABLE frontends ADD COLUMN maxconn INTEGER;" + 'maxconn': "ALTER TABLE frontends ADD COLUMN maxconn INTEGER;", + # Issue #38: SPOE filter directives (e.g. Coraza WAF) and frontend + # log-format were silently dropped on bulk-import / manual edit + # because the parser recognised only a fixed set of directives. + # These nullable TEXT columns persist them verbatim (multi-line for + # `filters`), mirroring the request_headers/options passthrough. + 'log_format': "ALTER TABLE frontends ADD COLUMN log_format TEXT;", + 'filters': "ALTER TABLE frontends ADD COLUMN filters TEXT;" } for col, query in frontend_columns.items(): @@ -1741,7 +1748,12 @@ async def ensure_agent_activity_logs_table(): # columns on letsencrypt_accounts/letsencrypt_orders/acme_challenges and the brand-new # letsencrypt_account_dns_credentials table (ensure_letsencrypt_dns_credentials step). # All additive + idempotent; default challenge_type 'http-01' keeps existing flows byte-identical. -SCHEMA_VERSION = 8 +# v1.8.8 (Issue #38 — SPOE filter + frontend log-format): bumped 8 -> 9 for the additive +# `log_format` + `filters` TEXT columns on `frontends` (frontend_columns loop). Without this +# bump, already-deployed databases (version >= 8) skip the whole migration run and never gain +# the columns, so the frontends SELECT/INSERT would fail. Additive + idempotent + nullable; +# existing rows stay NULL and render byte-identical. +SCHEMA_VERSION = 9 async def run_all_migrations(): diff --git a/backend/models/frontend.py b/backend/models/frontend.py index 048c95d..094cafa 100644 --- a/backend/models/frontend.py +++ b/backend/models/frontend.py @@ -85,6 +85,11 @@ class FrontendConfig(BaseModel): response_headers: Optional[str] = None options: Optional[str] = None tcp_request_rules: Optional[str] = None + # Issue #38: SPOE filter directives (Coraza WAF etc.) + frontend log-format. + # Passthrough TEXT (no validator) — SPOE `filter ... config ` legitimately + # references an operator-managed file, so the ACL `-f` guard must NOT apply here. + log_format: Optional[str] = None + filters: Optional[str] = None timeout_client: Optional[int] = None timeout_http_request: Optional[int] = None rate_limit: Optional[int] = None diff --git a/backend/routers/cluster.py b/backend/routers/cluster.py index e36157a..f149264 100644 --- a/backend/routers/cluster.py +++ b/backend/routers/cluster.py @@ -3696,17 +3696,19 @@ async def confirm_restore_config_version( # UPDATE existing frontend (ALL 8 parsed fields) # CRITICAL FIX: Include maxconn and timeout_client so UI shows restored values await conn.execute(""" - UPDATE frontends - SET bind_address = $1, bind_port = $2, default_backend = $3, + UPDATE frontends + SET bind_address = $1, bind_port = $2, default_backend = $3, mode = $4, ssl_enabled = $5, ssl_port = $6, maxconn = $7, timeout_client = $8, + log_format = $11, filters = $12, updated_at = CURRENT_TIMESTAMP, last_config_status = 'PENDING' WHERE id = $9 AND cluster_id = $10 - """, + """, parsed_fe.bind_address, parsed_fe.bind_port, parsed_fe.default_backend, parsed_fe.mode, parsed_fe.ssl_enabled, parsed_fe.ssl_port, parsed_fe.maxconn, parsed_fe.timeout_client, - fe_id, cluster_id + fe_id, cluster_id, + parsed_fe.log_format, parsed_fe.filters # Issue #38 ) changes_summary["frontends_updated"] += 1 logger.info(f"RESTORE: Updated frontend '{parsed_fe.name}' (SSL: {parsed_fe.ssl_enabled}, maxconn: {parsed_fe.maxconn})") @@ -3714,16 +3716,17 @@ async def confirm_restore_config_version( # CREATE new frontend (ALL 8 parsed fields) # CRITICAL FIX: Include maxconn and timeout_client so UI shows restored values await conn.execute(""" - INSERT INTO frontends + INSERT INTO frontends (name, bind_address, bind_port, default_backend, mode, ssl_enabled, ssl_port, maxconn, timeout_client, - cluster_id, is_active, last_config_status, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, TRUE, 'PENDING', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) - """, + cluster_id, log_format, filters, is_active, last_config_status, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, TRUE, 'PENDING', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + """, parsed_fe.name, parsed_fe.bind_address, parsed_fe.bind_port, parsed_fe.default_backend, parsed_fe.mode, parsed_fe.ssl_enabled, parsed_fe.ssl_port, parsed_fe.maxconn, parsed_fe.timeout_client, - cluster_id + cluster_id, + parsed_fe.log_format, parsed_fe.filters # Issue #38 ) changes_summary["frontends_created"] += 1 logger.info(f"RESTORE: Created frontend '{parsed_fe.name}' (SSL: {parsed_fe.ssl_enabled}, maxconn: {parsed_fe.maxconn})") diff --git a/backend/routers/config.py b/backend/routers/config.py index a833612..c7bcf55 100644 --- a/backend/routers/config.py +++ b/backend/routers/config.py @@ -855,6 +855,9 @@ async def parse_bulk_config( "response_headers": frontend.response_headers, "options": frontend.options, "tcp_request_rules": frontend.tcp_request_rules, + # Issue #38: SPOE filters + frontend log-format + "log_format": frontend.log_format, + "filters": frontend.filters, # CRITICAL: SSL Advanced Options (parsed from bind directive) "ssl_alpn": frontend.ssl_alpn, "ssl_npn": frontend.ssl_npn, @@ -1089,7 +1092,50 @@ async def parse_bulk_config( # Add auto-assignment info at the beginning enhanced_warnings = ssl_auto_assign_info + enhanced_warnings - + + # ───────────────────────────────────────────────────────────────── + # Issue #38: SPOE pre-flight advisories. Surface, at preview time, the + # SPOE configurations that would FAIL HAProxy's `haproxy -c` at apply so + # the operator sees them BEFORE importing. Cluster-aware: the referenced + # SPOE engine config (e.g. coraza.cfg) is a sibling of the cluster's + # haproxy_config_path, which HAProxy OpenManager does not provision. + # ───────────────────────────────────────────────────────────────── + try: + _cfg_path = await conn.fetchval( + "SELECT haproxy_config_path FROM haproxy_clusters WHERE id = $1", + request.cluster_id, + ) or "/etc/haproxy/haproxy.cfg" + _cfg_dir = _cfg_path.rsplit("/", 1)[0] or "/etc/haproxy" + for _fe in frontends_data: + _rh = _fe.get("request_headers") or "" + _filters = _fe.get("filters") or "" + # engines declared by `filter spoe engine config ` + _declared_engines = set(re.findall( + r"filter\s+spoe\s+engine\s+(\S+)", _filters, re.IGNORECASE)) + # engines referenced by `... send-spoe-group ` + _used_engines = set(re.findall( + r"send-spoe-group\s+(\S+)", _rh, re.IGNORECASE)) + _missing = _used_engines - _declared_engines + if _missing: + enhanced_warnings.append( + f"⚠️ Frontend '{_fe['name']}': 'send-spoe-group' references SPOE " + f"engine(s) {', '.join(sorted(_missing))} but no matching " + f"'filter spoe engine ...' line was found. HAProxy will " + f"reject this at apply with \"unable to find SPOE engine\". Add the " + f"filter line to this frontend." + ) + for _path in re.findall( + r"filter\s+spoe\s+engine\s+\S+\s+config\s+(\S+)", + _filters, re.IGNORECASE): + enhanced_warnings.append( + f"ℹ️ Frontend '{_fe['name']}': SPOE engine config '{_path}' and its " + f"SPOA backend must exist on the HAProxy host (cluster config dir: " + f"{_cfg_dir}). HAProxy OpenManager preserves the filter directive but " + f"does not provision these files; otherwise 'haproxy -c' fails at apply." + ) + except Exception as _spoe_adv_err: + logger.warning(f"SPOE advisory generation skipped: {_spoe_adv_err}") + # BULK IMPORT MVP: Check existing entities for UPSERT detection # Mark each entity as new or update for UI display # CRITICAL: Only mark as UPDATE if there are actual field changes @@ -1150,7 +1196,17 @@ async def parse_bulk_config( if frontend.get("tcp_request_rules") and frontend["tcp_request_rules"] != existing["tcp_request_rules"]: has_changes = True changes["tcp_request_rules"] = {"old": existing["tcp_request_rules"], "new": frontend["tcp_request_rules"]} - + # Issue #38: SPOE filters + log-format change detection. REQUIRED for + # persistence (not just display): without it, an import that only adds + # a `filter`/`log-format` to an existing frontend would be flagged + # "no change" and the directive would never be written to the DB. + if frontend.get("log_format") and frontend["log_format"] != existing.get("log_format"): + has_changes = True + changes["log_format"] = {"old": existing.get("log_format"), "new": frontend["log_format"]} + if frontend.get("filters") and frontend["filters"] != existing.get("filters"): + has_changes = True + changes["filters"] = {"old": existing.get("filters"), "new": frontend["filters"]} + # CRITICAL: SSL Advanced Options change detection if frontend.get("ssl_alpn") is not None and frontend.get("ssl_alpn") != existing.get("ssl_alpn"): has_changes = True @@ -2094,7 +2150,18 @@ async def bulk_create_entities( update_fields.append(f"options = ${param_index}") update_values.append(frontend_data["options"]) param_index += 1 - + + # Issue #38: SPOE filters + frontend log-format (merge strategy) + if frontend_data.get("log_format") and frontend_data["log_format"] != existing_full.get("log_format"): + update_fields.append(f"log_format = ${param_index}") + update_values.append(frontend_data["log_format"]) + param_index += 1 + + if frontend_data.get("filters") and frontend_data["filters"] != existing_full.get("filters"): + update_fields.append(f"filters = ${param_index}") + update_values.append(frontend_data["filters"]) + param_index += 1 + # CRITICAL FIX: Update SSL advanced options (alpn, npn, ciphers, etc.) # These are parsed from bind directive and should be preserved in database if "ssl_alpn" in frontend_data and frontend_data.get("ssl_alpn") != existing_full.get("ssl_alpn"): @@ -2214,9 +2281,10 @@ async def bulk_create_entities( timeout_client, timeout_http_request, maxconn, request_headers, response_headers, tcp_request_rules, options, rate_limit, compression, log_separate, monitor_uri, - cluster_id, acl_rules, use_backend_rules, redirect_rules, updated_at - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, CURRENT_TIMESTAMP) + cluster_id, acl_rules, use_backend_rules, redirect_rules, + log_format, filters, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, CURRENT_TIMESTAMP) RETURNING id """, frontend_data["name"], @@ -2257,7 +2325,9 @@ async def bulk_create_entities( request.cluster_id, json.dumps(frontend_data.get("acl_rules", [])), # acl_rules json.dumps(frontend_data.get("use_backend_rules", [])), # use_backend_rules - json.dumps([]) # redirect_rules + json.dumps([]), # redirect_rules + frontend_data.get("log_format"), # Issue #38 + frontend_data.get("filters") # Issue #38 ) created_entities["frontends"].append({ diff --git a/backend/routers/frontend.py b/backend/routers/frontend.py index eaae35e..f113b50 100644 --- a/backend/routers/frontend.py +++ b/backend/routers/frontend.py @@ -408,6 +408,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -424,6 +425,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -453,6 +455,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -465,6 +468,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -480,6 +484,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -492,6 +497,7 @@ async def get_frontends( ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni, acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, + log_format, filters, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, maxconn, is_active, created_at, updated_at, cluster_id, last_config_status @@ -601,6 +607,8 @@ async def get_frontends( "response_headers": f.get("response_headers"), "options": f.get("options"), "tcp_request_rules": f.get("tcp_request_rules"), + "log_format": f.get("log_format"), # Issue #38 + "filters": f.get("filters"), # Issue #38 "timeout_client": f.get("timeout_client"), "timeout_http_request": f.get("timeout_http_request"), "rate_limit": f.get("rate_limit"), @@ -735,18 +743,18 @@ async def create_frontend(frontend: FrontendConfig, request: Request, authorizat acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, - cluster_id, maxconn, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, CURRENT_TIMESTAMP) + cluster_id, maxconn, log_format, filters, updated_at + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, CURRENT_TIMESTAMP) RETURNING id - """, frontend.name, frontend.bind_address, frontend.bind_port, + """, frontend.name, frontend.bind_address, frontend.bind_port, frontend.default_backend, frontend.mode, frontend.ssl_enabled, frontend.ssl_certificate_id, ssl_cert_ids_json, frontend.ssl_port, frontend.ssl_cert_path, frontend.ssl_cert, frontend.ssl_verify, - frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites, + frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites, frontend.ssl_min_ver, frontend.ssl_max_ver, frontend.ssl_strict_sni, json.dumps(frontend.acl_rules or []), json.dumps(frontend.redirect_rules or []), json.dumps(frontend.use_backend_rules or []), frontend.request_headers, frontend.response_headers, filtered_options, frontend.tcp_request_rules, frontend.timeout_client, frontend.timeout_http_request, frontend.rate_limit, frontend.compression, frontend.log_separate, frontend.monitor_uri, - frontend.cluster_id, frontend.maxconn) + frontend.cluster_id, frontend.maxconn, frontend.log_format, frontend.filters) # If cluster_id provided, create new config version for agents sync_results = [] @@ -1060,9 +1068,10 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R acl_rules = $20, redirect_rules = $21, use_backend_rules = $22, request_headers = $23, response_headers = $24, options = $25, tcp_request_rules = $26, timeout_client = $27, timeout_http_request = $28, rate_limit = $29, compression = $30, log_separate = $31, monitor_uri = $32, - cluster_id = $33, maxconn = $34, updated_at = CURRENT_TIMESTAMP - WHERE id = $35 - """, frontend.name, frontend.bind_address, frontend.bind_port, + cluster_id = $33, maxconn = $34, log_format = $35, filters = $36, + updated_at = CURRENT_TIMESTAMP + WHERE id = $37 + """, frontend.name, frontend.bind_address, frontend.bind_port, frontend.default_backend, frontend.mode, ssl_enabled, ssl_certificate_id, ssl_cert_ids_json, ssl_port, ssl_cert_path, ssl_cert, ssl_verify, frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites, @@ -1070,7 +1079,7 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R json.dumps(frontend.acl_rules or []), json.dumps(frontend.redirect_rules or []), json.dumps(frontend.use_backend_rules or []), frontend.request_headers, frontend.response_headers, filtered_options, frontend.tcp_request_rules, frontend.timeout_client, frontend.timeout_http_request, frontend.rate_limit, frontend.compression, frontend.log_separate, frontend.monitor_uri, - frontend.cluster_id, frontend.maxconn, frontend_id) + frontend.cluster_id, frontend.maxconn, frontend.log_format, frontend.filters, frontend_id) # Debug: Check what was actually saved updated_frontend = await conn.fetchrow(""" @@ -1124,6 +1133,8 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R "response_headers": frontend.response_headers, "options": filtered_options, "tcp_request_rules": frontend.tcp_request_rules, + "log_format": frontend.log_format, # Issue #38 + "filters": frontend.filters, # Issue #38 "timeout_client": frontend.timeout_client, "timeout_http_request": frontend.timeout_http_request, "rate_limit": frontend.rate_limit, diff --git a/backend/services/frontend_service.py b/backend/services/frontend_service.py index f0f7a56..97b5c28 100644 --- a/backend/services/frontend_service.py +++ b/backend/services/frontend_service.py @@ -56,14 +56,14 @@ async def create_frontend_row( acl_rules, redirect_rules, use_backend_rules, request_headers, response_headers, options, tcp_request_rules, timeout_client, timeout_http_request, rate_limit, compression, log_separate, monitor_uri, - cluster_id, maxconn, updated_at + cluster_id, maxconn, log_format, filters, updated_at ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, - $33, $34, CURRENT_TIMESTAMP + $33, $34, $35, $36, CURRENT_TIMESTAMP ) RETURNING id """, @@ -101,6 +101,8 @@ async def create_frontend_row( getattr(payload, "monitor_uri", None), cluster_id, getattr(payload, "maxconn", None), + getattr(payload, "log_format", None), # Issue #38 + getattr(payload, "filters", None), # Issue #38 ) if mark_pending: diff --git a/backend/services/haproxy_config.py b/backend/services/haproxy_config.py index ecdf232..0e62fd7 100644 --- a/backend/services/haproxy_config.py +++ b/backend/services/haproxy_config.py @@ -393,6 +393,12 @@ def _categorize_haproxy_directive(line: str) -> str: return "prelude" if s.startswith("acl "): return "acl" + # Issue #38: SPOE (and other) `filter` directives must be declared BEFORE + # the `http-request send-spoe-group` rules that use them, otherwise HAProxy + # fails with "unable to find SPOE engine". Own bucket, flushed right after + # `prelude` and before tcp_req/acl/http_req (see flush order below). + if s.startswith("filter "): + return "filter" if s.startswith("stick-table") or s.startswith("stick "): return "stick" if s.startswith("tcp-request"): @@ -414,6 +420,7 @@ def _categorize_haproxy_directive(line: str) -> str: or s.startswith("compression ") or s.startswith("monitor-uri") or s.startswith("log ") + or s.startswith("log-format") # Issue #38: log-format / log-format-sd or s.startswith("description ") or s.startswith("disabled") or s.startswith("enabled") @@ -903,7 +910,7 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An # "stick-table already declared"). # ───────────────────────────────────────────────────────────────── _fe_buckets: Dict[str, List[str]] = { - "prelude": [], "stick": [], "tcp_req": [], + "prelude": [], "filter": [], "stick": [], "tcp_req": [], "acl": [], "http_req": [], "http_resp": [], "redirect": [], "use_be": [], "default_be": [], } @@ -996,6 +1003,17 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An if line_stripped and line_stripped not in ('[]', '{}', 'null', 'None'): _emit_fe(f" {line_stripped}") + # Issue #38: emit frontend log-format and SPOE (etc.) filter directives. + # `log_format` routes to the `prelude` bucket, `filters` to the `filter` + # bucket (both via _emit_fe → _categorize_haproxy_directive), guaranteeing + # `filter ...` is rendered before the `http-request send-spoe-group` rules. + for _fld in ('log_format', 'filters'): + if frontend.get(_fld): + for line in frontend[_fld].split('\n'): + line_stripped = line.strip() + if line_stripped and line_stripped not in ('[]', '{}', 'null', 'None'): + _emit_fe(f" {line_stripped}") + # CRITICAL: Validate frontend-backend mode compatibility if frontend.get('default_backend'): default_backend_name = frontend['default_backend'].strip() if frontend['default_backend'] else '' @@ -1223,6 +1241,7 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An # ───────────────────────────────────────────────────────────── for _bucket_key in ( "prelude", + "filter", "stick", "tcp_req", "acl", diff --git a/backend/tests/test_site_wizard_round11.py b/backend/tests/test_site_wizard_round11.py index f04c4a3..37a14ef 100644 --- a/backend/tests/test_site_wizard_round11.py +++ b/backend/tests/test_site_wizard_round11.py @@ -276,14 +276,16 @@ def test_categorize_routes_directives_correctly(): def test_emit_buckets_flushed_in_canonical_order(): """The flush block at end of frontend processing must list buckets - in: prelude → stick → tcp_req → acl → http_req → http_resp → + in: prelude → filter → stick → tcp_req → acl → http_req → http_resp → redirect → use_be → default_be. Pre-fix `http-request` rules interleaved with `use_backend` rules in source order, producing - HAProxy parser warnings.""" + HAProxy parser warnings. (Issue #38 added the `filter` bucket, flushed + right after `prelude` so SPOE `filter` lines precede `send-spoe-group`.)""" src = _gen_src() flush_match = re.search( r'for\s+_bucket_key\s+in\s+\(\s*' r'"prelude"\s*,\s*' + r'"filter"\s*,\s*' r'"stick"\s*,\s*' r'"tcp_req"\s*,\s*' r'"acl"\s*,\s*' diff --git a/backend/tests/test_spoe_filter_import.py b/backend/tests/test_spoe_filter_import.py new file mode 100644 index 0000000..2f5b57d --- /dev/null +++ b/backend/tests/test_spoe_filter_import.py @@ -0,0 +1,203 @@ +""" +Issue #38 regression tests: HAProxy SPOE `filter` + frontend `log-format` support. + +Bug: the bulk-config parser recognised only a fixed set of frontend directives, +so `filter spoe engine coraza config ...` and `log-format ...` were silently +dropped on import / manual edit. This regenerated a config missing the SPOE +engine definition, so HAProxy failed with +"unable to find SPOE engine 'coraza' used by the send-spoe-group 'coraza-req'". + +These tests verify the end-to-end fix without requiring a database: +1. parser captures `filter` + `log-format` into the new ParsedFrontend fields; +2. `http-request send-spoe-group` is still preserved (regression guard); +3. the generator's directive categoriser + bucket flush order emit `filter` + BEFORE the `http-request send-spoe-group` rules and keep `log-format`; +4. reject/rollback restores the new columns; +5. a non-SPOE frontend is completely unaffected (zero-impact). +""" +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from utils.haproxy_config_parser import parse_haproxy_config, ParsedFrontend +from services.haproxy_config import _categorize_haproxy_directive +from models.frontend import FrontendConfig + + +# The exact frontend/backend config reported in Issue #38 (Coraza-SPOA). +ISSUE_38_CONFIG = r""" +frontend web-frontend + bind *:8073 + mode http + log-format "%ci:%cp\ [%t]\ %ft\ %b/%s\ %ST\ %B\ %{+Q}r\ %[var(txn.coraza.id)]\ waf-hit:\ %[var(txn.coraza.fail)]" + filter spoe engine coraza config /etc/haproxy/coraza.cfg + http-request set-var(txn.coraza.app) str(haproxy_waf) + http-request send-spoe-group coraza coraza-req + http-request deny if { var(txn.coraza.fail) -m int eq 1 } + default_backend web-backend + +backend web-backend + balance roundrobin + mode http + server server1 192.168.1.10:443 weight 100 ssl verify none + +backend coraza-spoa + mode tcp + option spop-check + server coraza_spoa 192.168.12.21:9000 +""" + + +def _get_frontend(parse_result, name): + for fe in parse_result.frontends: + if fe.name == name: + return fe + return None + + +class TestParserCapturesSpoe: + def test_filter_and_log_format_captured(self): + result = parse_haproxy_config(ISSUE_38_CONFIG) + fe = _get_frontend(result, "web-frontend") + assert fe is not None, "web-frontend should be parsed and kept" + assert fe.filters is not None + assert "filter spoe engine coraza config /etc/haproxy/coraza.cfg" in fe.filters + assert fe.log_format is not None + assert fe.log_format.startswith("log-format") + # the escaped/quoted format string must be preserved verbatim + assert "%[var(txn.coraza.fail)]" in fe.log_format + + def test_send_spoe_group_still_preserved(self): + # Regression guard: http-request rules (incl. send-spoe-group) must + # still be collected into request_headers as before. + result = parse_haproxy_config(ISSUE_38_CONFIG) + fe = _get_frontend(result, "web-frontend") + assert fe.request_headers is not None + assert "send-spoe-group coraza coraza-req" in fe.request_headers + + def test_multiple_filters_preserved_in_order(self): + cfg = """ +frontend f1 + bind *:80 + mode http + filter compression + filter spoe engine coraza config /etc/haproxy/coraza.cfg + default_backend b1 + +backend b1 + mode http + server s1 10.0.0.1:80 +""" + fe = _get_frontend(parse_haproxy_config(cfg), "f1") + lines = fe.filters.split("\n") + assert lines == [ + "filter compression", + "filter spoe engine coraza config /etc/haproxy/coraza.cfg", + ] + + def test_log_format_sd_variant_captured(self): + cfg = """ +frontend f1 + bind *:80 + mode http + log-format-sd "[exampleSDID@1234 field=value]" + default_backend b1 + +backend b1 + mode http + server s1 10.0.0.1:80 +""" + fe = _get_frontend(parse_haproxy_config(cfg), "f1") + assert fe.log_format is not None + assert fe.log_format.startswith("log-format-sd") + + +class TestGeneratorOrderingContract: + """The generator routes directives into ordered buckets. Verify SPOE + correctness at the (pure) categoriser + documented flush-order level.""" + + def test_filter_routes_to_filter_bucket(self): + assert _categorize_haproxy_directive(" filter spoe engine coraza config /x.cfg") == "filter" + + def test_send_spoe_group_routes_to_http_req(self): + assert _categorize_haproxy_directive(" http-request send-spoe-group coraza coraza-req") == "http_req" + + def test_log_format_routes_to_prelude(self): + assert _categorize_haproxy_directive(' log-format "%ci:%cp"') == "prelude" + assert _categorize_haproxy_directive(' log-format-sd "[x]"') == "prelude" + + def test_flush_order_places_filter_before_http_req(self): + # The bucket flush order is the single source of truth for emission + # ordering. Assert `filter` is flushed before `http_req` (and after + # `prelude`), guaranteeing `filter ...` renders before + # `http-request send-spoe-group ...`. + src = _read_source("services/haproxy_config.py") + m = re.search(r"for _bucket_key in \((.*?)\):", src, re.DOTALL) + assert m, "bucket flush loop not found" + order = re.findall(r'"(\w+)"', m.group(1)) + assert "filter" in order, "new 'filter' bucket missing from flush order" + assert order.index("prelude") < order.index("filter") < order.index("http_req") + + +class TestModelAndRollback: + def test_model_has_passthrough_fields(self): + fc = FrontendConfig( + name="f", bind_port=80, + filters="filter spoe engine coraza config /etc/haproxy/coraza.cfg", + log_format='log-format "%ci"', + ) + assert fc.filters.startswith("filter spoe") + assert fc.log_format.startswith("log-format") + + def test_dataclass_defaults_none(self): + fe = ParsedFrontend(name="f") + assert fe.filters is None + assert fe.log_format is None + + def test_rollback_restores_new_columns(self): + # Reject/rollback of a frontend UPDATE must restore the new columns, + # otherwise the rejected (new) filters/log_format would persist. + src = _read_source("utils/entity_snapshot.py") + assert "log_format = $" in src + assert "filters = $" in src + assert "old_values.get('log_format')" in src + assert "old_values.get('filters')" in src + + +class TestZeroImpact: + def test_non_spoe_frontend_unaffected(self): + cfg = """ +frontend plain + bind *:80 + mode http + option httplog + default_backend b1 + +backend b1 + mode http + server s1 10.0.0.1:80 +""" + fe = _get_frontend(parse_haproxy_config(cfg), "plain") + # No filter / log-format present → new fields stay None (no behaviour change) + assert fe.filters is None + assert fe.log_format is None + + def test_spop_check_backend_roundtrips_without_warning(self): + result = parse_haproxy_config(ISSUE_38_CONFIG) + be = next((b for b in result.backends if b.name == "coraza-spoa"), None) + assert be is not None, "coraza-spoa backend should import" + assert be.mode == "tcp" + assert be.options and "option spop-check" in be.options + # spop-check is now a known option → no spurious 'unknown option' warning + assert not any( + "coraza-spoa" in w and "spop-check" in w and "Unknown" in w + for w in result.warnings + ) + + +def _read_source(relpath): + base = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(base, relpath), "r", encoding="utf-8") as fh: + return fh.read() diff --git a/backend/utils/entity_snapshot.py b/backend/utils/entity_snapshot.py index 3ae4bcf..80e5152 100644 --- a/backend/utils/entity_snapshot.py +++ b/backend/utils/entity_snapshot.py @@ -329,9 +329,10 @@ async def _rollback_update( tcp_request_rules = $26, timeout_client = $27, timeout_http_request = $28, rate_limit = $29, compression = $30, log_separate = $31, monitor_uri = $32, maxconn = $33, - cluster_id = $34, is_active = $35, last_config_status = $36, + cluster_id = $34, is_active = $35, last_config_status = $36, + log_format = $37, filters = $38, updated_at = CURRENT_TIMESTAMP - WHERE id = $37 + WHERE id = $39 """, old_values.get('name'), old_values.get('bind_address'), @@ -369,6 +370,8 @@ async def _rollback_update( old_values.get('cluster_id'), old_values.get('is_active'), old_values.get('last_config_status'), + old_values.get('log_format'), # Issue #38 + old_values.get('filters'), # Issue #38 entity_id ) diff --git a/backend/utils/haproxy_config_parser.py b/backend/utils/haproxy_config_parser.py index fe15bb0..fbfe904 100644 --- a/backend/utils/haproxy_config_parser.py +++ b/backend/utils/haproxy_config_parser.py @@ -105,6 +105,11 @@ class ParsedFrontend: response_headers: Optional[str] = None options: Optional[str] = None # HAProxy frontend options (option httplog, option forwardfor, etc.) tcp_request_rules: Optional[str] = None # TCP request directives (for TCP mode) + # Issue #38: SPOE (and other) filter directives + frontend log-format. + # Stored as full directive lines; `filters` is newline-joined to preserve + # ordering when multiple `filter ...` lines exist. + log_format: Optional[str] = None # `log-format` / `log-format-sd` line(s) + filters: Optional[str] = None # `filter ...` line(s), e.g. `filter spoe engine coraza config ...` @dataclass @@ -256,8 +261,23 @@ class HAProxyConfigParser: acl_rules_list = [] use_backend_rules_list = [] tcp_request_rules_list = [] + filters_list = [] + log_format_list = [] for line in lines: + # Issue #38: capture `filter ...` (SPOE/Coraza etc.) and + # `log-format`/`log-format-sd` directives. Pre-fix these matched + # no branch below and were silently dropped, so an imported SPOE + # config lost `filter spoe engine coraza ...` (→ HAProxy fatal + # "unable to find SPOE engine") and the frontend log-format. + # `continue` isolates them from the header/option handling below. + if line.startswith('filter '): + filters_list.append(line.strip()) + continue + if re.match(r'^log-format(-sd)?\s', line, re.IGNORECASE): + log_format_list.append(line.strip()) + continue + # Parse bind directive # IMPORTANT: Handle multiple bind lines correctly # Example: bind *:1002 (HTTP) and bind *:443 ssl (HTTPS) @@ -540,6 +560,13 @@ class HAProxyConfigParser: if tcp_request_rules_list: frontend.tcp_request_rules = '\n'.join(tcp_request_rules_list) + # Issue #38: assign captured SPOE filters + log-format + if filters_list: + frontend.filters = '\n'.join(filters_list) + + if log_format_list: + frontend.log_format = '\n'.join(log_format_list) + self.frontends.append(frontend) logger.info(f"Parsed frontend: {name} -> {frontend.default_backend}") @@ -666,9 +693,14 @@ class HAProxyConfigParser: 'transparent', 'abortonclose', 'allbackups', 'checkcache', 'clitcpka', 'srvtcpka', 'http-no-delay', 'socket-stats', 'tcp-smart-accept', 'tcp-smart-connect', 'independant-streams', 'log-separate-errors', - 'log-health-checks', 'accept-invalid-http-request', 'accept-invalid-http-response' + 'log-health-checks', 'accept-invalid-http-request', 'accept-invalid-http-response', + # Issue #38: SPOP health check for SPOE agent backends + # (e.g. coraza-spoa). Already collected below regardless, but + # listing it suppresses the spurious "unknown option" warning + # for the exact SPOE use-case. + 'spop-check' ] - + if option_name not in valid_options: # Unknown/invalid option - add warning but still collect it self.warnings.append( diff --git a/backend/version.json b/backend/version.json index 95e61d9..682865e 100644 --- a/backend/version.json +++ b/backend/version.json @@ -1,5 +1,5 @@ { - "version": "1.8.7", - "releaseName": "Version reporting single-source fix", - "releaseDate": "2026-07-09" + "version": "1.8.8", + "releaseName": "SPOE filter + frontend log-format support (Issue #38)", + "releaseDate": "2026-07-10" } diff --git a/frontend/package.json b/frontend/package.json index be31c5a..40e4b1b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "haproxy-openmanager-frontend", - "version": "1.8.7", + "version": "1.8.8", "description": "HAProxy Load Balancer Management UI", "license": "AGPL-3.0-or-later", "dependencies": { diff --git a/frontend/src/components/BulkConfigImport.js b/frontend/src/components/BulkConfigImport.js index c330c27..7db5740 100644 --- a/frontend/src/components/BulkConfigImport.js +++ b/frontend/src/components/BulkConfigImport.js @@ -458,6 +458,8 @@ backend web-backend {record.request_headers && Req Headers} {record.response_headers && Resp Headers} {record.tcp_request_rules && TCP Rules} + {record.filters && Filters} + {record.log_format && Log Format} {record.acl_rules && record.acl_rules.length > 0 && {record.acl_rules.length} ACLs} {record.use_backend_rules && record.use_backend_rules.length > 0 && {record.use_backend_rules.length} Routes} @@ -1076,8 +1078,9 @@ backend web-backend size="small" expandable={{ expandedRowRender: (frontend) => { - const hasDetails = frontend.request_headers || frontend.response_headers || - frontend.options || frontend.tcp_request_rules || + const hasDetails = frontend.request_headers || frontend.response_headers || + frontend.options || frontend.tcp_request_rules || + frontend.filters || frontend.log_format || (frontend.acl_rules && frontend.acl_rules.length > 0) || (frontend.use_backend_rules && frontend.use_backend_rules.length > 0); @@ -1157,12 +1160,52 @@ backend web-backend } > - )} + {/* Issue #38: SPOE filters */} + {frontend.filters && ( + + Filters (SPOE/WAF) + {frontend._changes?.filters && ( + + {frontend._changes.filters.old ? 'CHANGED' : 'NEW'} + + )} + + } + > + + + )} + {/* Issue #38: frontend log-format */} + {frontend.log_format && ( + + Log Format + {frontend._changes?.log_format && ( + + {frontend._changes.log_format.old ? 'CHANGED' : 'NEW'} + + )} + + } + > + + + )} {frontend.acl_rules && frontend.acl_rules.length > 0 && ( {frontend.acl_rules.map((acl, idx) => ( diff --git a/frontend/src/components/FrontendManagement.js b/frontend/src/components/FrontendManagement.js index 5d86906..a208691 100644 --- a/frontend/src/components/FrontendManagement.js +++ b/frontend/src/components/FrontendManagement.js @@ -642,7 +642,11 @@ const FrontendManagement = () => { // Explicitly set options field to handle null/undefined case (NEW field) options: frontend.options || '', // BUGFIX: Explicitly set tcp_request_rules field to handle null/undefined case - tcp_request_rules: frontend.tcp_request_rules || '' + tcp_request_rules: frontend.tcp_request_rules || '', + // Issue #38: SPOE filters + frontend log-format (null → '' so the + // TextAreas populate on edit and round-trip on save, preventing null-wipe) + log_format: frontend.log_format || '', + filters: frontend.filters || '' }); // Update SSL field visibility after setting values @@ -2250,6 +2254,40 @@ tcp-request connection reject if { src -f /etc/haproxy/blacklist.lst }`} + + {/* Issue #38: SPOE filters + frontend log-format */} + + + +