feat(haproxy): preserve SPOE filter + frontend log-format on import/edit (v1.8.8, Issue #38)

Bulk import / manual edit silently dropped `filter spoe engine ...` (Coraza WAF)
and frontend `log-format` because the parser recognised only a fixed directive
set. The regenerated config then missed the SPOE engine, so HAProxy failed with
"unable to find SPOE engine 'coraza' used by the send-spoe-group".

- parser: capture `filter` + `log-format`/`log-format-sd` into new ParsedFrontend fields
- db: additive nullable `log_format` + `filters` TEXT columns on frontends (SCHEMA_VERSION 8->9)
- generator: new `filter` bucket flushed before http-request rules so `filter` precedes
  `send-spoe-group`; `log-format` kept in prelude
- bulk import: preview dict, change-detection, persist (create + merge-update); cluster-aware
  SPOE pre-flight advisories (missing-filter + host-prerequisite) surfaced in the UI
- manual CRUD: full round-trip (get/create/update) incl. React form fields (no null-wipe)
- reject/rollback: restore the new columns; restore path + wizard helper kept in parity
- backend `option spop-check` recognised (suppresses spurious warning for coraza-spoa)
- tests: test_spoe_filter_import.py; full suite green (1079 passed)
This commit is contained in:
taylanbakircioglu
2026-07-10 18:34:36 +03:00
parent 60f4fa71ed
commit 9e2ea04777
15 changed files with 487 additions and 44 deletions
+14 -2
View File
@@ -194,7 +194,14 @@ async def ensure_agents_table():
'use_backend_rules': "ALTER TABLE frontends ADD COLUMN use_backend_rules JSONB DEFAULT '[]'::jsonb;",
'request_headers': "ALTER TABLE frontends ADD COLUMN request_headers TEXT;",
'response_headers': "ALTER TABLE frontends ADD COLUMN response_headers TEXT;",
'maxconn': "ALTER TABLE frontends ADD COLUMN maxconn INTEGER;"
'maxconn': "ALTER TABLE frontends ADD COLUMN maxconn INTEGER;",
# Issue #38: SPOE filter directives (e.g. Coraza WAF) and frontend
# log-format were silently dropped on bulk-import / manual edit
# because the parser recognised only a fixed set of directives.
# These nullable TEXT columns persist them verbatim (multi-line for
# `filters`), mirroring the request_headers/options passthrough.
'log_format': "ALTER TABLE frontends ADD COLUMN log_format TEXT;",
'filters': "ALTER TABLE frontends ADD COLUMN filters TEXT;"
}
for col, query in frontend_columns.items():
@@ -1741,7 +1748,12 @@ async def ensure_agent_activity_logs_table():
# columns on letsencrypt_accounts/letsencrypt_orders/acme_challenges and the brand-new
# letsencrypt_account_dns_credentials table (ensure_letsencrypt_dns_credentials step).
# All additive + idempotent; default challenge_type 'http-01' keeps existing flows byte-identical.
SCHEMA_VERSION = 8
# v1.8.8 (Issue #38 — SPOE filter + frontend log-format): bumped 8 -> 9 for the additive
# `log_format` + `filters` TEXT columns on `frontends` (frontend_columns loop). Without this
# bump, already-deployed databases (version >= 8) skip the whole migration run and never gain
# the columns, so the frontends SELECT/INSERT would fail. Additive + idempotent + nullable;
# existing rows stay NULL and render byte-identical.
SCHEMA_VERSION = 9
async def run_all_migrations():
+5
View File
@@ -85,6 +85,11 @@ class FrontendConfig(BaseModel):
response_headers: Optional[str] = None
options: Optional[str] = None
tcp_request_rules: Optional[str] = None
# Issue #38: SPOE filter directives (Coraza WAF etc.) + frontend log-format.
# Passthrough TEXT (no validator) — SPOE `filter ... config <path>` legitimately
# references an operator-managed file, so the ACL `-f` guard must NOT apply here.
log_format: Optional[str] = None
filters: Optional[str] = None
timeout_client: Optional[int] = None
timeout_http_request: Optional[int] = None
rate_limit: Optional[int] = None
+12 -9
View File
@@ -3696,17 +3696,19 @@ async def confirm_restore_config_version(
# UPDATE existing frontend (ALL 8 parsed fields)
# CRITICAL FIX: Include maxconn and timeout_client so UI shows restored values
await conn.execute("""
UPDATE frontends
SET bind_address = $1, bind_port = $2, default_backend = $3,
UPDATE frontends
SET bind_address = $1, bind_port = $2, default_backend = $3,
mode = $4, ssl_enabled = $5, ssl_port = $6,
maxconn = $7, timeout_client = $8,
log_format = $11, filters = $12,
updated_at = CURRENT_TIMESTAMP, last_config_status = 'PENDING'
WHERE id = $9 AND cluster_id = $10
""",
""",
parsed_fe.bind_address, parsed_fe.bind_port, parsed_fe.default_backend,
parsed_fe.mode, parsed_fe.ssl_enabled, parsed_fe.ssl_port,
parsed_fe.maxconn, parsed_fe.timeout_client,
fe_id, cluster_id
fe_id, cluster_id,
parsed_fe.log_format, parsed_fe.filters # Issue #38
)
changes_summary["frontends_updated"] += 1
logger.info(f"RESTORE: Updated frontend '{parsed_fe.name}' (SSL: {parsed_fe.ssl_enabled}, maxconn: {parsed_fe.maxconn})")
@@ -3714,16 +3716,17 @@ async def confirm_restore_config_version(
# CREATE new frontend (ALL 8 parsed fields)
# CRITICAL FIX: Include maxconn and timeout_client so UI shows restored values
await conn.execute("""
INSERT INTO frontends
INSERT INTO frontends
(name, bind_address, bind_port, default_backend, mode, ssl_enabled, ssl_port,
maxconn, timeout_client,
cluster_id, is_active, last_config_status, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, TRUE, 'PENDING', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
cluster_id, log_format, filters, is_active, last_config_status, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, TRUE, 'PENDING', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
""",
parsed_fe.name, parsed_fe.bind_address, parsed_fe.bind_port,
parsed_fe.default_backend, parsed_fe.mode, parsed_fe.ssl_enabled, parsed_fe.ssl_port,
parsed_fe.maxconn, parsed_fe.timeout_client,
cluster_id
cluster_id,
parsed_fe.log_format, parsed_fe.filters # Issue #38
)
changes_summary["frontends_created"] += 1
logger.info(f"RESTORE: Created frontend '{parsed_fe.name}' (SSL: {parsed_fe.ssl_enabled}, maxconn: {parsed_fe.maxconn})")
+77 -7
View File
@@ -855,6 +855,9 @@ async def parse_bulk_config(
"response_headers": frontend.response_headers,
"options": frontend.options,
"tcp_request_rules": frontend.tcp_request_rules,
# Issue #38: SPOE filters + frontend log-format
"log_format": frontend.log_format,
"filters": frontend.filters,
# CRITICAL: SSL Advanced Options (parsed from bind directive)
"ssl_alpn": frontend.ssl_alpn,
"ssl_npn": frontend.ssl_npn,
@@ -1089,7 +1092,50 @@ async def parse_bulk_config(
# Add auto-assignment info at the beginning
enhanced_warnings = ssl_auto_assign_info + enhanced_warnings
# ─────────────────────────────────────────────────────────────────
# Issue #38: SPOE pre-flight advisories. Surface, at preview time, the
# SPOE configurations that would FAIL HAProxy's `haproxy -c` at apply so
# the operator sees them BEFORE importing. Cluster-aware: the referenced
# SPOE engine config (e.g. coraza.cfg) is a sibling of the cluster's
# haproxy_config_path, which HAProxy OpenManager does not provision.
# ─────────────────────────────────────────────────────────────────
try:
_cfg_path = await conn.fetchval(
"SELECT haproxy_config_path FROM haproxy_clusters WHERE id = $1",
request.cluster_id,
) or "/etc/haproxy/haproxy.cfg"
_cfg_dir = _cfg_path.rsplit("/", 1)[0] or "/etc/haproxy"
for _fe in frontends_data:
_rh = _fe.get("request_headers") or ""
_filters = _fe.get("filters") or ""
# engines declared by `filter spoe engine <name> config <path>`
_declared_engines = set(re.findall(
r"filter\s+spoe\s+engine\s+(\S+)", _filters, re.IGNORECASE))
# engines referenced by `... send-spoe-group <name> <group>`
_used_engines = set(re.findall(
r"send-spoe-group\s+(\S+)", _rh, re.IGNORECASE))
_missing = _used_engines - _declared_engines
if _missing:
enhanced_warnings.append(
f"⚠️ Frontend '{_fe['name']}': 'send-spoe-group' references SPOE "
f"engine(s) {', '.join(sorted(_missing))} but no matching "
f"'filter spoe engine <name> ...' line was found. HAProxy will "
f"reject this at apply with \"unable to find SPOE engine\". Add the "
f"filter line to this frontend."
)
for _path in re.findall(
r"filter\s+spoe\s+engine\s+\S+\s+config\s+(\S+)",
_filters, re.IGNORECASE):
enhanced_warnings.append(
f"️ Frontend '{_fe['name']}': SPOE engine config '{_path}' and its "
f"SPOA backend must exist on the HAProxy host (cluster config dir: "
f"{_cfg_dir}). HAProxy OpenManager preserves the filter directive but "
f"does not provision these files; otherwise 'haproxy -c' fails at apply."
)
except Exception as _spoe_adv_err:
logger.warning(f"SPOE advisory generation skipped: {_spoe_adv_err}")
# BULK IMPORT MVP: Check existing entities for UPSERT detection
# Mark each entity as new or update for UI display
# CRITICAL: Only mark as UPDATE if there are actual field changes
@@ -1150,7 +1196,17 @@ async def parse_bulk_config(
if frontend.get("tcp_request_rules") and frontend["tcp_request_rules"] != existing["tcp_request_rules"]:
has_changes = True
changes["tcp_request_rules"] = {"old": existing["tcp_request_rules"], "new": frontend["tcp_request_rules"]}
# Issue #38: SPOE filters + log-format change detection. REQUIRED for
# persistence (not just display): without it, an import that only adds
# a `filter`/`log-format` to an existing frontend would be flagged
# "no change" and the directive would never be written to the DB.
if frontend.get("log_format") and frontend["log_format"] != existing.get("log_format"):
has_changes = True
changes["log_format"] = {"old": existing.get("log_format"), "new": frontend["log_format"]}
if frontend.get("filters") and frontend["filters"] != existing.get("filters"):
has_changes = True
changes["filters"] = {"old": existing.get("filters"), "new": frontend["filters"]}
# CRITICAL: SSL Advanced Options change detection
if frontend.get("ssl_alpn") is not None and frontend.get("ssl_alpn") != existing.get("ssl_alpn"):
has_changes = True
@@ -2094,7 +2150,18 @@ async def bulk_create_entities(
update_fields.append(f"options = ${param_index}")
update_values.append(frontend_data["options"])
param_index += 1
# Issue #38: SPOE filters + frontend log-format (merge strategy)
if frontend_data.get("log_format") and frontend_data["log_format"] != existing_full.get("log_format"):
update_fields.append(f"log_format = ${param_index}")
update_values.append(frontend_data["log_format"])
param_index += 1
if frontend_data.get("filters") and frontend_data["filters"] != existing_full.get("filters"):
update_fields.append(f"filters = ${param_index}")
update_values.append(frontend_data["filters"])
param_index += 1
# CRITICAL FIX: Update SSL advanced options (alpn, npn, ciphers, etc.)
# These are parsed from bind directive and should be preserved in database
if "ssl_alpn" in frontend_data and frontend_data.get("ssl_alpn") != existing_full.get("ssl_alpn"):
@@ -2214,9 +2281,10 @@ async def bulk_create_entities(
timeout_client, timeout_http_request, maxconn,
request_headers, response_headers, tcp_request_rules, options,
rate_limit, compression, log_separate, monitor_uri,
cluster_id, acl_rules, use_backend_rules, redirect_rules, updated_at
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, CURRENT_TIMESTAMP)
cluster_id, acl_rules, use_backend_rules, redirect_rules,
log_format, filters, updated_at
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, CURRENT_TIMESTAMP)
RETURNING id
""",
frontend_data["name"],
@@ -2257,7 +2325,9 @@ async def bulk_create_entities(
request.cluster_id,
json.dumps(frontend_data.get("acl_rules", [])), # acl_rules
json.dumps(frontend_data.get("use_backend_rules", [])), # use_backend_rules
json.dumps([]) # redirect_rules
json.dumps([]), # redirect_rules
frontend_data.get("log_format"), # Issue #38
frontend_data.get("filters") # Issue #38
)
created_entities["frontends"].append({
+20 -9
View File
@@ -408,6 +408,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -424,6 +425,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -453,6 +455,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -465,6 +468,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -480,6 +484,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -492,6 +497,7 @@ async def get_frontends(
ssl_alpn, ssl_npn, ssl_ciphers, ssl_ciphersuites, ssl_min_ver, ssl_max_ver, ssl_strict_sni,
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules,
log_format, filters,
timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
maxconn, is_active, created_at, updated_at, cluster_id, last_config_status
@@ -601,6 +607,8 @@ async def get_frontends(
"response_headers": f.get("response_headers"),
"options": f.get("options"),
"tcp_request_rules": f.get("tcp_request_rules"),
"log_format": f.get("log_format"), # Issue #38
"filters": f.get("filters"), # Issue #38
"timeout_client": f.get("timeout_client"),
"timeout_http_request": f.get("timeout_http_request"),
"rate_limit": f.get("rate_limit"),
@@ -735,18 +743,18 @@ async def create_frontend(frontend: FrontendConfig, request: Request, authorizat
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules, timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
cluster_id, maxconn, updated_at
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, CURRENT_TIMESTAMP)
cluster_id, maxconn, log_format, filters, updated_at
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, CURRENT_TIMESTAMP)
RETURNING id
""", frontend.name, frontend.bind_address, frontend.bind_port,
""", frontend.name, frontend.bind_address, frontend.bind_port,
frontend.default_backend, frontend.mode, frontend.ssl_enabled,
frontend.ssl_certificate_id, ssl_cert_ids_json, frontend.ssl_port, frontend.ssl_cert_path, frontend.ssl_cert, frontend.ssl_verify,
frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites,
frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites,
frontend.ssl_min_ver, frontend.ssl_max_ver, frontend.ssl_strict_sni,
json.dumps(frontend.acl_rules or []), json.dumps(frontend.redirect_rules or []), json.dumps(frontend.use_backend_rules or []),
frontend.request_headers, frontend.response_headers, filtered_options, frontend.tcp_request_rules, frontend.timeout_client, frontend.timeout_http_request,
frontend.rate_limit, frontend.compression, frontend.log_separate, frontend.monitor_uri,
frontend.cluster_id, frontend.maxconn)
frontend.cluster_id, frontend.maxconn, frontend.log_format, frontend.filters)
# If cluster_id provided, create new config version for agents
sync_results = []
@@ -1060,9 +1068,10 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R
acl_rules = $20, redirect_rules = $21, use_backend_rules = $22,
request_headers = $23, response_headers = $24, options = $25, tcp_request_rules = $26, timeout_client = $27, timeout_http_request = $28,
rate_limit = $29, compression = $30, log_separate = $31, monitor_uri = $32,
cluster_id = $33, maxconn = $34, updated_at = CURRENT_TIMESTAMP
WHERE id = $35
""", frontend.name, frontend.bind_address, frontend.bind_port,
cluster_id = $33, maxconn = $34, log_format = $35, filters = $36,
updated_at = CURRENT_TIMESTAMP
WHERE id = $37
""", frontend.name, frontend.bind_address, frontend.bind_port,
frontend.default_backend, frontend.mode, ssl_enabled,
ssl_certificate_id, ssl_cert_ids_json, ssl_port, ssl_cert_path, ssl_cert, ssl_verify,
frontend.ssl_alpn, frontend.ssl_npn, frontend.ssl_ciphers, frontend.ssl_ciphersuites,
@@ -1070,7 +1079,7 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R
json.dumps(frontend.acl_rules or []), json.dumps(frontend.redirect_rules or []), json.dumps(frontend.use_backend_rules or []),
frontend.request_headers, frontend.response_headers, filtered_options, frontend.tcp_request_rules, frontend.timeout_client, frontend.timeout_http_request,
frontend.rate_limit, frontend.compression, frontend.log_separate, frontend.monitor_uri,
frontend.cluster_id, frontend.maxconn, frontend_id)
frontend.cluster_id, frontend.maxconn, frontend.log_format, frontend.filters, frontend_id)
# Debug: Check what was actually saved
updated_frontend = await conn.fetchrow("""
@@ -1124,6 +1133,8 @@ async def update_frontend(frontend_id: int, frontend: FrontendConfig, request: R
"response_headers": frontend.response_headers,
"options": filtered_options,
"tcp_request_rules": frontend.tcp_request_rules,
"log_format": frontend.log_format, # Issue #38
"filters": frontend.filters, # Issue #38
"timeout_client": frontend.timeout_client,
"timeout_http_request": frontend.timeout_http_request,
"rate_limit": frontend.rate_limit,
+4 -2
View File
@@ -56,14 +56,14 @@ async def create_frontend_row(
acl_rules, redirect_rules, use_backend_rules,
request_headers, response_headers, options, tcp_request_rules, timeout_client, timeout_http_request,
rate_limit, compression, log_separate, monitor_uri,
cluster_id, maxconn, updated_at
cluster_id, maxconn, log_format, filters, updated_at
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12,
$13, $14, $15, $16, $17, $18, $19,
$20, $21, $22,
$23, $24, $25, $26, $27, $28,
$29, $30, $31, $32,
$33, $34, CURRENT_TIMESTAMP
$33, $34, $35, $36, CURRENT_TIMESTAMP
)
RETURNING id
""",
@@ -101,6 +101,8 @@ async def create_frontend_row(
getattr(payload, "monitor_uri", None),
cluster_id,
getattr(payload, "maxconn", None),
getattr(payload, "log_format", None), # Issue #38
getattr(payload, "filters", None), # Issue #38
)
if mark_pending:
+20 -1
View File
@@ -393,6 +393,12 @@ def _categorize_haproxy_directive(line: str) -> str:
return "prelude"
if s.startswith("acl "):
return "acl"
# Issue #38: SPOE (and other) `filter` directives must be declared BEFORE
# the `http-request send-spoe-group` rules that use them, otherwise HAProxy
# fails with "unable to find SPOE engine". Own bucket, flushed right after
# `prelude` and before tcp_req/acl/http_req (see flush order below).
if s.startswith("filter "):
return "filter"
if s.startswith("stick-table") or s.startswith("stick "):
return "stick"
if s.startswith("tcp-request"):
@@ -414,6 +420,7 @@ def _categorize_haproxy_directive(line: str) -> str:
or s.startswith("compression ")
or s.startswith("monitor-uri")
or s.startswith("log ")
or s.startswith("log-format") # Issue #38: log-format / log-format-sd
or s.startswith("description ")
or s.startswith("disabled")
or s.startswith("enabled")
@@ -903,7 +910,7 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An
# "stick-table already declared").
# ─────────────────────────────────────────────────────────────────
_fe_buckets: Dict[str, List[str]] = {
"prelude": [], "stick": [], "tcp_req": [],
"prelude": [], "filter": [], "stick": [], "tcp_req": [],
"acl": [], "http_req": [], "http_resp": [],
"redirect": [], "use_be": [], "default_be": [],
}
@@ -996,6 +1003,17 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An
if line_stripped and line_stripped not in ('[]', '{}', 'null', 'None'):
_emit_fe(f" {line_stripped}")
# Issue #38: emit frontend log-format and SPOE (etc.) filter directives.
# `log_format` routes to the `prelude` bucket, `filters` to the `filter`
# bucket (both via _emit_fe → _categorize_haproxy_directive), guaranteeing
# `filter ...` is rendered before the `http-request send-spoe-group` rules.
for _fld in ('log_format', 'filters'):
if frontend.get(_fld):
for line in frontend[_fld].split('\n'):
line_stripped = line.strip()
if line_stripped and line_stripped not in ('[]', '{}', 'null', 'None'):
_emit_fe(f" {line_stripped}")
# CRITICAL: Validate frontend-backend mode compatibility
if frontend.get('default_backend'):
default_backend_name = frontend['default_backend'].strip() if frontend['default_backend'] else ''
@@ -1223,6 +1241,7 @@ async def generate_haproxy_config_for_cluster(cluster_id: int, conn: Optional[An
# ─────────────────────────────────────────────────────────────
for _bucket_key in (
"prelude",
"filter",
"stick",
"tcp_req",
"acl",
+4 -2
View File
@@ -276,14 +276,16 @@ def test_categorize_routes_directives_correctly():
def test_emit_buckets_flushed_in_canonical_order():
"""The flush block at end of frontend processing must list buckets
in: prelude stick tcp_req acl http_req http_resp
in: prelude filter stick tcp_req acl http_req http_resp
redirect use_be default_be. Pre-fix `http-request` rules
interleaved with `use_backend` rules in source order, producing
HAProxy parser warnings."""
HAProxy parser warnings. (Issue #38 added the `filter` bucket, flushed
right after `prelude` so SPOE `filter` lines precede `send-spoe-group`.)"""
src = _gen_src()
flush_match = re.search(
r'for\s+_bucket_key\s+in\s+\(\s*'
r'"prelude"\s*,\s*'
r'"filter"\s*,\s*'
r'"stick"\s*,\s*'
r'"tcp_req"\s*,\s*'
r'"acl"\s*,\s*'
+203
View File
@@ -0,0 +1,203 @@
"""
Issue #38 regression tests: HAProxy SPOE `filter` + frontend `log-format` support.
Bug: the bulk-config parser recognised only a fixed set of frontend directives,
so `filter spoe engine coraza config ...` and `log-format ...` were silently
dropped on import / manual edit. This regenerated a config missing the SPOE
engine definition, so HAProxy failed with
"unable to find SPOE engine 'coraza' used by the send-spoe-group 'coraza-req'".
These tests verify the end-to-end fix without requiring a database:
1. parser captures `filter` + `log-format` into the new ParsedFrontend fields;
2. `http-request send-spoe-group` is still preserved (regression guard);
3. the generator's directive categoriser + bucket flush order emit `filter`
BEFORE the `http-request send-spoe-group` rules and keep `log-format`;
4. reject/rollback restores the new columns;
5. a non-SPOE frontend is completely unaffected (zero-impact).
"""
import os
import re
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from utils.haproxy_config_parser import parse_haproxy_config, ParsedFrontend
from services.haproxy_config import _categorize_haproxy_directive
from models.frontend import FrontendConfig
# The exact frontend/backend config reported in Issue #38 (Coraza-SPOA).
ISSUE_38_CONFIG = r"""
frontend web-frontend
bind *:8073
mode http
log-format "%ci:%cp\ [%t]\ %ft\ %b/%s\ %ST\ %B\ %{+Q}r\ %[var(txn.coraza.id)]\ waf-hit:\ %[var(txn.coraza.fail)]"
filter spoe engine coraza config /etc/haproxy/coraza.cfg
http-request set-var(txn.coraza.app) str(haproxy_waf)
http-request send-spoe-group coraza coraza-req
http-request deny if { var(txn.coraza.fail) -m int eq 1 }
default_backend web-backend
backend web-backend
balance roundrobin
mode http
server server1 192.168.1.10:443 weight 100 ssl verify none
backend coraza-spoa
mode tcp
option spop-check
server coraza_spoa 192.168.12.21:9000
"""
def _get_frontend(parse_result, name):
for fe in parse_result.frontends:
if fe.name == name:
return fe
return None
class TestParserCapturesSpoe:
def test_filter_and_log_format_captured(self):
result = parse_haproxy_config(ISSUE_38_CONFIG)
fe = _get_frontend(result, "web-frontend")
assert fe is not None, "web-frontend should be parsed and kept"
assert fe.filters is not None
assert "filter spoe engine coraza config /etc/haproxy/coraza.cfg" in fe.filters
assert fe.log_format is not None
assert fe.log_format.startswith("log-format")
# the escaped/quoted format string must be preserved verbatim
assert "%[var(txn.coraza.fail)]" in fe.log_format
def test_send_spoe_group_still_preserved(self):
# Regression guard: http-request rules (incl. send-spoe-group) must
# still be collected into request_headers as before.
result = parse_haproxy_config(ISSUE_38_CONFIG)
fe = _get_frontend(result, "web-frontend")
assert fe.request_headers is not None
assert "send-spoe-group coraza coraza-req" in fe.request_headers
def test_multiple_filters_preserved_in_order(self):
cfg = """
frontend f1
bind *:80
mode http
filter compression
filter spoe engine coraza config /etc/haproxy/coraza.cfg
default_backend b1
backend b1
mode http
server s1 10.0.0.1:80
"""
fe = _get_frontend(parse_haproxy_config(cfg), "f1")
lines = fe.filters.split("\n")
assert lines == [
"filter compression",
"filter spoe engine coraza config /etc/haproxy/coraza.cfg",
]
def test_log_format_sd_variant_captured(self):
cfg = """
frontend f1
bind *:80
mode http
log-format-sd "[exampleSDID@1234 field=value]"
default_backend b1
backend b1
mode http
server s1 10.0.0.1:80
"""
fe = _get_frontend(parse_haproxy_config(cfg), "f1")
assert fe.log_format is not None
assert fe.log_format.startswith("log-format-sd")
class TestGeneratorOrderingContract:
"""The generator routes directives into ordered buckets. Verify SPOE
correctness at the (pure) categoriser + documented flush-order level."""
def test_filter_routes_to_filter_bucket(self):
assert _categorize_haproxy_directive(" filter spoe engine coraza config /x.cfg") == "filter"
def test_send_spoe_group_routes_to_http_req(self):
assert _categorize_haproxy_directive(" http-request send-spoe-group coraza coraza-req") == "http_req"
def test_log_format_routes_to_prelude(self):
assert _categorize_haproxy_directive(' log-format "%ci:%cp"') == "prelude"
assert _categorize_haproxy_directive(' log-format-sd "[x]"') == "prelude"
def test_flush_order_places_filter_before_http_req(self):
# The bucket flush order is the single source of truth for emission
# ordering. Assert `filter` is flushed before `http_req` (and after
# `prelude`), guaranteeing `filter ...` renders before
# `http-request send-spoe-group ...`.
src = _read_source("services/haproxy_config.py")
m = re.search(r"for _bucket_key in \((.*?)\):", src, re.DOTALL)
assert m, "bucket flush loop not found"
order = re.findall(r'"(\w+)"', m.group(1))
assert "filter" in order, "new 'filter' bucket missing from flush order"
assert order.index("prelude") < order.index("filter") < order.index("http_req")
class TestModelAndRollback:
def test_model_has_passthrough_fields(self):
fc = FrontendConfig(
name="f", bind_port=80,
filters="filter spoe engine coraza config /etc/haproxy/coraza.cfg",
log_format='log-format "%ci"',
)
assert fc.filters.startswith("filter spoe")
assert fc.log_format.startswith("log-format")
def test_dataclass_defaults_none(self):
fe = ParsedFrontend(name="f")
assert fe.filters is None
assert fe.log_format is None
def test_rollback_restores_new_columns(self):
# Reject/rollback of a frontend UPDATE must restore the new columns,
# otherwise the rejected (new) filters/log_format would persist.
src = _read_source("utils/entity_snapshot.py")
assert "log_format = $" in src
assert "filters = $" in src
assert "old_values.get('log_format')" in src
assert "old_values.get('filters')" in src
class TestZeroImpact:
def test_non_spoe_frontend_unaffected(self):
cfg = """
frontend plain
bind *:80
mode http
option httplog
default_backend b1
backend b1
mode http
server s1 10.0.0.1:80
"""
fe = _get_frontend(parse_haproxy_config(cfg), "plain")
# No filter / log-format present → new fields stay None (no behaviour change)
assert fe.filters is None
assert fe.log_format is None
def test_spop_check_backend_roundtrips_without_warning(self):
result = parse_haproxy_config(ISSUE_38_CONFIG)
be = next((b for b in result.backends if b.name == "coraza-spoa"), None)
assert be is not None, "coraza-spoa backend should import"
assert be.mode == "tcp"
assert be.options and "option spop-check" in be.options
# spop-check is now a known option → no spurious 'unknown option' warning
assert not any(
"coraza-spoa" in w and "spop-check" in w and "Unknown" in w
for w in result.warnings
)
def _read_source(relpath):
base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
with open(os.path.join(base, relpath), "r", encoding="utf-8") as fh:
return fh.read()
+5 -2
View File
@@ -329,9 +329,10 @@ async def _rollback_update(
tcp_request_rules = $26, timeout_client = $27, timeout_http_request = $28,
rate_limit = $29, compression = $30, log_separate = $31,
monitor_uri = $32, maxconn = $33,
cluster_id = $34, is_active = $35, last_config_status = $36,
cluster_id = $34, is_active = $35, last_config_status = $36,
log_format = $37, filters = $38,
updated_at = CURRENT_TIMESTAMP
WHERE id = $37
WHERE id = $39
""",
old_values.get('name'),
old_values.get('bind_address'),
@@ -369,6 +370,8 @@ async def _rollback_update(
old_values.get('cluster_id'),
old_values.get('is_active'),
old_values.get('last_config_status'),
old_values.get('log_format'), # Issue #38
old_values.get('filters'), # Issue #38
entity_id
)
+34 -2
View File
@@ -105,6 +105,11 @@ class ParsedFrontend:
response_headers: Optional[str] = None
options: Optional[str] = None # HAProxy frontend options (option httplog, option forwardfor, etc.)
tcp_request_rules: Optional[str] = None # TCP request directives (for TCP mode)
# Issue #38: SPOE (and other) filter directives + frontend log-format.
# Stored as full directive lines; `filters` is newline-joined to preserve
# ordering when multiple `filter ...` lines exist.
log_format: Optional[str] = None # `log-format` / `log-format-sd` line(s)
filters: Optional[str] = None # `filter ...` line(s), e.g. `filter spoe engine coraza config ...`
@dataclass
@@ -256,8 +261,23 @@ class HAProxyConfigParser:
acl_rules_list = []
use_backend_rules_list = []
tcp_request_rules_list = []
filters_list = []
log_format_list = []
for line in lines:
# Issue #38: capture `filter ...` (SPOE/Coraza etc.) and
# `log-format`/`log-format-sd` directives. Pre-fix these matched
# no branch below and were silently dropped, so an imported SPOE
# config lost `filter spoe engine coraza ...` (→ HAProxy fatal
# "unable to find SPOE engine") and the frontend log-format.
# `continue` isolates them from the header/option handling below.
if line.startswith('filter '):
filters_list.append(line.strip())
continue
if re.match(r'^log-format(-sd)?\s', line, re.IGNORECASE):
log_format_list.append(line.strip())
continue
# Parse bind directive
# IMPORTANT: Handle multiple bind lines correctly
# Example: bind *:1002 (HTTP) and bind *:443 ssl (HTTPS)
@@ -540,6 +560,13 @@ class HAProxyConfigParser:
if tcp_request_rules_list:
frontend.tcp_request_rules = '\n'.join(tcp_request_rules_list)
# Issue #38: assign captured SPOE filters + log-format
if filters_list:
frontend.filters = '\n'.join(filters_list)
if log_format_list:
frontend.log_format = '\n'.join(log_format_list)
self.frontends.append(frontend)
logger.info(f"Parsed frontend: {name} -> {frontend.default_backend}")
@@ -666,9 +693,14 @@ class HAProxyConfigParser:
'transparent', 'abortonclose', 'allbackups', 'checkcache', 'clitcpka',
'srvtcpka', 'http-no-delay', 'socket-stats', 'tcp-smart-accept',
'tcp-smart-connect', 'independant-streams', 'log-separate-errors',
'log-health-checks', 'accept-invalid-http-request', 'accept-invalid-http-response'
'log-health-checks', 'accept-invalid-http-request', 'accept-invalid-http-response',
# Issue #38: SPOP health check for SPOE agent backends
# (e.g. coraza-spoa). Already collected below regardless, but
# listing it suppresses the spurious "unknown option" warning
# for the exact SPOE use-case.
'spop-check'
]
if option_name not in valid_options:
# Unknown/invalid option - add warning but still collect it
self.warnings.append(
+3 -3
View File
@@ -1,5 +1,5 @@
{
"version": "1.8.7",
"releaseName": "Version reporting single-source fix",
"releaseDate": "2026-07-09"
"version": "1.8.8",
"releaseName": "SPOE filter + frontend log-format support (Issue #38)",
"releaseDate": "2026-07-10"
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "haproxy-openmanager-frontend",
"version": "1.8.7",
"version": "1.8.8",
"description": "HAProxy Load Balancer Management UI",
"license": "AGPL-3.0-or-later",
"dependencies": {
+46 -3
View File
@@ -458,6 +458,8 @@ backend web-backend
{record.request_headers && <Tag color="blue">Req Headers</Tag>}
{record.response_headers && <Tag color="green">Resp Headers</Tag>}
{record.tcp_request_rules && <Tag color="purple">TCP Rules</Tag>}
{record.filters && <Tag color="magenta">Filters</Tag>}
{record.log_format && <Tag color="geekblue">Log Format</Tag>}
{record.acl_rules && record.acl_rules.length > 0 && <Tag color="orange">{record.acl_rules.length} ACLs</Tag>}
{record.use_backend_rules && record.use_backend_rules.length > 0 && <Tag color="cyan">{record.use_backend_rules.length} Routes</Tag>}
</Space>
@@ -1076,8 +1078,9 @@ backend web-backend
size="small"
expandable={{
expandedRowRender: (frontend) => {
const hasDetails = frontend.request_headers || frontend.response_headers ||
frontend.options || frontend.tcp_request_rules ||
const hasDetails = frontend.request_headers || frontend.response_headers ||
frontend.options || frontend.tcp_request_rules ||
frontend.filters || frontend.log_format ||
(frontend.acl_rules && frontend.acl_rules.length > 0) ||
(frontend.use_backend_rules && frontend.use_backend_rules.length > 0);
@@ -1157,12 +1160,52 @@ backend web-backend
</span>
}
>
<MultiLineDiffRenderer
<MultiLineDiffRenderer
value={frontend.tcp_request_rules}
changeInfo={frontend._changes?.tcp_request_rules}
/>
</Descriptions.Item>
)}
{/* Issue #38: SPOE filters */}
{frontend.filters && (
<Descriptions.Item
label={
<span>
Filters (SPOE/WAF)
{frontend._changes?.filters && (
<Tag color="green" style={{ marginLeft: 8, fontSize: '10px' }}>
{frontend._changes.filters.old ? 'CHANGED' : 'NEW'}
</Tag>
)}
</span>
}
>
<MultiLineDiffRenderer
value={frontend.filters}
changeInfo={frontend._changes?.filters}
/>
</Descriptions.Item>
)}
{/* Issue #38: frontend log-format */}
{frontend.log_format && (
<Descriptions.Item
label={
<span>
Log Format
{frontend._changes?.log_format && (
<Tag color="green" style={{ marginLeft: 8, fontSize: '10px' }}>
{frontend._changes.log_format.old ? 'CHANGED' : 'NEW'}
</Tag>
)}
</span>
}
>
<MultiLineDiffRenderer
value={frontend.log_format}
changeInfo={frontend._changes?.log_format}
/>
</Descriptions.Item>
)}
{frontend.acl_rules && frontend.acl_rules.length > 0 && (
<Descriptions.Item label={`ACL Rules (${frontend.acl_rules.length})`}>
{frontend.acl_rules.map((acl, idx) => (
+39 -1
View File
@@ -642,7 +642,11 @@ const FrontendManagement = () => {
// Explicitly set options field to handle null/undefined case (NEW field)
options: frontend.options || '',
// BUGFIX: Explicitly set tcp_request_rules field to handle null/undefined case
tcp_request_rules: frontend.tcp_request_rules || ''
tcp_request_rules: frontend.tcp_request_rules || '',
// Issue #38: SPOE filters + frontend log-format (null → '' so the
// TextAreas populate on edit and round-trip on save, preventing null-wipe)
log_format: frontend.log_format || '',
filters: frontend.filters || ''
});
// Update SSL field visibility after setting values
@@ -2250,6 +2254,40 @@ tcp-request connection reject if { src -f /etc/haproxy/blacklist.lst }`}
</Form.Item>
</Col>
</Row>
{/* Issue #38: SPOE filters + frontend log-format */}
<Row gutter={16}>
<Col span={24}>
<Form.Item
name="filters"
label="Filters (SPOE / WAF)"
extra="HAProxy filter directives (one per line). Emitted before send-spoe-group rules."
tooltip="e.g. Coraza WAF via SPOE. The referenced engine config file and its SPOA backend must exist on the HAProxy host."
>
<TextArea
rows={3}
placeholder={`Examples:
filter spoe engine coraza config /etc/haproxy/coraza.cfg`}
/>
</Form.Item>
</Col>
</Row>
<Row gutter={16}>
<Col span={24}>
<Form.Item
name="log_format"
label="Custom Log Format"
extra="HAProxy log-format / log-format-sd directive (kept verbatim)"
tooltip="Overrides option httplog/tcplog. Use the full directive including the quoted format string."
>
<TextArea
rows={3}
placeholder={'log-format "%ci:%cp [%t] %ft %b/%s %ST %B %{+Q}r"'}
/>
</Form.Item>
</Col>
</Row>
</Panel>
</Collapse>