Files
haproxy-openmanager/backend/tests/test_request_log_router_auth.py
T
mustafa.ulukaya ef26860df9 feat(logging): unified request/response log with configurable retention (v1.11.0)
Until now the only record of what happened was `user_activity_logs`, which
stores non-GET 2xx operations with no bodies. When something failed you could
see that a counter went up, never what was sent or what came back.

This adds one queryable timeline covering both directions:

- inbound: every API call, including GETs and including 4xx/5xx, with the
  user, client IP, status, duration and — redacted, size-capped — the request
  and response bodies.
- outbound: every HTTP call the backend makes, tagged with who it went to
  (ACME/Let's Encrypt, Cloudflare, GoDaddy, HAProxy stats, agents, the ACME
  diagnostics probe).

Outbound rows inherit the inbound request's id, so one operator action and the
CA/DNS calls it triggered read as a single trace: opening a failed "Request
Certificate" shows the exact POST /acme/new-order and the CA's 429 underneath.

Implementation notes:

- Capture is a pure-ASGI middleware that TEES the request and response streams
  rather than draining them. `await request.body()` inside a BaseHTTPMiddleware
  would consume the receive channel and break the raw-body agent heartbeat
  handler. Registered last so it is outermost: it then sees the final
  client-visible response and seeds correlation_id_context before the error
  handler reads it.
- Rows are written by a batching background writer with a bounded queue, so the
  request path never awaits the database and a saturated logger drops rows
  visibly (surfaced on the page) instead of blocking. Redaction runs on the
  writer, off the request coroutine.
- Secrets never land: headers are an allowlist with Authorization/Cookie kept
  only as a presence marker; body keys and value shapes are redacted
  (passwords, tokens, api_token, API keys, private-key PEMs, JWTs); the ACME
  JWS request body is never stored, because a stored protected+signature pair
  is a replayable credential — a summary is logged instead; DNS-provider errors
  record only the exception type; the ACME HTTP-01 challenge endpoint is
  excluded so key_authorization is never captured.
- Retention is operator-configurable in Settings -> Request Log: separate day
  counts for successful and failed rows (7 / 30) plus a hard row cap (500k),
  whichever is reached first. Pruned in batches under a Postgres advisory lock,
  with the day counts bound as parameters, never interpolated.
- New permissions requestlog.read / requestlog.manage. super_admin and
  security_admin get both, operator gets read, viewer gets neither.

Schema: one new table (request_logs) plus its settings seed, SCHEMA_VERSION
10 -> 11, auto-migrated. No existing table altered, no agent or rendered-config
change. Kill switches: REQUEST_LOG_ENABLED=false (middleware never registered)
or the `enabled` toggle in Settings.

Tests: 245 new (7 backend files + 1 frontend), full suite 1655 backend +
17 frontend passing.
2026-08-11 02:36:03 +03:00

177 lines
6.9 KiB
Python

"""v1.11.0: the request log API is gated, and its routes resolve.
Two distinct failure modes are pinned here.
**Auth.** The table holds redacted-but-real request and response bodies for
every user, so an unauthenticated or under-privileged caller must never get a
row. There is no database in this suite, so the behavioural checks assert only
that an anonymous call is rejected before any DB work — which is exactly the
property that matters — and a source scan covers the per-endpoint permission.
**Route order.** `/{log_id}` is a single-segment path and FastAPI matches in
declaration order, so declaring it before `/settings`, `/stats` or `/purge`
makes those three unreachable (they parse as a log id and 422). This is the
mirror image of the shadowing trap already present in routers/settings.py.
"""
import os
import re
import sys
import pytest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_BACKEND = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
_ROUTER = os.path.join(_BACKEND, "routers", "request_logs.py")
_MAIN = os.path.join(_BACKEND, "main.py")
REJECT = (401, 403, 422)
@pytest.fixture(scope="module")
def src():
with open(_ROUTER, encoding="utf-8") as f:
return f.read()
# --------------------------------------------------------------------------
# Behavioural: nothing is readable without credentials
# --------------------------------------------------------------------------
@pytest.mark.parametrize("method,path", [
("get", "/api/request-logs"),
("get", "/api/request-logs/1"),
("get", "/api/request-logs/stats"),
("get", "/api/request-logs/settings"),
("put", "/api/request-logs/settings"),
("post", "/api/request-logs/purge"),
])
def test_anonymous_access_is_rejected(client, method, path):
res = getattr(client, method)(path) if method != "put" else client.put(path, json={})
assert res.status_code in REJECT, (
f"{method.upper()} {path} returned {res.status_code} without an Authorization "
f"header — the request log contains captured bodies for every user"
)
def test_a_garbage_token_is_rejected(client):
res = client.get("/api/request-logs", headers={"authorization": "Bearer not-a-token"})
assert res.status_code in REJECT
# --------------------------------------------------------------------------
# Source scan: per-endpoint permission
# --------------------------------------------------------------------------
def _handler_body(src, decorator):
start = src.index(decorator)
rest = src[start + len(decorator):]
end = rest.find("\n@router.")
return rest if end == -1 else rest[:end]
@pytest.mark.parametrize("decorator,action", [
('@router.get("/settings")', "manage"),
('@router.put("/settings")', "manage"),
('@router.get("/stats")', "read"),
('@router.post("/purge")', "manage"),
('@router.get("")', "read"),
('@router.get("/{log_id}")', "read"),
])
def test_every_endpoint_enforces_its_permission(src, decorator, action):
body = _handler_body(src, decorator)
assert f'_require(authorization, "{action}")' in body, (
f"{decorator} does not enforce requestlog.{action}"
)
def test_require_helper_raises_403_not_a_silent_pass(src):
helper = src.split("async def _require", 1)[1].split("\nasync def ", 1)[0]
assert "check_user_permission" in helper
assert "status_code=403" in helper
assert "current_user=current_user" in helper, (
"the admin bypass is skipped, so every call pays an extra SELECT on users"
)
# --------------------------------------------------------------------------
# Route declaration order
# --------------------------------------------------------------------------
@pytest.mark.parametrize("literal", ['@router.get("/settings")', '@router.put("/settings")',
'@router.get("/stats")', '@router.post("/purge")'])
def test_literal_routes_are_declared_before_the_catch_all(src, literal):
catch_all = src.index('@router.get("/{log_id}")')
assert src.index(literal) < catch_all, (
f"{literal} is declared after GET /{{log_id}}. FastAPI matches in declaration "
f"order and /{{log_id}} is a single-segment path, so it would swallow this route "
f"and the request would fail parsing 'settings' as an int."
)
def test_list_route_is_declared_before_the_catch_all(src):
assert src.index('@router.get("")') < src.index('@router.get("/{log_id}")')
# --------------------------------------------------------------------------
# Query construction
# --------------------------------------------------------------------------
def test_filters_are_bound_never_interpolated(src):
"""User-supplied filters reach the WHERE clause; they must arrive as $n
parameters."""
body = _handler_body(src, '@router.get("")')
# The only f-string interpolation allowed into SQL is the placeholder index
# and the assembled clause list, never a raw value.
for match in re.findall(r'add\("([^"]+)"', body):
assert "{n}" in match, f"filter clause {match!r} does not use a bound placeholder"
def test_list_endpoint_scopes_non_privileged_callers_to_themselves(src):
body = _handler_body(src, '@router.get("")')
assert "if not can_manage:" in body
assert "direction = 'inbound' AND user_id =" in body, (
"a caller with only requestlog.read can see every other user's captured request "
"bodies"
)
def test_detail_endpoint_applies_the_same_scoping(src):
body = _handler_body(src, '@router.get("/{log_id}")')
assert "can_manage" in body
assert "404" in body, (
"the detail endpoint should 404 rather than 403 for a row the caller may not see, "
"so it does not confirm which ids exist"
)
def test_list_response_omits_bodies(src):
"""A 200-row page carrying two 8 KB JSONB blobs per row is a multi-megabyte
response; bodies belong to the detail endpoint."""
columns = src.split("_LIST_COLUMNS = ", 1)[1].split('"""', 2)[1]
assert "request_body," not in columns
assert "response_body," not in columns
assert "request_body_bytes" in columns, "the size is still useful in the list"
def test_count_is_bounded(src):
body = _handler_body(src, '@router.get("")')
assert "LIMIT {count_cap}" in body or "count_cap" in body, (
"an unbounded COUNT(*) over request_logs is a sequential scan on every page change"
)
assert "total_is_estimate" in body
# --------------------------------------------------------------------------
# Registration
# --------------------------------------------------------------------------
def test_router_is_registered_in_main():
with open(_MAIN, encoding="utf-8") as f:
main_src = f.read()
assert "from routers.request_logs import router as request_logs_router" in main_src
assert "app.include_router(request_logs_router)" in main_src, (
"the router is imported but never mounted, so every endpoint 404s"
)