mirror of
https://github.com/PerpetualSoftware/pad.git
synced 2026-09-22 18:43:45 +00:00
xarmian
98c8b78d06
Plug MCP traffic and OAuth flow events into pad's existing
internal/metrics Prometheus surface, plus a Grafana dashboard.
Metrics (all under pad_*):
- Counters: mcp_tool_calls_total{user_id,tool,status},
mcp_authz_denials_total{reason}, oauth_flows_total{stage},
oauth_token_revocations_total{reason}
- Histograms: mcp_tool_call_duration_seconds{tool},
oauth_flow_duration_seconds{stage}, oauth_token_ttl_seconds
- Gauges: mcp_active_sessions, oauth_active_tokens (callback collector)
Wiring seams: MCPAuditLog (per-call), MCPBearerAuth (audience denials),
emitMCPAuditDenied (rate-limit denials), RequireWorkspaceAccess (gated
to MCP-origin via context — workspace_not_in_allowlist + not_a_member),
OAuth handlers (per-stage flow events + per-handler latency), and
internal/oauth/storage.go via a new SetRevocationObserver hook so the
OAuth package stays metrics-naive.
Cmd/pad wires both observers via Server.wireOAuthMetricsObserver(),
called from both SetMetrics and SetOAuthServer for order-independence.
Store helpers added (with full test coverage):
- CountActiveOAuthAccessTokens — backs the active-tokens gauge
- OldestAccessTokenIssuedAtByRequestID — backs the TTL observation
Grafana dashboard at monitoring/grafana/mcp.json: 13 panels across MCP
traffic + OAuth flow rows (rate-by-tool, p50/p95/p99 latency, status
breakdown, denial reasons, active sessions, top-10 users, OAuth flow
events by stage, OAuth handler p95, active tokens, revocations by
reason, TTL p50/p95).
Codex review caught one HIGH issue (round 1, fixed in same commit):
the active-tokens collector originally emitted NewInvalidMetric on
provider error, which propagates through Registry.Gather() and fails
the entire /metrics scrape via promhttp's default error handler.
Switched to log + skip-the-sample so a transient SQLite blip drops
ONE gauge for one scrape rather than the whole observability surface.
Added TestRegisterOAuthActiveTokensCollector_ErrorIsScrapeSafe to pin
the contract.
Tests cover increments, histogram bucket placement, callback collector
freshness across mutations + error path, observer hook firing on user-
initiated revocation + rotation + nil-safety, and per-helper unit tests
for the server-side metric emission.
Verified with `make check` (golangci-lint + go test ./... + web build).
291 lines
11 KiB
JSON
291 lines
11 KiB
JSON
{
|
|
"__comment": "Grafana dashboard for pad MCP + OAuth observability (PLAN-943 TASK-961). Import via Grafana UI or provisioned dashboards. Datasource is templated — set $datasource on import to your Prometheus source. Refresh interval: 30s. Time range default: last 6h.",
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "grafana",
|
|
"uid": "-- Grafana --"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"description": "MCP traffic + OAuth flow observability for pad-cloud. Pairs with the audit log at /console/admin/mcp-audit for per-row drill-down.",
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"id": 100,
|
|
"panels": [],
|
|
"title": "MCP traffic",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "MCP tool-call rate broken out by tool. Spikes here without a corresponding active-sessions change usually means one client looping; cross-reference the audit log.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": { "drawStyle": "line", "fillOpacity": 10, "lineWidth": 1 },
|
|
"unit": "reqps"
|
|
}
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
|
"id": 1,
|
|
"options": { "legend": { "displayMode": "table", "placement": "right", "calcs": ["mean", "max"] } },
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "sum by (tool) (rate(pad_mcp_tool_calls_total[5m]))",
|
|
"legendFormat": "{{tool}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "MCP tool-call rate by tool",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "p50 / p95 / p99 tool-call latency. Watch p99 — the audit log records latency_ms per row but doesn't aggregate.",
|
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
|
"id": 2,
|
|
"options": { "legend": { "displayMode": "table", "placement": "right" } },
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.50, sum by (le, tool) (rate(pad_mcp_tool_call_duration_seconds_bucket[5m])))",
|
|
"legendFormat": "p50 {{tool}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.95, sum by (le, tool) (rate(pad_mcp_tool_call_duration_seconds_bucket[5m])))",
|
|
"legendFormat": "p95 {{tool}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.99, sum by (le, tool) (rate(pad_mcp_tool_call_duration_seconds_bucket[5m])))",
|
|
"legendFormat": "p99 {{tool}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "MCP tool-call latency (p50/p95/p99)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Per-status outcome breakdown. Sustained spikes in 'denied' or 'error' usually pair with an attention item in the audit log.",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": { "mode": "palette-classic" },
|
|
"custom": { "drawStyle": "bars", "fillOpacity": 60, "stacking": { "mode": "normal" } },
|
|
"unit": "reqps"
|
|
}
|
|
},
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 },
|
|
"id": 3,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "sum by (status) (rate(pad_mcp_tool_calls_total[5m]))",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "MCP outcomes by status",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Authorization denials by reason. Sustained 'audience_mismatch' = client misconfigured; sustained 'rate_limited' = consider raising the per-token bucket; sustained 'workspace_not_in_allowlist' = client requesting workspaces the user didn't grant.",
|
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 },
|
|
"id": 4,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "sum by (reason) (rate(pad_mcp_authz_denials_total[5m]))",
|
|
"legendFormat": "{{reason}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "MCP authorization denials by reason",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Active Streamable HTTP sessions (initialize - DELETE). Drift up over time? Probably client crashes; expected to recover at server restart.",
|
|
"fieldConfig": { "defaults": { "unit": "short" } },
|
|
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 17 },
|
|
"id": 5,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "pad_mcp_active_sessions",
|
|
"legendFormat": "active sessions",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "MCP active sessions",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Top users by tool-call rate over the last 5 minutes. High-cardinality panel — useful for spotting one client driving most of the load.",
|
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
|
"gridPos": { "h": 6, "w": 16, "x": 8, "y": 17 },
|
|
"id": 6,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "topk(10, sum by (user_id) (rate(pad_mcp_tool_calls_total[5m])))",
|
|
"legendFormat": "{{user_id}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Top 10 MCP users by tool-call rate",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
|
|
"id": 200,
|
|
"panels": [],
|
|
"title": "OAuth flows",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Authorization-flow stage transitions. Healthy ratio: completed >> abandoned >> failed. A sudden 'failed' spike = clients with bad config; sustained high 'abandoned' = consent UI clarity issue.",
|
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
|
|
"id": 7,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "sum by (stage) (rate(pad_oauth_flows_total[5m]))",
|
|
"legendFormat": "{{stage}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "OAuth flow events by stage",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "p95 latency per OAuth handler. Slow 'token' = fosite signing or DB; slow 'authorize' = workspace-list fetch.",
|
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
|
|
"id": 8,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.95, sum by (le, stage) (rate(pad_oauth_flow_duration_seconds_bucket[5m])))",
|
|
"legendFormat": "p95 {{stage}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "OAuth flow latency (p95)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Active access tokens (active=1 in storage). Caveat: includes tokens past their expires_in but not yet pruned; expected to be an upper bound.",
|
|
"fieldConfig": { "defaults": { "unit": "short" } },
|
|
"gridPos": { "h": 6, "w": 8, "x": 0, "y": 32 },
|
|
"id": 9,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "pad_oauth_active_tokens",
|
|
"legendFormat": "active tokens",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "OAuth active tokens",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Token revocations by reason. Sudden 'rotated' spike = clients churning refresh tokens (expected on bursts); 'replayed' is the canary for theft detection.",
|
|
"fieldConfig": { "defaults": { "unit": "reqps" } },
|
|
"gridPos": { "h": 6, "w": 8, "x": 8, "y": 32 },
|
|
"id": 10,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "sum by (reason) (rate(pad_oauth_token_revocations_total[5m]))",
|
|
"legendFormat": "{{reason}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "OAuth token revocations by reason",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"description": "Token TTL distribution at revocation time. p50 near the configured access-token TTL = healthy natural-expiry pattern; p50 near zero = something is revoking tokens immediately (likely abuse mitigation or misconfig).",
|
|
"fieldConfig": { "defaults": { "unit": "s" } },
|
|
"gridPos": { "h": 6, "w": 8, "x": 16, "y": 32 },
|
|
"id": 11,
|
|
"targets": [
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.50, sum by (le) (rate(pad_oauth_token_ttl_seconds_bucket[1h])))",
|
|
"legendFormat": "p50",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
|
"expr": "histogram_quantile(0.95, sum by (le) (rate(pad_oauth_token_ttl_seconds_bucket[1h])))",
|
|
"legendFormat": "p95",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "OAuth token TTL at revocation (p50/p95)",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"refresh": "30s",
|
|
"schemaVersion": 39,
|
|
"tags": ["pad", "mcp", "oauth"],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": { "selected": false, "text": "Prometheus", "value": "Prometheus" },
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"multi": false,
|
|
"name": "datasource",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
}
|
|
]
|
|
},
|
|
"time": { "from": "now-6h", "to": "now" },
|
|
"timezone": "",
|
|
"title": "Pad — MCP + OAuth observability",
|
|
"uid": "pad-mcp-oauth",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|