From 4855095446eda1877de558e688635528456271d9 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 6 Aug 2026 21:59:59 +0800 Subject: [PATCH] obs: mirror log attributes into loki lines (#5776) Co-authored-by: heihutu --- .docker/observability/README.md | 15 ++++++++++++++ .docker/observability/README_ZH.md | 9 +++++++++ .../observability/otel-collector-config.yaml | 20 +++++++++++++++++-- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.docker/observability/README.md b/.docker/observability/README.md index cb4b9a881..8781c1b4c 100644 --- a/.docker/observability/README.md +++ b/.docker/observability/README.md @@ -170,6 +170,10 @@ Important behavior notes: - Logs and metrics usually appear during startup, so seeing those two signals first is expected. +- The OpenTelemetry bridge sends `tracing` fields as log attributes. Loki stores + those attributes as structured metadata, and the Collector also mirrors the + common troubleshooting fields into the log line so simple line filters can + find them. - Visible trace data usually requires real HTTP/S3/gRPC request traffic after startup, because request-path spans are created on demand. - `RUSTFS_OBS_LOGGER_LEVEL=info` keeps the top-level request span but filters @@ -195,6 +199,17 @@ curl -I http://127.0.0.1:9000/health/ready # Jaeger: http://localhost:16686 ``` +For a structured RustFS log such as an inter-node RPC authentication failure, +the Loki line now includes fields such as `event`, `component`, `subsystem`, +`failure_reason`, `rpc_service`, `rpc_method`, and `expected_audience`. Useful +LogQL checks: + +```logql +{service_name="RustFS"} |= "RPC signature verification failed" +{service_name="RustFS"} |= "failure_reason=" +{service_name="RustFS"} | failure_reason != "" +``` + If logs and metrics are present but traces are sparse, the most common cause is "no real request traffic yet" or "`info` level filtered nested spans", not an OTLP routing failure. diff --git a/.docker/observability/README_ZH.md b/.docker/observability/README_ZH.md index f42674f6c..22d46ee78 100644 --- a/.docker/observability/README_ZH.md +++ b/.docker/observability/README_ZH.md @@ -169,6 +169,7 @@ RustFS 会自动在该基础 URL 后补全： 需要注意： - 启动阶段通常会先看到日志和指标，因此“先有日志/指标、后有 trace”是正常现象。 +- OpenTelemetry bridge 会把 `tracing` 字段作为日志 attributes 发送。Loki 会将这些 attributes 存为 structured metadata，同时 Collector 会把常用排障字段镜像进日志行，方便用简单的行内容过滤直接查到。 - 可见的 trace 数据通常依赖启动后的真实 HTTP/S3/gRPC 请求流量，因为请求路径上的 span 是按需创建的。 - `RUSTFS_OBS_LOGGER_LEVEL=info` 会保留顶层请求 span，但会过滤掉很多 `debug` 级别的嵌套 span。 如果 Tempo 或 Jaeger 中的 trace 看起来很稀疏，建议先改成 `RUSTFS_OBS_LOGGER_LEVEL=debug`，再判断是否是 collector 或 Tempo 问题。 @@ -192,6 +193,14 @@ curl -I http://127.0.0.1:9000/health/ready # Jaeger: http://localhost:16686 ``` +对于 RustFS 结构化日志，例如节点间 RPC 鉴权失败，Loki 日志行现在会包含 `event`、`component`、`subsystem`、`failure_reason`、`rpc_service`、`rpc_method`、`expected_audience` 等字段。常用 LogQL 检查： + +```logql +{service_name="RustFS"} |= "RPC signature verification failed" +{service_name="RustFS"} |= "failure_reason=" +{service_name="RustFS"} | failure_reason != "" +``` + 如果日志和指标已经正常，但 trace 仍然稀疏，最常见的原因通常是 “还没有真实请求流量”或“`info` 级别过滤了嵌套 span”，而不是 OTLP 路由失败。 diff --git a/.docker/observability/otel-collector-config.yaml b/.docker/observability/otel-collector-config.yaml index da566c364..b9ae67032 100644 --- a/.docker/observability/otel-collector-config.yaml +++ b/.docker/observability/otel-collector-config.yaml @@ -29,11 +29,27 @@ processors: limit_mib: 1024 spike_limit_mib: 256 transform/logs: + error_mode: ignore log_statements: - context: log statements: - - set(attributes["message"], body.string) - - set(attributes["log.body"], body.string) + - set(attributes["message"], body.string) where IsString(body) + - set(attributes["log.body"], body.string) where IsString(body) + - set(body, Concat([body, " event=", attributes["event"]], "")) where IsString(body) and attributes["event"] != nil + - set(body, Concat([body, " component=", attributes["component"]], "")) where IsString(body) and attributes["component"] != nil + - set(body, Concat([body, " subsystem=", attributes["subsystem"]], "")) where IsString(body) and attributes["subsystem"] != nil + - set(body, Concat([body, " state=", attributes["state"]], "")) where IsString(body) and attributes["state"] != nil + - set(body, Concat([body, " result=", attributes["result"]], "")) where IsString(body) and attributes["result"] != nil + - set(body, Concat([body, " reason=", attributes["reason"]], "")) where IsString(body) and attributes["reason"] != nil + - set(body, Concat([body, " failure_reason=", attributes["failure_reason"]], "")) where IsString(body) and attributes["failure_reason"] != nil + - set(body, Concat([body, " rpc_path=", attributes["rpc_path"]], "")) where IsString(body) and attributes["rpc_path"] != nil + - set(body, Concat([body, " rpc_service=", attributes["rpc_service"]], "")) where IsString(body) and attributes["rpc_service"] != nil + - set(body, Concat([body, " rpc_method=", attributes["rpc_method"]], "")) where IsString(body) and attributes["rpc_method"] != nil + - set(body, Concat([body, " expected_audience=", attributes["expected_audience"]], "")) where IsString(body) and attributes["expected_audience"] != nil + - set(body, Concat([body, " peer_addr=", attributes["peer_addr"]], "")) where IsString(body) and attributes["peer_addr"] != nil + - set(body, Concat([body, " replay_scope_bootstrap_allowed=", attributes["replay_scope_bootstrap_allowed"]], "")) where IsString(body) and attributes["replay_scope_bootstrap_allowed"] != nil + - set(body, Concat([body, " error=", attributes["error"]], "")) where IsString(body) and attributes["error"] != nil + - set(body, Concat([body, " exception_message=", attributes["exception.message"]], "")) where IsString(body) and attributes["exception.message"] != nil exporters: otlp/tempo: