Refactor Telemetry Initialization and Environment Utilities (#811)

* improve code for metrics

* improve code for metrics

* fix

* fix

* Refactor telemetry initialization and environment functions ordering

- Reorder functions in envs.rs by type size (8-bit to 64-bit, signed before unsigned) and add missing variants like get_env_opt_u16.
- Optimize init_telemetry to support three modes: stdout logging (default error level with span tracing), file rolling logs (size-based with retention), and HTTP-based observability with sub-endpoints (trace, metric, log) falling back to unified endpoint.
- Fix stdout logging issue by retaining WorkerGuard in OtelGuard to prevent premature release of async writer threads.
- Enhance observability mode with HTTP protocol, compression, and proper resource management.
- Update OtelGuard to include tracing_guard for stdout and flexi_logger_handles for file logging.
- Improve error handling and configuration extraction in OtelConfig.

* fix

* up

* fix

* fix

* improve code for obs

* fix

* fix

.docker/observability/docker-compose.yml

@@ -16,7 +16,7 @@ services:
 
   tempo-init:
     image: busybox:latest
-    command: ["sh", "-c", "chown -R 10001:10001 /var/tempo"]
+    command: [ "sh", "-c", "chown -R 10001:10001 /var/tempo" ]
     volumes:
       - ./tempo-data:/var/tempo
     user: root
@@ -39,7 +39,7 @@ services:
       - otel-network
 
   otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.129.1
+    image: otel/opentelemetry-collector-contrib:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
@@ -55,7 +55,7 @@ services:
     networks:
       - otel-network
   jaeger:
-    image: jaegertracing/jaeger:2.8.0
+    image: jaegertracing/jaeger:latest
     environment:
       - TZ=Asia/Shanghai
     ports:
@@ -65,17 +65,21 @@ services:
     networks:
       - otel-network
   prometheus:
-    image: prom/prometheus:v3.4.2
+    image: prom/prometheus:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--web.enable-otlp-receiver' # Enable OTLP
+      - '--enable-feature=promql-experimental-functions' # Enable info()
     networks:
       - otel-network
   loki:
-    image: grafana/loki:3.5.1
+    image: grafana/loki:latest
     environment:
       - TZ=Asia/Shanghai
     volumes:
@@ -86,7 +90,7 @@ services:
     networks:
       - otel-network
   grafana:
-    image: grafana/grafana:12.0.2
+    image: grafana/grafana:latest
     ports:
       - "3000:3000"  # Web UI
     volumes:

.docker/observability/grafana-datasources.yaml

@@ -29,4 +29,80 @@ datasources:
       serviceMap:
         datasourceUid: prometheus
       streamingEnabled:
-        search: true
+        search: true
+      tracesToLogsV2:
+        # Field with an internal link pointing to a logs data source in Grafana.
+        # datasourceUid value must match the uid value of the logs data source.
+        datasourceUid: 'loki'
+        spanStartTimeShift: '-1h'
+        spanEndTimeShift: '1h'
+        tags: [ 'job', 'instance', 'pod', 'namespace' ]
+        filterByTraceID: false
+        filterBySpanID: false
+        customQuery: true
+        query: 'method="$${__span.tags.method}"'
+        tracesToMetrics:
+          datasourceUid: 'prom'
+          spanStartTimeShift: '-1h'
+          spanEndTimeShift: '1h'
+          tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
+          queries:
+            - name: 'Sample query'
+              query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
+        tracesToProfiles:
+          datasourceUid: 'grafana-pyroscope-datasource'
+          tags: [ 'job', 'instance', 'pod', 'namespace' ]
+          profileTypeId: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds'
+          customQuery: true
+          query: 'method="$${__span.tags.method}"'
+        serviceMap:
+          datasourceUid: 'prometheus'
+        nodeGraph:
+          enabled: true
+        search:
+          hide: false
+        traceQuery:
+          timeShiftEnabled: true
+          spanStartTimeShift: '-1h'
+          spanEndTimeShift: '1h'
+        spanBar:
+          type: 'Tag'
+          tag: 'http.path'
+        streamingEnabled:
+          search: true
+  - name: Jaeger
+    type: jaeger
+    uid: Jaeger
+    url: http://jaeger:16686
+    basicAuth: false
+    access: proxy
+    readOnly: false
+    isDefault: false
+    jsonData:
+      tracesToLogsV2:
+        # Field with an internal link pointing to a logs data source in Grafana.
+        # datasourceUid value must match the uid value of the logs data source.
+        datasourceUid: 'loki'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: [ 'job', 'instance', 'pod', 'namespace' ]
+        filterByTraceID: false
+        filterBySpanID: false
+        customQuery: true
+        query: 'method="$${__span.tags.method}"'
+      tracesToMetrics:
+        datasourceUid: 'prom'
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: [ { key: 'service.name', value: 'service' }, { key: 'job' } ]
+        queries:
+          - name: 'Sample query'
+            query: 'sum(rate(traces_spanmetrics_latency_bucket{$$__tags}[5m]))'
+      nodeGraph:
+        enabled: true
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      spanBar:
+        type: 'None'

.docker/observability/loki-config.yaml

@@ -63,6 +63,7 @@ ruler:
 frontend:
   encoding: protobuf
 
+
 # By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
 # analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
 #

.docker/observability/otel-collector-config.yaml

@@ -43,7 +43,6 @@ exporters:
     send_timestamps: true  # 发送时间戳
     # enable_open_metrics: true
   otlphttp/loki: # Loki 导出器，用于日志数据
-    # endpoint: "http://loki:3100/otlp/v1/logs"
     endpoint: "http://loki:3100/otlp/v1/logs"
     tls:
       insecure: true

.docker/observability/prometheus.yml

@@ -13,16 +13,43 @@
 # limitations under the License.
 
 global:
-  scrape_interval: 5s  # 刮取间隔
+  scrape_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
 
 scrape_configs:
   - job_name: 'otel-collector'
     static_configs:
-      - targets: [ 'otel-collector:8888' ]  # 从 Collector 刮取指标
+      - targets: [ 'otel-collector:8888' ]  # Scrape metrics from Collector
   - job_name: 'otel-metrics'
     static_configs:
-      - targets: [ 'otel-collector:8889' ]  # 应用指标
+      - targets: [ 'otel-collector:8889' ]  # Application indicators
   - job_name: 'tempo'
     static_configs:
-      - targets: [ 'tempo:3200' ]
-      
+      - targets: [ 'tempo:3200' ]  # Scrape metrics from Tempo
+
+otlp:
+  # Recommended attributes to be promoted to labels.
+  promote_resource_attributes:
+    - service.instance.id
+    - service.name
+    - service.namespace
+    - cloud.availability_zone
+    - cloud.region
+    - container.name
+    - deployment.environment.name
+    - k8s.cluster.name
+    - k8s.container.name
+    - k8s.cronjob.name
+    - k8s.daemonset.name
+    - k8s.deployment.name
+    - k8s.job.name
+    - k8s.namespace.name
+    - k8s.pod.name
+    - k8s.replicaset.name
+    - k8s.statefulset.name
+  # Ingest OTLP data keeping all characters in metric/label names.
+  translation_strategy: NoUTF8EscapingWithSuffixes
+
+storage:
+  # OTLP is a push-based protocol, Out of order samples is a common scenario.
+  tsdb:
+    out_of_order_time_window: 30m