Refactor telemetry initialization for non-production environments (#789)

* add dep `scopeguard`

* improve for tracing

* fix

* fix

* improve code for import

* add logger trace id

* fix

* fix

* fix

* fix

* fix

crates/obs/src/telemetry.rs

@@ -74,6 +74,8 @@ pub struct OtelGuard {
     logger_provider: Option<SdkLoggerProvider>,
     // Add a flexi_logger handle to keep the logging alive
     _flexi_logger_handles: Option<flexi_logger::LoggerHandle>,
+    // WorkerGuard for writing tracing files
+    _tracing_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
 }
 
 // Implement debug manually and avoid relying on all fields to implement debug
@@ -84,6 +86,7 @@ impl std::fmt::Debug for OtelGuard {
             .field("meter_provider", &self.meter_provider.is_some())
             .field("logger_provider", &self.logger_provider.is_some())
             .field("_flexi_logger_handles", &self._flexi_logger_handles.is_some())
+            .field("_tracing_guard", &self._tracing_guard.is_some())
             .finish()
     }
 }
@@ -277,13 +280,18 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
                     .with_thread_names(true)
                     .with_thread_ids(true)
                     .with_file(true)
-                    .with_line_number(true);
-
-                // Only add full span events tracking in the development environment
-                if !is_production {
-                    layer = layer.with_span_events(FmtSpan::FULL);
-                }
+                    .with_line_number(true)
+                    .json()
+                    .with_current_span(true)
+                    .with_span_list(true);
+                let span_event = if is_production {
+                    FmtSpan::CLOSE
+                } else {
+                    // Only add full span events tracking in the development environment
+                    FmtSpan::FULL
+                };
 
+                layer = layer.with_span_events(span_event);
                 layer.with_filter(build_env_filter(logger_level, None))
             };
 
@@ -321,6 +329,7 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
             meter_provider: Some(meter_provider),
             logger_provider: Some(logger_provider),
             _flexi_logger_handles: flexi_logger_handle,
+            _tracing_guard: None,
         };
     }
 
@@ -352,6 +361,47 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
         }
     }
 
+    if endpoint.is_empty() && !is_production {
+        // Create a file appender (rolling by day), add the -tracing suffix to the file name to avoid conflicts
+        let file_appender = tracing_appender::rolling::hourly(log_directory, format!("{log_filename}-tracing.log"));
+        let (nb_writer, guard) = tracing_appender::non_blocking(file_appender);
+
+        let enable_color = std::io::stdout().is_terminal();
+        let fmt_layer = tracing_subscriber::fmt::layer()
+            .with_timer(LocalTime::rfc_3339())
+            .with_target(true)
+            .with_ansi(enable_color)
+            .with_thread_names(true)
+            .with_thread_ids(true)
+            .with_file(true)
+            .with_line_number(true)
+            .with_writer(nb_writer) // Specify writing file
+            .json()
+            .with_current_span(true)
+            .with_span_list(true)
+            .with_span_events(FmtSpan::CLOSE); // Log span lifecycle events, including trace_id
+
+        let env_filter = build_env_filter(logger_level, None);
+
+        // Use registry() to register fmt_layer directly to ensure trace_id is output to the log
+        tracing_subscriber::registry()
+            .with(env_filter)
+            .with(ErrorLayer::default())
+            .with(fmt_layer)
+            .init();
+
+        info!("Tracing telemetry initialized for non-production with trace_id logging.");
+        IS_OBSERVABILITY_ENABLED.set(false).ok();
+
+        return OtelGuard {
+            tracer_provider: None,
+            meter_provider: None,
+            logger_provider: None,
+            _flexi_logger_handles: None,
+            _tracing_guard: Some(guard),
+        };
+    }
+
     // Build log cutting conditions
     let rotation_criterion = match (config.log_rotation_time.as_deref(), config.log_rotation_size_mb) {
         // Cut by time and size at the same time
@@ -447,7 +497,6 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
 
     // Environment-aware stdout configuration
     flexi_logger_builder = flexi_logger_builder.duplicate_to_stdout(level_filter);
-
     // Only add stdout formatting and startup messages in non-production environments
     if !is_production {
         flexi_logger_builder = flexi_logger_builder
@@ -496,6 +545,7 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
         meter_provider: None,
         logger_provider: None,
         _flexi_logger_handles: flexi_logger_handle,
+        _tracing_guard: None,
     }
 }