mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-26 05:56:50 +00:00
feat(obs): add init_obs_with_config API and signature guard test (#2175)
Signed-off-by: houseme <housemecn@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -25,20 +25,27 @@
|
||||
//!
|
||||
//! The function [`init_local_logging`] is the single entry point for both
|
||||
//! cases; callers do **not** need to distinguish between stdout and file modes.
|
||||
//!
|
||||
//! The file-backed mode delegates retention and compression to
|
||||
//! [`crate::cleaner`], which keeps the logging setup code focused on subscriber
|
||||
//! construction while still allowing periodic housekeeping in the background.
|
||||
|
||||
use super::guard::OtelGuard;
|
||||
use crate::TelemetryError;
|
||||
use crate::cleaner::LogCleaner;
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::OBSERVABILITY_METRIC_ENABLED;
|
||||
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
|
||||
DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL,
|
||||
DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
|
||||
DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM, DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN,
|
||||
DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES,
|
||||
DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS, DEFAULT_OBS_LOG_PARALLEL_COMPRESS,
|
||||
DEFAULT_OBS_LOG_PARALLEL_WORKERS, DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL, DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP,
|
||||
DEFAULT_OBS_LOG_ZSTD_WORKERS,
|
||||
};
|
||||
use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED};
|
||||
use std::sync::Arc;
|
||||
@@ -111,6 +118,8 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
let env_filter = build_env_filter(logger_level, None);
|
||||
let (nb, guard) = tracing_appender::non_blocking(std::io::stdout());
|
||||
|
||||
// Keep stdout formatting JSON-shaped even in local-only mode so operators
|
||||
// can ship the same log schema to external collectors if needed.
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
@@ -131,7 +140,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
.with(fmt_layer)
|
||||
.init();
|
||||
|
||||
OBSERVABILITY_METRIC_ENABLED.set(false).ok();
|
||||
set_observability_metric_enabled(false);
|
||||
counter!("rustfs.start.total").increment(1);
|
||||
info!("Init stdout logging (level: {})", logger_level);
|
||||
|
||||
@@ -154,6 +163,10 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
/// Called by [`init_local_logging`] when a log directory is present.
|
||||
/// Handles directory creation, permission enforcement (Unix), file appender
|
||||
/// setup, optional stdout mirror, and log-cleanup task spawning.
|
||||
///
|
||||
/// The function intentionally performs all fallible filesystem preparation
|
||||
/// before registering the subscriber so startup failures are reported early and
|
||||
/// do not leave partially initialized tracing state behind.
|
||||
fn init_file_logging_internal(
|
||||
config: &OtelConfig,
|
||||
log_directory: &str,
|
||||
@@ -181,11 +194,9 @@ fn init_file_logging_internal(
|
||||
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
|
||||
.to_lowercase();
|
||||
|
||||
// Determine match mode from config, defaulting to Suffix
|
||||
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
|
||||
Some("prefix") => FileMatchMode::Prefix,
|
||||
_ => FileMatchMode::Suffix,
|
||||
};
|
||||
// Match mode controls how the rolling filename is recognized later by the
|
||||
// cleaner. Suffix mode fits timestamp-prefixed filenames especially well.
|
||||
let match_mode = FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE));
|
||||
|
||||
let rotation = match rotation_str.as_str() {
|
||||
"minutely" => Rotation::Minutely,
|
||||
@@ -207,7 +218,8 @@ fn init_file_logging_internal(
|
||||
let env_filter = build_env_filter(logger_level, None);
|
||||
let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL };
|
||||
|
||||
// File layer writes JSON without ANSI codes.
|
||||
// File output stays machine-readable and free of ANSI sequences so the
|
||||
// resulting files are safe to parse or ship to log processors.
|
||||
let file_layer = tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
@@ -223,7 +235,8 @@ fn init_file_logging_internal(
|
||||
.with_span_events(span_events.clone());
|
||||
|
||||
// Optional stdout mirror: enabled explicitly via `log_stdout_enabled`, or
|
||||
// unconditionally in non-production environments.
|
||||
// unconditionally in non-production environments so developers still see
|
||||
// immediate terminal output while file rotation remains enabled.
|
||||
let (stdout_layer, stdout_guard) = if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
|
||||
let (stdout_nb, stdout_guard) = tracing_appender::non_blocking(std::io::stdout());
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
@@ -255,7 +268,7 @@ fn init_file_logging_internal(
|
||||
.with(stdout_layer)
|
||||
.init();
|
||||
|
||||
OBSERVABILITY_METRIC_ENABLED.set(false).ok();
|
||||
set_observability_metric_enabled(false);
|
||||
|
||||
// ── 5. Start background cleanup task ─────────────────────────────────────
|
||||
let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
|
||||
@@ -284,6 +297,8 @@ fn init_file_logging_internal(
|
||||
/// Tightens permissions to `0755` if the directory is more permissive.
|
||||
/// This prevents world-writable log directories from being a security hazard.
|
||||
/// No-ops if permissions are already `0755` or stricter.
|
||||
///
|
||||
/// The function never broadens permissions; it is strictly a hardening step.
|
||||
#[cfg(unix)]
|
||||
pub fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> {
|
||||
use std::fs::Permissions;
|
||||
@@ -325,6 +340,10 @@ pub fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError>
|
||||
/// Tokio runtime and should be aborted (via the returned `JoinHandle`) when
|
||||
/// the application shuts down.
|
||||
///
|
||||
/// The asynchronous loop itself remains lightweight: each cleanup pass is
|
||||
/// delegated to `spawn_blocking` because directory traversal, compression, and
|
||||
/// deletion are inherently blocking filesystem operations.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Observability config containing cleanup parameters.
|
||||
/// * `log_directory` - Directory path of the rolling log files.
|
||||
@@ -341,16 +360,14 @@ pub fn spawn_cleanup_task(
|
||||
keep_files: usize,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let log_dir = std::path::PathBuf::from(log_directory);
|
||||
// Use suffix matching for log files like "2026-03-01-06-21.rustfs.log"
|
||||
// where "rustfs.log" is the suffix.
|
||||
// Use suffix matching for log files like `2026-03-01-06-21.rustfs.log`
|
||||
// where `rustfs.log` is the stable suffix generated by the rolling appender.
|
||||
let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
|
||||
let active_filename = file_pattern.clone();
|
||||
|
||||
// Determine match mode from config, defaulting to Suffix
|
||||
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
|
||||
Some("prefix") => FileMatchMode::Prefix,
|
||||
_ => FileMatchMode::Suffix,
|
||||
};
|
||||
// Determine match mode from config, defaulting to the repository-wide
|
||||
// observability setting when the caller leaves it unset.
|
||||
let match_mode = FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE));
|
||||
|
||||
let max_total_size = config
|
||||
.log_max_total_size_bytes
|
||||
@@ -362,6 +379,21 @@ pub fn spawn_cleanup_task(
|
||||
let gzip_level = config
|
||||
.log_gzip_compression_level
|
||||
.unwrap_or(DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL);
|
||||
let compression_algorithm = CompressionAlgorithm::from_config_str(
|
||||
config
|
||||
.log_compression_algorithm
|
||||
.as_deref()
|
||||
.unwrap_or(DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM),
|
||||
);
|
||||
let parallel_compress = config.log_parallel_compress.unwrap_or(DEFAULT_OBS_LOG_PARALLEL_COMPRESS);
|
||||
let parallel_workers = config.log_parallel_workers.unwrap_or(DEFAULT_OBS_LOG_PARALLEL_WORKERS);
|
||||
let zstd_level = config
|
||||
.log_zstd_compression_level
|
||||
.unwrap_or(DEFAULT_OBS_LOG_ZSTD_COMPRESSION_LEVEL);
|
||||
let zstd_fallback_to_gzip = config
|
||||
.log_zstd_fallback_to_gzip
|
||||
.unwrap_or(DEFAULT_OBS_LOG_ZSTD_FALLBACK_TO_GZIP);
|
||||
let zstd_workers = config.log_zstd_workers.unwrap_or(DEFAULT_OBS_LOG_ZSTD_WORKERS);
|
||||
let retention_days = config
|
||||
.log_compressed_file_retention_days
|
||||
.unwrap_or(DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS);
|
||||
@@ -387,6 +419,14 @@ pub fn spawn_cleanup_task(
|
||||
.max_single_file_size_bytes(max_single_file_size)
|
||||
.compress_old_files(compress)
|
||||
.gzip_compression_level(gzip_level)
|
||||
// Compression behavior stays fully config-driven, but the builder
|
||||
// clamps unsafe numeric values and preserves sensible defaults.
|
||||
.compression_algorithm(compression_algorithm)
|
||||
.parallel_compress(parallel_compress)
|
||||
.parallel_workers(parallel_workers)
|
||||
.zstd_compression_level(zstd_level)
|
||||
.zstd_fallback_to_gzip(zstd_fallback_to_gzip)
|
||||
.zstd_workers(zstd_workers)
|
||||
.compressed_file_retention_days(retention_days)
|
||||
.exclude_patterns(exclude_patterns)
|
||||
.delete_empty_files(delete_empty)
|
||||
@@ -395,17 +435,37 @@ pub fn spawn_cleanup_task(
|
||||
.build(),
|
||||
);
|
||||
|
||||
info!(
|
||||
compression_algorithm = %compression_algorithm,
|
||||
parallel_compress,
|
||||
parallel_workers,
|
||||
zstd_level,
|
||||
zstd_fallback_to_gzip,
|
||||
zstd_workers,
|
||||
"log cleaner compression profile configured"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
|
||||
loop {
|
||||
// Wait for the next scheduled tick before dispatching another pass.
|
||||
// The blocking filesystem work runs on a dedicated blocking thread.
|
||||
interval.tick().await;
|
||||
let cleaner_clone = cleaner.clone();
|
||||
let result = tokio::task::spawn_blocking(move || cleaner_clone.cleanup()).await;
|
||||
|
||||
match result {
|
||||
Ok(Ok(_)) => {} // Success
|
||||
Ok(Err(e)) => tracing::warn!("Log cleanup failed: {}", e),
|
||||
Err(e) => tracing::warn!("Log cleanup task panicked: {}", e),
|
||||
Ok(Ok(_)) => {
|
||||
counter!(METRIC_LOG_CLEANER_RUNS_TOTAL).increment(1);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
counter!(METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL).increment(1);
|
||||
tracing::warn!("Log cleanup failed: {}", e);
|
||||
}
|
||||
Err(e) => {
|
||||
counter!(METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL).increment(1);
|
||||
tracing::warn!("Log cleanup task panicked: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -418,6 +478,7 @@ mod tests {
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
/// Invalid file names should be reported as errors instead of panicking.
|
||||
fn test_init_file_logging_invalid_filename_does_not_panic() {
|
||||
let temp_dir = tempdir().expect("create temp dir");
|
||||
let temp_path = temp_dir.path().to_str().expect("temp dir path is utf-8");
|
||||
|
||||
@@ -31,10 +31,14 @@
|
||||
//!
|
||||
//! All exporters use **HTTP binary** (Protobuf) encoding with **gzip**
|
||||
//! compression for efficiency over the wire.
|
||||
//!
|
||||
//! If log export is not configured, this module deliberately falls back to the
|
||||
//! same rolling-file logging path used by the local backend so applications can
|
||||
//! combine OTLP traces/metrics with on-disk logs.
|
||||
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::OBSERVABILITY_METRIC_ENABLED;
|
||||
use crate::global::set_observability_metric_enabled;
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::guard::OtelGuard;
|
||||
// Import helper functions from local.rs (sibling module)
|
||||
@@ -53,7 +57,7 @@ use opentelemetry_sdk::{
|
||||
metrics::{PeriodicReader, SdkMeterProvider},
|
||||
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
|
||||
};
|
||||
use rustfs_config::observability::DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES;
|
||||
use rustfs_config::observability::{DEFAULT_OBS_LOG_MATCH_MODE, DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES};
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED,
|
||||
DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
|
||||
@@ -97,6 +101,8 @@ pub(super) fn init_observability_http(
|
||||
is_production: bool,
|
||||
) -> Result<OtelGuard, TelemetryError> {
|
||||
// ── Resource & sampling ──────────────────────────────────────────────────
|
||||
// Build the common resource once so all enabled signals report the same
|
||||
// service identity and deployment metadata.
|
||||
let res = build_resource(config);
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME).to_owned();
|
||||
let use_stdout = config.use_stdout.unwrap_or(!is_production);
|
||||
@@ -134,8 +140,9 @@ pub(super) fn init_observability_http(
|
||||
}
|
||||
});
|
||||
|
||||
// If log_endpoint is not explicitly set, fallback to root_ep/v1/logs ONLY if root_ep is present.
|
||||
// If both are empty, log_ep is empty, which triggers the fallback to file logging logic.
|
||||
// If `log_endpoint` is not explicitly set, fall back to `root_ep/v1/logs`
|
||||
// only when a root endpoint exists. An empty result intentionally triggers
|
||||
// the file-logging path below instead of silently disabling application logs.
|
||||
let log_ep: String = config
|
||||
.log_endpoint
|
||||
.as_deref()
|
||||
@@ -159,6 +166,8 @@ pub(super) fn init_observability_http(
|
||||
let profiling_agent = init_profiler(config);
|
||||
|
||||
// ── Logger Logic ──────────────────────────────────────────────────────────
|
||||
// Logging is the only signal that may intentionally route to either OTLP
|
||||
// or local files depending on configuration completeness.
|
||||
let mut logger_provider: Option<SdkLoggerProvider> = None;
|
||||
let mut otel_bridge = None;
|
||||
let mut file_layer_opt = None; // File layer (File mode)
|
||||
@@ -179,11 +188,14 @@ pub(super) fn init_observability_http(
|
||||
.as_ref()
|
||||
.map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
|
||||
|
||||
// Note: We do NOT create a separate `fmt_layer_opt` here; stdout behavior is driven by the provider.
|
||||
// No separate formatting layer is added here; when OTLP logging is
|
||||
// active, the OpenTelemetry bridge is the authoritative sink for
|
||||
// `tracing` events unless local file logging is needed as a fallback.
|
||||
}
|
||||
let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL };
|
||||
// ── Case 2: File Logging
|
||||
// Supplement: If log_directory is set and no OTLP log endpoint is configured, we enable file logging logic.
|
||||
// If a log directory is configured and OTLP log export is unavailable, use
|
||||
// the same rolling-file behavior as the local-only telemetry backend.
|
||||
if let Some(log_directory) = config.log_directory.as_deref().filter(|s| !s.is_empty())
|
||||
&& logger_provider.is_none()
|
||||
{
|
||||
@@ -204,10 +216,7 @@ pub(super) fn init_observability_http(
|
||||
.as_deref()
|
||||
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
|
||||
.to_lowercase();
|
||||
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
|
||||
Some("prefix") => FileMatchMode::Prefix,
|
||||
_ => FileMatchMode::Suffix,
|
||||
};
|
||||
let match_mode = FileMatchMode::from_config_str(config.log_match_mode.as_deref().unwrap_or(DEFAULT_OBS_LOG_MATCH_MODE));
|
||||
let rotation = match rotation_str.as_str() {
|
||||
"minutely" => Rotation::Minutely,
|
||||
"hourly" => Rotation::Hourly,
|
||||
@@ -241,7 +250,8 @@ pub(super) fn init_observability_http(
|
||||
.with_filter(build_env_filter(logger_level, None)),
|
||||
);
|
||||
|
||||
// Cleanup task
|
||||
// The cleanup task keeps rotated files bounded while the OTLP trace and
|
||||
// metric exporters continue to operate independently.
|
||||
cleanup_handle = Some(spawn_cleanup_task(config, log_directory, log_filename, keep_files));
|
||||
|
||||
info!(
|
||||
@@ -256,8 +266,9 @@ pub(super) fn init_observability_http(
|
||||
.map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())));
|
||||
let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone()));
|
||||
|
||||
// Optional stdout mirror (matching init_file_logging_internal logic)
|
||||
// This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules.
|
||||
// Optional stdout mirror (matching `init_file_logging_internal` logic).
|
||||
// This is separate from OTLP stdout exporting; it only affects local human
|
||||
// readable output for the tracing subscriber.
|
||||
if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
|
||||
let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout());
|
||||
stdout_guard = Some(stdout_g);
|
||||
@@ -309,6 +320,8 @@ pub(super) fn init_observability_http(
|
||||
/// Build an optional [`SdkTracerProvider`] for the given trace endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or trace export is disabled.
|
||||
/// When enabled, the provider is also registered as the global tracer provider
|
||||
/// and installs the W3C trace-context propagator.
|
||||
fn build_tracer_provider(
|
||||
trace_ep: &str,
|
||||
config: &OtelConfig,
|
||||
@@ -344,6 +357,10 @@ fn build_tracer_provider(
|
||||
Ok(Some(provider))
|
||||
}
|
||||
|
||||
/// Convert a configured sample ratio into the SDK sampler strategy.
|
||||
///
|
||||
/// Invalid or non-finite ratios fall back to `AlwaysOn` so telemetry does not
|
||||
/// disappear due to configuration mistakes.
|
||||
fn build_tracer_sampler(sample_ratio: f64) -> Sampler {
|
||||
if sample_ratio.is_finite() && (0.0..=1.0).contains(&sample_ratio) {
|
||||
Sampler::TraceIdRatioBased(sample_ratio)
|
||||
@@ -355,6 +372,8 @@ fn build_tracer_sampler(sample_ratio: f64) -> Sampler {
|
||||
/// Build an optional [`SdkMeterProvider`] for the given metrics endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or metric export is disabled.
|
||||
/// The provider is paired with the crate's metrics recorder so `metrics` crate
|
||||
/// instruments flow into OpenTelemetry readers.
|
||||
fn build_meter_provider(
|
||||
metric_ep: &str,
|
||||
config: &OtelConfig,
|
||||
@@ -394,13 +413,14 @@ fn build_meter_provider(
|
||||
|
||||
global::set_meter_provider(provider.clone());
|
||||
metrics::set_global_recorder(recorder).map_err(|e| TelemetryError::InstallMetricsRecorder(e.to_string()))?;
|
||||
OBSERVABILITY_METRIC_ENABLED.set(true).ok();
|
||||
set_observability_metric_enabled(true);
|
||||
Ok(Some(provider))
|
||||
}
|
||||
|
||||
/// Build an optional [`SdkLoggerProvider`] for the given log endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or log export is disabled.
|
||||
/// The caller wraps the resulting provider in an OpenTelemetry tracing bridge.
|
||||
fn build_logger_provider(
|
||||
log_ep: &str,
|
||||
config: &OtelConfig,
|
||||
@@ -427,8 +447,10 @@ fn build_logger_provider(
|
||||
Ok(Some(builder.build()))
|
||||
}
|
||||
|
||||
/// Starts the Pyroscope continuous profiling agent if `ENV_OBS_PROFILING_ENDPOINT` is set.
|
||||
/// No-op and returns None on non-unix platforms.
|
||||
/// Start the Pyroscope continuous profiling agent when profiling is enabled.
|
||||
///
|
||||
/// Returns `None` on non-Unix platforms, when the feature is disabled, or when
|
||||
/// no usable profiling endpoint is configured.
|
||||
#[cfg(unix)]
|
||||
fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>> {
|
||||
use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend};
|
||||
@@ -468,6 +490,9 @@ fn init_profiler(config: &OtelConfig) -> Option<pyroscope::PyroscopeAgent<pyrosc
|
||||
}
|
||||
|
||||
/// Create a stdout periodic metrics reader for the given interval.
|
||||
///
|
||||
/// This helper is primarily used for local development and diagnostics when
|
||||
/// operators want to see exported metric points without an OTLP collector.
|
||||
fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout::MetricExporter> {
|
||||
PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default())
|
||||
.with_interval(Duration::from_secs(interval))
|
||||
@@ -479,6 +504,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
/// Valid ratios should produce trace-id-ratio sampling.
|
||||
fn test_build_tracer_sampler_uses_trace_ratio_for_valid_values() {
|
||||
let sampler = build_tracer_sampler(0.0);
|
||||
assert!(format!("{sampler:?}").contains("TraceIdRatioBased"));
|
||||
@@ -491,6 +517,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Invalid ratios should degrade to the safest non-dropping sampler.
|
||||
fn test_build_tracer_sampler_rejects_invalid_ratio_with_always_on() {
|
||||
let sampler = build_tracer_sampler(-0.1);
|
||||
assert!(format!("{sampler:?}").contains("AlwaysOn"));
|
||||
|
||||
@@ -19,7 +19,12 @@
|
||||
//! log files do not grow indefinitely by rotating them when they exceed a configured size.
|
||||
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::global::{
|
||||
METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES, METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS,
|
||||
METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL, METRIC_LOG_CLEANER_ROTATION_TOTAL,
|
||||
};
|
||||
use jiff::Zoned;
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -186,6 +191,7 @@ impl RollingAppender {
|
||||
}
|
||||
|
||||
fn roll(&mut self) -> io::Result<()> {
|
||||
let rotate_started = std::time::Instant::now();
|
||||
// 1. Close current file first to ensure all buffers are flushed to OS (if any)
|
||||
// and handle released.
|
||||
if let Some(mut file) = self.file.take()
|
||||
@@ -250,6 +256,9 @@ impl RollingAppender {
|
||||
// This overrides whatever open_file() derived from mtime, ensuring
|
||||
// we stick to the logical rotation time.
|
||||
self.last_roll_ts = now.timestamp().as_second();
|
||||
counter!(METRIC_LOG_CLEANER_ROTATION_TOTAL).increment(1);
|
||||
histogram!(METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS).record(rotate_started.elapsed().as_secs_f64());
|
||||
gauge!(METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES).set(self.size as f64);
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -281,6 +290,8 @@ impl RollingAppender {
|
||||
"RollingAppender: Failed to rotate log file after {} retries. Error: {:?}",
|
||||
MAX_RETRIES, last_error
|
||||
);
|
||||
counter!(METRIC_LOG_CLEANER_ROTATION_FAILURES_TOTAL).increment(1);
|
||||
histogram!(METRIC_LOG_CLEANER_ROTATION_DURATION_SECONDS).record(rotate_started.elapsed().as_secs_f64());
|
||||
|
||||
// Attempt to re-open existing active file to allow continued writing
|
||||
self.open_file()?;
|
||||
@@ -314,6 +325,7 @@ impl Write for RollingAppender {
|
||||
if let Some(file) = &mut self.file {
|
||||
let n = file.write(buf)?;
|
||||
self.size += n as u64;
|
||||
gauge!(METRIC_LOG_CLEANER_ACTIVE_FILE_SIZE_BYTES).set(self.size as f64);
|
||||
Ok(n)
|
||||
} else {
|
||||
Err(io::Error::other("Failed to open log file"))
|
||||
|
||||
Reference in New Issue
Block a user