mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-25 21:46:50 +00:00
feat(obs): add advanced log management configuration (#2016)
Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com> Co-authored-by: 唐小鸭 <tangtang1251@qq.com>
This commit is contained in:
heihutu
@@ -0,0 +1,94 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Log filtering utilities for tracing subscribers.
|
||||
//!
|
||||
//! This module provides helper functions for building `EnvFilter` instances
|
||||
//! used across different logging backends (stdout, file, OpenTelemetry).
|
||||
|
||||
use smallvec::SmallVec;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
/// Build an `EnvFilter` from the given log level string.
|
||||
///
|
||||
/// If the `RUST_LOG` environment variable is set, it takes precedence over the
|
||||
/// provided `logger_level`. For non-verbose levels (`info`, `warn`, `error`),
|
||||
/// noisy internal crates (`hyper`, `tonic`, `h2`, `reqwest`, `tower`) are
|
||||
/// automatically silenced to reduce log noise.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `logger_level` - The desired log level string (e.g., `"info"`, `"debug"`).
|
||||
/// * `default_level` - An optional override that replaces `logger_level` as the
|
||||
/// base directive; useful when the caller wants to force a specific level
|
||||
/// regardless of what is stored in config.
|
||||
///
|
||||
/// # Returns
|
||||
/// A configured `EnvFilter` ready to be attached to a `tracing_subscriber` registry.
|
||||
pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter {
|
||||
let level = default_level.unwrap_or(logger_level);
|
||||
let mut filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level));
|
||||
|
||||
// Suppress chatty infrastructure crates unless the operator explicitly
|
||||
// requests trace/debug output.
|
||||
if !matches!(logger_level, "trace" | "debug") {
|
||||
let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
|
||||
for directive in directives {
|
||||
filter = filter.add_directive(format!("{directive}=off").parse().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_env_filter_default_level_overrides() {
|
||||
// Ensure that providing a default_level uses it instead of logger_level.
|
||||
let filter = build_env_filter("debug", Some("error"));
|
||||
// The Debug output uses `LevelFilter::ERROR` for the error level directive.
|
||||
let dbg = format!("{filter:?}");
|
||||
assert!(
|
||||
dbg.contains("LevelFilter::ERROR"),
|
||||
"Expected 'LevelFilter::ERROR' in filter debug output: {dbg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_env_filter_suppresses_noisy_crates() {
|
||||
// For info level, hyper/tonic/etc. should be suppressed with OFF.
|
||||
let filter = build_env_filter("info", None);
|
||||
let dbg = format!("{filter:?}");
|
||||
// The Debug output uses `LevelFilter::OFF` for suppressed crates.
|
||||
assert!(
|
||||
dbg.contains("LevelFilter::OFF"),
|
||||
"Expected 'LevelFilter::OFF' suppression directives in filter: {dbg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_env_filter_debug_no_suppression() {
|
||||
// For debug level, our code does NOT inject any OFF directives.
|
||||
let filter = build_env_filter("debug", None);
|
||||
let dbg = format!("{filter:?}");
|
||||
// Verify the filter builds without panicking and contains the debug level.
|
||||
assert!(!dbg.is_empty());
|
||||
assert!(
|
||||
dbg.contains("LevelFilter::DEBUG"),
|
||||
"Expected 'LevelFilter::DEBUG' in filter debug output: {dbg}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RAII guard for OpenTelemetry provider lifecycle management.
|
||||
//!
|
||||
//! [`OtelGuard`] holds all runtime resources created during telemetry
|
||||
//! initialisation. Dropping it triggers an ordered shutdown:
|
||||
//!
|
||||
//! 1. Tracer provider — flushes pending spans.
|
||||
//! 2. Meter provider — flushes pending metrics.
|
||||
//! 3. Logger provider — flushes pending log records.
|
||||
//! 4. Cleanup task — aborted to prevent lingering background work.
|
||||
//! 5. Tracing worker guard — flushes buffered log lines written by
|
||||
//! `tracing_appender`.
|
||||
|
||||
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
|
||||
|
||||
/// RAII guard that owns all active OpenTelemetry providers and the
|
||||
/// `tracing_appender` worker guard.
|
||||
///
|
||||
/// Construct this via the `init_*` functions in [`crate::telemetry`] rather
|
||||
/// than directly. The guard must be kept alive for the entire duration of the
|
||||
/// application — once dropped, all telemetry pipelines are shut down.
|
||||
pub struct OtelGuard {
|
||||
/// Optional tracer provider for distributed tracing.
|
||||
pub(crate) tracer_provider: Option<SdkTracerProvider>,
|
||||
/// Optional meter provider for metrics collection.
|
||||
pub(crate) meter_provider: Option<SdkMeterProvider>,
|
||||
/// Optional logger provider for OTLP log export.
|
||||
pub(crate) logger_provider: Option<SdkLoggerProvider>,
|
||||
/// Worker guard that keeps the non-blocking `tracing_appender` thread
|
||||
/// alive. Dropping it blocks until all buffered records are flushed.
|
||||
pub(crate) tracing_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
|
||||
/// Optional guard for stdout logging; kept separate to allow independent flushing and shutdown.
|
||||
pub(crate) stdout_guard: Option<tracing_appender::non_blocking::WorkerGuard>,
|
||||
/// Handle to the background log-cleanup task; aborted on drop.
|
||||
pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OtelGuard {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("OtelGuard")
|
||||
.field("tracer_provider", &self.tracer_provider.is_some())
|
||||
.field("meter_provider", &self.meter_provider.is_some())
|
||||
.field("logger_provider", &self.logger_provider.is_some())
|
||||
.field("tracing_guard", &self.tracing_guard.is_some())
|
||||
.field("stdout_guard", &self.stdout_guard.is_some())
|
||||
.field("cleanup_handle", &self.cleanup_handle.is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OtelGuard {
|
||||
/// Shut down all telemetry providers in order.
|
||||
///
|
||||
/// Errors during shutdown are printed to `stderr` so they are visible even
|
||||
/// after the tracing subscriber has been torn down.
|
||||
fn drop(&mut self) {
|
||||
if let Some(provider) = self.tracer_provider.take()
|
||||
&& let Err(err) = provider.shutdown()
|
||||
{
|
||||
eprintln!("Tracer shutdown error: {err:?}");
|
||||
}
|
||||
|
||||
if let Some(provider) = self.meter_provider.take()
|
||||
&& let Err(err) = provider.shutdown()
|
||||
{
|
||||
eprintln!("Meter shutdown error: {err:?}");
|
||||
}
|
||||
|
||||
if let Some(provider) = self.logger_provider.take()
|
||||
&& let Err(err) = provider.shutdown()
|
||||
{
|
||||
eprintln!("Logger shutdown error: {err:?}");
|
||||
}
|
||||
|
||||
if let Some(handle) = self.cleanup_handle.take() {
|
||||
handle.abort();
|
||||
eprintln!("Log cleanup task stopped");
|
||||
}
|
||||
|
||||
if let Some(guard) = self.tracing_guard.take() {
|
||||
drop(guard);
|
||||
eprintln!("Tracing guard dropped, flushing logs.");
|
||||
}
|
||||
|
||||
if let Some(guard) = self.stdout_guard.take() {
|
||||
drop(guard);
|
||||
eprintln!("Stdout guard dropped, flushing logs.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Local logging backend: stdout-only or file-rolling with optional stdout mirror.
|
||||
//!
|
||||
//! # Behaviour
|
||||
//!
|
||||
//! | Condition | Result |
|
||||
//! |----------------------------------|----------------------------------------------|
|
||||
//! | No log directory configured | JSON logs written to **stdout only** |
|
||||
//! | Log directory configured | JSON logs written to **rolling file**; |
|
||||
//! | | stdout mirror enabled when `log_stdout_enabled` |
|
||||
//! | | is `true` or environment is non-production |
|
||||
//!
|
||||
//! The function [`init_local_logging`] is the single entry point for both
|
||||
//! cases; callers do **not** need to distinguish between stdout and file modes.
|
||||
|
||||
use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::OBSERVABILITY_METRIC_ENABLED;
|
||||
use crate::log_cleanup::LogCleaner;
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
|
||||
DEFAULT_OBS_LOG_DELETE_EMPTY_FILES, DEFAULT_OBS_LOG_DRY_RUN, DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL,
|
||||
DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES, DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS,
|
||||
};
|
||||
use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED};
|
||||
use std::{fs, io::IsTerminal, time::Duration};
|
||||
use tracing::info;
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_subscriber::{
|
||||
fmt::{format::FmtSpan, time::LocalTime},
|
||||
layer::SubscriberExt,
|
||||
util::SubscriberInitExt,
|
||||
};
|
||||
|
||||
use super::guard::OtelGuard;
|
||||
|
||||
/// Initialize local logging (stdout-only or file-rolling).
|
||||
///
|
||||
/// When `log_directory` is empty or `None` in the config the function sets up
|
||||
/// a non-blocking JSON subscriber that writes to **stdout** and returns
|
||||
/// immediately — no file I/O, no cleanup task.
|
||||
///
|
||||
/// When a log directory is provided the function additionally:
|
||||
/// 1. Creates the directory (including on Unix, enforces `0755` permissions).
|
||||
/// 2. Attaches a rolling-file appender (daily or hourly based on
|
||||
/// `log_rotation_time`).
|
||||
/// 3. Optionally mirrors output to stdout based on `log_stdout_enabled`.
|
||||
/// 4. Spawns a background cleanup task that periodically removes or compresses
|
||||
/// old log files according to the cleanup configuration in [`OtelConfig`].
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Observability configuration, fully populated from environment variables.
|
||||
/// * `logger_level` - Effective log level string (e.g., `"info"`).
|
||||
/// * `is_production` - Whether the runtime environment is production; controls
|
||||
/// span verbosity and stdout mirroring defaults.
|
||||
///
|
||||
/// # Returns
|
||||
/// An [`OtelGuard`] that keeps the `tracing_appender` worker alive and holds
|
||||
/// a handle to the cleanup task (if started). Dropping the guard flushes
|
||||
/// in-flight logs and stops the cleanup task.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`TelemetryError`] if the log directory cannot be created or its
|
||||
/// permissions cannot be set (Unix only).
|
||||
pub(super) fn init_local_logging(
|
||||
config: &OtelConfig,
|
||||
logger_level: &str,
|
||||
is_production: bool,
|
||||
) -> Result<OtelGuard, TelemetryError> {
|
||||
// Determine the effective log directory. An absent or empty value means
|
||||
// stdout-only mode: we skip file setup entirely.
|
||||
let log_dir_str = config.log_directory.as_deref().filter(|s| !s.is_empty());
|
||||
|
||||
if let Some(log_directory) = log_dir_str {
|
||||
init_file_logging_internal(config, log_directory, logger_level, is_production)
|
||||
} else {
|
||||
Ok(init_stdout_only(config, logger_level, is_production))
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Stdout-only ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Set up a non-blocking stdout JSON subscriber with no file I/O.
|
||||
///
|
||||
/// Used when no log directory has been configured. The subscriber formats
|
||||
/// every log record as a JSON line, including RFC-3339 timestamps, thread
|
||||
/// identifiers, file/line information, and span context.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `_config` - Unused at the moment; reserved for future configuration.
|
||||
/// * `logger_level` - Effective log level string.
|
||||
/// * `is_production` - Controls span event verbosity.
|
||||
fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: bool) -> OtelGuard {
|
||||
let env_filter = build_env_filter(logger_level, None);
|
||||
let (nb, guard) = tracing_appender::non_blocking(std::io::stdout());
|
||||
|
||||
let fmt_layer = tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
.with_ansi(std::io::stdout().is_terminal())
|
||||
.with_thread_names(true)
|
||||
.with_thread_ids(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_writer(nb)
|
||||
.json()
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_span_events(if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL });
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer)
|
||||
.init();
|
||||
|
||||
OBSERVABILITY_METRIC_ENABLED.set(false).ok();
|
||||
counter!("rustfs.start.total").increment(1);
|
||||
info!("Init stdout logging (level: {})", logger_level);
|
||||
|
||||
OtelGuard {
|
||||
tracer_provider: None,
|
||||
meter_provider: None,
|
||||
logger_provider: None,
|
||||
tracing_guard: Some(guard),
|
||||
stdout_guard: None,
|
||||
cleanup_handle: None,
|
||||
}
|
||||
}
|
||||
|
||||
// ─── File-rolling ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Internal implementation for file-based rolling log setup.
|
||||
///
|
||||
/// Called by [`init_local_logging`] when a log directory is present.
|
||||
/// Handles directory creation, permission enforcement (Unix), file appender
|
||||
/// setup, optional stdout mirror, and log-cleanup task spawning.
|
||||
fn init_file_logging_internal(
|
||||
config: &OtelConfig,
|
||||
log_directory: &str,
|
||||
logger_level: &str,
|
||||
is_production: bool,
|
||||
) -> Result<OtelGuard, TelemetryError> {
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
|
||||
let log_filename = config.log_filename.as_deref().unwrap_or(service_name);
|
||||
let keep_files = config.log_keep_files.unwrap_or(DEFAULT_LOG_KEEP_FILES);
|
||||
|
||||
// ── 1. Ensure the log directory exists ───────────────────────────────────
|
||||
if let Err(e) = fs::create_dir_all(log_directory) {
|
||||
return Err(TelemetryError::Io(e.to_string()));
|
||||
}
|
||||
|
||||
// ── 2. Enforce directory permissions (Unix only) ─────────────────────────
|
||||
#[cfg(unix)]
|
||||
ensure_dir_permissions(log_directory)?;
|
||||
|
||||
// ── 3. Choose rotation strategy ──────────────────────────────────────────
|
||||
// `log_rotation_time` drives the rolling-appender rotation period.
|
||||
let rotation = config
|
||||
.log_rotation_time
|
||||
.as_deref()
|
||||
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
|
||||
.to_lowercase();
|
||||
use tracing_appender::rolling::{RollingFileAppender, Rotation};
|
||||
let file_appender = {
|
||||
let rotation = match rotation.as_str() {
|
||||
"minutely" => Rotation::MINUTELY,
|
||||
"hourly" => Rotation::HOURLY,
|
||||
_ => Rotation::DAILY,
|
||||
};
|
||||
RollingFileAppender::builder()
|
||||
.rotation(rotation)
|
||||
.filename_suffix(log_filename)
|
||||
.max_log_files(keep_files)
|
||||
.build(log_directory)
|
||||
.expect("failed to initialize rolling file appender")
|
||||
};
|
||||
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
// ── 4. Build subscriber layers ────────────────────────────────────────────
|
||||
let env_filter = build_env_filter(logger_level, None);
|
||||
let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL };
|
||||
|
||||
// File layer writes JSON without ANSI codes.
|
||||
let file_layer = tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
.with_ansi(false)
|
||||
.with_thread_names(true)
|
||||
.with_thread_ids(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_writer(non_blocking)
|
||||
.json()
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_span_events(span_events.clone());
|
||||
|
||||
// Optional stdout mirror: enabled explicitly via `log_stdout_enabled`, or
|
||||
// unconditionally in non-production environments.
|
||||
let (stdout_layer, stdout_guard) = if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production {
|
||||
let (stdout_nb, stdout_guard) = tracing_appender::non_blocking(std::io::stdout());
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
(
|
||||
Some(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
.with_ansi(enable_color)
|
||||
.with_thread_names(true)
|
||||
.with_thread_ids(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_writer(stdout_nb) // .json()
|
||||
// .with_current_span(true)
|
||||
// .with_span_list(true)
|
||||
.with_span_events(span_events),
|
||||
),
|
||||
Some(stdout_guard),
|
||||
)
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer)
|
||||
.with(stdout_layer)
|
||||
.init();
|
||||
|
||||
OBSERVABILITY_METRIC_ENABLED.set(false).ok();
|
||||
|
||||
// ── 5. Start background cleanup task ─────────────────────────────────────
|
||||
let cleanup_handle = spawn_cleanup_task(config, log_directory, log_filename, keep_files);
|
||||
|
||||
info!(
|
||||
"Init file logging at '{}', rotation: {}, keep {} files",
|
||||
log_directory, rotation, keep_files
|
||||
);
|
||||
|
||||
Ok(OtelGuard {
|
||||
tracer_provider: None,
|
||||
meter_provider: None,
|
||||
logger_provider: None,
|
||||
tracing_guard: Some(guard),
|
||||
stdout_guard,
|
||||
cleanup_handle: Some(cleanup_handle),
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Directory permissions (Unix) ─────────────────────────────────────────────
|
||||
|
||||
/// Ensure the log directory has at most `0755` permissions (Unix only).
|
||||
///
|
||||
/// Tightens permissions to `0755` if the directory is more permissive.
|
||||
/// This prevents world-writable log directories from being a security hazard.
|
||||
/// No-ops if permissions are already `0755` or stricter.
|
||||
#[cfg(unix)]
|
||||
fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> {
|
||||
use std::fs::Permissions;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let desired: u32 = 0o755;
|
||||
match fs::metadata(log_directory) {
|
||||
Ok(meta) => {
|
||||
let current = meta.permissions().mode() & 0o777;
|
||||
// Only tighten to 0755 if existing permissions are looser than target.
|
||||
if (current & !desired) != 0 {
|
||||
if let Err(e) = fs::set_permissions(log_directory, Permissions::from_mode(desired)) {
|
||||
return Err(TelemetryError::SetPermissions(format!(
|
||||
"dir='{log_directory}', want={desired:#o}, have={current:#o}, err={e}"
|
||||
)));
|
||||
}
|
||||
// Second verification pass to confirm the change took effect.
|
||||
if let Ok(meta2) = fs::metadata(log_directory) {
|
||||
let after = meta2.permissions().mode() & 0o777;
|
||||
if after != desired {
|
||||
return Err(TelemetryError::SetPermissions(format!(
|
||||
"dir='{log_directory}', want={desired:#o}, after={after:#o}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(TelemetryError::Io(format!("stat '{log_directory}' failed: {e}"))),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Cleanup task ─────────────────────────────────────────────────────────────
|
||||
|
||||
/// Spawn a background task that periodically cleans up old log files.
|
||||
///
|
||||
/// All cleanup parameters are derived from [`OtelConfig`] fields, with
|
||||
/// sensible defaults when fields are absent. The task runs on the current
|
||||
/// Tokio runtime and should be aborted (via the returned `JoinHandle`) when
|
||||
/// the application shuts down.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Observability config containing cleanup parameters.
|
||||
/// * `log_directory` - Directory path of the rolling log files.
|
||||
/// * `log_filename` - Base filename (used as the file prefix for matching).
|
||||
/// * `keep_files` - Legacy keep-files count; used as fallback when the new
|
||||
/// `log_keep_count` field is absent.
|
||||
///
|
||||
/// # Returns
|
||||
/// A [`tokio::task::JoinHandle`] for the spawned cleanup loop.
|
||||
fn spawn_cleanup_task(
|
||||
config: &OtelConfig,
|
||||
log_directory: &str,
|
||||
log_filename: &str,
|
||||
keep_files: usize,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let log_dir = std::path::PathBuf::from(log_directory);
|
||||
let file_prefix = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
|
||||
let keep_count = config.log_keep_count.unwrap_or(keep_files);
|
||||
let max_total_size = config
|
||||
.log_max_total_size_bytes
|
||||
.unwrap_or(DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES * keep_count as u64);
|
||||
let max_single_file_size = config
|
||||
.log_max_single_file_size_bytes
|
||||
.unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
|
||||
let compress = config.log_compress_old_files.unwrap_or(DEFAULT_OBS_LOG_COMPRESS_OLD_FILES);
|
||||
let gzip_level = config
|
||||
.log_gzip_compression_level
|
||||
.unwrap_or(DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL);
|
||||
let retention_days = config
|
||||
.log_compressed_file_retention_days
|
||||
.unwrap_or(DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS);
|
||||
let exclude_patterns = config
|
||||
.log_exclude_patterns
|
||||
.as_deref()
|
||||
.map(|s| s.split(',').map(|p| p.trim().to_string()).collect())
|
||||
.unwrap_or_default();
|
||||
let delete_empty = config.log_delete_empty_files.unwrap_or(DEFAULT_OBS_LOG_DELETE_EMPTY_FILES);
|
||||
let min_age = config
|
||||
.log_min_file_age_seconds
|
||||
.unwrap_or(DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS);
|
||||
let dry_run = config.log_dry_run.unwrap_or(DEFAULT_OBS_LOG_DRY_RUN);
|
||||
let cleanup_interval = config
|
||||
.log_cleanup_interval_seconds
|
||||
.unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS);
|
||||
|
||||
let cleaner = LogCleaner::new(
|
||||
log_dir,
|
||||
file_prefix,
|
||||
keep_count,
|
||||
max_total_size,
|
||||
max_single_file_size,
|
||||
compress,
|
||||
gzip_level,
|
||||
retention_days,
|
||||
exclude_patterns,
|
||||
delete_empty,
|
||||
min_age,
|
||||
dry_run,
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(e) = cleaner.cleanup() {
|
||||
tracing::warn!("Log cleanup failed: {}", e);
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Telemetry initialisation module for RustFS.
|
||||
//!
|
||||
//! This module is the single entry point for all observability backends.
|
||||
//! Callers should use [`init_telemetry`] and keep the returned [`OtelGuard`]
|
||||
//! alive for the lifetime of the application.
|
||||
//!
|
||||
//! ## Architecture
|
||||
//!
|
||||
//! The module is split into focused sub-modules:
|
||||
//!
|
||||
//! | Sub-module | Responsibility |
|
||||
//! |--------------|---------------------------------------------------------|
|
||||
//! | `guard` | [`OtelGuard`] RAII type for provider lifecycle |
|
||||
//! | `filter` | `EnvFilter` construction helpers |
|
||||
//! | `resource` | OpenTelemetry `Resource` builder |
|
||||
//! | `local` | Local logging: stdout-only **or** rolling-file |
|
||||
//! | `otel` | Full OTLP/HTTP pipeline (traces + metrics + logs) |
|
||||
//!
|
||||
//! ## Routing rules (evaluated in order)
|
||||
//!
|
||||
//! 1. **OpenTelemetry** — if any OTLP endpoint is configured, the full HTTP
|
||||
//! pipeline is initialised via [`otel::init_observability_http`].
|
||||
//! 2. **File logging** — if `RUSTFS_OBS_LOG_DIRECTORY` (or `log_directory` /
|
||||
//! `log_dir` in config) is set to a non-empty value, rolling-file logging is
|
||||
//! initialised together with an optional stdout mirror.
|
||||
//! 3. **Stdout only** — default fallback; no file I/O, no remote export.
|
||||
|
||||
mod filter;
|
||||
mod guard;
|
||||
mod local;
|
||||
mod otel;
|
||||
mod recorder;
|
||||
mod resource;
|
||||
|
||||
use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
pub use guard::OtelGuard;
|
||||
pub use recorder::Recorder;
|
||||
use rustfs_config::observability::ENV_OBS_LOG_DIRECTORY;
|
||||
use rustfs_config::{DEFAULT_LOG_LEVEL, ENVIRONMENT, observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION};
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
|
||||
/// Initialize the telemetry subsystem according to the provided configuration.
|
||||
///
|
||||
/// Evaluates three routing rules in priority order and delegates to the
|
||||
/// appropriate backend:
|
||||
///
|
||||
/// 1. If any OTLP endpoint is set, initialises the full
|
||||
/// OpenTelemetry HTTP pipeline (traces + metrics + logs).
|
||||
/// 2. If a log directory is explicitly configured via the
|
||||
/// `RUSTFS_OBS_LOG_DIRECTORY` environment variable, initialises
|
||||
/// rolling-file logging with an optional stdout mirror.
|
||||
/// 3. Otherwise, falls back to stdout-only JSON logging.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Observability configuration, typically built from environment
|
||||
/// variables via [`OtelConfig::extract_otel_config_from_env`].
|
||||
///
|
||||
/// # Returns
|
||||
/// An [`OtelGuard`] that must be kept alive for the duration of the
|
||||
/// application. Dropping it triggers ordered shutdown of all providers.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`TelemetryError`] when a backend fails to initialise (e.g., cannot
|
||||
/// create the log directory, or an OTLP exporter cannot connect).
|
||||
pub(crate) fn init_telemetry(config: &OtelConfig) -> Result<OtelGuard, TelemetryError> {
|
||||
let environment = config.environment.as_deref().unwrap_or(ENVIRONMENT);
|
||||
let is_production = environment.eq_ignore_ascii_case(DEFAULT_OBS_ENVIRONMENT_PRODUCTION);
|
||||
let logger_level = config.logger_level.as_deref().unwrap_or(DEFAULT_LOG_LEVEL);
|
||||
|
||||
// ── Rule 1: OpenTelemetry HTTP pipeline ───────────────────────────────────
|
||||
// Activated when at least one OTLP endpoint is non-empty.
|
||||
let has_obs = !config.endpoint.is_empty()
|
||||
|| config.trace_endpoint.as_deref().map(|s| !s.is_empty()).unwrap_or(false)
|
||||
|| config.metric_endpoint.as_deref().map(|s| !s.is_empty()).unwrap_or(false)
|
||||
|| config.log_endpoint.as_deref().map(|s| !s.is_empty()).unwrap_or(false);
|
||||
|
||||
if has_obs {
|
||||
return otel::init_observability_http(config, logger_level, is_production);
|
||||
}
|
||||
|
||||
// ── Rule 2 & 3: Local logging (file or stdout) ────────────────────────────
|
||||
// `init_local_logging` internally decides between file and stdout mode
|
||||
// based on whether a log directory is configured.
|
||||
//
|
||||
// We check the environment variable here (rather than relying solely on the
|
||||
// config struct) to honour dynamic overrides set after config construction.
|
||||
let user_set_log_dir = get_env_opt_str(ENV_OBS_LOG_DIRECTORY);
|
||||
let effective_config = if user_set_log_dir.as_deref().filter(|d| !d.is_empty()).is_some() {
|
||||
// Environment variable is set: ensure the config reflects it so that
|
||||
// `init_local_logging` picks up the value even if the struct was built
|
||||
// before the env var was set.
|
||||
std::borrow::Cow::Owned(OtelConfig {
|
||||
log_directory: user_set_log_dir,
|
||||
..config.clone()
|
||||
})
|
||||
} else {
|
||||
std::borrow::Cow::Borrowed(config)
|
||||
};
|
||||
|
||||
local::init_local_logging(&effective_config, logger_level, is_production)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rustfs_config::observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION;
|
||||
use rustfs_config::{ENVIRONMENT, USE_STDOUT};
|
||||
|
||||
#[test]
|
||||
fn test_production_environment_detection() {
|
||||
// Verify that case-insensitive comparison correctly identifies production.
|
||||
let production_envs = ["production", "PRODUCTION", "Production"];
|
||||
for env_value in production_envs {
|
||||
let is_production = env_value.eq_ignore_ascii_case(DEFAULT_OBS_ENVIRONMENT_PRODUCTION);
|
||||
assert!(is_production, "Should detect '{env_value}' as production environment");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_production_environment_detection() {
|
||||
// Verify that non-production environments are not misidentified.
|
||||
let non_production_envs = ["development", "test", "staging", "dev", "local"];
|
||||
for env_value in non_production_envs {
|
||||
let is_production = env_value.eq_ignore_ascii_case(DEFAULT_OBS_ENVIRONMENT_PRODUCTION);
|
||||
assert!(!is_production, "Should not detect '{env_value}' as production environment");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stdout_behavior_logic() {
|
||||
// Validate the stdout-enable logic for different environment/config combinations.
|
||||
struct TestCase {
|
||||
is_production: bool,
|
||||
config_use_stdout: Option<bool>,
|
||||
expected_use_stdout: bool,
|
||||
description: &'static str,
|
||||
}
|
||||
|
||||
let test_cases = [
|
||||
TestCase {
|
||||
is_production: true,
|
||||
config_use_stdout: None,
|
||||
expected_use_stdout: false,
|
||||
description: "Production with no config should disable stdout",
|
||||
},
|
||||
TestCase {
|
||||
is_production: false,
|
||||
config_use_stdout: None,
|
||||
expected_use_stdout: USE_STDOUT,
|
||||
description: "Non-production with no config should use default",
|
||||
},
|
||||
TestCase {
|
||||
is_production: true,
|
||||
config_use_stdout: Some(true),
|
||||
expected_use_stdout: true,
|
||||
description: "Production with explicit true should enable stdout",
|
||||
},
|
||||
TestCase {
|
||||
is_production: true,
|
||||
config_use_stdout: Some(false),
|
||||
expected_use_stdout: false,
|
||||
description: "Production with explicit false should disable stdout",
|
||||
},
|
||||
TestCase {
|
||||
is_production: false,
|
||||
config_use_stdout: Some(true),
|
||||
expected_use_stdout: true,
|
||||
description: "Non-production with explicit true should enable stdout",
|
||||
},
|
||||
];
|
||||
|
||||
for case in &test_cases {
|
||||
let default_use_stdout = if case.is_production { false } else { USE_STDOUT };
|
||||
let actual = case.config_use_stdout.unwrap_or(default_use_stdout);
|
||||
assert_eq!(actual, case.expected_use_stdout, "Test case failed: {}", case.description);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_log_level_filter_mapping_logic() {
|
||||
// Validate the log level string → tracing level mapping used in filters.
|
||||
let test_cases = [
|
||||
("trace", "Trace"),
|
||||
("debug", "Debug"),
|
||||
("info", "Info"),
|
||||
("warn", "Warn"),
|
||||
("warning", "Warn"),
|
||||
("error", "Error"),
|
||||
("off", "None"),
|
||||
("invalid_level", "Info"),
|
||||
];
|
||||
|
||||
for (input, expected) in test_cases {
|
||||
let mapped = match input.to_lowercase().as_str() {
|
||||
"trace" => "Trace",
|
||||
"debug" => "Debug",
|
||||
"info" => "Info",
|
||||
"warn" | "warning" => "Warn",
|
||||
"error" => "Error",
|
||||
"off" => "None",
|
||||
_ => "Info",
|
||||
};
|
||||
assert_eq!(mapped, expected, "Log level '{input}' should map to '{expected}'");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_otel_config_environment_defaults() {
|
||||
// Verify that environment field defaults behave correctly.
|
||||
use crate::config::OtelConfig;
|
||||
let config = OtelConfig {
|
||||
endpoint: "".to_string(),
|
||||
use_stdout: None,
|
||||
environment: Some("production".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let environment = config.environment.as_deref().unwrap_or(ENVIRONMENT);
|
||||
assert_eq!(environment, "production");
|
||||
|
||||
let dev_config = OtelConfig {
|
||||
endpoint: "".to_string(),
|
||||
use_stdout: None,
|
||||
environment: Some("development".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let dev_environment = dev_config.environment.as_deref().unwrap_or(ENVIRONMENT);
|
||||
assert_eq!(dev_environment, "development");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! OpenTelemetry HTTP exporter initialisation.
|
||||
//!
|
||||
//! This module sets up full OTLP/HTTP pipelines for:
|
||||
//! - **Traces** via [`opentelemetry_otlp::SpanExporter`]
|
||||
//! - **Metrics** via [`opentelemetry_otlp::MetricExporter`]
|
||||
//! - **Logs** via [`opentelemetry_otlp::LogExporter`]
|
||||
//!
|
||||
//! Each signal has a dedicated endpoint field in [`OtelConfig`]. When a
|
||||
//! per-signal endpoint is absent, the function falls back to appending the
|
||||
//! standard OTLP path suffix to the root `endpoint` field:
|
||||
//!
|
||||
//! | Signal | Fallback path |
|
||||
//! |---------|-----------------|
|
||||
//! | Traces | `/v1/traces` |
|
||||
//! | Metrics | `/v1/metrics` |
|
||||
//! | Logs | `/v1/logs` |
|
||||
//!
|
||||
//! All exporters use **HTTP binary** (Protobuf) encoding with **gzip**
|
||||
//! compression for efficiency over the wire.
|
||||
|
||||
use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::OBSERVABILITY_METRIC_ENABLED;
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::guard::OtelGuard;
|
||||
use crate::telemetry::recorder::Recorder;
|
||||
use crate::telemetry::resource::build_resource;
|
||||
use metrics::counter;
|
||||
use opentelemetry::{global, trace::TracerProvider};
|
||||
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
|
||||
use opentelemetry_otlp::{Compression, Protocol, WithExportConfig, WithHttpConfig};
|
||||
use opentelemetry_sdk::propagation::TraceContextPropagator;
|
||||
use opentelemetry_sdk::{
|
||||
logs::SdkLoggerProvider,
|
||||
metrics::{PeriodicReader, SdkMeterProvider},
|
||||
trace::{RandomIdGenerator, Sampler, SdkTracerProvider},
|
||||
};
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED,
|
||||
DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO,
|
||||
};
|
||||
use std::{io::IsTerminal, time::Duration};
|
||||
use tracing::info;
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer};
|
||||
use tracing_subscriber::{
|
||||
Layer,
|
||||
fmt::{format::FmtSpan, time::LocalTime},
|
||||
layer::SubscriberExt,
|
||||
util::SubscriberInitExt,
|
||||
};
|
||||
|
||||
/// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs).
|
||||
///
|
||||
/// This function is invoked when at least one OTLP endpoint has been
|
||||
/// configured. It creates exporters, wires them into SDK providers, installs
|
||||
/// a global tracer/meter, and builds a `tracing_subscriber` registry that
|
||||
/// bridges Rust's `tracing` macros to the OTLP pipelines.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `config` - Fully populated observability configuration.
|
||||
/// * `logger_level` - Effective log level string (e.g., `"info"`).
|
||||
/// * `is_production` - Controls span verbosity and stdout layer defaults.
|
||||
///
|
||||
/// # Returns
|
||||
/// An [`OtelGuard`] owning all created providers. Dropping it triggers an
|
||||
/// ordered shutdown and flushes all pending telemetry data.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns [`TelemetryError`] if any exporter or provider fails to build.
|
||||
///
|
||||
/// # Note
|
||||
/// This function is intentionally kept unchanged from the pre-refactor
|
||||
/// implementation to preserve existing OTLP behaviour.
|
||||
pub(super) fn init_observability_http(
|
||||
config: &OtelConfig,
|
||||
logger_level: &str,
|
||||
is_production: bool,
|
||||
) -> Result<OtelGuard, TelemetryError> {
|
||||
// ── Resource & sampling ──────────────────────────────────────────────────
|
||||
let res = build_resource(config);
|
||||
let service_name = config.service_name.as_deref().unwrap_or(APP_NAME).to_owned();
|
||||
let use_stdout = config.use_stdout.unwrap_or(!is_production);
|
||||
let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO);
|
||||
let sampler = if (0.0..1.0).contains(&sample_ratio) {
|
||||
Sampler::TraceIdRatioBased(sample_ratio)
|
||||
} else {
|
||||
Sampler::AlwaysOn
|
||||
};
|
||||
|
||||
// ── Endpoint resolution ───────────────────────────────────────────────────
|
||||
// Each signal may have a dedicated endpoint; if absent, fall back to the
|
||||
// root endpoint with the standard OTLP path suffix appended.
|
||||
let root_ep = config.endpoint.clone();
|
||||
|
||||
let trace_ep: String = config
|
||||
.trace_endpoint
|
||||
.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| format!("{root_ep}/v1/traces"));
|
||||
|
||||
let metric_ep: String = config
|
||||
.metric_endpoint
|
||||
.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| format!("{root_ep}/v1/metrics"));
|
||||
|
||||
let log_ep: String = config
|
||||
.log_endpoint
|
||||
.as_deref()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_else(|| format!("{root_ep}/v1/logs"));
|
||||
|
||||
// ── Tracer provider (HTTP) ────────────────────────────────────────────────
|
||||
let tracer_provider = build_tracer_provider(&trace_ep, config, res.clone(), sampler, use_stdout)?;
|
||||
|
||||
// ── Meter provider (HTTP) ─────────────────────────────────────────────────
|
||||
let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?;
|
||||
|
||||
// ── Logger provider (HTTP) ────────────────────────────────────────────────
|
||||
let logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?;
|
||||
|
||||
// ── Tracing subscriber registry ───────────────────────────────────────────
|
||||
// Build an optional stdout formatting layer. When `log_stdout_enabled` is
|
||||
// false the field is `None` and tracing-subscriber will skip it.
|
||||
let fmt_layer_opt = if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) {
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
let span_event = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL };
|
||||
let layer = tracing_subscriber::fmt::layer()
|
||||
.with_timer(LocalTime::rfc_3339())
|
||||
.with_target(true)
|
||||
.with_ansi(enable_color)
|
||||
.with_thread_names(true)
|
||||
.with_thread_ids(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.json()
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_span_events(span_event)
|
||||
.with_filter(build_env_filter(logger_level, None));
|
||||
Some(layer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let filter = build_env_filter(logger_level, None);
|
||||
let otel_bridge = logger_provider
|
||||
.as_ref()
|
||||
.map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None)));
|
||||
let tracer_layer = tracer_provider
|
||||
.as_ref()
|
||||
.map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string())));
|
||||
let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone()));
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer_opt)
|
||||
.with(tracer_layer)
|
||||
.with(otel_bridge)
|
||||
.with(metrics_layer)
|
||||
.init();
|
||||
|
||||
counter!("rustfs.start.total").increment(1);
|
||||
info!(
|
||||
"Init observability (HTTP): trace='{}', metric='{}', log='{}'",
|
||||
trace_ep, metric_ep, log_ep
|
||||
);
|
||||
|
||||
Ok(OtelGuard {
|
||||
tracer_provider,
|
||||
meter_provider,
|
||||
logger_provider,
|
||||
tracing_guard: None,
|
||||
stdout_guard: None,
|
||||
cleanup_handle: None,
|
||||
})
|
||||
}
|
||||
|
||||
// ─── Private builder helpers ──────────────────────────────────────────────────
|
||||
|
||||
/// Build an optional [`SdkTracerProvider`] for the given trace endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or trace export is disabled.
|
||||
fn build_tracer_provider(
|
||||
trace_ep: &str,
|
||||
config: &OtelConfig,
|
||||
res: opentelemetry_sdk::Resource,
|
||||
sampler: Sampler,
|
||||
use_stdout: bool,
|
||||
) -> Result<Option<SdkTracerProvider>, TelemetryError> {
|
||||
if trace_ep.is_empty() || !config.traces_export_enabled.unwrap_or(DEFAULT_OBS_TRACES_EXPORT_ENABLED) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let exporter = opentelemetry_otlp::SpanExporter::builder()
|
||||
.with_http()
|
||||
.with_endpoint(trace_ep)
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.with_compression(Compression::Gzip)
|
||||
.build()
|
||||
.map_err(|e| TelemetryError::BuildSpanExporter(e.to_string()))?;
|
||||
|
||||
let mut builder = SdkTracerProvider::builder()
|
||||
.with_sampler(sampler)
|
||||
.with_id_generator(RandomIdGenerator::default())
|
||||
.with_resource(res)
|
||||
.with_batch_exporter(exporter);
|
||||
|
||||
if use_stdout {
|
||||
builder = builder.with_batch_exporter(opentelemetry_stdout::SpanExporter::default());
|
||||
}
|
||||
|
||||
let provider = builder.build();
|
||||
global::set_tracer_provider(provider.clone());
|
||||
global::set_text_map_propagator(TraceContextPropagator::new());
|
||||
Ok(Some(provider))
|
||||
}
|
||||
|
||||
/// Build an optional [`SdkMeterProvider`] for the given metrics endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or metric export is disabled.
|
||||
fn build_meter_provider(
|
||||
metric_ep: &str,
|
||||
config: &OtelConfig,
|
||||
res: opentelemetry_sdk::Resource,
|
||||
service_name: &str,
|
||||
use_stdout: bool,
|
||||
) -> Result<Option<SdkMeterProvider>, TelemetryError> {
|
||||
if metric_ep.is_empty() || !config.metrics_export_enabled.unwrap_or(DEFAULT_OBS_METRICS_EXPORT_ENABLED) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let exporter = opentelemetry_otlp::MetricExporter::builder()
|
||||
.with_http()
|
||||
.with_endpoint(metric_ep)
|
||||
.with_temporality(opentelemetry_sdk::metrics::Temporality::default())
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.with_compression(Compression::Gzip)
|
||||
.build()
|
||||
.map_err(|e| TelemetryError::BuildMetricExporter(e.to_string()))?;
|
||||
|
||||
let meter_interval = config.meter_interval.unwrap_or(METER_INTERVAL);
|
||||
|
||||
let (provider, recorder) = Recorder::builder(service_name.to_string())
|
||||
.with_meter_provider(|b: opentelemetry_sdk::metrics::MeterProviderBuilder| {
|
||||
let b = b.with_resource(res).with_reader(
|
||||
PeriodicReader::builder(exporter)
|
||||
.with_interval(Duration::from_secs(meter_interval))
|
||||
.build(),
|
||||
);
|
||||
if use_stdout {
|
||||
b.with_reader(create_periodic_reader(meter_interval))
|
||||
} else {
|
||||
b
|
||||
}
|
||||
})
|
||||
.build();
|
||||
|
||||
global::set_meter_provider(provider.clone() as SdkMeterProvider);
|
||||
metrics::set_global_recorder(recorder).map_err(|e| TelemetryError::InstallMetricsRecorder(e.to_string()))?;
|
||||
OBSERVABILITY_METRIC_ENABLED.set(true).ok();
|
||||
Ok(Some(provider))
|
||||
}
|
||||
|
||||
/// Build an optional [`SdkLoggerProvider`] for the given log endpoint.
|
||||
///
|
||||
/// Returns `None` when the endpoint is empty or log export is disabled.
|
||||
fn build_logger_provider(
|
||||
log_ep: &str,
|
||||
config: &OtelConfig,
|
||||
res: opentelemetry_sdk::Resource,
|
||||
use_stdout: bool,
|
||||
) -> Result<Option<SdkLoggerProvider>, TelemetryError> {
|
||||
if log_ep.is_empty() || !config.logs_export_enabled.unwrap_or(DEFAULT_OBS_LOGS_EXPORT_ENABLED) {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let exporter = opentelemetry_otlp::LogExporter::builder()
|
||||
.with_http()
|
||||
.with_endpoint(log_ep)
|
||||
.with_protocol(Protocol::HttpBinary)
|
||||
.with_compression(Compression::Gzip)
|
||||
.build()
|
||||
.map_err(|e| TelemetryError::BuildLogExporter(e.to_string()))?;
|
||||
|
||||
let mut builder = SdkLoggerProvider::builder().with_resource(res);
|
||||
builder = builder.with_batch_exporter(exporter);
|
||||
if use_stdout {
|
||||
builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default());
|
||||
}
|
||||
Ok(Some(builder.build()))
|
||||
}
|
||||
|
||||
/// Create a stdout periodic metrics reader for the given interval.
|
||||
fn create_periodic_reader(interval: u64) -> PeriodicReader<opentelemetry_stdout::MetricExporter> {
|
||||
PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default())
|
||||
.with_interval(Duration::from_secs(interval))
|
||||
.build()
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::GlobalError;
|
||||
use metrics::{Counter, CounterFn, Gauge, GaugeFn, Histogram, HistogramFn, Key, KeyName, Metadata, SharedString, Unit};
|
||||
use opentelemetry::{
|
||||
InstrumentationScope, InstrumentationScopeBuilder, KeyValue, global,
|
||||
metrics::{Meter, MeterProvider},
|
||||
};
|
||||
use opentelemetry_sdk::metrics::{MeterProviderBuilder, SdkMeterProvider};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::HashMap,
|
||||
ops::Deref,
|
||||
sync::{
|
||||
Arc, Mutex, RwLock,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
};
|
||||
use tracing::error;
|
||||
|
||||
macro_rules! configure_builder {
|
||||
($builder:expr, $metadata:expr) => {{
|
||||
let mut builder = $builder;
|
||||
if let Some(metadata) = $metadata {
|
||||
if let Some(unit) = metadata.unit {
|
||||
builder = builder.with_unit(unit.as_canonical_label());
|
||||
}
|
||||
builder = builder.with_description(metadata.description.to_string());
|
||||
}
|
||||
builder
|
||||
}};
|
||||
}
|
||||
|
||||
/// A builder for constructing a [`Recorder`].
|
||||
#[derive(Debug)]
|
||||
pub struct Builder {
|
||||
builder: MeterProviderBuilder,
|
||||
scope: InstrumentationScopeBuilder,
|
||||
}
|
||||
|
||||
impl Builder {
|
||||
/// Runs the closure (`f`) to modify the [`MeterProviderBuilder`] to build a
|
||||
/// [`MeterProvider`](MeterProvider).
|
||||
pub fn with_meter_provider(mut self, f: impl FnOnce(MeterProviderBuilder) -> MeterProviderBuilder) -> Self {
|
||||
self.builder = f(self.builder);
|
||||
self
|
||||
}
|
||||
|
||||
/// Modify the [`InstrumentationScope`] to provide additional metadata from the
|
||||
/// closure (`f`).
|
||||
pub fn with_instrumentation_scope(
|
||||
mut self,
|
||||
f: impl FnOnce(InstrumentationScopeBuilder) -> InstrumentationScopeBuilder,
|
||||
) -> Self {
|
||||
self.scope = f(self.scope);
|
||||
self
|
||||
}
|
||||
|
||||
/// Consumes the builder and builds a new [`Recorder`] and returns
|
||||
/// a [`SdkMeterProvider`].
|
||||
///
|
||||
/// A [`SdkMeterProvider`] is provided so you have the responsibility to
|
||||
/// do whatever you need to do with it.
|
||||
///
|
||||
/// This will not install the recorder as the global recorder for
|
||||
/// the [`metrics`] crate, use [`Builder::install`]. This will not install a meter
|
||||
/// provider to [`global`], use [`Builder::install_global`].
|
||||
pub fn build(self) -> (SdkMeterProvider, Recorder) {
|
||||
let provider = self.builder.build();
|
||||
let meter = provider.meter_with_scope(self.scope.build());
|
||||
|
||||
(provider, Recorder::with_meter(meter))
|
||||
}
|
||||
|
||||
/// Builds a [`Recorder`] and sets it as the global recorder for the [`metrics`]
|
||||
/// crate.
|
||||
///
|
||||
/// This method will not call [`global::set_meter_provider`] for OpenTelemetry and
|
||||
/// will be returned as the first element in the return's type tuple.
|
||||
pub fn install(self) -> Result<(SdkMeterProvider, Recorder), GlobalError> {
|
||||
let (provider, recorder) = self.build();
|
||||
metrics::set_global_recorder(recorder.clone())?;
|
||||
|
||||
Ok((provider, recorder))
|
||||
}
|
||||
|
||||
/// Builds the [`Recorder`] to record metrics to OpenTelemetry, set the global
|
||||
/// recorder for the [`metrics`] crate, and calls [`global::set_meter_provider`]
|
||||
/// to set the constructed [`SdkMeterProvider`].
|
||||
pub fn install_global(self) -> Result<Recorder, GlobalError> {
|
||||
let (provider, recorder) = self.install()?;
|
||||
global::set_meter_provider(provider);
|
||||
|
||||
Ok(recorder)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MetricMetadata {
|
||||
unit: Option<Unit>,
|
||||
description: SharedString,
|
||||
}
|
||||
|
||||
/// A standard recorder that implements [`metrics::Recorder`].
|
||||
///
|
||||
/// This instance implements <code>[`Deref`]\<Target = [`Meter`]\></code>, so
|
||||
/// you can still interact with the SDK's initialized [`Meter`] instance.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Recorder {
|
||||
meter: Meter,
|
||||
metrics_metadata: Arc<Mutex<HashMap<KeyName, MetricMetadata>>>,
|
||||
// cache metric handlers as to not reregister on each call
|
||||
cached_counters: Arc<RwLock<HashMap<Key, Counter>>>,
|
||||
cached_gauges: Arc<RwLock<HashMap<Key, Gauge>>>,
|
||||
cached_histograms: Arc<RwLock<HashMap<Key, Histogram>>>,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
/// Creates a new [`Builder`] with a given name for instrumentation.
|
||||
pub fn builder<S: Into<Cow<'static, str>>>(name: S) -> Builder {
|
||||
Builder {
|
||||
builder: MeterProviderBuilder::default(),
|
||||
scope: InstrumentationScope::builder(name.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a [`Recorder`] with an already established [`Meter`].
|
||||
pub fn with_meter(meter: Meter) -> Self {
|
||||
Recorder {
|
||||
meter,
|
||||
metrics_metadata: Default::default(),
|
||||
cached_counters: Default::default(),
|
||||
cached_gauges: Default::default(),
|
||||
cached_histograms: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_cached_metric<T: Clone>(lock: &RwLock<HashMap<Key, T>>, key: &Key, metric_type: &str) -> Option<T> {
|
||||
let cache = match lock.read() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
error!("{} cache read lock poisoned: {}", metric_type, e);
|
||||
e.into_inner()
|
||||
}
|
||||
};
|
||||
cache.get(key).cloned()
|
||||
}
|
||||
|
||||
fn insert_cached_metric<T: Clone>(lock: &RwLock<HashMap<Key, T>>, key: Key, value: T, metric_type: &str) -> T {
|
||||
let mut cache = match lock.write() {
|
||||
Ok(g) => g,
|
||||
Err(e) => {
|
||||
error!("{} cache write lock poisoned: {}", metric_type, e);
|
||||
e.into_inner()
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(v) = cache.get(&key) {
|
||||
return v.clone();
|
||||
}
|
||||
cache.insert(key, value.clone());
|
||||
value
|
||||
}
|
||||
|
||||
fn with_metadata_lock<F, R>(&self, f: F) -> R
|
||||
where
|
||||
F: FnOnce(&mut HashMap<KeyName, MetricMetadata>) -> R,
|
||||
{
|
||||
let mut guard = self.metrics_metadata.lock().unwrap_or_else(|e| {
|
||||
error!("metrics_metadata lock poisoned: {}", e);
|
||||
e.into_inner()
|
||||
});
|
||||
f(&mut guard)
|
||||
}
|
||||
|
||||
fn describe_metric(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.with_metadata_lock(|metadata| {
|
||||
metadata.insert(key, MetricMetadata { unit, description });
|
||||
});
|
||||
}
|
||||
|
||||
fn get_metadata_for_builder(&self, key_name: &str) -> Option<MetricMetadata> {
|
||||
self.with_metadata_lock(|metadata| metadata.remove(key_name))
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Recorder {
|
||||
type Target = Meter;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.meter
|
||||
}
|
||||
}
|
||||
|
||||
impl metrics::Recorder for Recorder {
|
||||
fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe_metric(key, unit, description);
|
||||
}
|
||||
|
||||
fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe_metric(key, unit, description);
|
||||
}
|
||||
|
||||
fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe_metric(key, unit, description);
|
||||
}
|
||||
|
||||
fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
|
||||
if let Some(counter) = Self::get_cached_metric(&self.cached_counters, key, "counter") {
|
||||
return counter;
|
||||
}
|
||||
|
||||
let builder = self.meter.u64_counter(key.name().to_owned());
|
||||
let metadata = self.get_metadata_for_builder(key.name());
|
||||
let builder = configure_builder!(builder, metadata);
|
||||
|
||||
let counter = builder.build();
|
||||
let labels = key
|
||||
.labels()
|
||||
.map(|label| KeyValue::new(label.key().to_owned(), label.value().to_owned()))
|
||||
.collect();
|
||||
|
||||
let handle = Counter::from_arc(Arc::new(WrappedCounter {
|
||||
counter,
|
||||
labels,
|
||||
value: AtomicU64::new(0),
|
||||
}));
|
||||
|
||||
Self::insert_cached_metric(&self.cached_counters, key.clone(), handle, "counter")
|
||||
}
|
||||
|
||||
fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge {
|
||||
if let Some(gauge) = Self::get_cached_metric(&self.cached_gauges, key, "gauge") {
|
||||
return gauge;
|
||||
}
|
||||
|
||||
let builder = self.meter.f64_gauge(key.name().to_owned());
|
||||
let metadata = self.get_metadata_for_builder(key.name());
|
||||
let builder = configure_builder!(builder, metadata);
|
||||
|
||||
let gauge = builder.build();
|
||||
let labels = key
|
||||
.labels()
|
||||
.map(|label| KeyValue::new(label.key().to_owned(), label.value().to_owned()))
|
||||
.collect();
|
||||
|
||||
let handle = Gauge::from_arc(Arc::new(WrappedGauge {
|
||||
gauge,
|
||||
labels,
|
||||
value: AtomicU64::new(0),
|
||||
}));
|
||||
|
||||
Self::insert_cached_metric(&self.cached_gauges, key.clone(), handle, "gauge")
|
||||
}
|
||||
|
||||
fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram {
|
||||
if let Some(histogram) = Self::get_cached_metric(&self.cached_histograms, key, "histogram") {
|
||||
return histogram;
|
||||
}
|
||||
|
||||
let builder = self.meter.f64_histogram(key.name().to_owned());
|
||||
let metadata = self.get_metadata_for_builder(key.name());
|
||||
let builder = configure_builder!(builder, metadata);
|
||||
|
||||
let histogram = builder.build();
|
||||
let labels = key
|
||||
.labels()
|
||||
.map(|label| KeyValue::new(label.key().to_owned(), label.value().to_owned()))
|
||||
.collect();
|
||||
|
||||
let handle = Histogram::from_arc(Arc::new(WrappedHistogram { histogram, labels }));
|
||||
|
||||
Self::insert_cached_metric(&self.cached_histograms, key.clone(), handle, "histogram")
|
||||
}
|
||||
}
|
||||
|
||||
struct WrappedCounter {
|
||||
counter: opentelemetry::metrics::Counter<u64>,
|
||||
labels: Vec<KeyValue>,
|
||||
value: AtomicU64,
|
||||
}
|
||||
|
||||
impl CounterFn for WrappedCounter {
|
||||
fn increment(&self, value: u64) {
|
||||
self.value.fetch_add(value, Ordering::Relaxed);
|
||||
self.counter.add(value, &self.labels);
|
||||
}
|
||||
|
||||
fn absolute(&self, value: u64) {
|
||||
let prev = self.value.swap(value, Ordering::Relaxed);
|
||||
let diff = value.saturating_sub(prev);
|
||||
self.counter.add(diff, &self.labels);
|
||||
}
|
||||
}
|
||||
|
||||
struct WrappedGauge {
|
||||
gauge: opentelemetry::metrics::Gauge<f64>,
|
||||
labels: Vec<KeyValue>,
|
||||
value: AtomicU64,
|
||||
}
|
||||
|
||||
impl GaugeFn for WrappedGauge {
|
||||
fn increment(&self, value: f64) {
|
||||
let mut current = self.value.load(Ordering::Relaxed);
|
||||
let mut new = f64::from_bits(current) + value;
|
||||
while let Err(val) = self
|
||||
.value
|
||||
.compare_exchange(current, new.to_bits(), Ordering::AcqRel, Ordering::Relaxed)
|
||||
{
|
||||
current = val;
|
||||
new = f64::from_bits(current) + value;
|
||||
}
|
||||
|
||||
self.gauge.record(new, &self.labels);
|
||||
}
|
||||
|
||||
fn decrement(&self, value: f64) {
|
||||
let mut current = self.value.load(Ordering::Relaxed);
|
||||
let mut new = f64::from_bits(current) - value;
|
||||
while let Err(val) = self
|
||||
.value
|
||||
.compare_exchange(current, new.to_bits(), Ordering::AcqRel, Ordering::Relaxed)
|
||||
{
|
||||
current = val;
|
||||
new = f64::from_bits(current) - value;
|
||||
}
|
||||
|
||||
self.gauge.record(new, &self.labels);
|
||||
}
|
||||
|
||||
fn set(&self, value: f64) {
|
||||
self.value.store(value.to_bits(), Ordering::Relaxed);
|
||||
self.gauge.record(value, &self.labels);
|
||||
}
|
||||
}
|
||||
|
||||
struct WrappedHistogram {
|
||||
histogram: opentelemetry::metrics::Histogram<f64>,
|
||||
labels: Vec<KeyValue>,
|
||||
}
|
||||
|
||||
impl HistogramFn for WrappedHistogram {
|
||||
fn record(&self, value: f64) {
|
||||
self.histogram.record(value, &self.labels);
|
||||
}
|
||||
|
||||
fn record_many(&self, value: f64, count: usize) {
|
||||
for _ in 0..count {
|
||||
self.histogram.record(value, &self.labels);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use metrics::Recorder as _;
|
||||
use opentelemetry_sdk::metrics::Temporality;
|
||||
|
||||
fn test_recorder() -> Recorder {
|
||||
let exporter = opentelemetry_stdout::MetricExporterBuilder::default()
|
||||
.with_temporality(Temporality::Cumulative)
|
||||
.build();
|
||||
|
||||
let (_provider, recorder) = Recorder::builder("test")
|
||||
.with_meter_provider(|b| b.with_periodic_exporter(exporter))
|
||||
.build();
|
||||
|
||||
recorder
|
||||
}
|
||||
|
||||
fn test_metadata() -> Metadata<'static> {
|
||||
Metadata::new(module_path!(), metrics::Level::INFO, None)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standard_usage() {
|
||||
let exporter = opentelemetry_stdout::MetricExporterBuilder::default()
|
||||
.with_temporality(Temporality::Cumulative)
|
||||
.build();
|
||||
|
||||
let (provider, recorder) = Recorder::builder("my-app")
|
||||
.with_meter_provider(|builder| builder.with_periodic_exporter(exporter))
|
||||
.build();
|
||||
|
||||
global::set_meter_provider(provider.clone());
|
||||
metrics::set_global_recorder(recorder).unwrap();
|
||||
|
||||
let counter = metrics::counter!("my-counter");
|
||||
counter.increment(1);
|
||||
|
||||
provider.force_flush().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counter_cached_on_repeated_registration() {
|
||||
let recorder = test_recorder();
|
||||
let key = Key::from_name("requests_total");
|
||||
let meta = test_metadata();
|
||||
|
||||
let _first = recorder.register_counter(&key, &meta);
|
||||
let _second = recorder.register_counter(&key, &meta);
|
||||
|
||||
let cache = recorder.cached_counters.read().unwrap();
|
||||
assert_eq!(cache.len(), 1, "counter should be cached and inserted only once");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gauge_cached_on_repeated_registration() {
|
||||
let recorder = test_recorder();
|
||||
let key = Key::from_name("active_connections");
|
||||
let meta = test_metadata();
|
||||
|
||||
let _first = recorder.register_gauge(&key, &meta);
|
||||
let _second = recorder.register_gauge(&key, &meta);
|
||||
|
||||
let cache = recorder.cached_gauges.read().unwrap();
|
||||
assert_eq!(cache.len(), 1, "gauge should be cached and inserted only once");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_cached_on_repeated_registration() {
|
||||
let recorder = test_recorder();
|
||||
let key = Key::from_name("request_duration");
|
||||
let meta = test_metadata();
|
||||
|
||||
let _first = recorder.register_histogram(&key, &meta);
|
||||
let _second = recorder.register_histogram(&key, &meta);
|
||||
|
||||
let cache = recorder.cached_histograms.read().unwrap();
|
||||
assert_eq!(cache.len(), 1, "histogram should be cached and inserted only once");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_register_counter_inserts_once() {
|
||||
let recorder = test_recorder();
|
||||
let key = Key::from_name("concurrent_counter");
|
||||
let shared = Arc::new(recorder);
|
||||
let barrier = Arc::new(std::sync::Barrier::new(10));
|
||||
|
||||
let handles: Vec<_> = (0..10)
|
||||
.map(|_| {
|
||||
let r = Arc::clone(&shared);
|
||||
let k = key.clone();
|
||||
let b = Arc::clone(&barrier);
|
||||
std::thread::spawn(move || {
|
||||
b.wait();
|
||||
let _ = r.register_counter(&k, &test_metadata());
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for h in handles {
|
||||
h.join().unwrap();
|
||||
}
|
||||
|
||||
let cache = shared.cached_counters.read().unwrap();
|
||||
assert_eq!(cache.len(), 1, "concurrent registrations should produce exactly one cache entry");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! OpenTelemetry [`Resource`] construction for RustFS.
|
||||
//!
|
||||
//! A `Resource` describes the entity producing telemetry data. The resource
|
||||
//! built here includes the service name, service version, deployment
|
||||
//! environment, and the local machine IP address so that data can be
|
||||
//! correlated across services in a distributed system.
|
||||
|
||||
use crate::config::OtelConfig;
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry_sdk::Resource;
|
||||
use opentelemetry_semantic_conventions::{
|
||||
SCHEMA_URL,
|
||||
attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION},
|
||||
};
|
||||
use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
|
||||
use rustfs_utils::get_local_ip_with_default;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Build an OpenTelemetry [`Resource`] populated from the provided config.
|
||||
///
|
||||
/// The resource carries the following attributes:
|
||||
/// - `service.name` — from `config.service_name`, defaulting to [`APP_NAME`].
|
||||
/// - `service.version` — from `config.service_version`, defaulting to
|
||||
/// [`SERVICE_VERSION`].
|
||||
/// - `deployment.environment` — from `config.environment`, defaulting to
|
||||
/// [`ENVIRONMENT`].
|
||||
/// - `network.local.address` — the primary local IP of the current host,
|
||||
/// useful for identifying individual nodes in a cluster.
|
||||
///
|
||||
/// All attributes are attached to the resource using the semantic conventions
|
||||
/// schema URL to ensure compatibility with standard OTLP backends.
|
||||
pub(super) fn build_resource(config: &OtelConfig) -> Resource {
|
||||
Resource::builder()
|
||||
.with_service_name(Cow::Borrowed(config.service_name.as_deref().unwrap_or(APP_NAME)).to_string())
|
||||
.with_schema_url(
|
||||
[
|
||||
KeyValue::new(
|
||||
OTEL_SERVICE_VERSION,
|
||||
Cow::Borrowed(config.service_version.as_deref().unwrap_or(SERVICE_VERSION)).to_string(),
|
||||
),
|
||||
KeyValue::new(
|
||||
DEPLOYMENT_ENVIRONMENT_NAME,
|
||||
Cow::Borrowed(config.environment.as_deref().unwrap_or(ENVIRONMENT)).to_string(),
|
||||
),
|
||||
KeyValue::new(NETWORK_LOCAL_ADDRESS, get_local_ip_with_default()),
|
||||
],
|
||||
SCHEMA_URL,
|
||||
)
|
||||
.build()
|
||||
}
|
||||
Reference in New Issue
Block a user