gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-23 12:49:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(obs): standardize runtime logging batch one (#3438)

Browse Source
This commit is contained in:
cxymds
2026-06-14 18:55:14 +08:00
committed by GitHub
parent bed4c7288b
commit 8f6b1d47b5
5 changed files with 199 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/obs/src/logging.rs
+82
View File
@@ -83,6 +83,19 @@ mod tests {
}
}
fn assert_source_contains(rel_path: &str, required_patterns: &[&str]) {
let path = workspace_root().join(rel_path);
let source = fs::read_to_string(&path).unwrap_or_else(|err| panic!("failed to read {}: {}", path.display(), err));
for pattern in required_patterns {
assert!(
source.contains(pattern),
"missing required logging governance pattern `{}` in {}",
pattern,
path.display()
);
}
}
#[test]
fn logging_redaction_rules_are_valid() {
assert!(validate_logging_redaction_rules().is_ok());
@@ -156,6 +169,75 @@ mod tests {
);
}
#[test]
fn startup_fatal_stderr_uses_single_formatter_for_pre_observability_failures() {
assert_source_contains(
"rustfs/src/main.rs",
&[
"fn format_fatal_stderr_message(context: &str, error: impl std::fmt::Display) -> String",
"fn emit_fatal_stderr(context: &str, error: impl std::fmt::Display)",
"emit_fatal_stderr(\"Server runtime failed\", e)",
"emit_fatal_stderr(\"Command parse failed\", e)",
"emit_fatal_stderr(\"Observability initialization failed\", &e)",
],
);
}
#[test]
fn observability_runtime_logging_uses_tracing_for_fallback_and_guard_shutdown() {
assert_no_unmasked_access_key_logging(
"crates/obs/src/telemetry/local.rs",
&[
"[WARN] Failed to initialize file observability logging",
"Falling back to stdout logging.",
],
);
assert_source_contains(
"crates/obs/src/telemetry/local.rs",
&[
"warn!(",
"state = \"fallback_to_stdout\"",
"failed_sink = \"file\"",
"sink = \"stdout\"",
],
);
assert_no_unmasked_access_key_logging(
"crates/obs/src/telemetry/guard.rs",
&[
"eprintln!(\"Tracer shutdown error: {err:?}\")",
"eprintln!(\"Meter shutdown error: {err:?}\")",
"eprintln!(\"Logger shutdown error: {err:?}\")",
"eprintln!(\"Log cleanup task stopped\")",
"eprintln!(\"Tracing guard dropped, flushing logs.\")",
"eprintln!(\"Stdout guard dropped, flushing logs.\")",
],
);
assert_source_contains(
"crates/obs/src/telemetry/guard.rs",
&[
"EVENT_OBS_GUARD_SHUTDOWN",
"resource = \"tracer_provider\"",
"resource = \"meter_provider\"",
"resource = \"logger_provider\"",
"resource = \"log_cleaner\"",
"resource = \"tracing_guard\"",
"resource = \"stdout_guard\"",
],
);
}
#[test]
fn low_level_logging_sink_stderr_exceptions_remain_explicit() {
assert_source_contains(
"crates/obs/src/telemetry/rolling.rs",
&[
"Failed to flush log file before rotation",
"RollingAppender: Failed to rotate log file after",
"RollingAppender: failed to rotate log file",
],
);
}
#[test]
fn audit_notify_runtime_logging_does_not_use_previous_sentence_first_noise_patterns() {
assert_no_unmasked_access_key_logging(
crates/obs/src/telemetry/guard.rs
+76 -10
View File
@@ -27,6 +27,7 @@
//! 7. Stdout worker guard — flushes buffered log lines written to stdout.
use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider};
use tracing::{debug, error};
#[cfg(all(
feature = "pyroscope",
@@ -43,6 +44,10 @@ pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyro
#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
pub(crate) type MemoryProfilingAgent = ();
const LOG_COMPONENT_OBS: &str = "obs";
const LOG_SUBSYSTEM_GUARD: &str = "guard";
const EVENT_OBS_GUARD_SHUTDOWN: &str = "obs_guard_shutdown";
/// RAII guard that owns all active OpenTelemetry providers and the
/// `tracing_appender` worker guard.
///
@@ -85,25 +90,49 @@ impl std::fmt::Debug for OtelGuard {
impl Drop for OtelGuard {
/// Shut down all telemetry providers in order.
///
/// Errors during shutdown are printed to `stderr` so they are visible even
/// after the tracing subscriber has been torn down.
/// Errors are emitted before tracing resources are dropped so shutdown
/// diagnostics remain structured and low-noise.
fn drop(&mut self) {
if let Some(provider) = self.tracer_provider.take()
&& let Err(err) = provider.shutdown()
{
eprintln!("Tracer shutdown error: {err:?}");
error!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "tracer_provider",
result = "shutdown_failed",
error = ?err,
"observability guard shutdown failed"
);
}
if let Some(provider) = self.meter_provider.take()
&& let Err(err) = provider.shutdown()
{
eprintln!("Meter shutdown error: {err:?}");
error!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "meter_provider",
result = "shutdown_failed",
error = ?err,
"observability guard shutdown failed"
);
}
if let Some(provider) = self.logger_provider.take()
&& let Err(err) = provider.shutdown()
{
eprintln!("Logger shutdown error: {err:?}");
error!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "logger_provider",
result = "shutdown_failed",
error = ?err,
"observability guard shutdown failed"
);
}
#[cfg(all(
@@ -112,7 +141,15 @@ impl Drop for OtelGuard {
))]
if let Some(agent) = self.profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Profiling agent stop error: {err:?}"),
Err(err) => error!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "profiling_agent",
result = "shutdown_failed",
error = ?err,
"observability guard shutdown failed"
),
Ok(stopped) => {
stopped.shutdown();
}
@@ -122,7 +159,15 @@ impl Drop for OtelGuard {
#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
if let Some(agent) = self.memory_profiling_agent.take() {
match agent.stop() {
Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"),
Err(err) => error!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "memory_profiling_agent",
result = "shutdown_failed",
error = ?err,
"observability guard shutdown failed"
),
Ok(stopped) => {
stopped.shutdown();
}
@@ -130,18 +175,39 @@ impl Drop for OtelGuard {
}
if let Some(handle) = self.cleanup_handle.take() {
debug!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "log_cleaner",
result = "abort_requested",
"observability guard resource shutdown requested"
);
handle.abort();
eprintln!("Log cleanup task stopped");
}
if let Some(guard) = self.tracing_guard.take() {
debug!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "tracing_guard",
result = "flush_requested",
"observability guard resource flush requested"
);
drop(guard);
eprintln!("Tracing guard dropped, flushing logs.");
}
if let Some(guard) = self.stdout_guard.take() {
debug!(
event = EVENT_OBS_GUARD_SHUTDOWN,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_GUARD,
resource = "stdout_guard",
result = "flush_requested",
"observability guard resource flush requested"
);
drop(guard);
eprintln!("Stdout guard dropped, flushing logs.");
}
}
}
crates/obs/src/telemetry/local.rs
+14 -5
View File
@@ -51,7 +51,7 @@ use rustfs_config::{APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME,
use std::sync::Arc;
use std::{fs, io::IsTerminal, time::Duration};
use tracing::Subscriber;
use tracing::info;
use tracing::{info, warn};
use tracing_error::ErrorLayer;
use tracing_subscriber::{
fmt::{format::FmtSpan, time::LocalTime},
@@ -126,8 +126,9 @@ pub(super) fn init_local_logging(
match init_file_logging_internal(config, log_directory, logger_level, is_production) {
Ok(guard) => Ok(guard),
Err(error) if should_fallback_to_stdout(&error) => {
let guard = init_stdout_only(config, logger_level, is_production);
emit_file_logging_fallback_warning(log_directory, &error);
Ok(init_stdout_only(config, logger_level, is_production))
Ok(guard)
}
Err(error) => Err(error),
}
@@ -361,9 +362,17 @@ pub(super) fn should_fallback_to_stdout(error: &TelemetryError) -> bool {
}
pub(super) fn emit_file_logging_fallback_warning(log_directory: &str, error: &TelemetryError) {
eprintln!(
"[WARN] Failed to initialize file observability logging at '{}': {}. Falling back to stdout logging.",
log_directory, error
warn!(
event = EVENT_LOCAL_LOGGING_STATE,
component = LOG_COMPONENT_OBS,
subsystem = LOG_SUBSYSTEM_LOCAL_LOGGING,
state = "fallback_to_stdout",
backend = "local",
failed_sink = "file",
sink = "stdout",
log_directory,
error = %error,
"local logging state changed"
);
}