diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 34a807204..856de8723 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -71,7 +71,7 @@ zstd = { workspace = true, features = ["zstdmt"] } sysinfo = { workspace = true } nvml-wrapper = { workspace = true, optional = true } -[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +[target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies] pyroscope = { workspace = true } jemalloc_pprof = { workspace = true } diff --git a/crates/obs/src/telemetry/guard.rs b/crates/obs/src/telemetry/guard.rs index 75c23c800..033103ba3 100644 --- a/crates/obs/src/telemetry/guard.rs +++ b/crates/obs/src/telemetry/guard.rs @@ -41,9 +41,9 @@ pub struct OtelGuard { pub(crate) meter_provider: Option, /// Optional logger provider for OTLP log export. pub(crate) logger_provider: Option, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] pub(crate) profiling_agent: Option>, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] pub(crate) memory_profiling_agent: Option>, /// Handle to the background log-cleanup task; aborted on drop. pub(crate) cleanup_handle: Option>, @@ -60,9 +60,9 @@ impl std::fmt::Debug for OtelGuard { s.field("tracer_provider", &self.tracer_provider.is_some()) .field("meter_provider", &self.meter_provider.is_some()) .field("logger_provider", &self.logger_provider.is_some()); - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] s.field("profiling_agent", &self.profiling_agent.is_some()); - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] s.field("memory_profiling_agent", &self.memory_profiling_agent.is_some()); s.field("cleanup_handle", &self.cleanup_handle.is_some()) .field("tracing_guard", &self.tracing_guard.is_some()) @@ -95,7 +95,7 @@ impl Drop for OtelGuard { eprintln!("Logger shutdown error: {err:?}"); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] if let Some(agent) = self.profiling_agent.take() { match agent.stop() { Err(err) => eprintln!("Profiling agent stop error: {err:?}"), @@ -105,7 +105,7 @@ impl Drop for OtelGuard { } } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] if let Some(agent) = self.memory_profiling_agent.take() { match agent.stop() { Err(err) => eprintln!("Memory profiling agent stop error: {err:?}"), diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index 393748c77..f5bc20042 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -155,9 +155,9 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo tracer_provider: None, meter_provider: None, logger_provider: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent: None, tracing_guard: Some(guard), stdout_guard: None, @@ -291,9 +291,9 @@ fn init_file_logging_internal( tracer_provider: None, meter_provider: None, logger_provider: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent: None, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent: None, tracing_guard: Some(guard), stdout_guard, diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index acba1fb2f..083b99602 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -164,10 +164,10 @@ pub(super) fn init_observability_http( // ── Meter provider (HTTP) ───────────────────────────────────────────────── let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?; - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] let profiling_agent = init_profiler(config); - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] let memory_profiling_agent = init_memory_profiler(config); // ── Logger Logic ────────────────────────────────────────────────────────── @@ -210,7 +210,7 @@ pub(super) fn init_observability_http( let file_logging_result = (|| -> Result<_, TelemetryError> { fs::create_dir_all(log_directory).map_err(|e| TelemetryError::Io(e.to_string()))?; - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(unix)] crate::telemetry::local::ensure_dir_permissions(log_directory)?; let rotation_str = config @@ -317,9 +317,9 @@ pub(super) fn init_observability_http( tracer_provider, meter_provider, logger_provider, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] profiling_agent, - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] memory_profiling_agent, tracing_guard, stdout_guard, @@ -496,7 +496,7 @@ fn build_logger_provider( /// Returns `None` when profiling export is disabled, when no usable /// profiling endpoint is configured, or when building or starting the agent /// fails. -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] fn init_profiler(config: &OtelConfig) -> Option> { use pyroscope::backend::{BackendConfig, PprofConfig, pprof_backend}; use pyroscope::pyroscope::PyroscopeAgentBuilder; @@ -541,7 +541,7 @@ fn init_profiler(config: &OtelConfig) -> Option Option> { use pyroscope::backend::jemalloc_backend; use pyroscope::pyroscope::PyroscopeAgentBuilder; diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 185f83a19..9b53de4cc 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -173,16 +173,18 @@ hashbrown = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] libsystemd.workspace = true -[target.'cfg(not(any(target_os = "linux", target_os = "macos")))'.dependencies] +[target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies] mimalloc = { workspace = true } libmimalloc-sys = { version = "0.1.49", features = ["extended"] } -# jemalloc + pprof-based profiling on linux and macos. [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +pprof = { workspace = true } + +# jemalloc-based allocator and memory profiling are only enabled on validated linux-gnu-x86_64 targets. +[target.'cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))'.dependencies] tikv-jemallocator = { workspace = true } tikv-jemalloc-ctl = { workspace = true } jemalloc_pprof = { workspace = true } -pprof = { workspace = true } [dev-dependencies] uuid = { workspace = true, features = ["v4"] } diff --git a/rustfs/src/admin/handlers/profile.rs b/rustfs/src/admin/handlers/profile.rs index be337aca1..0c33bd558 100644 --- a/rustfs/src/admin/handlers/profile.rs +++ b/rustfs/src/admin/handlers/profile.rs @@ -15,10 +15,10 @@ use crate::admin::{auth::validate_admin_request, router::Operation}; use crate::auth::{check_key_valid, get_session_token}; use crate::server::RemoteAddr; -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] use http::HeaderMap; use http::StatusCode; -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] use http::header::CONTENT_TYPE; use matchit::Params; use rustfs_policy::policy::action::{Action, AdminAction}; @@ -52,7 +52,7 @@ impl Operation for TriggerProfileCPU { authorize_profile_request(&req).await?; info!("Triggering CPU profile dump via S3 request..."); - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))))] { return Ok(S3Response::new(( StatusCode::NOT_IMPLEMENTED, @@ -64,7 +64,7 @@ impl Operation for TriggerProfileCPU { ))); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] { let dur = std::time::Duration::from_secs(60); match crate::profiling::dump_cpu_pprof_for(dur).await { @@ -86,7 +86,7 @@ impl Operation for TriggerProfileMemory { authorize_profile_request(&req).await?; info!("Triggering Memory profile dump via S3 request..."); - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] { return Ok(S3Response::new(( StatusCode::NOT_IMPLEMENTED, @@ -94,7 +94,7 @@ impl Operation for TriggerProfileMemory { ))); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] { match crate::profiling::dump_memory_pprof_now().await { Ok(path) => { diff --git a/rustfs/src/admin/handlers/profile_admin.rs b/rustfs/src/admin/handlers/profile_admin.rs index 8741713ae..d56a16c9f 100644 --- a/rustfs/src/admin/handlers/profile_admin.rs +++ b/rustfs/src/admin/handlers/profile_admin.rs @@ -70,7 +70,7 @@ impl Operation for ProfileHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize_profile_request(&req).await?; - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))))] { let requested_url = req.uri.to_string(); let target_os = std::env::consts::OS; @@ -82,7 +82,7 @@ impl Operation for ProfileHandler { return Ok(S3Response::new((StatusCode::NOT_IMPLEMENTED, Body::from(msg)))); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] { use rustfs_config::{DEFAULT_CPU_FREQ, ENV_CPU_FREQ}; use rustfs_utils::get_env_usize; @@ -172,9 +172,9 @@ impl Operation for ProfileStatusHandler { async fn call(&self, req: S3Request, _params: Params<'_, '_>) -> S3Result> { authorize_profile_request(&req).await?; - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))))] let message = format!("CPU profiling is not supported on {} platform", std::env::consts::OS); - #[cfg(not(any(target_os = "linux", target_os = "macos")))] + #[cfg(not(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))))] let status = HashMap::from([ ("enabled", "false"), ("status", "not_supported"), @@ -182,7 +182,7 @@ impl Operation for ProfileStatusHandler { ("message", message.as_str()), ]); - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] let status = { use rustfs_config::{DEFAULT_ENABLE_PROFILING, ENV_ENABLE_PROFILING}; use rustfs_utils::get_env_bool; diff --git a/rustfs/src/allocator_reclaim.rs b/rustfs/src/allocator_reclaim.rs index 99fe99bd8..6602d4161 100644 --- a/rustfs/src/allocator_reclaim.rs +++ b/rustfs/src/allocator_reclaim.rs @@ -18,12 +18,15 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, warn}; pub fn allocator_backend() -> &'static str { - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] { "jemalloc" } - #[cfg(all(not(target_os = "windows"), not(any(target_os = "linux", target_os = "macos"))))] + #[cfg(all( + not(target_os = "windows"), + not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) + ))] { "mimalloc" } @@ -82,7 +85,10 @@ fn reclaimable_work_snapshot() -> ReclaimableWorkSnapshot { } } -#[cfg(all(not(target_os = "windows"), not(any(target_os = "linux", target_os = "macos"))))] +#[cfg(all( + not(target_os = "windows"), + not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")) +))] #[allow(unsafe_code)] fn collect_allocator_memory(force: bool) -> Result<(), String> { // SAFETY: `mi_collect` is provided by the active global allocator backend @@ -94,7 +100,7 @@ fn collect_allocator_memory(force: bool) -> Result<(), String> { Ok(()) } -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] fn collect_allocator_memory(_force: bool) -> Result<(), String> { let _ = tikv_jemalloc_ctl::background_thread::write(true); tikv_jemalloc_ctl::epoch::advance().map_err(|err| err.to_string())?; diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index c6993c8d0..293b49797 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -73,11 +73,11 @@ const ENV_SCANNER_ENABLED: &str = "RUSTFS_SCANNER_ENABLED"; const ENV_SCANNER_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_SCANNER"; const ENV_HEAL_ENABLED: &str = "RUSTFS_HEAL_ENABLED"; const ENV_HEAL_ENABLED_DEPRECATED: &str = "RUSTFS_ENABLE_HEAL"; -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(not(any(target_os = "linux", target_os = "macos")))] +#[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; diff --git a/rustfs/src/profiling.rs b/rustfs/src/profiling.rs index 03e9df234..c2d170cee 100644 --- a/rustfs/src/profiling.rs +++ b/rustfs/src/profiling.rs @@ -62,7 +62,6 @@ pub use unsupported_impl::{dump_cpu_pprof_for, dump_memory_pprof_now, init_from_ #[cfg(any(target_os = "linux", target_os = "macos"))] mod linux_impl { - use jemalloc_pprof::PROF_CTL; use pprof::protos::Message; use rustfs_config::{ DEFAULT_CPU_DURATION_SECS, DEFAULT_CPU_FREQ, DEFAULT_CPU_INTERVAL_SECS, DEFAULT_CPU_MODE, DEFAULT_ENABLE_PROFILING, @@ -78,7 +77,7 @@ mod linux_impl { use tokio::sync::Mutex; use tokio::time::sleep; use tokio_util::sync::CancellationToken; - use tracing::{debug, error, info, warn}; + use tracing::{debug, info, warn}; static CPU_CONT_GUARD: OnceLock>>>> = OnceLock::new(); static PROFILING_CANCEL_TOKEN: OnceLock = OnceLock::new(); @@ -153,11 +152,12 @@ mod linux_impl { } // Public API: dump memory pprof now (jemalloc) + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] pub async fn dump_memory_pprof_now() -> Result { let out = output_dir().join(format!("mem_profile_{}.pb", ts())); let mut f = File::create(&out).map_err(|e| format!("create file failed: {e}"))?; - let prof_ctl_cell = PROF_CTL + let prof_ctl_cell = jemalloc_pprof::PROF_CTL .as_ref() .ok_or_else(|| "jemalloc profiling control not available".to_string())?; let mut prof_ctl = prof_ctl_cell.lock().await; @@ -172,7 +172,13 @@ mod linux_impl { Ok(out) } + #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] + pub async fn dump_memory_pprof_now() -> Result { + Err(memory_profiling_unsupported_message()) + } + // Jemalloc status check (No forced placement, only status observation) + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] pub async fn check_jemalloc_profiling() { use tikv_jemalloc_ctl::{config, epoch, stats}; @@ -190,7 +196,7 @@ mod linux_impl { Err(_) => debug!("MALLOC_CONF is not set"), } - if let Some(lock) = PROF_CTL.as_ref() { + if let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() { let ctl = lock.lock().await; info!(activated = ctl.activated(), "jemalloc profiling status"); } else { @@ -213,6 +219,11 @@ mod linux_impl { show!("active", stats::active::read()); } + #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] + pub async fn check_jemalloc_profiling() { + debug!("jemalloc profiling status check skipped on unsupported target"); + } + // Internal: start continuous CPU profiling async fn start_cpu_continuous(freq_hz: i32) { let cell = CPU_CONT_GUARD.get_or_init(|| Arc::new(Mutex::new(None))).clone(); @@ -283,6 +294,7 @@ mod linux_impl { } // Internal: start periodic memory dump when jemalloc profiling is active + #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))] async fn start_memory_periodic(interval: Duration, token: CancellationToken) { info!(?interval, "start periodic memory pprof dump"); tokio::spawn(async move { @@ -295,7 +307,7 @@ mod linux_impl { _ = sleep(interval) => {} } - let Some(lock) = PROF_CTL.as_ref() else { + let Some(lock) = jemalloc_pprof::PROF_CTL.as_ref() else { debug!("skip memory dump: PROF_CTL not available"); continue; }; @@ -305,28 +317,32 @@ mod linux_impl { debug!("skip memory dump: jemalloc profiling not active"); continue; } - let out = output_dir().join(format!("mem_profile_periodic_{}.pb", ts())); match File::create(&out) { Err(e) => { - error!("periodic mem dump create file failed: {}", e); + tracing::error!("periodic mem dump create file failed: {}", e); continue; } Ok(mut f) => match ctl.dump_pprof() { Ok(bytes) => { if let Err(e) = f.write_all(&bytes) { - error!("periodic mem dump write failed: {}", e); + tracing::error!("periodic mem dump write failed: {}", e); } else { info!("periodic memory profile dumped to {}", out.display()); } } - Err(e) => error!("periodic mem dump failed: {}", e), + Err(e) => tracing::error!("periodic mem dump failed: {}", e), }, } } }); } + #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] + async fn start_memory_periodic(_interval: Duration, _token: CancellationToken) { + debug!("periodic memory profiling skipped on unsupported target"); + } + // Public: unified init entry, avoid duplication/conflict pub async fn init_from_env() { let enabled = get_env_bool(ENV_ENABLE_PROFILING, DEFAULT_ENABLE_PROFILING); @@ -361,6 +377,16 @@ mod linux_impl { } } + #[cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))] + fn memory_profiling_unsupported_message() -> String { + let target_env = option_env!("CARGO_CFG_TARGET_ENV").unwrap_or("unknown"); + format!( + "Memory profiling is only supported on linux x86_64 gnu. target_os={}, target_env={target_env}, target_arch={}", + std::env::consts::OS, + std::env::consts::ARCH + ) + } + /// Stop all background profiling tasks pub fn shutdown_profiling() { if let Some(token) = PROFILING_CANCEL_TOKEN.get() {