feat: replace jemalloc with mimalloc (#4174)

* feat: replace jemalloc with mimalloc

* docs: record allocator rounds5 retest

* fix(replication): satisfy clippy unwrap lints

* docs: keep allocator migration plan local only

* feat(profiling): rely on pyroscope cpu profiling

* refactor(profiling): centralize unsupported pprof responses

* chore(deps): update s3s revision

crates/config/src/constants/profiler.rs

@@ -15,7 +15,8 @@
 /// Profiler related environment variable names and default values
 pub const ENV_ENABLE_PROFILING: &str = "RUSTFS_ENABLE_PROFILING";
 
-// CPU profiling
+// Legacy local CPU pprof controls. Local pprof dumps are currently unsupported;
+// Pyroscope export is configured through observability settings.
 pub const ENV_CPU_MODE: &str = "RUSTFS_PROF_CPU_MODE"; // off|continuous|periodic
 /// Frequency of CPU profiling samples
 pub const ENV_CPU_FREQ: &str = "RUSTFS_PROF_CPU_FREQ";
@@ -24,7 +25,7 @@ pub const ENV_CPU_INTERVAL_SECS: &str = "RUSTFS_PROF_CPU_INTERVAL_SECS";
 /// Duration of each CPU profiling session (for periodic mode)
 pub const ENV_CPU_DURATION_SECS: &str = "RUSTFS_PROF_CPU_DURATION_SECS";
 
-/// Memory profiling (jemalloc)
+/// Periodic memory pprof dumps (currently unsupported with mimalloc)
 pub const ENV_MEM_PERIODIC: &str = "RUSTFS_PROF_MEM_PERIODIC";
 /// Interval between memory profiling snapshots (for periodic mode)
 pub const ENV_MEM_INTERVAL_SECS: &str = "RUSTFS_PROF_MEM_INTERVAL_SECS";
@@ -34,7 +35,7 @@ pub const ENV_OUTPUT_DIR: &str = "RUSTFS_PROF_OUTPUT_DIR";
 
 /// Defaults for profiler settings
 pub const DEFAULT_ENABLE_PROFILING: bool = false;
-/// CPU profiling
+/// Legacy local CPU pprof mode
 pub const DEFAULT_CPU_MODE: &str = "off";
 /// Frequency of CPU profiling samples
 pub const DEFAULT_CPU_FREQ: usize = 100;
@@ -42,7 +43,7 @@ pub const DEFAULT_CPU_FREQ: usize = 100;
 pub const DEFAULT_CPU_INTERVAL_SECS: u64 = 300;
 /// Duration of each CPU profiling session (for periodic mode)
 pub const DEFAULT_CPU_DURATION_SECS: u64 = 60;
-/// Memory profiling (jemalloc)
+/// Periodic memory pprof dumps (currently unsupported with mimalloc)
 pub const DEFAULT_MEM_PERIODIC: bool = false;
 /// Interval between memory profiling snapshots (for periodic mode)
 pub const DEFAULT_MEM_INTERVAL_SECS: u64 = 300;

crates/extension-schema/src/lib.rs

@@ -782,7 +782,7 @@ mod tests {
         let contract = OpsProfilerContract {
             mode: OpsProfilerContractMode::CapabilityDescription,
             backends: vec![
-                profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true),
+                profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true),
                 profiler_backend("memory_pprof", OpsProfilerBackendStatus::Disabled, false),
                 profiler_backend("ebpf", OpsProfilerBackendStatus::Unsupported, false),
                 profiler_backend("future_kernel_profiler", OpsProfilerBackendStatus::Unknown, false),
@@ -803,7 +803,7 @@ mod tests {
     fn ops_profiler_schema_serializes_stable_json_shape() {
         let contract = OpsProfilerContract {
             mode: OpsProfilerContractMode::CapabilityDescription,
-            backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+            backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
         };
 
         let value = serde_json::to_value(contract).expect("ops profiler schema should serialize");
@@ -813,7 +813,7 @@ mod tests {
             json!({
                 "mode": "capability_description",
                 "backends": [{
-                    "backend": "cpu_pprof",
+                    "backend": "pyroscope",
                     "status": "enabled",
                     "supports_profile_export": true,
                     "redaction_required": ["secret", "token", "local_path", "host"],
@@ -860,7 +860,7 @@ mod tests {
             contract: OpsProfilerContract {
                 mode: OpsProfilerContractMode::CapabilityDescription,
                 backends: vec![
-                    profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true),
+                    profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true),
                     profiler_backend("memory_pprof", OpsProfilerBackendStatus::Disabled, false),
                     profiler_backend("ebpf", OpsProfilerBackendStatus::Unsupported, false),
                 ],
@@ -898,7 +898,7 @@ mod tests {
             },
             contract: OpsProfilerContract {
                 mode: OpsProfilerContractMode::CapabilityDescription,
-                backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+                backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
             },
         };
 
@@ -916,7 +916,7 @@ mod tests {
                 "contract": {
                     "mode": "capability_description",
                     "backends": [{
-                        "backend": "cpu_pprof",
+                        "backend": "pyroscope",
                         "status": "enabled",
                         "supports_profile_export": true,
                         "redaction_required": ["secret", "token", "local_path", "host"],
@@ -942,7 +942,7 @@ mod tests {
             },
             contract: OpsProfilerContract {
                 mode: OpsProfilerContractMode::CapabilityDescription,
-                backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+                backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
             },
         };
 
@@ -976,7 +976,7 @@ mod tests {
     fn rejects_ops_profiler_execution_requests_and_empty_backends() {
         let mut contract = OpsProfilerContract {
             mode: OpsProfilerContractMode::ExecutionRequest,
-            backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+            backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
         };
 
         assert_eq!(
@@ -998,7 +998,7 @@ mod tests {
         let err = serde_json::from_value::<OpsProfilerContract>(json!({
             "mode": "capability_description",
             "backends": [{
-                "backend": "cpu_pprof",
+                "backend": "pyroscope",
                 "status": "enabled",
                 "supports_profile_export": true,
                 "redaction_required": ["local_path"]
@@ -1014,7 +1014,7 @@ mod tests {
         let err = serde_json::from_value::<OpsProfilerContract>(json!({
             "mode": "capability_description",
             "backends": [{
-                "backend": "cpu_pprof",
+                "backend": "pyroscope",
                 "status": "enabled",
                 "supports_profile_export": true,
                 "redaction_required": ["local_path"],
@@ -1035,7 +1035,7 @@ mod tests {
     fn rejects_ops_profiler_invalid_backend_and_redaction_shape() {
         let mut contract = OpsProfilerContract {
             mode: OpsProfilerContractMode::CapabilityDescription,
-            backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+            backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
         };
         contract.backends[0].backend = OpsProfilerBackendName::new(" ");
 
@@ -1044,15 +1044,15 @@ mod tests {
             ExtensionContractError::EmptyOpsProfilerBackend
         );
 
-        contract.backends[0] = profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true);
+        contract.backends[0] = profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true);
         contract
             .backends
-            .push(profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Disabled, false));
+            .push(profiler_backend("pyroscope", OpsProfilerBackendStatus::Disabled, false));
 
         assert_eq!(
             validate_ops_profiler_contract(&contract).expect_err("duplicate backends should fail validation"),
             ExtensionContractError::DuplicateOpsProfilerBackend {
-                backend: "cpu_pprof".to_string()
+                backend: "pyroscope".to_string()
             }
         );
 
@@ -1062,7 +1062,7 @@ mod tests {
         assert_eq!(
             validate_ops_profiler_contract(&contract).expect_err("duplicate redaction fields should fail validation"),
             ExtensionContractError::DuplicateOpsProfilerRedactionField {
-                backend: "cpu_pprof".to_string(),
+                backend: "pyroscope".to_string(),
                 field: OpsProfilerRedactionField::Host
             }
         );
@@ -1072,7 +1072,7 @@ mod tests {
         assert_eq!(
             validate_ops_profiler_contract(&contract).expect_err("profile export requires local path redaction"),
             ExtensionContractError::OpsProfilerMissingLocalPathRedaction {
-                backend: "cpu_pprof".to_string()
+                backend: "pyroscope".to_string()
             }
         );
     }
@@ -1081,14 +1081,14 @@ mod tests {
     fn rejects_ops_profiler_missing_provenance_boundary() {
         let mut contract = OpsProfilerContract {
             mode: OpsProfilerContractMode::CapabilityDescription,
-            backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
+            backends: vec![profiler_backend("pyroscope", OpsProfilerBackendStatus::Enabled, true)],
         };
         contract.backends[0].provenance.source = " ".to_string();
 
         assert_eq!(
             validate_ops_profiler_contract(&contract).expect_err("provenance source should be required"),
             ExtensionContractError::EmptyOpsProfilerProvenanceSource {
-                backend: "cpu_pprof".to_string()
+                backend: "pyroscope".to_string()
             }
         );
 
@@ -1098,7 +1098,7 @@ mod tests {
         assert_eq!(
             validate_ops_profiler_contract(&contract).expect_err("collection boundary should be required"),
             ExtensionContractError::EmptyOpsProfilerCollectionBoundary {
-                backend: "cpu_pprof".to_string()
+                backend: "pyroscope".to_string()
             }
         );
     }

crates/obs/Cargo.toml

@@ -28,7 +28,7 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
 [features]
 default = []
 gpu = ["dep:nvml-wrapper"]
-pyroscope = ["dep:jemalloc_pprof", "dep:pyroscope"]
+pyroscope = ["dep:pyroscope"]
 
 [lints]
 workspace = true
@@ -83,7 +83,6 @@ nvml-wrapper = { workspace = true, optional = true }
 
 [target.'cfg(any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
 pyroscope = { workspace = true, features = ["backend-pprof-rs"], optional = true }
-jemalloc_pprof = { workspace = true, optional = true }
 
 
 [dev-dependencies]

crates/obs/src/telemetry/guard.rs

@@ -39,10 +39,6 @@ pub(crate) type ProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope:
     any(target_os = "macos", all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
 )))]
 pub(crate) type ProfilingAgent = ();
-#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-pub(crate) type MemoryProfilingAgent = pyroscope::PyroscopeAgent<pyroscope::pyroscope::PyroscopeAgentRunning>;
-#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
-pub(crate) type MemoryProfilingAgent = ();
 
 const LOG_COMPONENT_OBS: &str = "obs";
 const LOG_SUBSYSTEM_GUARD: &str = "guard";
@@ -67,7 +63,6 @@ pub struct OtelGuard {
     /// Optional logger provider for OTLP log export.
     pub(crate) logger_provider: Option<SdkLoggerProvider>,
     pub(crate) profiling_agent: Option<ProfilingAgent>,
-    pub(crate) memory_profiling_agent: Option<MemoryProfilingAgent>,
     /// Handle to the background log-cleanup task; aborted on drop.
     pub(crate) cleanup_handle: Option<tokio::task::JoinHandle<()>>,
     /// Worker guard that keeps the non-blocking `tracing_appender` thread
@@ -84,7 +79,6 @@ impl std::fmt::Debug for OtelGuard {
             .field("meter_provider", &self.meter_provider.is_some())
             .field("logger_provider", &self.logger_provider.is_some())
             .field("profiling_agent", &self.profiling_agent.is_some())
-            .field("memory_profiling_agent", &self.memory_profiling_agent.is_some())
             .field("cleanup_handle", &self.cleanup_handle.is_some())
             .field("tracing_guard", &self.tracing_guard.is_some())
             .field("stdout_guard", &self.stdout_guard.is_some())
@@ -161,30 +155,6 @@ impl Drop for OtelGuard {
             }
         }
 
-        #[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-        if let Some(agent) = self.memory_profiling_agent.take() {
-            match agent.stop() {
-                Err(err) => {
-                    if tracing::dispatcher::has_been_set() {
-                        error!(
-                            event = EVENT_OBS_GUARD_SHUTDOWN,
-                            component = LOG_COMPONENT_OBS,
-                            subsystem = LOG_SUBSYSTEM_GUARD,
-                            resource = "memory_profiling_agent",
-                            result = "shutdown_failed",
-                            error = %err,
-                            "observability guard shutdown failed"
-                        );
-                    } else {
-                        eprintln!("{}", format_guard_shutdown_stderr_message("memory_profiling_agent", err));
-                    }
-                }
-                Ok(stopped) => {
-                    stopped.shutdown();
-                }
-            }
-        }
-
         if let Some(handle) = self.cleanup_handle.take() {
             debug!(
                 event = EVENT_OBS_GUARD_SHUTDOWN,

crates/obs/src/telemetry/local.rs

@@ -271,7 +271,6 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
         meter_provider: None,
         logger_provider: None,
         profiling_agent: None,
-        memory_profiling_agent: None,
         tracing_guard: Some(guard),
         stdout_guard: None,
         cleanup_handle: None,
@@ -389,7 +388,6 @@ fn init_file_logging_internal(
         meter_provider: None,
         logger_provider: None,
         profiling_agent: None,
-        memory_profiling_agent: None,
         tracing_guard: Some(guard),
         stdout_guard,
         cleanup_handle: Some(cleanup_handle),

crates/obs/src/telemetry/otel.rs

@@ -40,7 +40,7 @@ use crate::cleaner::types::FileMatchMode;
 use crate::config::OtelConfig;
 use crate::global::set_observability_metric_enabled;
 use crate::telemetry::filter::build_env_filter;
-use crate::telemetry::guard::{MemoryProfilingAgent, OtelGuard, ProfilingAgent};
+use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
 use crate::telemetry::local::{build_json_log_layer, spawn_cleanup_task};
 use crate::telemetry::recorder::Recorder;
 use crate::telemetry::resource::build_resource;
@@ -173,7 +173,6 @@ pub(super) fn init_observability_http(
     let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?;
 
     let profiling_agent = init_profiler(config);
-    let memory_profiling_agent = init_memory_profiler(config);
 
     // ── Logger Logic ──────────────────────────────────────────────────────────
     // Logging is the only signal that may intentionally route to either OTLP
@@ -317,7 +316,6 @@ pub(super) fn init_observability_http(
         meter_provider,
         logger_provider,
         profiling_agent,
-        memory_profiling_agent,
         tracing_guard,
         stdout_guard,
         cleanup_handle,
@@ -563,65 +561,6 @@ fn init_profiler(_config: &OtelConfig) -> Option<ProfilingAgent> {
     None
 }
 
-/// Initialise a Pyroscope agent for continuous **memory** profiling via jemalloc.
-///
-/// This is only available on `linux + gnu + x86_64` where tikv-jemallocator
-/// is the global allocator and `jemalloc_pprof::PROF_CTL` is accessible.
-///
-/// Returns `None` when profiling export is disabled, the endpoint is missing,
-/// jemalloc profiling is not activated, or the agent fails to build/start.
-#[cfg(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
-fn init_memory_profiler(config: &OtelConfig) -> Option<MemoryProfilingAgent> {
-    use pyroscope::backend::jemalloc_backend;
-    use pyroscope::pyroscope::PyroscopeAgentBuilder;
-    use rustfs_config::VERSION;
-
-    if !config
-        .profiling_export_enabled
-        .unwrap_or(rustfs_config::DEFAULT_OBS_PROFILING_EXPORT_ENABLED)
-    {
-        return None;
-    }
-
-    let endpoint = config.profiling_endpoint.as_ref()?.as_str();
-    if endpoint.is_empty() {
-        return None;
-    }
-
-    // Verify jemalloc profiling is available and activated
-    {
-        let prof_ctl = jemalloc_pprof::PROF_CTL.as_ref()?;
-        let ctl = prof_ctl.try_lock().ok()?;
-        if !ctl.activated() {
-            eprintln!("Memory profiling skipped: jemalloc profiling is not activated");
-            return None;
-        }
-    }
-
-    let backend = jemalloc_backend();
-    let service_name = config.service_name.as_deref().unwrap_or(APP_NAME);
-    let version = config.service_version.as_deref().unwrap_or(VERSION);
-    let sample_rate = 100;
-
-    let agent = PyroscopeAgentBuilder::new(endpoint, service_name, sample_rate, "pyroscope-rs", "1.0.1", backend)
-        .tags(vec![("version", version), ("profile_type", "memory")])
-        .build()
-        .ok()?;
-
-    match agent.start() {
-        Ok(agent) => Some(agent),
-        Err(err) => {
-            eprintln!("Memory profiling agent start error: {err:?}");
-            None
-        }
-    }
-}
-
-#[cfg(not(all(feature = "pyroscope", target_os = "linux", target_env = "gnu", target_arch = "x86_64")))]
-fn init_memory_profiler(_config: &OtelConfig) -> Option<MemoryProfilingAgent> {
-    None
-}
-
 /// Create a stdout periodic metrics reader for the given interval.
 ///
 /// This helper is primarily used for local development and diagnostics when

crates/targets/src/catalog/extension.rs

@@ -123,8 +123,8 @@ pub fn builtin_ops_profiler_contract() -> OpsProfilerContract {
     OpsProfilerContract {
         mode: OpsProfilerContractMode::CapabilityDescription,
         backends: vec![
-            builtin_ops_profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true),
-            builtin_ops_profiler_backend("memory_pprof", OpsProfilerBackendStatus::Disabled, true),
+            builtin_ops_profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Unsupported, false),
+            builtin_ops_profiler_backend("memory_pprof", OpsProfilerBackendStatus::Unsupported, false),
             builtin_ops_profiler_backend("ebpf", OpsProfilerBackendStatus::Unsupported, false),
         ],
     }
@@ -391,8 +391,8 @@ mod tests {
         assert_eq!(
             backends,
             vec![
-                ("cpu_pprof", OpsProfilerBackendStatus::Enabled, true),
-                ("memory_pprof", OpsProfilerBackendStatus::Disabled, true),
+                ("cpu_pprof", OpsProfilerBackendStatus::Unsupported, false),
+                ("memory_pprof", OpsProfilerBackendStatus::Unsupported, false),
                 ("ebpf", OpsProfilerBackendStatus::Unsupported, false),
             ]
         );

crates/targets/src/runtime/ops_profiler.rs

@@ -172,13 +172,13 @@ mod tests {
         assert_eq!(registry.registered_backend_count(), contract.backends.len());
         assert_eq!(
             registry.registration_for("cpu_pprof").map(|registration| registration.status),
-            Some(OpsProfilerBackendStatus::Enabled)
+            Some(OpsProfilerBackendStatus::Unsupported)
         );
         assert_eq!(
             registry
                 .registration_for("memory_pprof")
                 .map(|registration| registration.supports_profile_export),
-            Some(true)
+            Some(false)
         );
         assert_eq!(
             registry.authorize_read(OpsProfilerReadRequest {