perf(memory): add reclaim signals and cache controls (#2689)

rustfs/Cargo.toml

@@ -171,6 +171,7 @@ libsystemd.workspace = true
 
 [target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
 mimalloc = { workspace = true }
+libmimalloc-sys = { version = "0.1.47", features = ["extended"] }
 
 
 

rustfs/src/allocator_reclaim.rs

@@ -0,0 +1,243 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(unsafe_code)]
+
+use metrics::{counter, gauge, histogram};
+use std::time::Duration;
+use tokio_util::sync::CancellationToken;
+use tracing::{debug, warn};
+
+pub fn allocator_backend() -> &'static str {
+    #[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
+    {
+        "jemalloc"
+    }
+
+    #[cfg(all(
+        not(target_os = "windows"),
+        not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+    ))]
+    {
+        "mimalloc"
+    }
+
+    #[cfg(target_os = "windows")]
+    {
+        "mimalloc-windows"
+    }
+}
+
+fn active_requests() -> u64 {
+    crate::server::active_http_requests()
+}
+
+fn current_delete_tail_activity() -> u64 {
+    crate::delete_tail_activity::current_delete_tail_activity()
+}
+
+fn current_scanner_activity() -> u64 {
+    rustfs_scanner::current_scanner_activity()
+}
+
+fn current_heal_activity() -> u64 {
+    rustfs_heal::current_heal_active_tasks() + rustfs_heal::current_heal_queue_length()
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+struct ReclaimableWorkSnapshot {
+    active_requests: u64,
+    delete_tail_activity: u64,
+    scanner_activity: u64,
+    heal_activity: u64,
+    ec_inflight_bytes: u64,
+    get_buffered_bytes: u64,
+}
+
+impl ReclaimableWorkSnapshot {
+    fn active_signal_count(self) -> u64 {
+        u64::from(self.active_requests > 0)
+            + u64::from(self.delete_tail_activity > 0)
+            + u64::from(self.scanner_activity > 0)
+            + u64::from(self.heal_activity > 0)
+            + u64::from(self.ec_inflight_bytes > 0)
+            + u64::from(self.get_buffered_bytes > 0)
+    }
+}
+
+fn reclaimable_work_snapshot() -> ReclaimableWorkSnapshot {
+    ReclaimableWorkSnapshot {
+        active_requests: active_requests(),
+        delete_tail_activity: current_delete_tail_activity(),
+        scanner_activity: current_scanner_activity(),
+        heal_activity: current_heal_activity(),
+        ec_inflight_bytes: rustfs_io_metrics::current_ec_encode_inflight_bytes(),
+        get_buffered_bytes: rustfs_io_metrics::current_get_object_buffered_bytes(),
+    }
+}
+
+#[cfg(all(
+    not(target_os = "windows"),
+    not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))
+))]
+fn collect_allocator_memory(force: bool) -> Result<(), String> {
+    // SAFETY: `mi_collect` is provided by the active global allocator backend
+    // on this target family. It is explicitly intended to reclaim retained
+    // pages/segments and does not require additional invariants from the caller.
+    unsafe {
+        libmimalloc_sys::mi_collect(force);
+    }
+    Ok(())
+}
+
+#[cfg(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))]
+fn collect_allocator_memory(_force: bool) -> Result<(), String> {
+    #[cfg(not(target_os = "macos"))]
+    let _ = tikv_jemalloc_ctl::background_thread::write(true);
+    tikv_jemalloc_ctl::epoch::advance().map_err(|err| err.to_string())?;
+    Ok(())
+}
+
+#[cfg(target_os = "windows")]
+fn collect_allocator_memory(_force: bool) -> Result<(), String> {
+    Err("allocator reclaim is not supported on Windows".to_string())
+}
+
+fn run_allocator_reclaim(force: bool) {
+    let backend = allocator_backend();
+    let start = std::time::Instant::now();
+
+    match collect_allocator_memory(force) {
+        Ok(()) => {
+            counter!("rustfs_memory_allocator_reclaim_total", "backend" => backend.to_string(), "result" => "ok".to_string())
+                .increment(1);
+            histogram!(
+                "rustfs_memory_allocator_reclaim_duration_seconds",
+                "backend" => backend.to_string(),
+                "result" => "ok".to_string()
+            )
+            .record(start.elapsed().as_secs_f64());
+        }
+        Err(err) => {
+            counter!(
+                "rustfs_memory_allocator_reclaim_total",
+                "backend" => backend.to_string(),
+                "result" => "err".to_string()
+            )
+            .increment(1);
+            warn!(backend, force, error = %err, "allocator reclaim failed");
+        }
+    }
+}
+
+pub fn init_allocator_reclaim(ctx: CancellationToken) {
+    let backend = allocator_backend();
+    let enabled = rustfs_utils::get_env_bool(
+        rustfs_config::ENV_ALLOCATOR_RECLAIM_ENABLED,
+        rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_ENABLED,
+    );
+    gauge!("rustfs_memory_allocator_reclaim_enabled").set(if enabled { 1.0 } else { 0.0 });
+    counter!("rustfs_memory_allocator_backend_info", "backend" => backend.to_string()).increment(1);
+
+    if !enabled {
+        debug!("allocator reclaim loop disabled");
+        return;
+    }
+
+    let configured_force =
+        rustfs_utils::get_env_bool(rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE, rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_FORCE);
+    let force = if backend == "jemalloc" && configured_force {
+        warn!(
+            backend,
+            env = rustfs_config::ENV_ALLOCATOR_RECLAIM_FORCE,
+            "allocator reclaim force mode is not supported on jemalloc backend; ignoring configured force flag"
+        );
+        false
+    } else {
+        configured_force
+    };
+    let idle_intervals = rustfs_utils::get_env_u64(
+        rustfs_config::ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS,
+        rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS,
+    )
+    .max(1);
+    let interval = Duration::from_secs(
+        rustfs_utils::get_env_u64(
+            rustfs_config::ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS,
+            rustfs_config::DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS,
+        )
+        .max(1),
+    );
+
+    tokio::spawn(async move {
+        let mut ticker = tokio::time::interval(interval);
+        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+        let mut idle_streak = 0_u64;
+
+        loop {
+            tokio::select! {
+                _ = ctx.cancelled() => {
+                    debug!("allocator reclaim loop cancelled");
+                    break;
+                }
+                _ = ticker.tick() => {
+                    let snapshot = reclaimable_work_snapshot();
+                    let active_signal_count = snapshot.active_signal_count();
+                    gauge!("rustfs_memory_allocator_reclaim_active_requests").set(snapshot.active_requests as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_delete_tail_activity_current").set(snapshot.delete_tail_activity as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_scanner_activity_current").set(snapshot.scanner_activity as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_heal_activity_current").set(snapshot.heal_activity as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_ec_inflight_bytes_current").set(snapshot.ec_inflight_bytes as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_get_buffered_bytes_current").set(snapshot.get_buffered_bytes as f64);
+                    gauge!("rustfs_memory_allocator_reclaim_reclaimable_work_current").set(active_signal_count as f64);
+                    if active_signal_count == 0 {
+                        idle_streak = idle_streak.saturating_add(1);
+                        gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(idle_streak as f64);
+                    } else {
+                        idle_streak = 0;
+                        gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0);
+                    }
+
+                    if idle_streak >= idle_intervals {
+                        run_allocator_reclaim(force);
+                        idle_streak = 0;
+                        gauge!("rustfs_memory_allocator_reclaim_idle_streak").set(0.0);
+                    } else {
+                        let reason = if active_signal_count > 0 {
+                            "work_inflight"
+                        } else {
+                            "idle_window"
+                        };
+                        counter!("rustfs_memory_allocator_reclaim_skipped_total", "reason" => reason.to_string()).increment(1);
+                    }
+                }
+            }
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{allocator_backend, reclaimable_work_snapshot};
+
+    #[test]
+    fn allocator_backend_name_is_available() {
+        assert!(!allocator_backend().is_empty());
+    }
+
+    #[test]
+    fn reclaimable_work_snapshot_is_collectable() {
+        let _ = reclaimable_work_snapshot();
+    }
+}

rustfs/src/app/object_usecase.rs

@@ -16,6 +16,7 @@
 
 use crate::app::context::{AppContext, default_notify_interface, get_global_app_context};
 use crate::config::RustFSBufferConfig;
+use crate::delete_tail_activity::{DeleteTailActivityGuard, DeleteTailStage};
 use crate::error::ApiError;
 use crate::storage::access::{PostObjectRequestMarker, authorize_request, has_bypass_governance_header, req_info_mut};
 use crate::storage::concurrency::{
@@ -36,7 +37,7 @@ use bytes::Bytes;
 use datafusion::arrow::{
     csv::WriterBuilder as CsvWriterBuilder, json::WriterBuilder as JsonWriterBuilder, json::writer::JsonArray,
 };
-use futures::{StreamExt, stream};
+use futures::StreamExt;
 use http::{HeaderMap, HeaderValue, StatusCode};
 use md5::Context as Md5Context;
 use metrics::{counter, histogram};
@@ -118,7 +119,7 @@ use std::collections::HashMap;
 use std::ops::Add;
 use std::path::Path;
 use std::str::FromStr;
-use std::sync::{Arc, Mutex};
+use std::sync::{Arc, Mutex, OnceLock};
 use std::time::Duration;
 use time::{OffsetDateTime, format_description::well_known::Rfc3339};
 use tokio::io::{AsyncRead, ReadBuf};
@@ -213,6 +214,7 @@ async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &
     let Some(existing) = existing else {
         return;
     };
+    let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
 
     let je = if opts.delete_prefix {
         rustfs_ecstore::bucket::lifecycle::tier_sweeper::transitioned_force_delete_journal_entry(&existing.transitioned_object)
@@ -255,6 +257,38 @@ pin_project! {
     }
 }
 
+pin_project! {
+    struct MemoryTrackedBytesStream {
+        bytes: Bytes,
+        emitted: bool,
+        _guard: Option<rustfs_io_metrics::MemoryGaugeGuard>,
+    }
+}
+
+impl MemoryTrackedBytesStream {
+    fn new(bytes: Bytes, guard: Option<rustfs_io_metrics::MemoryGaugeGuard>) -> Self {
+        Self {
+            bytes,
+            emitted: false,
+            _guard: guard,
+        }
+    }
+}
+
+impl futures::Stream for MemoryTrackedBytesStream {
+    type Item = std::io::Result<Bytes>;
+
+    fn poll_next(self: std::pin::Pin<&mut Self>, _cx: &mut std::task::Context<'_>) -> std::task::Poll<Option<Self::Item>> {
+        let this = self.project();
+        if *this.emitted {
+            return std::task::Poll::Ready(None);
+        }
+
+        *this.emitted = true;
+        std::task::Poll::Ready(Some(Ok(this.bytes.clone())))
+    }
+}
+
 impl<R> ExtractArchiveEtagReader<R> {
     fn new(inner: R, etag: Arc<Mutex<Option<String>>>) -> Self {
         Self {
@@ -348,6 +382,16 @@ fn should_use_zero_copy(size: i64, headers: &HeaderMap) -> bool {
     true
 }
 
+fn object_seek_support_threshold() -> usize {
+    static OBJECT_SEEK_SUPPORT_THRESHOLD: OnceLock<usize> = OnceLock::new();
+    *OBJECT_SEEK_SUPPORT_THRESHOLD.get_or_init(|| {
+        rustfs_utils::get_env_usize(
+            rustfs_config::ENV_OBJECT_SEEK_SUPPORT_THRESHOLD,
+            rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD,
+        )
+    })
+}
+
 #[cfg(test)]
 mod deadlock_request_guard_tests {
     use super::DeadlockRequestGuard;
@@ -994,8 +1038,10 @@ impl DefaultObjectUsecase {
     }
 
     fn build_memory_blob(buf: Vec<u8>, response_content_length: i64, _optimal_buffer_size: usize) -> Option<StreamingBlob> {
+        let guard = rustfs_io_metrics::track_get_object_buffered_bytes(buf.len());
+        let bytes = Bytes::from(buf);
         Some(StreamingBlob::wrap(bytes_stream(
-            stream::once(async move { Ok::<Bytes, std::io::Error>(Bytes::from(buf)) }),
+            MemoryTrackedBytesStream::new(bytes, guard),
             response_content_length as usize,
         )))
     }
@@ -1211,11 +1257,9 @@ impl DefaultObjectUsecase {
 
         let info = reader.object_info;
 
-        use rustfs_io_metrics::{record_memory_copy_saved, record_zero_copy_read};
+        use rustfs_io_metrics::record_zero_copy_read;
         let read_duration = read_start.elapsed();
-        let estimated_saved = (info.size * 2) as usize;
         record_zero_copy_read(info.size as usize, read_duration.as_secs_f64() * 1000.0);
-        record_memory_copy_saved(estimated_saved);
 
         manager.record_disk_operation(info.size as u64, read_duration, true).await;
 
@@ -1483,7 +1527,7 @@ impl DefaultObjectUsecase {
         R: AsyncRead + Send + Sync + Unpin + 'static,
     {
         if encryption_applied {
-            let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD;
+            let seekable_object_size_threshold = object_seek_support_threshold();
             let should_buffer_encrypted_object = response_content_length > 0
                 && response_content_length <= seekable_object_size_threshold as i64
                 && part_number.is_none()
@@ -1514,7 +1558,7 @@ impl DefaultObjectUsecase {
             return Ok(Self::build_reader_blob(final_stream, response_content_length, optimal_buffer_size));
         }
 
-        let seekable_object_size_threshold = rustfs_config::DEFAULT_OBJECT_SEEK_SUPPORT_THRESHOLD;
+        let seekable_object_size_threshold = object_seek_support_threshold();
         let should_provide_seek_support = response_content_length > 0
             && response_content_length <= seekable_object_size_threshold as i64
             && part_number.is_none()
@@ -2114,9 +2158,9 @@ impl DefaultObjectUsecase {
 
         let request_id = req
             .extensions
-            .get::<crate::storage::request_context::RequestContext>()
+            .get::<request_context::RequestContext>()
             .map(|ctx| ctx.request_id.clone())
-            .unwrap_or_else(|| crate::storage::request_context::RequestContext::fallback().request_id);
+            .unwrap_or_else(|| request_context::RequestContext::fallback().request_id);
         let bootstrap = Self::init_get_object_bootstrap(&req.input.bucket, &req.input.key, &request_id)?;
         let timeout_config = bootstrap.timeout_config;
         let wrapper = bootstrap.wrapper;
@@ -3083,6 +3127,7 @@ impl DefaultObjectUsecase {
                 && (dobj.delete_marker_replication_status() == ReplicationStatusType::Pending
                     || dobj.version_purge_status() == VersionPurgeStatusType::Pending)
             {
+                let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
                 let mut dobj = dobj.clone();
                 if is_dir_object(dobj.object_name.as_str()) && dobj.version_id.is_none() {
                     dobj.version_id = Some(Uuid::nil());
@@ -3104,11 +3149,9 @@ impl DefaultObjectUsecase {
             .as_ref()
             .map(|context| context.notify())
             .unwrap_or_else(default_notify_interface);
-        let request_context = req
-            .extensions
-            .get::<crate::storage::request_context::RequestContext>()
-            .cloned();
+        let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
         spawn_background_with_context(request_context, async move {
+            let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify);
             for res in delete_results {
                 if let Some(dobj) = res.delete_object {
                     let event_name = if dobj.delete_marker {
@@ -3298,6 +3341,7 @@ impl DefaultObjectUsecase {
         let deleted_replication_info = existing_object_info
             .as_ref()
             .filter(|_| should_use_existing_delete_replication_info(&opts));
+        let _delete_tail_guard = DeleteTailActivityGuard::new(DeleteTailStage::Tail);
         let deleted_object_source = deleted_replication_info.unwrap_or(&obj_info);
         let replication_state_source =
             delete_replication_state_source(&opts, existing_object_info.as_ref(), deleted_object_source);
@@ -3311,6 +3355,7 @@ impl DefaultObjectUsecase {
         };
 
         if schedule_delete_replication {
+            let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Replication);
             let mut deleted_object = DeletedObjectReplicationInfo {
                 delete_object: rustfs_ecstore::store_api::DeletedObject {
                     delete_marker: deleted_object_source.delete_marker && !deleted_delete_marker_version,
@@ -4321,10 +4366,7 @@ impl DefaultObjectUsecase {
             };
 
             let notify = notify.clone();
-            let request_context = req
-                .extensions
-                .get::<crate::storage::request_context::RequestContext>()
-                .cloned();
+            let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
             spawn_background_with_context(request_context, async move {
                 notify.notify(event_args).await;
             });

rustfs/src/delete_tail_activity.rs

@@ -0,0 +1,115 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use metrics::{counter, gauge, histogram};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+static DELETE_TAIL_TOTAL: AtomicU64 = AtomicU64::new(0);
+static DELETE_CLEANUP_TOTAL: AtomicU64 = AtomicU64::new(0);
+static DELETE_REPLICATION_TOTAL: AtomicU64 = AtomicU64::new(0);
+static DELETE_NOTIFY_TOTAL: AtomicU64 = AtomicU64::new(0);
+
+#[derive(Clone, Copy, Debug)]
+pub enum DeleteTailStage {
+    Tail,
+    Cleanup,
+    Replication,
+    Notify,
+}
+
+impl DeleteTailStage {
+    const fn as_str(self) -> &'static str {
+        match self {
+            Self::Tail => "tail",
+            Self::Cleanup => "cleanup",
+            Self::Replication => "replication",
+            Self::Notify => "notify",
+        }
+    }
+}
+
+fn stage_counter(stage: DeleteTailStage) -> &'static AtomicU64 {
+    match stage {
+        DeleteTailStage::Tail => &DELETE_TAIL_TOTAL,
+        DeleteTailStage::Cleanup => &DELETE_CLEANUP_TOTAL,
+        DeleteTailStage::Replication => &DELETE_REPLICATION_TOTAL,
+        DeleteTailStage::Notify => &DELETE_NOTIFY_TOTAL,
+    }
+}
+
+#[derive(Debug)]
+pub struct DeleteTailActivityGuard {
+    stage: DeleteTailStage,
+    started_at: std::time::Instant,
+}
+
+impl DeleteTailActivityGuard {
+    pub fn new(stage: DeleteTailStage) -> Self {
+        let total = stage_counter(stage).fetch_add(1, Ordering::Relaxed) + 1;
+        gauge!(
+            "rustfs_delete_tail_activity_inflight_current",
+            "stage" => stage.as_str().to_string()
+        )
+        .set(total as f64);
+        gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64);
+        counter!(
+            "rustfs_delete_tail_activity_started_total",
+            "stage" => stage.as_str().to_string()
+        )
+        .increment(1);
+        Self {
+            stage,
+            started_at: std::time::Instant::now(),
+        }
+    }
+}
+
+impl Drop for DeleteTailActivityGuard {
+    fn drop(&mut self) {
+        let previous = stage_counter(self.stage).fetch_sub(1, Ordering::Relaxed);
+        let next = previous.saturating_sub(1);
+        gauge!(
+            "rustfs_delete_tail_activity_inflight_current",
+            "stage" => self.stage.as_str().to_string()
+        )
+        .set(next as f64);
+        gauge!("rustfs_delete_tail_activity_total_inflight_current").set(current_delete_tail_activity() as f64);
+        histogram!(
+            "rustfs_delete_tail_activity_duration_seconds",
+            "stage" => self.stage.as_str().to_string()
+        )
+        .record(self.started_at.elapsed().as_secs_f64());
+    }
+}
+
+pub fn current_delete_tail_activity() -> u64 {
+    DELETE_TAIL_TOTAL.load(Ordering::Relaxed)
+        + DELETE_CLEANUP_TOTAL.load(Ordering::Relaxed)
+        + DELETE_REPLICATION_TOTAL.load(Ordering::Relaxed)
+        + DELETE_NOTIFY_TOTAL.load(Ordering::Relaxed)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{DeleteTailActivityGuard, DeleteTailStage, current_delete_tail_activity};
+
+    #[test]
+    fn delete_tail_activity_guard_tracks_total_activity() {
+        let before = current_delete_tail_activity();
+        let guard = DeleteTailActivityGuard::new(DeleteTailStage::Cleanup);
+        assert_eq!(current_delete_tail_activity(), before + 1);
+        drop(guard);
+        assert_eq!(current_delete_tail_activity(), before);
+    }
+}

rustfs/src/lib.rs

@@ -51,15 +51,18 @@
 //! tests, and then shut it down.
 
 pub mod admin;
+pub mod allocator_reclaim;
 pub mod app;
 pub mod auth;
 pub mod auth_keystone;
 pub mod capacity;
 pub mod config;
+pub mod delete_tail_activity;
 pub mod embedded;
 pub mod error;
 pub mod init;
 pub mod license;
+pub mod memory_observability;
 pub mod profiling;
 #[cfg(any(feature = "ftps", feature = "webdav"))]
 pub mod protocols;

rustfs/src/main.rs

@@ -560,10 +560,12 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
     print_server_info();
 
     init_update_check();
+    rustfs::allocator_reclaim::init_allocator_reclaim(ctx.clone());
 
     if rustfs_obs::observability_metric_enabled() {
         // Initialize metrics system
         init_metrics_runtime(ctx.clone());
+        rustfs::memory_observability::init_memory_observability(ctx.clone());
 
         // Initialize auto-tuner for performance optimization (optional)
         rustfs::init::init_auto_tuner(ctx.clone()).await;

rustfs/src/memory_observability.rs

@@ -0,0 +1,209 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_io_metrics::{
+    record_cgroup_memory_split, record_cpu_usage, record_memory_usage, record_process_memory_split,
+    snapshot_process_resource_and_system,
+};
+use std::collections::HashMap;
+use std::path::Path;
+use std::sync::{Mutex, OnceLock};
+use std::time::Duration;
+use sysinfo::System;
+use tokio_util::sync::CancellationToken;
+use tracing::debug;
+
+static MEMORY_SYSTEM: OnceLock<Mutex<System>> = OnceLock::new();
+
+const ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS: &str = "RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS";
+const DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS: u64 = 15;
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+struct CgroupMemorySnapshot {
+    current_bytes: Option<u64>,
+    limit_bytes: Option<u64>,
+    anon_bytes: Option<u64>,
+    file_bytes: Option<u64>,
+    active_file_bytes: Option<u64>,
+    inactive_file_bytes: Option<u64>,
+}
+
+fn memory_system() -> &'static Mutex<System> {
+    MEMORY_SYSTEM.get_or_init(|| Mutex::new(System::new()))
+}
+
+fn refresh_total_memory() -> u64 {
+    let mut system = memory_system().lock().unwrap_or_else(|poisoned| poisoned.into_inner());
+    system.refresh_memory();
+    system.total_memory()
+}
+
+fn read_optional_u64(path: &Path) -> Option<u64> {
+    let content = std::fs::read_to_string(path).ok()?;
+    let trimmed = content.trim();
+    if trimmed.is_empty() || trimmed == "max" {
+        return None;
+    }
+    trimmed.parse::<u64>().ok()
+}
+
+fn parse_kv_stats(content: &str) -> HashMap<String, u64> {
+    content
+        .lines()
+        .filter_map(|line| {
+            let mut parts = line.split_whitespace();
+            let key = parts.next()?;
+            let value = parts.next()?.parse::<u64>().ok()?;
+            Some((key.to_string(), value))
+        })
+        .collect()
+}
+
+fn read_cgroup_v2() -> Option<CgroupMemorySnapshot> {
+    let root = Path::new("/sys/fs/cgroup");
+    let stat_path = root.join("memory.stat");
+    if !stat_path.exists() {
+        return None;
+    }
+
+    let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?);
+    Some(CgroupMemorySnapshot {
+        current_bytes: read_optional_u64(&root.join("memory.current")),
+        limit_bytes: read_optional_u64(&root.join("memory.max")),
+        anon_bytes: stats.get("anon").copied(),
+        file_bytes: stats.get("file").copied(),
+        active_file_bytes: stats.get("active_file").copied(),
+        inactive_file_bytes: stats.get("inactive_file").copied(),
+    })
+}
+
+fn read_cgroup_v1() -> Option<CgroupMemorySnapshot> {
+    let root = Path::new("/sys/fs/cgroup/memory");
+    let stat_path = root.join("memory.stat");
+    if !stat_path.exists() {
+        return None;
+    }
+
+    let stats = parse_kv_stats(&std::fs::read_to_string(&stat_path).ok()?);
+    Some(CgroupMemorySnapshot {
+        current_bytes: read_optional_u64(&root.join("memory.usage_in_bytes")),
+        limit_bytes: read_optional_u64(&root.join("memory.limit_in_bytes")),
+        anon_bytes: stats.get("total_rss").copied().or_else(|| stats.get("rss").copied()),
+        file_bytes: stats.get("total_cache").copied().or_else(|| stats.get("cache").copied()),
+        active_file_bytes: stats
+            .get("total_active_file")
+            .copied()
+            .or_else(|| stats.get("active_file").copied()),
+        inactive_file_bytes: stats
+            .get("total_inactive_file")
+            .copied()
+            .or_else(|| stats.get("inactive_file").copied()),
+    })
+}
+
+fn read_cgroup_memory_snapshot() -> Option<CgroupMemorySnapshot> {
+    read_cgroup_v2().or_else(read_cgroup_v1)
+}
+
+async fn record_memory_snapshot() {
+    match tokio::task::spawn_blocking(|| {
+        let (resource, process) = snapshot_process_resource_and_system();
+        let total_memory = refresh_total_memory();
+        let cgroup = read_cgroup_memory_snapshot();
+        (resource, process, total_memory, cgroup)
+    })
+    .await
+    {
+        Ok((resource, process, total_memory, cgroup)) => {
+            record_memory_usage(process.resident_memory_bytes, total_memory);
+            record_cpu_usage(resource.cpu_percent);
+            record_process_memory_split(process.resident_memory_bytes, process.virtual_memory_bytes);
+
+            if let Some(cgroup) = cgroup {
+                record_cgroup_memory_split(
+                    cgroup.current_bytes,
+                    cgroup.limit_bytes,
+                    cgroup.anon_bytes,
+                    cgroup.file_bytes,
+                    cgroup.active_file_bytes,
+                    cgroup.inactive_file_bytes,
+                );
+            }
+        }
+        Err(err) => {
+            debug!(error = ?err, "memory observability sampler task failed");
+        }
+    }
+}
+
+pub fn init_memory_observability(ctx: CancellationToken) {
+    let interval_secs =
+        rustfs_utils::get_env_u64(ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS, DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS);
+    let interval = Duration::from_secs(interval_secs.max(1));
+
+    tokio::spawn(async move {
+        let mut ticker = tokio::time::interval(interval);
+        ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+
+        loop {
+            tokio::select! {
+                _ = ctx.cancelled() => {
+                    debug!("memory observability sampler cancelled");
+                    break;
+                }
+                _ = ticker.tick() => {
+                    record_memory_snapshot().await;
+                }
+            }
+        }
+    });
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{CgroupMemorySnapshot, parse_kv_stats, read_optional_u64};
+    use std::fs;
+    use std::path::PathBuf;
+
+    #[test]
+    fn parse_kv_stats_extracts_numeric_pairs() {
+        let parsed = parse_kv_stats("anon 12\nfile 34\nactive_file 56\n");
+        assert_eq!(parsed.get("anon").copied(), Some(12));
+        assert_eq!(parsed.get("file").copied(), Some(34));
+        assert_eq!(parsed.get("active_file").copied(), Some(56));
+    }
+
+    #[test]
+    fn read_optional_u64_parses_numeric_and_max_values() {
+        let tempdir = tempfile::tempdir().expect("tempdir");
+        let value_path: PathBuf = tempdir.path().join("value");
+        let max_path: PathBuf = tempdir.path().join("max");
+        fs::write(&value_path, "123\n").expect("write numeric");
+        fs::write(&max_path, "max\n").expect("write max");
+
+        assert_eq!(read_optional_u64(&value_path), Some(123));
+        assert_eq!(read_optional_u64(&max_path), None);
+    }
+
+    #[test]
+    fn cgroup_memory_snapshot_defaults_are_empty() {
+        let snapshot = CgroupMemorySnapshot::default();
+        assert_eq!(snapshot.current_bytes, None);
+        assert_eq!(snapshot.limit_bytes, None);
+        assert_eq!(snapshot.anon_bytes, None);
+        assert_eq!(snapshot.file_bytes, None);
+        assert_eq!(snapshot.active_file_bytes, None);
+        assert_eq!(snapshot.inactive_file_bytes, None);
+    }
+}

rustfs/src/server/http.rs

@@ -122,6 +122,10 @@ fn record_active_http_requests(delta: i64) {
     gauge!(METRIC_HTTP_SERVER_ACTIVE_REQUESTS).set(next as f64);
 }
 
+pub(crate) fn active_http_requests() -> u64 {
+    ACTIVE_HTTP_REQUESTS.load(Ordering::Relaxed)
+}
+
 pub async fn start_http_server(
     config: &config::Config,
     readiness: Arc<GlobalReadiness>,

rustfs/src/server/mod.rs

@@ -39,6 +39,7 @@ pub use service_state::ShutdownSignal;
 pub use service_state::wait_for_shutdown;
 
 // Items only used within the library crate (admin handlers, server/http.rs, etc.).
+pub(crate) use http::active_http_requests;
 pub(crate) use module_switch::{
     ModuleSwitchSnapshot, ModuleSwitchSource, PersistedModuleSwitches, current_module_switch_snapshot,
     refresh_persisted_module_switches_from_store, save_persisted_module_switches_to_store, validate_module_switch_update,