feat(rpc): expose and auto-size replay cache capacity (#5781)

* feat(metrics): expose replay cache pressure

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(rpc): auto-size replay cache capacity

Co-Authored-By: heihutu <heihutu@gmail.com>

* feat(cache): split runtime memory feature

Co-Authored-By: heihutu <heihutu@gmail.com>

---------

Co-authored-by: heihutu <heihutu@gmail.com>

crates/object-data-cache/Cargo.toml

@@ -28,21 +28,33 @@ categories = ["web-programming", "development-tools"]
 doctest = false
 
 [features]
-default = []
-hotpath = ["hotpath/hotpath", "hotpath/tokio"]
+default = ["cache"]
+runtime-memory = ["dep:sysinfo"]
+cache = [
+    "runtime-memory",
+    "dep:bytes",
+    "dep:metrics",
+    "dep:moka",
+    "dep:starshard",
+    "dep:thiserror",
+    "dep:tokio",
+    "dep:tracing",
+    "sysinfo/multithread",
+]
+hotpath = ["cache", "hotpath/hotpath", "hotpath/tokio"]
 hotpath-alloc = ["hotpath", "hotpath/hotpath-alloc"]
 hotpath-cpu = ["hotpath", "hotpath/hotpath-cpu"]
 
 [dependencies]
 hotpath.workspace = true
-bytes = { workspace = true, features = ["serde"] }
-metrics = { workspace = true }
-moka = { workspace = true, features = ["future"] }
-starshard = { workspace = true, features = ["rayon", "async", "serde"] }
-sysinfo = { workspace = true, features = ["multithread"] }
-thiserror.workspace = true
-tokio = { workspace = true, features = ["sync", "time", "fs", "rt-multi-thread"] }
-tracing.workspace = true
+bytes = { workspace = true, optional = true, features = ["serde"] }
+metrics = { workspace = true, optional = true }
+moka = { workspace = true, optional = true, features = ["future"] }
+starshard = { workspace = true, optional = true, features = ["rayon", "async", "serde"] }
+sysinfo = { workspace = true, optional = true }
+thiserror = { workspace = true, optional = true }
+tokio = { workspace = true, optional = true, features = ["sync", "time", "fs", "rt-multi-thread"] }
+tracing = { workspace = true, optional = true }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["html_reports"] }

crates/object-data-cache/src/config.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use crate::error::ObjectDataCacheConfigError;
-use crate::memory::{EffectiveMemory, MemoryBasis, resolve_effective_memory};
+use crate::{EffectiveMemory, MemoryBasis, resolve_effective_memory};
 use std::sync::Once;
 use std::time::Duration;
 
@@ -296,7 +296,7 @@ mod tests {
         clamp_derived_max_bytes,
     };
     use crate::error::ObjectDataCacheConfigError;
-    use crate::memory::{EffectiveMemory, MemoryBasis};
+    use crate::{EffectiveMemory, MemoryBasis};
     use std::time::Duration;
 
     #[test]

crates/object-data-cache/src/lib.rs

@@ -41,27 +41,52 @@
 //! the cache disabled for those buckets. Adding timing noise is not a viable
 //! mitigation: it would cost exactly the latency the cache exists to save.
 
+#[cfg(feature = "cache")]
 pub mod backend;
+#[cfg(feature = "cache")]
 pub mod cache;
+#[cfg(feature = "cache")]
 pub mod config;
+#[cfg(feature = "cache")]
 pub mod entry;
+#[cfg(feature = "cache")]
 pub mod error;
+#[cfg(feature = "cache")]
 pub mod index;
+#[cfg(feature = "cache")]
 pub mod key;
+#[cfg(feature = "cache")]
 pub mod memory;
+#[cfg(feature = "cache")]
 pub mod metrics;
+#[cfg(feature = "cache")]
 pub mod moka_backend;
+#[cfg(feature = "cache")]
 pub mod noop;
+#[cfg(feature = "runtime-memory")]
+mod runtime_memory;
+#[cfg(feature = "cache")]
 pub mod singleflight;
+#[cfg(feature = "cache")]
 pub mod starshard_index;
+#[cfg(feature = "cache")]
 pub mod stats;
 
+#[cfg(feature = "cache")]
 pub use cache::{
     ObjectDataCache, ObjectDataCacheBodyReservation, ObjectDataCacheFillResult, ObjectDataCacheGetPlan,
     ObjectDataCacheGetRequest, ObjectDataCacheInvalidationReason, ObjectDataCacheInvalidationResult, ObjectDataCacheLookup,
     ObjectDataCacheReservedBody,
 };
+#[cfg(feature = "cache")]
 pub use config::{ObjectDataCacheConfig, ObjectDataCacheMode};
+#[cfg(feature = "cache")]
 pub use error::ObjectDataCacheConfigError;
+#[cfg(feature = "cache")]
 pub use key::{NULL_VERSION_ID, ObjectDataCacheBodyVariant, ObjectDataCacheIdentity, ObjectDataCacheKey};
+#[cfg(feature = "runtime-memory")]
+pub use runtime_memory::{
+    EffectiveMemory, MemoryBasis, effective_memory_from_system, resolve_effective_memory, select_effective_memory,
+};
+#[cfg(feature = "cache")]
 pub use stats::{ObjectDataCacheStats, ObjectDataCacheStatsSnapshot};

crates/object-data-cache/src/memory.rs

@@ -14,6 +14,7 @@
 
 use crate::config::ObjectDataCacheConfig;
 use crate::metrics::record_memory_pressure;
+use crate::runtime_memory::resolve_effective_memory;
 use crate::stats::ObjectDataCacheStats;
 use bytes::Bytes;
 use std::hint::spin_loop;
@@ -22,7 +23,6 @@ use std::sync::Arc;
 use std::sync::Mutex;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::time::{Duration, Instant};
-use sysinfo::System;
 
 const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5);
 const DEFAULT_TELEMETRY_STALENESS: Duration = Duration::from_secs(15);
@@ -31,73 +31,6 @@ const DEFAULT_TELEMETRY_STALENESS: Duration = Duration::from_secs(15);
 // exhaustion safely skips only the cache fill.
 const MAX_STATE_RETRIES: usize = 8;
 
-/// Source used to resolve the effective memory limits.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) enum MemoryBasis {
-    /// Limits come from host memory reported by sysinfo.
-    Host,
-    /// Limits come from a constraining cgroup (container) memory limit.
-    Cgroup,
-}
-
-impl MemoryBasis {
-    pub(crate) const fn as_str(self) -> &'static str {
-        match self {
-            Self::Host => "host",
-            Self::Cgroup => "cgroup",
-        }
-    }
-}
-
-/// Effective memory totals after reconciling host memory with cgroup limits.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) struct EffectiveMemory {
-    /// Effective total memory in bytes.
-    pub(crate) total_bytes: u64,
-    /// Effective available memory in bytes.
-    pub(crate) available_bytes: u64,
-    /// Whether the limits are host- or cgroup-derived.
-    pub(crate) basis: MemoryBasis,
-}
-
-/// Reconciles host memory with an optional cgroup limit.
-///
-/// `cgroup` carries `(total_memory, free_memory)` as reported for the cgroup
-/// hierarchy (sysinfo already caps `total_memory` at the host total). A cgroup
-/// only counts when it actually constrains below the host, so an unlimited
-/// cgroup transparently falls back to host values.
-pub(crate) fn select_effective_memory(host_total: u64, host_available: u64, cgroup: Option<(u64, u64)>) -> EffectiveMemory {
-    match cgroup {
-        Some((cgroup_total, cgroup_free)) if cgroup_total > 0 && cgroup_total < host_total => EffectiveMemory {
-            total_bytes: cgroup_total,
-            available_bytes: cgroup_free.min(cgroup_total),
-            basis: MemoryBasis::Cgroup,
-        },
-        _ => EffectiveMemory {
-            total_bytes: host_total,
-            available_bytes: host_available,
-            basis: MemoryBasis::Host,
-        },
-    }
-}
-
-/// Resolves the effective memory from an already-refreshed system handle.
-///
-/// `cgroup_limits()` is computed fresh on each call and is only implemented on
-/// Linux (it returns `None` elsewhere), so non-Linux hosts always use host
-/// values.
-pub(crate) fn effective_memory_from_system(system: &System) -> EffectiveMemory {
-    let cgroup = system.cgroup_limits().map(|limits| (limits.total_memory, limits.free_memory));
-    select_effective_memory(system.total_memory(), system.available_memory(), cgroup)
-}
-
-/// Resolves the effective memory using a fresh, memory-refreshed system handle.
-pub(crate) fn resolve_effective_memory() -> EffectiveMemory {
-    let mut system = System::new();
-    system.refresh_memory();
-    effective_memory_from_system(&system)
-}
-
 /// Immutable memory snapshot used by the cache fill gate.
 #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
 pub struct ObjectDataCacheMemorySnapshot {
@@ -797,10 +730,7 @@ fn lock_or_recover<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
 
 #[cfg(test)]
 mod tests {
-    use super::{
-        DEFAULT_TELEMETRY_STALENESS, MemoryBasis, ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot,
-        select_effective_memory,
-    };
+    use super::{DEFAULT_TELEMETRY_STALENESS, ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot};
     use crate::config::ObjectDataCacheConfig;
     use crate::stats::ObjectDataCacheStats;
     use bytes::Bytes;
@@ -1240,42 +1170,6 @@ mod tests {
         assert_eq!(gate.claimed_bytes_for_test(), 0);
     }
 
-    #[test]
-    fn select_effective_memory_prefers_constraining_cgroup() {
-        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB)));
-
-        assert_eq!(effective.basis, MemoryBasis::Cgroup);
-        assert_eq!(effective.total_bytes, 2 * GIB);
-        assert_eq!(effective.available_bytes, GIB);
-    }
-
-    #[test]
-    fn select_effective_memory_ignores_non_constraining_cgroup() {
-        // A cgroup total equal to (or above) the host total means no real limit.
-        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((64 * GIB, 10 * GIB)));
-
-        assert_eq!(effective.basis, MemoryBasis::Host);
-        assert_eq!(effective.total_bytes, 64 * GIB);
-        assert_eq!(effective.available_bytes, 40 * GIB);
-    }
-
-    #[test]
-    fn select_effective_memory_falls_back_to_host_without_cgroup() {
-        let effective = select_effective_memory(8 * GIB, 4 * GIB, None);
-
-        assert_eq!(effective.basis, MemoryBasis::Host);
-        assert_eq!(effective.total_bytes, 8 * GIB);
-        assert_eq!(effective.available_bytes, 4 * GIB);
-    }
-
-    #[test]
-    fn select_effective_memory_caps_available_at_total() {
-        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, 3 * GIB)));
-
-        assert_eq!(effective.total_bytes, 2 * GIB);
-        assert_eq!(effective.available_bytes, 2 * GIB);
-    }
-
     #[test]
     fn gate_pauses_fill_when_container_memory_is_low() {
         // Simulate a pod-sized snapshot (256 MiB total, 16 MiB free): below the

crates/object-data-cache/src/runtime_memory.rs

@@ -0,0 +1,124 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sysinfo::System;
+
+/// Source used to resolve the effective memory limits.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MemoryBasis {
+    /// Limits come from host memory reported by sysinfo.
+    Host,
+    /// Limits come from a constraining cgroup (container) memory limit.
+    Cgroup,
+}
+
+impl MemoryBasis {
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::Host => "host",
+            Self::Cgroup => "cgroup",
+        }
+    }
+}
+
+/// Effective memory totals after reconciling host memory with cgroup limits.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct EffectiveMemory {
+    /// Effective total memory in bytes.
+    pub total_bytes: u64,
+    /// Effective available memory in bytes.
+    pub available_bytes: u64,
+    /// Whether the limits are host- or cgroup-derived.
+    pub basis: MemoryBasis,
+}
+
+/// Reconciles host memory with an optional cgroup limit.
+///
+/// `cgroup` carries `(total_memory, free_memory)` as reported for the cgroup
+/// hierarchy (sysinfo already caps `total_memory` at the host total). A cgroup
+/// only counts when it actually constrains below the host, so an unlimited
+/// cgroup transparently falls back to host values.
+pub fn select_effective_memory(host_total: u64, host_available: u64, cgroup: Option<(u64, u64)>) -> EffectiveMemory {
+    match cgroup {
+        Some((cgroup_total, cgroup_free)) if cgroup_total > 0 && cgroup_total < host_total => EffectiveMemory {
+            total_bytes: cgroup_total,
+            available_bytes: cgroup_free.min(cgroup_total),
+            basis: MemoryBasis::Cgroup,
+        },
+        _ => EffectiveMemory {
+            total_bytes: host_total,
+            available_bytes: host_available,
+            basis: MemoryBasis::Host,
+        },
+    }
+}
+
+/// Resolves the effective memory from an already-refreshed system handle.
+///
+/// `cgroup_limits()` is computed fresh on each call and is only implemented on
+/// Linux (it returns `None` elsewhere), so non-Linux hosts always use host
+/// values.
+pub fn effective_memory_from_system(system: &System) -> EffectiveMemory {
+    let cgroup = system.cgroup_limits().map(|limits| (limits.total_memory, limits.free_memory));
+    select_effective_memory(system.total_memory(), system.available_memory(), cgroup)
+}
+
+/// Resolves the effective memory using a fresh, memory-refreshed system handle.
+pub fn resolve_effective_memory() -> EffectiveMemory {
+    let mut system = System::new();
+    system.refresh_memory();
+    effective_memory_from_system(&system)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{MemoryBasis, select_effective_memory};
+
+    const GIB: u64 = 1024 * 1024 * 1024;
+
+    #[test]
+    fn select_effective_memory_prefers_constraining_cgroup() {
+        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB)));
+
+        assert_eq!(effective.basis, MemoryBasis::Cgroup);
+        assert_eq!(effective.total_bytes, 2 * GIB);
+        assert_eq!(effective.available_bytes, GIB);
+    }
+
+    #[test]
+    fn select_effective_memory_ignores_non_constraining_cgroup() {
+        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((64 * GIB, 10 * GIB)));
+
+        assert_eq!(effective.basis, MemoryBasis::Host);
+        assert_eq!(effective.total_bytes, 64 * GIB);
+        assert_eq!(effective.available_bytes, 40 * GIB);
+    }
+
+    #[test]
+    fn select_effective_memory_falls_back_to_host_without_cgroup() {
+        let effective = select_effective_memory(8 * GIB, 4 * GIB, None);
+
+        assert_eq!(effective.basis, MemoryBasis::Host);
+        assert_eq!(effective.total_bytes, 8 * GIB);
+        assert_eq!(effective.available_bytes, 4 * GIB);
+    }
+
+    #[test]
+    fn select_effective_memory_caps_available_at_total() {
+        let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, 3 * GIB)));
+
+        assert_eq!(effective.total_bytes, 2 * GIB);
+        assert_eq!(effective.available_bytes, 2 * GIB);
+    }
+}