From 904554b41708b165d929529df6b84656cf9d010b Mon Sep 17 00:00:00 2001 From: houseme Date: Fri, 10 Jul 2026 17:17:32 +0800 Subject: [PATCH] fix(object-data-cache): make memory container-aware and bound cache config (#4655) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(object-data-cache): bound cache config and make memory container-aware Harden the object data cache configuration path so a single bad input degrades to the disabled adapter instead of OOM-killing pods, panicking at boot, or silently mis-sizing the cache (backlog#1110/1113/1114/1115/ 1127/1140/1130). - ODC-05: resolve capacity and the fill gate from the effective (container-aware) memory. Prefer sysinfo cgroup_limits() and use min(host, cgroup) as the total plus cgroup-derived availability, falling back to host values when no cgroup limit constrains. Log the resolved capacity and its basis once at startup. - ODC-08: cap ttl/time_to_idle at 30 days in validate() so an unbounded Duration can no longer trip moka's ~1000-year builder assertion. - ODC-09: drop the max_entry_bytes floor from the derived-capacity clamp so MAX_ENTRY_BYTES can no longer inflate total capacity above the safety clamp; reject an entry larger than the resolved capacity instead. - ODC-10: require an explicit max_bytes to clear max_entry_bytes plus the weigher overhead so a fillable-but-unretainable cache is rejected. - ODC-22: reject max_entry_bytes at/above the u32 weigher boundary. - ODC-35: warn (not reject) when time_to_idle exceeds ttl and document the min(ttl, time_to_idle) expiry interaction. - ODC-25: give numeric env overrides two-valued semantics via a new rustfs_utils::get_env_parse_outcome; a malformed value now disables the whole cache with one aggregated warning instead of silently keeping defaults. Co-Authored-By: heihutu * fix(object-data-cache): require identity_keys_max of at least 2 The identity index admits a new key by evicting the oldest one, so a budget of 1 evicts the previous key on every fill and can never hold two live keys of one object at once — a versioned bucket alternating two versions then hits a permanent 0% hit rate. Reject the degenerate value at config validation time, where the adapter turns it into a disabled cache with a warning, since the bounded-eviction policy cannot rescue it at runtime. Handed off from the identity-index batch (backlog#1128), which changed the overflow policy but could not touch validate(). Refs: backlog#1115, backlog#1128 Co-Authored-By: heihutu * fix(object-data-cache): let a zero free-memory floor opt out of the gate Making the memory gate container-aware turned it live in CI: the runners are Kubernetes pods, so the gate now reads the pod's cgroup free memory, finds it below the 20% floor, and refuses the fill. Tests that assert a fill succeeds then failed on CI while passing on a developer host, which has no cgroup and falls back to host memory. The gate is behaving correctly — the tests were the ones depending on a live memory reading. Treat min_free_memory_percent = 0 as a deliberate opt-out rather than an invalid value: allows_fill returns early before it touches any snapshot, so admission becomes independent of where the suite runs. Operators gain the same escape hatch. Every test that requires a fill to succeed now sets the floor to 0. The tests that exercise the gate keep it enabled via memory_gated_config, so the ODC-05 coverage they provide is preserved rather than short-circuited. Refs: backlog#1110 Co-Authored-By: heihutu * fix(object-data-cache): opt the usecase fill tests out of the memory gate The previous commit exempted the fill-dependent tests it could find, but missed the six adapters built inside object_usecase.rs. Fixing the first batch moved nextest's fail-fast point forward and CI surfaced them: build_get_object_body_with_cache_materializes_once_and_hits_later and ..._uses_cached_body_without_reader_preread both assert a fill lands, and both ran with the default 20% free-memory floor. Set the floor to 0 on all six, matching the sibling test modules. Verified by pinning the gate's snapshot to 0% available — harsher than any CI pod — and confirming these tests still pass, which shows the exemption path never reads memory at all. An exhaustive sweep over every fill-enabled ObjectDataCacheConfig in the tree now shows no remaining site: the only unexempted ones are fill_enabled()'s matches! arm and two adapter tests that assert the adapter is disabled and therefore never fill. Refs: backlog#1110 Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- Cargo.lock | 3 +- crates/object-data-cache/Cargo.toml | 1 + crates/object-data-cache/src/cache.rs | 3 + crates/object-data-cache/src/config.rs | 275 ++++++++++++++++-- crates/object-data-cache/src/error.rs | 24 ++ crates/object-data-cache/src/memory.rs | 160 +++++++++- crates/object-data-cache/src/moka_backend.rs | 15 +- crates/utils/src/envs.rs | 60 +++- rustfs/src/app/object_data_cache/adapter.rs | 168 ++++++++++- rustfs/src/app/object_data_cache/body.rs | 2 + rustfs/src/app/object_data_cache/hook.rs | 2 + .../src/app/object_data_cache/invalidation.rs | 2 + rustfs/src/app/object_usecase.rs | 12 + 13 files changed, 686 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ac30bff2..693502e8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9575,6 +9575,7 @@ dependencies = [ "sysinfo", "thiserror 2.0.18", "tokio", + "tracing", ] [[package]] @@ -10030,7 +10031,7 @@ dependencies = [ [[package]] name = "rustfs-uring" version = "0.1.0" -source = "git+https://github.com/rustfs/uring?rev=39018c09cde98b380658f1a1511349a3574daddf#39018c09cde98b380658f1a1511349a3574daddf" +source = "git+https://github.com/rustfs/uring?rev=f577ba0bfd6c3d74f9bdc54b573b8fcb45d6bc22#f577ba0bfd6c3d74f9bdc54b573b8fcb45d6bc22" dependencies = [ "io-uring", "libc", diff --git a/crates/object-data-cache/Cargo.toml b/crates/object-data-cache/Cargo.toml index be4303461..2e9b57502 100644 --- a/crates/object-data-cache/Cargo.toml +++ b/crates/object-data-cache/Cargo.toml @@ -35,6 +35,7 @@ starshard.workspace = true sysinfo = { workspace = true, features = ["multithread"] } thiserror.workspace = true tokio = { workspace = true, features = ["sync"] } +tracing.workspace = true [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt", "time"] } diff --git a/crates/object-data-cache/src/cache.rs b/crates/object-data-cache/src/cache.rs index 9437df50b..e06f45de2 100644 --- a/crates/object-data-cache/src/cache.rs +++ b/crates/object-data-cache/src/cache.rs @@ -348,6 +348,9 @@ mod tests { let config = ObjectDataCacheConfig { mode: ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading, which differs + // between a developer host and a CI container. + min_free_memory_percent: 0, ..ObjectDataCacheConfig::default() }; ObjectDataCache::new(config).expect("fill-enabled cache config should initialize") diff --git a/crates/object-data-cache/src/config.rs b/crates/object-data-cache/src/config.rs index d5ea7bc73..b6782c69d 100644 --- a/crates/object-data-cache/src/config.rs +++ b/crates/object-data-cache/src/config.rs @@ -13,12 +13,32 @@ // limitations under the License. use crate::error::ObjectDataCacheConfigError; +use crate::memory::{MemoryBasis, resolve_effective_memory}; +use std::sync::Once; use std::time::Duration; -use sysinfo::System; const DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP: u64 = 10; const DEFAULT_DERIVED_MAX_BYTES_CAP: u64 = 64 * 1024 * 1024 * 1024; +/// Upper bound (seconds) for `ttl` / `time_to_idle`. Kept far below moka's +/// ~1000-year builder assertion while remaining a sane operational cap so a +/// bad env var degrades to the disabled adapter instead of panicking at boot. +const MAX_DURATION_SECS: u64 = 30 * 24 * 60 * 60; + +/// Overhead reserved on top of a cached body when validating an explicit +/// `max_bytes`. moka's weigher charges key bytes + a small per-entry overhead +/// on top of the body, so `max_bytes` must clear `max_entry_bytes` by at least +/// this margin for the entry to ever be retained. +const ENTRY_WEIGHT_OVERHEAD_BYTES: u64 = 4096; + +/// Upper bound for `max_entry_bytes`. moka weighers return `u32`, so an entry +/// above ~4 GiB would be under-weighted and bypass capacity accounting; stay +/// below `u32::MAX` with room for the weigher overhead. +const MAX_ENTRY_BYTES_LIMIT: u64 = u32::MAX as u64 - ENTRY_WEIGHT_OVERHEAD_BYTES; + +/// Guards the one-shot startup log of the resolved cache capacity. +static RESOLVED_CAPACITY_LOGGED: Once = Once::new(); + /// Runtime mode for the object data cache. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ObjectDataCacheMode { @@ -56,16 +76,25 @@ pub struct ObjectDataCacheConfig { /// Maximum cacheable entry size in bytes. pub max_entry_bytes: u64, /// Time-to-live for a cache entry. + /// + /// moka expires an entry at `min(ttl, time_to_idle-since-last-access)`, so + /// a `time_to_idle` larger than `ttl` never takes effect. pub ttl: Duration, /// Time-to-idle for a cache entry. + /// + /// See [`ttl`](Self::ttl): expiration uses `min(ttl, time_to_idle)`, so + /// setting `time_to_idle` above `ttl` is inert. pub time_to_idle: Duration, - /// Minimum free memory percent before fill is paused. + /// Minimum free memory percent before fill is paused. Zero disables the + /// memory gate entirely, which makes fill admission independent of the + /// host's or container's live memory reading. pub min_free_memory_percent: u8, /// Fill concurrency multiplier applied to CPU count. pub fill_concurrency_per_cpu: u16, /// Absolute fill concurrency cap. pub fill_concurrency_max: u16, - /// Conservative cap for keys attached to one object identity. + /// Conservative cap for keys attached to one object identity. Must be at + /// least 2: the index admits a new key by evicting the oldest one. pub identity_keys_max: u16, } @@ -111,16 +140,27 @@ impl ObjectDataCacheConfig { return Ok(self.max_bytes); } - let mut system = System::new(); - system.refresh_memory(); - let total_memory = system.total_memory(); + // Resolve capacity from the effective (container-aware) total memory so + // a pod with a cgroup limit far below the node RAM does not size the + // cache to the node. + let effective = resolve_effective_memory(); + let total_memory = effective.total_bytes; let derived = total_memory.saturating_mul(u64::from(self.max_memory_percent)) / 100; - let resolved = clamp_derived_max_bytes(derived, total_memory, self.max_entry_bytes); + let resolved = clamp_derived_max_bytes(derived, total_memory); if resolved == 0 { return Err(ObjectDataCacheConfigError::ZeroResolvedMaxBytes); } + // The derived capacity is no longer floored by `max_entry_bytes` (that + // used to silently inflate the cache above the safety clamp). If a + // single entry cannot fit, reject rather than inflate. + if self.max_entry_bytes > resolved { + return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity); + } + + log_resolved_capacity_once(resolved, total_memory, effective.basis); + Ok(resolved) } @@ -134,15 +174,46 @@ impl ObjectDataCacheConfig { return Err(ObjectDataCacheConfigError::ZeroMaxEntryBytes); } + if self.max_entry_bytes > MAX_ENTRY_BYTES_LIMIT { + return Err(ObjectDataCacheConfigError::MaxEntryBytesTooLarge); + } + + // An explicit capacity must leave room for a full entry plus the + // weigher overhead, otherwise moka can never retain the entry while + // fills still report success. + if self.max_bytes > 0 && self.max_bytes < self.max_entry_bytes.saturating_add(ENTRY_WEIGHT_OVERHEAD_BYTES) { + return Err(ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes); + } + if self.ttl.is_zero() { return Err(ObjectDataCacheConfigError::ZeroTimeToLiveSecs); } + if self.ttl.as_secs() > MAX_DURATION_SECS { + return Err(ObjectDataCacheConfigError::TimeToLiveTooLarge); + } + if self.time_to_idle.is_zero() { return Err(ObjectDataCacheConfigError::ZeroTimeToIdleSecs); } - if self.min_free_memory_percent == 0 || self.min_free_memory_percent > 100 { + if self.time_to_idle.as_secs() > MAX_DURATION_SECS { + return Err(ObjectDataCacheConfigError::TimeToIdleTooLarge); + } + + // moka expires at min(ttl, time_to_idle); a larger time_to_idle is + // inert. Warn instead of rejecting so a benign misconfiguration still + // starts the cache. + if self.time_to_idle > self.ttl { + tracing::warn!( + time_to_idle_secs = self.time_to_idle.as_secs(), + ttl_secs = self.ttl.as_secs(), + "object data cache time_to_idle exceeds ttl; moka expires at min(ttl, time_to_idle) so the larger time_to_idle has no effect" + ); + } + + // Zero is a deliberate opt-out of the memory gate, not an invalid value. + if self.min_free_memory_percent > 100 { return Err(ObjectDataCacheConfigError::InvalidMinFreeMemoryPercent); } @@ -162,15 +233,33 @@ impl ObjectDataCacheConfig { return Err(ObjectDataCacheConfigError::ZeroIdentityKeysMax); } + // The identity index evicts the oldest key to admit a new one, so a + // budget of 1 evicts the previous key on every fill and can never hold + // two live keys of one object at once. + if self.identity_keys_max < 2 { + return Err(ObjectDataCacheConfigError::IdentityKeysMaxTooSmall); + } + Ok(()) } } -fn clamp_derived_max_bytes(derived: u64, total_memory: u64, max_entry_bytes: u64) -> u64 { +fn clamp_derived_max_bytes(derived: u64, total_memory: u64) -> u64 { let percent_cap = total_memory.saturating_mul(DEFAULT_DERIVED_MAX_MEMORY_PERCENT_CAP) / 100; - let safe_cap = percent_cap.min(DEFAULT_DERIVED_MAX_BYTES_CAP).max(max_entry_bytes); + let safe_cap = percent_cap.min(DEFAULT_DERIVED_MAX_BYTES_CAP); - derived.min(safe_cap).max(max_entry_bytes) + derived.min(safe_cap) +} + +fn log_resolved_capacity_once(resolved_max_bytes: u64, effective_total_bytes: u64, basis: MemoryBasis) { + RESOLVED_CAPACITY_LOGGED.call_once(|| { + tracing::info!( + resolved_max_bytes, + effective_total_bytes, + basis = basis.as_str(), + "object data cache resolved capacity" + ); + }); } #[cfg(test)] @@ -260,6 +349,54 @@ mod tests { assert_eq!(err, ObjectDataCacheConfigError::FillConcurrencyMaxTooSmall); } + #[test] + fn validate_accepts_zero_min_free_memory_percent_as_gate_opt_out() { + let config = ObjectDataCacheConfig { + min_free_memory_percent: 0, + ..ObjectDataCacheConfig::default() + }; + + assert!(config.validate().is_ok()); + } + + #[test] + fn validate_rejects_min_free_memory_percent_above_100() { + let config = ObjectDataCacheConfig { + min_free_memory_percent: 101, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("a free-memory floor above 100% is unsatisfiable"); + + assert_eq!(err, ObjectDataCacheConfigError::InvalidMinFreeMemoryPercent); + } + + #[test] + fn validate_rejects_single_key_identity_budget() { + let config = ObjectDataCacheConfig { + identity_keys_max: 1, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("a one-key identity budget evicts the previous key on every fill"); + + assert_eq!(err, ObjectDataCacheConfigError::IdentityKeysMaxTooSmall); + } + + #[test] + fn validate_accepts_two_key_identity_budget() { + let config = ObjectDataCacheConfig { + identity_keys_max: 2, + ..ObjectDataCacheConfig::default() + }; + + assert!(config.validate().is_ok()); + } + #[test] fn validate_accepts_explicit_byte_cap() { let config = ObjectDataCacheConfig { @@ -287,25 +424,129 @@ mod tests { } #[test] - fn resolved_max_bytes_is_at_least_max_entry_bytes() { + fn derived_capacity_is_not_inflated_by_max_entry_bytes() { + // 512 MiB effective memory with a tiny percent yields a small derived + // capacity. A large max_entry_bytes must NOT raise the total capacity + // (the old `.max(max_entry_bytes)` floor did exactly that). + let host = 512_u64 * 1024 * 1024; + let derived = host / 100; + let resolved = clamp_derived_max_bytes(derived, host); + + assert_eq!(resolved, derived); + assert!(resolved < 1024 * 1024 * 1024); + } + + #[test] + fn resolved_max_bytes_rejects_entry_larger_than_capacity() { + // Derived capacity is clamped to at most 64 GiB, so a 128 GiB entry cap + // can never fit regardless of the test host's memory. let config = ObjectDataCacheConfig { max_bytes: 0, - max_memory_percent: 1, - max_entry_bytes: 8_388_608, + max_memory_percent: 100, + max_entry_bytes: 128 * 1024 * 1024 * 1024, ..ObjectDataCacheConfig::default() }; - let resolved = config.resolved_max_bytes().expect("derived capacity should stay positive"); + let err = config + .resolved_max_bytes() + .expect_err("entry cap above the derived capacity must be rejected"); - assert!(resolved >= config.max_entry_bytes); + assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsCapacity); } #[test] fn derived_max_bytes_clamps_to_v3_safe_cap() { let one_tib = 1024_u64 * 1024 * 1024 * 1024; let derived = one_tib / 2; - let resolved = clamp_derived_max_bytes(derived, one_tib, 1_048_576); + let resolved = clamp_derived_max_bytes(derived, one_tib); assert_eq!(resolved, DEFAULT_DERIVED_MAX_BYTES_CAP); } + + #[test] + fn validate_rejects_ttl_above_upper_bound() { + let config = ObjectDataCacheConfig { + ttl: Duration::from_secs(u64::MAX), + ..ObjectDataCacheConfig::default() + }; + + let err = config.validate().expect_err("ttl above the operational cap must be rejected"); + + assert_eq!(err, ObjectDataCacheConfigError::TimeToLiveTooLarge); + } + + #[test] + fn validate_rejects_time_to_idle_above_upper_bound() { + let config = ObjectDataCacheConfig { + time_to_idle: Duration::from_secs(u64::MAX), + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("time-to-idle above the operational cap must be rejected"); + + assert_eq!(err, ObjectDataCacheConfigError::TimeToIdleTooLarge); + } + + #[test] + fn validate_rejects_entry_above_weigher_limit() { + let config = ObjectDataCacheConfig { + max_entry_bytes: u32::MAX as u64, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("entry size at the u32 weigher boundary must be rejected"); + + assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesTooLarge); + } + + #[test] + fn validate_rejects_entry_larger_than_explicit_max_bytes() { + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::HitOnly, + max_bytes: 2 * 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 8 * 1024 * 1024, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("an entry larger than max_bytes can never be retained"); + + assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes); + } + + #[test] + fn validate_rejects_entry_equal_to_explicit_max_bytes() { + // Equal fails because the weigher adds key bytes + per-entry overhead. + let config = ObjectDataCacheConfig { + mode: ObjectDataCacheMode::HitOnly, + max_bytes: 4 * 1024 * 1024, + max_memory_percent: 0, + max_entry_bytes: 4 * 1024 * 1024, + ..ObjectDataCacheConfig::default() + }; + + let err = config + .validate() + .expect_err("max_entry_bytes must clear max_bytes by the weigher overhead"); + + assert_eq!(err, ObjectDataCacheConfigError::MaxEntryBytesExceedsMaxBytes); + } + + #[test] + fn validate_accepts_time_to_idle_greater_than_ttl() { + // A time_to_idle above ttl is inert (warned, not rejected). + let config = ObjectDataCacheConfig { + ttl: Duration::from_secs(30), + time_to_idle: Duration::from_secs(60), + ..ObjectDataCacheConfig::default() + }; + + assert!(config.validate().is_ok()); + } } diff --git a/crates/object-data-cache/src/error.rs b/crates/object-data-cache/src/error.rs index 2f4eda4c2..d63e3d442 100644 --- a/crates/object-data-cache/src/error.rs +++ b/crates/object-data-cache/src/error.rs @@ -25,14 +25,34 @@ pub enum ObjectDataCacheConfigError { #[error("object data cache max_entry_bytes must be greater than 0")] ZeroMaxEntryBytes, + /// The configured entry size exceeds the moka weigher's u32 accounting range. + #[error("object data cache max_entry_bytes must stay below 4 GiB so the capacity weigher does not under-count entries")] + MaxEntryBytesTooLarge, + + /// The configured entry size cannot fit inside the explicit byte capacity. + #[error("object data cache max_entry_bytes plus weigher overhead must not exceed max_bytes")] + MaxEntryBytesExceedsMaxBytes, + + /// The configured entry size exceeds the resolved (derived) cache capacity. + #[error("object data cache max_entry_bytes must not exceed the resolved cache capacity")] + MaxEntryBytesExceedsCapacity, + /// The configured time-to-live cannot be zero. #[error("object data cache ttl_secs must be greater than 0")] ZeroTimeToLiveSecs, + /// The configured time-to-live exceeds the supported upper bound. + #[error("object data cache ttl_secs must not exceed 2592000 seconds (30 days)")] + TimeToLiveTooLarge, + /// The configured time-to-idle cannot be zero. #[error("object data cache time_to_idle_secs must be greater than 0")] ZeroTimeToIdleSecs, + /// The configured time-to-idle exceeds the supported upper bound. + #[error("object data cache time_to_idle_secs must not exceed 2592000 seconds (30 days)")] + TimeToIdleTooLarge, + /// The configured minimum free memory percentage exceeded the supported range. #[error("object data cache min_free_memory_percent must be in 1..=100")] InvalidMinFreeMemoryPercent, @@ -53,6 +73,10 @@ pub enum ObjectDataCacheConfigError { #[error("object data cache identity_keys_max must be greater than 0")] ZeroIdentityKeysMax, + /// A single-key identity budget evicts the previous key on every fill. + #[error("object data cache identity_keys_max must be at least 2")] + IdentityKeysMaxTooSmall, + /// Failed to resolve a non-zero cache capacity from the runtime environment. #[error("object data cache could not resolve a positive max capacity")] ZeroResolvedMaxBytes, diff --git a/crates/object-data-cache/src/memory.rs b/crates/object-data-cache/src/memory.rs index 5b4c99256..a395cd1fd 100644 --- a/crates/object-data-cache/src/memory.rs +++ b/crates/object-data-cache/src/memory.rs @@ -22,6 +22,73 @@ use sysinfo::System; const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(5); +/// Source used to resolve the effective memory limits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MemoryBasis { + /// Limits come from host memory reported by sysinfo. + Host, + /// Limits come from a constraining cgroup (container) memory limit. + Cgroup, +} + +impl MemoryBasis { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Host => "host", + Self::Cgroup => "cgroup", + } + } +} + +/// Effective memory totals after reconciling host memory with cgroup limits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct EffectiveMemory { + /// Effective total memory in bytes. + pub(crate) total_bytes: u64, + /// Effective available memory in bytes. + pub(crate) available_bytes: u64, + /// Whether the limits are host- or cgroup-derived. + pub(crate) basis: MemoryBasis, +} + +/// Reconciles host memory with an optional cgroup limit. +/// +/// `cgroup` carries `(total_memory, free_memory)` as reported for the cgroup +/// hierarchy (sysinfo already caps `total_memory` at the host total). A cgroup +/// only counts when it actually constrains below the host, so an unlimited +/// cgroup transparently falls back to host values. +pub(crate) fn select_effective_memory(host_total: u64, host_available: u64, cgroup: Option<(u64, u64)>) -> EffectiveMemory { + match cgroup { + Some((cgroup_total, cgroup_free)) if cgroup_total > 0 && cgroup_total < host_total => EffectiveMemory { + total_bytes: cgroup_total, + available_bytes: cgroup_free.min(cgroup_total), + basis: MemoryBasis::Cgroup, + }, + _ => EffectiveMemory { + total_bytes: host_total, + available_bytes: host_available, + basis: MemoryBasis::Host, + }, + } +} + +/// Resolves the effective memory from an already-refreshed system handle. +/// +/// `cgroup_limits()` is computed fresh on each call and is only implemented on +/// Linux (it returns `None` elsewhere), so non-Linux hosts always use host +/// values. +pub(crate) fn effective_memory_from_system(system: &System) -> EffectiveMemory { + let cgroup = system.cgroup_limits().map(|limits| (limits.total_memory, limits.free_memory)); + select_effective_memory(system.total_memory(), system.available_memory(), cgroup) +} + +/// Resolves the effective memory using a fresh, memory-refreshed system handle. +pub(crate) fn resolve_effective_memory() -> EffectiveMemory { + let mut system = System::new(); + system.refresh_memory(); + effective_memory_from_system(&system) +} + /// Immutable memory snapshot used by the cache fill gate. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct ObjectDataCacheMemorySnapshot { @@ -62,9 +129,10 @@ impl ObjectDataCacheMemoryGate { pub fn new(config: &ObjectDataCacheConfig, stats: Arc) -> Self { let mut system = System::new(); system.refresh_memory(); + let effective = effective_memory_from_system(&system); let snapshot = ObjectDataCacheMemorySnapshot { - total_bytes: system.total_memory(), - available_bytes: system.available_memory(), + total_bytes: effective.total_bytes, + available_bytes: effective.available_bytes, }; Self { @@ -106,9 +174,10 @@ impl ObjectDataCacheMemoryGate { let mut system = lock_or_recover(&self.system); system.refresh_memory(); + let effective = effective_memory_from_system(&system); let snapshot = ObjectDataCacheMemorySnapshot { - total_bytes: system.total_memory(), - available_bytes: system.available_memory(), + total_bytes: effective.total_bytes, + available_bytes: effective.available_bytes, }; self.snapshot_total_bytes.store(snapshot.total_bytes, Ordering::Relaxed); @@ -119,6 +188,12 @@ impl ObjectDataCacheMemoryGate { /// Returns true when the fill path may proceed under current memory pressure. pub fn allows_fill(&self, required_bytes: u64) -> bool { + // A zero floor opts out of the gate, so fill admission never depends on + // a live memory reading — which differs between a host and a container. + if self.min_free_memory_percent == 0 { + return true; + } + self.refresh_if_stale(); let snapshot = self.snapshot(); if snapshot.total_bytes == 0 { @@ -152,11 +227,65 @@ fn lock_or_recover(mutex: &Mutex) -> std::sync::MutexGuard<'_, T> { #[cfg(test)] mod tests { - use super::{ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot}; + use super::{MemoryBasis, ObjectDataCacheMemoryGate, ObjectDataCacheMemorySnapshot, select_effective_memory}; use crate::config::ObjectDataCacheConfig; use crate::stats::ObjectDataCacheStats; use std::sync::Arc; + const GIB: u64 = 1024 * 1024 * 1024; + + #[test] + fn select_effective_memory_prefers_constraining_cgroup() { + let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, GIB))); + + assert_eq!(effective.basis, MemoryBasis::Cgroup); + assert_eq!(effective.total_bytes, 2 * GIB); + assert_eq!(effective.available_bytes, GIB); + } + + #[test] + fn select_effective_memory_ignores_non_constraining_cgroup() { + // A cgroup total equal to (or above) the host total means no real limit. + let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((64 * GIB, 10 * GIB))); + + assert_eq!(effective.basis, MemoryBasis::Host); + assert_eq!(effective.total_bytes, 64 * GIB); + assert_eq!(effective.available_bytes, 40 * GIB); + } + + #[test] + fn select_effective_memory_falls_back_to_host_without_cgroup() { + let effective = select_effective_memory(8 * GIB, 4 * GIB, None); + + assert_eq!(effective.basis, MemoryBasis::Host); + assert_eq!(effective.total_bytes, 8 * GIB); + assert_eq!(effective.available_bytes, 4 * GIB); + } + + #[test] + fn select_effective_memory_caps_available_at_total() { + let effective = select_effective_memory(64 * GIB, 40 * GIB, Some((2 * GIB, 3 * GIB))); + + assert_eq!(effective.total_bytes, 2 * GIB); + assert_eq!(effective.available_bytes, 2 * GIB); + } + + #[test] + fn gate_pauses_fill_when_container_memory_is_low() { + // Simulate a pod-sized snapshot (256 MiB total, 16 MiB free): below the + // default 20% free-memory floor, so fills must pause even though a node + // would have plenty of headroom. + let stats = Arc::new(ObjectDataCacheStats::default()); + let gate = ObjectDataCacheMemoryGate::new(&ObjectDataCacheConfig::default(), Arc::clone(&stats)); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 256 * 1024 * 1024, + available_bytes: 16 * 1024 * 1024, + })); + + assert!(!gate.allows_fill(1024)); + assert_eq!(stats.snapshot().memory_pressure_events, 1); + } + #[test] fn allows_fill_when_memory_snapshot_has_headroom() { let stats = Arc::new(ObjectDataCacheStats::default()); @@ -169,6 +298,27 @@ mod tests { assert!(gate.allows_fill(100)); } + #[test] + fn zero_min_free_percent_disables_the_gate() { + // A pod-sized snapshot far below any floor: the gate is opted out, so + // fill admission stays independent of the live memory reading. + let stats = Arc::new(ObjectDataCacheStats::default()); + let gate = ObjectDataCacheMemoryGate::new( + &ObjectDataCacheConfig { + min_free_memory_percent: 0, + ..ObjectDataCacheConfig::default() + }, + Arc::clone(&stats), + ); + gate.set_test_snapshot(Some(ObjectDataCacheMemorySnapshot { + total_bytes: 1_000, + available_bytes: 1, + })); + + assert!(gate.allows_fill(512)); + assert_eq!(stats.snapshot().memory_pressure_events, 0); + } + #[test] fn blocks_fill_under_memory_pressure() { let stats = Arc::new(ObjectDataCacheStats::default()); diff --git a/crates/object-data-cache/src/moka_backend.rs b/crates/object-data-cache/src/moka_backend.rs index dba17740f..e8e9d13ae 100644 --- a/crates/object-data-cache/src/moka_backend.rs +++ b/crates/object-data-cache/src/moka_backend.rs @@ -275,6 +275,9 @@ mod tests { use std::sync::Arc; use std::time::Duration; + /// Fills here must succeed regardless of the live memory reading, which + /// differs between a developer host and a CI container, so the gate is + /// opted out of. Tests that exercise the gate re-enable it explicitly. fn enabled_config() -> ObjectDataCacheConfig { ObjectDataCacheConfig { mode: ObjectDataCacheMode::FillMaterializeEnabled, @@ -283,13 +286,20 @@ mod tests { max_entry_bytes: 1_048_576, ttl: Duration::from_millis(100), time_to_idle: Duration::from_millis(100), - min_free_memory_percent: 20, + min_free_memory_percent: 0, fill_concurrency_per_cpu: 1, fill_concurrency_max: 32, identity_keys_max: 16, } } + fn memory_gated_config() -> ObjectDataCacheConfig { + ObjectDataCacheConfig { + min_free_memory_percent: 20, + ..enabled_config() + } + } + fn cacheable_plan(object: &str, etag: &str) -> ObjectDataCacheGetPlan { ObjectDataCacheGetPlan::Cacheable { key: ObjectDataCacheKey::new("bucket", object, None, etag, 5, ObjectDataCacheBodyVariant::FullObjectPlainV1), @@ -394,7 +404,8 @@ mod tests { #[tokio::test] async fn moka_backend_skips_fill_under_memory_pressure() { let stats = Arc::new(ObjectDataCacheStats::default()); - let backend = MokaBackend::new(&enabled_config(), Arc::clone(&stats)).expect("moka backend should build"); + // The gate must be enabled here, otherwise allows_fill short-circuits. + let backend = MokaBackend::new(&memory_gated_config(), Arc::clone(&stats)).expect("moka backend should build"); backend .memory_gate .set_test_snapshot(Some(crate::memory::ObjectDataCacheMemorySnapshot { diff --git a/crates/utils/src/envs.rs b/crates/utils/src/envs.rs index 7e36fcf73..82e93c5b7 100644 --- a/crates/utils/src/envs.rs +++ b/crates/utils/src/envs.rs @@ -533,6 +533,49 @@ pub fn get_env_opt_u64(key: &str) -> Option { parse_env_value(key) } +/// Outcome of reading and parsing an environment variable. +/// +/// Unlike the `get_env_opt_*` helpers, this distinguishes an absent variable +/// from one that is present but fails to parse, so callers can treat a +/// malformed value as a hard configuration error instead of silently falling +/// back to a default. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EnvParseOutcome { + /// The variable (and any alias) is not set. + Absent, + /// The variable is set but its value failed to parse into `T`. + Invalid, + /// The variable is set and parsed successfully. + Parsed(T), +} + +/// Read an environment variable and report whether it is absent, present but +/// malformed, or parsed successfully. +/// +/// #Parameters +/// - `key`: The environment variable key to look up (deprecated aliases apply). +/// +/// #Returns +/// - `EnvParseOutcome`: `Absent`, `Invalid`, or `Parsed(value)`. +pub fn get_env_parse_outcome(key: &str) -> EnvParseOutcome +where + T: std::str::FromStr, +{ + let Some((used_key, value)) = resolve_env_with_aliases(key, &[]) else { + return EnvParseOutcome::Absent; + }; + + match value.parse::() { + Ok(parsed) => EnvParseOutcome::Parsed(parsed), + Err(_) => { + log_once(&format!("env_invalid_value:{used_key}"), || { + format!("Invalid {} value for {used_key}: {value}. Treating as unset.", type_name::()) + }); + EnvParseOutcome::Invalid + } + } +} + /// Retrieve an environment variable as a specific type, with a default value if not set or parsing fails. /// /// #Parameters @@ -687,8 +730,8 @@ pub fn apply_external_env_compat() -> ExternalEnvCompatReport { #[cfg(test)] mod tests { use super::{ - apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_bool_with_aliases, get_env_f64, - get_env_i32_with_aliases, get_env_opt_f64, get_env_opt_u64, get_env_str, get_env_u64, + EnvParseOutcome, apply_external_env_compat, build_external_env_compat_report_from_entries, get_env_bool_with_aliases, + get_env_f64, get_env_i32_with_aliases, get_env_opt_f64, get_env_opt_u64, get_env_parse_outcome, get_env_str, get_env_u64, }; fn source_key(suffix: &str) -> String { @@ -828,6 +871,19 @@ mod tests { }); } + #[test] + fn env_parse_outcome_distinguishes_absent_invalid_and_parsed() { + temp_env::with_var_unset("RUSTFS_TEST_OUTCOME", || { + assert_eq!(get_env_parse_outcome::("RUSTFS_TEST_OUTCOME"), EnvParseOutcome::Absent); + }); + temp_env::with_var("RUSTFS_TEST_OUTCOME", Some("not-a-u64"), || { + assert_eq!(get_env_parse_outcome::("RUSTFS_TEST_OUTCOME"), EnvParseOutcome::Invalid); + }); + temp_env::with_var("RUSTFS_TEST_OUTCOME", Some("42"), || { + assert_eq!(get_env_parse_outcome::("RUSTFS_TEST_OUTCOME"), EnvParseOutcome::Parsed(42)); + }); + } + #[test] fn apply_external_env_compat_copies_missing_rustfs_keys() { temp_env::with_var("MINIO_ROOT_USER", Some("compat-admin"), || { diff --git a/rustfs/src/app/object_data_cache/adapter.rs b/rustfs/src/app/object_data_cache/adapter.rs index 14d2ad415..87b843b89 100644 --- a/rustfs/src/app/object_data_cache/adapter.rs +++ b/rustfs/src/app/object_data_cache/adapter.rs @@ -52,8 +52,21 @@ impl ObjectDataCacheAdapter { /// Creates an adapter from runtime environment variables, falling back to /// no-op on invalid input so startup behavior stays conservative. + /// + /// A malformed value on any single key (bad mode string, failed validation, + /// or an unparseable numeric override) disables the whole cache rather than + /// silently starting with defaults the operator never chose. pub(crate) fn from_env_or_disabled() -> Arc { - Self::from_config_or_disabled(object_data_cache_config_from_env(), "env") + match object_data_cache_config_from_env() { + Ok(config) => Self::from_config_or_disabled(config, "env"), + Err(invalid_keys) => { + warn!( + invalid_keys = %invalid_keys.join(","), + "object data cache disabled because one or more environment variables are malformed" + ); + Self::disabled_arc() + } + } } /// Creates a disabled no-op adapter. @@ -131,20 +144,51 @@ impl Default for ObjectDataCacheAdapter { } } -fn object_data_cache_config_from_env() -> ObjectDataCacheConfig { - object_data_cache_config_from_values(ObjectDataCacheEnvValues { +/// Reads a numeric env override with three-valued semantics: absent keeps the +/// default (`None`), a valid value is applied, and a malformed value records the +/// key in `invalid_keys` so the caller can invalidate the whole config. +fn take_numeric_env(key: &'static str, invalid_keys: &mut Vec<&'static str>) -> Option +where + T: std::str::FromStr, +{ + match rustfs_utils::get_env_parse_outcome::(key) { + rustfs_utils::EnvParseOutcome::Parsed(value) => Some(value), + rustfs_utils::EnvParseOutcome::Absent => None, + rustfs_utils::EnvParseOutcome::Invalid => { + invalid_keys.push(key); + None + } + } +} + +fn object_data_cache_config_from_env() -> Result> { + let mut invalid_keys: Vec<&'static str> = Vec::new(); + + let values = ObjectDataCacheEnvValues { enabled: rustfs_utils::get_env_opt_bool(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE), mode: rustfs_utils::get_env_opt_str(rustfs_config::ENV_OBJECT_DATA_CACHE_MODE), - max_bytes: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_BYTES), - max_memory_percent: rustfs_utils::get_env_opt_u8(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT), - max_entry_bytes: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES), - ttl_secs: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_TTL_SECS), - time_to_idle_secs: rustfs_utils::get_env_opt_u64(rustfs_config::ENV_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS), - min_free_memory_percent: rustfs_utils::get_env_opt_u8(rustfs_config::ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT), - fill_concurrency_per_cpu: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU), - fill_concurrency_max: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX), - identity_keys_max: rustfs_utils::get_env_opt_u16(rustfs_config::ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX), - }) + max_bytes: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_BYTES, &mut invalid_keys), + max_memory_percent: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT, &mut invalid_keys), + max_entry_bytes: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES, &mut invalid_keys), + ttl_secs: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_TTL_SECS, &mut invalid_keys), + time_to_idle_secs: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS, &mut invalid_keys), + min_free_memory_percent: take_numeric_env( + rustfs_config::ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT, + &mut invalid_keys, + ), + fill_concurrency_per_cpu: take_numeric_env( + rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU, + &mut invalid_keys, + ), + fill_concurrency_max: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX, &mut invalid_keys), + identity_keys_max: take_numeric_env(rustfs_config::ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX, &mut invalid_keys), + }; + + if !invalid_keys.is_empty() { + return Err(invalid_keys); + } + + Ok(object_data_cache_config_from_values(values)) } fn object_data_cache_config_from_values(values: ObjectDataCacheEnvValues) -> ObjectDataCacheConfig { @@ -217,11 +261,41 @@ fn parse_object_data_cache_mode(value: &str) -> Option { #[cfg(test)] mod tests { use super::{ - ObjectDataCacheAdapter, ObjectDataCacheEnvValues, object_data_cache_config_from_values, parse_object_data_cache_mode, + ObjectDataCacheAdapter, ObjectDataCacheEnvValues, object_data_cache_config_from_env, + object_data_cache_config_from_values, parse_object_data_cache_mode, }; use rustfs_object_data_cache::{ObjectDataCacheConfig, ObjectDataCacheMode}; use std::time::Duration; + /// All object-data-cache env keys, unset by default. Tests override the + /// entries they care about so the surrounding process environment cannot + /// influence the outcome. + fn all_env_unset() -> Vec<(&'static str, Option<&'static str>)> { + vec![ + (rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MODE, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_BYTES, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_MEMORY_PERCENT, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_TTL_SECS, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_TIME_TO_IDLE_SECS, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX, None), + (rustfs_config::ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX, None), + ] + } + + fn with_env_overrides(overrides: &[(&'static str, &'static str)]) -> Vec<(&'static str, Option<&'static str>)> { + let mut vars = all_env_unset(); + for (key, value) in overrides { + if let Some(entry) = vars.iter_mut().find(|(name, _)| name == key) { + entry.1 = Some(value); + } + } + vars + } + #[test] fn disabled_adapter_exposes_disabled_engine() { let adapter = ObjectDataCacheAdapter::disabled(); @@ -342,4 +416,70 @@ mod tests { assert!(adapter.is_disabled()); } + + #[test] + fn overlong_ttl_builds_disabled_adapter_without_panic() { + // moka's builder asserts on durations beyond ~1000 years; validate() + // must reject an out-of-range ttl so the adapter degrades to disabled + // rather than panicking AppContext::new at boot. + let adapter = ObjectDataCacheAdapter::from_config_or_disabled( + ObjectDataCacheConfig { + mode: ObjectDataCacheMode::FillMaterializeEnabled, + max_bytes: 8 * 1024 * 1024, + ttl: Duration::from_secs(u64::MAX), + ..ObjectDataCacheConfig::default() + }, + "test", + ); + + assert!(adapter.is_disabled()); + } + + #[test] + fn object_data_cache_env_absent_uses_defaults() { + temp_env::with_vars(all_env_unset(), || { + let config = object_data_cache_config_from_env().expect("absent env should build the default config"); + + assert_eq!(config, ObjectDataCacheConfig::default()); + }); + } + + #[test] + fn object_data_cache_env_valid_numeric_value_is_applied() { + let vars = with_env_overrides(&[ + (rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, "true"), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES, "2097152"), + ]); + temp_env::with_vars(vars, || { + let config = object_data_cache_config_from_env().expect("valid numeric env should be applied"); + + assert_eq!(config.mode, ObjectDataCacheMode::HitOnly); + assert_eq!(config.max_entry_bytes, 2_097_152); + }); + } + + #[test] + fn object_data_cache_env_malformed_numeric_invalidates_config() { + let key = rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES; + let vars = with_env_overrides(&[(rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, "true"), (key, "not-a-number")]); + temp_env::with_vars(vars, || { + let invalid_keys = + object_data_cache_config_from_env().expect_err("a malformed numeric key must invalidate the config"); + + assert!(invalid_keys.contains(&key)); + }); + } + + #[test] + fn object_data_cache_env_malformed_numeric_builds_disabled_adapter() { + let vars = with_env_overrides(&[ + (rustfs_config::ENV_OBJECT_DATA_CACHE_ENABLE, "true"), + (rustfs_config::ENV_OBJECT_DATA_CACHE_MAX_ENTRY_BYTES, "not-a-number"), + ]); + temp_env::with_vars(vars, || { + let adapter = ObjectDataCacheAdapter::from_env_or_disabled(); + + assert!(adapter.is_disabled()); + }); + } } diff --git a/rustfs/src/app/object_data_cache/body.rs b/rustfs/src/app/object_data_cache/body.rs index 70689e70f..13c0c4d0e 100644 --- a/rustfs/src/app/object_data_cache/body.rs +++ b/rustfs/src/app/object_data_cache/body.rs @@ -92,6 +92,8 @@ mod tests { let config = ObjectDataCacheConfig { mode: ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..ObjectDataCacheConfig::default() }; ObjectDataCacheAdapter::new(config).expect("fill-enabled config should build adapter") diff --git a/rustfs/src/app/object_data_cache/hook.rs b/rustfs/src/app/object_data_cache/hook.rs index ee7b74115..df21f08ba 100644 --- a/rustfs/src/app/object_data_cache/hook.rs +++ b/rustfs/src/app/object_data_cache/hook.rs @@ -86,6 +86,8 @@ mod tests { ObjectDataCacheAdapter::new(ObjectDataCacheConfig { mode: ObjectDataCacheMode::FillBufferedOnly, max_bytes: 4 * 1024 * 1024, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..ObjectDataCacheConfig::default() }) .expect("adapter"), diff --git a/rustfs/src/app/object_data_cache/invalidation.rs b/rustfs/src/app/object_data_cache/invalidation.rs index 0ea479053..c9934923c 100644 --- a/rustfs/src/app/object_data_cache/invalidation.rs +++ b/rustfs/src/app/object_data_cache/invalidation.rs @@ -128,6 +128,8 @@ mod tests { let config = ObjectDataCacheConfig { mode: ObjectDataCacheMode::FillMaterializeEnabled, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..ObjectDataCacheConfig::default() }; ObjectDataCacheAdapter::new(config).expect("enabled config should build adapter") diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 8c0638a8c..4b710ef59 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -7463,6 +7463,8 @@ mod tests { crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("fill-enabled cache adapter should initialize"); @@ -7519,6 +7521,8 @@ mod tests { crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("fill-enabled cache adapter should initialize"); @@ -7583,6 +7587,8 @@ mod tests { crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("fill-enabled cache adapter should initialize"); @@ -7652,6 +7658,8 @@ mod tests { crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillBufferedOnly, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("fill-enabled cache adapter should initialize"); @@ -7715,6 +7723,8 @@ mod tests { crate::app::object_data_cache::ObjectDataCacheAdapter::new(rustfs_object_data_cache::ObjectDataCacheConfig { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, max_bytes: 8_388_608, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("materialize-fill cache adapter should initialize"); @@ -7786,6 +7796,8 @@ mod tests { mode: rustfs_object_data_cache::ObjectDataCacheMode::FillMaterializeEnabled, max_bytes: 8_388_608, max_entry_bytes: 4, + // Fill must not depend on the live memory reading (host vs container). + min_free_memory_percent: 0, ..rustfs_object_data_cache::ObjectDataCacheConfig::default() }) .expect("materialize-fill cache adapter should initialize");