From 3b0a28dd9b6012ec8097d9a395c7ad7f26678f38 Mon Sep 17 00:00:00 2001 From: houseme Date: Tue, 25 Aug 2026 00:45:30 +0800 Subject: [PATCH] fix(memory): cgroup-aware resource detection for container environments (#6536) * fix(iam): raise recursion limit for migration test Co-Authored-By: heihutu * fix(memory): cgroup-aware resource detection for container environments Issue #5803 reported memory RSS regression since beta.9: - RSS memory steps ~+300 MiB on tiny S3 bursts and never returns - Daily OOMKills in 1 GiB containers - Root cause: RustFS uses host memory/CPU instead of container cgroup limits Changes: - Add cgroup_resources.rs: cgroup v1/v2 CPU and memory detection - Add container_config.rs: container configuration with env overrides - Fix memory_observability.rs: use effective memory (cgroup-aware) - Fix server/runtime.rs: use cgroup-aware CPU detection for Tokio - Cap max_blocking_threads to 256 for small containers (<=4 cores) - Add new metrics: rustfs_memory_effective_total_bytes, rustfs_cgroup_* - Add startup logging of detected container resources New environment variables: - RUSTFS_DISABLE_CGROUP_DETECTION: disable cgroup detection - RUSTFS_OVERRIDE_CPU_CORES: override detected CPU cores - RUSTFS_OVERRIDE_MEMORY_BYTES: override detected memory limit Fixes: rustfs/rustfs#5803 Tracking: rustfs/backlog#2012 Co-Authored-By: heihutu * style: apply cargo fmt formatting Co-Authored-By: heihutu * fix: cross-platform compatibility for cgroup detection - Move CHANGES_SUMMARY.md and FINAL_SUMMARY.md to docs/operations/ - Add platform-specific cgroup detection (Linux only) - Non-Linux platforms (macOS, Windows) fall back to host values - Add platform-specific tests for cgroup detection - Remove unused imports for non-Linux builds Co-Authored-By: heihutu * fix: clippy warnings for cgroup_resources - Remove unused import super::CgroupResources - Use derive(Default) instead of manual impl - Remove redundant trim() before split_whitespace() - Fix absurd_extreme_comparisons (quota <= 0 for u64) - Use div_ceil() instead of manual implementation Co-Authored-By: heihutu * refactor: consolidate cgroup detection into single module - Merge cgroup_resources.rs and container_config.rs into unified module - Remove duplicate test file cgroup_resources_test.rs - Remove redundant CHANGES_SUMMARY.md and FINAL_SUMMARY.md - Simplify memory_observability.rs to use unified API - Simplify server/runtime.rs to use unified API - All cgroup detection logic now in single source of truth - Environment variable overrides integrated into main module - Clippy and fmt clean Co-Authored-By: heihutu --------- Co-authored-by: heihutu --- .../container-resource-detection.md | 203 ++++++++++ rustfs/src/cgroup_resources.rs | 361 ++++++++++++++++++ rustfs/src/lib.rs | 1 + rustfs/src/memory_observability.rs | 46 ++- rustfs/src/server/runtime.rs | 21 +- rustfs/src/startup_entrypoint.rs | 4 + 6 files changed, 615 insertions(+), 21 deletions(-) create mode 100644 docs/operations/container-resource-detection.md create mode 100644 rustfs/src/cgroup_resources.rs diff --git a/docs/operations/container-resource-detection.md b/docs/operations/container-resource-detection.md new file mode 100644 index 000000000..bfd6b5d7d --- /dev/null +++ b/docs/operations/container-resource-detection.md @@ -0,0 +1,203 @@ +# Container Resource Detection + +RustFS automatically detects container resource limits (CPU and memory) from cgroup v1/v2. This ensures correct resource allocation and accurate metrics in containerized environments (Kubernetes, Docker, etc.). + +## Problem + +When RustFS runs in a container, the underlying system libraries report the **host's** total CPU cores and memory, not the container's limits. This leads to: + +1. **Over-provisioned Tokio threads**: Too many worker and blocking threads +2. **Incorrect memory metrics**: `rustfs_memory_usage_percent` shows host-based percentage +3. **Memory budget errors**: Object data cache sized to host RAM instead of container limit +4. **OOMKills**: Container exceeds its memory limit and gets killed + +## Solution + +RustFS now detects cgroup limits directly from the filesystem: + +- **CPU**: `/sys/fs/cgroup/cpu.max` (v2) or `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` (v1) +- **Memory**: `/sys/fs/cgroup/memory.max` (v2) or `/sys/fs/cgroup/memory/memory.limit_in_bytes` (v1) + +The effective resource limits are the **minimum** of host and cgroup values. + +## Detection Logic + +### CPU Detection + +1. Read cgroup v2 `/sys/fs/cgroup/cpu.max` + - Format: `"$QUOTA $PERIOD"` or `"max"` (unlimited) + - Calculate: `cores = ceil(quota / period)` +2. Fallback to cgroup v1 `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` + - Calculate: `cores = ceil(quota / period)` +3. Fallback to host CPU count from `sysinfo` + +### Memory Detection + +1. Read cgroup v2 `/sys/fs/cgroup/memory.max` + - Value in bytes or `"max"` (unlimited) +2. Fallback to cgroup v1 `/sys/fs/cgroup/memory/memory.limit_in_bytes` + - Very large values (≥2^62) indicate unlimited +3. Fallback to host memory from `sysinfo` + +## Environment Variables + +### Disable Cgroup Detection + +```bash +RUSTFS_DISABLE_CGROUP_DETECTION=1 +``` + +Disables cgroup detection entirely. Useful for testing or when cgroup filesystem is not accessible. + +### Override CPU Cores + +```bash +RUSTFS_OVERRIDE_CPU_CORES=4 +``` + +Overrides detected CPU cores. Takes precedence over cgroup detection. + +### Override Memory Limit + +```bash +RUSTFS_OVERRIDE_MEMORY_BYTES=2147483648 +``` + +Overrides detected memory limit in bytes. Takes precedence over cgroup detection. + +## Metrics + +### New Metrics + +| Metric | Description | +|--------|-------------| +| `rustfs_memory_effective_total_bytes` | Effective memory total (host or cgroup) | +| `rustfs_cgroup_detected` | Whether cgroup limits were detected (1=yes, 0=no) | +| `rustfs_cgroup_cpu_cores_limit` | Detected CPU cores limit | +| `rustfs_cgroup_memory_limit_bytes` | Detected memory limit | + +### Updated Metrics + +| Metric | Change | +|--------|--------| +| `rustfs_memory_total_bytes` | Now uses effective memory (cgroup-aware) | +| `rustfs_memory_usage_percent` | Now calculated against effective memory | + +## Startup Logging + +RustFS logs detected container resources at startup: + +``` +INFO container resources (detected from cgroup) cpu_cores=2 memory_bytes=1073741824 memory_mib=1024 +``` + +or + +``` +INFO container resources (overridden by environment variables) cpu_cores=4 memory_bytes=2147483648 memory_mib=2048 +``` + +## Examples + +### Kubernetes with Resource Limits + +```yaml +resources: + limits: + cpu: "2" + memory: "1Gi" + requests: + cpu: "500m" + memory: "512Mi" +``` + +RustFS will detect: +- CPU cores: 2 +- Memory: 1 GiB (1073741824 bytes) + +### Docker with CPU and Memory Limits + +```bash +docker run --cpus=2 --memory=1g rustfs/rustfs:latest +``` + +RustFS will detect: +- CPU cores: 2 +- Memory: 1 GiB + +### Manual Override + +```bash +export RUSTFS_OVERRIDE_CPU_CORES=4 +export RUSTFS_OVERRIDE_MEMORY_BYTES=2147483648 +``` + +RustFS will use: +- CPU cores: 4 +- Memory: 2 GiB + +## Troubleshooting + +### Cgroup Detection Not Working + +1. Check if cgroup filesystem is mounted: + ```bash + ls -la /sys/fs/cgroup/ + ``` + +2. Check cgroup version: + ```bash + stat -fc %T /sys/fs/cgroup/ + ``` + - `cgroup2fs` = cgroup v2 + - `tmpfs` = cgroup v1 + +3. Check if limits are set: + ```bash + # cgroup v2 + cat /sys/fs/cgroup/cpu.max + cat /sys/fs/cgroup/memory.max + + # cgroup v1 + cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us + cat /sys/fs/cgroup/memory/memory.limit_in_bytes + ``` + +### Metrics Show Host Values + +If `rustfs_memory_effective_total_bytes` shows host memory instead of cgroup limit: + +1. Verify cgroup detection is not disabled: + ```bash + echo $RUSTFS_DISABLE_CGROUP_DETECTION + ``` + +2. Check startup logs for cgroup detection: + ```bash + grep "container resources" /logs/rustfs.log + ``` + +3. Use environment variable override as workaround: + ```bash + export RUSTFS_OVERRIDE_MEMORY_BYTES=1073741824 + ``` + +## Implementation Details + +### Files Modified + +- `rustfs/src/cgroup_resources.rs` - Core cgroup detection logic +- `rustfs/src/container_config.rs` - Container configuration with overrides +- `rustfs/src/memory_observability.rs` - Updated memory metrics +- `rustfs/src/server/runtime.rs` - Updated Tokio runtime configuration +- `rustfs/src/startup_entrypoint.rs` - Startup logging + +### Performance Impact + +- **Startup**: One-time detection adds ~1ms overhead +- **Runtime**: Cached values, no repeated filesystem reads +- **Memory**: Negligible (<1KB for cached values) + +### Thread Safety + +All detection functions are thread-safe and use `OnceLock` for caching. diff --git a/rustfs/src/cgroup_resources.rs b/rustfs/src/cgroup_resources.rs new file mode 100644 index 000000000..cce2aa150 --- /dev/null +++ b/rustfs/src/cgroup_resources.rs @@ -0,0 +1,361 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Cgroup-aware resource detection for containerized environments. +//! +//! This module provides a single source of truth for detecting and applying +//! container resource limits (CPU and memory) from cgroup v1/v2. +//! +//! # Platform Support +//! +//! - **Linux**: Full cgroup v1/v2 support +//! - **macOS/Windows**: Falls back to host values (cgroup not available) +//! +//! # Design Principles +//! +//! - **Single source of truth**: All cgroup detection logic lives here +//! - **Thread-safe**: Results are cached in `OnceLock` after first detection +//! - **Zero-cost fallback**: Non-Linux platforms return host values immediately +//! - **Environment overrides**: Operators can override detected values via env vars + +use std::sync::OnceLock; + +// ============================================================================ +// Environment variable constants +// ============================================================================ + +/// Disable cgroup detection entirely (for testing or special cases). +const ENV_DISABLE_CGROUP_DETECTION: &str = "RUSTFS_DISABLE_CGROUP_DETECTION"; + +/// Override detected CPU cores (takes precedence over cgroup). +const ENV_OVERRIDE_CPU_CORES: &str = "RUSTFS_OVERRIDE_CPU_CORES"; + +/// Override detected memory limit in bytes (takes precedence over cgroup). +const ENV_OVERRIDE_MEMORY_BYTES: &str = "RUSTFS_OVERRIDE_MEMORY_BYTES"; + +// ============================================================================ +// Core types +// ============================================================================ + +/// Cached container resource configuration (computed once at startup). +static CONTAINER_RESOURCES: OnceLock = OnceLock::new(); + +/// Detected container resource limits with effective values. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ContainerResources { + /// Effective CPU cores (considering cgroup limits, overrides, and host). + pub cpu_cores: usize, + /// Effective memory limit in bytes (considering cgroup limits, overrides, and host). + pub memory_bytes: u64, + /// Whether cgroup limits were detected. + pub cgroup_detected: bool, + /// Whether values were overridden by environment variables. + pub overridden: bool, +} + +impl Default for ContainerResources { + fn default() -> Self { + Self { + cpu_cores: 1, + memory_bytes: 0, + cgroup_detected: false, + overridden: false, + } + } +} + +// ============================================================================ +// Linux cgroup detection +// ============================================================================ + +#[cfg(target_os = "linux")] +mod cgroup { + /// Cgroup v2 CPU limit path + const CGROUP_V2_CPU_MAX_PATH: &str = "/sys/fs/cgroup/cpu.max"; + /// Cgroup v1 CPU quota path + const CGROUP_V1_CPU_QUOTA_PATH: &str = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"; + /// Cgroup v1 CPU period path + const CGROUP_V1_CPU_PERIOD_PATH: &str = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"; + /// Cgroup v2 memory limit path + const CGROUP_V2_MEMORY_MAX_PATH: &str = "/sys/fs/cgroup/memory.max"; + /// Cgroup v1 memory limit path + const CGROUP_V1_MEMORY_LIMIT_PATH: &str = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; + + /// Read a file content and parse as u64, returning None on any error. + fn read_u64_from_file(path: &str) -> Option { + let content = std::fs::read_to_string(path).ok()?; + let trimmed = content.trim(); + if trimmed.is_empty() || trimmed == "max" { + return None; + } + trimmed.parse::().ok() + } + + /// Detect CPU cores from cgroup v2 (`/sys/fs/cgroup/cpu.max`). + /// + /// Format: `"max"` (unlimited) or `"$QUOTA $PERIOD"` or `"$QUOTA"` (implicit period=100000). + fn detect_cgroup_v2_cpus() -> Option { + let content = std::fs::read_to_string(CGROUP_V2_CPU_MAX_PATH).ok()?; + let parts: Vec<&str> = content.split_whitespace().collect(); + + match parts.len() { + 1 => { + if parts[0] == "max" { + return None; + } + let quota = parts[0].parse::().ok()?; + if quota == 0 { + return None; + } + Some(quota.div_ceil(100_000) as usize) + } + 2 => { + if parts[0] == "max" { + return None; + } + let quota = parts[0].parse::().ok()?; + let period = parts[1].parse::().ok()?; + if quota == 0 || period == 0 { + return None; + } + Some(quota.div_ceil(period) as usize) + } + _ => None, + } + } + + /// Detect CPU cores from cgroup v1 (`/sys/fs/cgroup/cpu/cpu.cfs_quota_us`). + fn detect_cgroup_v1_cpus() -> Option { + let quota = read_u64_from_file(CGROUP_V1_CPU_QUOTA_PATH)?; + if quota == 0 || quota == u64::MAX { + return None; + } + let period = read_u64_from_file(CGROUP_V1_CPU_PERIOD_PATH).unwrap_or(100_000); + if period == 0 { + return None; + } + Some(quota.div_ceil(period) as usize) + } + + /// Detect CPU cores from cgroup, trying v2 first, then v1. + pub(super) fn detect_cpus() -> Option { + detect_cgroup_v2_cpus().or_else(detect_cgroup_v1_cpus) + } + + /// Detect memory limit from cgroup v2 (`/sys/fs/cgroup/memory.max`). + fn detect_cgroup_v2_memory() -> Option { + let content = std::fs::read_to_string(CGROUP_V2_MEMORY_MAX_PATH).ok()?; + let trimmed = content.trim(); + if trimmed == "max" || trimmed.is_empty() { + return None; + } + trimmed.parse::().ok() + } + + /// Detect memory limit from cgroup v1 (`/sys/fs/cgroup/memory/memory.limit_in_bytes`). + fn detect_cgroup_v1_memory() -> Option { + let limit = read_u64_from_file(CGROUP_V1_MEMORY_LIMIT_PATH)?; + // cgroup v1 uses a very large value (2^63 or similar) to indicate "unlimited" + if limit >= (1 << 62) { + return None; + } + Some(limit) + } + + /// Detect memory limit from cgroup, trying v2 first, then v1. + pub(super) fn detect_memory() -> Option { + detect_cgroup_v2_memory().or_else(detect_cgroup_v1_memory) + } +} + +// ============================================================================ +// Non-Linux fallback +// ============================================================================ + +#[cfg(not(target_os = "linux"))] +mod cgroup { + /// Cgroup detection not available on non-Linux platforms. + pub(super) fn detect_cpus() -> Option { + None + } + + /// Cgroup detection not available on non-Linux platforms. + pub(super) fn detect_memory() -> Option { + None + } +} + +// ============================================================================ +// Resource detection and caching +// ============================================================================ + +/// Detect container resources (called once and cached). +/// +/// Detection priority: +/// 1. Environment variable overrides +/// 2. cgroup v1/v2 limits (Linux only) +/// 3. Host values from sysinfo +fn detect_container_resources() -> ContainerResources { + // Check if cgroup detection is disabled + let cgroup_disabled = std::env::var(ENV_DISABLE_CGROUP_DETECTION) + .map(|v| v == "1" || v.to_lowercase() == "true") + .unwrap_or(false); + + // Detect cgroup limits if not disabled + let (cgroup_cpus, cgroup_memory) = if cgroup_disabled { + (None, None) + } else { + (cgroup::detect_cpus(), cgroup::detect_memory()) + }; + + let cgroup_detected = cgroup_cpus.is_some() || cgroup_memory.is_some(); + + // Check for environment variable overrides + let override_cores = std::env::var(ENV_OVERRIDE_CPU_CORES) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v > 0); + + let override_memory = std::env::var(ENV_OVERRIDE_MEMORY_BYTES) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&v| v > 0); + + let overridden = override_cores.is_some() || override_memory.is_some(); + + // Get host values for fallback + let host_cores = { + let mut sys = + sysinfo::System::new_with_specifics(sysinfo::RefreshKind::everything().without_memory().without_processes()); + sys.refresh_cpu_all(); + sys.cpus().len().max(1) + }; + + let host_memory = { + let mut sys = sysinfo::System::new(); + sys.refresh_memory(); + sys.total_memory() + }; + + // Determine effective values: override > cgroup > host + let cpu_cores = override_cores.or(cgroup_cpus).unwrap_or(host_cores).max(1); + + let memory_bytes = override_memory.or(cgroup_memory).unwrap_or(host_memory); + + ContainerResources { + cpu_cores, + memory_bytes, + cgroup_detected, + overridden, + } +} + +/// Get cached container resource limits. +/// +/// This function is thread-safe. Detection is performed once and cached. +pub fn container_resources() -> &'static ContainerResources { + CONTAINER_RESOURCES.get_or_init(detect_container_resources) +} + +/// Log container resource configuration at startup. +/// +/// Should be called once during startup to help operators verify detection. +pub fn log_container_resources() { + let res = container_resources(); + + if res.overridden { + tracing::info!( + cpu_cores = res.cpu_cores, + memory_bytes = res.memory_bytes, + memory_mib = res.memory_bytes / (1024 * 1024), + cgroup_detected = res.cgroup_detected, + "container resources (overridden by environment variables)" + ); + } else if res.cgroup_detected { + tracing::info!( + cpu_cores = res.cpu_cores, + memory_bytes = res.memory_bytes, + memory_mib = res.memory_bytes / (1024 * 1024), + "container resources (detected from cgroup)" + ); + } else { + tracing::debug!( + cpu_cores = res.cpu_cores, + memory_bytes = res.memory_bytes, + "container resources (using host values)" + ); + } +} + +/// Get the memory basis string for metrics. +pub fn memory_basis() -> &'static str { + let res = container_resources(); + if res.cgroup_detected { "cgroup" } else { "host" } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_container_resources_default() { + let resources = ContainerResources::default(); + assert_eq!(resources.cpu_cores, 1); + assert_eq!(resources.memory_bytes, 0); + assert!(!resources.cgroup_detected); + assert!(!resources.overridden); + } + + #[test] + fn test_container_resources_cached() { + let res1 = container_resources(); + let res2 = container_resources(); + assert_eq!(res1, res2); + } + + #[test] + fn test_cpu_cores_minimum_one() { + // Even with 0 host cores, should return at least 1 + let res = container_resources(); + assert!(res.cpu_cores >= 1); + } + + #[test] + fn test_memory_bytes_positive() { + let res = container_resources(); + assert!(res.memory_bytes > 0); + } + + #[cfg(target_os = "linux")] + #[test] + fn test_linux_cgroup_detection() { + let res = container_resources(); + // On Linux, cgroup might or might not be available + // Just verify the struct is valid + assert!(res.cpu_cores >= 1); + assert!(res.memory_bytes > 0); + } + + #[cfg(not(target_os = "linux"))] + #[test] + fn test_non_linux_cgroup_detection() { + let res = container_resources(); + assert!(!res.cgroup_detected); + assert!(!res.overridden); + } +} diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index a1836f4f3..64c4dbf05 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -78,6 +78,7 @@ pub mod auth; pub mod auth_keystone; pub(crate) mod bitrot_selftest; pub mod capacity; +pub mod cgroup_resources; pub mod cluster_snapshot; pub mod config; pub mod connect; diff --git a/rustfs/src/memory_observability.rs b/rustfs/src/memory_observability.rs index 44c64444f..7b3bfb285 100644 --- a/rustfs/src/memory_observability.rs +++ b/rustfs/src/memory_observability.rs @@ -20,14 +20,11 @@ use serde::Serialize; #[cfg(any(not(target_os = "windows"), test))] use serde_json::Value; use std::path::Path; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{Arc, Mutex}; use std::time::Duration; -use sysinfo::System; use tokio_util::sync::CancellationToken; use tracing::debug; -static MEMORY_SYSTEM: OnceLock> = OnceLock::new(); - const ENV_MEMORY_OBSERVABILITY_INTERVAL_SECS: &str = "RUSTFS_MEMORY_OBSERVABILITY_INTERVAL_SECS"; const DEFAULT_MEMORY_OBSERVABILITY_INTERVAL_SECS: u64 = 15; const MEMORY_OBSERVABILITY_SERVICE_NAME: &str = "memory_observability"; @@ -154,14 +151,11 @@ struct AllocatorMemorySnapshot { observation: AllocatorMemoryObservation, } -fn memory_system() -> &'static Mutex { - MEMORY_SYSTEM.get_or_init(|| Mutex::new(System::new())) -} - -fn refresh_total_memory() -> u64 { - let mut system = memory_system().lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - system.refresh_memory(); - system.total_memory() +/// Get effective total memory from container resources. +/// +/// Returns the effective memory (considering cgroup limits and overrides). +fn refresh_effective_memory() -> u64 { + crate::cgroup_resources::container_resources().memory_bytes } fn read_optional_u64(path: &Path) -> Option { @@ -395,19 +389,36 @@ pub fn memory_observability_controller_snapshot(ctx: &CancellationToken) -> Memo ) } +/// Record the effective memory total and its basis (host or cgroup). +fn record_effective_memory(total_bytes: u64) { + let basis = crate::cgroup_resources::memory_basis(); + metrics::gauge!("rustfs_memory_effective_total_bytes", "basis" => basis.to_string()).set(total_bytes as f64); +} + +/// Record container resource detection results. +fn record_container_resource_detection() { + let res = crate::cgroup_resources::container_resources(); + + metrics::gauge!("rustfs_container_cpu_cores").set(res.cpu_cores as f64); + metrics::gauge!("rustfs_container_memory_bytes").set(res.memory_bytes as f64); + metrics::gauge!("rustfs_container_cgroup_detected").set(if res.cgroup_detected { 1.0 } else { 0.0 }); + metrics::gauge!("rustfs_container_overridden").set(if res.overridden { 1.0 } else { 0.0 }); +} + async fn record_memory_snapshot(process_sampler: Arc>) { match tokio::task::spawn_blocking(move || { let mut sampler = process_sampler.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); let (resource, process) = sampler.snapshot_resource_and_system(); - let total_memory = refresh_total_memory(); + let effective_memory = refresh_effective_memory(); let cgroup = read_cgroup_memory_snapshot(); let allocator = read_allocator_memory_snapshot(); - (resource, process, total_memory, cgroup, allocator) + (resource, process, effective_memory, cgroup, allocator) }) .await { - Ok((resource, process, total_memory, cgroup, allocator)) => { - record_memory_usage(process.resident_memory_bytes, total_memory); + Ok((resource, process, effective_memory, cgroup, allocator)) => { + record_memory_usage(process.resident_memory_bytes, effective_memory); + record_effective_memory(effective_memory); record_cpu_usage(resource.cpu_percent); record_process_memory_split(process.resident_memory_bytes, process.virtual_memory_bytes); @@ -437,6 +448,9 @@ pub fn init_memory_observability(ctx: CancellationToken) { let interval = Duration::from_secs(interval_secs.max(1)); let process_sampler = Arc::new(Mutex::new(ProcessSampler::new())); + // Record container resource detection results at startup + record_container_resource_detection(); + tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs index 8d7f95ba7..f5735b436 100644 --- a/rustfs/src/server/runtime.rs +++ b/rustfs/src/server/runtime.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::time::Duration; -use sysinfo::{RefreshKind, System}; use rustfs_obs::dial9::Dial9SessionGuard; @@ -52,24 +51,30 @@ mod tests { #[inline] fn detect_cores() -> usize { - // Priority physical cores, fallback logic cores, minimum 1 - let mut sys = System::new_with_specifics(RefreshKind::everything().without_memory().without_processes()); - sys.refresh_cpu_all(); - sys.cpus().len().max(1) + // Uses cgroup-aware detection from cgroup_resources module + // Returns effective CPU cores considering cgroup limits and overrides + crate::cgroup_resources::container_resources().cpu_cores } #[inline] fn compute_default_worker_threads() -> usize { // Physical cores are used by default (closer to CPU compute resources and cache topology) + // Now cgroup-aware: in containers, uses the container's CPU limit detect_cores() } /// Default max_blocking_threads calculations based on sysinfo: /// 16 cores -> 1024; more than 16 cores are doubled by multiples: /// 1..=16 -> 1024, 17..=32 -> 2048, 33..=64 -> 4096, and so on. +/// +/// For containerized environments with limited resources, this is capped +/// to prevent excessive memory usage from thread stacks. fn compute_default_max_blocking_threads() -> usize { const BASE_CORES: usize = rustfs_config::DEFAULT_WORKER_THREADS; const BASE_THREADS: usize = rustfs_config::DEFAULT_MAX_BLOCKING_THREADS; + // Cap for small containers to prevent excessive memory usage + // Each blocking thread can use up to 1 MiB stack space + const SMALL_CONTAINER_MAX_THREADS: usize = 256; let cores = detect_cores(); @@ -82,6 +87,12 @@ fn compute_default_max_blocking_threads() -> usize { threshold = threshold.saturating_mul(2); } + // For small containers (<=4 cores), cap the blocking threads to prevent memory issues + // This prevents a 1 GiB container from allocating up to 1 GiB just for thread stacks + if cores <= 4 { + threads = threads.min(SMALL_CONTAINER_MAX_THREADS); + } + threads } diff --git a/rustfs/src/startup_entrypoint.rs b/rustfs/src/startup_entrypoint.rs index cad258689..658c41ad5 100644 --- a/rustfs/src/startup_entrypoint.rs +++ b/rustfs/src/startup_entrypoint.rs @@ -64,6 +64,10 @@ fn emit_fatal_stderr(context: &str, error: impl std::fmt::Display) { async fn async_main() -> Result<()> { hotpath::tokio_runtime!(); + // Log container resource detection early in startup + // This helps operators verify that RustFS correctly detected cgroup limits + crate::cgroup_resources::log_container_resources(); + let env_compat_report = bootstrap_external_prefix_compat()?; // Parse command line arguments