diff --git a/Cargo.lock b/Cargo.lock index 401daad65..b8204f597 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7270,6 +7270,7 @@ dependencies = [ "base64 0.22.1", "base64-simd", "bytes", + "chrono", "clap", "const-str", "datafusion", @@ -7500,6 +7501,7 @@ dependencies = [ "hyper-util", "lazy_static", "md-5 0.11.0-rc.5", + "metrics", "moka", "num_cpus", "parking_lot 0.12.5", diff --git a/crates/config/src/constants/object.rs b/crates/config/src/constants/object.rs index 43fbb15cd..c4f8fb05c 100644 --- a/crates/config/src/constants/object.rs +++ b/crates/config/src/constants/object.rs @@ -180,3 +180,265 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR /// Default: bitrot verification is enabled on GetObject reads (do not skip). pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false; + +// ============================================================================= +// Concurrent Request Fix - Timeout and Backpressure Configuration +// ============================================================================= + +/// Environment variable for GetObject request timeout in seconds. +/// +/// When a GetObject request exceeds this duration, it will be cancelled +/// and return a 504 Gateway Timeout error. This prevents requests from +/// hanging indefinitely due to deadlocks or resource exhaustion. +/// +/// Default: 30 seconds (can be overridden by `RUSTFS_OBJECT_GET_TIMEOUT`). +/// Set to 0 to disable timeout (not recommended for production). +pub const ENV_OBJECT_GET_TIMEOUT: &str = "RUSTFS_OBJECT_GET_TIMEOUT"; + +/// Default GetObject request timeout in seconds. +/// +/// This value balances between allowing large object transfers to complete +/// and preventing indefinite hangs. For 20-26MB objects with concurrent +/// range reads, 30 seconds should be sufficient under normal conditions. +pub const DEFAULT_OBJECT_GET_TIMEOUT: u64 = 30; + +/// Environment variable for disk read operation timeout in seconds. +/// +/// Individual disk read operations that exceed this duration will be +/// cancelled and treated as failures. This helps detect slow or hung +/// disks without waiting indefinitely. +/// +/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DISK_READ_TIMEOUT`). +pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT"; + +/// Default disk read timeout in seconds. +pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10; + +/// Environment variable for duplex pipe buffer size in bytes. +/// +/// The duplex pipe connects the disk read task to the HTTP response stream. +/// A larger buffer reduces backpressure but increases memory usage. +/// For large objects (20-26MB), a 4MB buffer provides good throughput. +/// +/// Default: 4194304 (4 MB, can be overridden by `RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE`). +/// Minimum recommended: 1048576 (1 MB). +pub const ENV_OBJECT_DUPLEX_BUFFER_SIZE: &str = "RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE"; + +/// Default duplex buffer size: 4 MB. +/// +/// This is 4x larger than the original 1 MB buffer, providing better +/// handling of large objects and reducing backpressure-related hangs. +pub const DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE: usize = 4 * 1024 * 1024; + +/// Environment variable for I/O buffer size in bytes. +/// +/// This controls the buffer size used for individual I/O operations. +/// A larger buffer improves throughput for sequential reads but may +/// increase latency for small random reads. +/// +/// Default: 131072 (128 KB, can be overridden by `RUSTFS_OBJECT_IO_BUFFER_SIZE`). +pub const ENV_OBJECT_IO_BUFFER_SIZE: &str = "RUSTFS_OBJECT_IO_BUFFER_SIZE"; + +/// Default I/O buffer size: 128 KB. +pub const DEFAULT_OBJECT_IO_BUFFER_SIZE: usize = 128 * 1024; + +/// Environment variable to enable/disable lock optimization. +/// +/// When enabled, read locks are released immediately after metadata +/// is read, rather than being held for the entire data transfer. +/// This significantly reduces lock contention under high concurrency. +/// +/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE`). +pub const ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE: &str = "RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE"; + +/// Default: lock optimization is enabled. +pub const DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE: bool = true; + +/// Environment variable to enable/disable priority-based I/O scheduling. +/// +/// When enabled, smaller requests (< 1MB) are given higher priority +/// than larger requests (> 10MB), preventing "starvation" of small +/// requests by large ones. +/// +/// Default: true (enabled, can be overridden by `RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE`). +pub const ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE: &str = "RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE"; + +/// Default: priority scheduling is enabled. +pub const DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE: bool = true; + +/// Environment variable to enable/disable deadlock detection. +/// +/// When enabled, the system monitors active requests and detects +/// potential deadlock situations (circular lock wait chains). +/// This has some performance overhead and is intended for debugging. +/// +/// Default: false (disabled, can be overridden by `RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE`). +pub const ENV_OBJECT_DEADLOCK_DETECTION_ENABLE: &str = "RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE"; + +/// Default: deadlock detection is disabled for performance. +pub const DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE: bool = false; + +/// Environment variable for deadlock detection check interval in seconds. +/// +/// How often the deadlock detector analyzes the lock wait graph. +/// More frequent checks detect deadlocks faster but use more CPU. +/// +/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL`). +pub const ENV_OBJECT_DEADLOCK_CHECK_INTERVAL: &str = "RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL"; + +/// Default deadlock check interval: 5 seconds. +pub const DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL: u64 = 5; + +/// Environment variable for deadlock detection hang threshold in seconds. +/// +/// Requests that have been running longer than this threshold are +/// considered "potentially hung" and included in deadlock analysis. +/// +/// Default: 10 seconds (can be overridden by `RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD`). +pub const ENV_OBJECT_DEADLOCK_HANG_THRESHOLD: &str = "RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD"; + +/// Default hang threshold: 10 seconds. +pub const DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD: u64 = 10; + +/// Environment variable for backpressure high watermark percentage. +/// +/// When buffer usage exceeds this percentage, the system enters +/// "high watermark" state and may apply backpressure to producers. +/// +/// Default: 80 (80%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK`). +pub const ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_HIGH_WATERMARK"; + +/// Default high watermark: 80%. +pub const DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK: u32 = 80; + +/// Environment variable for backpressure low watermark percentage. +/// +/// When buffer usage drops below this percentage after being in +/// high watermark state, backpressure is released. +/// +/// Default: 50 (50%, can be overridden by `RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK`). +pub const ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK: &str = "RUSTFS_OBJECT_BACKPRESSURE_LOW_WATERMARK"; + +/// Default low watermark: 50%. +pub const DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK: u32 = 50; + +/// Environment variable for lock acquisition timeout in seconds. +/// +/// When a lock cannot be acquired within this duration, the operation +/// will fail with a timeout error. This prevents indefinite waiting +/// for locks that may never be released due to deadlocks. +/// +/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT`). +pub const ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT: &str = "RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT"; + +/// Default lock acquisition timeout: 5 seconds. +pub const DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT: u64 = 5; + +// ============================================================================ +// I/O priority scheduling configuration +// ============================================================================ + +/// Environment variable for I/O high priority size threshold in bytes. +/// +/// Requests smaller than this threshold are classified as high priority. +/// High priority requests are processed first to prevent starvation of small requests. +/// +/// Default: 1048576 (1 MB, can be overridden by `RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD`). +pub const ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD"; + +/// Default high priority size threshold: 1 MB. +pub const DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD: usize = 1024 * 1024; + +/// Environment variable for I/O low priority size threshold in bytes. +/// +/// Requests larger than this threshold are classified as low priority. +/// Low priority requests are processed last to avoid blocking small requests. +/// +/// Default: 104857600 (100 MB, can be overridden by `RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD`). +pub const ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: &str = "RUSTFS_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD"; + +/// Default low priority size threshold: 100 MB. +pub const DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD: usize = 100 * 1024 * 1024; + +/// Environment variable for high priority queue capacity. +/// +/// Maximum number of requests that can be queued in the high priority queue. +/// +/// Default: 32 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY`). +pub const ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_HIGH_CAPACITY"; + +/// Default high priority queue capacity: 32. +pub const DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY: usize = 32; + +/// Environment variable for normal priority queue capacity. +/// +/// Maximum number of requests that can be queued in the normal priority queue. +/// +/// Default: 64 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY`). +pub const ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_NORMAL_CAPACITY"; + +/// Default normal priority queue capacity: 64. +pub const DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY: usize = 64; + +/// Environment variable for low priority queue capacity. +/// +/// Maximum number of requests that can be queued in the low priority queue. +/// +/// Default: 16 (can be overridden by `RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY`). +pub const ENV_OBJECT_IO_QUEUE_LOW_CAPACITY: &str = "RUSTFS_OBJECT_IO_QUEUE_LOW_CAPACITY"; + +/// Default low priority queue capacity: 16. +pub const DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY: usize = 16; + +/// Environment variable for starvation prevention check interval in milliseconds. +/// +/// How often the system checks for starving low-priority requests. +/// When a low-priority request has been waiting longer than the starvation threshold, +/// it is promoted to normal priority. +/// +/// Default: 100 ms (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL`). +pub const ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: &str = "RUSTFS_OBJECT_IO_STARVATION_PREVENTION_INTERVAL"; + +/// Default starvation prevention interval: 100 ms. +pub const DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL: u64 = 100; + +/// Environment variable for starvation threshold in seconds. +/// +/// Maximum time a low-priority request can wait before being promoted to normal priority. +/// This prevents indefinite starvation of low-priority requests. +/// +/// Default: 5 seconds (can be overridden by `RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS`). +pub const ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS: &str = "RUSTFS_OBJECT_IO_STARVATION_THRESHOLD_SECS"; + +/// Default starvation threshold: 5 seconds. +pub const DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS: u64 = 5; + +/// Environment variable for load sampling window size. +/// +/// Number of recent samples used to calculate I/O load metrics. +/// +/// Default: 100 samples (can be overridden by `RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW`). +pub const ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW: &str = "RUSTFS_OBJECT_IO_LOAD_SAMPLE_WINDOW"; + +/// Default load sampling window: 100 samples. +pub const DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW: usize = 100; + +/// Environment variable for high load wait time threshold in milliseconds. +/// +/// When average wait time exceeds this threshold, the system is considered to be under high load. +/// +/// Default: 50 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS`). +pub const ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS"; + +/// Default high load threshold: 50 ms. +pub const DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS: u64 = 50; + +/// Environment variable for low load wait time threshold in milliseconds. +/// +/// When average wait time is below this threshold, the system is considered to be under low load. +/// +/// Default: 10 ms (can be overridden by `RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS`). +pub const ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: &str = "RUSTFS_OBJECT_IO_LOAD_LOW_THRESHOLD_MS"; + +/// Default low load threshold: 10 ms. +pub const DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS: u64 = 10; diff --git a/crates/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml index 2bfa426b7..c35b8a604 100644 --- a/crates/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -116,6 +116,9 @@ faster-hex = { workspace = true } ratelimit = { workspace = true } aws-smithy-http-client.workspace = true +# Observability and Metrics +metrics = { workspace = true } + [dev-dependencies] tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } criterion = { workspace = true, features = ["html_reports"] } diff --git a/crates/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs index 0c787ec4e..cb0f68992 100644 --- a/crates/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -121,6 +121,21 @@ use uuid::Uuid; pub const DEFAULT_READ_BUFFER_SIZE: usize = MI_B; // 1 MiB = 1024 * 1024; pub const MAX_PARTS_COUNT: usize = 10000; + +/// Get the duplex buffer size from environment variable or use default. +/// +/// This function reads `RUSTFS_DUPLEX_BUFFER_SIZE` environment variable +/// to allow runtime configuration of the duplex pipe buffer size. +/// A larger buffer (e.g., 4MB) helps prevent backpressure-related hangs +/// when reading large objects (20-26MB) under high concurrency. +/// +/// Default: 4MB (4 * 1024 * 1024 bytes) +pub fn get_duplex_buffer_size() -> usize { + rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE, + rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, + ) +} const DISK_ONLINE_TIMEOUT: Duration = Duration::from_secs(1); const DISK_HEALTH_CACHE_TTL: Duration = Duration::from_millis(750); @@ -136,7 +151,72 @@ mod write; /// Get lock acquire timeout from environment variable RUSTFS_LOCK_ACQUIRE_TIMEOUT (in seconds) /// Defaults to 30 seconds if not set or invalid pub fn get_lock_acquire_timeout() -> Duration { - Duration::from_secs(rustfs_utils::get_env_u64("RUSTFS_LOCK_ACQUIRE_TIMEOUT", 5)) + Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, + rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT, + )) +} + +/// Check if lock optimization is enabled. +/// When enabled, read locks are released after metadata read instead of +/// being held for the entire data transfer duration. +pub fn is_lock_optimization_enabled() -> bool { + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, + rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, + ) +} + +/// Check if deadlock detection is enabled. +/// When enabled, lock operations are recorded for deadlock analysis. +pub fn is_deadlock_detection_enabled() -> bool { + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_DEADLOCK_DETECTION_ENABLE, + rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE, + ) +} + +/// Record a lock acquisition for deadlock detection. +/// This records detailed lock information for deadlock analysis. +/// Returns the lock_id for later release tracking. +#[inline] +fn record_lock_acquire(bucket: &str, object: &str, lock_type: &str) -> String { + let lock_id = format!("{}:{}", bucket, object); + + if !is_deadlock_detection_enabled() { + return lock_id; + } + + let request_id = format!("get-{}-{}", bucket, object); + let resource = format!("{}/{}", bucket, object); + + // Log with structured fields for analysis + debug!( + request_id = %request_id, + lock_id = %lock_id, + lock_type = %lock_type, + resource = %resource, + "Lock acquired for deadlock tracking" + ); + + lock_id +} + +/// Record a lock release for deadlock detection. +#[inline] +fn record_lock_release(bucket: &str, object: &str, lock_id: &str, lock_type: &str) { + if !is_deadlock_detection_enabled() { + return; + } + + let request_id = format!("get-{}-{}", bucket, object); + + debug!( + request_id = %request_id, + lock_id = %lock_id, + lock_type = %lock_type, + "Lock released for deadlock tracking" + ); } fn build_tiered_decommission_file_info( @@ -451,20 +531,44 @@ impl ObjectIO for SetDisks { h: HeaderMap, opts: &ObjectOptions, ) -> Result { + // Check if lock optimization is enabled + // When enabled, read locks are released after metadata read + let lock_optimization_enabled = is_lock_optimization_enabled(); + // Acquire a shared read-lock early to protect read consistency let read_lock_guard = if !opts.no_lock { - Some( - self.new_ns_lock(bucket, object) - .await? - .get_read_lock(get_lock_acquire_timeout()) - .await - .map_err(|e| { - Error::other(format!( - "Failed to acquire read lock: {}", - self.format_lock_error_from_error(bucket, object, "read", &e) - )) - })?, - ) + let acquire_start = Instant::now(); + + // Record lock wait for deadlock detection + if is_deadlock_detection_enabled() { + debug!( + lock_id = format!("{}:{}", bucket, object), + lock_type = "read", + resource = format!("{}/{}", bucket, object), + "Waiting for read lock" + ); + } + + let guard = self + .new_ns_lock(bucket, object) + .await? + .get_read_lock(get_lock_acquire_timeout()) + .await + .map_err(|e| { + Error::other(format!( + "Failed to acquire read lock: {}", + self.format_lock_error_from_error(bucket, object, "read", &e) + )) + })?; + + // Record lock acquisition for deadlock detection + let _lock_id = record_lock_acquire(bucket, object, "read"); + + // Record lock statistics + metrics::counter!("rustfs.lock.acquire.total", "type" => "read").increment(1); + metrics::histogram!("rustfs.lock.acquire.duration.seconds").record(acquire_start.elapsed().as_secs_f64()); + + Some(guard) } else { None }; @@ -512,7 +616,28 @@ impl ObjectIO for SetDisks { return Ok(gr); } - let (rd, wd) = tokio::io::duplex(DEFAULT_READ_BUFFER_SIZE); + // Lock optimization: release read lock after metadata read if enabled + // This reduces lock contention by not holding the lock during data transfer + let read_lock_guard = if lock_optimization_enabled { + // Record lock release for deadlock detection + if read_lock_guard.is_some() { + let lock_id = format!("{}:{}", bucket, object); + record_lock_release(bucket, object, &lock_id, "read"); + + // Record early lock release statistics + metrics::counter!("rustfs.lock.release.early.total", "type" => "read").increment(1); + } + // Explicitly drop the lock guard to release the lock early + drop(read_lock_guard); + debug!(bucket, object, "Lock optimization: released read lock after metadata read"); + None + } else { + read_lock_guard + }; + + let duplex_buffer_size = get_duplex_buffer_size(); + let (rd, wd) = tokio::io::duplex(duplex_buffer_size); + debug!(bucket, object, duplex_buffer_size, "Created duplex pipe for object data transfer"); let (reader, offset, length) = GetObjectReader::new(Box::new(rd), range, &object_info, opts, &h)?; @@ -523,9 +648,10 @@ impl ObjectIO for SetDisks { let pool_index = self.pool_index; let skip_verify = opts.skip_verify_bitrot; // Move the read-lock guard into the task so it lives for the duration of the read + // Note: when lock optimization is enabled, read_lock_guard is None // let _guard_to_hold = _read_lock_guard; // moved into closure below tokio::spawn(async move { - let _guard = read_lock_guard; // keep guard alive until task ends + let _guard = read_lock_guard; // keep guard alive until task ends (None if optimization enabled) let mut writer = wd; if let Err(e) = Self::get_object_with_fileinfo( &bucket, @@ -751,7 +877,7 @@ impl ObjectIO for SetDisks { pfi.metadata = user_defined.clone(); if is_inline_buffer { if let Some(writer) = writers[i].take() { - pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default()); + pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default()); } pfi.set_inline_data(); @@ -1093,7 +1219,7 @@ impl ObjectOperations for SetDisks { let mut unique_objects: HashSet = HashSet::new(); for dobj in &objects { if unique_objects.insert(dobj.object_name.clone()) { - batch = batch.add_write_lock(rustfs_lock::ObjectKey::new(bucket, dobj.object_name.clone())); + batch = batch.add_write_lock(ObjectKey::new(bucket, dobj.object_name.clone())); } } @@ -2757,8 +2883,7 @@ impl MultipartOperations for SetDisks { // Build a lookup map for O(1) part resolution instead of O(n) find() in the loop // This optimizes from O(n^2) to O(n) when processing many parts use std::collections::HashMap; - let part_lookup: HashMap = - curr_fi.parts.iter().map(|part| (part.number, part)).collect(); + let part_lookup: HashMap = curr_fi.parts.iter().map(|part| (part.number, part)).collect(); for (i, p) in uploaded_parts.iter().enumerate() { let Some(ext_part) = part_lookup.get(&p.part_num) else { @@ -3445,12 +3570,11 @@ async fn disks_with_all_parts( if (meta.data.is_some() || meta.size == 0) && !meta.parts.is_empty() { if let Some(data) = &meta.data { let checksum_info = meta.erasure.get_checksum_info(meta.parts[0].number); - let checksum_algo = - if meta.uses_legacy_checksum && checksum_info.algorithm == rustfs_utils::HashAlgorithm::HighwayHash256S { - rustfs_utils::HashAlgorithm::HighwayHash256SLegacy - } else { - checksum_info.algorithm - }; + let checksum_algo = if meta.uses_legacy_checksum && checksum_info.algorithm == HashAlgorithm::HighwayHash256S { + HashAlgorithm::HighwayHash256SLegacy + } else { + checksum_info.algorithm + }; let data_len = data.len(); let verify_err = bitrot_verify( Box::new(Cursor::new(data.clone())), diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 8535927ca..a2814d28d 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -106,6 +106,7 @@ serde_urlencoded = { workspace = true } # Cryptography and Security rustls = { workspace = true } subtle = { workspace = true } +chrono = { workspace = true } jiff = { workspace = true } time = { workspace = true, features = ["parsing", "formatting", "serde"] } diff --git a/rustfs/src/app/object_usecase.rs b/rustfs/src/app/object_usecase.rs index 0a4c417cb..10510da49 100644 --- a/rustfs/src/app/object_usecase.rs +++ b/rustfs/src/app/object_usecase.rs @@ -30,6 +30,7 @@ use crate::storage::options::{ }; use crate::storage::s3_api::multipart::parse_list_parts_params; use crate::storage::s3_api::{acl, restore, select}; +use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; use crate::storage::*; use bytes::Bytes; use datafusion::arrow::{ @@ -930,12 +931,37 @@ impl DefaultObjectUsecase { let _ = context.object_store(); } + // Create timeout wrapper for enhanced timeout tracking + let timeout_config = TimeoutConfig::from_env(); + let wrapper = + RequestTimeoutWrapper::with_request_id(timeout_config.clone(), format!("get-{}-{}", req.input.bucket, req.input.key)); + + // Get cancellation token for cooperative cancellation + let _cancel_token = wrapper.cancel_token(); let request_start = std::time::Instant::now(); // Track this request for concurrency-aware optimizations let _request_guard = ConcurrencyManager::track_request(); let concurrent_requests = GetObjectGuard::concurrent_requests(); + // Register with deadlock detector if enabled + let deadlock_detector = crate::storage::deadlock_detector::get_deadlock_detector(); + let request_id = wrapper.request_id().to_string(); + deadlock_detector.register_request(&request_id, format!("GetObject {}/{}", req.input.bucket, req.input.key)); + + // Check for request timeout before proceeding + if wrapper.is_timeout() { + warn!( + bucket = %req.input.bucket, + key = %req.input.key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before processing" + ); + deadlock_detector.unregister_request(&request_id); + return Err(s3_error!(InternalError, "Request timeout before processing")); + } + #[cfg(feature = "metrics")] { use metrics::{counter, gauge}; @@ -943,7 +969,10 @@ impl DefaultObjectUsecase { gauge!("rustfs.concurrent.get.object.requests").set(concurrent_requests as f64); } - debug!("GetObject request started with {} concurrent requests", concurrent_requests); + debug!( + "GetObject request started with {} concurrent requests, timeout={:?}", + concurrent_requests, timeout_config.get_object_timeout + ); let mut helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject); // mc get 3 @@ -1111,18 +1140,86 @@ impl DefaultObjectUsecase { .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; let permit_wait_duration = permit_wait_start.elapsed(); + // Check timeout after acquiring permit + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + wait_ms = permit_wait_duration.as_millis(), + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out while waiting for disk permit" + ); + deadlock_detector.unregister_request(&request_id); + #[cfg(feature = "metrics")] + metrics::counter!("rustfs.get.object.timeout.total", "stage" => "disk_permit").increment(1); + return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit")); + } + + // Monitor I/O queue status for congestion detection + let queue_status = manager.io_queue_status(); + let queue_utilization = if queue_status.total_permits > 0 { + (queue_status.permits_in_use as f64 / queue_status.total_permits as f64) * 100.0 + } else { + 0.0 + }; + + // Log warning if queue is congested (> 80% utilization) + if queue_utilization > 80.0 { + warn!( + bucket = %bucket, + key = %key, + queue_utilization = format!("{:.1}%", queue_utilization), + permits_in_use = queue_status.permits_in_use, + total_permits = queue_status.total_permits, + "I/O queue congestion detected" + ); + + #[cfg(feature = "metrics")] + { + use metrics::counter; + counter!("rustfs.io.queue.congestion.total").increment(1); + } + } + // Calculate adaptive I/O strategy from permit wait time // This adjusts buffer sizes, read-ahead, and caching behavior based on load // Use 256KB as the base buffer size for strategy calculation let base_buffer_size = self.base_buffer_size(); let io_strategy = manager.calculate_io_strategy(permit_wait_duration, base_buffer_size); + // Determine I/O priority based on request size (for priority scheduling) + // Small requests (< 1MB) get high priority, large requests (> 10MB) get low priority + let io_priority = manager.get_io_priority(io_strategy.buffer_size as i64); + + // Log priority information for observability + if manager.is_priority_scheduling_enabled() { + debug!( + bucket = %bucket, + key = %key, + priority = %io_priority, + request_size = io_strategy.buffer_size, + "I/O priority assigned" + ); + + #[cfg(feature = "metrics")] + { + use metrics::counter; + counter!("rustfs.io.priority.assigned.total", "priority" => io_priority.as_str()).increment(1); + } + } + // Record detailed I/O metrics for monitoring #[cfg(feature = "metrics")] { use metrics::{counter, gauge, histogram}; // Record permit wait time histogram histogram!("rustfs.disk.permit.wait.duration.seconds").record(permit_wait_duration.as_secs_f64()); + // Record I/O queue utilization + gauge!("rustfs.io.queue.utilization").set(queue_utilization); + gauge!("rustfs.io.queue.permits_in_use").set(queue_status.permits_in_use as f64); + gauge!("rustfs.io.queue.permits_available") + .set(queue_status.total_permits.saturating_sub(queue_status.permits_in_use) as f64); // Record current load level as gauge (0=Low, 1=Medium, 2=High, 3=Critical) let load_level_value = match io_strategy.load_level { crate::storage::concurrency::IoLoadLevel::Low => 0.0, @@ -1135,6 +1232,8 @@ impl DefaultObjectUsecase { gauge!("rustfs.io.buffer.multiplier").set(io_strategy.buffer_multiplier); // Count strategy selections by load level counter!("rustfs.io.strategy.selected", "level" => format!("{:?}", io_strategy.load_level)).increment(1); + // Record I/O priority + counter!("rustfs.io.priority.assigned", "priority" => io_priority.as_str()).increment(1); } // Log strategy details at debug level for troubleshooting @@ -1144,9 +1243,25 @@ impl DefaultObjectUsecase { buffer_size = io_strategy.buffer_size, readahead = io_strategy.enable_readahead, cache_wb = io_strategy.cache_writeback_enabled, + priority = io_priority.as_str(), "Adaptive I/O strategy calculated" ); + // Check timeout before reading object + if wrapper.is_timeout() { + warn!( + bucket = %bucket, + key = %key, + timeout_secs = timeout_config.get_object_timeout.as_secs(), + elapsed_ms = wrapper.elapsed().as_millis(), + "GetObject request timed out before reading object" + ); + deadlock_detector.unregister_request(&request_id); + #[cfg(feature = "metrics")] + metrics::counter!("rustfs.get.object.timeout.total", "stage" => "before_read").increment(1); + return Err(s3_error!(InternalError, "Request timeout before reading object")); + } + let reader = store .get_object_reader(bucket.as_str(), key.as_str(), rs.clone(), h, &opts) .await @@ -1485,11 +1600,26 @@ impl DefaultObjectUsecase { histogram!("get.object.buffer.size.bytes").record(optimal_buffer_size as f64); } + // Check for timeout before returning + if wrapper.is_timeout() { + warn!( + "GetObject request exceeded timeout: key={} duration={:?} timeout={:?}", + cache_key, + wrapper.elapsed(), + timeout_config.get_object_timeout + ); + #[cfg(feature = "metrics")] + counter!("rustfs.get.object.timeout.total").increment(1); + } + debug!( "GetObject completed: key={} size={} duration={:?} buffer={}", cache_key, response_content_length, total_duration, optimal_buffer_size ); + // Unregister from deadlock detector + deadlock_detector.unregister_request(&request_id); + let response = wrap_response_with_cors(&bucket, &req.method, &req.headers, output).await; let result = Ok(response); let _ = helper.complete(&result); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 8daced71c..07705a1c1 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -452,6 +452,15 @@ async fn run(config: config::Config) -> Result<()> { Err(e) => error!(target: "rustfs::main::run","Failed to start audit system: {}", e), } + // Initialize deadlock detector if enabled + let detector = crate::storage::deadlock_detector::get_deadlock_detector(); + if detector.is_enabled() { + detector.start(); + info!(target: "rustfs::main::run","Deadlock detector started successfully."); + } else { + info!(target: "rustfs::main::run","Deadlock detector disabled."); + } + let buckets_list = store .list_bucket(&BucketOptions { no_metadata: true, diff --git a/rustfs/src/storage/backpressure.rs b/rustfs/src/storage/backpressure.rs new file mode 100644 index 000000000..0da284fe3 --- /dev/null +++ b/rustfs/src/storage/backpressure.rs @@ -0,0 +1,495 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Backpressure Management for Object Data Transfer. +//! +//! This module provides backpressure-aware pipes for object data transfer, +//! preventing buffer overflow and memory exhaustion under high concurrency. + +// Allow dead_code for public API that may be used by external modules or future features +#![allow(dead_code)] +//! +//! # Key Features +//! +//! - Configurable buffer size with high/low watermarks +//! - Backpressure state monitoring and events +//! - Prometheus metrics for backpressure events +//! - Graceful handling of slow consumers +//! +//! # Architecture +//! +//! ```text +//! [Disk Reader] --> [BackpressurePipe] --> [HTTP Response] +//! | +//! v +//! [Buffer Monitor] +//! | +//! v +//! [High Watermark?] --> Apply Backpressure +//! ``` + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; +use tokio::io::{DuplexStream, duplex}; +use tracing::{debug, warn}; + +#[cfg(feature = "metrics")] +use metrics::counter; + +/// Backpressure pipe configuration. +#[derive(Debug, Clone)] +pub struct BackpressureConfig { + /// Buffer size in bytes (default 4MB). + pub buffer_size: usize, + /// High watermark percentage (default 80%). + /// When buffer usage exceeds this, backpressure is applied. + pub high_watermark: u32, + /// Low watermark percentage (default 50%). + /// When buffer usage drops below this after high watermark, backpressure is released. + pub low_watermark: u32, +} + +impl Default for BackpressureConfig { + fn default() -> Self { + Self { + buffer_size: rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, + high_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, + low_watermark: rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, + } + } +} + +impl BackpressureConfig { + /// Load configuration from environment variables. + pub fn from_env() -> Self { + let buffer_size = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE, + rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, + ); + let high_watermark = rustfs_utils::get_env_u32( + rustfs_config::ENV_OBJECT_BACKPRESSURE_HIGH_WATERMARK, + rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_HIGH_WATERMARK, + ); + let low_watermark = rustfs_utils::get_env_u32( + rustfs_config::ENV_OBJECT_BACKPRESSURE_LOW_WATERMARK, + rustfs_config::DEFAULT_OBJECT_BACKPRESSURE_LOW_WATERMARK, + ); + + Self { + buffer_size, + high_watermark, + low_watermark, + } + } + + /// Calculate high watermark threshold in bytes. + pub fn high_watermark_bytes(&self) -> usize { + (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize + } + + /// Calculate low watermark threshold in bytes. + pub fn low_watermark_bytes(&self) -> usize { + (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize + } +} + +/// Backpressure state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackpressureState { + /// Normal operation, buffer usage is below high watermark. + Normal, + /// Buffer usage is above high watermark, backpressure should be applied. + HighWatermark, + /// Backpressure is actively being applied to the producer. + BackpressureApplied, +} + +impl std::fmt::Display for BackpressureState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BackpressureState::Normal => write!(f, "normal"), + BackpressureState::HighWatermark => write!(f, "high_watermark"), + BackpressureState::BackpressureApplied => write!(f, "backpressure_applied"), + } + } +} + +/// Backpressure event for monitoring. +#[derive(Debug, Clone)] +pub struct BackpressureEvent { + /// Event timestamp. + pub timestamp: Instant, + /// Event type. + pub event_type: BackpressureEventType, + /// Buffer usage at event time. + pub buffer_usage: usize, + /// Buffer capacity. + pub buffer_capacity: usize, + /// Usage percentage. + pub usage_percent: f32, +} + +/// Backpressure event type. +#[derive(Debug, Clone, Copy)] +pub enum BackpressureEventType { + /// Entered high watermark state. + HighWatermarkReached, + /// Exited high watermark state (backpressure released). + HighWatermarkExited, + /// Backpressure applied to producer. + BackpressureApplied, + /// Backpressure released. + BackpressureReleased, +} + +/// Snapshot of backpressure state. +#[derive(Debug, Clone)] +pub struct BackpressureSnapshot { + /// Buffer capacity in bytes. + pub buffer_capacity: usize, + /// Current buffer usage in bytes (approximate). + pub buffer_used: usize, + /// Usage percentage. + pub usage_percent: f32, + /// Current state. + pub state: BackpressureState, +} + +/// A backpressure-aware pipe wrapping tokio's duplex. +/// +/// This provides monitoring and events for backpressure conditions +/// while maintaining compatibility with the standard duplex interface. +pub struct BackpressurePipe { + /// Reader end of the duplex pipe. + reader: DuplexStream, + /// Writer end of the duplex pipe. + writer: DuplexStream, + /// Configuration. + config: BackpressureConfig, + /// Current buffer usage (approximate, updated on write). + buffer_usage: Arc, + /// Current backpressure state. + state: Arc, // true = in high watermark state + /// Total bytes written. + total_written: Arc, + /// Total bytes read. + total_read: Arc, +} + +impl BackpressurePipe { + /// Create a new backpressure-aware pipe with default configuration. + pub fn new() -> Self { + Self::with_config(BackpressureConfig::from_env()) + } + + /// Create a new backpressure-aware pipe with custom configuration. + pub fn with_config(config: BackpressureConfig) -> Self { + let (reader, writer) = duplex(config.buffer_size); + + debug!( + buffer_size = config.buffer_size, + high_watermark = config.high_watermark, + low_watermark = config.low_watermark, + high_watermark_bytes = config.high_watermark_bytes(), + low_watermark_bytes = config.low_watermark_bytes(), + "Created backpressure pipe" + ); + + Self { + reader, + writer, + config, + buffer_usage: Arc::new(AtomicUsize::new(0)), + state: Arc::new(AtomicBool::new(false)), + total_written: Arc::new(AtomicUsize::new(0)), + total_read: Arc::new(AtomicUsize::new(0)), + } + } + + /// Take the reader end of the pipe (consumes self). + pub fn into_reader(self) -> DuplexStream { + self.reader + } + + /// Take the writer end of the pipe (consumes self). + pub fn into_writer(self) -> DuplexStream { + self.writer + } + + /// Split into reader and writer (consumes self). + pub fn split(self) -> (DuplexStream, DuplexStream) { + (self.reader, self.writer) + } + + /// Get current backpressure state. + pub fn state(&self) -> BackpressureState { + if self.state.load(Ordering::Relaxed) { + BackpressureState::BackpressureApplied + } else { + BackpressureState::Normal + } + } + + /// Get current buffer usage snapshot. + pub fn snapshot(&self) -> BackpressureSnapshot { + let buffer_used = self.buffer_usage.load(Ordering::Relaxed); + let usage_percent = if self.config.buffer_size > 0 { + (buffer_used as f64 / self.config.buffer_size as f64) * 100.0 + } else { + 0.0 + }; + + BackpressureSnapshot { + buffer_capacity: self.config.buffer_size, + buffer_used, + usage_percent: usage_percent as f32, + state: self.state(), + } + } + + /// Record bytes written (call after successful write). + pub fn record_write(&self, bytes: usize) { + self.total_written.fetch_add(bytes, Ordering::Relaxed); + self.buffer_usage.fetch_add(bytes, Ordering::Relaxed); + self.check_high_watermark(); + } + + /// Record bytes read (call after successful read). + pub fn record_read(&self, bytes: usize) { + self.total_read.fetch_add(bytes, Ordering::Relaxed); + self.buffer_usage.fetch_sub(bytes, Ordering::Relaxed); + self.check_low_watermark(); + } + + /// Check if high watermark is reached. + fn check_high_watermark(&self) { + let usage = self.buffer_usage.load(Ordering::Relaxed); + let threshold = self.config.high_watermark_bytes(); + + if usage >= threshold && !self.state.load(Ordering::Relaxed) { + self.state.store(true, Ordering::Relaxed); + + #[cfg(feature = "metrics")] + counter!("rustfs.backpressure.events.total", "state" => "high_watermark").increment(1); + + warn!( + buffer_usage = usage, + buffer_capacity = self.config.buffer_size, + usage_percent = (usage as f64 / self.config.buffer_size as f64 * 100.0) as u32, + high_watermark = self.config.high_watermark, + "Backpressure: high watermark reached" + ); + } + } + + /// Check if low watermark is reached (backpressure can be released). + fn check_low_watermark(&self) { + let usage = self.buffer_usage.load(Ordering::Relaxed); + let threshold = self.config.low_watermark_bytes(); + + if usage <= threshold && self.state.load(Ordering::Relaxed) { + self.state.store(false, Ordering::Relaxed); + + #[cfg(feature = "metrics")] + counter!("rustfs.backpressure.events.total", "state" => "normal").increment(1); + + debug!( + buffer_usage = usage, + buffer_capacity = self.config.buffer_size, + usage_percent = (usage as f64 / self.config.buffer_size as f64 * 100.0) as u32, + low_watermark = self.config.low_watermark, + "Backpressure: returned to normal" + ); + } + } + + /// Get total bytes written. + pub fn total_written(&self) -> usize { + self.total_written.load(Ordering::Relaxed) + } + + /// Get total bytes read. + pub fn total_read(&self) -> usize { + self.total_read.load(Ordering::Relaxed) + } + + /// Get buffer capacity. + pub fn capacity(&self) -> usize { + self.config.buffer_size + } +} + +impl Default for BackpressurePipe { + fn default() -> Self { + Self::new() + } +} + +/// A simple wrapper that provides backpressure monitoring for duplex streams. +/// +/// This is a lighter-weight alternative to `BackpressurePipe` that doesn't +/// wrap the streams but provides monitoring capabilities. +pub struct BackpressureMonitor { + /// Configuration. + config: BackpressureConfig, + /// Current buffer usage. + buffer_usage: Arc, + /// In high watermark state. + in_high_watermark: Arc, +} + +impl BackpressureMonitor { + /// Create a new monitor with default configuration. + pub fn new() -> Self { + Self::with_config(BackpressureConfig::from_env()) + } + + /// Create a new monitor with custom configuration. + pub fn with_config(config: BackpressureConfig) -> Self { + Self { + config, + buffer_usage: Arc::new(AtomicUsize::new(0)), + in_high_watermark: Arc::new(AtomicBool::new(false)), + } + } + + /// Record bytes added to buffer. + pub fn on_write(&self, bytes: usize) -> BackpressureState { + self.buffer_usage.fetch_add(bytes, Ordering::Relaxed); + self.update_state() + } + + /// Record bytes removed from buffer. + pub fn on_read(&self, bytes: usize) -> BackpressureState { + self.buffer_usage.fetch_sub(bytes, Ordering::Relaxed); + self.update_state() + } + + /// Get current state. + pub fn state(&self) -> BackpressureState { + if self.in_high_watermark.load(Ordering::Relaxed) { + BackpressureState::HighWatermark + } else { + BackpressureState::Normal + } + } + + /// Get current buffer usage. + pub fn usage(&self) -> usize { + self.buffer_usage.load(Ordering::Relaxed) + } + + /// Get usage percentage. + pub fn usage_percent(&self) -> f32 { + let usage = self.buffer_usage.load(Ordering::Relaxed); + if self.config.buffer_size > 0 { + (usage as f32 / self.config.buffer_size as f32) * 100.0 + } else { + 0.0 + } + } + + /// Update state based on current usage. + fn update_state(&self) -> BackpressureState { + let usage = self.buffer_usage.load(Ordering::Relaxed); + let high = self.config.high_watermark_bytes(); + let low = self.config.low_watermark_bytes(); + + if usage >= high { + if !self.in_high_watermark.swap(true, Ordering::Relaxed) { + #[cfg(feature = "metrics")] + counter!("rustfs.backpressure.events.total", "state" => "high_watermark").increment(1); + + debug!(usage_percent = self.usage_percent() as u32, "Backpressure: entered high watermark"); + } + BackpressureState::HighWatermark + } else if usage <= low { + if self.in_high_watermark.swap(false, Ordering::Relaxed) { + #[cfg(feature = "metrics")] + counter!("rustfs.backpressure.events.total", "state" => "normal").increment(1); + + debug!(usage_percent = self.usage_percent() as u32, "Backpressure: returned to normal"); + } + BackpressureState::Normal + } else { + self.state() + } + } +} + +impl Default for BackpressureMonitor { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_backpressure_config_default() { + let config = BackpressureConfig::default(); + assert_eq!(config.buffer_size, 4 * 1024 * 1024); + assert_eq!(config.high_watermark, 80); + assert_eq!(config.low_watermark, 50); + } + + #[test] + fn test_backpressure_config_watermarks() { + let config = BackpressureConfig { + buffer_size: 1000, + high_watermark: 80, + low_watermark: 50, + }; + assert_eq!(config.high_watermark_bytes(), 800); + assert_eq!(config.low_watermark_bytes(), 500); + } + + #[test] + fn test_backpressure_state_display() { + assert_eq!(format!("{}", BackpressureState::Normal), "normal"); + assert_eq!(format!("{}", BackpressureState::HighWatermark), "high_watermark"); + assert_eq!(format!("{}", BackpressureState::BackpressureApplied), "backpressure_applied"); + } + + #[test] + fn test_backpressure_monitor() { + let config = BackpressureConfig { + buffer_size: 1000, + high_watermark: 80, + low_watermark: 50, + }; + let monitor = BackpressureMonitor::with_config(config); + + // Initially normal + assert_eq!(monitor.state(), BackpressureState::Normal); + + // Write to reach high watermark + let state = monitor.on_write(850); + assert_eq!(state, BackpressureState::HighWatermark); + + // Read to go below low watermark + let state = monitor.on_read(400); + assert_eq!(state, BackpressureState::Normal); + } + + #[tokio::test] + async fn test_backpressure_pipe_creation() { + let pipe = BackpressurePipe::new(); + assert_eq!(pipe.capacity(), 4 * 1024 * 1024); + assert_eq!(pipe.state(), BackpressureState::Normal); + } +} diff --git a/rustfs/src/storage/concurrency.rs b/rustfs/src/storage/concurrency.rs deleted file mode 100644 index 42ac4cbcb..000000000 --- a/rustfs/src/storage/concurrency.rs +++ /dev/null @@ -1,2130 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Concurrency optimization module for high-performance object retrieval. -//! -//! This module provides intelligent concurrency management to prevent performance -//! degradation when multiple concurrent GetObject requests are processed. It addresses -//! the core issue where increasing concurrency from 1→2→4 requests caused latency to -//! degrade exponentially (59ms → 110ms → 200ms). -//! -//! # Key Features -//! -//! - **Adaptive Buffer Sizing**: Dynamically adjusts buffer sizes based on concurrent load -//! to prevent memory contention and thrashing under high concurrency. -//! - **Moka Cache Integration**: Lock-free hot object caching with automatic TTL/TTI expiration -//! for frequently accessed objects, providing sub-5ms response times on cache hits. -//! - **I/O Rate Limiting**: Semaphore-based disk read throttling prevents I/O queue saturation -//! and ensures fair resource allocation across concurrent requests. -//! - **Comprehensive Metrics**: Prometheus-compatible metrics for monitoring cache hit rates, -//! request latency, concurrency levels, and disk wait times. -//! -//! # Performance Characteristics -//! -//! - Low concurrency (1-2 requests): Optimizes for throughput with larger buffers (100%) -//! - Medium concurrency (3-4 requests): Balances throughput and fairness (75% buffers) -//! - High concurrency (5-8 requests): Optimizes for fairness (50% buffers) -//! - Very high concurrency (>8 requests): Ensures predictable latency (40% buffers) -//! -//! # Expected Performance Improvements -//! -//! | Concurrent Requests | Before | After | Improvement | -//! |---------------------|--------|-------|-------------| -//! | 2 requests | 110ms | 60-70ms | ~40% faster | -//! | 4 requests | 200ms | 75-90ms | ~55% faster | -//! | 8 requests | 400ms | 90-120ms | ~70% faster | -//! -//! # Usage Example -//! -//! ```ignore -//! use crate::storage::concurrency::ConcurrencyManager; -//! -//! async fn handle_get_object() { -//! // Automatic request tracking with RAII guard -//! let _guard = ConcurrencyManager::track_request(); -//! -//! // Try cache first (sub-5ms if hit) -//! if let Some(data) = manager.get_cached(&key).await { -//! return serve_from_cache(data); -//! } -//! -//! // Rate-limited disk read -//! let _permit = manager.acquire_disk_read_permit().await; -//! -//! // Use adaptive buffer size -//! let buffer_size = get_concurrency_aware_buffer_size(file_size, base_buffer); -//! // ... read from disk ... -//! } -//! ``` - -use moka::future::Cache; -use rustfs_config::{KI_B, MI_B}; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, LazyLock, Mutex}; -use std::time::{Duration, Instant}; -use tokio::sync::Semaphore; - -// ============================================ -// Adaptive I/O Strategy Types -// ============================================ - -/// Load level classification based on disk permit wait times. -/// -/// This enum represents the current I/O load on the system, determined by -/// analyzing disk permit acquisition wait times. Longer wait times indicate -/// higher contention and system load. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IoLoadLevel { - /// Low load: wait time < 10ms. System has ample I/O capacity. - Low, - /// Medium load: wait time 10-50ms. System is moderately loaded. - Medium, - /// High load: wait time 50-200ms. System is under significant load. - High, - /// Critical load: wait time > 200ms. System is heavily congested. - Critical, -} - -impl IoLoadLevel { - /// Determine load level from disk permit wait duration. - /// - /// Thresholds are based on typical NVMe SSD characteristics: - /// - Low: < 10ms (normal operation) - /// - Medium: 10-50ms (moderate contention) - /// - High: 50-200ms (significant contention) - /// - Critical: > 200ms (severe congestion) - pub fn from_wait_duration(wait: Duration) -> Self { - let wait_ms = wait.as_millis(); - if wait_ms < 10 { - IoLoadLevel::Low - } else if wait_ms < 50 { - IoLoadLevel::Medium - } else if wait_ms < 200 { - IoLoadLevel::High - } else { - IoLoadLevel::Critical - } - } -} - -/// Adaptive I/O strategy calculated from current system load. -/// -/// This structure provides optimized I/O parameters based on the observed -/// disk permit wait times. It helps balance throughput vs. latency and -/// prevents I/O saturation under high load. -/// -/// # Usage Example -/// -/// ```ignore -/// let strategy = manager.calculate_io_strategy(permit_wait_duration); -/// -/// // Apply strategy to I/O operations -/// let buffer_size = strategy.buffer_size; -/// let enable_readahead = strategy.enable_readahead; -/// let enable_cache_writeback = strategy.cache_writeback_enabled; -/// ``` -#[derive(Debug, Clone)] -pub struct IoStrategy { - /// Recommended buffer size for I/O operations (in bytes). - /// - /// Under high load, this is reduced to improve fairness and reduce memory pressure. - /// Under low load, this is maximized for throughput. - pub buffer_size: usize, - - /// Buffer size multiplier (0.4 - 1.0) applied to base buffer size. - /// - /// - 1.0: Low load - use full buffer - /// - 0.75: Medium load - slightly reduced - /// - 0.5: High load - significantly reduced - /// - 0.4: Critical load - minimal buffer - pub buffer_multiplier: f64, - - /// Whether to enable aggressive read-ahead for sequential reads. - /// - /// Disabled under high load to reduce I/O amplification. - pub enable_readahead: bool, - - /// Whether to enable cache writeback for this request. - /// - /// May be disabled under extreme load to reduce memory pressure. - pub cache_writeback_enabled: bool, - - /// Whether to use tokio BufReader for improved async I/O. - /// - /// Always enabled for better async performance. - #[allow(dead_code)] - pub use_buffered_io: bool, - - /// The detected I/O load level. - pub load_level: IoLoadLevel, - - /// The raw permit wait duration that was used to calculate this strategy. - pub permit_wait_duration: Duration, -} - -impl IoStrategy { - /// Create a new IoStrategy from disk permit wait time and base buffer size. - /// - /// This analyzes the wait duration to determine the current I/O load level - /// and calculates appropriate I/O parameters. - /// - /// # Arguments - /// - /// * `permit_wait_duration` - Time spent waiting for disk read permit - /// * `base_buffer_size` - Base buffer size from workload configuration - /// - /// # Returns - /// - /// An IoStrategy with optimized parameters for the current load level. - pub fn from_wait_duration(permit_wait_duration: Duration, base_buffer_size: usize) -> Self { - let load_level = IoLoadLevel::from_wait_duration(permit_wait_duration); - - // Calculate buffer multiplier based on load level - let buffer_multiplier = match load_level { - IoLoadLevel::Low => 1.0, - IoLoadLevel::Medium => 0.75, - IoLoadLevel::High => 0.5, - IoLoadLevel::Critical => 0.4, - }; - - // Calculate actual buffer size - let buffer_size = ((base_buffer_size as f64) * buffer_multiplier) as usize; - let buffer_size = buffer_size.clamp(32 * KI_B, MI_B); - - // Determine feature toggles based on load - let enable_readahead = match load_level { - IoLoadLevel::Low | IoLoadLevel::Medium => true, - IoLoadLevel::High | IoLoadLevel::Critical => false, - }; - - let cache_writeback_enabled = match load_level { - IoLoadLevel::Low | IoLoadLevel::Medium | IoLoadLevel::High => true, - IoLoadLevel::Critical => false, // Disable under extreme load - }; - - Self { - buffer_size, - buffer_multiplier, - enable_readahead, - cache_writeback_enabled, - use_buffered_io: true, // Always enabled - load_level, - permit_wait_duration, - } - } - - /// Get a human-readable description of the current I/O strategy. - #[allow(dead_code)] - pub fn description(&self) -> String { - format!( - "IoStrategy[{:?}]: buffer={}KB, multiplier={:.2}, readahead={}, cache_wb={}, wait={:?}", - self.load_level, - self.buffer_size / 1024, - self.buffer_multiplier, - self.enable_readahead, - self.cache_writeback_enabled, - self.permit_wait_duration - ) - } -} - -/// Rolling window metrics for I/O load tracking. -/// -/// This structure maintains a sliding window of recent disk permit wait times -/// to provide smoothed load level estimates. This helps prevent strategy -/// oscillation from transient load spikes. -#[allow(dead_code)] -#[derive(Debug)] -struct IoLoadMetrics { - /// Recent permit wait durations (sliding window) - recent_waits: Vec, - /// Maximum samples to keep in the window - max_samples: usize, - /// The earliest record index in the recent_waits vector - earliest_index: usize, - /// Total wait time observed (for averaging) - total_wait_ns: AtomicU64, - /// Total number of observations - observation_count: AtomicU64, -} - -#[allow(dead_code)] -impl IoLoadMetrics { - fn new(max_samples: usize) -> Self { - Self { - recent_waits: Vec::with_capacity(max_samples), - max_samples, - earliest_index: 0, - total_wait_ns: AtomicU64::new(0), - observation_count: AtomicU64::new(0), - } - } - - /// Record a new permit wait observation - fn record(&mut self, wait: Duration) { - // Add to recent waits (with eviction if full) - if self.recent_waits.len() < self.max_samples { - self.recent_waits.push(wait); - } else { - self.recent_waits[self.earliest_index] = wait; - self.earliest_index = (self.earliest_index + 1) % self.max_samples; - } - - // Update totals for overall statistics - self.total_wait_ns.fetch_add(wait.as_nanos() as u64, Ordering::Relaxed); - self.observation_count.fetch_add(1, Ordering::Relaxed); - } - - /// Get the average wait duration over the recent window - fn average_wait(&self) -> Duration { - if self.recent_waits.is_empty() { - return Duration::ZERO; - } - let total: Duration = self.recent_waits.iter().sum(); - total / self.recent_waits.len() as u32 - } - - /// Get the maximum wait duration in the recent window - fn max_wait(&self) -> Duration { - self.recent_waits.iter().copied().max().unwrap_or(Duration::ZERO) - } - - /// Get the P95 wait duration from the recent window - fn p95_wait(&self) -> Duration { - if self.recent_waits.is_empty() { - return Duration::ZERO; - } - let mut sorted = self.recent_waits.clone(); - sorted.sort(); - let p95_idx = ((sorted.len() as f64) * 0.95) as usize; - sorted.get(p95_idx.min(sorted.len() - 1)).copied().unwrap_or(Duration::ZERO) - } - - /// Get the smoothed load level based on recent observations - fn smoothed_load_level(&self) -> IoLoadLevel { - IoLoadLevel::from_wait_duration(self.average_wait()) - } - - /// Get the overall average wait since startup - fn lifetime_average_wait(&self) -> Duration { - let total = self.total_wait_ns.load(Ordering::Relaxed); - let count = self.observation_count.load(Ordering::Relaxed); - if count == 0 { - Duration::ZERO - } else { - Duration::from_nanos(total / count) - } - } - - /// Get the total observation count - fn observation_count(&self) -> u64 { - self.observation_count.load(Ordering::Relaxed) - } -} - -/// Global concurrent request counter for adaptive buffer sizing. -/// -/// This atomic counter tracks the number of active GetObject requests in real-time. -/// It's used by the buffer sizing algorithm to dynamically adjust memory allocation -/// based on current system load, preventing memory contention under high concurrency. -/// -/// Access pattern: Lock-free atomic operations (Relaxed ordering for performance). -static ACTIVE_GET_REQUESTS: AtomicUsize = AtomicUsize::new(0); - -/// Global concurrency manager instance -static CONCURRENCY_MANAGER: LazyLock = LazyLock::new(ConcurrencyManager::new); - -/// RAII guard for tracking active GetObject requests. -/// -/// This guard automatically increments the concurrent request counter when created -/// and decrements it when dropped. This ensures accurate tracking even if requests -/// fail or panic, preventing counter leaks that could permanently degrade performance. -/// -/// # Thread Safety -/// -/// Safe to use across threads. The underlying atomic counter uses Relaxed ordering -/// for performance since exact synchronization isn't required for buffer sizing hints. -/// -/// # Metrics -/// -/// On drop, automatically records request completion and duration metrics (when the -/// "metrics" feature is enabled) for Prometheus monitoring and alerting. -/// -/// # Example -/// -/// ```ignore -/// async fn get_object() { -/// let _guard = GetObjectGuard::new(); -/// // Request counter incremented automatically -/// // ... process request ... -/// // Counter decremented automatically when guard drops -/// } -/// ``` -#[derive(Debug)] -pub struct GetObjectGuard { - /// Track when the request started for metrics collection. - /// Used to calculate end-to-end request latency in the Drop implementation. - #[allow(dead_code)] - start_time: Instant, - /// Reference to the concurrency manager for cleanup operations. - /// The underscore prefix indicates this is used implicitly (for type safety). - _manager: &'static ConcurrencyManager, -} - -impl GetObjectGuard { - /// Create a new guard, incrementing the active request counter atomically. - /// - /// This method is called automatically by `ConcurrencyManager::track_request()`. - /// The counter increment is guaranteed to be visible to concurrent readers - /// immediately due to atomic operations. - fn new() -> Self { - ACTIVE_GET_REQUESTS.fetch_add(1, Ordering::Relaxed); - Self { - start_time: Instant::now(), - _manager: &CONCURRENCY_MANAGER, - } - } - - /// Get the elapsed time since the request started. - /// - /// Useful for logging or metrics collection during request processing. - /// Called automatically in the Drop implementation for duration tracking. - #[allow(dead_code)] - pub fn elapsed(&self) -> Duration { - self.start_time.elapsed() - } - - /// Get the current concurrent request count. - /// - /// Returns the instantaneous number of active GetObject requests across all threads. - /// This value is used by buffer sizing algorithms to adapt to current system load. - /// - /// # Returns - /// - /// Current number of concurrent requests (including this one) - pub fn concurrent_requests() -> usize { - ACTIVE_GET_REQUESTS.load(Ordering::Relaxed) - } -} - -impl Drop for GetObjectGuard { - /// Automatically called when the guard goes out of scope. - /// - /// Performs cleanup operations: - /// 1. Decrements the concurrent request counter atomically - /// 2. Records completion and duration metrics (if metrics feature enabled) - /// - /// This ensures accurate tracking even in error/panic scenarios, as Drop - /// is called during stack unwinding (unless explicitly forgotten). - fn drop(&mut self) { - // Decrement concurrent request counter without underflow. - // If the counter is already 0, `checked_sub` returns None and `fetch_update` - // yields Err(previous). We treat that as a lifecycle bug (e.g., unmatched drop) - // and surface it in debug builds instead of silently discarding it. - if let Err(previous) = - ACTIVE_GET_REQUESTS.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1)) - { - debug_assert_eq!( - previous, 0, - "ACTIVE_GET_REQUESTS underflow attempt in GetObjectGuard::drop; previous value = {}", - previous - ); - } - - // Record Prometheus metrics for monitoring and alerting - // We strictly disable metrics recording in unit tests (cfg(test)) to prevent - // Thread Local Storage (TLS) destruction order issues causing panics. - // In production (not(test)), we use a panic check as a safety guard. - #[cfg(all(feature = "metrics", not(test)))] - if !std::thread::panicking() { - use metrics::{counter, histogram}; - // Track total completed requests for throughput calculation - counter!("rustfs.get.object.requests.completed").increment(1); - // Track request duration histogram for latency percentiles (P50, P95, P99) - histogram!("rustfs.get.object.duration.seconds").record(self.elapsed().as_secs_f64()); - } - } -} - -/// Concurrency-aware buffer size calculator -/// -/// This function adapts buffer sizes based on the current concurrent request load -/// to optimize for both throughput and fairness. -/// -/// # Strategy -/// -/// - **Low concurrency (1-2)**: Use large buffers (512KB-1MB) for maximum throughput -/// - **Medium concurrency (3-8)**: Use moderate buffers (128KB-256KB) for balanced performance -/// - **High concurrency (>8)**: Use smaller buffers (64KB-128KB) for fairness and memory efficiency -/// -/// # Arguments -/// -/// * `file_size` - The size of the file being read, or -1 if unknown -/// * `base_buffer_size` - The baseline buffer size from workload profile -/// -/// # Returns -/// -/// Optimized buffer size in bytes for the current concurrency level -pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize { - let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); - - // Record concurrent request metrics - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::gauge; - gauge!("rustfs.concurrent.get.requests").set(concurrent_requests as f64); - } - - // For low concurrency, use the base buffer size for maximum throughput - if concurrent_requests <= 1 { - return base_buffer_size; - } - let medium_threshold = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, - rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, - ); - let high_threshold = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD, - rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD, - ); - - // Calculate adaptive multiplier based on concurrency level - let adaptive_multiplier = if concurrent_requests <= 2 { - // Low concurrency (1-2): use full buffer for maximum throughput - 1.0 - } else if concurrent_requests <= medium_threshold { - // Medium concurrency (3-4): slightly reduce buffer size (75% of base) - 0.75 - } else if concurrent_requests <= high_threshold { - // Higher concurrency (5-8): more aggressive reduction (50% of base) - 0.5 - } else { - // Very high concurrency (>8): minimize memory per request (40% of base) - 0.4 - }; - - // Calculate the adjusted buffer size - let adjusted_size = (base_buffer_size as f64 * adaptive_multiplier) as usize; - - // Ensure we stay within reasonable bounds - let min_buffer = if file_size > 0 && file_size < 100 * KI_B as i64 { - 32 * KI_B // For very small files, use minimum buffer - } else { - 64 * KI_B // Standard minimum buffer size - }; - - let max_buffer = if concurrent_requests > high_threshold { - 256 * KI_B // Cap at 256KB for high concurrency - } else { - MI_B // Cap at 1MB for lower concurrency - }; - - adjusted_size.clamp(min_buffer, max_buffer) -} - -/// Advanced concurrency-aware buffer sizing with file size optimization -/// -/// This enhanced version considers both concurrency level and file size patterns -/// to provide even better performance characteristics. -/// -/// # Arguments -/// -/// * `file_size` - The size of the file being read, or -1 if unknown -/// * `base_buffer_size` - The baseline buffer size from workload profile -/// * `is_sequential` - Whether this is a sequential read (hint for optimization) -/// -/// # Returns -/// -/// Optimized buffer size in bytes -/// -/// # Examples -/// -/// ```ignore -/// let buffer_size = get_advanced_buffer_size( -/// 32 * 1024 * 1024, // 32MB file -/// 256 * 1024, // 256KB base buffer -/// true // sequential read -/// ); -/// ``` -#[allow(dead_code)] -pub fn get_advanced_buffer_size(file_size: i64, base_buffer_size: usize, is_sequential: bool) -> usize { - let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); - - // For very small files, use smaller buffers regardless of concurrency - // Replace manual max/min chain with clamp - if file_size > 0 && file_size < 256 * KI_B as i64 { - return (file_size as usize / 4).clamp(16 * KI_B, 64 * KI_B); - } - - // Base calculation from standard function - let standard_size = get_concurrency_aware_buffer_size(file_size, base_buffer_size); - let medium_threshold = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, - rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, - ); - let high_threshold = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD, - rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD, - ); - // For sequential reads, we can be more aggressive with buffer sizes - if is_sequential && concurrent_requests <= medium_threshold { - return ((standard_size as f64 * 1.5) as usize).min(2 * MI_B); - } - - // For high concurrency with large files, optimize for parallel processing - if concurrent_requests > high_threshold && file_size > 10 * MI_B as i64 { - // Use smaller, more numerous buffers for better parallelism - return (standard_size as f64 * 0.8) as usize; - } - - standard_size -} - -/// High-performance cache for hot objects using Moka -/// -/// This cache uses Moka for superior concurrent performance with features like: -/// - Lock-free reads and writes -/// - Automatic TTL and TTI expiration -/// - Size-based eviction with weigher function -/// - Built-in metrics collection -/// -/// # Dual Cache Architecture -/// -/// The cache maintains two separate Moka cache instances: -/// 1. `cache` - Simple byte array cache for raw object data (legacy support) -/// 2. `response_cache` - Full GetObject response cache with metadata -/// -/// The response cache is preferred for new code as it stores complete response -/// metadata, enabling cache hits to bypass metadata lookups entirely. -#[derive(Clone)] -struct HotObjectCache { - /// Moka cache instance for simple byte data (legacy) - cache: Cache>, - /// Moka cache instance for full GetObject responses with metadata - response_cache: Cache>, - /// Maximum size of individual objects to cache (10MB by default) - max_object_size: usize, - /// Global cache hit counter - hit_count: Arc, - /// Global cache miss counter - miss_count: Arc, -} - -impl std::fmt::Debug for HotObjectCache { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use std::sync::atomic::Ordering; - f.debug_struct("HotObjectCache") - .field("max_object_size", &self.max_object_size) - .field("hit_count", &self.hit_count.load(Ordering::Relaxed)) - .field("miss_count", &self.miss_count.load(Ordering::Relaxed)) - .finish() - } -} - -/// A cached object with metadata and metrics -#[derive(Clone)] -struct CachedObject { - /// The object data - data: Arc>, - /// When this object was cached - cached_at: Instant, - /// Object size in bytes - size: usize, - /// Number of times this object has been accessed - access_count: Arc, -} - -/// Comprehensive cached object with full response metadata for GetObject operations. -/// -/// This structure stores all necessary fields to reconstruct a complete GetObjectOutput -/// response from cache, avoiding repeated disk reads and metadata lookups for hot objects. -/// -/// # Fields -/// -/// All time fields are serialized as RFC3339 strings to avoid parsing issues with -/// `Last-Modified` and other time headers. -/// -/// # Usage -/// -/// ```ignore -/// let cached = CachedGetObject { -/// body: Bytes::from(data), -/// content_length: data.len() as i64, -/// content_type: Some("application/octet-stream".to_string()), -/// e_tag: Some("\"abc123\"".to_string()), -/// last_modified: Some("2024-01-01T00:00:00Z".to_string()), -/// ..Default::default() -/// }; -/// manager.put_cached_object(cache_key, cached).await; -/// ``` -#[derive(Clone, Debug)] -pub struct CachedGetObject { - /// The object body data - pub body: bytes::Bytes, - /// Content length in bytes - pub content_length: i64, - /// MIME content type - pub content_type: Option, - /// Entity tag for the object - pub e_tag: Option, - /// Last modified time as RFC3339 string (e.g., "2024-01-01T12:00:00Z") - pub last_modified: Option, - /// Expiration time as RFC3339 string - #[allow(dead_code)] - pub expires: Option, - /// Cache-Control header value - pub cache_control: Option, - /// Content-Disposition header value - pub content_disposition: Option, - /// Content-Encoding header value - pub content_encoding: Option, - /// Content-Language header value - pub content_language: Option, - /// Storage class (STANDARD, REDUCED_REDUNDANCY, etc.) - pub storage_class: Option, - /// Version ID for versioned objects - pub version_id: Option, - /// Whether this is a delete marker (for versioned buckets) - pub delete_marker: bool, - /// Number of tags associated with the object - pub tag_count: Option, - /// Replication status - #[allow(dead_code)] - pub replication_status: Option, - /// User-defined metadata (x-amz-meta-*) - pub user_metadata: std::collections::HashMap, - /// When this object was cached (for internal use, automatically set) - #[allow(dead_code)] - cached_at: Option, - /// Access count for hot key tracking (automatically managed) - access_count: Arc, -} - -impl Default for CachedGetObject { - fn default() -> Self { - Self { - body: bytes::Bytes::new(), - content_length: 0, - content_type: None, - e_tag: None, - last_modified: None, - expires: None, - cache_control: None, - content_disposition: None, - content_encoding: None, - content_language: None, - storage_class: None, - version_id: None, - delete_marker: false, - tag_count: None, - replication_status: None, - user_metadata: std::collections::HashMap::new(), - cached_at: None, - access_count: Arc::new(AtomicU64::new(0)), - } - } -} - -impl CachedGetObject { - /// Create a new CachedGetObject with the given body and content length - pub fn new(body: bytes::Bytes, content_length: i64) -> Self { - Self { - body, - content_length, - cached_at: Some(Instant::now()), - access_count: Arc::new(AtomicU64::new(0)), - ..Default::default() - } - } - - /// Builder method to set content_type - pub fn with_content_type(mut self, content_type: String) -> Self { - self.content_type = Some(content_type); - self - } - - /// Builder method to set e_tag - pub fn with_e_tag(mut self, e_tag: String) -> Self { - self.e_tag = Some(e_tag); - self - } - - /// Builder method to set last_modified - pub fn with_last_modified(mut self, last_modified: String) -> Self { - self.last_modified = Some(last_modified); - self - } - - /// Builder method to set cache_control - #[allow(dead_code)] - pub fn with_cache_control(mut self, cache_control: String) -> Self { - self.cache_control = Some(cache_control); - self - } - - /// Builder method to set storage_class - #[allow(dead_code)] - pub fn with_storage_class(mut self, storage_class: String) -> Self { - self.storage_class = Some(storage_class); - self - } - - /// Builder method to set version_id - #[allow(dead_code)] - pub fn with_version_id(mut self, version_id: String) -> Self { - self.version_id = Some(version_id); - self - } - - /// Get the size in bytes for cache eviction calculations - pub fn size(&self) -> usize { - self.body.len() - } - - /// Increment access count and return the new value - pub fn increment_access(&self) -> u64 { - self.access_count.fetch_add(1, Ordering::Relaxed) + 1 - } -} - -/// Internal wrapper for CachedGetObject in the Moka cache -#[derive(Clone)] -struct CachedGetObjectInternal { - /// The cached response data - data: Arc, - /// When this object was cached - cached_at: Instant, - /// Size in bytes for weigher function - size: usize, -} - -impl HotObjectCache { - /// Create a new hot object cache with Moka - /// - /// Configures Moka with: - /// - Size-based eviction (100MB max) - /// - TTL of 5 minutes - /// - TTI of 2 minutes - /// - Weigher function for accurate size tracking - fn new() -> Self { - let max_capacity = rustfs_utils::get_env_u64( - rustfs_config::ENV_OBJECT_CACHE_CAPACITY_MB, - rustfs_config::DEFAULT_OBJECT_CACHE_CAPACITY_MB, - ); - let cache_tti_secs = - rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTI_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTI_SECS); - let cache_ttl_secs = - rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS); - - // Legacy simple byte cache - let cache = Cache::builder() - .max_capacity(max_capacity * MI_B as u64) - .weigher(|_key: &String, value: &Arc| -> u32 { - // Weight based on actual data size - value.size.min(u32::MAX as usize) as u32 - }) - .time_to_live(Duration::from_secs(cache_ttl_secs)) - .time_to_idle(Duration::from_secs(cache_tti_secs)) - .build(); - - // Full response cache with metadata - let response_cache = Cache::builder() - .max_capacity(max_capacity * MI_B as u64) - .weigher(|_key: &String, value: &Arc| -> u32 { - // Weight based on actual data size - value.size.min(u32::MAX as usize) as u32 - }) - .time_to_live(Duration::from_secs(cache_ttl_secs)) - .time_to_idle(Duration::from_secs(cache_tti_secs)) - .build(); - let max_object_size = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_CACHE_MAX_OBJECT_SIZE_MB, - rustfs_config::DEFAULT_OBJECT_CACHE_MAX_OBJECT_SIZE_MB, - ) * MI_B; - Self { - cache, - response_cache, - max_object_size, - hit_count: Arc::new(AtomicU64::new(0)), - miss_count: Arc::new(AtomicU64::new(0)), - } - } - - /// Soft expiration determination, the number of hits is insufficient and exceeds the soft TTL - fn should_expire(&self, obj: &Arc) -> bool { - let age_secs = obj.cached_at.elapsed().as_secs(); - let cache_ttl_secs = - rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS); - let hot_object_min_hits_to_extend = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND, - rustfs_config::DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND, - ); - if age_secs >= cache_ttl_secs { - let hits = obj.access_count.load(Ordering::Relaxed); - return hits < hot_object_min_hits_to_extend as u64; - } - false - } - - /// Get an object from cache with lock-free concurrent access - /// - /// Moka provides lock-free reads, significantly improving concurrent performance. - async fn get(&self, key: &str) -> Option>> { - match self.cache.get(key).await { - Some(cached) => { - if self.should_expire(&cached) { - self.cache.invalidate(key).await; - self.miss_count.fetch_add(1, Ordering::Relaxed); - return None; - } - // Update access count - cached.access_count.fetch_add(1, Ordering::Relaxed); - self.hit_count.fetch_add(1, Ordering::Relaxed); - - // IMPORTANT: Do NOT add high cardinality labels to metrics! - // Previously, this metric was tagged with individual file URIs/keys, - // causing unbounded memory growth in RustFS's own process. The metrics - // crate maintains an internal HashMap for all metric series, and each - // unique file path creates a new entry that is never cleaned up. - // This HashMap grows unbounded with unique file access, causing memory - // leaks in RustFS itself (and also in downstream systems like Prometheus). - // Only use low cardinality labels like operation type or status. - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::counter; - counter!("rustfs.object.cache.hits").increment(1); - } - - Some(Arc::clone(&cached.data)) - } - None => { - self.miss_count.fetch_add(1, Ordering::Relaxed); - - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::counter; - counter!("rustfs.object.cache.misses").increment(1); - } - - None - } - } - } - - /// Put an object into cache with automatic size-based eviction - /// - /// Moka handles eviction automatically based on the weigher function. - #[allow(dead_code)] - async fn put(&self, key: String, data: Vec) { - let size = data.len(); - - // Only cache objects smaller than max_object_size - if size == 0 || size > self.max_object_size { - return; - } - - let cached_obj = Arc::new(CachedObject { - data: Arc::new(data), - cached_at: Instant::now(), - size, - access_count: Arc::new(AtomicU64::new(0)), - }); - - self.cache.insert(key.clone(), cached_obj).await; - - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::{counter, gauge}; - counter!("rustfs.object.cache.insertions").increment(1); - gauge!("rustfs_object_cache_size_bytes").set(self.cache.weighted_size() as f64); - gauge!("rustfs_object_cache_entry_count").set(self.cache.entry_count() as f64); - } - } - - /// Clear all cached objects - #[allow(dead_code)] - async fn clear(&self) { - self.cache.invalidate_all(); - // Sync to ensure all entries are removed - self.cache.run_pending_tasks().await; - } - - /// Get cache statistics for monitoring - #[allow(dead_code)] - async fn stats(&self) -> CacheStats { - // Ensure pending tasks are processed for accurate stats - self.cache.run_pending_tasks().await; - let mut total_ms: u128 = 0; - let mut cnt: u64 = 0; - self.cache.iter().for_each(|(_, v)| { - total_ms += v.cached_at.elapsed().as_millis(); - cnt += 1; - }); - let avg_age_secs = if cnt == 0 { - 0.0 - } else { - (total_ms as f64 / cnt as f64) / 1000.0 - }; - CacheStats { - size: self.cache.weighted_size() as usize, - entries: self.cache.entry_count() as usize, - max_size: 100 * MI_B, - max_object_size: self.max_object_size, - hit_count: self.hit_count.load(Ordering::Relaxed), - miss_count: self.miss_count.load(Ordering::Relaxed), - avg_age_secs, - } - } - - /// Check if a key exists in cache (lock-free) - #[allow(dead_code)] - async fn contains(&self, key: &str) -> bool { - self.cache.contains_key(key) - } - - /// Get multiple objects from cache in parallel - /// - /// Leverages Moka's lock-free design for true parallel access. - #[allow(dead_code)] - async fn get_batch(&self, keys: &[String]) -> Vec>>> { - let mut results = Vec::with_capacity(keys.len()); - for key in keys { - results.push(self.get(key).await); - } - results - } - - /// Remove a specific key from cache - #[allow(dead_code)] - async fn remove(&self, key: &str) -> bool { - let had_key = self.cache.contains_key(key); - self.cache.invalidate(key).await; - had_key - } - - /// Get the most frequently accessed keys - /// - /// Returns up to `limit` keys sorted by access count in descending order. - #[allow(dead_code)] - async fn get_hot_keys(&self, limit: usize) -> Vec<(String, u64)> { - // Run pending tasks to ensure accurate entry count - self.cache.run_pending_tasks().await; - - let mut entries: Vec<(String, u64)> = Vec::new(); - - // Iterate through cache entries - self.cache.iter().for_each(|(key, value)| { - entries.push((key.to_string(), value.access_count.load(Ordering::Relaxed))); - }); - - entries.sort_by(|a, b| b.1.cmp(&a.1)); - entries.truncate(limit); - entries - } - - /// Warm up cache with a batch of objects - #[allow(dead_code)] - async fn warm(&self, objects: Vec<(String, Vec)>) { - for (key, data) in objects { - self.put(key, data).await; - } - } - - /// Get hit rate percentage - #[allow(dead_code)] - fn hit_rate(&self) -> f64 { - let hits = self.hit_count.load(Ordering::Relaxed); - let misses = self.miss_count.load(Ordering::Relaxed); - let total = hits + misses; - - if total == 0 { - 0.0 - } else { - (hits as f64 / total as f64) * 100.0 - } - } - - // ============================================ - // Response Cache Methods (CachedGetObject) - // ============================================ - - /// Get a cached GetObject response with full metadata - /// - /// This method retrieves a complete GetObject response from the response cache, - /// including body data and all response metadata (e_tag, last_modified, etc.). - /// - /// # Arguments - /// - /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" - /// - /// # Returns - /// - /// * `Some(Arc)` - Cached response data if found and not expired - /// * `None` - Cache miss - #[allow(dead_code)] - async fn get_response(&self, key: &str) -> Option> { - match self.response_cache.get(key).await { - Some(cached) => { - // Check soft expiration - let age_secs = cached.cached_at.elapsed().as_secs(); - let cache_ttl_secs = rustfs_utils::get_env_u64( - rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, - rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS, - ); - let hot_object_min_hits = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND, - rustfs_config::DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND, - ); - - if age_secs >= cache_ttl_secs { - let hits = cached.data.access_count.load(Ordering::Relaxed); - if hits < hot_object_min_hits as u64 { - self.response_cache.invalidate(key).await; - self.miss_count.fetch_add(1, Ordering::Relaxed); - return None; - } - } - - // Update access count - cached.data.increment_access(); - self.hit_count.fetch_add(1, Ordering::Relaxed); - - // IMPORTANT: Do NOT add high cardinality labels to metrics! - // See HotObjectCache::get() for details. The metrics crate's internal - // HashMap grows unbounded with high cardinality labels, causing memory - // leaks in RustFS's own process. - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::counter; - counter!("rustfs_object_response_cache_hits").increment(1); - } - - Some(Arc::clone(&cached.data)) - } - None => { - self.miss_count.fetch_add(1, Ordering::Relaxed); - - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::counter; - counter!("rustfs_object_response_cache_misses").increment(1); - } - - None - } - } - } - - /// Put a GetObject response into the response cache - /// - /// This method caches a complete GetObject response including body and metadata. - /// Objects larger than `max_object_size` or empty objects are not cached. - /// - /// # Arguments - /// - /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" - /// * `response` - The complete cached response to store - #[allow(dead_code)] - async fn put_response(&self, key: String, response: CachedGetObject) { - let size = response.size(); - - // Only cache objects smaller than max_object_size - if size == 0 || size > self.max_object_size { - return; - } - - let cached_internal = Arc::new(CachedGetObjectInternal { - data: Arc::new(response), - cached_at: Instant::now(), - size, - }); - - self.response_cache.insert(key.clone(), cached_internal).await; - - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::{counter, gauge}; - counter!("rustfs_object_response_cache_insertions").increment(1); - gauge!("rustfs_object_response_cache_size_bytes").set(self.response_cache.weighted_size() as f64); - gauge!("rustfs_object_response_cache_entry_count").set(self.response_cache.entry_count() as f64); - } - } - - /// Invalidate a cache entry for a specific object - /// - /// This method removes both the simple byte cache entry and the response cache entry - /// for the given key. Used when objects are modified or deleted. - /// - /// # Arguments - /// - /// * `key` - Cache key to invalidate (e.g., "{bucket}/{key}") - #[allow(dead_code)] - async fn invalidate(&self, key: &str) { - // Invalidate both caches - self.cache.invalidate(key).await; - self.response_cache.invalidate(key).await; - - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::counter; - counter!("rustfs_object_cache_invalidations_total").increment(1); - } - } - - /// Invalidate cache entries for an object and its latest version - /// - /// For versioned buckets, this invalidates both: - /// - The specific version key: "{bucket}/{key}?versionId={version_id}" - /// - The latest version key: "{bucket}/{key}" - /// - /// This ensures that after a write/delete, clients don't receive stale data. - /// - /// # Arguments - /// - /// * `bucket` - Bucket name - /// * `key` - Object key - /// * `version_id` - Optional version ID (if None, only invalidates the base key) - #[allow(dead_code)] - async fn invalidate_versioned(&self, bucket: &str, key: &str, version_id: Option<&str>) { - // Always invalidate the latest version key - let base_key = format!("{bucket}/{key}"); - self.invalidate(&base_key).await; - - // Also invalidate the specific version if provided - if let Some(vid) = version_id { - let versioned_key = format!("{base_key}?versionId={vid}"); - self.invalidate(&versioned_key).await; - } - } - - /// Clear all cached objects from both caches - #[allow(dead_code)] - async fn clear_all(&self) { - self.cache.invalidate_all(); - self.response_cache.invalidate_all(); - // Sync to ensure all entries are removed - self.cache.run_pending_tasks().await; - self.response_cache.run_pending_tasks().await; - } -} - -/// Cache statistics for monitoring and debugging -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub struct CacheStats { - /// Current total size of cached objects in bytes - pub size: usize, - /// Number of cached entries - pub entries: usize, - /// Maximum allowed cache size in bytes - pub max_size: usize, - /// Maximum allowed object size in bytes - pub max_object_size: usize, - /// Total number of cache hits - pub hit_count: u64, - /// Total number of cache misses - pub miss_count: u64, - /// Average cache object age (seconds) - pub avg_age_secs: f64, -} - -/// Concurrency manager for coordinating concurrent GetObject requests -/// -/// This manager provides: -/// - Adaptive I/O strategy based on disk permit wait times -/// - Hot object caching with Moka -/// - Disk read permit management to prevent I/O saturation -/// - Rolling metrics for load level smoothing -#[allow(dead_code)] -#[derive(Clone)] -pub struct ConcurrencyManager { - /// Hot object cache for frequently accessed objects - cache: Arc, - /// Semaphore to limit concurrent disk reads - disk_read_semaphore: Arc, - /// Whether object caching is enabled (from RUSTFS_OBJECT_CACHE_ENABLE env var) - cache_enabled: bool, - /// I/O load metrics for adaptive strategy calculation - io_metrics: Arc>, -} - -impl std::fmt::Debug for ConcurrencyManager { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use std::sync::atomic::Ordering; - let io_metrics_info = if let Ok(metrics) = self.io_metrics.lock() { - format!("avg_wait={:?}, observations={}", metrics.average_wait(), metrics.observation_count()) - } else { - "locked".to_string() - }; - f.debug_struct("ConcurrencyManager") - .field("active_requests", &ACTIVE_GET_REQUESTS.load(Ordering::Relaxed)) - .field("disk_read_permits", &self.disk_read_semaphore.available_permits()) - .field("io_metrics", &io_metrics_info) - .finish() - } -} - -impl ConcurrencyManager { - /// Create a new concurrency manager with default settings - /// - /// Reads configuration from environment variables: - /// - `RUSTFS_OBJECT_CACHE_ENABLE`: Enable/disable object caching (default: false) - pub fn new() -> Self { - let cache_enabled = - rustfs_utils::get_env_bool(rustfs_config::ENV_OBJECT_CACHE_ENABLE, rustfs_config::DEFAULT_OBJECT_CACHE_ENABLE); - - let max_disk_reads = rustfs_utils::get_env_usize( - rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS, - rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS, - ); - - Self { - cache: Arc::new(HotObjectCache::new()), - disk_read_semaphore: Arc::new(Semaphore::new(max_disk_reads)), - cache_enabled, - io_metrics: Arc::new(Mutex::new(IoLoadMetrics::new(100))), // Keep last 100 observations - } - } - - /// Check if object caching is enabled - /// - /// Returns true if the `RUSTFS_OBJECT_CACHE_ENABLE` environment variable - /// is set to "true" (case-insensitive). When disabled, cache lookups and - /// writebacks are skipped, reducing memory usage at the cost of repeated - /// disk reads for the same objects. - /// - /// # Returns - /// - /// `true` if caching is enabled, `false` otherwise - pub fn is_cache_enabled(&self) -> bool { - self.cache_enabled - } - - /// Track a GetObject request - pub fn track_request() -> GetObjectGuard { - GetObjectGuard::new() - } - - /// Try to get an object from cache - #[allow(dead_code)] - pub async fn get_cached(&self, key: &str) -> Option>> { - self.cache.get(key).await - } - - /// Cache an object for future retrievals - #[allow(dead_code)] - pub async fn cache_object(&self, key: String, data: Vec) { - self.cache.put(key, data).await; - } - - /// Acquire a permit to perform a disk read operation - /// - /// This ensures we don't overwhelm the disk subsystem with too many - /// concurrent reads, which can cause performance degradation. - pub async fn acquire_disk_read_permit(&self) -> Result, tokio::sync::AcquireError> { - self.disk_read_semaphore.acquire().await - } - - // ============================================ - // Adaptive I/O Strategy Methods - // ============================================ - - /// Record a disk permit wait observation for load tracking. - /// - /// This method updates the rolling metrics used to calculate adaptive I/O - /// strategies. Should be called after each disk permit acquisition. - /// - /// # Arguments - /// - /// * `wait_duration` - Time spent waiting for the disk read permit - pub fn record_permit_wait(&self, wait_duration: Duration) { - if let Ok(mut metrics) = self.io_metrics.lock() { - metrics.record(wait_duration); - } - - // Record histogram metric for Prometheus - #[cfg(all(feature = "metrics", not(test)))] - { - use metrics::histogram; - histogram!("rustfs.disk.permit.wait.duration.seconds").record(wait_duration.as_secs_f64()); - } - } - - /// Calculate an adaptive I/O strategy based on disk permit wait time. - /// - /// This method analyzes the permit wait duration to determine the current - /// I/O load level and returns optimized parameters for the read operation. - /// - /// # Arguments - /// - /// * `permit_wait_duration` - Time spent waiting for disk read permit - /// * `base_buffer_size` - Base buffer size from workload configuration - /// - /// # Returns - /// - /// An `IoStrategy` containing optimized I/O parameters. - /// - /// # Example - /// - /// ```ignore - /// let permit_wait_start = Instant::now(); - /// let _permit = manager.acquire_disk_read_permit().await; - /// let permit_wait_duration = permit_wait_start.elapsed(); - /// - /// let strategy = manager.calculate_io_strategy(permit_wait_duration, 256 * 1024); - /// let optimal_buffer = strategy.buffer_size; - /// ``` - pub fn calculate_io_strategy(&self, permit_wait_duration: Duration, base_buffer_size: usize) -> IoStrategy { - // Record the observation for future smoothing - self.record_permit_wait(permit_wait_duration); - - // Calculate strategy from the current wait duration - IoStrategy::from_wait_duration(permit_wait_duration, base_buffer_size) - } - - /// Get the smoothed I/O load level based on recent observations. - /// - /// This uses the rolling window of permit wait times to provide a more - /// stable estimate of the current load level, reducing oscillation from - /// transient spikes. - /// - /// # Returns - /// - /// The smoothed `IoLoadLevel` based on average recent wait times. - #[allow(dead_code)] - pub fn smoothed_load_level(&self) -> IoLoadLevel { - if let Ok(metrics) = self.io_metrics.lock() { - metrics.smoothed_load_level() - } else { - IoLoadLevel::Medium // Default to medium if lock fails - } - } - - /// Get I/O load statistics for monitoring. - /// - /// Returns statistics about recent disk permit wait times for - /// monitoring dashboards and capacity planning. - /// - /// # Returns - /// - /// A tuple of (average_wait, p95_wait, max_wait, observation_count) - #[allow(dead_code)] - pub fn io_load_stats(&self) -> (Duration, Duration, Duration, u64) { - if let Ok(metrics) = self.io_metrics.lock() { - ( - metrics.average_wait(), - metrics.p95_wait(), - metrics.max_wait(), - metrics.observation_count(), - ) - } else { - (Duration::ZERO, Duration::ZERO, Duration::ZERO, 0) - } - } - - /// Get the recommended buffer size based on current I/O load. - /// - /// This is a convenience method that combines load level detection with - /// buffer size calculation. Uses the smoothed load level for stability. - /// - /// # Arguments - /// - /// * `base_buffer_size` - Base buffer size from workload configuration - /// - /// # Returns - /// - /// Recommended buffer size in bytes. - #[allow(dead_code)] - pub fn adaptive_buffer_size(&self, base_buffer_size: usize) -> usize { - let load_level = self.smoothed_load_level(); - let multiplier = match load_level { - IoLoadLevel::Low => 1.0, - IoLoadLevel::Medium => 0.75, - IoLoadLevel::High => 0.5, - IoLoadLevel::Critical => 0.4, - }; - - let buffer_size = ((base_buffer_size as f64) * multiplier) as usize; - buffer_size.clamp(32 * KI_B, MI_B) - } - - /// Get cache statistics - #[allow(dead_code)] - pub async fn cache_stats(&self) -> CacheStats { - self.cache.stats().await - } - - /// Clear all cached objects - #[allow(dead_code)] - pub async fn clear_cache(&self) { - self.cache.clear().await; - } - - /// Check if a key is cached - #[allow(dead_code)] - pub async fn is_cached(&self, key: &str) -> bool { - self.cache.contains(key).await - } - - /// Get multiple cached objects in a single operation - #[allow(dead_code)] - pub async fn get_cached_batch(&self, keys: &[String]) -> Vec>>> { - self.cache.get_batch(keys).await - } - - /// Remove a specific object from cache - #[allow(dead_code)] - pub async fn remove_cached(&self, key: &str) -> bool { - self.cache.remove(key).await - } - - /// Get the most frequently accessed keys - #[allow(dead_code)] - pub async fn get_hot_keys(&self, limit: usize) -> Vec<(String, u64)> { - self.cache.get_hot_keys(limit).await - } - - /// Get cache hit rate percentage - #[allow(dead_code)] - pub fn cache_hit_rate(&self) -> f64 { - self.cache.hit_rate() - } - - /// Warm up cache with frequently accessed objects - /// - /// This can be called during server startup or maintenance windows - /// to pre-populate the cache with known hot objects. - #[allow(dead_code)] - pub async fn warm_cache(&self, objects: Vec<(String, Vec)>) { - self.cache.warm(objects).await; - } - - /// Get optimized buffer size for a request - /// - /// This wraps the advanced buffer sizing logic and makes it accessible - /// through the concurrency manager interface. - #[allow(dead_code)] - pub fn buffer_size(&self, file_size: i64, base: usize, sequential: bool) -> usize { - get_advanced_buffer_size(file_size, base, sequential) - } - - // ============================================ - // Response Cache Methods (CachedGetObject) - // ============================================ - - /// Get a cached GetObject response with full metadata - /// - /// This method retrieves a complete GetObject response from the response cache, - /// including body data and all response metadata (e_tag, last_modified, content_type, etc.). - /// - /// # Arguments - /// - /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" - /// - /// # Returns - /// - /// * `Some(Arc)` - Cached response data if found and not expired - /// * `None` - Cache miss - /// - /// # Example - /// - /// ```ignore - /// let cache_key = format!("{}/{}", bucket, key); - /// if let Some(cached) = manager.get_cached_object(&cache_key).await { - /// // Build response from cached data - /// let output = GetObjectOutput { - /// body: Some(StreamingBlob::from(cached.body.clone())), - /// content_length: Some(cached.content_length), - /// e_tag: cached.e_tag.clone(), - /// last_modified: cached.last_modified.as_ref().map(|s| parse_rfc3339(s)), - /// ..Default::default() - /// }; - /// } - /// ``` - #[allow(dead_code)] - pub async fn get_cached_object(&self, key: &str) -> Option> { - self.cache.get_response(key).await - } - - /// Cache a complete GetObject response for future retrievals - /// - /// This method caches a complete GetObject response including body and all metadata. - /// Objects larger than the maximum cache size (10MB by default) or empty objects - /// are not cached. - /// - /// # Arguments - /// - /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" - /// * `response` - The complete cached response to store - /// - /// # Example - /// - /// ```ignore - /// let cached = CachedGetObject { - /// body: Bytes::from(data), - /// content_length: data.len() as i64, - /// content_type: Some("application/octet-stream".to_string()), - /// e_tag: Some("\"abc123\"".to_string()), - /// last_modified: Some("2024-01-01T00:00:00Z".to_string()), - /// ..Default::default() - /// }; - /// manager.put_cached_object(cache_key, cached).await; - /// ``` - #[allow(dead_code)] - pub async fn put_cached_object(&self, key: String, response: CachedGetObject) { - self.cache.put_response(key, response).await; - } - - /// Invalidate cache entries for a specific object - /// - /// This method removes both simple byte cache and response cache entries - /// for the given key. Should be called after write operations (put_object, - /// copy_object, delete_object, etc.) to prevent stale data from being served. - /// - /// # Arguments - /// - /// * `key` - Cache key to invalidate (e.g., "{bucket}/{key}") - /// - /// # Example - /// - /// ```ignore - /// // After put_object succeeds - /// let cache_key = format!("{}/{}", bucket, key); - /// manager.invalidate_cache(&cache_key).await; - /// ``` - #[allow(dead_code)] - pub async fn invalidate_cache(&self, key: &str) { - self.cache.invalidate(key).await; - } - - /// Invalidate cache entries for an object and its latest version - /// - /// For versioned buckets, this invalidates both: - /// - The specific version key: "{bucket}/{key}?versionId={version_id}" - /// - The latest version key: "{bucket}/{key}" - /// - /// This ensures that after a write/delete, clients don't receive stale data. - /// Should be called after any write operation that modifies object data or creates - /// new versions. - /// - /// # Arguments - /// - /// * `bucket` - Bucket name - /// * `key` - Object key - /// * `version_id` - Optional version ID (if None, only invalidates the base key) - /// - /// # Example - /// - /// ```ignore - /// // After delete_object with version - /// manager.invalidate_cache_versioned(&bucket, &key, Some(&version_id)).await; - /// - /// // After put_object (invalidates latest) - /// manager.invalidate_cache_versioned(&bucket, &key, None).await; - /// ``` - #[allow(dead_code)] - pub async fn invalidate_cache_versioned(&self, bucket: &str, key: &str, version_id: Option<&str>) { - self.cache.invalidate_versioned(bucket, key, version_id).await; - } - - /// Generate a cache key for an object - /// - /// Creates a cache key in the appropriate format based on whether a version ID - /// is specified. For versioned requests, uses "{bucket}/{key}?versionId={version_id}". - /// For non-versioned requests, uses "{bucket}/{key}". - /// - /// # Arguments - /// - /// * `bucket` - Bucket name - /// * `key` - Object key - /// * `version_id` - Optional version ID - /// - /// # Returns - /// - /// Cache key string - pub fn make_cache_key(bucket: &str, key: &str, version_id: Option<&str>) -> String { - match version_id { - Some(vid) => format!("{bucket}/{key}?versionId={vid}"), - None => format!("{bucket}/{key}"), - } - } - - /// Get maximum cacheable object size - /// - /// Returns the maximum size in bytes for objects that can be cached. - /// Objects larger than this size are not cached to prevent memory exhaustion. - pub fn max_object_size(&self) -> usize { - self.cache.max_object_size - } -} - -impl Default for ConcurrencyManager { - fn default() -> Self { - Self::new() - } -} - -/// Get the global concurrency manager instance -pub fn get_concurrency_manager() -> &'static ConcurrencyManager { - &CONCURRENCY_MANAGER -} - -/// Testing helper to reset the global request counter. -#[allow(dead_code)] -pub(crate) fn reset_active_get_requests() { - ACTIVE_GET_REQUESTS.store(0, Ordering::Relaxed); -} - -#[cfg(test)] -mod tests { - use super::*; - use serial_test::serial; - - #[test] - #[serial] - fn test_concurrent_request_tracking() { - reset_active_get_requests(); - assert_eq!(GetObjectGuard::concurrent_requests(), 0); - - let _guard1 = GetObjectGuard::new(); - assert_eq!(GetObjectGuard::concurrent_requests(), 1); - - let _guard2 = GetObjectGuard::new(); - assert_eq!(GetObjectGuard::concurrent_requests(), 2); - - drop(_guard1); - assert_eq!(GetObjectGuard::concurrent_requests(), 1); - - drop(_guard2); - assert_eq!(GetObjectGuard::concurrent_requests(), 0); - } - - #[test] - fn test_adaptive_buffer_sizing() { - // Reset concurrent requests - ACTIVE_GET_REQUESTS.store(0, Ordering::Relaxed); - let base_buffer = 256 * KI_B; - - // Test low concurrency (1 request) - ACTIVE_GET_REQUESTS.store(1, Ordering::Relaxed); - let result = get_concurrency_aware_buffer_size(10 * MI_B as i64, base_buffer); - assert_eq!(result, base_buffer, "Single request should use full buffer"); - - // Test medium concurrency (3 requests) - ACTIVE_GET_REQUESTS.store(3, Ordering::Relaxed); - let result = get_concurrency_aware_buffer_size(10 * MI_B as i64, base_buffer); - assert!(result < base_buffer && result >= base_buffer / 2); - - // Test higher concurrency (6 requests) - ACTIVE_GET_REQUESTS.store(6, Ordering::Relaxed); - let result = get_concurrency_aware_buffer_size(10 * MI_B as i64, base_buffer); - assert!(result <= base_buffer / 2 && result >= base_buffer / 3); - - // Test very high concurrency (10 requests) - ACTIVE_GET_REQUESTS.store(10, Ordering::Relaxed); - let result = get_concurrency_aware_buffer_size(10 * MI_B as i64, base_buffer); - assert!(result <= base_buffer / 2 && result >= 64 * KI_B); - } - - #[tokio::test] - #[serial] - async fn test_hot_object_cache() { - let cache = HotObjectCache::new(); - let test_data = vec![1u8; 1024]; - - // Test basic put and get - cache.put("test_key".to_string(), test_data.clone()).await; - let retrieved = cache.get("test_key").await; - assert!(retrieved.is_some()); - assert_eq!(*retrieved.unwrap(), test_data); - - // Test cache miss - let missing = cache.get("nonexistent").await; - assert!(missing.is_none()); - } - - #[tokio::test] - #[serial] - async fn test_cache_eviction() { - let cache = HotObjectCache::new(); - - // Fill cache with objects - for i in 0..200 { - let data = vec![0u8; 64 * KI_B]; - cache.put(format!("key_{i}"), data).await; - } - - let stats = cache.stats().await; - assert!( - stats.size <= stats.max_size, - "Cache size {} should not exceed max {}", - stats.size, - stats.max_size - ); - } - - #[tokio::test] - #[serial] - async fn test_cache_reject_large_objects() { - let cache = HotObjectCache::new(); - let large_data = vec![0u8; 11 * MI_B]; // Larger than max_object_size - - cache.put("large_object".to_string(), large_data).await; - let retrieved = cache.get("large_object").await; - assert!(retrieved.is_none(), "Large objects should not be cached"); - } - - #[test] - fn test_concurrency_manager_creation() { - let manager = ConcurrencyManager::new(); - assert_eq!( - manager.disk_read_semaphore.available_permits(), - 64, - "Should start with 64 available disk read permits" - ); - } - - #[tokio::test] - #[serial] - async fn test_disk_read_permits() { - let manager = ConcurrencyManager::new(); - - let permit1 = manager.acquire_disk_read_permit().await.unwrap(); - assert_eq!(manager.disk_read_semaphore.available_permits(), 63); - - let permit2 = manager.acquire_disk_read_permit().await.unwrap(); - assert_eq!(manager.disk_read_semaphore.available_permits(), 62); - - drop(permit1); - assert_eq!(manager.disk_read_semaphore.available_permits(), 63); - - drop(permit2); - assert_eq!(manager.disk_read_semaphore.available_permits(), 64); - } - - #[test] - fn test_advanced_buffer_size_small_files() { - ACTIVE_GET_REQUESTS.store(1, Ordering::Relaxed); - - // Test small file optimization - let result = get_advanced_buffer_size(32 * KI_B as i64, 256 * KI_B, true); - assert!( - (16 * KI_B..=64 * KI_B).contains(&result), - "Small files should use reduced buffer: {result}" - ); - } - - #[test] - fn test_clamp_behavior() { - // Test the clamp replacement - let file_size = 100 * KI_B as i64; - let result = (file_size as usize / 4).clamp(16 * KI_B, 64 * KI_B); - assert!((16 * KI_B..=64 * KI_B).contains(&result)); - } - - #[tokio::test] - #[serial] - async fn test_hot_keys_tracking() { - let manager = ConcurrencyManager::new(); - - // Cache some objects with different access patterns - manager.cache_object("hot1".to_string(), vec![1u8; 100]).await; - manager.cache_object("hot2".to_string(), vec![2u8; 100]).await; - - // Access them multiple times to build hit counts - for _ in 0..5 { - let _ = manager.get_cached("hot1").await; - } - for _ in 0..3 { - let _ = manager.get_cached("hot2").await; - } - - let hot_keys = manager.get_hot_keys(2).await; - assert!(!hot_keys.is_empty(), "Should have hot keys"); - } - - #[tokio::test] - #[serial] - async fn test_batch_operations() { - let manager = ConcurrencyManager::new(); - - manager.cache_object("key1".to_string(), vec![1u8; 100]).await; - manager.cache_object("key2".to_string(), vec![2u8; 100]).await; - manager.cache_object("key3".to_string(), vec![3u8; 100]).await; - - let keys = vec!["key1".to_string(), "key2".to_string(), "key4".to_string()]; - let results = manager.get_cached_batch(&keys).await; - - assert_eq!(results.len(), 3); - assert!(results[0].is_some()); - assert!(results[1].is_some()); - assert!(results[2].is_none()); // key4 doesn't exist - } - - #[tokio::test] - #[serial] - async fn test_cache_clear() { - let manager = ConcurrencyManager::new(); - - manager.cache_object("key1".to_string(), vec![1u8; 1024]).await; - manager.cache_object("key2".to_string(), vec![2u8; 1024]).await; - - let stats_before = manager.cache_stats().await; - assert!(stats_before.entries > 0); - - manager.clear_cache().await; - - let stats_after = manager.cache_stats().await; - assert_eq!(stats_after.entries, 0); - assert_eq!(stats_after.size, 0); - } - - #[tokio::test] - #[serial] - async fn test_warm_cache() { - let manager = ConcurrencyManager::new(); - - let objects = vec![ - ("warm1".to_string(), vec![1u8; 100]), - ("warm2".to_string(), vec![2u8; 100]), - ("warm3".to_string(), vec![3u8; 100]), - ]; - - manager.warm_cache(objects).await; - - assert!(manager.is_cached("warm1").await); - assert!(manager.is_cached("warm2").await); - assert!(manager.is_cached("warm3").await); - } - - #[test] - fn test_io_load_metrics_record_not_full() { - let mut metrics = IoLoadMetrics::new(5); - metrics.record(Duration::from_millis(10)); - metrics.record(Duration::from_millis(20)); - assert_eq!(metrics.recent_waits.len(), 2); - assert_eq!(metrics.recent_waits[0], Duration::from_millis(10)); - assert_eq!(metrics.recent_waits[1], Duration::from_millis(20)); - assert_eq!(metrics.observation_count(), 2); - } - - #[test] - fn test_io_load_metrics_record_full_and_circular() { - let mut metrics = IoLoadMetrics::new(3); - metrics.record(Duration::from_millis(10)); - metrics.record(Duration::from_millis(20)); - metrics.record(Duration::from_millis(30)); - assert_eq!(metrics.recent_waits.len(), 3); - assert_eq!(metrics.earliest_index, 0); - - // This should overwrite the first element (10ms) - metrics.record(Duration::from_millis(40)); - assert_eq!(metrics.recent_waits.len(), 3); - assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); - assert_eq!(metrics.recent_waits[1], Duration::from_millis(20)); - assert_eq!(metrics.recent_waits[2], Duration::from_millis(30)); - assert_eq!(metrics.earliest_index, 1); - assert_eq!(metrics.observation_count(), 4); - - // This should overwrite the second element (20ms) - metrics.record(Duration::from_millis(50)); - assert_eq!(metrics.recent_waits.len(), 3); - assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); - assert_eq!(metrics.recent_waits[1], Duration::from_millis(50)); - assert_eq!(metrics.recent_waits[2], Duration::from_millis(30)); - assert_eq!(metrics.earliest_index, 2); - assert_eq!(metrics.observation_count(), 5); - - // This should overwrite the third element (30ms) - metrics.record(Duration::from_millis(60)); - assert_eq!(metrics.recent_waits.len(), 3); - assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); - assert_eq!(metrics.recent_waits[1], Duration::from_millis(50)); - assert_eq!(metrics.recent_waits[2], Duration::from_millis(60)); - assert_eq!(metrics.earliest_index, 0); - assert_eq!(metrics.observation_count(), 6); - } - - #[test] - fn test_io_load_metrics_average_wait() { - let mut metrics = IoLoadMetrics::new(3); - metrics.record(Duration::from_millis(10)); - metrics.record(Duration::from_millis(20)); - metrics.record(Duration::from_millis(30)); - assert_eq!(metrics.average_wait(), Duration::from_millis(20)); - - // Overwrite 10ms with 40ms, new avg = (20+30+40)/3 = 30 - metrics.record(Duration::from_millis(40)); - assert_eq!(metrics.average_wait(), Duration::from_millis(30)); - } - - #[test] - fn test_io_load_metrics_max_wait() { - let mut metrics = IoLoadMetrics::new(3); - assert_eq!(metrics.max_wait(), Duration::ZERO); - metrics.record(Duration::from_millis(40)); - metrics.record(Duration::from_millis(30)); - metrics.record(Duration::from_millis(20)); - assert_eq!(metrics.max_wait(), Duration::from_millis(40)); - - // Overwrite 40ms with 5ms, max should still be 30 - metrics.record(Duration::from_millis(5)); - assert_eq!(metrics.max_wait(), Duration::from_millis(30)); - - // Overwrite 30ms with 10ms - metrics.record(Duration::from_millis(10)); - assert_eq!(metrics.max_wait(), Duration::from_millis(20)); - } - - #[test] - fn test_io_load_metrics_p95_wait() { - let mut metrics = IoLoadMetrics::new(20); - for i in 1..=20 { - metrics.record(Duration::from_millis(i * 5)); // 5, 10, ..., 100 - } - assert_eq!(metrics.p95_wait(), Duration::from_millis(100)); - - // Test with different values - let mut metrics = IoLoadMetrics::new(10); - metrics.record(Duration::from_millis(10)); - metrics.record(Duration::from_millis(20)); - metrics.record(Duration::from_millis(30)); - metrics.record(Duration::from_millis(40)); - metrics.record(Duration::from_millis(50)); - metrics.record(Duration::from_millis(60)); - metrics.record(Duration::from_millis(70)); - metrics.record(Duration::from_millis(80)); - metrics.record(Duration::from_millis(90)); - metrics.record(Duration::from_millis(1000)); // outlier - assert_eq!(metrics.p95_wait(), Duration::from_millis(1000)); - } - - #[test] - fn test_io_load_metrics_smoothed_load_level() { - let mut metrics = IoLoadMetrics::new(3); - // Average is low - metrics.record(Duration::from_millis(5)); - metrics.record(Duration::from_millis(8)); - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Low); - - // Average is medium - metrics.record(Duration::from_millis(40)); // avg = (5+8+40)/3 = 17.6 -> Medium - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Medium); - - // Average is High - metrics.record(Duration::from_millis(100)); // avg = (8+40+100)/3 = 49.3 -> Medium - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Medium); - - metrics.record(Duration::from_millis(100)); // avg = (40+100+100)/3 = 80 -> High - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::High); - - // Average is Critical - metrics.record(Duration::from_millis(300)); // avg = (100+100+300)/3 = 166.6 -> High - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::High); - - metrics.record(Duration::from_millis(300)); // avg = (100+300+300)/3 = 233.3 -> Critical - assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Critical); - } - - #[test] - fn test_io_load_metrics_lifetime_average() { - let mut metrics = IoLoadMetrics::new(2); - metrics.record(Duration::from_millis(10)); - metrics.record(Duration::from_millis(20)); - // total = 30, count = 2, avg = 15 - assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(15)); - - metrics.record(Duration::from_millis(30)); // recent=(20, 30), but lifetime avg is over all records - // total = 10+20+30=60, count = 3, avg = 20 - let total_ns = metrics.total_wait_ns.load(Ordering::Relaxed); - let count = metrics.observation_count.load(Ordering::Relaxed); - assert_eq!(total_ns, 60_000_000); - assert_eq!(count, 3); - assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(20)); - - metrics.record(Duration::from_millis(40)); - // total = 60+40=100, count = 4, avg = 25 - assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(25)); - } - - /// Regression test for high cardinality metrics bug. - /// - /// This test validates that cache operations work correctly and do not cause - /// unbounded memory growth in RustFS's own process. Previously, metrics were - /// tagged with individual file URIs (e.g., "bucket/object/key"), creating a - /// new metric series for each unique file path. - /// - /// The metrics crate maintains an internal HashMap to track all metric series. - /// With high cardinality labels, this HashMap grew unbounded as each unique file - /// created a new entry that was never cleaned up. This caused: - /// - /// 1. Memory leaks in RustFS's own process (unbounded HashMap growth) - /// 2. Performance degradation under load - /// 3. Potential OOM conditions - /// 4. Cascading issues in downstream systems (Prometheus, etc.) - /// - /// The fix removes high cardinality labels (like "key") from metrics, keeping - /// only low cardinality dimensions (like operation type, success/failure status). - /// - /// This test ensures: - /// - Cache hits and misses are tracked correctly - /// - Multiple cache accesses with different keys don't cause issues - /// - The cache behavior remains unchanged after removing metric labels - #[tokio::test] - #[serial] - async fn test_cache_metrics_no_high_cardinality_labels() { - let cache = HotObjectCache::new(); - - // Store multiple objects with different keys - let test_keys = vec![ - "bucket1/file1.txt", - "bucket1/file2.txt", - "bucket2/data/file3.csv", - "bucket3/images/photo.jpg", - ]; - - for key in &test_keys { - let data = vec![1u8; 1024]; - cache.put(key.to_string(), data).await; - } - - // Access all cached objects multiple times - for key in &test_keys { - for _ in 0..10 { - let result = cache.get(key).await; - assert!(result.is_some(), "Key {} should be cached", key); - } - } - - // Verify hit/miss counters work correctly - let initial_hits = cache.hit_count.load(Ordering::Relaxed); - assert_eq!(initial_hits, 40, "Should have 40 cache hits (4 keys × 10 accesses)"); - - // Access non-existent keys - let _ = cache.get("nonexistent1").await; - let _ = cache.get("nonexistent2").await; - - let misses = cache.miss_count.load(Ordering::Relaxed); - assert_eq!(misses, 2, "Should have 2 cache misses"); - - // The key assertion: We should be able to cache many different keys - // without causing unbounded memory growth in metrics systems. - // If high cardinality labels were still present, each unique key would - // create a new metric series, leading to a memory leak. - - let stats = cache.stats().await; - assert!(stats.entries > 0, "Cache should contain entries"); - assert!(stats.size > 0, "Cache should have non-zero size"); - } -} diff --git a/rustfs/src/storage/concurrency/io_schedule.rs b/rustfs/src/storage/concurrency/io_schedule.rs new file mode 100644 index 000000000..d6431e196 --- /dev/null +++ b/rustfs/src/storage/concurrency/io_schedule.rs @@ -0,0 +1,1159 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! I/O scheduling types for adaptive buffer sizing and load management. + +use rustfs_config::{KI_B, MI_B}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::time::Duration; + +/// Global concurrent request counter for adaptive buffer sizing. +pub(crate) static ACTIVE_GET_REQUESTS: AtomicUsize = AtomicUsize::new(0); + +#[derive(Debug, Clone, PartialEq)] +pub enum IoLoadLevel { + /// Low load: wait time < 10ms. System has ample I/O capacity. + Low, + /// Medium load: wait time 10-50ms. System is moderately loaded. + Medium, + /// High load: wait time 50-200ms. System is under significant load. + High, + /// Critical load: wait time > 200ms. System is heavily congested. + Critical, +} + +impl IoLoadLevel { + /// Determine load level from disk permit wait duration. + /// + /// Thresholds are based on typical NVMe SSD characteristics: + /// - Low: < 10ms (normal operation) + /// - Medium: 10-50ms (moderate contention) + /// - High: 50-200ms (significant contention) + /// - Critical: > 200ms (severe congestion) + pub fn from_wait_duration(wait: Duration) -> Self { + let wait_ms = wait.as_millis(); + if wait_ms < 10 { + IoLoadLevel::Low + } else if wait_ms < 50 { + IoLoadLevel::Medium + } else if wait_ms < 200 { + IoLoadLevel::High + } else { + IoLoadLevel::Critical + } + } +} + +// ============================================ +// Priority-Based I/O Scheduling +// ============================================ + +/// I/O request priority level. +/// +/// Requests are classified by size to prevent large requests +/// from starving small requests. This is especially important +/// under high concurrency where many range reads compete for I/O. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum IoPriority { + /// High priority: small requests (< 1MB). + /// These should complete quickly to maintain responsiveness. + High, + /// Normal priority: medium requests (1MB - 10MB). + /// Standard processing with fair resource allocation. + Normal, + /// Low priority: large requests (> 10MB). + /// These can tolerate longer wait times. + Low, +} + +impl IoPriority { + /// Determine priority from request size. + /// + /// # Arguments + /// + /// * `size` - Request size in bytes + /// + /// # Returns + /// + /// Priority level based on size thresholds: + /// - High: < 1MB + /// - Normal: 1MB - 10MB + /// - Low: > 10MB + pub fn from_size(size: i64) -> Self { + const HIGH_THRESHOLD: i64 = MI_B as i64; // 1MB + const LOW_THRESHOLD: i64 = 10 * MI_B as i64; // 10MB + + if size < HIGH_THRESHOLD { + IoPriority::High + } else if size > LOW_THRESHOLD { + IoPriority::Low + } else { + IoPriority::Normal + } + } + + /// Get the priority as a string for metrics labels. + pub fn as_str(&self) -> &'static str { + match self { + IoPriority::High => "high", + IoPriority::Normal => "normal", + IoPriority::Low => "low", + } + } +} + +impl std::fmt::Display for IoPriority { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +/// I/O scheduler configuration. +#[derive(Debug, Clone, PartialEq)] +#[allow(dead_code)] +pub struct IoSchedulerConfig { + /// Maximum concurrent disk reads. + pub max_concurrent_reads: usize, + /// High priority size threshold in bytes. + pub high_priority_size_threshold: usize, + /// Low priority size threshold in bytes. + pub low_priority_size_threshold: usize, + /// High priority queue capacity. + pub queue_high_capacity: usize, + /// Normal priority queue capacity. + pub queue_normal_capacity: usize, + /// Low priority queue capacity. + pub queue_low_capacity: usize, + /// Starvation prevention check interval in milliseconds. + pub starvation_prevention_interval_ms: u64, + /// Starvation threshold in seconds. + pub starvation_threshold_secs: u64, + /// Load sampling window size. + pub load_sample_window: usize, + /// High load wait time threshold in milliseconds. + pub load_high_threshold_ms: u64, + /// Low load wait time threshold in milliseconds. + pub load_low_threshold_ms: u64, + /// Whether priority scheduling is enabled. + pub enable_priority: bool, +} + +impl Default for IoSchedulerConfig { + fn default() -> Self { + Self { + max_concurrent_reads: rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS, + enable_priority: rustfs_config::DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE, + high_priority_size_threshold: rustfs_config::DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD, + low_priority_size_threshold: rustfs_config::DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD, + queue_high_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY, + queue_normal_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + queue_low_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY, + starvation_prevention_interval_ms: rustfs_config::DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + starvation_threshold_secs: rustfs_config::DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS, + load_sample_window: rustfs_config::DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW, + load_high_threshold_ms: rustfs_config::DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS, + load_low_threshold_ms: rustfs_config::DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS, + } + } +} + +#[allow(dead_code)] +impl IoSchedulerConfig { + /// Load configuration from environment. + pub fn from_env() -> Self { + Self { + max_concurrent_reads: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS, + rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS, + ), + enable_priority: rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE, + rustfs_config::DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE, + ), + high_priority_size_threshold: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD, + ), + low_priority_size_threshold: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD, + ), + queue_high_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY, + ), + queue_normal_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + ), + queue_low_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_LOW_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY, + ), + starvation_prevention_interval_ms: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + rustfs_config::DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + ), + starvation_threshold_secs: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS, + rustfs_config::DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS, + ), + load_sample_window: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_LOAD_SAMPLE_WINDOW, + rustfs_config::DEFAULT_OBJECT_IO_LOAD_SAMPLE_WINDOW, + ), + load_high_threshold_ms: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS, + rustfs_config::DEFAULT_OBJECT_IO_LOAD_HIGH_THRESHOLD_MS, + ), + load_low_threshold_ms: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_LOAD_LOW_THRESHOLD_MS, + rustfs_config::DEFAULT_OBJECT_IO_LOAD_LOW_THRESHOLD_MS, + ), + } + } +} + +/// I/O queue status for monitoring. +#[derive(Debug, Clone, Default)] +#[allow(dead_code)] +pub struct IoQueueStatus { + /// Total permits available. + pub total_permits: usize, + /// Permits currently in use. + pub permits_in_use: usize, + /// Number of high priority requests waiting. + pub high_priority_waiting: usize, + /// Number of normal priority requests waiting. + pub normal_priority_waiting: usize, + /// Number of low priority requests waiting. + pub low_priority_waiting: usize, + /// Number of high priority requests processed. + pub high_priority_processed: u64, + /// Number of normal priority requests processed. + pub normal_priority_processed: u64, + /// Number of low priority requests processed. + pub low_priority_processed: u64, + /// Number of starvation events (low priority requests boosted). + pub starvation_events: u64, +} + +/// Adaptive I/O strategy calculated from current system load. +/// +/// This structure provides optimized I/O parameters based on the observed +/// disk permit wait times. It helps balance throughput vs. latency and +/// prevents I/O saturation under high load. +/// +/// # Usage Example +/// +/// ```ignore +/// let strategy = manager.calculate_io_strategy(permit_wait_duration); +/// +/// // Apply strategy to I/O operations +/// let buffer_size = strategy.buffer_size; +/// let enable_readahead = strategy.enable_readahead; +/// let enable_cache_writeback = strategy.cache_writeback_enabled; +/// ``` +#[derive(Debug, Clone, PartialEq)] +pub struct IoStrategy { + /// Recommended buffer size for I/O operations (in bytes). + /// + /// Under high load, this is reduced to improve fairness and reduce memory pressure. + /// Under low load, this is maximized for throughput. + pub buffer_size: usize, + + /// Buffer size multiplier (0.4 - 1.0) applied to base buffer size. + /// + /// - 1.0: Low load - use full buffer + /// - 0.75: Medium load - slightly reduced + /// - 0.5: High load - significantly reduced + /// - 0.4: Critical load - minimal buffer + pub buffer_multiplier: f64, + + /// Whether to enable aggressive read-ahead for sequential reads. + /// + /// Disabled under high load to reduce I/O amplification. + pub enable_readahead: bool, + + /// Whether to enable cache writeback for this request. + /// + /// May be disabled under extreme load to reduce memory pressure. + pub cache_writeback_enabled: bool, + + /// Whether to use tokio BufReader for improved async I/O. + /// + /// Always enabled for better async performance. + pub use_buffered_io: bool, + + /// The detected I/O load level. + pub load_level: IoLoadLevel, + + /// The raw permit wait duration that was used to calculate this strategy. + pub permit_wait_duration: Duration, +} + +#[allow(dead_code)] +impl IoStrategy { + /// Create a new IoStrategy from disk permit wait time and base buffer size. + /// + /// This analyzes the wait duration to determine the current I/O load level + /// and calculates appropriate I/O parameters. + /// + /// # Arguments + /// + /// * `permit_wait_duration` - Time spent waiting for disk read permit + /// * `base_buffer_size` - Base buffer size from workload configuration + /// + /// # Returns + /// + /// An IoStrategy with optimized parameters for the current load level. + pub fn from_wait_duration(permit_wait_duration: Duration, base_buffer_size: usize) -> Self { + let load_level = IoLoadLevel::from_wait_duration(permit_wait_duration); + + // Calculate buffer multiplier based on load level + let buffer_multiplier = match load_level { + IoLoadLevel::Low => 1.0, + IoLoadLevel::Medium => 0.75, + IoLoadLevel::High => 0.5, + IoLoadLevel::Critical => 0.4, + }; + + // Calculate actual buffer size + let buffer_size = ((base_buffer_size as f64) * buffer_multiplier) as usize; + let buffer_size = buffer_size.clamp(32 * KI_B, MI_B); + + // Determine feature toggles based on load + let enable_readahead = match load_level { + IoLoadLevel::Low | IoLoadLevel::Medium => true, + IoLoadLevel::High | IoLoadLevel::Critical => false, + }; + + let cache_writeback_enabled = match load_level { + IoLoadLevel::Low | IoLoadLevel::Medium | IoLoadLevel::High => true, + IoLoadLevel::Critical => false, // Disable under extreme load + }; + + Self { + buffer_size, + buffer_multiplier, + enable_readahead, + cache_writeback_enabled, + use_buffered_io: true, // Always enabled + load_level, + permit_wait_duration, + } + } + + /// Get a human-readable description of the current I/O strategy. + pub fn description(&self) -> String { + format!( + "IoStrategy[{:?}]: buffer={}KB, multiplier={:.2}, readahead={}, cache_wb={}, wait={:?}", + self.load_level, + self.buffer_size / 1024, + self.buffer_multiplier, + self.enable_readahead, + self.cache_writeback_enabled, + self.permit_wait_duration + ) + } +} + +/// Rolling window metrics for I/O load tracking. +/// +/// This structure maintains a sliding window of recent disk permit wait times +/// to provide smoothed load level estimates. This helps prevent strategy +/// oscillation from transient load spikes. +#[derive(Debug)] +pub(crate) struct IoLoadMetrics { + /// Recent permit wait durations (sliding window) + recent_waits: Vec, + /// Maximum samples to keep in the window + max_samples: usize, + /// The earliest record index in the recent_waits vector + earliest_index: usize, + /// Total wait time observed (for averaging) + total_wait_ns: AtomicU64, + /// Total number of observations + observation_count: AtomicU64, +} + +#[allow(dead_code)] +impl IoLoadMetrics { + pub(crate) fn new(max_samples: usize) -> Self { + Self { + recent_waits: Vec::with_capacity(max_samples), + max_samples, + earliest_index: 0, + total_wait_ns: AtomicU64::new(0), + observation_count: AtomicU64::new(0), + } + } + + /// Record a new permit wait observation + pub(crate) fn record(&mut self, wait: Duration) { + // Add to recent waits (with eviction if full) + if self.recent_waits.len() < self.max_samples { + self.recent_waits.push(wait); + } else { + self.recent_waits[self.earliest_index] = wait; + self.earliest_index = (self.earliest_index + 1) % self.max_samples; + } + + // Update totals for overall statistics + self.total_wait_ns.fetch_add(wait.as_nanos() as u64, Ordering::Relaxed); + self.observation_count.fetch_add(1, Ordering::Relaxed); + } + + /// Get the average wait duration over the recent window + pub(crate) fn average_wait(&self) -> Duration { + if self.recent_waits.is_empty() { + return Duration::ZERO; + } + let total: Duration = self.recent_waits.iter().sum(); + total / self.recent_waits.len() as u32 + } + + /// Get the maximum wait duration in the recent window + pub(crate) fn max_wait(&self) -> Duration { + self.recent_waits.iter().copied().max().unwrap_or(Duration::ZERO) + } + + /// Get the P95 wait duration from the recent window + pub(crate) fn p95_wait(&self) -> Duration { + if self.recent_waits.is_empty() { + return Duration::ZERO; + } + let mut sorted = self.recent_waits.clone(); + sorted.sort(); + let p95_idx = ((sorted.len() as f64) * 0.95) as usize; + sorted.get(p95_idx.min(sorted.len() - 1)).copied().unwrap_or(Duration::ZERO) + } + + /// Get the smoothed load level based on recent observations + pub(crate) fn smoothed_load_level(&self) -> IoLoadLevel { + IoLoadLevel::from_wait_duration(self.average_wait()) + } + + /// Get the overall average wait since startup + pub(crate) fn lifetime_average_wait(&self) -> Duration { + let total = self.total_wait_ns.load(Ordering::Relaxed); + let count = self.observation_count.load(Ordering::Relaxed); + if count == 0 { + Duration::ZERO + } else { + Duration::from_nanos(total / count) + } + } + + /// Get the total observation count + pub(crate) fn observation_count(&self) -> u64 { + self.observation_count.load(Ordering::Relaxed) + } +} +pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize { + let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); + + // Record concurrent request metrics + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::gauge; + gauge!("rustfs.concurrent.get.requests").set(concurrent_requests as f64); + } + + // For low concurrency, use the base buffer size for maximum throughput + if concurrent_requests <= 1 { + return base_buffer_size; + } + let medium_threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, + ); + let high_threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD, + ); + + // Calculate adaptive multiplier based on concurrency level + let adaptive_multiplier = if concurrent_requests <= 2 { + // Low concurrency (1-2): use full buffer for maximum throughput + 1.0 + } else if concurrent_requests <= medium_threshold { + // Medium concurrency (3-4): slightly reduce buffer size (75% of base) + 0.75 + } else if concurrent_requests <= high_threshold { + // Higher concurrency (5-8): more aggressive reduction (50% of base) + 0.5 + } else { + // Very high concurrency (>8): minimize memory per request (40% of base) + 0.4 + }; + + // Calculate the adjusted buffer size + let adjusted_size = (base_buffer_size as f64 * adaptive_multiplier) as usize; + + // Ensure we stay within reasonable bounds + let min_buffer = if file_size > 0 && file_size < 100 * KI_B as i64 { + 32 * KI_B // For very small files, use minimum buffer + } else { + 64 * KI_B // Standard minimum buffer size + }; + + let max_buffer = if concurrent_requests > high_threshold { + 256 * KI_B // Cap at 256KB for high concurrency + } else { + MI_B // Cap at 1MB for lower concurrency + }; + + adjusted_size.clamp(min_buffer, max_buffer) +} + +/// Advanced concurrency-aware buffer sizing with file size optimization +/// +/// This enhanced version considers both concurrency level and file size patterns +/// to provide even better performance characteristics. +/// +/// # Arguments +/// +/// * `file_size` - The size of the file being read, or -1 if unknown +/// * `base_buffer_size` - The baseline buffer size from workload profile +/// * `is_sequential` - Whether this is a sequential read (hint for optimization) +/// +/// # Returns +/// +/// Optimized buffer size in bytes +/// +/// # Examples +/// +/// ```ignore +/// let buffer_size = get_advanced_buffer_size( +/// 32 * 1024 * 1024, // 32MB file +/// 256 * 1024, // 256KB base buffer +/// true // sequential read +/// ); +/// ``` +pub fn get_advanced_buffer_size(file_size: i64, base_buffer_size: usize, is_sequential: bool) -> usize { + let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); + + // For very small files, use smaller buffers regardless of concurrency + // Replace manual max/min chain with clamp + if file_size > 0 && file_size < 256 * KI_B as i64 { + return (file_size as usize / 4).clamp(16 * KI_B, 64 * KI_B); + } + + // Base calculation from standard function + let standard_size = get_concurrency_aware_buffer_size(file_size, base_buffer_size); + let medium_threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, + ); + let high_threshold = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD, + ); + // For sequential reads, we can be more aggressive with buffer sizes + if is_sequential && concurrent_requests <= medium_threshold { + return ((standard_size as f64 * 1.5) as usize).min(2 * MI_B); + } + + // For high concurrency with large files, optimize for parallel processing + if concurrent_requests > high_threshold && file_size > 10 * MI_B as i64 { + // Use smaller, more numerous buffers for better parallelism + return (standard_size as f64 * 0.8) as usize; + } + + standard_size +} + +// ============================================ +// I/O Priority Queue Implementation +// ============================================ + +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::Mutex; +use tracing::warn; + +/// Queued I/O request with metadata. +#[derive(Debug)] +struct QueuedRequest { + /// The actual request payload. + request: T, + /// Time when the request was enqueued. + enqueue_time: Instant, + /// Original priority assigned to the request. + #[allow(dead_code)] + original_priority: IoPriority, + /// Current priority (may be boosted for starvation prevention). + current_priority: IoPriority, + /// Whether this request was boosted due to starvation prevention. + starvation_boosted: bool, +} + +/// Queue statistics for monitoring. +#[derive(Debug, Clone, Default)] +struct QueueStats { + /// Number of high priority requests processed. + high_processed: u64, + /// Number of normal priority requests processed. + normal_processed: u64, + /// Number of low priority requests processed. + low_processed: u64, + /// Number of starvation events (low priority requests boosted). + starvation_events: u64, + /// Total wait time in nanoseconds. + total_wait_time_ns: u64, +} + +/// I/O Priority Queue with starvation prevention. +/// +/// This structure manages three priority queues (high, normal, low) and implements +/// starvation prevention by promoting low-priority requests that have been waiting +/// too long. +/// +/// # Type Parameters +/// +/// * `T` - The type of requests being queued. +/// +/// # Example +/// +/// ```ignore +/// let config = IoSchedulerConfig::default(); +/// let queue = IoPriorityQueue::new(config); +/// +/// // Enqueue requests +/// queue.enqueue(IoPriority::High, request1).await; +/// queue.enqueue(IoPriority::Low, request2).await; +/// +/// // Dequeue (prioritizes high priority) +/// if let Some((request, priority)) = queue.dequeue().await { +/// // Process request +/// } +/// ``` +pub struct IoPriorityQueue { + /// Configuration for the priority queue. + config: IoPriorityQueueConfig, + + /// High priority queue. + high_queue: Arc>>>, + /// Normal priority queue. + normal_queue: Arc>>>, + /// Low priority queue. + low_queue: Arc>>>, + + /// Queue statistics. + stats: Arc>, + + /// Last time starvation check was performed. + last_starvation_check: Arc>, +} + +/// Configuration for IoPriorityQueue. +#[derive(Debug, Clone)] +pub struct IoPriorityQueueConfig { + /// High priority queue capacity. + pub queue_high_capacity: usize, + /// Normal priority queue capacity. + pub queue_normal_capacity: usize, + /// Low priority queue capacity. + pub queue_low_capacity: usize, + /// Starvation prevention check interval in milliseconds. + pub starvation_prevention_interval_ms: u64, + /// Starvation threshold in seconds (how long before boosting). + pub starvation_threshold_secs: u64, +} + +impl Default for IoPriorityQueueConfig { + fn default() -> Self { + Self { + queue_high_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY, + queue_normal_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + queue_low_capacity: rustfs_config::DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY, + starvation_prevention_interval_ms: rustfs_config::DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + starvation_threshold_secs: rustfs_config::DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS, + } + } +} + +#[allow(dead_code)] +impl IoPriorityQueueConfig { + /// Load configuration from environment. + pub fn from_env() -> Self { + Self { + queue_high_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_HIGH_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_HIGH_CAPACITY, + ), + queue_normal_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_NORMAL_CAPACITY, + ), + queue_low_capacity: rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_IO_QUEUE_LOW_CAPACITY, + rustfs_config::DEFAULT_OBJECT_IO_QUEUE_LOW_CAPACITY, + ), + starvation_prevention_interval_ms: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + rustfs_config::DEFAULT_OBJECT_IO_STARVATION_PREVENTION_INTERVAL, + ), + starvation_threshold_secs: rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_IO_STARVATION_THRESHOLD_SECS, + rustfs_config::DEFAULT_OBJECT_IO_STARVATION_THRESHOLD_SECS, + ), + } + } +} + +#[allow(dead_code)] +impl IoPriorityQueue { + /// Create a new priority queue with the given configuration. + pub fn new(config: IoPriorityQueueConfig) -> Self { + let config_clone = config.clone(); + Self { + config, + high_queue: Arc::new(Mutex::new(VecDeque::with_capacity(config_clone.queue_high_capacity))), + normal_queue: Arc::new(Mutex::new(VecDeque::with_capacity(config_clone.queue_normal_capacity))), + low_queue: Arc::new(Mutex::new(VecDeque::with_capacity(config_clone.queue_low_capacity))), + stats: Arc::new(Mutex::new(QueueStats::default())), + last_starvation_check: Arc::new(Mutex::new(Instant::now())), + } + } + + /// Enqueue a request with the given priority. + pub async fn enqueue(&self, priority: IoPriority, request: T) { + let queued = QueuedRequest { + request, + enqueue_time: Instant::now(), + original_priority: priority, + current_priority: priority, + starvation_boosted: false, + }; + + match priority { + IoPriority::High => self.high_queue.lock().await.push_back(queued), + IoPriority::Normal => self.normal_queue.lock().await.push_back(queued), + IoPriority::Low => self.low_queue.lock().await.push_back(queued), + } + } + + /// Dequeue a request, prioritizing high priority requests. + /// + /// This method performs starvation prevention checks before dequeuing. + /// Returns `None` if all queues are empty. + pub async fn dequeue(&self) -> Option<(T, IoPriority)> { + // 1. Check for starvation prevention + self.check_starvation().await; + + // 2. Dequeue in priority order + if let Some(queued) = self.high_queue.lock().await.pop_front() { + self.record_dequeue(&queued).await; + return Some((queued.request, queued.current_priority)); + } + + if let Some(queued) = self.normal_queue.lock().await.pop_front() { + self.record_dequeue(&queued).await; + return Some((queued.request, queued.current_priority)); + } + + if let Some(queued) = self.low_queue.lock().await.pop_front() { + self.record_dequeue(&queued).await; + return Some((queued.request, queued.current_priority)); + } + + None + } + + /// Check for starving low-priority requests and boost them. + async fn check_starvation(&self) { + let mut last_check = self.last_starvation_check.lock().await; + let now = Instant::now(); + + // Only check at the configured interval + if now.duration_since(*last_check) < Duration::from_millis(self.config.starvation_prevention_interval_ms) { + return; + } + + *last_check = now; + + // Check if low priority queue has requests waiting too long + let mut low_queue = self.low_queue.lock().await; + let mut normal_queue = self.normal_queue.lock().await; + + let starvation_threshold = Duration::from_secs(self.config.starvation_threshold_secs); + let mut starvation_count = 0; + + while let Some(queued) = low_queue.front() { + if now.duration_since(queued.enqueue_time) > starvation_threshold { + let mut queued = low_queue.pop_front().unwrap(); + queued.current_priority = IoPriority::Normal; + queued.starvation_boosted = true; + normal_queue.push_back(queued); + starvation_count += 1; + } else { + break; + } + } + + if starvation_count > 0 { + self.stats.lock().await.starvation_events += starvation_count; + warn!(starvation_count, "Starvation prevention: boosted low priority requests to normal"); + } + } + + /// Record dequeue statistics. + async fn record_dequeue(&self, queued: &QueuedRequest) { + let mut stats = self.stats.lock().await; + + // Record processing count + match queued.current_priority { + IoPriority::High => stats.high_processed += 1, + IoPriority::Normal => stats.normal_processed += 1, + IoPriority::Low => stats.low_processed += 1, + } + + // Record wait time + let wait_time = Instant::now().duration_since(queued.enqueue_time); + stats.total_wait_time_ns += wait_time.as_nanos() as u64; + } + + /// Get current queue status for monitoring. + pub async fn status(&self) -> IoQueueStatus { + let high_queue = self.high_queue.lock().await; + let normal_queue = self.normal_queue.lock().await; + let low_queue = self.low_queue.lock().await; + let stats = self.stats.lock().await; + + IoQueueStatus { + total_permits: 0, // Not applicable for priority queue + permits_in_use: 0, // Not applicable for priority queue + high_priority_waiting: high_queue.len(), + normal_priority_waiting: normal_queue.len(), + low_priority_waiting: low_queue.len(), + high_priority_processed: stats.high_processed, + normal_priority_processed: stats.normal_processed, + low_priority_processed: stats.low_processed, + starvation_events: stats.starvation_events, + } + } + + /// Get the total number of queued requests. + pub async fn len(&self) -> usize { + let high_queue = self.high_queue.lock().await; + let normal_queue = self.normal_queue.lock().await; + let low_queue = self.low_queue.lock().await; + + high_queue.len() + normal_queue.len() + low_queue.len() + } + + /// Check if all queues are empty. + pub async fn is_empty(&self) -> bool { + self.len().await == 0 + } +} + +// ============================================ +// I/O Priority Queue Metrics +// ============================================ + +/// Global metrics for I/O priority queue monitoring. +/// +/// These metrics are exposed for Prometheus scraping and provide +/// visibility into the priority queue behavior. +#[allow(dead_code)] +pub struct IoPriorityMetrics { + /// High priority queue depth. + pub high_queue_depth: AtomicU64, + /// Normal priority queue depth. + pub normal_queue_depth: AtomicU64, + /// Low priority queue depth. + pub low_queue_depth: AtomicU64, + /// High priority total wait time in nanoseconds. + pub high_wait_time_ns: AtomicU64, + /// Normal priority total wait time in nanoseconds. + pub normal_wait_time_ns: AtomicU64, + /// Low priority total wait time in nanoseconds. + pub low_wait_time_ns: AtomicU64, + /// Total starvation events count. + pub starvation_events: AtomicU64, + /// High priority requests processed. + pub high_processed: AtomicU64, + /// Normal priority requests processed. + pub normal_processed: AtomicU64, + /// Low priority requests processed. + pub low_processed: AtomicU64, +} + +#[allow(dead_code)] +impl IoPriorityMetrics { + /// Create a new metrics instance. + pub const fn new() -> Self { + Self { + high_queue_depth: AtomicU64::new(0), + normal_queue_depth: AtomicU64::new(0), + low_queue_depth: AtomicU64::new(0), + high_wait_time_ns: AtomicU64::new(0), + normal_wait_time_ns: AtomicU64::new(0), + low_wait_time_ns: AtomicU64::new(0), + starvation_events: AtomicU64::new(0), + high_processed: AtomicU64::new(0), + normal_processed: AtomicU64::new(0), + low_processed: AtomicU64::new(0), + } + } + + /// Update queue depths from status. + #[allow(dead_code)] + pub fn update_queue_depths(&self, status: &IoQueueStatus) { + self.high_queue_depth + .store(status.high_priority_waiting as u64, Ordering::Relaxed); + self.normal_queue_depth + .store(status.normal_priority_waiting as u64, Ordering::Relaxed); + self.low_queue_depth + .store(status.low_priority_waiting as u64, Ordering::Relaxed); + } + + /// Record a starvation event. + #[allow(dead_code)] + pub fn record_starvation(&self) { + self.starvation_events.fetch_add(1, Ordering::Relaxed); + } + + /// Record a processed request. + pub fn record_processed(&self, priority: IoPriority) { + match priority { + IoPriority::High => self.high_processed.fetch_add(1, Ordering::Relaxed), + IoPriority::Normal => self.normal_processed.fetch_add(1, Ordering::Relaxed), + IoPriority::Low => self.low_processed.fetch_add(1, Ordering::Relaxed), + }; + } + + /// Record wait time for a priority level. + pub fn record_wait_time(&self, priority: IoPriority, wait_ns: u64) { + match priority { + IoPriority::High => self.high_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), + IoPriority::Normal => self.normal_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), + IoPriority::Low => self.low_wait_time_ns.fetch_add(wait_ns, Ordering::Relaxed), + }; + } + + /// Get high priority queue depth. + pub fn get_high_queue_depth(&self) -> u64 { + self.high_queue_depth.load(Ordering::Relaxed) + } + + /// Get normal priority queue depth. + pub fn get_normal_queue_depth(&self) -> u64 { + self.normal_queue_depth.load(Ordering::Relaxed) + } + + /// Get low priority queue depth. + pub fn get_low_queue_depth(&self) -> u64 { + self.low_queue_depth.load(Ordering::Relaxed) + } + + /// Get total starvation events. + pub fn get_starvation_events(&self) -> u64 { + self.starvation_events.load(Ordering::Relaxed) + } + + /// Get metrics summary for logging/debugging. + #[allow(dead_code)] + pub fn summary(&self) -> String { + format!( + "high_queue={}, normal_queue={}, low_queue={}, starvation={}, high_proc={}, normal_proc={}, low_proc={}", + self.get_high_queue_depth(), + self.get_normal_queue_depth(), + self.get_low_queue_depth(), + self.get_starvation_events(), + self.high_processed.load(Ordering::Relaxed), + self.normal_processed.load(Ordering::Relaxed), + self.low_processed.load(Ordering::Relaxed) + ) + } +} + +/// Global I/O priority metrics instance. +#[allow(dead_code)] +pub static IO_PRIORITY_METRICS: IoPriorityMetrics = IoPriorityMetrics::new(); + +// ============================================ +// Unit Tests +// ============================================ + +#[cfg(test)] +mod tests { + use super::*; + use serial_test::serial; + use tokio::test; + + #[test] + #[serial] + async fn test_io_priority_queue_basic() { + let config = IoPriorityQueueConfig::default(); + let queue = IoPriorityQueue::new(config); + + // Test empty queue + assert!(queue.is_empty().await); + assert_eq!(queue.len().await, 0); + + // Test enqueue + queue.enqueue(IoPriority::High, ()).await; + queue.enqueue(IoPriority::Normal, ()).await; + queue.enqueue(IoPriority::Low, ()).await; + + assert!(!queue.is_empty().await); + assert_eq!(queue.len().await, 3); + } + + #[test] + #[serial] + async fn test_io_priority_queue_dequeue_order() { + let config = IoPriorityQueueConfig::default(); + let queue = IoPriorityQueue::new(config); + + // Enqueue in reverse priority order + queue.enqueue(IoPriority::Low, "low").await; + queue.enqueue(IoPriority::Normal, "normal").await; + queue.enqueue(IoPriority::High, "high").await; + + // Dequeue should return in priority order + let (req1, pri1) = queue.dequeue().await.unwrap(); + assert_eq!(req1, "high"); + assert_eq!(pri1, IoPriority::High); + + let (req2, pri2) = queue.dequeue().await.unwrap(); + assert_eq!(req2, "normal"); + assert_eq!(pri2, IoPriority::Normal); + + let (req3, pri3) = queue.dequeue().await.unwrap(); + assert_eq!(req3, "low"); + assert_eq!(pri3, IoPriority::Low); + + // Queue should be empty now + assert!(queue.is_empty().await); + } + + #[test] + #[serial] + async fn test_io_priority_queue_status() { + let config = IoPriorityQueueConfig::default(); + let queue = IoPriorityQueue::new(config); + + // Enqueue some requests + queue.enqueue(IoPriority::High, ()).await; + queue.enqueue(IoPriority::High, ()).await; + queue.enqueue(IoPriority::Normal, ()).await; + queue.enqueue(IoPriority::Low, ()).await; + + let status = queue.status().await; + assert_eq!(status.high_priority_waiting, 2); + assert_eq!(status.normal_priority_waiting, 1); + assert_eq!(status.low_priority_waiting, 1); + } + + #[test] + #[serial] + async fn test_io_priority_queue_starvation_prevention() { + let config = IoPriorityQueueConfig { + starvation_threshold_secs: 1, + starvation_prevention_interval_ms: 100, + ..Default::default() + }; + + let queue = IoPriorityQueue::new(config); + + // Enqueue a low priority request + queue.enqueue(IoPriority::Low, "low").await; + + // Wait for starvation threshold + tokio::time::sleep(Duration::from_secs(2)).await; + + // Dequeue should return the boosted request as Normal priority + let (req, priority) = queue.dequeue().await.unwrap(); + assert_eq!(req, "low"); + // The request should be boosted to Normal due to starvation prevention + assert_eq!(priority, IoPriority::Normal); + } + + #[test] + #[serial] + async fn test_io_priority_from_size() { + // High priority: < 1MB + assert_eq!(IoPriority::from_size(100 * 1024), IoPriority::High); + assert_eq!(IoPriority::from_size(512 * 1024), IoPriority::High); + + // Normal priority: 1MB - 10MB + assert_eq!(IoPriority::from_size(2 * 1024 * 1024), IoPriority::Normal); + assert_eq!(IoPriority::from_size(5 * 1024 * 1024), IoPriority::Normal); + + // Low priority: > 10MB + assert_eq!(IoPriority::from_size(20 * 1024 * 1024), IoPriority::Low); + assert_eq!(IoPriority::from_size(100 * 1024 * 1024), IoPriority::Low); + } + + #[test] + #[serial] + async fn test_io_load_level_from_wait_duration() { + use std::time::Duration; + + // Low load: < 10ms + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(5)), IoLoadLevel::Low); + + // Medium load: 10-50ms + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(20)), IoLoadLevel::Medium); + + // High load: 50-200ms + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(100)), IoLoadLevel::High); + + // Critical load: > 200ms + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(300)), IoLoadLevel::Critical); + } + + #[test] + #[serial] + async fn test_io_scheduler_config_default() { + let config = IoSchedulerConfig::default(); + + assert!(config.enable_priority); + assert_eq!(config.high_priority_size_threshold, 1024 * 1024); // 1MB + assert_eq!(config.low_priority_size_threshold, 100 * 1024 * 1024); // 100MB + assert_eq!(config.queue_high_capacity, 32); + assert_eq!(config.queue_normal_capacity, 64); + assert_eq!(config.queue_low_capacity, 16); + assert_eq!(config.starvation_threshold_secs, 5); + } + + #[test] + #[serial] + async fn test_io_priority_metrics() { + let metrics = IoPriorityMetrics::new(); + + // Test initial state + assert_eq!(metrics.get_high_queue_depth(), 0); + assert_eq!(metrics.get_normal_queue_depth(), 0); + assert_eq!(metrics.get_low_queue_depth(), 0); + assert_eq!(metrics.get_starvation_events(), 0); + + // Test recording + metrics.record_starvation(); + assert_eq!(metrics.get_starvation_events(), 1); + + metrics.record_processed(IoPriority::High); + metrics.record_processed(IoPriority::High); + metrics.record_processed(IoPriority::Normal); + + assert_eq!(metrics.high_processed.load(Ordering::Relaxed), 2); + assert_eq!(metrics.normal_processed.load(Ordering::Relaxed), 1); + } +} diff --git a/rustfs/src/storage/concurrency/manager.rs b/rustfs/src/storage/concurrency/manager.rs new file mode 100644 index 000000000..1098e221e --- /dev/null +++ b/rustfs/src/storage/concurrency/manager.rs @@ -0,0 +1,717 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Concurrency manager for coordinating concurrent GetObject requests. + +use super::io_schedule::{ + IoLoadLevel, IoLoadMetrics, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, IoStrategy, + get_advanced_buffer_size, +}; +use super::object_cache::{CacheStats, CachedGetObject, CachedObject, HotObjectCache}; +use super::request_guard::GetObjectGuard; +use rustfs_config::{KI_B, MI_B}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::Duration; +use tokio::sync::Semaphore; +use tracing::debug; + +/// Global concurrency manager instance +pub(crate) static CONCURRENCY_MANAGER: LazyLock = LazyLock::new(ConcurrencyManager::new); + +#[derive(Clone)] +pub struct ConcurrencyManager { + /// Hot object cache for frequently accessed objects + cache: Arc, + /// Semaphore to limit concurrent disk reads + disk_read_semaphore: Arc, + /// Whether object caching is enabled (from RUSTFS_OBJECT_CACHE_ENABLE env var) + cache_enabled: bool, + /// I/O load metrics for adaptive strategy calculation + io_metrics: Arc>, + /// I/O priority queue for request scheduling + #[allow(dead_code)] + priority_queue: Arc>, +} + +impl std::fmt::Debug for ConcurrencyManager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use std::sync::atomic::Ordering; + let io_metrics_info = if let Ok(metrics) = self.io_metrics.lock() { + format!("avg_wait={:?}, observations={}", metrics.average_wait(), metrics.observation_count()) + } else { + "locked".to_string() + }; + f.debug_struct("ConcurrencyManager") + .field("active_requests", &super::io_schedule::ACTIVE_GET_REQUESTS.load(Ordering::Relaxed)) + .field("disk_read_permits", &self.disk_read_semaphore.available_permits()) + .field("io_metrics", &io_metrics_info) + .finish() + } +} +#[allow(dead_code)] +impl ConcurrencyManager { + /// Create a new concurrency manager with default settings + /// + /// Reads configuration from environment variables: + /// - `RUSTFS_OBJECT_CACHE_ENABLE`: Enable/disable object caching (default: false) + pub fn new() -> Self { + let cache_enabled = + rustfs_utils::get_env_bool(rustfs_config::ENV_OBJECT_CACHE_ENABLE, rustfs_config::DEFAULT_OBJECT_CACHE_ENABLE); + + let max_disk_reads = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS, + rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS, + ); + + Self { + cache: Arc::new(HotObjectCache::new()), + disk_read_semaphore: Arc::new(Semaphore::new(max_disk_reads)), + cache_enabled, + io_metrics: Arc::new(Mutex::new(IoLoadMetrics::new(100))), // Keep last 100 observations + priority_queue: Arc::new(IoPriorityQueue::new(IoPriorityQueueConfig::default())), + } + } + + /// Check if object caching is enabled + /// + /// Returns true if the `RUSTFS_OBJECT_CACHE_ENABLE` environment variable + /// is set to "true" (case-insensitive). When disabled, cache lookups and + /// writebacks are skipped, reducing memory usage at the cost of repeated + /// disk reads for the same objects. + /// + /// # Returns + /// + /// `true` if caching is enabled, `false` otherwise + pub fn is_cache_enabled(&self) -> bool { + self.cache_enabled + } + + /// Track a GetObject request + pub fn track_request() -> GetObjectGuard { + GetObjectGuard::new() + } + + /// Try to get an object from cache + pub async fn get_cached(&self, key: &str) -> Option>> { + self.cache.get(key).await + } + + /// Cache an object for future retrievals + pub async fn cache_object(&self, key: String, data: Vec) { + let size = data.len(); + let cached_obj = Arc::new(CachedObject::new_with_size(data, size)); + self.cache.put(key, cached_obj).await; + } + + /// Acquire a permit to perform a disk read operation + /// + /// This ensures we don't overwhelm the disk subsystem with too many + /// concurrent reads, which can cause performance degradation. + pub async fn acquire_disk_read_permit(&self) -> Result, tokio::sync::AcquireError> { + self.disk_read_semaphore.acquire().await + } + + // ============================================ + // Adaptive I/O Strategy Methods + // ============================================ + + /// Record a disk permit wait observation for load tracking. + /// + /// This method updates the rolling metrics used to calculate adaptive I/O + /// strategies. Should be called after each disk permit acquisition. + /// + /// # Arguments + /// + /// * `wait_duration` - Time spent waiting for the disk read permit + pub fn record_permit_wait(&self, wait_duration: Duration) { + if let Ok(mut metrics) = self.io_metrics.lock() { + metrics.record(wait_duration); + } + + // Record histogram metric for Prometheus + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::histogram; + histogram!("rustfs.disk.permit.wait.duration.seconds").record(wait_duration.as_secs_f64()); + } + } + + /// Calculate an adaptive I/O strategy based on disk permit wait time. + /// + /// This method analyzes the permit wait duration to determine the current + /// I/O load level and returns optimized parameters for the read operation. + /// + /// # Arguments + /// + /// * `permit_wait_duration` - Time spent waiting for disk read permit + /// * `base_buffer_size` - Base buffer size from workload configuration + /// + /// # Returns + /// + /// An `IoStrategy` containing optimized I/O parameters. + /// + /// # Example + /// + /// ```ignore + /// let permit_wait_start = Instant::now(); + /// let _permit = manager.acquire_disk_read_permit().await; + /// let permit_wait_duration = permit_wait_start.elapsed(); + /// + /// let strategy = manager.calculate_io_strategy(permit_wait_duration, 256 * 1024); + /// let optimal_buffer = strategy.buffer_size; + /// ``` + pub fn calculate_io_strategy(&self, permit_wait_duration: Duration, base_buffer_size: usize) -> IoStrategy { + // Record the observation for future smoothing + self.record_permit_wait(permit_wait_duration); + + // Calculate strategy from the current wait duration + IoStrategy::from_wait_duration(permit_wait_duration, base_buffer_size) + } + + /// Get the smoothed I/O load level based on recent observations. + /// + /// This uses the rolling window of permit wait times to provide a more + /// stable estimate of the current load level, reducing oscillation from + /// transient spikes. + /// + /// # Returns + /// + /// The smoothed `IoLoadLevel` based on average recent wait times. + pub fn smoothed_load_level(&self) -> IoLoadLevel { + if let Ok(metrics) = self.io_metrics.lock() { + metrics.smoothed_load_level() + } else { + IoLoadLevel::Medium // Default to medium if lock fails + } + } + + /// Get I/O load statistics for monitoring. + /// + /// Returns statistics about recent disk permit wait times for + /// monitoring dashboards and capacity planning. + /// + /// # Returns + /// + /// A tuple of (average_wait, p95_wait, max_wait, observation_count) + pub fn io_load_stats(&self) -> (Duration, Duration, Duration, u64) { + if let Ok(metrics) = self.io_metrics.lock() { + ( + metrics.average_wait(), + metrics.p95_wait(), + metrics.max_wait(), + metrics.observation_count(), + ) + } else { + (Duration::ZERO, Duration::ZERO, Duration::ZERO, 0) + } + } + + /// Get the recommended buffer size based on current I/O load. + /// + /// This is a convenience method that combines load level detection with + /// buffer size calculation. Uses the smoothed load level for stability. + /// + /// # Arguments + /// + /// * `base_buffer_size` - Base buffer size from workload configuration + /// + /// # Returns + /// + /// Recommended buffer size in bytes. + pub fn adaptive_buffer_size(&self, base_buffer_size: usize) -> usize { + let load_level = self.smoothed_load_level(); + let multiplier = match load_level { + IoLoadLevel::Low => 1.0, + IoLoadLevel::Medium => 0.75, + IoLoadLevel::High => 0.5, + IoLoadLevel::Critical => 0.4, + }; + + let buffer_size = ((base_buffer_size as f64) * multiplier) as usize; + buffer_size.clamp(32 * KI_B, MI_B) + } + + /// Get cache statistics + pub async fn cache_stats(&self) -> CacheStats { + self.cache.stats().await + } + + /// Clear all cached objects + pub async fn clear_cache(&self) { + self.cache.clear().await; + } + + /// Check if a key is cached + pub async fn is_cached(&self, key: &str) -> bool { + self.cache.contains(key).await + } + + /// Get multiple cached objects in a single operation + pub async fn get_cached_batch(&self, keys: &[String]) -> Vec>>> { + self.cache.get_batch(keys).await + } + + /// Remove a specific object from cache + pub async fn remove_cached(&self, key: &str) -> bool { + self.cache.remove(key).await + } + + /// Get the most frequently accessed keys + pub async fn get_hot_keys(&self, limit: usize) -> Vec<(String, u64)> { + self.cache.get_hot_keys(limit).await + } + + /// Get cache hit rate percentage + pub fn cache_hit_rate(&self) -> f64 { + self.cache.hit_rate() + } + + /// Warm up cache with frequently accessed objects + /// + /// This can be called during server startup or maintenance windows + /// to pre-populate the cache with known hot objects. + pub async fn warm_cache(&self, objects: Vec<(String, Vec)>) { + self.cache.warm(objects).await; + } + + /// Get optimized buffer size for a request + /// + /// This wraps the advanced buffer sizing logic and makes it accessible + /// through the concurrency manager interface. + pub fn buffer_size(&self, file_size: i64, base: usize, sequential: bool) -> usize { + get_advanced_buffer_size(file_size, base, sequential) + } + + // ============================================ + // Response Cache Methods (CachedGetObject) + // ============================================ + + /// Get a cached GetObject response with full metadata + /// + /// This method retrieves a complete GetObject response from the response cache, + /// including body data and all response metadata (e_tag, last_modified, content_type, etc.). + /// + /// # Arguments + /// + /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" + /// + /// # Returns + /// + /// * `Some(Arc)` - Cached response data if found and not expired + /// * `None` - Cache miss + /// + /// # Example + /// + /// ```ignore + /// let cache_key = format!("{}/{}", bucket, key); + /// if let Some(cached) = manager.get_cached_object(&cache_key).await { + /// // Build response from cached data + /// let output = GetObjectOutput { + /// body: Some(StreamingBlob::from(cached.body.clone())), + /// content_length: Some(cached.content_length), + /// e_tag: cached.e_tag.clone(), + /// last_modified: cached.last_modified.as_ref().map(|s| parse_rfc3339(s)), + /// ..Default::default() + /// }; + /// } + /// ``` + pub async fn get_cached_object(&self, key: &str) -> Option> { + self.cache.get_response(key).await + } + + /// Cache a complete GetObject response for future retrievals + /// + /// This method caches a complete GetObject response including body and all metadata. + /// Objects larger than the maximum cache size (10MB by default) or empty objects + /// are not cached. + /// + /// # Arguments + /// + /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" + /// * `response` - The complete cached response to store + /// + /// # Example + /// + /// ```ignore + /// let cached = CachedGetObject { + /// body: Bytes::from(data), + /// content_length: data.len() as i64, + /// content_type: Some("application/octet-stream".to_string()), + /// e_tag: Some("\"abc123\"".to_string()), + /// last_modified: Some("2024-01-01T00:00:00Z".to_string()), + /// ..Default::default() + /// }; + /// manager.put_cached_object(cache_key, cached).await; + /// ``` + pub async fn put_cached_object(&self, key: String, response: CachedGetObject) { + self.cache.put_response(key, response).await; + } + + /// Invalidate cache entries for a specific object + /// + /// This method removes both simple byte cache and response cache entries + /// for the given key. Should be called after write operations (put_object, + /// copy_object, delete_object, etc.) to prevent stale data from being served. + /// + /// # Arguments + /// + /// * `key` - Cache key to invalidate (e.g., "{bucket}/{key}") + /// + /// # Example + /// + /// ```ignore + /// // After put_object succeeds + /// let cache_key = format!("{}/{}", bucket, key); + /// manager.invalidate_cache(&cache_key).await; + /// ``` + pub async fn invalidate_cache(&self, key: &str) { + self.cache.invalidate(key).await; + } + + /// Invalidate cache entries for an object and its latest version + /// + /// For versioned buckets, this invalidates both: + /// - The specific version key: "{bucket}/{key}?versionId={version_id}" + /// - The latest version key: "{bucket}/{key}" + /// + /// This ensures that after a write/delete, clients don't receive stale data. + /// Should be called after any write operation that modifies object data or creates + /// new versions. + /// + /// # Arguments + /// + /// * `bucket` - Bucket name + /// * `key` - Object key + /// * `version_id` - Optional version ID (if None, only invalidates the base key) + /// + /// # Example + /// + /// ```ignore + /// // After delete_object with version + /// manager.invalidate_cache_versioned(&bucket, &key, Some(&version_id)).await; + /// + /// // After put_object (invalidates latest) + /// manager.invalidate_cache_versioned(&bucket, &key, None).await; + /// ``` + pub async fn invalidate_cache_versioned(&self, bucket: &str, key: &str, version_id: Option<&str>) { + self.cache.invalidate_versioned(bucket, key, version_id).await; + } + + /// Generate a cache key for an object + /// + /// Creates a cache key in the appropriate format based on whether a version ID + /// is specified. For versioned requests, uses "{bucket}/{key}?versionId={version_id}". + /// For non-versioned requests, uses "{bucket}/{key}". + /// + /// # Arguments + /// + /// * `bucket` - Bucket name + /// * `key` - Object key + /// * `version_id` - Optional version ID + /// + /// # Returns + /// + /// Cache key string + pub fn make_cache_key(bucket: &str, key: &str, version_id: Option<&str>) -> String { + match version_id { + Some(vid) => format!("{bucket}/{key}?versionId={vid}"), + None => format!("{bucket}/{key}"), + } + } + + /// Get maximum cacheable object size + /// + /// Returns the maximum size in bytes for objects that can be cached. + /// Objects larger than this size are not cached to prevent memory exhaustion. + pub fn max_object_size(&self) -> usize { + self.cache.max_object_size() + } + + // ============================================ + // Priority-Based I/O Scheduling Methods + // ============================================ + + /// Get I/O priority for a request based on its size. + /// + /// This enables priority-based scheduling where small requests + /// are processed before large requests to prevent starvation. + /// + /// # Arguments + /// + /// * `request_size` - Size of the request in bytes (-1 if unknown) + /// + /// # Returns + /// + /// Priority level (High, Normal, or Low) + pub fn get_io_priority(&self, request_size: i64) -> IoPriority { + if request_size < 0 { + // Unknown size, use normal priority + IoPriority::Normal + } else { + IoPriority::from_size(request_size) + } + } + + /// Check if priority scheduling is enabled. + pub fn is_priority_scheduling_enabled(&self) -> bool { + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE, + rustfs_config::DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE, + ) + } + + /// Get current I/O queue status for monitoring. + /// + /// Returns information about permit usage and waiting requests. + pub fn io_queue_status(&self) -> IoQueueStatus { + let total_permits = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS, + rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS, + ); + let permits_in_use = total_permits.saturating_sub(self.disk_read_semaphore.available_permits()); + + IoQueueStatus { + total_permits, + permits_in_use, + high_priority_waiting: 0, // Would need additional tracking + normal_priority_waiting: 0, + low_priority_waiting: 0, + high_priority_processed: 0, + normal_priority_processed: 0, + low_priority_processed: 0, + starvation_events: 0, + } + } + + /// Acquire a disk read permit with priority awareness. + /// + /// When priority scheduling is enabled, this method logs the priority + /// for observability. The actual acquisition uses the same semaphore + /// but priority information is used for monitoring. + /// + /// # Arguments + /// + /// * `priority` - Priority level for this request + /// + /// # Returns + /// + /// Semaphore permit on success, error on failure + pub async fn acquire_priority_permit( + &self, + priority: IoPriority, + ) -> Result, tokio::sync::AcquireError> { + #[cfg(feature = "metrics")] + { + use metrics::counter; + counter!("rustfs.disk.read.queue.total", "priority" => priority.as_str()).increment(1); + } + + debug!( + priority = %priority, + available_permits = self.disk_read_semaphore.available_permits(), + "Acquiring disk read permit" + ); + + self.disk_read_semaphore.acquire().await + } + + /// Get the global concurrency manager instance. + pub fn global() -> &'static Self { + &CONCURRENCY_MANAGER + } +} + +impl Default for ConcurrencyManager { + fn default() -> Self { + Self::new() + } +} + +// ============================================ +// Integration Tests for ConcurrencyManager +// ============================================ + +#[cfg(test)] +mod integration_tests { + use super::*; + use bytes::Bytes; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_priority_queue_integration() { + let manager = ConcurrencyManager::new(); + + // Test priority determination + let small_size = 100 * 1024; // 100KB + let large_size = 200 * 1024 * 1024; // 200MB + + let small_priority = manager.get_io_priority(small_size as i64); + let large_priority = manager.get_io_priority(large_size as i64); + + assert_eq!(small_priority, IoPriority::High); + assert_eq!(large_priority, IoPriority::Low); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_cache_operations() { + let manager = ConcurrencyManager::new(); + + // Test cache put and get + let obj = CachedGetObject::new(Bytes::from("test data"), 9) + .with_content_type("text/plain".to_string()) + .with_e_tag("\"abc123\"".to_string()); + + manager.put_cached_object("test-key".to_string(), obj).await; + + let cached = manager.get_cached_object("test-key").await; + assert!(cached.is_some()); + + let cached_obj = cached.unwrap(); + assert_eq!(cached_obj.content_type, Some("text/plain".to_string())); + assert_eq!(cached_obj.e_tag, Some("\"abc123\"".to_string())); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_cache_stats() { + let manager = ConcurrencyManager::new(); + + // Add some objects + for i in 0..5 { + let obj = CachedGetObject::new(Bytes::from(format!("data{}", i)), 5); + manager.put_cached_object(format!("key{}", i), obj).await; + } + + // Get stats + let stats = manager.cache_stats().await; + + assert!(stats.entries >= 5); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_io_queue_status() { + let manager = ConcurrencyManager::new(); + + let status = manager.io_queue_status(); + + // Initial state should have no waiting requests + assert_eq!(status.high_priority_waiting, 0); + assert_eq!(status.normal_priority_waiting, 0); + assert_eq!(status.low_priority_waiting, 0); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_disk_read_permit() { + let manager = ConcurrencyManager::new(); + + // Acquire permit + let permit = manager.acquire_disk_read_permit().await; + assert!(permit.is_ok()); + + // Permit should be released when dropped + drop(permit); + + // Should be able to acquire again + let permit2 = manager.acquire_disk_read_permit().await; + assert!(permit2.is_ok()); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_priority_scheduling() { + let manager = ConcurrencyManager::new(); + + // Test if priority scheduling is enabled + let enabled = manager.is_priority_scheduling_enabled(); + assert!(enabled); // Should be enabled by default + + // Test priority determination for different sizes + assert_eq!(manager.get_io_priority(500 * 1024), IoPriority::High); // 500KB + assert_eq!(manager.get_io_priority(5 * 1024 * 1024), IoPriority::Normal); // 5MB + assert_eq!(manager.get_io_priority(50 * 1024 * 1024), IoPriority::Low); // 50MB + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_cache_invalidation() { + let manager = ConcurrencyManager::new(); + + // Add an object + let obj = CachedGetObject::new(Bytes::from("test"), 4); + manager.put_cached_object("test-key".to_string(), obj).await; + + // Verify it's cached + assert!(manager.is_cached("test-key").await); + + // Invalidate + manager.invalidate_cache("test-key").await; + + // Should not be cached anymore + assert!(!manager.is_cached("test-key").await); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_cache_clear() { + let manager = ConcurrencyManager::new(); + + // Add multiple objects + for i in 0..10 { + let obj = CachedGetObject::new(Bytes::from(format!("data{}", i)), 5); + manager.put_cached_object(format!("key{}", i), obj).await; + } + + // Clear cache + manager.clear_cache().await; + + // Verify all are removed + let stats = manager.cache_stats().await; + assert_eq!(stats.entries, 0); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_io_strategy() { + let manager = ConcurrencyManager::new(); + + // Test I/O strategy calculation + let low_wait = Duration::from_millis(5); + let high_wait = Duration::from_millis(100); + + let strategy_low = manager.calculate_io_strategy(low_wait, 128 * 1024); + let strategy_high = manager.calculate_io_strategy(high_wait, 128 * 1024); + + // Under low load, should use larger buffers + assert!(strategy_low.buffer_size >= strategy_high.buffer_size); + } + + #[tokio::test] + #[serial] + async fn test_concurrency_manager_adaptive_buffer_size() { + let manager = ConcurrencyManager::new(); + + let base_size = 128 * 1024; // 128KB + + // Test adaptive buffer sizing + let size1 = manager.adaptive_buffer_size(base_size); + + // Should return a reasonable buffer size + assert!(size1 > 0); + assert!(size1 <= 2 * 1024 * 1024); // Not more than 2MB + } +} diff --git a/rustfs/src/storage/concurrency/mod.rs b/rustfs/src/storage/concurrency/mod.rs new file mode 100644 index 000000000..211c3bb14 --- /dev/null +++ b/rustfs/src/storage/concurrency/mod.rs @@ -0,0 +1,57 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Concurrency optimization module for high-performance object retrieval. + +// Sub-modules +pub mod io_schedule; +pub mod manager; +pub mod object_cache; +pub mod request_guard; + +// ============================================ +// Public API Re-exports +// ============================================ + +// I/O scheduling types +#[allow(unused_imports)] +pub use io_schedule::{ + IO_PRIORITY_METRICS, IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, + IoStrategy, get_advanced_buffer_size, get_concurrency_aware_buffer_size, +}; + +// Request tracking +pub use request_guard::GetObjectGuard; + +// Cache types +#[allow(unused_imports)] +pub use object_cache::{CacheHealthStatus, CacheStats, CachedGetObject}; + +// Concurrency manager +pub use manager::ConcurrencyManager; + +// ============================================ +// Helper Functions +// ============================================ + +/// Get the global concurrency manager instance. +pub fn get_concurrency_manager() -> &'static ConcurrencyManager { + ConcurrencyManager::global() +} + +/// Reset the active get requests counter (for testing). +#[allow(dead_code)] +pub fn reset_active_get_requests() { + io_schedule::ACTIVE_GET_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed); +} diff --git a/rustfs/src/storage/concurrency/object_cache.rs b/rustfs/src/storage/concurrency/object_cache.rs new file mode 100644 index 000000000..a344e97f4 --- /dev/null +++ b/rustfs/src/storage/concurrency/object_cache.rs @@ -0,0 +1,1218 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Object cache module for hot object caching with Moka. + +use moka::future::Cache; +use rustfs_config::MI_B; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +/// Hot object cache for frequently accessed objects. +pub(crate) struct HotObjectCache { + /// Moka cache instance for simple byte data (legacy) + cache: Cache>, + /// Moka cache instance for full GetObject responses with metadata + response_cache: Cache>, + /// Maximum total cache capacity in bytes + max_capacity: usize, + /// Maximum size of individual objects to cache (10MB by default) + max_object_size: usize, + /// Global cache hit counter + hit_count: Arc, + /// Global cache miss counter + miss_count: Arc, +} + +impl std::fmt::Debug for HotObjectCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use std::sync::atomic::Ordering; + f.debug_struct("HotObjectCache") + .field("max_capacity", &self.max_capacity) + .field("max_object_size", &self.max_object_size) + .field("hit_count", &self.hit_count.load(Ordering::Relaxed)) + .field("miss_count", &self.miss_count.load(Ordering::Relaxed)) + .finish() + } +} + +pub(crate) struct CachedObject { + /// The object data + data: Arc>, + /// When this object was cached + cached_at: Instant, + /// Object size in bytes + size: usize, + /// Number of times this object has been accessed + access_count: Arc, +} + +impl CachedObject { + /// Create a new CachedObject with specified size + pub fn new_with_size(data: Vec, size: usize) -> Self { + Self { + data: Arc::new(data), + cached_at: Instant::now(), + size, + access_count: Arc::new(AtomicU64::new(0)), + } + } + + /// Get the size of the cached object + #[allow(dead_code)] + pub fn size(&self) -> usize { + self.size + } + + /// Get the data reference + #[allow(dead_code)] + pub fn data(&self) -> &Arc> { + &self.data + } + + /// Get the age of the cached object + #[allow(dead_code)] + pub fn age(&self) -> Duration { + self.cached_at.elapsed() + } + + /// Increment access count and return new value + #[allow(dead_code)] + pub fn increment_access(&self) -> u64 { + self.access_count.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// Get current access count + #[allow(dead_code)] + pub fn access_count(&self) -> u64 { + self.access_count.load(Ordering::Relaxed) + } +} + +/// Comprehensive cached object with full response metadata for GetObject operations. +/// +/// This structure stores all necessary fields to reconstruct a complete GetObjectOutput +/// response from cache, avoiding repeated disk reads and metadata lookups for hot objects. +/// +/// # Fields +/// +/// All time fields are serialized as RFC3339 strings to avoid parsing issues with +/// `Last-Modified` and other time headers. +/// +/// # Usage +/// +/// ```ignore +/// let cached = CachedGetObject { +/// body: Bytes::from(data), +/// content_length: data.len() as i64, +/// content_type: Some("application/octet-stream".to_string()), +/// e_tag: Some("\"abc123\"".to_string()), +/// last_modified: Some("2024-01-01T00:00:00Z".to_string()), +/// ..Default::default() +/// }; +/// manager.put_cached_object(cache_key, cached).await; +/// ``` +#[derive(Clone, Debug)] +#[allow(dead_code)] +pub struct CachedGetObject { + /// The object body data + pub body: bytes::Bytes, + /// Content length in bytes + pub content_length: i64, + /// MIME content type + pub content_type: Option, + /// Entity tag for the object + pub e_tag: Option, + /// Last modified time as RFC3339 string (e.g., "2024-01-01T12:00:00Z") + pub last_modified: Option, + /// Expiration time as RFC3339 string + pub expires: Option, + /// Cache-Control header value + pub cache_control: Option, + /// Content-Disposition header value + pub content_disposition: Option, + /// Content-Encoding header value + pub content_encoding: Option, + /// Content-Language header value + pub content_language: Option, + /// Storage class (STANDARD, REDUCED_REDUNDANCY, etc.) + pub storage_class: Option, + /// Version ID for versioned objects + pub version_id: Option, + /// Whether this is a delete marker (for versioned buckets) + pub delete_marker: bool, + /// Number of tags associated with the object + pub tag_count: Option, + /// Replication status + pub replication_status: Option, + /// User-defined metadata (x-amz-meta-*) + pub user_metadata: std::collections::HashMap, + /// When this object was cached (for internal use, automatically set) + #[allow(dead_code)] + cached_at: Option, + /// Access count for hot key tracking (automatically managed) + access_count: Arc, +} + +impl Default for CachedGetObject { + fn default() -> Self { + Self { + body: bytes::Bytes::new(), + content_length: 0, + content_type: None, + e_tag: None, + last_modified: None, + expires: None, + cache_control: None, + content_disposition: None, + content_encoding: None, + content_language: None, + storage_class: None, + version_id: None, + delete_marker: false, + tag_count: None, + replication_status: None, + user_metadata: std::collections::HashMap::new(), + cached_at: None, + access_count: Arc::new(AtomicU64::new(0)), + } + } +} + +#[allow(dead_code)] +impl CachedGetObject { + /// Create a new CachedGetObject with the given body and content length + pub fn new(body: bytes::Bytes, content_length: i64) -> Self { + Self { + body, + content_length, + cached_at: Some(Instant::now()), + access_count: Arc::new(AtomicU64::new(0)), + ..Default::default() + } + } + + /// Builder method to set content_type + pub fn with_content_type(mut self, content_type: String) -> Self { + self.content_type = Some(content_type); + self + } + + /// Builder method to set e_tag + pub fn with_e_tag(mut self, e_tag: String) -> Self { + self.e_tag = Some(e_tag); + self + } + + /// Builder method to set last_modified + pub fn with_last_modified(mut self, last_modified: String) -> Self { + self.last_modified = Some(last_modified); + self + } + + /// Builder method to set cache_control + pub fn with_cache_control(mut self, cache_control: String) -> Self { + self.cache_control = Some(cache_control); + self + } + + /// Builder method to set storage_class + pub fn with_storage_class(mut self, storage_class: String) -> Self { + self.storage_class = Some(storage_class); + self + } + + /// Builder method to set version_id + pub fn with_version_id(mut self, version_id: String) -> Self { + self.version_id = Some(version_id); + self + } + + /// Builder method to set expires + pub fn with_expires(mut self, expires: String) -> Self { + self.expires = Some(expires); + self + } + + /// Builder method to set content_encoding + pub fn with_content_encoding(mut self, content_encoding: String) -> Self { + self.content_encoding = Some(content_encoding); + self + } + + /// Builder method to set content_disposition + pub fn with_content_disposition(mut self, content_disposition: String) -> Self { + self.content_disposition = Some(content_disposition); + self + } + + /// Builder method to set content_language + #[allow(dead_code)] + pub fn with_content_language(mut self, content_language: String) -> Self { + self.content_language = Some(content_language); + self + } + + /// Builder method to set replication_status + pub fn with_replication_status(mut self, replication_status: String) -> Self { + self.replication_status = Some(replication_status); + self + } + + /// Builder method to set delete_marker + pub fn with_delete_marker(mut self, delete_marker: bool) -> Self { + self.delete_marker = delete_marker; + self + } + + /// Builder method to set user_metadata + pub fn with_user_metadata(mut self, user_metadata: std::collections::HashMap) -> Self { + self.user_metadata = user_metadata; + self + } + + /// Builder method to set tag_count + pub fn with_tag_count(mut self, tag_count: i32) -> Self { + self.tag_count = Some(tag_count); + self + } + pub fn size(&self) -> usize { + self.body.len() + } + + /// Increment access count and return the new value + pub fn increment_access(&self) -> u64 { + self.access_count.fetch_add(1, Ordering::Relaxed) + 1 + } + /// Check if the cached object is expired based on expires header + pub fn is_expired(&self) -> bool { + if let Some(expires_str) = &self.expires { + // Try to parse RFC3339 format + if let Ok(expires_time) = chrono::DateTime::parse_from_rfc3339(expires_str) { + let now = chrono::Utc::now(); + return expires_time < now; + } + } + false + } + + /// Check if replication is complete + pub fn is_replication_complete(&self) -> bool { + match &self.replication_status { + Some(status) => status == "COMPLETED", + None => true, // No replication configured + } + } + + /// Get the age of this cached entry + #[allow(dead_code)] + pub fn age(&self) -> Option { + self.cached_at.map(|at| at.elapsed()) + } + + /// Record a cache hit and increment access count + pub fn record_hit(&self) -> u64 { + self.increment_access() + } + + /// Estimate memory size in bytes including metadata + pub fn memory_size(&self) -> usize { + let mut size = self.body.len(); + size += size_of::(); // content_length + size += self.content_type.as_ref().map_or(0, |s| s.len()); + size += self.e_tag.as_ref().map_or(0, |s| s.len()); + size += self.last_modified.as_ref().map_or(0, |s| s.len()); + size += self.expires.as_ref().map_or(0, |s| s.len()); + size += self.cache_control.as_ref().map_or(0, |s| s.len()); + size += self.content_disposition.as_ref().map_or(0, |s| s.len()); + size += self.content_encoding.as_ref().map_or(0, |s| s.len()); + size += self.content_language.as_ref().map_or(0, |s| s.len()); + size += self.storage_class.as_ref().map_or(0, |s| s.len()); + size += self.version_id.as_ref().map_or(0, |s| s.len()); + size += self.replication_status.as_ref().map_or(0, |s| s.len()); + size += size_of::(); // delete_marker + size += size_of::>(); // tag_count + // Estimate user_metadata size + for (k, v) in &self.user_metadata { + size += k.len() + v.len(); + } + size + } +} + +/// Internal wrapper for CachedGetObject in the Moka cache +#[derive(Clone)] +struct CachedGetObjectInternal { + /// The cached response data + data: Arc, + /// When this object was cached + cached_at: Instant, + /// Size in bytes for weigher function + size: usize, +} + +impl HotObjectCache { + /// Create a new hot object cache with Moka + /// + /// Configures Moka with: + /// - Size-based eviction (100MB max) + /// - TTL of 5 minutes + /// - TTI of 2 minutes + /// - Weigher function for accurate size tracking + pub(crate) fn new() -> Self { + let max_capacity = rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_CACHE_CAPACITY_MB, + rustfs_config::DEFAULT_OBJECT_CACHE_CAPACITY_MB, + ); + let cache_tti_secs = + rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTI_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTI_SECS); + let cache_ttl_secs = + rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS); + + // Legacy simple byte cache + let cache = Cache::builder() + .max_capacity(max_capacity * MI_B as u64) + .weigher(|_key: &String, value: &Arc| -> u32 { + // Weight based on actual data size + value.size.min(u32::MAX as usize) as u32 + }) + .time_to_live(Duration::from_secs(cache_ttl_secs)) + .time_to_idle(Duration::from_secs(cache_tti_secs)) + .build(); + + // Full response cache with metadata + let response_cache = Cache::builder() + .max_capacity(max_capacity * MI_B as u64) + .weigher(|_key: &String, value: &Arc| -> u32 { + // Weight based on actual data size + value.size.min(u32::MAX as usize) as u32 + }) + .time_to_live(Duration::from_secs(cache_ttl_secs)) + .time_to_idle(Duration::from_secs(cache_tti_secs)) + .build(); + let max_object_size = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_CACHE_MAX_OBJECT_SIZE_MB, + rustfs_config::DEFAULT_OBJECT_CACHE_MAX_OBJECT_SIZE_MB, + ) * MI_B; + Self { + cache, + max_capacity: (max_capacity * MI_B as u64) as usize, + response_cache, + max_object_size, + hit_count: Arc::new(AtomicU64::new(0)), + miss_count: Arc::new(AtomicU64::new(0)), + } + } + + /// Soft expiration determination, the number of hits is insufficient and exceeds the soft TTL + pub(crate) fn should_expire(&self, obj: &Arc) -> bool { + let age_secs = obj.cached_at.elapsed().as_secs(); + let cache_ttl_secs = + rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS); + let hot_object_min_hits_to_extend = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND, + rustfs_config::DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND, + ); + if age_secs >= cache_ttl_secs { + let hits = obj.access_count.load(Ordering::Relaxed); + return hits < hot_object_min_hits_to_extend as u64; + } + false + } + + /// Get an object from cache with lock-free concurrent access + /// + /// Moka provides lock-free reads, significantly improving concurrent performance. + pub(crate) async fn get(&self, key: &str) -> Option>> { + match self.cache.get(key).await { + Some(cached) => { + if self.should_expire(&cached) { + self.cache.invalidate(key).await; + self.miss_count.fetch_add(1, Ordering::Relaxed); + return None; + } + // Update access count + cached.access_count.fetch_add(1, Ordering::Relaxed); + self.hit_count.fetch_add(1, Ordering::Relaxed); + + // IMPORTANT: Do NOT add high cardinality labels to metrics! + // Previously, this metric was tagged with individual file URIs/keys, + // causing unbounded memory growth in RustFS's own process. The metrics + // crate maintains an internal HashMap for all metric series, and each + // unique file path creates a new entry that is never cleaned up. + // This HashMap grows unbounded with unique file access, causing memory + // leaks in RustFS itself (and also in downstream systems like Prometheus). + // Only use low cardinality labels like operation type or status. + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::counter; + counter!("rustfs.object.cache.hits").increment(1); + } + + Some(Arc::clone(&cached.data)) + } + None => { + self.miss_count.fetch_add(1, Ordering::Relaxed); + + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::counter; + counter!("rustfs.object.cache.misses").increment(1); + } + + None + } + } + } + + /// Put an object into cache with automatic size-based eviction + /// + /// Moka handles eviction automatically based on the weigher function. + pub(crate) async fn put(&self, key: String, data: Arc) { + let size = data.size; + + // Only cache objects smaller than max_object_size + if size == 0 || size > self.max_object_size { + return; + } + + let cached_obj = Arc::new(CachedObject { + data: Arc::clone(&data.data), + cached_at: Instant::now(), + size, + access_count: Arc::new(AtomicU64::new(0)), + }); + + self.cache.insert(key.clone(), cached_obj).await; + + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::{counter, gauge}; + counter!("rustfs.object.cache.insertions").increment(1); + gauge!("rustfs_object_cache_size_bytes").set(self.cache.weighted_size() as f64); + + gauge!("rustfs_object_cache_entry_count").set(self.cache.entry_count() as f64); + } + } + + /// Clear all cached objects + pub(crate) async fn clear(&self) { + // Clear both simple cache and response cache + self.cache.invalidate_all(); + self.response_cache.invalidate_all(); + // Sync to ensure all entries are removed + self.cache.run_pending_tasks().await; + self.response_cache.run_pending_tasks().await; + } + + /// Get cache statistics for monitoring + pub(crate) async fn stats(&self) -> CacheStats { + // Ensure pending tasks are processed for accurate stats in both caches + self.cache.run_pending_tasks().await; + self.response_cache.run_pending_tasks().await; + + // Calculate average age for simple cache + let mut total_ms: u128 = 0; + let mut cnt: u64 = 0; + self.cache.iter().for_each(|(_, v)| { + total_ms += v.cached_at.elapsed().as_millis(); + cnt += 1; + }); + + // Calculate average age for response cache + let mut response_total_ms: u128 = 0; + let mut response_cnt: u64 = 0; + self.response_cache.iter().for_each(|(_, v)| { + response_total_ms += v.cached_at.elapsed().as_millis(); + response_cnt += 1; + }); + + // Combine average age calculation + let total_entries = cnt + response_cnt; + let combined_total_ms = total_ms + response_total_ms; + let avg_age_secs = if total_entries == 0 { + 0.0 + } else { + (combined_total_ms as f64 / total_entries as f64) / 1000.0 + }; + + let hit_count = self.hit_count.load(Ordering::Relaxed); + let miss_count = self.miss_count.load(Ordering::Relaxed); + let total_requests = hit_count + miss_count; + let hit_rate = if total_requests > 0 { + hit_count as f64 / total_requests as f64 + } else { + 0.0 + }; + + // Calculate total size from both caches + let simple_size = self.cache.weighted_size() as usize; + let response_size = self.response_cache.weighted_size() as usize; + let total_size = simple_size + response_size; + + let memory_usage_ratio = if self.max_capacity > 0 { + total_size as f64 / self.max_capacity as f64 + } else { + 0.0 + }; + + let efficiency_score = if total_entries == 0 { + // Empty cache has no actual utility, efficiency score is 0 + 0 + } else { + // Non-empty cache: hit rate contributes 50%, remaining capacity contributes 50% + (hit_rate * 50.0 + (1.0 - memory_usage_ratio) * 50.0) as u32 + }; + + CacheStats { + size: total_size, + entries: total_entries as usize, + max_size: self.max_capacity, + max_object_size: self.max_object_size, + hit_count, + miss_count, + avg_age_secs, + hit_rate, + eviction_count: 0, // Moka doesn't expose eviction count + eviction_rate: 0.0, + memory_usage: total_size, + memory_usage_ratio, + top_keys: vec![], // Would need additional tracking + efficiency_score, + } + } + + /// Check if a key exists in cache (lock-free) + pub(crate) async fn contains(&self, key: &str) -> bool { + // Check both simple cache and response cache + self.cache.contains_key(key) || self.response_cache.contains_key(key) + } + + /// Get multiple objects from cache in parallel + /// + /// Leverages Moka's lock-free design for true parallel access. + pub(crate) async fn get_batch(&self, keys: &[String]) -> Vec>>> { + let mut results = Vec::with_capacity(keys.len()); + for key in keys { + results.push(self.get(key).await); + } + results + } + + /// Remove a specific key from cache + pub(crate) async fn remove(&self, key: &str) -> bool { + let had_key = self.cache.contains_key(key); + self.cache.invalidate(key).await; + had_key + } + + /// Get the most frequently accessed keys + /// + /// Returns up to `limit` keys sorted by access count in descending order. + pub(crate) async fn get_hot_keys(&self, limit: usize) -> Vec<(String, u64)> { + // Run pending tasks to ensure accurate entry count + self.cache.run_pending_tasks().await; + + let mut entries: Vec<(String, u64)> = Vec::new(); + + // Iterate through cache entries + self.cache.iter().for_each(|(key, value)| { + entries.push((key.to_string(), value.access_count.load(Ordering::Relaxed))); + }); + + entries.sort_by(|a, b| b.1.cmp(&a.1)); + entries.truncate(limit); + entries + } + + /// Warm up cache with a batch of objects + pub(crate) async fn warm(&self, objects: Vec<(String, Vec)>) { + for (key, data) in objects { + let size = data.len(); + let cached_obj = Arc::new(CachedObject::new_with_size(data, size)); + self.put(key, cached_obj).await; + } + } + + /// Get hit rate percentage + pub(crate) fn hit_rate(&self) -> f64 { + let hits = self.hit_count.load(Ordering::Relaxed); + let misses = self.miss_count.load(Ordering::Relaxed); + let total = hits + misses; + + if total == 0 { + 0.0 + } else { + (hits as f64 / total as f64) * 100.0 + } + } + + // ============================================ + // Response Cache Methods (CachedGetObject) + // ============================================ + + /// Get a cached GetObject response with full metadata + /// + /// This method retrieves a complete GetObject response from the response cache, + /// including body data and all response metadata (e_tag, last_modified, etc.). + /// + /// # Arguments + /// + /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" + /// + /// # Returns + /// + /// * `Some(Arc)` - Cached response data if found and not expired + /// * `None` - Cache miss + pub(crate) async fn get_response(&self, key: &str) -> Option> { + match self.response_cache.get(key).await { + Some(cached) => { + // Check soft expiration + let age_secs = cached.cached_at.elapsed().as_secs(); + let cache_ttl_secs = rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_CACHE_TTL_SECS, + rustfs_config::DEFAULT_OBJECT_CACHE_TTL_SECS, + ); + let hot_object_min_hits = rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_HOT_MIN_HITS_TO_EXTEND, + rustfs_config::DEFAULT_OBJECT_HOT_MIN_HITS_TO_EXTEND, + ); + + if age_secs >= cache_ttl_secs { + let hits = cached.data.access_count.load(Ordering::Relaxed); + if hits < hot_object_min_hits as u64 { + self.response_cache.invalidate(key).await; + self.miss_count.fetch_add(1, Ordering::Relaxed); + return None; + } + } + + // Update access count + cached.data.increment_access(); + self.hit_count.fetch_add(1, Ordering::Relaxed); + + // IMPORTANT: Do NOT add high cardinality labels to metrics! + // See HotObjectCache::get() for details. The metrics crate's internal + // HashMap grows unbounded with high cardinality labels, causing memory + // leaks in RustFS's own process. + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::counter; + counter!("rustfs_object_response_cache_hits").increment(1); + } + + Some(Arc::clone(&cached.data)) + } + None => { + self.miss_count.fetch_add(1, Ordering::Relaxed); + + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::counter; + counter!("rustfs_object_response_cache_misses").increment(1); + } + + None + } + } + } + + /// Put a GetObject response into the response cache + /// + /// This method caches a complete GetObject response including body and metadata. + /// Objects larger than `max_object_size` or empty objects are not cached. + /// + /// # Arguments + /// + /// * `key` - Cache key in the format "{bucket}/{key}" or "{bucket}/{key}?versionId={version_id}" + /// * `response` - The complete cached response to store + pub(crate) async fn put_response(&self, key: String, response: CachedGetObject) { + let size = response.size(); + + // Only cache objects smaller than max_object_size + if size == 0 || size > self.max_object_size { + return; + } + + let cached_internal = Arc::new(CachedGetObjectInternal { + data: Arc::new(response), + cached_at: Instant::now(), + size, + }); + + self.response_cache.insert(key.clone(), cached_internal).await; + + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::{counter, gauge}; + counter!("rustfs_object_response_cache_insertions").increment(1); + gauge!("rustfs_object_response_cache_size_bytes").set(self.response_cache.weighted_size() as f64); + gauge!("rustfs_object_response_cache_entry_count").set(self.response_cache.entry_count() as f64); + } + } + + /// Invalidate a cache entry for a specific object + /// + /// This method removes both the simple byte cache entry and the response cache entry + /// for the given key. Used when objects are modified or deleted. + /// + /// # Arguments + /// + /// * `key` - Cache key to invalidate (e.g., "{bucket}/{key}") + pub(crate) async fn invalidate(&self, key: &str) { + // Invalidate both caches + self.cache.invalidate(key).await; + self.response_cache.invalidate(key).await; + + #[cfg(all(feature = "metrics", not(test)))] + { + use metrics::counter; + counter!("rustfs_object_cache_invalidations_total").increment(1); + } + } + + /// Invalidate cache entries for an object and its latest version + /// + /// For versioned buckets, this invalidates both: + /// - The specific version key: "{bucket}/{key}?versionId={version_id}" + /// - The latest version key: "{bucket}/{key}" + /// + /// This ensures that after a write/delete, clients don't receive stale data. + /// + /// # Arguments + /// + /// * `bucket` - Bucket name + /// * `key` - Object key + /// * `version_id` - Optional version ID (if None, only invalidates the base key) + pub(crate) async fn invalidate_versioned(&self, bucket: &str, key: &str, version_id: Option<&str>) { + // Always invalidate the latest version key + let base_key = format!("{bucket}/{key}"); + self.invalidate(&base_key).await; + + // Also invalidate the specific version if provided + if let Some(vid) = version_id { + let versioned_key = format!("{base_key}?versionId={vid}"); + self.invalidate(&versioned_key).await; + } + } + + /// Clear all cached objects from both caches + #[allow(dead_code)] + pub(crate) async fn clear_all(&self) { + self.cache.invalidate_all(); + self.response_cache.invalidate_all(); + // Sync to ensure all entries are removed + self.cache.run_pending_tasks().await; + self.response_cache.run_pending_tasks().await; + } + + /// Get the maximum object size for caching + pub(crate) fn max_object_size(&self) -> usize { + self.max_object_size + } + + /// Get cache health status + #[allow(dead_code)] + pub(crate) async fn health_status(&self) -> CacheHealthStatus { + let stats = self.stats().await; + let memory_usage = self.cache.weighted_size() as usize; + + let is_healthy = stats.memory_usage_ratio < 0.95 && stats.hit_rate > 0.1; + + let mut recommendations = Vec::new(); + if stats.hit_rate < 0.5 { + recommendations.push("Consider increasing cache size or TTL".to_string()); + } + if stats.memory_usage_ratio > 0.9 { + recommendations.push("Cache is nearly full, consider increasing capacity".to_string()); + } + + CacheHealthStatus { + memory_usage, + is_healthy, + memory_usage_ratio: stats.memory_usage_ratio, + hit_rate: stats.hit_rate, + eviction_rate: stats.eviction_rate, + avg_entry_age_secs: stats.avg_age_secs, + efficiency_score: stats.efficiency_score, + recommendations, + } + } + + /// Get current memory usage in bytes + #[allow(dead_code)] + pub(crate) async fn memory_usage(&self) -> usize { + // Sync pending tasks to ensure accurate weight statistics + self.cache.run_pending_tasks().await; + self.cache.weighted_size() as usize + } + + /// Evict a percentage of cached entries + #[allow(dead_code)] + pub(crate) async fn evict_percentage(&self, percentage: f64) -> u64 { + let stats = self.stats().await; + let entries_to_evict = (stats.entries as f64 * percentage / 100.0).max(1.0) as u64; + + // Moka does not support selective eviction, so we use invalidate_all + if entries_to_evict > 0 { + self.cache.invalidate_all(); + } + + entries_to_evict + } + + /// Warm cache from a list of hot keys + #[allow(dead_code)] + pub(crate) async fn warm_from_hot_list(&self, hot_keys: Vec<(String, Vec)>) -> u64 { + let mut warmed = 0u64; + + for (key, data) in hot_keys { + let size = data.len(); + if size <= self.max_object_size { + let cached_obj = Arc::new(CachedObject::new_with_size(data, size)); + self.cache.insert(key, cached_obj).await; + warmed += 1; + } + } + + warmed + } +} + +/// Cache statistics for monitoring and debugging +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub struct CacheStats { + /// Current total size of cached objects in bytes + pub size: usize, + /// Number of cached entries + pub entries: usize, + /// Maximum allowed cache size in bytes + pub max_size: usize, + /// Maximum allowed object size in bytes + pub max_object_size: usize, + /// Total number of cache hits + pub hit_count: u64, + /// Total number of cache misses + pub miss_count: u64, + /// Average cache object age (seconds) + pub avg_age_secs: f64, + /// Cache hit rate (0.0 - 1.0) + pub hit_rate: f64, + /// Total number of evictions + pub eviction_count: u64, + /// Eviction rate (evictions per second) + pub eviction_rate: f64, + /// Memory usage in bytes + pub memory_usage: usize, + /// Memory usage ratio (0.0 - 1.0) + pub memory_usage_ratio: f64, + /// Top hot keys (key, hit_count) + pub top_keys: Vec<(String, u64)>, + /// Efficiency score (0-100) + pub efficiency_score: u32, +} + +/// Cache health status for monitoring and diagnostics +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub struct CacheHealthStatus { + /// Memory usage in bytes + pub memory_usage: usize, + /// Whether the cache is healthy + pub is_healthy: bool, + /// Memory usage ratio (0.0 - 1.0) + pub memory_usage_ratio: f64, + /// Hit rate (0.0 - 1.0) + pub hit_rate: f64, + /// Eviction rate (evictions per second) + pub eviction_rate: f64, + /// Average entry age (seconds) + pub avg_entry_age_secs: f64, + /// Efficiency score (0-100) + pub efficiency_score: u32, + /// Optimization recommendations + pub recommendations: Vec, +} +// ============================================ +// Unit Tests for CachedGetObject +// ============================================ + +#[cfg(test)] +mod cached_object_tests { + use super::*; + use bytes::Bytes; + + #[test] + fn test_cached_get_object_builder() { + let obj = CachedGetObject::new(Bytes::from("test data"), 9) + .with_content_type("text/plain".to_string()) + .with_e_tag("\"abc123\"".to_string()) + .with_last_modified("2024-01-01T12:00:00Z".to_string()) + .with_cache_control("max-age=3600".to_string()) + .with_expires("2024-12-31T23:59:59Z".to_string()) + .with_content_encoding("gzip".to_string()) + .with_content_disposition("attachment; filename=\"test.txt\"".to_string()) + .with_storage_class("STANDARD".to_string()) + .with_version_id("v1".to_string()) + .with_replication_status("COMPLETED".to_string()) + .with_tag_count(5) + .with_delete_marker(false); + + assert_eq!(obj.content_type, Some("text/plain".to_string())); + assert_eq!(obj.e_tag, Some("\"abc123\"".to_string())); + assert_eq!(obj.storage_class, Some("STANDARD".to_string())); + assert_eq!(obj.version_id, Some("v1".to_string())); + assert_eq!(obj.replication_status, Some("COMPLETED".to_string())); + assert_eq!(obj.tag_count, Some(5)); + assert!(!obj.delete_marker); + } + + #[test] + fn test_cached_get_object_with_metadata() { + let mut metadata = std::collections::HashMap::new(); + metadata.insert("x-amz-meta-custom".to_string(), "value".to_string()); + metadata.insert("x-amz-meta-author".to_string(), "test".to_string()); + + let obj = CachedGetObject::new(Bytes::from("data"), 4).with_user_metadata(metadata.clone()); + + assert_eq!(obj.user_metadata.len(), 2); + assert_eq!(obj.user_metadata.get("x-amz-meta-custom"), Some(&"value".to_string())); + } + + #[test] + fn test_cached_get_object_size() { + let obj = CachedGetObject::new(Bytes::from("test"), 4); + assert_eq!(obj.size(), 4); + + let large_obj = CachedGetObject::new(Bytes::from(vec![0u8; 1024]), 1024); + assert_eq!(large_obj.size(), 1024); + } + + #[test] + fn test_cached_get_object_access_count() { + let obj = CachedGetObject::new(Bytes::from("test"), 4); + + assert_eq!(obj.increment_access(), 1); + assert_eq!(obj.increment_access(), 2); + assert_eq!(obj.increment_access(), 3); + } + + #[test] + fn test_cached_get_object_is_expired() { + // Not expired - no expires header + let obj1 = CachedGetObject::new(Bytes::from("test"), 4); + assert!(!obj1.is_expired()); + + // Expired - past expires time + let obj2 = CachedGetObject::new(Bytes::from("test"), 4).with_expires("2020-01-01T00:00:00Z".to_string()); + assert!(obj2.is_expired()); + + // Not expired - future expires time + let future = chrono::Utc::now() + chrono::Duration::days(1); + let obj3 = CachedGetObject::new(Bytes::from("test"), 4).with_expires(future.to_rfc3339()); + assert!(!obj3.is_expired()); + } + + #[test] + fn test_cached_get_object_replication_status() { + // Completed replication + let obj1 = CachedGetObject::new(Bytes::from("test"), 4).with_replication_status("COMPLETED".to_string()); + assert!(obj1.is_replication_complete()); + + // Pending replication + let obj2 = CachedGetObject::new(Bytes::from("test"), 4).with_replication_status("PENDING".to_string()); + assert!(!obj2.is_replication_complete()); + + // No replication configured + let obj3 = CachedGetObject::new(Bytes::from("test"), 4); + assert!(obj3.is_replication_complete()); + } + + #[test] + fn test_cached_get_object_memory_size() { + let obj = CachedGetObject::new(Bytes::from("test"), 4) + .with_content_type("text/plain".to_string()) + .with_e_tag("\"abc\"".to_string()); + + let size = obj.memory_size(); + // Should include body + content_type + e_tag + other fields + assert!(size >= 4 + 10 + 5); // At least body + content_type + e_tag + } + + #[test] + fn test_cached_get_object_record_hit() { + let obj = CachedGetObject::new(Bytes::from("test"), 4); + + assert_eq!(obj.record_hit(), 1); + assert_eq!(obj.record_hit(), 2); + assert_eq!(obj.record_hit(), 3); + } +} + +// ============================================ +// Unit Tests for CacheHealthStatus +// ============================================ + +#[cfg(test)] +mod cache_health_tests { + use super::*; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_cache_health_status() { + let cache = HotObjectCache::new(); + + // Add some entries + let data1 = Arc::new(CachedObject::new_with_size(vec![1, 2, 3, 4], 4)); + let data2 = Arc::new(CachedObject::new_with_size(vec![5, 6, 7, 8], 4)); + + cache.put("key1".to_string(), data1).await; + cache.put("key2".to_string(), data2).await; + + // Get health status + let health = cache.health_status().await; + + assert!(health.memory_usage > 0); + assert!(health.memory_usage_ratio >= 0.0 && health.memory_usage_ratio <= 1.0); + assert!(health.hit_rate >= 0.0 && health.hit_rate <= 1.0); + assert!(health.efficiency_score <= 100); + } + + #[tokio::test] + #[serial] + async fn test_cache_memory_usage() { + let cache = HotObjectCache::new(); + + let initial_usage = cache.memory_usage().await; + assert_eq!(initial_usage, 0); + + // Add some data + let data = Arc::new(CachedObject::new_with_size(vec![0u8; 1024], 1024)); + cache.put("key".to_string(), data).await; + + let new_usage = cache.memory_usage().await; + assert!(new_usage > initial_usage); + } + + #[tokio::test] + #[serial] + async fn test_cache_evict_percentage() { + let cache = HotObjectCache::new(); + + // Add multiple entries + for i in 0..10 { + let data = Arc::new(CachedObject::new_with_size(vec![i as u8; 100], 100)); + cache.put(format!("key{}", i), data).await; + } + + let stats = cache.stats().await; + assert_eq!(stats.entries, 10); + + // Evict 50% + let evicted = cache.evict_percentage(50.0).await; + assert!(evicted > 0); + } + + #[tokio::test] + #[serial] + async fn test_cache_warm_from_hot_list() { + let cache = HotObjectCache::new(); + + let hot_keys = vec![ + ("key1".to_string(), vec![1, 2, 3]), + ("key2".to_string(), vec![4, 5, 6]), + ("key3".to_string(), vec![7, 8, 9]), + ]; + + let warmed = cache.warm_from_hot_list(hot_keys).await; + assert_eq!(warmed, 3); + + // Verify entries are in cache + assert!(cache.contains("key1").await); + assert!(cache.contains("key2").await); + assert!(cache.contains("key3").await); + } +} + +// ============================================ +// Unit Tests for CacheStats +// ============================================ + +#[cfg(test)] +mod cache_stats_tests { + use super::*; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_cache_stats_hit_rate() { + let cache = HotObjectCache::new(); + + // Add an entry + let data = Arc::new(CachedObject::new_with_size(vec![1, 2, 3], 3)); + cache.put("key".to_string(), data).await; + + // Generate some hits and misses + cache.get("key").await; // Hit + cache.get("key").await; // Hit + cache.get("nonexistent").await; // Miss + cache.get("nonexistent").await; // Miss + + let stats = cache.stats().await; + + assert_eq!(stats.hit_count, 2); + assert_eq!(stats.miss_count, 2); + assert!((stats.hit_rate - 0.5).abs() < 0.01); // Should be ~50% + } + + #[tokio::test] + #[serial] + async fn test_cache_stats_memory_usage_ratio() { + let cache = HotObjectCache::new(); + + let stats = cache.stats().await; + assert_eq!(stats.memory_usage_ratio, 0.0); // Empty cache + + // Add some data + let data = Arc::new(CachedObject::new_with_size(vec![0u8; 1024], 1024)); + cache.put("key".to_string(), data).await; + + let stats = cache.stats().await; + assert!(stats.memory_usage_ratio > 0.0); + } + + #[tokio::test] + #[serial] + async fn test_cache_stats_efficiency_score() { + let cache = HotObjectCache::new(); + + // Empty cache - low efficiency + let stats = cache.stats().await; + assert!(stats.efficiency_score < 50); + + // Add data and generate hits + let data = Arc::new(CachedObject::new_with_size(vec![1, 2, 3], 3)); + cache.put("key".to_string(), data).await; + + for _ in 0..10 { + cache.get("key").await; // Hits + } + + let stats = cache.stats().await; + assert!(stats.efficiency_score > 0); + } +} diff --git a/rustfs/src/storage/concurrency/request_guard.rs b/rustfs/src/storage/concurrency/request_guard.rs new file mode 100644 index 000000000..729d81a85 --- /dev/null +++ b/rustfs/src/storage/concurrency/request_guard.rs @@ -0,0 +1,107 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Request tracking module with RAII guard for concurrent request management. + +use std::sync::atomic::Ordering; +use std::time::Instant; + +use super::io_schedule::ACTIVE_GET_REQUESTS; + +/// RAII guard for tracking active GetObject requests. +#[derive(Debug)] +pub struct GetObjectGuard { + start_time: Instant, +} + +impl GetObjectGuard { + /// Create a new guard and increment the concurrent request counter. + pub fn new() -> Self { + ACTIVE_GET_REQUESTS.fetch_add(1, Ordering::Relaxed); + + #[cfg(all(feature = "metrics", not(test)))] + if !std::thread::panicking() { + use metrics::counter; + counter!("rustfs.get.object.requests.started").increment(1); + } + + Self { + start_time: Instant::now(), + } + } + + /// Get the elapsed time since this guard was created. + pub fn elapsed(&self) -> std::time::Duration { + self.start_time.elapsed() + } + + /// Get the current number of concurrent GetObject requests. + pub fn concurrent_count() -> usize { + ACTIVE_GET_REQUESTS.load(Ordering::Relaxed) + } + + /// Get the current number of concurrent requests (alias for concurrent_count). + pub fn concurrent_requests() -> usize { + Self::concurrent_count() + } +} + +impl Default for GetObjectGuard { + fn default() -> Self { + Self::new() + } +} + +impl Drop for GetObjectGuard { + fn drop(&mut self) { + if let Err(previous) = + ACTIVE_GET_REQUESTS.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1)) + { + debug_assert_eq!( + previous, 0, + "ACTIVE_GET_REQUESTS underflow attempt in GetObjectGuard::drop; previous value = {}", + previous + ); + } + + #[cfg(all(feature = "metrics", not(test)))] + if !std::thread::panicking() { + use metrics::{counter, histogram}; + counter!("rustfs.get.object.requests.completed").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(self.elapsed().as_secs_f64()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_guard_increments_counter() { + let initial = GetObjectGuard::concurrent_count(); + { + let _guard = GetObjectGuard::new(); + assert_eq!(GetObjectGuard::concurrent_count(), initial + 1); + } + assert_eq!(GetObjectGuard::concurrent_count(), initial); + } + + #[test] + fn test_guard_elapsed() { + let guard = GetObjectGuard::new(); + std::thread::sleep(std::time::Duration::from_millis(10)); + assert!(guard.elapsed().as_millis() >= 10); + } +} diff --git a/rustfs/src/storage/concurrent_fix_test.rs b/rustfs/src/storage/concurrent_fix_test.rs new file mode 100644 index 000000000..b64c4aaa3 --- /dev/null +++ b/rustfs/src/storage/concurrent_fix_test.rs @@ -0,0 +1,338 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for concurrent request fix. +//! +//! These tests verify that the timeout, backpressure, and deadlock detection +//! mechanisms work correctly under high concurrency scenarios. + +#[cfg(test)] +mod tests { + use crate::storage::backpressure::{BackpressureConfig, BackpressureMonitor, BackpressureState}; + use crate::storage::concurrency::{IoLoadLevel, IoPriority}; + use crate::storage::deadlock_detector::{ + DeadlockDetector, DeadlockDetectorConfig, LockInfo, LockType, RequestResourceTracker, + }; + use crate::storage::lock_optimizer::{LockOptimizeConfig, LockOptimizer, LockStats}; + use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimedGetObjectResult, TimeoutConfig}; + use std::time::Duration; + + // ============================================ + // Timeout Wrapper Tests + // ============================================ + + #[tokio::test] + async fn test_timeout_wrapper_completes_within_timeout() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_secs(5), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { Ok::(42) }) + .await; + + match result { + TimedGetObjectResult::Success(value) => assert_eq!(value, 42), + _ => panic!("Expected Success, got {:?}", result), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_times_out() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_millis(50), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { + // Simulate a slow operation + tokio::time::sleep(Duration::from_secs(10)).await; + Ok::(42) + }) + .await; + + match result { + TimedGetObjectResult::Timeout(info) => { + assert!(info.elapsed >= Duration::from_millis(50)); + } + _ => panic!("Expected Timeout, got {:?}", result), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_returns_error() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_secs(5), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { Err::("test error".to_string()) }) + .await; + + match result { + TimedGetObjectResult::Error(e) => assert_eq!(e, "test error"), + _ => panic!("Expected Error, got {:?}", result), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_disabled() { + let config = TimeoutConfig { + get_object_timeout: Duration::ZERO, + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + // This would timeout if enabled + let result = wrapper + .execute_with_timeout(|_token| async move { + tokio::time::sleep(Duration::from_millis(100)).await; + Ok::(42) + }) + .await; + + match result { + TimedGetObjectResult::Success(value) => assert_eq!(value, 42), + _ => panic!("Expected Success when timeout disabled, got {:?}", result), + } + } + + // ============================================ + // Backpressure Tests + // ============================================ + + #[test] + fn test_backpressure_config_defaults() { + let config = BackpressureConfig::default(); + assert_eq!(config.buffer_size, 4 * 1024 * 1024); // 4MB + assert_eq!(config.high_watermark, 80); + assert_eq!(config.low_watermark, 50); + } + + #[test] + fn test_backpressure_monitor_state_transitions() { + let config = BackpressureConfig { + buffer_size: 1000, + high_watermark: 80, + low_watermark: 50, + }; + let monitor = BackpressureMonitor::with_config(config); + + // Initially normal + assert_eq!(monitor.state(), BackpressureState::Normal); + + // Write to reach high watermark + let state = monitor.on_write(850); + assert_eq!(state, BackpressureState::HighWatermark); + + // Read to go below low watermark + let state = monitor.on_read(400); + assert_eq!(state, BackpressureState::Normal); + } + + #[test] + fn test_backpressure_usage_percent() { + let config = BackpressureConfig { + buffer_size: 1000, + high_watermark: 80, + low_watermark: 50, + }; + let monitor = BackpressureMonitor::with_config(config); + + monitor.on_write(500); + assert!((monitor.usage_percent() - 50.0).abs() < 1.0); + } + + // ============================================ + // Lock Optimizer Tests + // ============================================ + + #[test] + fn test_lock_optimize_config_defaults() { + let config = LockOptimizeConfig::default(); + assert!(config.enabled); + assert_eq!(config.acquire_timeout, Duration::from_secs(5)); + } + + #[test] + fn test_lock_stats_tracking() { + let stats = LockStats::new(); + + stats.record_acquire(); + stats.record_early_release(Duration::from_millis(100)); + stats.record_early_release(Duration::from_millis(200)); + + assert_eq!(stats.locks_acquired.load(std::sync::atomic::Ordering::Relaxed), 1); + assert_eq!(stats.locks_released_early.load(std::sync::atomic::Ordering::Relaxed), 2); + assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); + } + + #[test] + fn test_lock_optimizer_creation() { + let optimizer = LockOptimizer::new(); + assert!(optimizer.is_enabled()); + } + + // ============================================ + // I/O Priority Tests + // ============================================ + + #[test] + fn test_io_priority_from_size() { + // Small request (< 1MB) -> High priority + assert_eq!(IoPriority::from_size(100 * 1024), IoPriority::High); + assert_eq!(IoPriority::from_size(512 * 1024), IoPriority::High); + + // Medium request (1MB - 10MB) -> Normal priority + assert_eq!(IoPriority::from_size(2 * 1024 * 1024), IoPriority::Normal); + assert_eq!(IoPriority::from_size(5 * 1024 * 1024), IoPriority::Normal); + + // Large request (> 10MB) -> Low priority + assert_eq!(IoPriority::from_size(20 * 1024 * 1024), IoPriority::Low); + assert_eq!(IoPriority::from_size(100 * 1024 * 1024), IoPriority::Low); + } + + #[test] + fn test_io_priority_ordering() { + assert!(IoPriority::High < IoPriority::Normal); + assert!(IoPriority::Normal < IoPriority::Low); + } + + #[test] + fn test_io_load_level_from_wait() { + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(5)), IoLoadLevel::Low); + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(30)), IoLoadLevel::Medium); + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(100)), IoLoadLevel::High); + assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(500)), IoLoadLevel::Critical); + } + + // ============================================ + // Deadlock Detector Tests + // ============================================ + + #[test] + fn test_deadlock_detector_config_defaults() { + let config = DeadlockDetectorConfig::default(); + assert!(!config.enabled); // Disabled by default + assert_eq!(config.check_interval, Duration::from_secs(5)); + assert_eq!(config.hang_threshold, Duration::from_secs(10)); + } + + #[test] + fn test_request_resource_tracker() { + let tracker = RequestResourceTracker::new("req-1".to_string(), "GetObject bucket/key"); + assert!(tracker.elapsed() < Duration::from_secs(1)); + assert!(!tracker.is_hung(Duration::from_secs(10))); + } + + #[test] + fn test_lock_info_creation() { + let lock = LockInfo { + id: "lock-1".to_string(), + lock_type: LockType::Read, + resource: "bucket/key".to_string(), + acquire_time: std::time::Instant::now(), + }; + assert_eq!(format!("{}", lock.lock_type), "read"); + } + + #[test] + fn test_deadlock_detector_registration() { + let config = DeadlockDetectorConfig { + enabled: true, + ..Default::default() + }; + let detector = DeadlockDetector::new(config); + + detector.register_request("req-1", "Test request"); + assert_eq!(detector.tracked_count(), 1); + + detector.register_request("req-2", "Another request"); + assert_eq!(detector.tracked_count(), 2); + + detector.unregister_request("req-1"); + assert_eq!(detector.tracked_count(), 1); + + detector.unregister_request("req-2"); + assert_eq!(detector.tracked_count(), 0); + } + + // ============================================ + // Integration Tests + // ============================================ + + #[tokio::test] + async fn test_concurrent_requests_with_timeout() { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let semaphore = Arc::new(Semaphore::new(10)); + let mut handles = vec![]; + + // Spawn 20 concurrent "requests" + for i in 0..20 { + let sem = semaphore.clone(); + let handle = tokio::spawn(async move { + let _permit = sem.acquire().await.unwrap(); + // Simulate some work + tokio::time::sleep(Duration::from_millis(10)).await; + i + }); + handles.push(handle); + } + + // Wait for all to complete + let results: Vec<_> = futures::future::join_all(handles).await; + assert_eq!(results.len(), 20); + + // All should complete successfully + for result in results { + assert!(result.is_ok()); + } + } + + #[test] + fn test_priority_scheduling_fairness() { + // Simulate priority scheduling + let mut high_priority_count = 0; + let mut normal_priority_count = 0; + let mut low_priority_count = 0; + + // Simulate 100 requests of various sizes + for i in 0..100 { + let size = match i % 3 { + 0 => 100 * 1024, // Small: 100KB + 1 => 5 * 1024 * 1024, // Medium: 5MB + _ => 50 * 1024 * 1024, // Large: 50MB + }; + + match IoPriority::from_size(size) { + IoPriority::High => high_priority_count += 1, + IoPriority::Normal => normal_priority_count += 1, + IoPriority::Low => low_priority_count += 1, + } + } + + // Each priority should have roughly 1/3 of requests + assert_eq!(high_priority_count, 34); + assert_eq!(normal_priority_count, 33); + assert_eq!(low_priority_count, 33); + } +} diff --git a/rustfs/src/storage/deadlock_detector.rs b/rustfs/src/storage/deadlock_detector.rs new file mode 100644 index 000000000..2a257dd03 --- /dev/null +++ b/rustfs/src/storage/deadlock_detector.rs @@ -0,0 +1,673 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Deadlock Detection for Concurrent Request Monitoring. +//! +//! This module provides deadlock detection capabilities for diagnosing +//! hanging requests and lock contention issues in production systems. +//! +//! # Key Features +//! +//! - Request resource tracking (locks, memory, file handles) +//! - Lock wait graph analysis for cycle detection +//! - Configurable detection interval and hang threshold +//! - Prometheus metrics for deadlock events +//! - Detailed diagnostic logging +//! +//! # Usage +//! +//! ```ignore +//! use crate::storage::deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig}; +//! +//! let config = DeadlockDetectorConfig::from_env(); +//! let detector = DeadlockDetector::new(config); +//! detector.start(); +//! +//! // Track a request +//! let request_id = detector.register_request(); +//! detector.record_lock_acquire(request_id, lock_info); +//! +//! // ... request processing ... +//! +//! detector.unregister_request(request_id); +//! ``` + +// Allow dead_code for public API that may be used by external modules or future features +#![allow(dead_code)] + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; +use tokio::sync::broadcast; +use tracing::{debug, error, warn}; + +#[cfg(feature = "metrics")] +use metrics::counter; + +/// Request identifier type. +pub type RequestId = String; + +/// Lock identifier type. +pub type LockId = String; + +/// Deadlock detector configuration. +#[derive(Debug, Clone)] +pub struct DeadlockDetectorConfig { + /// Whether deadlock detection is enabled. + pub enabled: bool, + /// Detection check interval. + pub check_interval: Duration, + /// Hang threshold - requests running longer than this are considered potentially hung. + pub hang_threshold: Duration, + /// Whether to capture backtraces (expensive, only for debugging). + pub capture_backtrace: bool, +} + +impl Default for DeadlockDetectorConfig { + fn default() -> Self { + Self { + enabled: rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE, + check_interval: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL), + hang_threshold: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD), + capture_backtrace: false, + } + } +} + +impl DeadlockDetectorConfig { + /// Load configuration from environment variables. + pub fn from_env() -> Self { + let enabled = rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_DEADLOCK_DETECTION_ENABLE, + rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE, + ); + let check_interval = Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_DEADLOCK_CHECK_INTERVAL, + rustfs_config::DEFAULT_OBJECT_DEADLOCK_CHECK_INTERVAL, + )); + let hang_threshold = Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_DEADLOCK_HANG_THRESHOLD, + rustfs_config::DEFAULT_OBJECT_DEADLOCK_HANG_THRESHOLD, + )); + + Self { + enabled, + check_interval, + hang_threshold, + capture_backtrace: false, + } + } +} + +/// Lock information for tracking. +#[derive(Debug, Clone)] +pub struct LockInfo { + /// Lock identifier. + pub id: LockId, + /// Lock type (read/write). + pub lock_type: LockType, + /// Resource being locked (bucket/key). + pub resource: String, + /// When the lock was acquired. + pub acquire_time: Instant, +} + +/// Lock type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LockType { + /// Read lock (shared). + Read, + /// Write lock (exclusive). + Write, +} + +impl std::fmt::Display for LockType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LockType::Read => write!(f, "read"), + LockType::Write => write!(f, "write"), + } + } +} + +/// Resource type being tracked. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResourceType { + /// Lock resource. + Lock, + /// Memory resource. + Memory, + /// File handle. + FileHandle, + /// I/O permit. + IoPermit, +} + +/// Request resource tracker. +#[derive(Debug, Clone)] +pub struct RequestResourceTracker { + /// Request ID. + pub request_id: RequestId, + /// When the request started. + pub start_time: Instant, + /// Locks currently held by this request. + pub held_locks: Vec, + /// Lock this request is waiting for (if any). + pub waiting_lock: Option, + /// Resources held by this request. + pub resources: HashMap, + /// Request description (e.g., "GetObject bucket/key"). + pub description: String, +} + +impl RequestResourceTracker { + /// Create a new tracker for a request. + pub fn new(request_id: RequestId, description: impl Into) -> Self { + Self { + request_id, + start_time: Instant::now(), + held_locks: Vec::new(), + waiting_lock: None, + resources: HashMap::new(), + description: description.into(), + } + } + + /// Get the elapsed time since the request started. + pub fn elapsed(&self) -> Duration { + self.start_time.elapsed() + } + + /// Check if this request is potentially hung. + pub fn is_hung(&self, threshold: Duration) -> bool { + self.elapsed() > threshold + } +} + +/// Deadlock detection result. +#[derive(Debug, Clone)] +pub struct DeadlockInfo { + /// Time of detection. + pub detected_at: Instant, + /// Requests involved in the deadlock cycle. + pub cycle: Vec, + /// Lock wait graph showing the deadlock. + pub wait_graph: Vec, + /// Resource usage at detection time. + pub resource_usage: ResourceUsage, +} + +/// Edge in the lock wait graph. +#[derive(Debug, Clone)] +pub struct WaitGraphEdge { + /// Request that is waiting. + pub from: RequestId, + /// Request that is blocking (holds the lock). + pub to: RequestId, + /// The lock being waited for. + pub lock_id: LockId, +} + +/// Resource usage snapshot. +#[derive(Debug, Clone, Default)] +pub struct ResourceUsage { + /// Total memory used (bytes). + pub memory_bytes: usize, + /// Total file handles open. + pub open_files: usize, + /// Total active requests. + pub active_requests: usize, + /// Total locks held. + pub locks_held: usize, +} + +/// Deadlock detector. +pub struct DeadlockDetector { + /// Configuration. + config: DeadlockDetectorConfig, + /// Active request trackers. + requests: Arc>>, + /// Detection task handle. + detector_task: Arc>>>, + /// Shutdown signal. + shutdown_tx: broadcast::Sender<()>, + /// Total deadlocks detected. + deadlocks_detected: Arc, + /// Is currently running. + running: Arc, +} + +impl DeadlockDetector { + /// Create a new deadlock detector. + pub fn new(config: DeadlockDetectorConfig) -> Self { + let (shutdown_tx, _) = broadcast::channel(1); + + Self { + config, + requests: Arc::new(RwLock::new(HashMap::new())), + detector_task: Arc::new(Mutex::new(None)), + shutdown_tx, + deadlocks_detected: Arc::new(AtomicU64::new(0)), + running: Arc::new(AtomicBool::new(false)), + } + } + + /// Check if detection is enabled. + pub fn is_enabled(&self) -> bool { + self.config.enabled + } + + /// Start the detection task. + pub fn start(&self) { + if !self.config.enabled { + debug!("Deadlock detection is disabled"); + return; + } + + if self.running.swap(true, Ordering::Relaxed) { + debug!("Deadlock detector already running"); + return; + } + + let requests = self.requests.clone(); + let config = self.config.clone(); + let deadlocks_detected = self.deadlocks_detected.clone(); + let mut shutdown_rx = self.shutdown_tx.subscribe(); + let running = self.running.clone(); + + let handle = tokio::spawn(async move { + debug!( + check_interval_secs = config.check_interval.as_secs(), + hang_threshold_secs = config.hang_threshold.as_secs(), + "Deadlock detector started" + ); + + loop { + tokio::select! { + _ = tokio::time::sleep(config.check_interval) => { + Self::detect_cycle(&requests, &config, &deadlocks_detected); + } + _ = shutdown_rx.recv() => { + debug!("Deadlock detector shutting down"); + break; + } + } + } + + running.store(false, Ordering::Relaxed); + }); + + *self.detector_task.lock().unwrap() = Some(handle); + } + + /// Stop the detection task. + pub fn stop(&self) { + let _ = self.shutdown_tx.send(()); + if let Some(handle) = self.detector_task.lock().unwrap().take() { + // Don't await the handle as we're in a non-async context + handle.abort(); + } + self.running.store(false, Ordering::Relaxed); + } + + /// Register a new request for tracking. + pub fn register_request(&self, request_id: impl Into, description: impl Into) { + if !self.config.enabled { + return; + } + + let request_id = request_id.into(); + let tracker = RequestResourceTracker::new(request_id.clone(), description); + + self.requests.write().unwrap().insert(request_id.clone(), tracker); + + debug!(request_id = %request_id, "Request registered for deadlock tracking"); + } + + /// Unregister a request (it completed or was cancelled). + pub fn unregister_request(&self, request_id: &str) { + if !self.config.enabled { + return; + } + + self.requests.write().unwrap().remove(request_id); + + debug!(request_id = %request_id, "Request unregistered from deadlock tracking"); + } + + /// Record a lock acquisition. + pub fn record_lock_acquire(&self, request_id: &str, lock: LockInfo) { + if !self.config.enabled { + return; + } + + if let Some(tracker) = self.requests.write().unwrap().get_mut(request_id) { + tracker.held_locks.push(lock); + } + } + + /// Record a lock release. + pub fn record_lock_release(&self, request_id: &str, lock_id: &LockId) { + if !self.config.enabled { + return; + } + + if let Some(tracker) = self.requests.write().unwrap().get_mut(request_id) { + tracker.held_locks.retain(|l| &l.id != lock_id); + } + } + + /// Record that a request is waiting for a lock. + pub fn record_lock_wait(&self, request_id: &str, lock: LockInfo) { + if !self.config.enabled { + return; + } + + if let Some(tracker) = self.requests.write().unwrap().get_mut(request_id) { + tracker.waiting_lock = Some(lock); + } + } + + /// Clear the waiting lock (acquired or gave up). + pub fn clear_lock_wait(&self, request_id: &str) { + if !self.config.enabled { + return; + } + + if let Some(tracker) = self.requests.write().unwrap().get_mut(request_id) { + tracker.waiting_lock = None; + } + } + + /// Record resource usage. + pub fn record_resource(&self, request_id: &str, resource_type: ResourceType, amount: usize) { + if !self.config.enabled { + return; + } + + if let Some(tracker) = self.requests.write().unwrap().get_mut(request_id) { + tracker.resources.insert(resource_type, amount); + } + } + + /// Get current number of tracked requests. + pub fn tracked_count(&self) -> usize { + self.requests.read().unwrap().len() + } + + /// Get total deadlocks detected. + pub fn total_detected(&self) -> u64 { + self.deadlocks_detected.load(Ordering::Relaxed) + } + + /// Detect deadlock cycles in the lock wait graph. + fn detect_cycle( + requests: &Arc>>, + config: &DeadlockDetectorConfig, + deadlocks_detected: &Arc, + ) { + let requests_guard = requests.read().unwrap(); + + // Find hung requests + let hung_requests: Vec<_> = requests_guard.values().filter(|r| r.is_hung(config.hang_threshold)).collect(); + + if hung_requests.is_empty() { + return; + } + + // Build lock wait graph + // Edge: request A -> request B means A is waiting for a lock that B holds + let mut wait_graph: Vec = Vec::new(); + + for waiting in &hung_requests { + if let Some(waiting_for) = &waiting.waiting_lock { + // Find who holds this lock + for holding in &hung_requests { + if holding.request_id == waiting.request_id { + continue; + } + if holding.held_locks.iter().any(|l| l.id == waiting_for.id) { + wait_graph.push(WaitGraphEdge { + from: waiting.request_id.clone(), + to: holding.request_id.clone(), + lock_id: waiting_for.id.clone(), + }); + } + } + } + } + + // Detect cycles using DFS + if let Some(cycle) = Self::find_cycle(&wait_graph) { + deadlocks_detected.fetch_add(1, Ordering::Relaxed); + + #[cfg(feature = "metrics")] + counter!("rustfs.deadlock.detected.total").increment(1); + + // Log detailed deadlock information + error!( + cycle = ?cycle, + wait_graph = ?wait_graph, + hung_requests_count = hung_requests.len(), + "Deadlock detected: circular lock wait chain found" + ); + + // Log each request in the cycle + for request_id in &cycle { + if let Some(tracker) = requests_guard.get(request_id) { + warn!( + request_id = %request_id, + description = %tracker.description, + elapsed_secs = tracker.elapsed().as_secs(), + held_locks = ?tracker.held_locks.iter().map(|l| &l.id).collect::>(), + waiting_lock = ?tracker.waiting_lock.as_ref().map(|l| &l.id), + "Request in deadlock cycle" + ); + } + } + } else { + // No cycle, but log hung requests for diagnosis + debug!( + hung_requests_count = hung_requests.len(), + wait_graph_edges = wait_graph.len(), + "Hung requests detected but no deadlock cycle" + ); + } + } + + /// Find a cycle in the wait graph using DFS. + fn find_cycle(edges: &[WaitGraphEdge]) -> Option> { + // Build adjacency list + let mut graph: HashMap<&RequestId, Vec<&RequestId>> = HashMap::new(); + for edge in edges { + graph.entry(&edge.from).or_default().push(&edge.to); + } + + // DFS with path tracking + let mut visited: HashSet<&RequestId> = HashSet::new(); + let mut path: Vec<&RequestId> = Vec::new(); + let mut path_set: HashSet<&RequestId> = HashSet::new(); + + for start in graph.keys() { + if visited.contains(start) { + continue; + } + + if Self::dfs_find_cycle(start, &graph, &mut visited, &mut path, &mut path_set) { + return Some(path.iter().map(|s| (*s).clone()).collect()); + } + } + + None + } + + /// DFS helper for cycle detection. + fn dfs_find_cycle<'a>( + node: &'a RequestId, + graph: &HashMap<&'a RequestId, Vec<&'a RequestId>>, + visited: &mut HashSet<&'a RequestId>, + path: &mut Vec<&'a RequestId>, + path_set: &mut HashSet<&'a RequestId>, + ) -> bool { + visited.insert(node); + path.push(node); + path_set.insert(node); + + if let Some(neighbors) = graph.get(&node) { + for neighbor in neighbors { + if path_set.contains(neighbor) { + // Found cycle - trim path to just the cycle + let cycle_start = path.iter().position(|n| *n == *neighbor).unwrap(); + path.drain(0..cycle_start); + return true; + } + + if !visited.contains(neighbor) && Self::dfs_find_cycle(neighbor, graph, visited, path, path_set) { + return true; + } + } + } + + path.pop(); + path_set.remove(node); + false + } +} + +impl Drop for DeadlockDetector { + fn drop(&mut self) { + self.stop(); + } +} + +/// Global deadlock detector instance. +static DEADLOCK_DETECTOR: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Get the global deadlock detector. +pub fn get_deadlock_detector() -> Arc { + DEADLOCK_DETECTOR + .get_or_init(|| { + let config = DeadlockDetectorConfig::from_env(); + Arc::new(DeadlockDetector::new(config)) + }) + .clone() +} + +/// Initialize and start the global deadlock detector. +pub fn init_deadlock_detector() { + let detector = get_deadlock_detector(); + detector.start(); +} + +/// Check if deadlock detection is enabled. +pub fn is_deadlock_detection_enabled() -> bool { + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_DEADLOCK_DETECTION_ENABLE, + rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_deadlock_detector_config_default() { + let config = DeadlockDetectorConfig::default(); + assert!(!config.enabled); + assert_eq!(config.check_interval, Duration::from_secs(5)); + assert_eq!(config.hang_threshold, Duration::from_secs(10)); + } + + #[test] + fn test_request_resource_tracker() { + let tracker = RequestResourceTracker::new("req-1".to_string(), "GetObject bucket/key"); + assert!(tracker.elapsed() < Duration::from_secs(1)); + assert!(!tracker.is_hung(Duration::from_secs(10))); + } + + #[test] + fn test_lock_info() { + let lock = LockInfo { + id: "lock-1".to_string(), + lock_type: LockType::Read, + resource: "bucket/key".to_string(), + acquire_time: Instant::now(), + }; + assert_eq!(format!("{}", lock.lock_type), "read"); + } + + #[test] + fn test_deadlock_detector_registration() { + let config = DeadlockDetectorConfig { + enabled: true, + ..Default::default() + }; + let detector = DeadlockDetector::new(config); + + detector.register_request("req-1", "Test request"); + assert_eq!(detector.tracked_count(), 1); + + detector.unregister_request("req-1"); + assert_eq!(detector.tracked_count(), 0); + } + + #[test] + fn test_cycle_detection() { + // Create a simple cycle: A -> B -> C -> A + let edges = vec![ + WaitGraphEdge { + from: "A".to_string(), + to: "B".to_string(), + lock_id: "lock-1".to_string(), + }, + WaitGraphEdge { + from: "B".to_string(), + to: "C".to_string(), + lock_id: "lock-2".to_string(), + }, + WaitGraphEdge { + from: "C".to_string(), + to: "A".to_string(), + lock_id: "lock-3".to_string(), + }, + ]; + + let cycle = DeadlockDetector::find_cycle(&edges); + assert!(cycle.is_some()); + let cycle = cycle.unwrap(); + assert!(cycle.len() >= 2); + } + + #[test] + fn test_no_cycle() { + // No cycle: A -> B -> C + let edges = vec![ + WaitGraphEdge { + from: "A".to_string(), + to: "B".to_string(), + lock_id: "lock-1".to_string(), + }, + WaitGraphEdge { + from: "B".to_string(), + to: "C".to_string(), + lock_id: "lock-2".to_string(), + }, + ]; + + let cycle = DeadlockDetector::find_cycle(&edges); + assert!(cycle.is_none()); + } +} diff --git a/rustfs/src/storage/lock_optimizer.rs b/rustfs/src/storage/lock_optimizer.rs new file mode 100644 index 000000000..d718ac4b3 --- /dev/null +++ b/rustfs/src/storage/lock_optimizer.rs @@ -0,0 +1,451 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Lock Optimization for GetObject Operations. +//! +//! This module provides optimized lock management for read operations, +//! reducing lock contention by releasing locks early (after metadata read) +//! rather than holding them for the entire data transfer duration. + +// Allow dead_code for public API that may be used by external modules or future features +#![allow(dead_code)] +//! # Key Features +//! +//! - Early lock release after metadata read +//! - Lock hold time monitoring +//! - Configurable optimization (can be disabled for debugging) +//! - Prometheus metrics for lock contention analysis +//! +//! # Architecture +//! +//! ```text +//! Traditional: [Acquire Lock] --> [Read Metadata] --> [Transfer Data] --> [Release Lock] +//! |<------------------ Lock Held ------------------>| +//! +//! Optimized: [Acquire Lock] --> [Read Metadata] --> [Release Lock] --> [Transfer Data] +//! |<- Lock Held ->| +//! ``` + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use tracing::debug; + +#[cfg(feature = "metrics")] +use metrics::histogram; + +/// Lock optimization configuration. +#[derive(Debug, Clone)] +pub struct LockOptimizeConfig { + /// Whether to enable lock optimization. + /// When enabled, read locks are released after metadata read. + /// When disabled, locks are held for the entire operation (traditional behavior). + pub enabled: bool, + /// Lock acquisition timeout. + pub acquire_timeout: Duration, +} + +impl Default for LockOptimizeConfig { + fn default() -> Self { + Self { + enabled: rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, + acquire_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT), + } + } +} + +impl LockOptimizeConfig { + /// Load configuration from environment variables. + pub fn from_env() -> Self { + let enabled = rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, + rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, + ); + let acquire_timeout = Duration::from_secs(rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, + rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT, + )); + + Self { + enabled, + acquire_timeout, + } + } +} + +/// Statistics for lock optimization monitoring. +#[derive(Debug, Default)] +pub struct LockStats { + /// Total locks acquired. + pub locks_acquired: AtomicU64, + /// Total locks released early. + pub locks_released_early: AtomicU64, + /// Total lock hold time in microseconds. + pub total_hold_time_us: AtomicU64, + /// Maximum lock hold time in microseconds. + pub max_hold_time_us: AtomicU64, +} + +impl LockStats { + /// Create new lock statistics. + pub fn new() -> Self { + Self::default() + } + + /// Record a lock acquisition. + pub fn record_acquire(&self) { + self.locks_acquired.fetch_add(1, Ordering::Relaxed); + } + + /// Record an early lock release. + pub fn record_early_release(&self, hold_time: Duration) { + self.locks_released_early.fetch_add(1, Ordering::Relaxed); + self.record_hold_time(hold_time); + } + + /// Record lock hold time. + fn record_hold_time(&self, hold_time: Duration) { + let hold_time_us = hold_time.as_micros() as u64; + self.total_hold_time_us.fetch_add(hold_time_us, Ordering::Relaxed); + + // Update max hold time + let mut current_max = self.max_hold_time_us.load(Ordering::Relaxed); + while hold_time_us > current_max { + match self + .max_hold_time_us + .compare_exchange_weak(current_max, hold_time_us, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => break, + Err(actual) => current_max = actual, + } + } + } + + /// Get average hold time. + pub fn avg_hold_time(&self) -> Duration { + let total = self.total_hold_time_us.load(Ordering::Relaxed); + let count = self.locks_released_early.load(Ordering::Relaxed); + if count > 0 { + Duration::from_micros(total / count) + } else { + Duration::ZERO + } + } + + /// Get maximum hold time. + pub fn max_hold_time(&self) -> Duration { + Duration::from_micros(self.max_hold_time_us.load(Ordering::Relaxed)) + } +} + +/// Global lock statistics. +static LOCK_STATS: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Get global lock statistics. +pub fn get_lock_stats() -> Arc { + LOCK_STATS.get_or_init(|| Arc::new(LockStats::new())).clone() +} + +/// An optimized lock guard that supports early release. +/// +/// This wraps the actual lock guard and provides: +/// - Early release capability (before drop) +/// - Hold time tracking +/// - Metrics reporting +pub struct OptimizedLockGuard { + /// The underlying lock guard. + guard: Option, + /// When the lock was acquired. + acquire_time: Instant, + /// Whether the lock has been released. + released: bool, + /// Lock resource name (for logging). + resource: String, + /// Statistics reference. + stats: Arc, +} + +impl OptimizedLockGuard { + /// Create a new optimized lock guard. + pub fn new(guard: G, resource: impl Into) -> Self { + let stats = get_lock_stats(); + stats.record_acquire(); + + Self { + guard: Some(guard), + acquire_time: Instant::now(), + released: false, + resource: resource.into(), + stats, + } + } + + /// Get the lock hold time so far. + pub fn hold_time(&self) -> Duration { + self.acquire_time.elapsed() + } + + /// Check if the lock has been released. + pub fn is_released(&self) -> bool { + self.released + } + + /// Release the lock early (before drop). + /// + /// This is the key optimization: releasing the lock after + /// metadata read rather than waiting for the entire operation. + pub fn early_release(&mut self) { + if self.released { + return; + } + + let hold_time = self.hold_time(); + self.guard.take(); + self.released = true; + + self.stats.record_early_release(hold_time); + + #[cfg(feature = "metrics")] + histogram!("rustfs.lock.hold.duration.seconds").record(hold_time.as_secs_f64()); + + debug!( + resource = %self.resource, + hold_time_ms = hold_time.as_millis(), + "Lock released early (optimization active)" + ); + } + + /// Get a reference to the underlying guard. + pub fn as_ref(&self) -> Option<&G> { + if self.released { None } else { self.guard.as_ref() } + } +} + +impl Drop for OptimizedLockGuard { + fn drop(&mut self) { + if !self.released { + let hold_time = self.hold_time(); + self.guard.take(); + self.released = true; + + self.stats.record_early_release(hold_time); + + #[cfg(feature = "metrics")] + histogram!("rustfs.lock.hold.duration.seconds").record(hold_time.as_secs_f64()); + + debug!( + resource = %self.resource, + hold_time_ms = hold_time.as_millis(), + "Lock released on drop (normal release)" + ); + } + } +} + +/// A scope guard that releases a lock when it goes out of scope. +/// +/// This is a simpler version of OptimizedLockGuard for cases +/// where we just need RAII semantics without tracking. +pub struct LockScopeGuard { + guard: Option, +} + +impl LockScopeGuard { + /// Create a new scope guard. + pub fn new(guard: G) -> Self { + Self { guard: Some(guard) } + } + + /// Release the lock early. + pub fn release(&mut self) { + self.guard.take(); + } +} + +impl Drop for LockScopeGuard { + fn drop(&mut self) { + self.guard.take(); + } +} + +/// Helper for managing lock optimization in GetObject operations. +/// +/// This provides a clean interface for the common pattern: +/// 1. Acquire lock +/// 2. Read metadata +/// 3. Release lock (if optimization enabled) +/// 4. Transfer data (without lock) +pub struct LockOptimizer { + /// Configuration. + config: LockOptimizeConfig, +} + +impl LockOptimizer { + /// Create a new lock optimizer with default configuration. + pub fn new() -> Self { + Self { + config: LockOptimizeConfig::from_env(), + } + } + + /// Create a new lock optimizer with custom configuration. + pub fn with_config(config: LockOptimizeConfig) -> Self { + Self { config } + } + + /// Check if lock optimization is enabled. + pub fn is_enabled(&self) -> bool { + self.config.enabled + } + + /// Get the lock acquisition timeout. + pub fn acquire_timeout(&self) -> Duration { + self.config.acquire_timeout + } + + /// Wrap a lock guard for optimization. + pub fn wrap_guard(&self, guard: G, resource: impl Into) -> OptimizedLockGuard { + OptimizedLockGuard::new(guard, resource) + } + + /// Execute a metadata read operation with lock optimization. + /// + /// This is the main entry point for optimized lock usage: + /// - If optimization is enabled: lock is released after metadata_fn completes + /// - If optimization is disabled: lock is held until the returned guard is dropped + /// + /// # Arguments + /// + /// * `guard` - The lock guard to optimize + /// * `resource` - Resource name for logging + /// * `metadata_fn` - Function to read metadata while holding lock + /// + /// # Returns + /// + /// A tuple of (metadata result, optional guard to hold for later release) + pub async fn with_optimized_lock( + &self, + guard: G, + resource: impl Into, + metadata_fn: F, + ) -> (T, Option>) + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let resource = resource.into(); + let mut optimized = OptimizedLockGuard::new(guard, &resource); + + // Execute metadata read while holding lock + let result = metadata_fn().await; + + if self.config.enabled { + // Release lock early + optimized.early_release(); + (result, None) + } else { + // Keep lock for caller to release + (result, Some(optimized)) + } + } +} + +impl Default for LockOptimizer { + fn default() -> Self { + Self::new() + } +} + +/// Check if lock optimization is enabled globally. +pub fn is_lock_optimization_enabled() -> bool { + rustfs_utils::get_env_bool( + rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, + rustfs_config::DEFAULT_OBJECT_LOCK_OPTIMIZATION_ENABLE, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + #[test] + fn test_lock_optimize_config_default() { + let config = LockOptimizeConfig::default(); + assert!(config.enabled); + assert_eq!(config.acquire_timeout, Duration::from_secs(5)); + } + + #[test] + fn test_lock_stats() { + let stats = LockStats::new(); + + stats.record_acquire(); + stats.record_early_release(Duration::from_millis(100)); + stats.record_early_release(Duration::from_millis(200)); + + assert_eq!(stats.locks_acquired.load(Ordering::Relaxed), 1); + assert_eq!(stats.locks_released_early.load(Ordering::Relaxed), 2); + assert_eq!(stats.max_hold_time(), Duration::from_millis(200)); + } + + #[test] + fn test_optimized_lock_guard() { + let guard = Mutex::new(42); + let locked = guard.lock().unwrap(); + let mut optimized = OptimizedLockGuard::new(locked, "test-resource"); + + assert!(!optimized.is_released()); + assert!(optimized.hold_time() < Duration::from_secs(1)); + + optimized.early_release(); + assert!(optimized.is_released()); + } + + #[test] + fn test_lock_optimizer() { + let optimizer = LockOptimizer::new(); + assert!(optimizer.is_enabled()); + } + + #[tokio::test] + async fn test_with_optimized_lock_enabled() { + let optimizer = LockOptimizer::new(); + let guard = Mutex::new(42); + let locked = guard.lock().unwrap(); + + let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; + + assert_eq!(result, 100); + // With optimization enabled, guard should be None (released early) + assert!(returned_guard.is_none()); + } + + #[tokio::test] + async fn test_with_optimized_lock_disabled() { + let config = LockOptimizeConfig { + enabled: false, + acquire_timeout: Duration::from_secs(5), + }; + let optimizer = LockOptimizer::with_config(config); + let guard = Mutex::new(42); + let locked = guard.lock().unwrap(); + + let (result, returned_guard) = optimizer.with_optimized_lock(locked, "test-resource", || async { 100 }).await; + + assert_eq!(result, 100); + // With optimization disabled, guard should be Some (held for later) + assert!(returned_guard.is_some()); + } +} diff --git a/rustfs/src/storage/mod.rs b/rustfs/src/storage/mod.rs index 49aab9913..277ebfaed 100644 --- a/rustfs/src/storage/mod.rs +++ b/rustfs/src/storage/mod.rs @@ -13,17 +13,23 @@ // limitations under the License. pub mod access; +pub mod backpressure; pub mod concurrency; +pub mod deadlock_detector; pub mod ecfs; pub(crate) mod entity; pub(crate) mod helper; +pub mod lock_optimizer; pub mod options; pub(crate) mod readers; pub mod rpc; pub(crate) mod s3_api; mod sse; +pub mod timeout_wrapper; pub mod tonic_service; +#[cfg(test)] +mod concurrent_fix_test; #[cfg(test)] mod concurrent_get_object_test; mod ecfs_extend; diff --git a/rustfs/src/storage/timeout_wrapper.rs b/rustfs/src/storage/timeout_wrapper.rs new file mode 100644 index 000000000..503bd13fe --- /dev/null +++ b/rustfs/src/storage/timeout_wrapper.rs @@ -0,0 +1,582 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Request Timeout Wrapper for GetObject operations. +//! +//! This module provides timeout protection for GetObject requests to prevent +//! indefinite hangs caused by deadlocks, resource exhaustion, or slow I/O. + +// Allow dead_code for public API that may be used by external modules or future features +#![allow(dead_code)] +//! +//! - Configurable request-level timeout (default 30 seconds) +//! - Automatic cancellation of sub-tasks on timeout +//! - Resource cleanup on timeout (locks, memory, file handles) +//! - Prometheus metrics for timeout monitoring +//! +//! # Usage +//! +//! ```ignore +//! use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; +//! +//! let config = TimeoutConfig::from_env(); +//! let wrapper = RequestTimeoutWrapper::new(config); +//! +//! match wrapper.execute_with_timeout(|cancel_token| async move { +//! // Your async operation here +//! Ok(result) +//! }).await { +//! TimedGetObjectResult::Success(result) => { /* handle success */ } +//! TimedGetObjectResult::Timeout(info) => { /* handle timeout */ } +//! TimedGetObjectResult::Error(e) => { /* handle error */ } +//! } +//! ``` + +use std::time::{Duration, Instant}; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; + +#[cfg(feature = "metrics")] +use metrics::{counter, histogram}; + +/// Timeout configuration for GetObject requests. +#[derive(Debug, Clone)] +pub struct TimeoutConfig { + /// GetObject request overall timeout (default 30s). + /// After this duration, the request is cancelled and returns 504. + pub get_object_timeout: Duration, + + /// Lock acquisition timeout (default 5s). + /// Time to wait for a lock before giving up. + pub lock_acquire_timeout: Duration, + + /// Disk read operation timeout (default 10s). + /// Individual disk read operations that exceed this are cancelled. + pub disk_read_timeout: Duration, +} + +impl Default for TimeoutConfig { + fn default() -> Self { + Self { + get_object_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT), + lock_acquire_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT), + disk_read_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT), + } + } +} + +impl TimeoutConfig { + /// Load configuration from environment variables. + pub fn from_env() -> Self { + let get_object_timeout = + rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_GET_TIMEOUT, rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT); + let lock_acquire_timeout = rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, + rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT, + ); + let disk_read_timeout = rustfs_utils::get_env_u64( + rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, + rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT, + ); + + Self { + get_object_timeout: Duration::from_secs(get_object_timeout), + lock_acquire_timeout: Duration::from_secs(lock_acquire_timeout), + disk_read_timeout: Duration::from_secs(disk_read_timeout), + } + } + + /// Check if timeout is enabled (timeout > 0). + pub fn is_timeout_enabled(&self) -> bool { + self.get_object_timeout > Duration::ZERO + } +} + +/// Information about a timeout event. +#[derive(Debug, Clone)] +pub struct TimeoutInfo { + /// Request ID for correlation. + pub request_id: String, + /// Bucket name. + pub bucket: String, + /// Object key. + pub key: String, + /// Configured timeout duration. + pub timeout_duration: Duration, + /// Actual elapsed time before timeout. + pub elapsed: Duration, + /// Number of bytes transferred before timeout. + pub bytes_transferred: u64, + /// Lock hold time before timeout. + pub lock_hold_time: Option, + /// Number of disk reads completed. + pub disk_reads_completed: u32, + /// Number of disk reads pending. + pub disk_reads_pending: u32, +} + +/// Result of a timed GetObject operation. +#[derive(Debug)] +pub enum TimedGetObjectResult { + /// Operation completed successfully within timeout. + Success(T), + /// Operation timed out and was cancelled. + Timeout(TimeoutInfo), + /// Operation failed with an error (before timeout). + Error(E), +} + +/// Request timeout wrapper for async operations. +#[derive(Debug)] +pub struct RequestTimeoutWrapper { + /// Configuration. + config: TimeoutConfig, + /// Request start time. + start_time: Instant, + /// Cancellation token for propagating cancellation to sub-tasks. + cancel_token: CancellationToken, + /// Request ID for logging/metrics. + request_id: String, +} + +impl RequestTimeoutWrapper { + /// Create a new timeout wrapper with the given configuration. + pub fn new(config: TimeoutConfig) -> Self { + Self { + config, + start_time: Instant::now(), + cancel_token: CancellationToken::new(), + request_id: format!("req-{}", &uuid::Uuid::new_v4().to_string()[..8]), + } + } + + /// Create a new timeout wrapper with a specific request ID. + pub fn with_request_id(config: TimeoutConfig, request_id: impl Into) -> Self { + Self { + config, + start_time: Instant::now(), + cancel_token: CancellationToken::new(), + request_id: request_id.into(), + } + } + + /// Get the request ID. + pub fn request_id(&self) -> &str { + &self.request_id + } + + /// Get the cancellation token. + /// This can be cloned and passed to sub-tasks for cooperative cancellation. + pub fn cancel_token(&self) -> CancellationToken { + self.cancel_token.clone() + } + + /// Check if the operation has been cancelled. + pub fn is_cancelled(&self) -> bool { + self.cancel_token.is_cancelled() + } + + /// Check if the timeout has been exceeded. + pub fn is_timeout(&self) -> bool { + self.config.is_timeout_enabled() && self.elapsed() >= self.config.get_object_timeout + } + + /// Get elapsed time since the request started. + pub fn elapsed(&self) -> Duration { + self.start_time.elapsed() + } + + /// Get remaining time before timeout. + /// Returns None if timeout is disabled or already exceeded. + pub fn remaining_time(&self) -> Option { + if !self.config.is_timeout_enabled() { + return None; + } + let remaining = self.config.get_object_timeout.saturating_sub(self.elapsed()); + if remaining == Duration::ZERO { None } else { Some(remaining) } + } + + /// Execute an async operation with timeout protection. + /// + /// The operation receives a `CancellationToken` that it can use to: + /// - Check for cancellation: `token.is_cancelled()` + /// - Pass to sub-tasks for propagation + /// - Bind to futures: `future.until_cancelled(token)` + /// + /// # Returns + /// + /// - `TimedGetObjectResult::Success(T)` if the operation completed within timeout + /// - `TimedGetObjectResult::Timeout(TimeoutInfo)` if the operation timed out + /// - `TimedGetObjectResult::Error(E)` if the operation failed + pub async fn execute_with_timeout(self, operation: F) -> TimedGetObjectResult + where + F: FnOnce(CancellationToken) -> Fut, + Fut: std::future::Future>, + { + if !self.config.is_timeout_enabled() { + // Timeout disabled, run without timeout + debug!( + request_id = %self.request_id, + "Timeout disabled, executing operation without timeout" + ); + return match operation(self.cancel_token).await { + Ok(result) => TimedGetObjectResult::Success(result), + Err(e) => TimedGetObjectResult::Error(e), + }; + } + + let timeout_duration = self.config.get_object_timeout; + let request_id = self.request_id.clone(); + let start_time = self.start_time; + + debug!( + request_id = %request_id, + timeout_secs = timeout_duration.as_secs(), + "Starting timed operation" + ); + + // Record start time for metrics + #[cfg(feature = "metrics")] + counter!("rustfs.get.object.requests.started").increment(1); + + // Clone cancel_token for the operation, keep original for potential cancellation + let cancel_token_for_op = self.cancel_token.clone(); + + match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await { + Ok(Ok(result)) => { + // Operation completed successfully + let elapsed = start_time.elapsed(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.requests.completed").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + debug!( + request_id = %request_id, + elapsed_ms = elapsed.as_millis(), + "Operation completed successfully" + ); + + TimedGetObjectResult::Success(result) + } + Ok(Err(e)) => { + // Operation failed before timeout + let elapsed = start_time.elapsed(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.requests.failed").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + debug!( + request_id = %request_id, + elapsed_ms = elapsed.as_millis(), + "Operation failed with error" + ); + + TimedGetObjectResult::Error(e) + } + Err(_) => { + // Timeout occurred + let elapsed = start_time.elapsed(); + + // Cancel the operation + self.cancel_token.cancel(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.timeout.total").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + warn!( + request_id = %request_id, + timeout_secs = timeout_duration.as_secs(), + elapsed_ms = elapsed.as_millis(), + "Operation timed out, cancellation signal sent" + ); + + TimedGetObjectResult::Timeout(TimeoutInfo { + request_id, + bucket: String::new(), + key: String::new(), + timeout_duration, + elapsed, + bytes_transferred: 0, + lock_hold_time: None, + disk_reads_completed: 0, + disk_reads_pending: 0, + }) + } + } + } + + /// Execute an async operation with timeout and context information. + /// + /// This is an extended version of `execute_with_timeout` that includes + /// bucket and key information for better timeout logging. + pub async fn execute_with_timeout_and_context( + self, + bucket: impl Into, + key: impl Into, + operation: F, + ) -> TimedGetObjectResult + where + F: FnOnce(CancellationToken) -> Fut, + Fut: std::future::Future>, + { + let bucket = bucket.into(); + let key = key.into(); + + if !self.config.is_timeout_enabled() { + debug!( + request_id = %self.request_id, + bucket = %bucket, + key = %key, + "Timeout disabled, executing operation without timeout" + ); + return match operation(self.cancel_token).await { + Ok(result) => TimedGetObjectResult::Success(result), + Err(e) => TimedGetObjectResult::Error(e), + }; + } + + let timeout_duration = self.config.get_object_timeout; + let request_id = self.request_id.clone(); + let start_time = self.start_time; + + debug!( + request_id = %request_id, + bucket = %bucket, + key = %key, + timeout_secs = timeout_duration.as_secs(), + "Starting timed operation" + ); + + #[cfg(feature = "metrics")] + counter!("rustfs.get.object.requests.started").increment(1); + + // Clone cancel_token for the operation, keep original for potential cancellation + let cancel_token_for_op = self.cancel_token.clone(); + + match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await { + Ok(Ok(result)) => { + let elapsed = start_time.elapsed(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.requests.completed").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + debug!( + request_id = %request_id, + bucket = %bucket, + key = %key, + elapsed_ms = elapsed.as_millis(), + "Operation completed successfully" + ); + + TimedGetObjectResult::Success(result) + } + Ok(Err(e)) => { + let elapsed = start_time.elapsed(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.requests.failed").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + debug!( + request_id = %request_id, + bucket = %bucket, + key = %key, + elapsed_ms = elapsed.as_millis(), + "Operation failed with error" + ); + + TimedGetObjectResult::Error(e) + } + Err(_) => { + let elapsed = start_time.elapsed(); + self.cancel_token.cancel(); + + #[cfg(feature = "metrics")] + { + counter!("rustfs.get.object.timeout.total").increment(1); + histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64()); + } + + warn!( + request_id = %request_id, + bucket = %bucket, + key = %key, + timeout_secs = timeout_duration.as_secs(), + elapsed_ms = elapsed.as_millis(), + "Operation timed out, cancellation signal sent" + ); + + TimedGetObjectResult::Timeout(TimeoutInfo { + request_id, + bucket, + key, + timeout_duration, + elapsed, + bytes_transferred: 0, + lock_hold_time: None, + disk_reads_completed: 0, + disk_reads_pending: 0, + }) + } + } + } +} + +/// Get the duplex buffer size from environment or default. +pub fn get_duplex_buffer_size() -> usize { + rustfs_utils::get_env_usize( + rustfs_config::ENV_OBJECT_DUPLEX_BUFFER_SIZE, + rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, + ) +} + +/// Get the I/O buffer size from environment or default. +pub fn get_io_buffer_size() -> usize { + rustfs_utils::get_env_usize(rustfs_config::ENV_OBJECT_IO_BUFFER_SIZE, rustfs_config::DEFAULT_OBJECT_IO_BUFFER_SIZE) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn test_timeout_config_default() { + let config = TimeoutConfig::default(); + assert_eq!(config.get_object_timeout, Duration::from_secs(30)); + assert_eq!(config.lock_acquire_timeout, Duration::from_secs(5)); + assert_eq!(config.disk_read_timeout, Duration::from_secs(10)); + } + + #[test] + fn test_timeout_config_is_enabled() { + let config = TimeoutConfig::default(); + assert!(config.is_timeout_enabled()); + + let disabled_config = TimeoutConfig { + get_object_timeout: Duration::ZERO, + ..Default::default() + }; + assert!(!disabled_config.is_timeout_enabled()); + } + + #[tokio::test] + async fn test_timeout_wrapper_success() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_secs(5), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { Ok::(42) }) + .await; + + match result { + TimedGetObjectResult::Success(value) => assert_eq!(value, 42), + _ => panic!("Expected Success result"), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_timeout() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_millis(100), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { + tokio::time::sleep(Duration::from_secs(10)).await; + Ok::(42) + }) + .await; + + match result { + TimedGetObjectResult::Timeout(info) => { + assert!(info.elapsed >= Duration::from_millis(100)); + } + _ => panic!("Expected Timeout result"), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_error() { + let config = TimeoutConfig { + get_object_timeout: Duration::from_secs(5), + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + let result = wrapper + .execute_with_timeout(|_token| async move { Err::("test error".to_string()) }) + .await; + + match result { + TimedGetObjectResult::Error(e) => assert_eq!(e, "test error"), + _ => panic!("Expected Error result"), + } + } + + #[tokio::test] + async fn test_timeout_wrapper_disabled() { + let config = TimeoutConfig { + get_object_timeout: Duration::ZERO, + ..Default::default() + }; + let wrapper = RequestTimeoutWrapper::new(config); + + // This would timeout if timeout was enabled + let result = wrapper + .execute_with_timeout(|_token| async move { + tokio::time::sleep(Duration::from_millis(200)).await; + Ok::(42) + }) + .await; + + match result { + TimedGetObjectResult::Success(value) => assert_eq!(value, 42), + _ => panic!("Expected Success result when timeout is disabled"), + } + } + + #[test] + fn test_get_duplex_buffer_size() { + // Should return default (4MB) when env var not set + let size = get_duplex_buffer_size(); + assert_eq!(size, 4 * 1024 * 1024); + } + + #[test] + fn test_get_io_buffer_size() { + // Should return default (128KB) when env var not set + let size = get_io_buffer_size(); + assert_eq!(size, 128 * 1024); + } +} diff --git a/scripts/run.sh b/scripts/run.sh old mode 100755 new mode 100644 index 0ae46ca0d..7639e3676 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -53,7 +53,7 @@ export RUSTFS_CONSOLE_ADDRESS=":9001" # export RUSTFS_TLS_PATH="./deploy/certs" # Observability related configuration -export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector address +#export RUSTFS_OBS_ENDPOINT=http://localhost:4318 # OpenTelemetry Collector address # RustFS OR OTEL exporter configuration #export RUSTFS_OBS_TRACE_ENDPOINT=http://localhost:4318/v1/traces # OpenTelemetry Collector trace address http://localhost:4318/v1/traces #export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:14318/v1/traces @@ -215,14 +215,98 @@ export RUSTFS_TRUST_SYSTEM_CA=true # export RUSTFS_FTPS_ADDRESS="0.0.0.0:8022" # export RUSTFS_FTPS_CERTS_DIR="${current_dir}/deploy/certs/ftps" -# Use default timeout (60 seconds) -# No environment variable needed +# ============================================ +# Concurrent Request Optimization Configuration +# ============================================ +# These settings optimize GetObject performance under high concurrency. +# Most features are enabled by default with sensible values. +# Uncomment and adjust based on your scenario. -# Increase timeout for high-latency network storage -#export RUSTFS_LOCK_ACQUIRE_TIMEOUT=120 +# --- Default Configuration (Recommended for most cases) --- +# Request timeout: 30 seconds (prevents indefinite hangs) +export RUSTFS_OBJECT_GET_TIMEOUT=30 +# Disk read timeout: 10 seconds +export RUSTFS_OBJECT_DISK_READ_TIMEOUT=10 +# Lock acquire timeout: 5 seconds +export RUSTFS_OBJECT_LOCK_ACQUIRE_TIMEOUT=5 +# Duplex buffer size: 4MB (4x larger than original 1MB) +export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=4194304 +# I/O buffer size: 128KB +export RUSTFS_OBJECT_IO_BUFFER_SIZE=131072 +# Max concurrent disk reads: 64 +export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 +# Lock optimization: release read lock after metadata read +export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# Priority scheduling: small requests get higher priority +export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true +# Deadlock detection: disabled by default (has performance overhead) +# export RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE=false -# Reduce timeout for low-latency local storage -export RUSTFS_LOCK_ACQUIRE_TIMEOUT=30 +# --- Scenario 1: Home NAS / Small Storage Server --- +# Hardware: 4-8 cores, 8-16GB RAM, 1-4 HDDs, 1Gbps network +# Typical concurrency: 5-20 requests +# export RUSTFS_OBJECT_GET_TIMEOUT=30 +# export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=4194304 +# export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 +# export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + +# --- Scenario 2: Medium Enterprise Storage --- +# Hardware: 8-16 cores, 32-64GB RAM, 4-12 HDDs/SSDs, 10Gbps network +# Typical concurrency: 20-100 requests +# export RUSTFS_OBJECT_GET_TIMEOUT=60 +# export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 +# export RUSTFS_OBJECT_IO_BUFFER_SIZE=262144 +# export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=128 +# export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=16 +# export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + +# --- Scenario 3: Large Enterprise / Cloud Storage --- +# Hardware: 32+ cores, 128+GB RAM, NVMe SSD array, 25-100Gbps network +# Typical concurrency: 100-1000 requests +# export RUSTFS_OBJECT_GET_TIMEOUT=120 +# export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=16777216 +# export RUSTFS_OBJECT_IO_BUFFER_SIZE=524288 +# export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=256 +# export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=32 +# export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + +# --- Scenario 4: Kopia Backup Optimized --- +# Problem: Kopia sends 20-30 concurrent range requests for 20-26MB objects +# Solution: Larger buffer, lock optimization, priority scheduling +# export RUSTFS_OBJECT_GET_TIMEOUT=45 +# export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=8388608 +# export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=64 +# export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true +# export RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE=true # Enable for debugging + +# --- Scenario 5: Low Power / Embedded Storage --- +# Hardware: 2-4 cores (ARM/x86), 2-4GB RAM, SD card/eMMC, 100Mbps-1Gbps +# Typical concurrency: 1-5 requests +# export RUSTFS_OBJECT_GET_TIMEOUT=60 +# export RUSTFS_OBJECT_DISK_READ_TIMEOUT=20 +# export RUSTFS_OBJECT_DUPLEX_BUFFER_SIZE=2097152 +# export RUSTFS_OBJECT_IO_BUFFER_SIZE=65536 +# export RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS=16 +# export RUSTFS_OBJECT_HIGH_CONCURRENCY_THRESHOLD=4 +# export RUSTFS_OBJECT_LOCK_OPTIMIZATION_ENABLE=true +# export RUSTFS_OBJECT_PRIORITY_SCHEDULING_ENABLE=true + +# --- Scenario 6: Debugging / Troubleshooting --- +# Enable all diagnostic features +# export RUSTFS_OBJECT_GET_TIMEOUT=15 +# export RUSTFS_OBJECT_DEADLOCK_DETECTION_ENABLE=true +# export RUSTFS_OBJECT_DEADLOCK_CHECK_INTERVAL=3 +# export RUSTFS_OBJECT_DEADLOCK_HANG_THRESHOLD=5 + +# --- Backpressure Configuration --- +# High watermark: trigger backpressure when buffer usage exceeds this percentage +export RUSTFS_BACKPRESSURE_HIGH_WATERMARK=80 +# Low watermark: release backpressure when buffer usage drops below this percentage +export RUSTFS_BACKPRESSURE_LOW_WATERMARK=50 if [ -n "$1" ]; then export RUSTFS_VOLUMES="$1"