refactor: converge storage io hot paths (#3029)

* refactor(issue-633): clarify layered io control policies

* refactor(issue-633): consolidate timeout and deadlock layers

* refactor(issue-633): align storage backpressure metadata

* refactor(issue-633): unify storage backpressure transitions

* refactor(issue-633): simplify watermark transition API

* test(issue-633): add storage backpressure transition test

* refactor(issue-633): align storage pipe meta shape

* refactor(issue-633): enrich storage monitor metadata

* refactor(issue-633): finalize storage backpressure convergence

* refactor(issue-633): complete scheduler layer convergence

* refactor(issue-633): reduce concurrency facade config duplication

* refactor(issue-633): migrate storage callsites to final policy names

* chore(issue-633): apply final pre-commit normalization

* refactor(issue-633): unify timeout wrapper dynamic size path

* refactor(issue-633): make concurrency policies copyable

* refactor(issue-633): converge storage io hot paths

* fix(issue-633): honor storage timeout min bound

* fix(storage): avoid timeout calc panic on huge sizes

* refactor(storage): consolidate timeout checks and test attrs

* fix(storage): harden io scheduler core config mapping

* refactor(storage): eliminate patch-on-patch patterns and dead code

- Remove trivial accessor methods on ConcurrencyConfig that just return pub fields
- Remove dead BackpressureEvent/BackpressureEventType types from concurrency crate
- Fix io_schedule test using wrong constructor (from_core_config -> from_scheduler_config)
- Update manager.rs to use config fields directly instead of removed accessors

* fix: adopt review feedback for config guards

* test: remove needless struct update defaults

* fix: harden timeout policy and preserve api alias
This commit is contained in:
houseme
2026-05-20 20:00:23 +08:00
committed by GitHub
parent 45b04316b9
commit 375482a4b6
14 changed files with 988 additions and 702 deletions
+72 -51
View File
@@ -14,15 +14,17 @@
//! Backpressure management //! Backpressure management
use rustfs_io_core::{BackpressureMonitor as CoreBackpressureMonitor, BackpressureState}; use rustfs_io_core::{
BackpressureConfig as CoreBackpressureConfig, BackpressureMonitor as CoreBackpressureMonitor, BackpressureState,
};
use rustfs_io_metrics::backpressure_metrics; use rustfs_io_metrics::backpressure_metrics;
use std::sync::Arc; use std::sync::Arc;
use std::time::Instant; use std::time::Instant;
use tokio::io::{DuplexStream, duplex}; use tokio::io::{DuplexStream, duplex};
/// Backpressure configuration /// Facade policy for duplex-pipe watermark backpressure.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy)]
pub struct BackpressureConfig { pub struct PipeBackpressurePolicy {
/// Buffer size in bytes /// Buffer size in bytes
pub buffer_size: usize, pub buffer_size: usize,
/// High watermark percentage /// High watermark percentage
@@ -31,7 +33,7 @@ pub struct BackpressureConfig {
pub low_watermark: u32, pub low_watermark: u32,
} }
impl Default for BackpressureConfig { impl Default for PipeBackpressurePolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
buffer_size: 4 * 1024 * 1024, // 4MB buffer_size: 4 * 1024 * 1024, // 4MB
@@ -41,7 +43,7 @@ impl Default for BackpressureConfig {
} }
} }
impl BackpressureConfig { impl PipeBackpressurePolicy {
/// Calculate high watermark threshold in bytes /// Calculate high watermark threshold in bytes
pub fn high_watermark_bytes(&self) -> usize { pub fn high_watermark_bytes(&self) -> usize {
(self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize (self.buffer_size as u64 * self.high_watermark as u64 / 100) as usize
@@ -51,42 +53,59 @@ impl BackpressureConfig {
pub fn low_watermark_bytes(&self) -> usize { pub fn low_watermark_bytes(&self) -> usize {
(self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize (self.buffer_size as u64 * self.low_watermark as u64 / 100) as usize
} }
/// Convert the facade policy into the reusable io-core admission-pressure config.
///
/// The concurrency layer still owns duplex buffer sizing, but the shared
/// overload/admission primitive lives in `io-core`.
pub fn to_core_config(&self) -> CoreBackpressureConfig {
CoreBackpressureConfig {
max_concurrent: 32,
high_water_mark: self.high_watermark as f64 / 100.0,
low_water_mark: self.low_watermark as f64 / 100.0,
cooldown: std::time::Duration::from_millis(100),
enabled: true,
}
}
} }
/// Backpressure manager /// Backpressure manager
pub struct BackpressureManager { pub struct BackpressureManager {
config: BackpressureConfig, config: PipeBackpressurePolicy,
core_config: CoreBackpressureConfig,
monitor: Arc<CoreBackpressureMonitor>, monitor: Arc<CoreBackpressureMonitor>,
} }
impl BackpressureManager { impl BackpressureManager {
/// Create a new backpressure manager /// Create a new backpressure manager
pub fn new(buffer_size: usize, high_watermark: u32, low_watermark: u32) -> Self { pub fn new(buffer_size: usize, high_watermark: u32, low_watermark: u32) -> Self {
let config = BackpressureConfig { Self::from_policy(PipeBackpressurePolicy {
buffer_size, buffer_size,
high_watermark, high_watermark,
low_watermark, low_watermark,
}; })
}
let core_config = rustfs_io_core::BackpressureConfig {
max_concurrent: 32,
high_water_mark: high_watermark as f64 / 100.0,
low_water_mark: low_watermark as f64 / 100.0,
cooldown: std::time::Duration::from_millis(100),
enabled: true,
};
/// Create a new backpressure manager from the facade policy type.
pub fn from_policy(config: PipeBackpressurePolicy) -> Self {
let core_config = config.to_core_config();
Self { Self {
config, config,
core_config: core_config.clone(),
monitor: Arc::new(CoreBackpressureMonitor::new(core_config)), monitor: Arc::new(CoreBackpressureMonitor::new(core_config)),
} }
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &BackpressureConfig { pub fn config(&self) -> &PipeBackpressurePolicy {
&self.config &self.config
} }
/// Get the derived io-core admission-pressure configuration.
pub fn core_config(&self) -> &CoreBackpressureConfig {
&self.core_config
}
/// Get the monitor /// Get the monitor
pub fn monitor(&self) -> Arc<CoreBackpressureMonitor> { pub fn monitor(&self) -> Arc<CoreBackpressureMonitor> {
self.monitor.clone() self.monitor.clone()
@@ -94,7 +113,7 @@ impl BackpressureManager {
/// Create a backpressure pipe /// Create a backpressure pipe
pub fn create_pipe(&self) -> BackpressurePipe { pub fn create_pipe(&self) -> BackpressurePipe {
BackpressurePipe::new(self.config.clone(), self.monitor.clone()) BackpressurePipe::new(self.config, self.monitor.clone())
} }
/// Get current state /// Get current state
@@ -112,13 +131,24 @@ impl BackpressureManager {
pub struct BackpressurePipe { pub struct BackpressurePipe {
reader: DuplexStream, reader: DuplexStream,
writer: DuplexStream, writer: DuplexStream,
config: BackpressureConfig, config: PipeBackpressurePolicy,
monitor: Arc<CoreBackpressureMonitor>, monitor: Arc<CoreBackpressureMonitor>,
created_at: Instant, created_at: Instant,
} }
/// Shared pipe metadata snapshot for facade-level backpressure pipes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BackpressurePipeMeta {
/// Configured duplex buffer capacity in bytes.
pub buffer_capacity: usize,
/// Current backpressure state reported by the shared core monitor.
pub state: BackpressureState,
/// Age of the pipe since creation.
pub age: std::time::Duration,
}
impl BackpressurePipe { impl BackpressurePipe {
fn new(config: BackpressureConfig, monitor: Arc<CoreBackpressureMonitor>) -> Self { fn new(config: PipeBackpressurePolicy, monitor: Arc<CoreBackpressureMonitor>) -> Self {
let (reader, writer) = duplex(config.buffer_size); let (reader, writer) = duplex(config.buffer_size);
Self { Self {
@@ -146,7 +176,7 @@ impl BackpressurePipe {
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &BackpressureConfig { pub fn config(&self) -> &PipeBackpressurePolicy {
&self.config &self.config
} }
@@ -160,6 +190,15 @@ impl BackpressurePipe {
self.created_at.elapsed() self.created_at.elapsed()
} }
/// Get a compact metadata snapshot for the pipe.
pub fn meta(&self) -> BackpressurePipeMeta {
BackpressurePipeMeta {
buffer_capacity: self.config.buffer_size,
state: self.state(),
age: self.age(),
}
}
/// Check if should apply backpressure /// Check if should apply backpressure
pub fn should_apply_backpressure(&self) -> bool { pub fn should_apply_backpressure(&self) -> bool {
let should = self.monitor.should_apply_backpressure(); let should = self.monitor.should_apply_backpressure();
@@ -170,45 +209,26 @@ impl BackpressurePipe {
} }
} }
/// Backpressure event
#[allow(dead_code)]
#[derive(Debug, Clone)]
pub struct BackpressureEvent {
/// Event timestamp
pub timestamp: Instant,
/// Event type
pub event_type: BackpressureEventType,
/// Buffer usage
pub buffer_usage: usize,
/// Buffer capacity
pub buffer_capacity: usize,
}
/// Backpressure event type
#[allow(dead_code)]
#[derive(Debug, Clone, Copy)]
pub enum BackpressureEventType {
/// High watermark reached
HighWatermarkReached,
/// High watermark exited
HighWatermarkExited,
/// Backpressure applied
BackpressureApplied,
/// Backpressure released
BackpressureReleased,
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[test] #[test]
fn test_backpressure_config() { fn test_backpressure_config() {
let config = BackpressureConfig::default(); let config = PipeBackpressurePolicy::default();
assert_eq!(config.buffer_size, 4 * 1024 * 1024); assert_eq!(config.buffer_size, 4 * 1024 * 1024);
assert!(config.high_watermark > config.low_watermark); assert!(config.high_watermark > config.low_watermark);
} }
#[test]
fn test_backpressure_policy_to_core_config() {
let policy = PipeBackpressurePolicy::default();
let core = policy.to_core_config();
assert_eq!(core.high_water_mark, policy.high_watermark as f64 / 100.0);
assert_eq!(core.low_water_mark, policy.low_watermark as f64 / 100.0);
assert!(core.enabled);
}
#[test] #[test]
fn test_backpressure_manager() { fn test_backpressure_manager() {
let manager = BackpressureManager::new(1024, 80, 50); let manager = BackpressureManager::new(1024, 80, 50);
@@ -220,5 +240,6 @@ mod tests {
let manager = BackpressureManager::new(1024, 80, 50); let manager = BackpressureManager::new(1024, 80, 50);
let pipe = manager.create_pipe(); let pipe = manager.create_pipe();
assert_eq!(pipe.state(), BackpressureState::Normal); assert_eq!(pipe.state(), BackpressureState::Normal);
assert_eq!(pipe.meta().buffer_capacity, 1024);
} }
} }
+70 -84
View File
@@ -14,6 +14,10 @@
//! Configuration for concurrency management //! Configuration for concurrency management
use crate::{
backpressure::PipeBackpressurePolicy, deadlock::DeadlockMonitorPolicy, scheduler::SchedulerPolicy,
timeout::TimeoutManagerPolicy,
};
use std::time::Duration; use std::time::Duration;
/// Feature flags for concurrency modules /// Feature flags for concurrency modules
@@ -72,84 +76,39 @@ impl ConcurrencyFeatures {
} }
} }
/// Facade policy for lock manager behavior.
#[derive(Debug, Clone, Copy)]
pub struct LockManagerPolicy {
/// Enable lock optimization.
pub enabled: bool,
/// Lock acquisition timeout.
pub acquire_timeout: Duration,
}
impl Default for LockManagerPolicy {
fn default() -> Self {
Self {
enabled: true,
acquire_timeout: Duration::from_secs(5),
}
}
}
/// Main configuration for concurrency management /// Main configuration for concurrency management
#[derive(Debug, Clone)] #[derive(Debug, Clone, Default)]
pub struct ConcurrencyConfig { pub struct ConcurrencyConfig {
/// Feature flags /// Feature flags
pub features: ConcurrencyFeatures, pub features: ConcurrencyFeatures,
/// Timeout facade policy.
// Timeout configuration pub timeout_policy: TimeoutManagerPolicy,
/// Default timeout duration /// Lock facade policy.
pub default_timeout: Duration, pub lock_policy: LockManagerPolicy,
/// Maximum timeout duration /// Deadlock facade policy.
pub max_timeout: Duration, pub deadlock_policy: DeadlockMonitorPolicy,
/// Enable dynamic timeout /// Backpressure facade policy.
pub enable_dynamic_timeout: bool, pub backpressure_policy: PipeBackpressurePolicy,
/// Scheduler facade policy.
// Lock configuration pub scheduler_policy: SchedulerPolicy,
/// Enable lock optimization
pub enable_lock_optimization: bool,
/// Lock acquisition timeout
pub lock_acquire_timeout: Duration,
// Deadlock configuration
/// Enable deadlock detection
pub enable_deadlock_detection: bool,
/// Deadlock check interval
pub deadlock_check_interval: Duration,
/// Hang threshold
pub hang_threshold: Duration,
// Backpressure configuration
/// Buffer size for backpressure
pub backpressure_buffer_size: usize,
/// High watermark percentage
pub high_watermark: u32,
/// Low watermark percentage
pub low_watermark: u32,
// Scheduler configuration
/// Base buffer size for I/O
pub io_buffer_size: usize,
/// Maximum buffer size
pub max_buffer_size: usize,
/// High priority size threshold
pub high_priority_threshold: usize,
/// Low priority size threshold
pub low_priority_threshold: usize,
}
impl Default for ConcurrencyConfig {
fn default() -> Self {
Self {
features: ConcurrencyFeatures::default(),
// Timeout defaults
default_timeout: Duration::from_secs(30),
max_timeout: Duration::from_secs(300),
enable_dynamic_timeout: true,
// Lock defaults
enable_lock_optimization: true,
lock_acquire_timeout: Duration::from_secs(5),
// Deadlock defaults
enable_deadlock_detection: false,
deadlock_check_interval: Duration::from_secs(10),
hang_threshold: Duration::from_secs(60),
// Backpressure defaults
backpressure_buffer_size: 4 * 1024 * 1024, // 4MB
high_watermark: 80,
low_watermark: 50,
// Scheduler defaults
io_buffer_size: 64 * 1024, // 64KB
max_buffer_size: 4 * 1024 * 1024, // 4MB
high_priority_threshold: 1024 * 1024, // 1MB
low_priority_threshold: 10 * 1024 * 1024, // 10MB
}
}
} }
impl ConcurrencyConfig { impl ConcurrencyConfig {
@@ -161,25 +120,25 @@ impl ConcurrencyConfig {
if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_DEFAULT") if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_DEFAULT")
&& let Ok(secs) = val.parse::<u64>() && let Ok(secs) = val.parse::<u64>()
{ {
config.default_timeout = Duration::from_secs(secs); config.timeout_policy.default_timeout = Duration::from_secs(secs);
} }
if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_MAX") if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_MAX")
&& let Ok(secs) = val.parse::<u64>() && let Ok(secs) = val.parse::<u64>()
{ {
config.max_timeout = Duration::from_secs(secs); config.timeout_policy.max_timeout = Duration::from_secs(secs);
} }
if let Ok(val) = std::env::var("RUSTFS_BACKPRESSURE_BUFFER_SIZE") if let Ok(val) = std::env::var("RUSTFS_BACKPRESSURE_BUFFER_SIZE")
&& let Ok(size) = val.parse::<usize>() && let Ok(size) = val.parse::<usize>()
{ {
config.backpressure_buffer_size = size; config.backpressure_policy.buffer_size = size;
} }
if let Ok(val) = std::env::var("RUSTFS_IO_BUFFER_SIZE") if let Ok(val) = std::env::var("RUSTFS_IO_BUFFER_SIZE")
&& let Ok(size) = val.parse::<usize>() && let Ok(size) = val.parse::<usize>()
{ {
config.io_buffer_size = size; config.scheduler_policy.base_buffer_size = size;
} }
config config
@@ -187,18 +146,25 @@ impl ConcurrencyConfig {
/// Validate configuration /// Validate configuration
pub fn validate(&self) -> Result<(), ConfigError> { pub fn validate(&self) -> Result<(), ConfigError> {
if self.default_timeout > self.max_timeout { if self.timeout_policy.default_timeout > self.timeout_policy.max_timeout {
return Err(ConfigError::InvalidTimeout("default_timeout cannot exceed max_timeout".to_string())); return Err(ConfigError::InvalidTimeout("default_timeout cannot exceed max_timeout".to_string()));
} }
if self.timeout_policy.min_timeout > self.timeout_policy.max_timeout {
return Err(ConfigError::InvalidTimeout("min_timeout cannot exceed max_timeout".to_string()));
}
if self.high_watermark <= self.low_watermark || self.high_watermark > 100 { if self.backpressure_policy.high_watermark <= self.backpressure_policy.low_watermark
|| self.backpressure_policy.high_watermark > 100
{
return Err(ConfigError::InvalidBackpressure( return Err(ConfigError::InvalidBackpressure(
"high_watermark must be > low_watermark and <= 100".to_string(), "high_watermark must be > low_watermark and <= 100".to_string(),
)); ));
} }
if self.io_buffer_size > self.max_buffer_size { if self.scheduler_policy.base_buffer_size > self.scheduler_policy.max_buffer_size {
return Err(ConfigError::InvalidScheduler("io_buffer_size cannot exceed max_buffer_size".to_string())); return Err(ConfigError::InvalidScheduler(
"base_buffer_size cannot exceed max_buffer_size".to_string(),
));
} }
Ok(()) Ok(())
@@ -235,8 +201,12 @@ mod tests {
#[test] #[test]
fn test_invalid_timeout() { fn test_invalid_timeout() {
let config = ConcurrencyConfig { let config = ConcurrencyConfig {
default_timeout: Duration::from_secs(100), timeout_policy: TimeoutManagerPolicy {
max_timeout: Duration::from_secs(50), default_timeout: Duration::from_secs(100),
max_timeout: Duration::from_secs(50),
enable_dynamic: true,
..Default::default()
},
..Default::default() ..Default::default()
}; };
assert!( assert!(
@@ -245,6 +215,22 @@ mod tests {
); );
} }
#[test]
fn test_invalid_min_timeout() {
let config = ConcurrencyConfig {
timeout_policy: TimeoutManagerPolicy {
min_timeout: Duration::from_secs(100),
max_timeout: Duration::from_secs(50),
..Default::default()
},
..Default::default()
};
assert!(
config.validate().is_err(),
"validate() should return an error when min_timeout > max_timeout"
);
}
#[test] #[test]
fn test_features() { fn test_features() {
let features = ConcurrencyFeatures::all(); let features = ConcurrencyFeatures::all();
+47 -17
View File
@@ -14,15 +14,15 @@
//! Deadlock detection management //! Deadlock detection management
use rustfs_io_core::{DeadlockDetector as CoreDeadlockDetector, LockType}; use rustfs_io_core::{DeadlockDetector as CoreDeadlockDetector, DeadlockDetectorConfig as CoreDeadlockConfig, LockType};
use rustfs_io_metrics::deadlock_metrics; use rustfs_io_metrics::deadlock_metrics;
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
/// Deadlock configuration /// Facade policy for the concurrency-layer deadlock monitor.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy)]
pub struct DeadlockConfig { pub struct DeadlockMonitorPolicy {
/// Enable deadlock detection /// Enable deadlock detection
pub enabled: bool, pub enabled: bool,
/// Check interval /// Check interval
@@ -31,7 +31,7 @@ pub struct DeadlockConfig {
pub hang_threshold: Duration, pub hang_threshold: Duration,
} }
impl Default for DeadlockConfig { impl Default for DeadlockMonitorPolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
enabled: false, enabled: false,
@@ -41,9 +41,20 @@ impl Default for DeadlockConfig {
} }
} }
impl DeadlockMonitorPolicy {
/// Convert the facade policy into the reusable io-core deadlock config.
pub fn to_core_config(&self) -> CoreDeadlockConfig {
CoreDeadlockConfig {
enabled: self.enabled,
detection_interval: self.check_interval,
max_hold_time: self.hang_threshold,
}
}
}
/// Deadlock manager /// Deadlock manager
pub struct DeadlockManager { pub struct DeadlockManager {
config: DeadlockConfig, config: DeadlockMonitorPolicy,
detector: Arc<CoreDeadlockDetector>, detector: Arc<CoreDeadlockDetector>,
running: Arc<tokio::sync::Mutex<bool>>, running: Arc<tokio::sync::Mutex<bool>>,
} }
@@ -51,18 +62,16 @@ pub struct DeadlockManager {
impl DeadlockManager { impl DeadlockManager {
/// Create a new deadlock manager /// Create a new deadlock manager
pub fn new(enabled: bool, check_interval: Duration, hang_threshold: Duration) -> Self { pub fn new(enabled: bool, check_interval: Duration, hang_threshold: Duration) -> Self {
let config = DeadlockConfig { Self::from_policy(DeadlockMonitorPolicy {
enabled, enabled,
check_interval, check_interval,
hang_threshold, hang_threshold,
}; })
}
let core_config = rustfs_io_core::DeadlockDetectorConfig {
enabled,
detection_interval: check_interval,
max_hold_time: hang_threshold,
};
/// Create a new deadlock manager from the facade policy type.
pub fn from_policy(config: DeadlockMonitorPolicy) -> Self {
let core_config = config.to_core_config();
Self { Self {
config, config,
detector: Arc::new(CoreDeadlockDetector::new(core_config)), detector: Arc::new(CoreDeadlockDetector::new(core_config)),
@@ -71,7 +80,7 @@ impl DeadlockManager {
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &DeadlockConfig { pub fn config(&self) -> &DeadlockMonitorPolicy {
&self.config &self.config
} }
@@ -129,7 +138,11 @@ impl DeadlockManager {
} }
} }
/// Request tracker for tracking resources /// Lightweight compatibility wrapper for request-scoped deadlock bookkeeping.
///
/// This type intentionally stays minimal in the concurrency layer. Rich
/// request-level lock/resource diagnostics belong to
/// `rustfs::storage::deadlock_detector::RequestResourceTracker`.
pub struct RequestTracker { pub struct RequestTracker {
request_id: String, request_id: String,
description: String, description: String,
@@ -174,6 +187,11 @@ impl RequestTracker {
deadlock_metrics::record_lock_acquisition("read"); deadlock_metrics::record_lock_acquisition("read");
} }
/// Return a read-only view of tracked resource names.
pub fn resources(&self) -> &HashMap<String, Vec<String>> {
&self.resources
}
/// Record a lock release /// Record a lock release
pub fn record_lock_release(&mut self, lock_id: u64) { pub fn record_lock_release(&mut self, lock_id: u64) {
self.detector.record_release(lock_id); self.detector.record_release(lock_id);
@@ -196,12 +214,24 @@ mod tests {
assert!(!manager.config().enabled); assert!(!manager.config().enabled);
} }
#[test]
fn test_deadlock_policy_to_core_config() {
let policy = DeadlockMonitorPolicy::default();
let core = policy.to_core_config();
assert_eq!(core.enabled, policy.enabled);
assert_eq!(core.detection_interval, policy.check_interval);
assert_eq!(core.max_hold_time, policy.hang_threshold);
}
#[tokio::test] #[tokio::test]
async fn test_request_tracker() { async fn test_request_tracker() {
let manager = DeadlockManager::new(true, Duration::from_secs(10), Duration::from_secs(60)); let manager = DeadlockManager::new(true, Duration::from_secs(10), Duration::from_secs(60));
let tracker = manager.track_request("req-1".to_string(), "test request".to_string()); let mut tracker = manager.track_request("req-1".to_string(), "test request".to_string());
let lock_id = manager.register_lock(LockType::Mutex);
tracker.record_lock_acquire(lock_id, "bucket/key".to_string());
assert_eq!(tracker.request_id(), "req-1"); assert_eq!(tracker.request_id(), "req-1");
assert_eq!(tracker.description(), "test request"); assert_eq!(tracker.description(), "test request");
assert_eq!(tracker.resources().get("locks").map(Vec::len), Some(1));
} }
} }
+8 -8
View File
@@ -128,19 +128,19 @@ pub mod workers;
// Public module exports with feature gates // Public module exports with feature gates
#[cfg(feature = "timeout")] #[cfg(feature = "timeout")]
pub use timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager}; pub use timeout::{TimeoutGuard, TimeoutManager, TimeoutManagerPolicy};
#[cfg(feature = "lock")] #[cfg(feature = "lock")]
pub use lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard}; pub use lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
#[cfg(feature = "deadlock")] #[cfg(feature = "deadlock")]
pub use deadlock::{DeadlockConfig, DeadlockManager, RequestTracker}; pub use deadlock::{DeadlockManager, DeadlockMonitorPolicy, RequestTracker};
#[cfg(feature = "backpressure")] #[cfg(feature = "backpressure")]
pub use backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe}; pub use backpressure::{BackpressureManager, BackpressurePipe, PipeBackpressurePolicy};
#[cfg(feature = "scheduler")] #[cfg(feature = "scheduler")]
pub use scheduler::{IoStrategy, SchedulerConfig, SchedulerManager}; pub use scheduler::{IoStrategy, SchedulerManager, SchedulerPolicy};
// Configuration // Configuration
mod config; mod config;
@@ -155,19 +155,19 @@ pub mod prelude {
//! Prelude module for convenient imports //! Prelude module for convenient imports
#[cfg(feature = "timeout")] #[cfg(feature = "timeout")]
pub use crate::timeout::{TimeoutConfig, TimeoutGuard, TimeoutManager}; pub use crate::timeout::{TimeoutGuard, TimeoutManager, TimeoutManagerPolicy};
#[cfg(feature = "lock")] #[cfg(feature = "lock")]
pub use crate::lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard}; pub use crate::lock::{LockConfig, LockManager, LockScopeGuard, OptimizedLockGuard};
#[cfg(feature = "deadlock")] #[cfg(feature = "deadlock")]
pub use crate::deadlock::{DeadlockConfig, DeadlockManager, RequestTracker}; pub use crate::deadlock::{DeadlockManager, DeadlockMonitorPolicy, RequestTracker};
#[cfg(feature = "backpressure")] #[cfg(feature = "backpressure")]
pub use crate::backpressure::{BackpressureConfig, BackpressureManager, BackpressurePipe}; pub use crate::backpressure::{BackpressureManager, BackpressurePipe, PipeBackpressurePolicy};
#[cfg(feature = "scheduler")] #[cfg(feature = "scheduler")]
pub use crate::scheduler::{IoStrategy, SchedulerConfig, SchedulerManager}; pub use crate::scheduler::{IoStrategy, SchedulerManager, SchedulerPolicy};
pub use crate::{ConcurrencyConfig, ConcurrencyFeatures, ConcurrencyManager}; pub use crate::{ConcurrencyConfig, ConcurrencyFeatures, ConcurrencyManager};
} }
+7 -24
View File
@@ -85,39 +85,22 @@ impl ConcurrencyManager {
Self { Self {
#[cfg(feature = "timeout")] #[cfg(feature = "timeout")]
timeout: Arc::new(crate::timeout::TimeoutManager::new( timeout: Arc::new(crate::timeout::TimeoutManager::from_policy(config.timeout_policy)),
config.default_timeout,
config.max_timeout,
config.enable_dynamic_timeout,
)),
#[cfg(feature = "lock")] #[cfg(feature = "lock")]
lock: Arc::new(crate::lock::LockManager::new( lock: Arc::new(crate::lock::LockManager::new(
config.enable_lock_optimization, config.lock_policy.enabled,
config.lock_acquire_timeout, config.lock_policy.acquire_timeout,
)), )),
#[cfg(feature = "deadlock")] #[cfg(feature = "deadlock")]
deadlock: Arc::new(crate::deadlock::DeadlockManager::new( deadlock: Arc::new(crate::deadlock::DeadlockManager::from_policy(config.deadlock_policy)),
config.enable_deadlock_detection,
config.deadlock_check_interval,
config.hang_threshold,
)),
#[cfg(feature = "backpressure")] #[cfg(feature = "backpressure")]
backpressure: Arc::new(crate::backpressure::BackpressureManager::new( backpressure: Arc::new(crate::backpressure::BackpressureManager::from_policy(config.backpressure_policy)),
config.backpressure_buffer_size,
config.high_watermark,
config.low_watermark,
)),
#[cfg(feature = "scheduler")] #[cfg(feature = "scheduler")]
scheduler: Arc::new(crate::scheduler::SchedulerManager::new( scheduler: Arc::new(crate::scheduler::SchedulerManager::from_policy(config.scheduler_policy)),
config.io_buffer_size,
config.max_buffer_size,
config.high_priority_threshold,
config.low_priority_threshold,
)),
config, config,
} }
@@ -244,7 +227,7 @@ impl ConcurrencyManager {
pub async fn start(&self) { pub async fn start(&self) {
#[cfg(feature = "deadlock")] #[cfg(feature = "deadlock")]
{ {
if self.config.enable_deadlock_detection { if self.config.deadlock_policy.enabled {
self.deadlock.start().await; self.deadlock.start().await;
} }
} }
+47 -14
View File
@@ -22,9 +22,9 @@ use rustfs_io_metrics::io_metrics;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
/// Scheduler configuration /// Facade policy for the concurrency-layer scheduler manager.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy)]
pub struct SchedulerConfig { pub struct SchedulerPolicy {
/// Base buffer size /// Base buffer size
pub base_buffer_size: usize, pub base_buffer_size: usize,
/// Maximum buffer size /// Maximum buffer size
@@ -35,7 +35,7 @@ pub struct SchedulerConfig {
pub low_priority_threshold: usize, pub low_priority_threshold: usize,
} }
impl Default for SchedulerConfig { impl Default for SchedulerPolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
base_buffer_size: 64 * 1024, // 64KB base_buffer_size: 64 * 1024, // 64KB
@@ -46,9 +46,23 @@ impl Default for SchedulerConfig {
} }
} }
impl SchedulerPolicy {
/// Convert facade policy to io-core scheduler config.
pub fn to_core_config(&self) -> rustfs_io_core::IoSchedulerConfig {
rustfs_io_core::IoSchedulerConfig {
base_buffer_size: self.base_buffer_size,
max_buffer_size: self.max_buffer_size,
high_priority_size_threshold: self.high_priority_threshold,
low_priority_size_threshold: self.low_priority_threshold,
..rustfs_io_core::IoSchedulerConfig::default()
}
}
}
/// Scheduler manager /// Scheduler manager
pub struct SchedulerManager { pub struct SchedulerManager {
config: SchedulerConfig, config: SchedulerPolicy,
core_config: rustfs_io_core::IoSchedulerConfig,
scheduler: Arc<CoreIoScheduler>, scheduler: Arc<CoreIoScheduler>,
} }
@@ -60,26 +74,35 @@ impl SchedulerManager {
high_priority_threshold: usize, high_priority_threshold: usize,
low_priority_threshold: usize, low_priority_threshold: usize,
) -> Self { ) -> Self {
let config = SchedulerConfig { Self::from_policy(SchedulerPolicy {
base_buffer_size, base_buffer_size,
max_buffer_size, max_buffer_size,
high_priority_threshold, high_priority_threshold,
low_priority_threshold, low_priority_threshold,
}; })
}
let core_config = rustfs_io_core::IoSchedulerConfig::default(); /// Create a scheduler manager from facade policy.
pub fn from_policy(config: SchedulerPolicy) -> Self {
let core_config = config.to_core_config();
Self { Self {
config, config,
core_config: core_config.clone(),
scheduler: Arc::new(CoreIoScheduler::new(core_config)), scheduler: Arc::new(CoreIoScheduler::new(core_config)),
} }
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &SchedulerConfig { pub fn config(&self) -> &SchedulerPolicy {
&self.config &self.config
} }
/// Get the derived io-core scheduler config.
pub fn core_config(&self) -> &rustfs_io_core::IoSchedulerConfig {
&self.core_config
}
/// Get the scheduler /// Get the scheduler
pub fn scheduler(&self) -> Arc<CoreIoScheduler> { pub fn scheduler(&self) -> Arc<CoreIoScheduler> {
self.scheduler.clone() self.scheduler.clone()
@@ -87,7 +110,7 @@ impl SchedulerManager {
/// Create an I/O strategy /// Create an I/O strategy
pub fn create_strategy(&self) -> IoStrategy { pub fn create_strategy(&self) -> IoStrategy {
IoStrategy::new(self.config.clone(), self.scheduler.clone()) IoStrategy::new(self.config, self.scheduler.clone())
} }
/// Calculate buffer size /// Calculate buffer size
@@ -111,12 +134,12 @@ impl SchedulerManager {
/// I/O strategy /// I/O strategy
pub struct IoStrategy { pub struct IoStrategy {
config: SchedulerConfig, config: SchedulerPolicy,
scheduler: Arc<CoreIoScheduler>, scheduler: Arc<CoreIoScheduler>,
} }
impl IoStrategy { impl IoStrategy {
fn new(config: SchedulerConfig, scheduler: Arc<CoreIoScheduler>) -> Self { fn new(config: SchedulerPolicy, scheduler: Arc<CoreIoScheduler>) -> Self {
Self { config, scheduler } Self { config, scheduler }
} }
@@ -191,7 +214,7 @@ impl IoStrategy {
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &SchedulerConfig { pub fn config(&self) -> &SchedulerPolicy {
&self.config &self.config
} }
} }
@@ -202,10 +225,20 @@ mod tests {
#[test] #[test]
fn test_scheduler_config() { fn test_scheduler_config() {
let config = SchedulerConfig::default(); let config = SchedulerPolicy::default();
assert!(config.base_buffer_size < config.max_buffer_size); assert!(config.base_buffer_size < config.max_buffer_size);
} }
#[test]
fn test_scheduler_policy_to_core_config() {
let policy = SchedulerPolicy::default();
let core = policy.to_core_config();
assert_eq!(core.base_buffer_size, policy.base_buffer_size);
assert_eq!(core.max_buffer_size, policy.max_buffer_size);
assert_eq!(core.high_priority_size_threshold, policy.high_priority_threshold);
assert_eq!(core.low_priority_size_threshold, policy.low_priority_threshold);
}
#[test] #[test]
fn test_scheduler_manager() { fn test_scheduler_manager() {
let manager = SchedulerManager::new(1024, 4096, 512, 2048); let manager = SchedulerManager::new(1024, 4096, 512, 2048);
+89 -17
View File
@@ -14,60 +14,101 @@
//! Timeout management for operations //! Timeout management for operations
use rustfs_io_core::{TimeoutError, calculate_adaptive_timeout}; use rustfs_io_core::{TimeoutConfig as CoreTimeoutConfig, TimeoutError, calculate_adaptive_timeout};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
/// Timeout configuration /// Facade policy for the concurrency-layer timeout manager.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy)]
pub struct TimeoutConfig { pub struct TimeoutManagerPolicy {
/// Default timeout duration /// Default timeout duration
pub default_timeout: Duration, pub default_timeout: Duration,
/// Maximum timeout duration /// Maximum timeout duration
pub max_timeout: Duration, pub max_timeout: Duration,
/// Minimum timeout floor (prevents dynamic calculation from going too low).
pub min_timeout: Duration,
/// Enable dynamic timeout calculation /// Enable dynamic timeout calculation
pub enable_dynamic: bool, pub enable_dynamic: bool,
} }
impl Default for TimeoutConfig { impl Default for TimeoutManagerPolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
default_timeout: Duration::from_secs(30), default_timeout: Duration::from_secs(30),
max_timeout: Duration::from_secs(300), max_timeout: Duration::from_secs(300),
min_timeout: Duration::from_secs(5),
enable_dynamic: true, enable_dynamic: true,
} }
} }
} }
impl TimeoutManagerPolicy {
/// Convert the facade policy into the reusable io-core timeout configuration.
///
/// This keeps the concurrency layer explicitly wired to the shared core
/// timeout primitives without changing the facade's public behavior.
pub fn to_core_config(&self) -> CoreTimeoutConfig {
CoreTimeoutConfig {
base_timeout: self.default_timeout,
timeout_per_mb: Duration::ZERO,
max_timeout: self.max_timeout,
min_timeout: self.min_timeout,
get_object_timeout: self.default_timeout,
put_object_timeout: self.max_timeout,
list_objects_timeout: self.default_timeout,
enable_dynamic_timeout: self.enable_dynamic,
}
}
}
/// Timeout manager /// Timeout manager
pub struct TimeoutManager { pub struct TimeoutManager {
config: TimeoutConfig, config: TimeoutManagerPolicy,
core_config: CoreTimeoutConfig,
} }
impl TimeoutManager { impl TimeoutManager {
/// Create a new timeout manager /// Create a new timeout manager
pub fn new(default_timeout: Duration, max_timeout: Duration, enable_dynamic: bool) -> Self { pub fn new(default_timeout: Duration, max_timeout: Duration, enable_dynamic: bool) -> Self {
Self { let min_timeout = default_timeout.min(max_timeout);
config: TimeoutConfig { Self::from_policy(TimeoutManagerPolicy {
default_timeout, default_timeout,
max_timeout, max_timeout,
enable_dynamic, min_timeout,
}, enable_dynamic,
} })
}
/// Create a new timeout manager from the facade policy type.
pub fn from_policy(config: TimeoutManagerPolicy) -> Self {
let config = TimeoutManagerPolicy {
// Guard clamp(min, max) from panic when callers provide an
// out-of-order policy (or very small max_timeout).
min_timeout: config.min_timeout.min(config.max_timeout),
..config
};
let core_config = config.to_core_config();
Self { config, core_config }
} }
/// Get the configuration /// Get the configuration
pub fn config(&self) -> &TimeoutConfig { pub fn config(&self) -> &TimeoutManagerPolicy {
&self.config &self.config
} }
/// Get the derived io-core timeout configuration.
pub fn core_config(&self) -> &CoreTimeoutConfig {
&self.core_config
}
/// Calculate timeout for a given size /// Calculate timeout for a given size
pub fn calculate_timeout(&self, size: u64, _history: &[Duration]) -> Duration { pub fn calculate_timeout(&self, size: u64, _history: &[Duration]) -> Duration {
if !self.config.enable_dynamic { if !self.config.enable_dynamic {
return self.config.default_timeout; return self.config.default_timeout;
} }
calculate_adaptive_timeout(self.config.default_timeout, None, 0, size).min(self.config.max_timeout) calculate_adaptive_timeout(self.core_config.base_timeout, None, 0, size)
.clamp(self.core_config.min_timeout, self.core_config.max_timeout)
} }
/// Wrap an operation with timeout control /// Wrap an operation with timeout control
@@ -87,7 +128,7 @@ impl TimeoutManager {
/// Create a timeout guard for manual timeout control /// Create a timeout guard for manual timeout control
pub fn create_guard(&self, timeout: Option<Duration>) -> TimeoutGuard { pub fn create_guard(&self, timeout: Option<Duration>) -> TimeoutGuard {
TimeoutGuard::new(timeout.unwrap_or(self.config.default_timeout)) TimeoutGuard::new(timeout.unwrap_or(self.core_config.base_timeout))
} }
} }
@@ -134,10 +175,41 @@ mod tests {
#[test] #[test]
fn test_timeout_config() { fn test_timeout_config() {
let config = TimeoutConfig::default(); let config = TimeoutManagerPolicy::default();
assert!(config.default_timeout < config.max_timeout); assert!(config.default_timeout < config.max_timeout);
} }
#[test]
fn test_timeout_policy_to_core_config() {
let policy = TimeoutManagerPolicy::default();
let core = policy.to_core_config();
assert_eq!(core.base_timeout, policy.default_timeout);
assert_eq!(core.max_timeout, policy.max_timeout);
assert_eq!(core.min_timeout, policy.min_timeout);
assert_eq!(core.get_object_timeout, policy.default_timeout);
assert!(core.enable_dynamic_timeout);
}
#[test]
fn test_timeout_manager_new_sanitizes_min_timeout_with_small_max_timeout() {
let manager = TimeoutManager::new(Duration::from_secs(1), Duration::from_secs(1), true);
let timeout = manager.calculate_timeout(1024, &[]);
assert_eq!(timeout, Duration::from_secs(1));
}
#[test]
fn test_timeout_manager_from_policy_sanitizes_min_timeout() {
let manager = TimeoutManager::from_policy(TimeoutManagerPolicy {
default_timeout: Duration::from_secs(30),
max_timeout: Duration::from_secs(1),
min_timeout: Duration::from_secs(5),
enable_dynamic: true,
});
assert_eq!(manager.config().min_timeout, Duration::from_secs(1));
assert_eq!(manager.core_config().min_timeout, Duration::from_secs(1));
}
#[tokio::test] #[tokio::test]
async fn test_wrap_operation_success() { async fn test_wrap_operation_success() {
let manager = TimeoutManager::new(Duration::from_secs(5), Duration::from_secs(10), true); let manager = TimeoutManager::new(Duration::from_secs(5), Duration::from_secs(10), true);
+75 -43
View File
@@ -32,7 +32,7 @@ use crate::storage::options::{
use crate::storage::request_context::spawn_traced; use crate::storage::request_context::spawn_traced;
use crate::storage::s3_api::multipart::parse_list_parts_params; use crate::storage::s3_api::multipart::parse_list_parts_params;
use crate::storage::sse::{SSEType, build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error}; use crate::storage::sse::{SSEType, build_ssec_read_headers, encryption_material_to_metadata, map_get_object_reader_error};
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
use crate::storage::*; use crate::storage::*;
use bytes::Bytes; use bytes::Bytes;
use datafusion::arrow::{ use datafusion::arrow::{
@@ -159,7 +159,7 @@ impl Drop for DeadlockRequestGuard {
} }
struct GetObjectBootstrap { struct GetObjectBootstrap {
timeout_config: TimeoutConfig, timeout_config: GetObjectTimeoutPolicy,
wrapper: RequestTimeoutWrapper, wrapper: RequestTimeoutWrapper,
request_start: std::time::Instant, request_start: std::time::Instant,
request_guard: GetObjectGuard, request_guard: GetObjectGuard,
@@ -217,6 +217,12 @@ struct GetObjectOutputContext {
optimal_buffer_size: usize, optimal_buffer_size: usize,
} }
enum GetObjectTimeoutStage {
BeforeProcessing,
DiskPermitWait { permit_wait_duration: Duration },
BeforeRead,
}
async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) { async fn enqueue_transitioned_delete_cleanup(bucket: &str, object: &str, opts: &ObjectOptions, existing: Option<&ObjectInfo>) {
let Some(existing) = existing else { let Some(existing) = existing else {
return; return;
@@ -443,14 +449,14 @@ fn should_buffer_get_object_in_memory_with_threshold(
#[cfg(test)] #[cfg(test)]
mod deadlock_request_guard_tests { mod deadlock_request_guard_tests {
use super::DeadlockRequestGuard; use super::DeadlockRequestGuard;
use crate::storage::deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig}; use crate::storage::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy};
use std::sync::Arc; use std::sync::Arc;
#[test] #[test]
fn deadlock_request_guard_unregisters_on_drop() { fn deadlock_request_guard_unregisters_on_drop() {
let detector = Arc::new(DeadlockDetector::new(DeadlockDetectorConfig { let detector = Arc::new(DeadlockDetector::new(RequestHangDetectionPolicy {
enabled: true, enabled: true,
..DeadlockDetectorConfig::default() ..RequestHangDetectionPolicy::default()
})); }));
let request_id = "test-request-id".to_string(); let request_id = "test-request-id".to_string();
@@ -1112,7 +1118,7 @@ impl DefaultObjectUsecase {
} }
fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> { fn init_get_object_bootstrap(bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
let timeout_config = TimeoutConfig::from_env(); let timeout_config = GetObjectTimeoutPolicy::from_env();
let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string()); let wrapper = RequestTimeoutWrapper::with_request_id(timeout_config.clone(), request_id.to_string());
let request_start = std::time::Instant::now(); let request_start = std::time::Instant::now();
let request_guard = ConcurrencyManager::track_request(); let request_guard = ConcurrencyManager::track_request();
@@ -1123,16 +1129,7 @@ impl DefaultObjectUsecase {
deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}")); deadlock_detector.register_request(&request_id, format!("GetObject {bucket}/{key}"));
let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id); let deadlock_request_guard = DeadlockRequestGuard::new(deadlock_detector, request_id);
if wrapper.is_timeout() { Self::ensure_get_object_not_timed_out(&wrapper, &timeout_config, bucket, key, GetObjectTimeoutStage::BeforeProcessing)?;
warn!(
bucket = %bucket,
key = %key,
timeout_secs = timeout_config.get_object_timeout.as_secs(),
elapsed_ms = wrapper.elapsed().as_millis(),
"GetObject request timed out before processing"
);
return Err(s3_error!(InternalError, "Request timeout before processing"));
}
rustfs_io_metrics::record_get_object_request_start(concurrent_requests); rustfs_io_metrics::record_get_object_request_start(concurrent_requests);
@@ -1154,7 +1151,7 @@ impl DefaultObjectUsecase {
async fn acquire_get_object_io_planning<'a>( async fn acquire_get_object_io_planning<'a>(
manager: &'a ConcurrencyManager, manager: &'a ConcurrencyManager,
wrapper: &RequestTimeoutWrapper, wrapper: &RequestTimeoutWrapper,
timeout_config: &TimeoutConfig, timeout_config: &GetObjectTimeoutPolicy,
bucket: &str, bucket: &str,
key: &str, key: &str,
) -> S3Result<GetObjectIoPlanning<'a>> { ) -> S3Result<GetObjectIoPlanning<'a>> {
@@ -1165,19 +1162,13 @@ impl DefaultObjectUsecase {
.map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?; .map_err(|_| s3_error!(InternalError, "disk read semaphore closed"))?;
let permit_wait_duration = permit_wait_start.elapsed(); let permit_wait_duration = permit_wait_start.elapsed();
if wrapper.is_timeout() { Self::ensure_get_object_not_timed_out(
warn!( wrapper,
bucket = %bucket, timeout_config,
key = %key, bucket,
wait_ms = permit_wait_duration.as_millis(), key,
timeout_secs = timeout_config.get_object_timeout.as_secs(), GetObjectTimeoutStage::DiskPermitWait { permit_wait_duration },
elapsed_ms = wrapper.elapsed().as_millis(), )?;
"GetObject request timed out while waiting for disk permit"
);
rustfs_io_metrics::record_get_object_timeout(Some("disk_permit"), Some(wrapper.elapsed().as_secs_f64()));
return Err(s3_error!(InternalError, "Request timeout while waiting for disk permit"));
}
let queue_status = manager.io_queue_status(); let queue_status = manager.io_queue_status();
let queue_snapshot = GetObjectQueueSnapshot::from_available_permits( let queue_snapshot = GetObjectQueueSnapshot::from_available_permits(
@@ -1199,17 +1190,7 @@ impl DefaultObjectUsecase {
rustfs_io_metrics::record_io_queue_congestion(); rustfs_io_metrics::record_io_queue_congestion();
} }
if wrapper.is_timeout() { Self::ensure_get_object_not_timed_out(wrapper, timeout_config, bucket, key, GetObjectTimeoutStage::BeforeRead)?;
warn!(
bucket = %bucket,
key = %key,
timeout_secs = timeout_config.get_object_timeout.as_secs(),
elapsed_ms = wrapper.elapsed().as_millis(),
"GetObject request timed out before reading object"
);
rustfs_io_metrics::record_get_object_timeout(Some("before_read"), Some(wrapper.elapsed().as_secs_f64()));
return Err(s3_error!(InternalError, "Request timeout before reading object"));
}
Ok(GetObjectIoPlanning { Ok(GetObjectIoPlanning {
_disk_permit: disk_permit, _disk_permit: disk_permit,
@@ -1274,7 +1255,7 @@ impl DefaultObjectUsecase {
req: &S3Request<GetObjectInput>, req: &S3Request<GetObjectInput>,
manager: &'a ConcurrencyManager, manager: &'a ConcurrencyManager,
wrapper: &RequestTimeoutWrapper, wrapper: &RequestTimeoutWrapper,
timeout_config: &TimeoutConfig, timeout_config: &GetObjectTimeoutPolicy,
bucket: &str, bucket: &str,
key: &str, key: &str,
rs: Option<HTTPRangeSpec>, rs: Option<HTTPRangeSpec>,
@@ -2024,7 +2005,7 @@ impl DefaultObjectUsecase {
fn finalize_get_object_completion( fn finalize_get_object_completion(
wrapper: &RequestTimeoutWrapper, wrapper: &RequestTimeoutWrapper,
timeout_config: &TimeoutConfig, timeout_config: &GetObjectTimeoutPolicy,
total_duration: Duration, total_duration: Duration,
response_content_length: i64, response_content_length: i64,
optimal_buffer_size: usize, optimal_buffer_size: usize,
@@ -2052,6 +2033,57 @@ impl DefaultObjectUsecase {
); );
} }
fn ensure_get_object_not_timed_out(
wrapper: &RequestTimeoutWrapper,
timeout_config: &GetObjectTimeoutPolicy,
bucket: &str,
key: &str,
stage: GetObjectTimeoutStage,
) -> S3Result<()> {
if !wrapper.is_timeout() {
return Ok(());
}
let timeout_secs = timeout_config.get_object_timeout.as_secs();
let elapsed_ms = wrapper.elapsed().as_millis();
match stage {
GetObjectTimeoutStage::BeforeProcessing => {
warn!(
bucket = %bucket,
key = %key,
timeout_secs,
elapsed_ms,
"GetObject request timed out before processing"
);
Err(s3_error!(InternalError, "Request timeout before processing"))
}
GetObjectTimeoutStage::DiskPermitWait { permit_wait_duration } => {
warn!(
bucket = %bucket,
key = %key,
wait_ms = permit_wait_duration.as_millis(),
timeout_secs,
elapsed_ms,
"GetObject request timed out while waiting for disk permit"
);
rustfs_io_metrics::record_get_object_timeout(Some("disk_permit"), Some(wrapper.elapsed().as_secs_f64()));
Err(s3_error!(InternalError, "Request timeout while waiting for disk permit"))
}
GetObjectTimeoutStage::BeforeRead => {
warn!(
bucket = %bucket,
key = %key,
timeout_secs,
elapsed_ms,
"GetObject request timed out before reading object"
);
rustfs_io_metrics::record_get_object_timeout(Some("before_read"), Some(wrapper.elapsed().as_secs_f64()));
Err(s3_error!(InternalError, "Request timeout before reading object"))
}
}
}
async fn finalize_get_object_response( async fn finalize_get_object_response(
helper: OperationHelper, helper: OperationHelper,
bucket: &str, bucket: &str,
+201 -140
View File
@@ -17,9 +17,6 @@
//! This module provides backpressure-aware pipes for object data transfer, //! This module provides backpressure-aware pipes for object data transfer,
//! preventing buffer overflow and memory exhaustion under high concurrency. //! preventing buffer overflow and memory exhaustion under high concurrency.
// Allow dead_code for public API that may be used by external modules or future features
#![allow(dead_code)]
//!
//! # Key Features //! # Key Features
//! //!
//! - Configurable buffer size with high/low watermarks //! - Configurable buffer size with high/low watermarks
@@ -39,17 +36,16 @@
//! [High Watermark?] --> Apply Backpressure //! [High Watermark?] --> Apply Backpressure
//! ``` //! ```
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Instant; use std::time::{Duration, Instant};
use tokio::io::{DuplexStream, duplex}; use tokio::io::{DuplexStream, duplex};
use tracing::{debug, warn}; use tracing::{debug, warn};
use metrics::counter; use metrics::counter;
/// Backpressure pipe configuration. /// Object-transfer duplex pipe backpressure policy.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy)]
pub struct BackpressureConfig { pub struct ObjectPipeBackpressurePolicy {
/// Buffer size in bytes (default 4MB). /// Buffer size in bytes (default 4MB).
pub buffer_size: usize, pub buffer_size: usize,
/// High watermark percentage (default 80%). /// High watermark percentage (default 80%).
@@ -60,7 +56,7 @@ pub struct BackpressureConfig {
pub low_watermark: u32, pub low_watermark: u32,
} }
impl Default for BackpressureConfig { impl Default for ObjectPipeBackpressurePolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
buffer_size: rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE, buffer_size: rustfs_config::DEFAULT_OBJECT_DUPLEX_BUFFER_SIZE,
@@ -70,7 +66,7 @@ impl Default for BackpressureConfig {
} }
} }
impl BackpressureConfig { impl ObjectPipeBackpressurePolicy {
/// Load configuration from environment variables. /// Load configuration from environment variables.
pub fn from_env() -> Self { pub fn from_env() -> Self {
let buffer_size = rustfs_utils::get_env_usize( let buffer_size = rustfs_utils::get_env_usize(
@@ -125,45 +121,61 @@ impl std::fmt::Display for BackpressureState {
} }
} }
/// Backpressure event for monitoring. /// Compact metadata snapshot for object-transfer backpressure pipes.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BackpressureEvent { pub struct BackpressurePipeMeta {
/// Event timestamp.
pub timestamp: Instant,
/// Event type.
pub event_type: BackpressureEventType,
/// Buffer usage at event time.
pub buffer_usage: usize,
/// Buffer capacity.
pub buffer_capacity: usize,
/// Usage percentage.
pub usage_percent: f32,
}
/// Backpressure event type.
#[derive(Debug, Clone, Copy)]
pub enum BackpressureEventType {
/// Entered high watermark state.
HighWatermarkReached,
/// Exited high watermark state (backpressure released).
HighWatermarkExited,
/// Backpressure applied to producer.
BackpressureApplied,
/// Backpressure released.
BackpressureReleased,
}
/// Snapshot of backpressure state.
#[derive(Debug, Clone)]
pub struct BackpressureSnapshot {
/// Buffer capacity in bytes. /// Buffer capacity in bytes.
pub buffer_capacity: usize, pub buffer_capacity: usize,
/// Current buffer usage in bytes (approximate). /// Current backpressure state.
pub buffer_used: usize,
/// Usage percentage.
pub usage_percent: f32,
/// Current state.
pub state: BackpressureState, pub state: BackpressureState,
/// Age of the pipe since creation.
pub age: Duration,
}
/// Compact metadata snapshot for the lightweight backpressure monitor.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct BackpressureMonitorMeta {
/// Buffer capacity in bytes.
pub buffer_capacity: usize,
/// Current buffer usage percentage.
pub usage_percent: f32,
/// Current backpressure state.
pub state: BackpressureState,
}
fn calculate_usage_percent(usage: usize, capacity: usize) -> f32 {
if capacity > 0 {
(usage as f32 / capacity as f32) * 100.0
} else {
0.0
}
}
fn apply_watermark_transition(
in_high_watermark: &AtomicBool,
usage: usize,
high: usize,
low: usize,
) -> (BackpressureState, bool) {
let current = in_high_watermark.load(Ordering::Acquire);
let next_state = if usage >= high {
BackpressureState::HighWatermark
} else if usage <= low {
BackpressureState::Normal
} else if current {
BackpressureState::HighWatermark
} else {
BackpressureState::Normal
};
let next_is_high = matches!(next_state, BackpressureState::HighWatermark);
let changed = in_high_watermark.swap(next_is_high, Ordering::AcqRel) != next_is_high;
(next_state, changed)
}
fn saturating_sub_atomic(value: &AtomicUsize, delta: usize) {
value
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_sub(delta)))
.ok();
} }
/// A backpressure-aware pipe wrapping tokio's duplex. /// A backpressure-aware pipe wrapping tokio's duplex.
@@ -176,33 +188,41 @@ pub struct BackpressurePipe {
/// Writer end of the duplex pipe. /// Writer end of the duplex pipe.
writer: DuplexStream, writer: DuplexStream,
/// Configuration. /// Configuration.
config: BackpressureConfig, config: ObjectPipeBackpressurePolicy,
/// Current buffer usage (approximate, updated on write). /// Current buffer usage (approximate, updated on write).
buffer_usage: Arc<AtomicUsize>, buffer_usage: AtomicUsize,
/// Current backpressure state. /// Current backpressure state.
state: Arc<AtomicBool>, // true = in high watermark state state: AtomicBool, // true = in high watermark state
/// Total bytes written. /// Total bytes written.
total_written: Arc<AtomicUsize>, total_written: AtomicUsize,
/// Total bytes read. /// Total bytes read.
total_read: Arc<AtomicUsize>, total_read: AtomicUsize,
/// Cached high watermark threshold in bytes.
high_watermark_bytes: usize,
/// Cached low watermark threshold in bytes.
low_watermark_bytes: usize,
/// Pipe creation timestamp.
created_at: Instant,
} }
impl BackpressurePipe { impl BackpressurePipe {
/// Create a new backpressure-aware pipe with default configuration. /// Create a new backpressure-aware pipe with default configuration.
pub fn new() -> Self { pub fn new() -> Self {
Self::with_config(BackpressureConfig::from_env()) Self::with_config(ObjectPipeBackpressurePolicy::from_env())
} }
/// Create a new backpressure-aware pipe with custom configuration. /// Create a new backpressure-aware pipe with custom configuration.
pub fn with_config(config: BackpressureConfig) -> Self { pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self {
let (reader, writer) = duplex(config.buffer_size); let (reader, writer) = duplex(config.buffer_size);
let high_watermark_bytes = config.high_watermark_bytes();
let low_watermark_bytes = config.low_watermark_bytes();
debug!( debug!(
buffer_size = config.buffer_size, buffer_size = config.buffer_size,
high_watermark = config.high_watermark, high_watermark = config.high_watermark,
low_watermark = config.low_watermark, low_watermark = config.low_watermark,
high_watermark_bytes = config.high_watermark_bytes(), high_watermark_bytes,
low_watermark_bytes = config.low_watermark_bytes(), low_watermark_bytes,
"Created backpressure pipe" "Created backpressure pipe"
); );
@@ -210,10 +230,13 @@ impl BackpressurePipe {
reader, reader,
writer, writer,
config, config,
buffer_usage: Arc::new(AtomicUsize::new(0)), buffer_usage: AtomicUsize::new(0),
state: Arc::new(AtomicBool::new(false)), state: AtomicBool::new(false),
total_written: Arc::new(AtomicUsize::new(0)), total_written: AtomicUsize::new(0),
total_read: Arc::new(AtomicUsize::new(0)), total_read: AtomicUsize::new(0),
high_watermark_bytes,
low_watermark_bytes,
created_at: Instant::now(),
} }
} }
@@ -234,81 +257,79 @@ impl BackpressurePipe {
/// Get current backpressure state. /// Get current backpressure state.
pub fn state(&self) -> BackpressureState { pub fn state(&self) -> BackpressureState {
if self.state.load(Ordering::Relaxed) { if self.state.load(Ordering::Acquire) {
BackpressureState::BackpressureApplied BackpressureState::BackpressureApplied
} else { } else {
BackpressureState::Normal BackpressureState::Normal
} }
} }
/// Get current buffer usage snapshot. /// Get a compact metadata snapshot for the pipe.
pub fn snapshot(&self) -> BackpressureSnapshot { pub fn meta(&self) -> BackpressurePipeMeta {
let buffer_used = self.buffer_usage.load(Ordering::Relaxed); BackpressurePipeMeta {
let usage_percent = if self.config.buffer_size > 0 {
(buffer_used as f64 / self.config.buffer_size as f64) * 100.0
} else {
0.0
};
BackpressureSnapshot {
buffer_capacity: self.config.buffer_size, buffer_capacity: self.config.buffer_size,
buffer_used,
usage_percent: usage_percent as f32,
state: self.state(), state: self.state(),
age: self.age(),
} }
} }
/// Get the age of this pipe.
pub fn age(&self) -> Duration {
self.created_at.elapsed()
}
/// Get current buffer usage.
pub fn usage(&self) -> usize {
self.buffer_usage.load(Ordering::Acquire)
}
/// Record bytes written (call after successful write). /// Record bytes written (call after successful write).
pub fn record_write(&self, bytes: usize) { pub fn record_write(&self, bytes: usize) {
self.total_written.fetch_add(bytes, Ordering::Relaxed); self.total_written.fetch_add(bytes, Ordering::Relaxed);
self.buffer_usage.fetch_add(bytes, Ordering::Relaxed); self.buffer_usage.fetch_add(bytes, Ordering::Release);
self.check_high_watermark(); self.update_watermark_state();
} }
/// Record bytes read (call after successful read). /// Record bytes read (call after successful read).
pub fn record_read(&self, bytes: usize) { pub fn record_read(&self, bytes: usize) {
self.total_read.fetch_add(bytes, Ordering::Relaxed); self.total_read.fetch_add(bytes, Ordering::Relaxed);
self.buffer_usage.fetch_sub(bytes, Ordering::Relaxed); saturating_sub_atomic(&self.buffer_usage, bytes);
self.check_low_watermark(); self.update_watermark_state();
} }
/// Check if high watermark is reached. /// Update watermark state and emit transition signals.
fn check_high_watermark(&self) { fn update_watermark_state(&self) {
let usage = self.buffer_usage.load(Ordering::Relaxed); let usage = self.buffer_usage.load(Ordering::Acquire);
let threshold = self.config.high_watermark_bytes(); let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32;
let (next_state, changed) =
apply_watermark_transition(&self.state, usage, self.high_watermark_bytes, self.low_watermark_bytes);
if usage >= threshold && !self.state.load(Ordering::Relaxed) { if changed {
self.state.store(true, Ordering::Relaxed); match next_state {
BackpressureState::HighWatermark => {
counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1);
counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); warn!(
buffer_usage = usage,
buffer_capacity = self.config.buffer_size,
usage_percent,
high_watermark = self.config.high_watermark,
"Backpressure: high watermark reached"
);
}
BackpressureState::Normal => {
counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1);
warn!( debug!(
buffer_usage = usage, buffer_usage = usage,
buffer_capacity = self.config.buffer_size, buffer_capacity = self.config.buffer_size,
usage_percent = (usage as f64 / self.config.buffer_size as f64 * 100.0) as u32, usage_percent,
high_watermark = self.config.high_watermark, low_watermark = self.config.low_watermark,
"Backpressure: high watermark reached" "Backpressure: returned to normal"
); );
} }
} BackpressureState::BackpressureApplied => {}
}
/// Check if low watermark is reached (backpressure can be released).
fn check_low_watermark(&self) {
let usage = self.buffer_usage.load(Ordering::Relaxed);
let threshold = self.config.low_watermark_bytes();
if usage <= threshold && self.state.load(Ordering::Relaxed) {
self.state.store(false, Ordering::Relaxed);
counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1);
debug!(
buffer_usage = usage,
buffer_capacity = self.config.buffer_size,
usage_percent = (usage as f64 / self.config.buffer_size as f64 * 100.0) as u32,
low_watermark = self.config.low_watermark,
"Backpressure: returned to normal"
);
} }
} }
@@ -340,43 +361,51 @@ impl Default for BackpressurePipe {
/// wrap the streams but provides monitoring capabilities. /// wrap the streams but provides monitoring capabilities.
pub struct BackpressureMonitor { pub struct BackpressureMonitor {
/// Configuration. /// Configuration.
config: BackpressureConfig, config: ObjectPipeBackpressurePolicy,
/// Current buffer usage. /// Current buffer usage.
buffer_usage: Arc<AtomicUsize>, buffer_usage: AtomicUsize,
/// In high watermark state. /// In high watermark state.
in_high_watermark: Arc<AtomicBool>, in_high_watermark: AtomicBool,
/// Cached high watermark threshold in bytes.
high_watermark_bytes: usize,
/// Cached low watermark threshold in bytes.
low_watermark_bytes: usize,
} }
impl BackpressureMonitor { impl BackpressureMonitor {
/// Create a new monitor with default configuration. /// Create a new monitor with default configuration.
pub fn new() -> Self { pub fn new() -> Self {
Self::with_config(BackpressureConfig::from_env()) Self::with_config(ObjectPipeBackpressurePolicy::from_env())
} }
/// Create a new monitor with custom configuration. /// Create a new monitor with custom configuration.
pub fn with_config(config: BackpressureConfig) -> Self { pub fn with_config(config: ObjectPipeBackpressurePolicy) -> Self {
let high_watermark_bytes = config.high_watermark_bytes();
let low_watermark_bytes = config.low_watermark_bytes();
Self { Self {
config, config,
buffer_usage: Arc::new(AtomicUsize::new(0)), buffer_usage: AtomicUsize::new(0),
in_high_watermark: Arc::new(AtomicBool::new(false)), in_high_watermark: AtomicBool::new(false),
high_watermark_bytes,
low_watermark_bytes,
} }
} }
/// Record bytes added to buffer. /// Record bytes added to buffer.
pub fn on_write(&self, bytes: usize) -> BackpressureState { pub fn on_write(&self, bytes: usize) -> BackpressureState {
self.buffer_usage.fetch_add(bytes, Ordering::Relaxed); self.buffer_usage.fetch_add(bytes, Ordering::Release);
self.update_state() self.update_state()
} }
/// Record bytes removed from buffer. /// Record bytes removed from buffer.
pub fn on_read(&self, bytes: usize) -> BackpressureState { pub fn on_read(&self, bytes: usize) -> BackpressureState {
self.buffer_usage.fetch_sub(bytes, Ordering::Relaxed); saturating_sub_atomic(&self.buffer_usage, bytes);
self.update_state() self.update_state()
} }
/// Get current state. /// Get current state.
pub fn state(&self) -> BackpressureState { pub fn state(&self) -> BackpressureState {
if self.in_high_watermark.load(Ordering::Relaxed) { if self.in_high_watermark.load(Ordering::Acquire) {
BackpressureState::HighWatermark BackpressureState::HighWatermark
} else { } else {
BackpressureState::Normal BackpressureState::Normal
@@ -385,41 +414,46 @@ impl BackpressureMonitor {
/// Get current buffer usage. /// Get current buffer usage.
pub fn usage(&self) -> usize { pub fn usage(&self) -> usize {
self.buffer_usage.load(Ordering::Relaxed) self.buffer_usage.load(Ordering::Acquire)
} }
/// Get usage percentage. /// Get usage percentage.
pub fn usage_percent(&self) -> f32 { pub fn usage_percent(&self) -> f32 {
let usage = self.buffer_usage.load(Ordering::Relaxed); let usage = self.buffer_usage.load(Ordering::Acquire);
if self.config.buffer_size > 0 { calculate_usage_percent(usage, self.config.buffer_size)
(usage as f32 / self.config.buffer_size as f32) * 100.0 }
} else {
0.0 /// Get a compact metadata snapshot for the monitor.
pub fn meta(&self) -> BackpressureMonitorMeta {
let usage = self.buffer_usage.load(Ordering::Acquire);
BackpressureMonitorMeta {
buffer_capacity: self.config.buffer_size,
usage_percent: calculate_usage_percent(usage, self.config.buffer_size),
state: self.state(),
} }
} }
/// Update state based on current usage. /// Update state based on current usage.
fn update_state(&self) -> BackpressureState { fn update_state(&self) -> BackpressureState {
let usage = self.buffer_usage.load(Ordering::Relaxed); let usage = self.buffer_usage.load(Ordering::Acquire);
let high = self.config.high_watermark_bytes(); let usage_percent = calculate_usage_percent(usage, self.config.buffer_size) as u32;
let low = self.config.low_watermark_bytes(); let (next_state, changed) =
apply_watermark_transition(&self.in_high_watermark, usage, self.high_watermark_bytes, self.low_watermark_bytes);
if usage >= high { if matches!(next_state, BackpressureState::HighWatermark) {
if !self.in_high_watermark.swap(true, Ordering::Relaxed) { if changed {
counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1); counter!("rustfs_backpressure_events_total", "state" => "high_watermark").increment(1);
debug!(usage_percent = self.usage_percent() as u32, "Backpressure: entered high watermark"); debug!(usage_percent, "Backpressure: entered high watermark");
} }
BackpressureState::HighWatermark BackpressureState::HighWatermark
} else if usage <= low { } else {
if self.in_high_watermark.swap(false, Ordering::Relaxed) { if changed {
counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1); counter!("rustfs_backpressure_events_total", "state" => "normal").increment(1);
debug!(usage_percent = self.usage_percent() as u32, "Backpressure: returned to normal"); debug!(usage_percent, "Backpressure: returned to normal");
} }
BackpressureState::Normal BackpressureState::Normal
} else {
self.state()
} }
} }
} }
@@ -436,7 +470,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_config_default() { fn test_backpressure_config_default() {
let config = BackpressureConfig::default(); let config = ObjectPipeBackpressurePolicy::default();
assert_eq!(config.buffer_size, 4 * 1024 * 1024); assert_eq!(config.buffer_size, 4 * 1024 * 1024);
assert_eq!(config.high_watermark, 80); assert_eq!(config.high_watermark, 80);
assert_eq!(config.low_watermark, 50); assert_eq!(config.low_watermark, 50);
@@ -444,7 +478,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_config_watermarks() { fn test_backpressure_config_watermarks() {
let config = BackpressureConfig { let config = ObjectPipeBackpressurePolicy {
buffer_size: 1000, buffer_size: 1000,
high_watermark: 80, high_watermark: 80,
low_watermark: 50, low_watermark: 50,
@@ -462,7 +496,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_monitor() { fn test_backpressure_monitor() {
let config = BackpressureConfig { let config = ObjectPipeBackpressurePolicy {
buffer_size: 1000, buffer_size: 1000,
high_watermark: 80, high_watermark: 80,
low_watermark: 50, low_watermark: 50,
@@ -471,14 +505,18 @@ mod tests {
// Initially normal // Initially normal
assert_eq!(monitor.state(), BackpressureState::Normal); assert_eq!(monitor.state(), BackpressureState::Normal);
assert_eq!(monitor.meta().buffer_capacity, 1000);
assert_eq!(monitor.meta().usage_percent, 0.0);
// Write to reach high watermark // Write to reach high watermark
let state = monitor.on_write(850); let state = monitor.on_write(850);
assert_eq!(state, BackpressureState::HighWatermark); assert_eq!(state, BackpressureState::HighWatermark);
assert_eq!(monitor.meta().usage_percent, 85.0);
// Read to go below low watermark // Read to go below low watermark
let state = monitor.on_read(400); let state = monitor.on_read(400);
assert_eq!(state, BackpressureState::Normal); assert_eq!(state, BackpressureState::Normal);
assert_eq!(monitor.meta().usage_percent, 45.0);
} }
#[tokio::test] #[tokio::test]
@@ -486,5 +524,28 @@ mod tests {
let pipe = BackpressurePipe::new(); let pipe = BackpressurePipe::new();
assert_eq!(pipe.capacity(), 4 * 1024 * 1024); assert_eq!(pipe.capacity(), 4 * 1024 * 1024);
assert_eq!(pipe.state(), BackpressureState::Normal); assert_eq!(pipe.state(), BackpressureState::Normal);
assert_eq!(pipe.meta().buffer_capacity, 4 * 1024 * 1024);
assert!(pipe.meta().age <= pipe.age());
}
#[test]
fn test_backpressure_pipe_state_transitions() {
let config = ObjectPipeBackpressurePolicy {
buffer_size: 1000,
high_watermark: 80,
low_watermark: 50,
};
let pipe = BackpressurePipe::with_config(config);
assert_eq!(pipe.state(), BackpressureState::Normal);
assert_eq!(pipe.meta().state, BackpressureState::Normal);
pipe.record_write(850);
assert_eq!(pipe.state(), BackpressureState::BackpressureApplied);
assert_eq!(pipe.meta().state, BackpressureState::BackpressureApplied);
pipe.record_read(400);
assert_eq!(pipe.state(), BackpressureState::Normal);
assert_eq!(pipe.meta().state, BackpressureState::Normal);
} }
} }
+161 -47
View File
@@ -32,6 +32,7 @@
use rustfs_config::{KI_B, MI_B}; use rustfs_config::{KI_B, MI_B};
use rustfs_io_core::io_profile::{AccessPattern, StorageMedia, StorageProfile}; use rustfs_io_core::io_profile::{AccessPattern, StorageMedia, StorageProfile};
use rustfs_io_core::{IoPriorityQueueConfig as CoreIoPriorityQueueConfig, IoSchedulerConfig as CoreIoSchedulerConfig};
use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier}; use rustfs_io_metrics::bandwidth::{BandwidthSnapshot, BandwidthTier};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::Duration; use std::time::Duration;
@@ -129,8 +130,8 @@ impl IoPriority {
pub fn from_size(size: i64) -> Self { pub fn from_size(size: i64) -> Self {
Self::from_size_with_thresholds( Self::from_size_with_thresholds(
size, size,
IoSchedulerConfig::default().high_priority_size_threshold, rustfs_config::DEFAULT_OBJECT_IO_HIGH_PRIORITY_SIZE_THRESHOLD,
IoSchedulerConfig::default().low_priority_size_threshold, rustfs_config::DEFAULT_OBJECT_IO_LOW_PRIORITY_SIZE_THRESHOLD,
) )
} }
@@ -380,6 +381,32 @@ impl IoSchedulerConfig {
), ),
} }
} }
/// Convert storage-layer scheduler config to io-core scheduler config.
///
/// This keeps storage-specific policy loading in place while allowing
/// core scheduling components to consume a normalized shared config shape.
pub fn to_core_config(&self) -> CoreIoSchedulerConfig {
CoreIoSchedulerConfig {
max_concurrent_reads: self.max_concurrent_reads,
high_priority_size_threshold: self.high_priority_size_threshold,
low_priority_size_threshold: self.low_priority_size_threshold,
queue_high_capacity: self.queue_high_capacity,
queue_normal_capacity: self.queue_normal_capacity,
queue_low_capacity: self.queue_low_capacity,
starvation_prevention_interval_ms: self.starvation_prevention_interval_ms,
starvation_threshold_secs: self.starvation_threshold_secs,
load_sample_window: self.load_sample_window,
load_high_threshold_ms: self.load_high_threshold_ms,
load_low_threshold_ms: self.load_low_threshold_ms,
enable_priority: self.enable_priority,
storage_detection_enabled: self.storage_detection_enabled,
base_buffer_size: rustfs_config::DEFAULT_OBJECT_IO_BUFFER_SIZE,
max_buffer_size: MI_B,
min_buffer_size: 32 * KI_B,
..Default::default()
}
}
} }
/// I/O queue status for monitoring. /// I/O queue status for monitoring.
@@ -1271,27 +1298,38 @@ impl IoLoadMetrics {
self.observation_count.load(Ordering::Relaxed) self.observation_count.load(Ordering::Relaxed)
} }
} }
pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize { #[derive(Debug, Clone, Copy)]
let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); struct ConcurrencyThresholds {
medium: usize,
high: usize,
}
// Record concurrent request metrics fn load_concurrency_thresholds() -> ConcurrencyThresholds {
{ ConcurrencyThresholds {
use metrics::gauge; medium: rustfs_utils::get_env_usize(
gauge!("rustfs_concurrent_get_requests").set(concurrent_requests as f64); rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD,
rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD,
),
high: rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
),
} }
}
fn compute_concurrency_aware_buffer_size(
file_size: i64,
base_buffer_size: usize,
concurrent_requests: usize,
thresholds: ConcurrencyThresholds,
) -> usize {
let medium_threshold = thresholds.medium;
let high_threshold = thresholds.high;
// For low concurrency, use the base buffer size for maximum throughput // For low concurrency, use the base buffer size for maximum throughput
if concurrent_requests <= 1 { if concurrent_requests <= 1 {
return base_buffer_size; return base_buffer_size;
} }
let medium_threshold = rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD,
rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD,
);
let high_threshold = rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
);
// Calculate adaptive multiplier based on concurrency level // Calculate adaptive multiplier based on concurrency level
let adaptive_multiplier = if concurrent_requests <= 2 { let adaptive_multiplier = if concurrent_requests <= 2 {
@@ -1327,6 +1365,18 @@ pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize
adjusted_size.clamp(min_buffer, max_buffer) adjusted_size.clamp(min_buffer, max_buffer)
} }
pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize) -> usize {
let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed);
// Record concurrent request metrics
{
use metrics::gauge;
gauge!("rustfs_concurrent_get_requests").set(concurrent_requests as f64);
}
compute_concurrency_aware_buffer_size(file_size, base_buffer_size, concurrent_requests, load_concurrency_thresholds())
}
/// Advanced concurrency-aware buffer sizing with file size optimization /// Advanced concurrency-aware buffer sizing with file size optimization
/// ///
/// This enhanced version considers both concurrency level and file size patterns /// This enhanced version considers both concurrency level and file size patterns
@@ -1353,6 +1403,7 @@ pub fn get_concurrency_aware_buffer_size(file_size: i64, base_buffer_size: usize
/// ``` /// ```
pub fn get_advanced_buffer_size(file_size: i64, base_buffer_size: usize, is_sequential: bool) -> usize { pub fn get_advanced_buffer_size(file_size: i64, base_buffer_size: usize, is_sequential: bool) -> usize {
let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed); let concurrent_requests = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed);
let thresholds = load_concurrency_thresholds();
// For very small files, use smaller buffers regardless of concurrency // For very small files, use smaller buffers regardless of concurrency
// Replace manual max/min chain with clamp // Replace manual max/min chain with clamp
@@ -1361,15 +1412,10 @@ pub fn get_advanced_buffer_size(file_size: i64, base_buffer_size: usize, is_sequ
} }
// Base calculation from standard function // Base calculation from standard function
let standard_size = get_concurrency_aware_buffer_size(file_size, base_buffer_size); let standard_size = compute_concurrency_aware_buffer_size(file_size, base_buffer_size, concurrent_requests, thresholds);
let medium_threshold = rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, let medium_threshold = thresholds.medium;
rustfs_config::DEFAULT_OBJECT_MEDIUM_CONCURRENCY_THRESHOLD, let high_threshold = thresholds.high;
);
let high_threshold = rustfs_utils::get_env_usize(
rustfs_config::ENV_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
rustfs_config::DEFAULT_OBJECT_HIGH_CONCURRENCY_THRESHOLD,
);
// For sequential reads, we can be more aggressive with buffer sizes // For sequential reads, we can be more aggressive with buffer sizes
if is_sequential && concurrent_requests <= medium_threshold { if is_sequential && concurrent_requests <= medium_threshold {
return ((standard_size as f64 * 1.5) as usize).min(2 * MI_B); return ((standard_size as f64 * 1.5) as usize).min(2 * MI_B);
@@ -1523,6 +1569,28 @@ impl IoPriorityQueueConfig {
), ),
} }
} }
/// Convert storage-layer queue config to io-core queue config.
pub fn to_core_config(&self) -> CoreIoPriorityQueueConfig {
CoreIoPriorityQueueConfig {
high_capacity: self.queue_high_capacity,
normal_capacity: self.queue_normal_capacity,
low_capacity: self.queue_low_capacity,
starvation_interval: Duration::from_millis(self.starvation_prevention_interval_ms),
starvation_threshold: Duration::from_secs(self.starvation_threshold_secs),
}
}
/// Build queue config directly from a scheduler config.
pub fn from_scheduler_config(config: &IoSchedulerConfig) -> Self {
Self {
queue_high_capacity: config.queue_high_capacity,
queue_normal_capacity: config.queue_normal_capacity,
queue_low_capacity: config.queue_low_capacity,
starvation_prevention_interval_ms: config.starvation_prevention_interval_ms,
starvation_threshold_secs: config.starvation_threshold_secs,
}
}
} }
impl<T> IoPriorityQueue<T> { impl<T> IoPriorityQueue<T> {
@@ -1848,9 +1916,8 @@ pub fn get_buffer_size_opt_in(file_size: i64) -> usize {
mod tests { mod tests {
use super::*; use super::*;
use serial_test::serial; use serial_test::serial;
use tokio::test;
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_priority_queue_basic() { async fn test_io_priority_queue_basic() {
let config = IoPriorityQueueConfig::default(); let config = IoPriorityQueueConfig::default();
@@ -1869,7 +1936,7 @@ mod tests {
assert_eq!(queue.len().await, 3); assert_eq!(queue.len().await, 3);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_priority_queue_dequeue_order() { async fn test_io_priority_queue_dequeue_order() {
let config = IoPriorityQueueConfig::default(); let config = IoPriorityQueueConfig::default();
@@ -1897,7 +1964,7 @@ mod tests {
assert!(queue.is_empty().await); assert!(queue.is_empty().await);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_priority_queue_status() { async fn test_io_priority_queue_status() {
let config = IoPriorityQueueConfig::default(); let config = IoPriorityQueueConfig::default();
@@ -1915,7 +1982,7 @@ mod tests {
assert_eq!(status.low_priority_waiting, 1); assert_eq!(status.low_priority_waiting, 1);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_priority_queue_starvation_prevention() { async fn test_io_priority_queue_starvation_prevention() {
let config = IoPriorityQueueConfig { let config = IoPriorityQueueConfig {
@@ -1939,7 +2006,7 @@ mod tests {
assert_eq!(priority, IoPriority::Normal); assert_eq!(priority, IoPriority::Normal);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_priority_from_size() { async fn test_io_priority_from_size() {
// High priority: < 1MB // High priority: < 1MB
@@ -1955,7 +2022,7 @@ mod tests {
assert_eq!(IoPriority::from_size(100 * 1024 * 1024), IoPriority::Low); assert_eq!(IoPriority::from_size(100 * 1024 * 1024), IoPriority::Low);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_load_level_from_wait_duration() { async fn test_io_load_level_from_wait_duration() {
use std::time::Duration; use std::time::Duration;
@@ -1973,7 +2040,7 @@ mod tests {
assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(300)), IoLoadLevel::Critical); assert_eq!(IoLoadLevel::from_wait_duration(Duration::from_millis(300)), IoLoadLevel::Critical);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_io_scheduler_config_default() { async fn test_io_scheduler_config_default() {
let config = IoSchedulerConfig::default(); let config = IoSchedulerConfig::default();
@@ -1987,7 +2054,54 @@ mod tests {
assert_eq!(config.starvation_threshold_secs, 5); assert_eq!(config.starvation_threshold_secs, 5);
} }
#[test] #[tokio::test]
#[serial]
async fn test_io_scheduler_config_to_core_config() {
let config = IoSchedulerConfig::default();
let core = config.to_core_config();
assert_eq!(core.max_concurrent_reads, config.max_concurrent_reads);
assert_eq!(core.high_priority_size_threshold, config.high_priority_size_threshold);
assert_eq!(core.low_priority_size_threshold, config.low_priority_size_threshold);
assert_eq!(core.queue_high_capacity, config.queue_high_capacity);
assert_eq!(core.queue_normal_capacity, config.queue_normal_capacity);
assert_eq!(core.queue_low_capacity, config.queue_low_capacity);
assert_eq!(core.load_high_threshold_ms, config.load_high_threshold_ms);
assert_eq!(core.load_low_threshold_ms, config.load_low_threshold_ms);
assert_eq!(core.enable_priority, config.enable_priority);
}
#[tokio::test]
#[serial]
async fn test_io_priority_queue_config_to_core_config() {
let config = IoPriorityQueueConfig::default();
let core = config.to_core_config();
assert_eq!(core.high_capacity, config.queue_high_capacity);
assert_eq!(core.normal_capacity, config.queue_normal_capacity);
assert_eq!(core.low_capacity, config.queue_low_capacity);
assert_eq!(core.starvation_interval, Duration::from_millis(config.starvation_prevention_interval_ms));
assert_eq!(core.starvation_threshold, Duration::from_secs(config.starvation_threshold_secs));
}
#[tokio::test]
#[serial]
async fn test_io_priority_queue_config_from_scheduler_config() {
let scheduler_config = IoSchedulerConfig {
queue_high_capacity: 128,
queue_normal_capacity: 256,
queue_low_capacity: 512,
starvation_prevention_interval_ms: 2000,
starvation_threshold_secs: 120,
..Default::default()
};
let config = IoPriorityQueueConfig::from_scheduler_config(&scheduler_config);
assert_eq!(config.queue_high_capacity, 128);
assert_eq!(config.queue_normal_capacity, 256);
assert_eq!(config.queue_low_capacity, 512);
assert_eq!(config.starvation_prevention_interval_ms, 2000);
assert_eq!(config.starvation_threshold_secs, 120);
}
#[tokio::test]
#[serial] #[serial]
async fn test_io_priority_metrics() { async fn test_io_priority_metrics() {
let metrics = IoPriorityMetrics::new(); let metrics = IoPriorityMetrics::new();
@@ -2014,7 +2128,7 @@ mod tests {
// Multi-Factor Strategy Tests // Multi-Factor Strategy Tests
// ============================================ // ============================================
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_nvme_sequential_low_load() { async fn test_multi_factor_strategy_nvme_sequential_low_load() {
// NVMe + Sequential + Low load = maximum buffer size // NVMe + Sequential + Low load = maximum buffer size
@@ -2041,7 +2155,7 @@ mod tests {
assert_eq!(strategy.bandwidth_tier, BandwidthTier::High); assert_eq!(strategy.bandwidth_tier, BandwidthTier::High);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_hdd_random_high_load() { async fn test_multi_factor_strategy_hdd_random_high_load() {
// HDD + Random + High load = conservative buffer size // HDD + Random + High load = conservative buffer size
@@ -2068,7 +2182,7 @@ mod tests {
assert!(strategy.bandwidth_limited, "Low bandwidth should be marked"); assert!(strategy.bandwidth_limited, "Low bandwidth should be marked");
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_ssd_mixed_medium_load() { async fn test_multi_factor_strategy_ssd_mixed_medium_load() {
// SSD + Mixed + Medium load = moderate buffer // SSD + Mixed + Medium load = moderate buffer
@@ -2096,7 +2210,7 @@ mod tests {
assert_eq!(strategy.access_pattern, AccessPattern::Mixed); assert_eq!(strategy.access_pattern, AccessPattern::Mixed);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_critical_load_disables_features() { async fn test_multi_factor_strategy_critical_load_disables_features() {
// Any media + Critical load = minimal features // Any media + Critical load = minimal features
@@ -2121,7 +2235,7 @@ mod tests {
assert!(strategy.buffer_size < 200 * 1024, "Critical load should reduce buffer"); assert!(strategy.buffer_size < 200 * 1024, "Critical load should reduce buffer");
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_buffer_cap_enforcement() { async fn test_multi_factor_strategy_buffer_cap_enforcement() {
// Test that storage media caps are enforced // Test that storage media caps are enforced
@@ -2146,7 +2260,7 @@ mod tests {
assert!(strategy.debug_info.buffer_cap_applied, "Buffer cap should be applied"); assert!(strategy.debug_info.buffer_cap_applied, "Buffer cap should be applied");
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_bandwidth_low_reduces_buffer() { async fn test_multi_factor_strategy_bandwidth_low_reduces_buffer() {
// Low bandwidth should reduce buffer // Low bandwidth should reduce buffer
@@ -2170,7 +2284,7 @@ mod tests {
assert!(strategy.buffer_size < context.base_buffer_size, "Low bandwidth should reduce buffer"); assert!(strategy.buffer_size < context.base_buffer_size, "Low bandwidth should reduce buffer");
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_high_concurrency_reduction() { async fn test_multi_factor_strategy_high_concurrency_reduction() {
// High concurrency should reduce buffer // High concurrency should reduce buffer
@@ -2193,7 +2307,7 @@ mod tests {
assert!(strategy.buffer_size < context.base_buffer_size, "High concurrency should reduce buffer"); assert!(strategy.buffer_size < context.base_buffer_size, "High concurrency should reduce buffer");
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_sequential_boost() { async fn test_multi_factor_strategy_sequential_boost() {
// Sequential reads should get boost // Sequential reads should get boost
@@ -2235,7 +2349,7 @@ mod tests {
} }
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_unknown_media_conservative() { async fn test_multi_factor_strategy_unknown_media_conservative() {
// Unknown media should be conservative // Unknown media should be conservative
@@ -2261,7 +2375,7 @@ mod tests {
); );
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_priority_classification() { async fn test_multi_factor_strategy_priority_classification() {
// Test priority classification based on file size // Test priority classification based on file size
@@ -2308,7 +2422,7 @@ mod tests {
assert_eq!(large_strategy.priority, IoPriority::Low); assert_eq!(large_strategy.priority, IoPriority::Low);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_readahead_decision_matrix() { async fn test_multi_factor_strategy_readahead_decision_matrix() {
// Test readahead enable/disable logic // Test readahead enable/disable logic
@@ -2394,7 +2508,7 @@ mod tests {
} }
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_buffer_multiplier_stages() { async fn test_multi_factor_strategy_buffer_multiplier_stages() {
// Test that all multiplier stages are applied // Test that all multiplier stages are applied
@@ -2429,7 +2543,7 @@ mod tests {
assert!(strategy.should_reduce_for_bandwidth); assert!(strategy.should_reduce_for_bandwidth);
} }
#[test] #[tokio::test]
#[serial] #[serial]
async fn test_multi_factor_strategy_compatibility_path() { async fn test_multi_factor_strategy_compatibility_path() {
// Test that compatibility path (from_wait_duration) still works // Test that compatibility path (from_wait_duration) still works
+2 -8
View File
@@ -121,14 +121,8 @@ impl ConcurrencyManager {
// Keep 1000 samples for P95/P99 calculation // Keep 1000 samples for P95/P99 calculation
let metrics_collector = Arc::new(MetricsCollector::new(performance_metrics, 1000)); let metrics_collector = Arc::new(MetricsCollector::new(performance_metrics, 1000));
// Build priority queue config // Build queue config directly from scheduler config.
let queue_config = IoPriorityQueueConfig { let queue_config = IoPriorityQueueConfig::from_scheduler_config(&scheduler_config);
queue_high_capacity: scheduler_config.queue_high_capacity,
queue_normal_capacity: scheduler_config.queue_normal_capacity,
queue_low_capacity: scheduler_config.queue_low_capacity,
starvation_prevention_interval_ms: scheduler_config.starvation_prevention_interval_ms,
starvation_threshold_secs: scheduler_config.starvation_threshold_secs,
};
Self { Self {
disk_read_semaphore: Arc::new(Semaphore::new(max_disk_reads)), disk_read_semaphore: Arc::new(Semaphore::new(max_disk_reads)),
+12 -12
View File
@@ -19,13 +19,13 @@
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::storage::backpressure::{BackpressureConfig, BackpressureMonitor, BackpressureState}; use crate::storage::backpressure::{BackpressureMonitor, BackpressureState, ObjectPipeBackpressurePolicy};
use crate::storage::concurrency::{IoLoadLevel, IoPriority}; use crate::storage::concurrency::{IoLoadLevel, IoPriority};
use crate::storage::deadlock_detector::{ use crate::storage::deadlock_detector::{
DeadlockDetector, DeadlockDetectorConfig, LockInfo, LockType, RequestResourceTracker, DeadlockDetector, LockInfo, LockType, RequestHangDetectionPolicy, RequestResourceTracker,
}; };
use crate::storage::lock_optimizer::{LockOptimizeConfig, LockOptimizer, LockStats}; use crate::storage::lock_optimizer::{LockOptimizeConfig, LockOptimizer, LockStats};
use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimedGetObjectResult, TimeoutConfig}; use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper, TimedGetObjectResult};
use std::time::Duration; use std::time::Duration;
// ============================================ // ============================================
@@ -34,7 +34,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_completes_within_timeout() { async fn test_timeout_wrapper_completes_within_timeout() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(5), get_object_timeout: Duration::from_secs(5),
..Default::default() ..Default::default()
}; };
@@ -52,7 +52,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_times_out() { async fn test_timeout_wrapper_times_out() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_millis(50), get_object_timeout: Duration::from_millis(50),
..Default::default() ..Default::default()
}; };
@@ -76,7 +76,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_returns_error() { async fn test_timeout_wrapper_returns_error() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(5), get_object_timeout: Duration::from_secs(5),
..Default::default() ..Default::default()
}; };
@@ -94,7 +94,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_disabled() { async fn test_timeout_wrapper_disabled() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::ZERO, get_object_timeout: Duration::ZERO,
..Default::default() ..Default::default()
}; };
@@ -120,7 +120,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_config_defaults() { fn test_backpressure_config_defaults() {
let config = BackpressureConfig::default(); let config = ObjectPipeBackpressurePolicy::default();
assert_eq!(config.buffer_size, 4 * 1024 * 1024); // 4MB assert_eq!(config.buffer_size, 4 * 1024 * 1024); // 4MB
assert_eq!(config.high_watermark, 80); assert_eq!(config.high_watermark, 80);
assert_eq!(config.low_watermark, 50); assert_eq!(config.low_watermark, 50);
@@ -128,7 +128,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_monitor_state_transitions() { fn test_backpressure_monitor_state_transitions() {
let config = BackpressureConfig { let config = ObjectPipeBackpressurePolicy {
buffer_size: 1000, buffer_size: 1000,
high_watermark: 80, high_watermark: 80,
low_watermark: 50, low_watermark: 50,
@@ -149,7 +149,7 @@ mod tests {
#[test] #[test]
fn test_backpressure_usage_percent() { fn test_backpressure_usage_percent() {
let config = BackpressureConfig { let config = ObjectPipeBackpressurePolicy {
buffer_size: 1000, buffer_size: 1000,
high_watermark: 80, high_watermark: 80,
low_watermark: 50, low_watermark: 50,
@@ -229,7 +229,7 @@ mod tests {
#[test] #[test]
fn test_deadlock_detector_config_defaults() { fn test_deadlock_detector_config_defaults() {
let config = DeadlockDetectorConfig::default(); let config = RequestHangDetectionPolicy::default();
assert!(!config.enabled); // Disabled by default assert!(!config.enabled); // Disabled by default
assert_eq!(config.check_interval, Duration::from_secs(5)); assert_eq!(config.check_interval, Duration::from_secs(5));
assert_eq!(config.hang_threshold, Duration::from_secs(10)); assert_eq!(config.hang_threshold, Duration::from_secs(10));
@@ -255,7 +255,7 @@ mod tests {
#[test] #[test]
fn test_deadlock_detector_registration() { fn test_deadlock_detector_registration() {
let config = DeadlockDetectorConfig { let config = RequestHangDetectionPolicy {
enabled: true, enabled: true,
..Default::default() ..Default::default()
}; };
+27 -16
View File
@@ -40,9 +40,9 @@
//! # Usage //! # Usage
//! //!
//! ```ignore //! ```ignore
//! use crate::storage::deadlock_detector::{DeadlockDetector, DeadlockDetectorConfig}; //! use crate::storage::deadlock_detector::{DeadlockDetector, RequestHangDetectionPolicy};
//! //!
//! let config = DeadlockDetectorConfig::from_env(); //! let config = RequestHangDetectionPolicy::from_env();
//! let detector = DeadlockDetector::new(config); //! let detector = DeadlockDetector::new(config);
//! detector.start(); //! detector.start();
//! //!
@@ -66,6 +66,7 @@ use tokio::sync::broadcast;
use tracing::{debug, error, warn}; use tracing::{debug, error, warn};
use metrics::counter; use metrics::counter;
use rustfs_io_core::DeadlockDetectorConfig as CoreDeadlockConfig;
/// Request identifier type. /// Request identifier type.
pub type RequestId = String; pub type RequestId = String;
@@ -73,9 +74,9 @@ pub type RequestId = String;
/// Lock identifier type. /// Lock identifier type.
pub type LockId = String; pub type LockId = String;
/// Deadlock detector configuration. /// Request-level hang and deadlock diagnosis policy.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct DeadlockDetectorConfig { pub struct RequestHangDetectionPolicy {
/// Whether deadlock detection is enabled. /// Whether deadlock detection is enabled.
pub enabled: bool, pub enabled: bool,
/// Detection check interval. /// Detection check interval.
@@ -86,7 +87,7 @@ pub struct DeadlockDetectorConfig {
pub capture_backtrace: bool, pub capture_backtrace: bool,
} }
impl Default for DeadlockDetectorConfig { impl Default for RequestHangDetectionPolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
enabled: rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE, enabled: rustfs_config::DEFAULT_OBJECT_DEADLOCK_DETECTION_ENABLE,
@@ -97,7 +98,7 @@ impl Default for DeadlockDetectorConfig {
} }
} }
impl DeadlockDetectorConfig { impl RequestHangDetectionPolicy {
/// Load configuration from environment variables. /// Load configuration from environment variables.
pub fn from_env() -> Self { pub fn from_env() -> Self {
let enabled = rustfs_utils::get_env_bool( let enabled = rustfs_utils::get_env_bool(
@@ -120,6 +121,15 @@ impl DeadlockDetectorConfig {
capture_backtrace: false, capture_backtrace: false,
} }
} }
/// Convert the request-level policy into the shared io-core deadlock config.
pub fn to_core_config(&self) -> CoreDeadlockConfig {
CoreDeadlockConfig {
enabled: self.enabled,
detection_interval: self.check_interval,
max_hold_time: self.hang_threshold,
}
}
} }
/// Lock information for tracking. /// Lock information for tracking.
@@ -247,7 +257,7 @@ pub struct ResourceUsage {
/// Deadlock detector. /// Deadlock detector.
pub struct DeadlockDetector { pub struct DeadlockDetector {
/// Configuration. /// Configuration.
config: DeadlockDetectorConfig, config: RequestHangDetectionPolicy,
/// Active request trackers. /// Active request trackers.
requests: Arc<RwLock<HashMap<RequestId, RequestResourceTracker>>>, requests: Arc<RwLock<HashMap<RequestId, RequestResourceTracker>>>,
/// Detection task handle. /// Detection task handle.
@@ -262,7 +272,7 @@ pub struct DeadlockDetector {
impl DeadlockDetector { impl DeadlockDetector {
/// Create a new deadlock detector. /// Create a new deadlock detector.
pub fn new(config: DeadlockDetectorConfig) -> Self { pub fn new(config: RequestHangDetectionPolicy) -> Self {
let (shutdown_tx, _) = broadcast::channel(1); let (shutdown_tx, _) = broadcast::channel(1);
Self { Self {
@@ -426,7 +436,7 @@ impl DeadlockDetector {
/// Detect deadlock cycles in the lock wait graph. /// Detect deadlock cycles in the lock wait graph.
fn detect_cycle( fn detect_cycle(
requests: &Arc<RwLock<HashMap<RequestId, RequestResourceTracker>>>, requests: &Arc<RwLock<HashMap<RequestId, RequestResourceTracker>>>,
config: &DeadlockDetectorConfig, config: &RequestHangDetectionPolicy,
deadlocks_detected: &Arc<AtomicU64>, deadlocks_detected: &Arc<AtomicU64>,
) { ) {
let requests_guard = requests.read().unwrap(); let requests_guard = requests.read().unwrap();
@@ -499,13 +509,16 @@ impl DeadlockDetector {
/// Find a cycle in the wait graph using DFS. /// Find a cycle in the wait graph using DFS.
fn find_cycle(edges: &[WaitGraphEdge]) -> Option<Vec<RequestId>> { fn find_cycle(edges: &[WaitGraphEdge]) -> Option<Vec<RequestId>> {
// Build adjacency list if edges.is_empty() {
return None;
}
// Build adjacency list: from -> [to]
let mut graph: HashMap<&RequestId, Vec<&RequestId>> = HashMap::new(); let mut graph: HashMap<&RequestId, Vec<&RequestId>> = HashMap::new();
for edge in edges { for edge in edges {
graph.entry(&edge.from).or_default().push(&edge.to); graph.entry(&edge.from).or_default().push(&edge.to);
} }
// DFS with path tracking
let mut visited: HashSet<&RequestId> = HashSet::new(); let mut visited: HashSet<&RequestId> = HashSet::new();
let mut path: Vec<&RequestId> = Vec::new(); let mut path: Vec<&RequestId> = Vec::new();
let mut path_set: HashSet<&RequestId> = HashSet::new(); let mut path_set: HashSet<&RequestId> = HashSet::new();
@@ -514,7 +527,6 @@ impl DeadlockDetector {
if visited.contains(start) { if visited.contains(start) {
continue; continue;
} }
if Self::dfs_find_cycle(start, &graph, &mut visited, &mut path, &mut path_set) { if Self::dfs_find_cycle(start, &graph, &mut visited, &mut path, &mut path_set) {
return Some(path.iter().map(|s| (*s).clone()).collect()); return Some(path.iter().map(|s| (*s).clone()).collect());
} }
@@ -543,7 +555,6 @@ impl DeadlockDetector {
path.drain(0..cycle_start); path.drain(0..cycle_start);
return true; return true;
} }
if !visited.contains(neighbor) && Self::dfs_find_cycle(neighbor, graph, visited, path, path_set) { if !visited.contains(neighbor) && Self::dfs_find_cycle(neighbor, graph, visited, path, path_set) {
return true; return true;
} }
@@ -569,7 +580,7 @@ static DEADLOCK_DETECTOR: std::sync::OnceLock<Arc<DeadlockDetector>> = std::sync
pub fn get_deadlock_detector() -> Arc<DeadlockDetector> { pub fn get_deadlock_detector() -> Arc<DeadlockDetector> {
DEADLOCK_DETECTOR DEADLOCK_DETECTOR
.get_or_init(|| { .get_or_init(|| {
let config = DeadlockDetectorConfig::from_env(); let config = RequestHangDetectionPolicy::from_env();
Arc::new(DeadlockDetector::new(config)) Arc::new(DeadlockDetector::new(config))
}) })
.clone() .clone()
@@ -595,7 +606,7 @@ mod tests {
#[test] #[test]
fn test_deadlock_detector_config_default() { fn test_deadlock_detector_config_default() {
let config = DeadlockDetectorConfig::default(); let config = RequestHangDetectionPolicy::default();
assert!(!config.enabled); assert!(!config.enabled);
assert_eq!(config.check_interval, Duration::from_secs(5)); assert_eq!(config.check_interval, Duration::from_secs(5));
assert_eq!(config.hang_threshold, Duration::from_secs(10)); assert_eq!(config.hang_threshold, Duration::from_secs(10));
@@ -621,7 +632,7 @@ mod tests {
#[test] #[test]
fn test_deadlock_detector_registration() { fn test_deadlock_detector_registration() {
let config = DeadlockDetectorConfig { let config = RequestHangDetectionPolicy {
enabled: true, enabled: true,
..Default::default() ..Default::default()
}; };
+170 -221
View File
@@ -33,23 +33,14 @@
//! //!
//! - Configurable request-level timeout (default 30 seconds) //! - Configurable request-level timeout (default 30 seconds)
//! - Automatic cancellation of sub-tasks on timeout //! - Automatic cancellation of sub-tasks on timeout
//! - Resource cleanup on timeout (locks, memory, file handles)
//! - Timeout metrics emitted through `rustfs-io-metrics`
// Allow dead_code for public API that may be used by external modules or future features
#![allow(dead_code)]
//!
//! - Configurable request-level timeout (default 30 seconds)
//! - Automatic cancellation of sub-tasks on timeout
//! - Resource cleanup on timeout (locks, memory, file handles)
//! - Timeout metrics emitted through `rustfs-io-metrics` //! - Timeout metrics emitted through `rustfs-io-metrics`
//! //!
//! # Usage //! # Usage
//! //!
//! ```ignore //! ```ignore
//! use crate::storage::timeout_wrapper::{RequestTimeoutWrapper, TimeoutConfig}; //! use crate::storage::timeout_wrapper::{GetObjectTimeoutPolicy, RequestTimeoutWrapper};
//! //!
//! let config = TimeoutConfig::from_env(); //! let config = GetObjectTimeoutPolicy::from_env();
//! let wrapper = RequestTimeoutWrapper::new(config); //! let wrapper = RequestTimeoutWrapper::new(config);
//! //!
//! match wrapper.execute_with_timeout(|cancel_token| async move { //! match wrapper.execute_with_timeout(|cancel_token| async move {
@@ -66,23 +57,13 @@ use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
use tracing::{debug, warn}; use tracing::{debug, warn};
// Re-export types from rustfs_io_core for convenience /// Request-level timeout policy for GetObject.
/// Timeout configuration for GetObject requests.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct TimeoutConfig { pub struct GetObjectTimeoutPolicy {
/// GetObject request overall timeout (default 30s). /// GetObject request overall timeout (default 30s).
/// After this duration, the request is cancelled and returns 504. /// After this duration, the request is cancelled and returns 504.
pub get_object_timeout: Duration, pub get_object_timeout: Duration,
/// Lock acquisition timeout (default 5s).
/// Time to wait for a lock before giving up.
pub lock_acquire_timeout: Duration,
/// Disk read operation timeout (default 10s).
/// Individual disk read operations that exceed this are cancelled.
pub disk_read_timeout: Duration,
/// Enable dynamic timeout calculation based on object size /// Enable dynamic timeout calculation based on object size
pub enable_dynamic_timeout: bool, pub enable_dynamic_timeout: bool,
@@ -96,12 +77,14 @@ pub struct TimeoutConfig {
pub max_timeout: Duration, pub max_timeout: Duration,
} }
impl Default for TimeoutConfig { /// Backward-compatible alias for external callers using the previous name.
#[deprecated(note = "use GetObjectTimeoutPolicy instead")]
pub type TimeoutConfig = GetObjectTimeoutPolicy;
impl Default for GetObjectTimeoutPolicy {
fn default() -> Self { fn default() -> Self {
Self { Self {
get_object_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT), get_object_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT),
lock_acquire_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT),
disk_read_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT),
enable_dynamic_timeout: rustfs_config::DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE, enable_dynamic_timeout: rustfs_config::DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE,
bytes_per_second: rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND, bytes_per_second: rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND,
min_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT), min_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT),
@@ -110,21 +93,11 @@ impl Default for TimeoutConfig {
} }
} }
impl TimeoutConfig { impl GetObjectTimeoutPolicy {
/// Load configuration from environment variables. /// Load configuration from environment variables.
pub fn from_env() -> Self { pub fn from_env() -> Self {
let get_object_timeout = let get_object_timeout =
rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_GET_TIMEOUT, rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT); rustfs_utils::get_env_u64(rustfs_config::ENV_OBJECT_GET_TIMEOUT, rustfs_config::DEFAULT_OBJECT_GET_TIMEOUT);
let lock_acquire_timeout = rustfs_utils::get_env_u64(
rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT,
rustfs_config::DEFAULT_OBJECT_LOCK_ACQUIRE_TIMEOUT,
);
let disk_read_timeout = rustfs_utils::get_env_u64(
rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT,
rustfs_config::DEFAULT_OBJECT_DISK_READ_TIMEOUT,
);
// Dynamic timeout settings
let enable_dynamic_timeout = rustfs_utils::get_env_bool( let enable_dynamic_timeout = rustfs_utils::get_env_bool(
rustfs_config::ENV_OBJECT_DYNAMIC_TIMEOUT_ENABLE, rustfs_config::ENV_OBJECT_DYNAMIC_TIMEOUT_ENABLE,
rustfs_config::DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE, rustfs_config::DEFAULT_OBJECT_DYNAMIC_TIMEOUT_ENABLE,
@@ -138,8 +111,6 @@ impl TimeoutConfig {
Self { Self {
get_object_timeout: Duration::from_secs(get_object_timeout), get_object_timeout: Duration::from_secs(get_object_timeout),
lock_acquire_timeout: Duration::from_secs(lock_acquire_timeout),
disk_read_timeout: Duration::from_secs(disk_read_timeout),
enable_dynamic_timeout, enable_dynamic_timeout,
bytes_per_second, bytes_per_second,
min_timeout: Duration::from_secs(min_timeout_secs), min_timeout: Duration::from_secs(min_timeout_secs),
@@ -158,12 +129,17 @@ impl TimeoutConfig {
return self.get_object_timeout; return self.get_object_timeout;
} }
// Calculate timeout based on expected transfer speed // Keep storage-layer dynamic-timeout semantics local so request policy
// Add 50% buffer for network overhead and system load // bounds remain authoritative. We preserve the historical 1.5x envelope:
let estimated_seconds = (object_size / self.bytes_per_second) * 3 / 2; // object_size / (0.8 * bps) * 1.2 == object_size * 1.5 / bps.
//
// Ensure at least 1 second // Use integer math to avoid float->Duration conversion panics on extreme
let estimated_duration = Duration::from_secs(estimated_seconds.max(1)); // values and keep behavior predictable under saturation.
let bytes_per_second = self.bytes_per_second.max(1);
let numerator = (object_size as u128).saturating_mul(3);
let denominator = (bytes_per_second as u128).saturating_mul(2);
let estimated_secs = numerator.checked_div(denominator).unwrap_or(u128::MAX);
let estimated_duration = Duration::from_secs(estimated_secs.min(u64::MAX as u128) as u64);
// Clamp to min/max bounds // Clamp to min/max bounds
estimated_duration estimated_duration
@@ -220,59 +196,74 @@ pub enum TimedGetObjectResult<T, E> {
} }
/// Request timeout wrapper for async operations. /// Request timeout wrapper for async operations.
#[derive(Debug)]
pub struct RequestTimeoutWrapper { pub struct RequestTimeoutWrapper {
/// Configuration. /// Configuration.
config: TimeoutConfig, config: GetObjectTimeoutPolicy,
/// Request start time. /// Request start time.
start_time: Instant, start_time: Instant,
/// Optional operation size hint for dynamic timeout decisions.
operation_size: Option<u64>,
/// Cancellation token for propagating cancellation to sub-tasks. /// Cancellation token for propagating cancellation to sub-tasks.
cancel_token: CancellationToken, cancel_token: CancellationToken,
/// Request ID for logging/metrics. /// Request ID for logging/metrics.
request_id: String, request_id: String,
} }
impl std::fmt::Debug for RequestTimeoutWrapper {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RequestTimeoutWrapper")
.field("config", &self.config)
.field("elapsed", &self.elapsed())
.field("request_id", &self.request_id)
.finish()
}
}
impl RequestTimeoutWrapper { impl RequestTimeoutWrapper {
/// Create a new timeout wrapper with the given configuration. /// Create a new timeout wrapper with the given configuration.
/// ///
/// Note: This uses a sentinel request_id. Prefer `with_request_id()` to pass /// Note: This uses a sentinel request_id. Prefer `with_request_id()` to pass
/// the canonical request-id from `RequestContext`. /// the canonical request-id from `RequestContext`.
pub fn new(config: TimeoutConfig) -> Self { pub fn new(config: GetObjectTimeoutPolicy) -> Self {
Self { Self::new_with_parts(config, None, CancellationToken::new(), "no-request-id".to_string())
config,
start_time: Instant::now(),
cancel_token: CancellationToken::new(),
request_id: "no-request-id".to_string(),
}
} }
/// Create a new timeout wrapper with a specific request ID. /// Create a new timeout wrapper with a specific request ID.
pub fn with_request_id(config: TimeoutConfig, request_id: impl Into<String>) -> Self { pub fn with_request_id(config: GetObjectTimeoutPolicy, request_id: impl Into<String>) -> Self {
Self { Self::new_with_parts(config, None, CancellationToken::new(), request_id.into())
config,
start_time: Instant::now(),
cancel_token: CancellationToken::new(),
request_id: request_id.into(),
}
} }
/// Create a new timeout wrapper with operation size for dynamic timeout calculation. /// Create a new timeout wrapper with operation size for dynamic timeout calculation.
/// ///
/// Note: This uses a sentinel request_id. Prefer `with_request_id()` to pass /// Note: This uses a sentinel request_id. Prefer `with_request_id()` to pass
/// the canonical request-id from `RequestContext`. /// the canonical request-id from `RequestContext`.
pub fn with_operation_size(config: TimeoutConfig, operation_size: Option<u64>) -> Self { pub fn with_operation_size(config: GetObjectTimeoutPolicy, operation_size: Option<u64>) -> Self {
let _ = operation_size; Self::new_with_parts(config, operation_size, CancellationToken::new(), "no-request-id".to_string())
Self {
config,
start_time: Instant::now(),
cancel_token: CancellationToken::new(),
request_id: "no-request-id".to_string(),
}
} }
/// Get the configured timeout for this operation /// Get the configured timeout for this operation
pub fn get_timeout(&self, operation_size: Option<u64>) -> Duration { pub fn get_timeout(&self, operation_size_hint: Option<u64>) -> Duration {
self.config.get_timeout_for_operation(operation_size) self.config
.get_timeout_for_operation(self.effective_operation_size(operation_size_hint))
}
fn new_with_parts(
config: GetObjectTimeoutPolicy,
operation_size: Option<u64>,
cancel_token: CancellationToken,
request_id: String,
) -> Self {
Self {
config,
start_time: Instant::now(),
operation_size,
cancel_token,
request_id,
}
}
fn effective_operation_size(&self, operation_size_hint: Option<u64>) -> Option<u64> {
operation_size_hint.or(self.operation_size)
} }
/// Get the request ID. /// Get the request ID.
@@ -293,7 +284,7 @@ impl RequestTimeoutWrapper {
/// Check if the timeout has been exceeded. /// Check if the timeout has been exceeded.
pub fn is_timeout(&self) -> bool { pub fn is_timeout(&self) -> bool {
self.config.is_timeout_enabled() && self.elapsed() >= self.config.get_object_timeout self.config.is_timeout_enabled() && self.start_time.elapsed() >= self.get_timeout(None)
} }
/// Get elapsed time since the request started. /// Get elapsed time since the request started.
@@ -312,8 +303,10 @@ impl RequestTimeoutWrapper {
if !self.config.is_timeout_enabled() { if !self.config.is_timeout_enabled() {
return None; return None;
} }
let timeout = self.config.get_timeout_for_operation(operation_size); let timeout = self
let remaining = timeout.saturating_sub(self.elapsed()); .config
.get_timeout_for_operation(self.effective_operation_size(operation_size));
let remaining = timeout.saturating_sub(self.start_time.elapsed());
if remaining == Duration::ZERO { None } else { Some(remaining) } if remaining == Duration::ZERO { None } else { Some(remaining) }
} }
@@ -322,8 +315,10 @@ impl RequestTimeoutWrapper {
if !self.config.is_timeout_enabled() { if !self.config.is_timeout_enabled() {
return false; return false;
} }
let timeout = self.config.get_timeout_for_operation(operation_size); let timeout = self
self.elapsed() >= timeout .config
.get_timeout_for_operation(self.effective_operation_size(operation_size));
self.start_time.elapsed() >= timeout
} }
/// Execute an async operation with timeout protection. /// Execute an async operation with timeout protection.
@@ -343,95 +338,7 @@ impl RequestTimeoutWrapper {
F: FnOnce(CancellationToken) -> Fut, F: FnOnce(CancellationToken) -> Fut,
Fut: std::future::Future<Output = Result<T, E>>, Fut: std::future::Future<Output = Result<T, E>>,
{ {
if !self.config.is_timeout_enabled() { self.execute_with_timeout_internal(None, operation).await
// Timeout disabled, run without timeout
debug!(
request_id = %self.request_id,
"Timeout disabled, executing operation without timeout"
);
return match operation(self.cancel_token).await {
Ok(result) => TimedGetObjectResult::Success(result),
Err(e) => TimedGetObjectResult::Error(e),
};
}
let timeout_duration = self.config.get_object_timeout;
let request_id = self.request_id.clone();
let start_time = self.start_time;
debug!(
request_id = %request_id,
timeout_secs = timeout_duration.as_secs(),
"Starting timed operation"
);
// Record start time for metrics
rustfs_io_metrics::record_get_object_request_started();
// Clone cancel_token for the operation, keep original for potential cancellation
let cancel_token_for_op = self.cancel_token.clone();
match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await {
Ok(Ok(result)) => {
// Operation completed successfully
let elapsed = start_time.elapsed();
rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64());
debug!(
request_id = %request_id,
elapsed_ms = elapsed.as_millis(),
"Operation completed successfully"
);
TimedGetObjectResult::Success(result)
}
Ok(Err(e)) => {
// Operation failed before timeout
let elapsed = start_time.elapsed();
rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64());
debug!(
request_id = %request_id,
elapsed_ms = elapsed.as_millis(),
"Operation failed with error"
);
TimedGetObjectResult::Error(e)
}
Err(_) => {
// Timeout occurred
let elapsed = start_time.elapsed();
// Cancel the operation
self.cancel_token.cancel();
rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64()));
rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64());
warn!(
request_id = %request_id,
timeout_secs = timeout_duration.as_secs(),
elapsed_ms = elapsed.as_millis(),
"Operation timed out, cancellation signal sent"
);
TimedGetObjectResult::Timeout(TimeoutInfo {
request_id,
bucket: String::new(),
key: String::new(),
timeout_duration,
elapsed,
bytes_transferred: 0,
lock_hold_time: None,
disk_reads_completed: 0,
disk_reads_pending: 0,
object_size: None,
progress_percent: None,
})
}
}
} }
/// Execute an async operation with timeout and context information. /// Execute an async operation with timeout and context information.
@@ -450,86 +357,81 @@ impl RequestTimeoutWrapper {
{ {
let bucket = bucket.into(); let bucket = bucket.into();
let key = key.into(); let key = key.into();
self.execute_with_timeout_internal(Some((bucket, key)), operation).await
}
if !self.config.is_timeout_enabled() { async fn execute_with_timeout_internal<F, Fut, T, E>(
debug!( self,
context: Option<(String, String)>,
operation: F,
) -> TimedGetObjectResult<T, E>
where
F: FnOnce(CancellationToken) -> Fut,
Fut: std::future::Future<Output = Result<T, E>>,
{
let timeout_duration = self.get_timeout(None);
// Build a tracing span that carries request context for all log events.
let span = match &context {
Some((bucket, key)) => tracing::info_span!(
"timeout_operation",
request_id = %self.request_id, request_id = %self.request_id,
bucket = %bucket, bucket = %bucket,
key = %key, key = %key,
"Timeout disabled, executing operation without timeout" ),
); None => tracing::info_span!(
"timeout_operation",
request_id = %self.request_id,
),
};
let _guard = span.enter();
if !self.config.is_timeout_enabled() {
debug!("Timeout disabled, executing operation without timeout");
return match operation(self.cancel_token).await { return match operation(self.cancel_token).await {
Ok(result) => TimedGetObjectResult::Success(result), Ok(result) => TimedGetObjectResult::Success(result),
Err(e) => TimedGetObjectResult::Error(e), Err(e) => TimedGetObjectResult::Error(e),
}; };
} }
let timeout_duration = self.config.get_object_timeout; debug!(timeout_secs = timeout_duration.as_secs(), "Starting timed operation");
let request_id = self.request_id.clone();
let start_time = self.start_time;
debug!(
request_id = %request_id,
bucket = %bucket,
key = %key,
timeout_secs = timeout_duration.as_secs(),
"Starting timed operation"
);
rustfs_io_metrics::record_get_object_request_started(); rustfs_io_metrics::record_get_object_request_started();
// Clone cancel_token for the operation, keep original for potential cancellation
let cancel_token_for_op = self.cancel_token.clone(); let cancel_token_for_op = self.cancel_token.clone();
match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await { match tokio::time::timeout(timeout_duration, operation(cancel_token_for_op)).await {
Ok(Ok(result)) => { Ok(Ok(result)) => {
let elapsed = start_time.elapsed(); let elapsed = self.elapsed();
rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64());
debug!(elapsed_ms = elapsed.as_millis(), "Operation completed successfully");
debug!(
request_id = %request_id,
bucket = %bucket,
key = %key,
elapsed_ms = elapsed.as_millis(),
"Operation completed successfully"
);
TimedGetObjectResult::Success(result) TimedGetObjectResult::Success(result)
} }
Ok(Err(e)) => { Ok(Err(e)) => {
let elapsed = start_time.elapsed(); let elapsed = self.elapsed();
rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64());
debug!(elapsed_ms = elapsed.as_millis(), "Operation failed with error");
debug!(
request_id = %request_id,
bucket = %bucket,
key = %key,
elapsed_ms = elapsed.as_millis(),
"Operation failed with error"
);
TimedGetObjectResult::Error(e) TimedGetObjectResult::Error(e)
} }
Err(_) => { Err(_) => {
let elapsed = start_time.elapsed(); let elapsed = self.elapsed();
self.cancel_token.cancel(); self.cancel_token.cancel();
rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64())); rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64()));
rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64());
warn!( warn!(
request_id = %request_id,
bucket = %bucket,
key = %key,
timeout_secs = timeout_duration.as_secs(), timeout_secs = timeout_duration.as_secs(),
elapsed_ms = elapsed.as_millis(), elapsed_ms = elapsed.as_millis(),
"Operation timed out, cancellation signal sent" "Operation timed out, cancellation signal sent"
); );
let (bucket, key) = context.unwrap_or_default();
TimedGetObjectResult::Timeout(TimeoutInfo { TimedGetObjectResult::Timeout(TimeoutInfo {
request_id, request_id: self.request_id,
bucket, bucket,
key, key,
timeout_duration, timeout_duration,
@@ -538,7 +440,7 @@ impl RequestTimeoutWrapper {
lock_hold_time: None, lock_hold_time: None,
disk_reads_completed: 0, disk_reads_completed: 0,
disk_reads_pending: 0, disk_reads_pending: 0,
object_size: None, object_size: self.operation_size,
progress_percent: None, progress_percent: None,
}) })
} }
@@ -566,18 +468,16 @@ mod tests {
#[test] #[test]
fn test_timeout_config_default() { fn test_timeout_config_default() {
let config = TimeoutConfig::default(); let config = GetObjectTimeoutPolicy::default();
assert_eq!(config.get_object_timeout, Duration::from_secs(30)); assert_eq!(config.get_object_timeout, Duration::from_secs(30));
assert_eq!(config.lock_acquire_timeout, Duration::from_secs(5));
assert_eq!(config.disk_read_timeout, Duration::from_secs(10));
} }
#[test] #[test]
fn test_timeout_config_is_enabled() { fn test_timeout_config_is_enabled() {
let config = TimeoutConfig::default(); let config = GetObjectTimeoutPolicy::default();
assert!(config.is_timeout_enabled()); assert!(config.is_timeout_enabled());
let disabled_config = TimeoutConfig { let disabled_config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::ZERO, get_object_timeout: Duration::ZERO,
..Default::default() ..Default::default()
}; };
@@ -586,7 +486,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_success() { async fn test_timeout_wrapper_success() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(5), get_object_timeout: Duration::from_secs(5),
..Default::default() ..Default::default()
}; };
@@ -604,7 +504,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_timeout() { async fn test_timeout_wrapper_timeout() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_millis(100), get_object_timeout: Duration::from_millis(100),
..Default::default() ..Default::default()
}; };
@@ -627,7 +527,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_error() { async fn test_timeout_wrapper_error() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(5), get_object_timeout: Duration::from_secs(5),
..Default::default() ..Default::default()
}; };
@@ -645,7 +545,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_timeout_wrapper_disabled() { async fn test_timeout_wrapper_disabled() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::ZERO, get_object_timeout: Duration::ZERO,
..Default::default() ..Default::default()
}; };
@@ -681,7 +581,7 @@ mod tests {
#[test] #[test]
fn test_timeout_config_default_with_dynamic() { fn test_timeout_config_default_with_dynamic() {
let config = TimeoutConfig::default(); let config = GetObjectTimeoutPolicy::default();
assert!(config.enable_dynamic_timeout); assert!(config.enable_dynamic_timeout);
assert_eq!(config.bytes_per_second, rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND); assert_eq!(config.bytes_per_second, rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND);
assert_eq!(config.min_timeout, Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT)); assert_eq!(config.min_timeout, Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT));
@@ -690,7 +590,7 @@ mod tests {
#[test] #[test]
fn test_calculate_timeout_for_size() { fn test_calculate_timeout_for_size() {
let config = TimeoutConfig::default(); let config = GetObjectTimeoutPolicy::default();
// Test with small object (should use min timeout) // Test with small object (should use min timeout)
let small_timeout = config.calculate_timeout_for_size(1024); // 1KB let small_timeout = config.calculate_timeout_for_size(1024); // 1KB
@@ -707,9 +607,39 @@ mod tests {
assert!(huge_timeout <= Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MAX_TIMEOUT)); assert!(huge_timeout <= Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MAX_TIMEOUT));
} }
#[test]
fn test_calculate_timeout_for_size_respects_small_min_timeout() {
let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(30),
enable_dynamic_timeout: true,
bytes_per_second: 1024 * 1024, // 1MB/s
min_timeout: Duration::from_secs(1),
max_timeout: Duration::from_secs(30),
};
// Tiny object should still honor policy min_timeout (1s),
// instead of being raised by any external hard floor.
let timeout = config.calculate_timeout_for_size(1024); // 1KB
assert_eq!(timeout, Duration::from_secs(1));
}
#[test]
fn test_calculate_timeout_for_size_huge_object_does_not_panic_and_clamps() {
let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(300),
enable_dynamic_timeout: true,
bytes_per_second: 1,
min_timeout: Duration::from_secs(1),
max_timeout: Duration::from_secs(300),
};
let timeout = config.calculate_timeout_for_size(u64::MAX);
assert_eq!(timeout, Duration::from_secs(300));
}
#[test] #[test]
fn test_timeout_with_dynamic_disabled() { fn test_timeout_with_dynamic_disabled() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
enable_dynamic_timeout: false, enable_dynamic_timeout: false,
..Default::default() ..Default::default()
}; };
@@ -778,7 +708,7 @@ mod tests {
#[test] #[test]
fn test_should_timeout() { fn test_should_timeout() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_millis(100), get_object_timeout: Duration::from_millis(100),
..Default::default() ..Default::default()
}; };
@@ -795,7 +725,7 @@ mod tests {
#[test] #[test]
fn test_should_timeout_with_size() { fn test_should_timeout_with_size() {
let config = TimeoutConfig { let config = GetObjectTimeoutPolicy {
enable_dynamic_timeout: true, enable_dynamic_timeout: true,
bytes_per_second: 1024, // 1KB/s bytes_per_second: 1024, // 1KB/s
min_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT), min_timeout: Duration::from_secs(rustfs_config::DEFAULT_OBJECT_MIN_TIMEOUT),
@@ -811,4 +741,23 @@ mod tests {
// Large size should calculate longer timeout // Large size should calculate longer timeout
assert!(!wrapper.should_timeout(Some(10 * 1024 * 1024))); assert!(!wrapper.should_timeout(Some(10 * 1024 * 1024)));
} }
#[test]
fn test_wrapper_operation_size_hint_is_applied() {
let config = GetObjectTimeoutPolicy {
get_object_timeout: Duration::from_secs(300),
enable_dynamic_timeout: true,
bytes_per_second: 1024 * 1024,
min_timeout: Duration::from_secs(1),
max_timeout: Duration::from_secs(300),
};
let size = 100 * 1024 * 1024;
let wrapper = RequestTimeoutWrapper::with_operation_size(config.clone(), Some(size));
let expected_dynamic = config.get_timeout_for_operation(Some(size));
let baseline_no_size = config.get_timeout_for_operation(None);
assert_ne!(expected_dynamic, baseline_no_size);
assert_eq!(wrapper.get_timeout(None), expected_dynamic);
}
} }