gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-28 16:07:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: converge storage io hot paths (#3029)

Browse Source 
* refactor(issue-633): clarify layered io control policies

* refactor(issue-633): consolidate timeout and deadlock layers

* refactor(issue-633): align storage backpressure metadata

* refactor(issue-633): unify storage backpressure transitions

* refactor(issue-633): simplify watermark transition API

* test(issue-633): add storage backpressure transition test

* refactor(issue-633): align storage pipe meta shape

* refactor(issue-633): enrich storage monitor metadata

* refactor(issue-633): finalize storage backpressure convergence

* refactor(issue-633): complete scheduler layer convergence

* refactor(issue-633): reduce concurrency facade config duplication

* refactor(issue-633): migrate storage callsites to final policy names

* chore(issue-633): apply final pre-commit normalization

* refactor(issue-633): unify timeout wrapper dynamic size path

* refactor(issue-633): make concurrency policies copyable

* refactor(issue-633): converge storage io hot paths

* fix(issue-633): honor storage timeout min bound

* fix(storage): avoid timeout calc panic on huge sizes

* refactor(storage): consolidate timeout checks and test attrs

* fix(storage): harden io scheduler core config mapping

* refactor(storage): eliminate patch-on-patch patterns and dead code

- Remove trivial accessor methods on ConcurrencyConfig that just return pub fields
- Remove dead BackpressureEvent/BackpressureEventType types from concurrency crate
- Fix io_schedule test using wrong constructor (from_core_config -> from_scheduler_config)
- Update manager.rs to use config fields directly instead of removed accessors

* fix: adopt review feedback for config guards

* test: remove needless struct update defaults

* fix: harden timeout policy and preserve api alias
This commit is contained in:
houseme
2026-05-20 20:00:23 +08:00
committed by GitHub
parent 45b04316b9
commit 375482a4b6
14 changed files with 988 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/concurrency/src/config.rs
+70 -84
View File
@@ -14,6 +14,10 @@
//! Configuration for concurrency management
use crate::{
backpressure::PipeBackpressurePolicy, deadlock::DeadlockMonitorPolicy, scheduler::SchedulerPolicy,
timeout::TimeoutManagerPolicy,
};
use std::time::Duration;
/// Feature flags for concurrency modules
@@ -72,84 +76,39 @@ impl ConcurrencyFeatures {
}
}
/// Facade policy for lock manager behavior.
#[derive(Debug, Clone, Copy)]
pub struct LockManagerPolicy {
/// Enable lock optimization.
pub enabled: bool,
/// Lock acquisition timeout.
pub acquire_timeout: Duration,
}
impl Default for LockManagerPolicy {
fn default() -> Self {
Self {
enabled: true,
acquire_timeout: Duration::from_secs(5),
}
}
}
/// Main configuration for concurrency management
#[derive(Debug, Clone)]
#[derive(Debug, Clone, Default)]
pub struct ConcurrencyConfig {
/// Feature flags
pub features: ConcurrencyFeatures,
// Timeout configuration
/// Default timeout duration
pub default_timeout: Duration,
/// Maximum timeout duration
pub max_timeout: Duration,
/// Enable dynamic timeout
pub enable_dynamic_timeout: bool,
// Lock configuration
/// Enable lock optimization
pub enable_lock_optimization: bool,
/// Lock acquisition timeout
pub lock_acquire_timeout: Duration,
// Deadlock configuration
/// Enable deadlock detection
pub enable_deadlock_detection: bool,
/// Deadlock check interval
pub deadlock_check_interval: Duration,
/// Hang threshold
pub hang_threshold: Duration,
// Backpressure configuration
/// Buffer size for backpressure
pub backpressure_buffer_size: usize,
/// High watermark percentage
pub high_watermark: u32,
/// Low watermark percentage
pub low_watermark: u32,
// Scheduler configuration
/// Base buffer size for I/O
pub io_buffer_size: usize,
/// Maximum buffer size
pub max_buffer_size: usize,
/// High priority size threshold
pub high_priority_threshold: usize,
/// Low priority size threshold
pub low_priority_threshold: usize,
}
impl Default for ConcurrencyConfig {
fn default() -> Self {
Self {
features: ConcurrencyFeatures::default(),
// Timeout defaults
default_timeout: Duration::from_secs(30),
max_timeout: Duration::from_secs(300),
enable_dynamic_timeout: true,
// Lock defaults
enable_lock_optimization: true,
lock_acquire_timeout: Duration::from_secs(5),
// Deadlock defaults
enable_deadlock_detection: false,
deadlock_check_interval: Duration::from_secs(10),
hang_threshold: Duration::from_secs(60),
// Backpressure defaults
backpressure_buffer_size: 4 * 1024 * 1024, // 4MB
high_watermark: 80,
low_watermark: 50,
// Scheduler defaults
io_buffer_size: 64 * 1024, // 64KB
max_buffer_size: 4 * 1024 * 1024, // 4MB
high_priority_threshold: 1024 * 1024, // 1MB
low_priority_threshold: 10 * 1024 * 1024, // 10MB
}
}
/// Timeout facade policy.
pub timeout_policy: TimeoutManagerPolicy,
/// Lock facade policy.
pub lock_policy: LockManagerPolicy,
/// Deadlock facade policy.
pub deadlock_policy: DeadlockMonitorPolicy,
/// Backpressure facade policy.
pub backpressure_policy: PipeBackpressurePolicy,
/// Scheduler facade policy.
pub scheduler_policy: SchedulerPolicy,
}
impl ConcurrencyConfig {
@@ -161,25 +120,25 @@ impl ConcurrencyConfig {
if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_DEFAULT")
&& let Ok(secs) = val.parse::<u64>()
{
config.default_timeout = Duration::from_secs(secs);
config.timeout_policy.default_timeout = Duration::from_secs(secs);
}
if let Ok(val) = std::env::var("RUSTFS_TIMEOUT_MAX")
&& let Ok(secs) = val.parse::<u64>()
{
config.max_timeout = Duration::from_secs(secs);
config.timeout_policy.max_timeout = Duration::from_secs(secs);
}
if let Ok(val) = std::env::var("RUSTFS_BACKPRESSURE_BUFFER_SIZE")
&& let Ok(size) = val.parse::<usize>()
{
config.backpressure_buffer_size = size;
config.backpressure_policy.buffer_size = size;
}
if let Ok(val) = std::env::var("RUSTFS_IO_BUFFER_SIZE")
&& let Ok(size) = val.parse::<usize>()
{
config.io_buffer_size = size;
config.scheduler_policy.base_buffer_size = size;
}
config
@@ -187,18 +146,25 @@ impl ConcurrencyConfig {
/// Validate configuration
pub fn validate(&self) -> Result<(), ConfigError> {
if self.default_timeout > self.max_timeout {
if self.timeout_policy.default_timeout > self.timeout_policy.max_timeout {
return Err(ConfigError::InvalidTimeout("default_timeout cannot exceed max_timeout".to_string()));
}
if self.timeout_policy.min_timeout > self.timeout_policy.max_timeout {
return Err(ConfigError::InvalidTimeout("min_timeout cannot exceed max_timeout".to_string()));
}
if self.high_watermark <= self.low_watermark || self.high_watermark > 100 {
if self.backpressure_policy.high_watermark <= self.backpressure_policy.low_watermark
|| self.backpressure_policy.high_watermark > 100
{
return Err(ConfigError::InvalidBackpressure(
"high_watermark must be > low_watermark and <= 100".to_string(),
));
}
if self.io_buffer_size > self.max_buffer_size {
return Err(ConfigError::InvalidScheduler("io_buffer_size cannot exceed max_buffer_size".to_string()));
if self.scheduler_policy.base_buffer_size > self.scheduler_policy.max_buffer_size {
return Err(ConfigError::InvalidScheduler(
"base_buffer_size cannot exceed max_buffer_size".to_string(),
));
}
Ok(())
@@ -235,8 +201,12 @@ mod tests {
#[test]
fn test_invalid_timeout() {
let config = ConcurrencyConfig {
default_timeout: Duration::from_secs(100),
max_timeout: Duration::from_secs(50),
timeout_policy: TimeoutManagerPolicy {
default_timeout: Duration::from_secs(100),
max_timeout: Duration::from_secs(50),
enable_dynamic: true,
..Default::default()
},
..Default::default()
};
assert!(
@@ -245,6 +215,22 @@ mod tests {
);
}
#[test]
fn test_invalid_min_timeout() {
let config = ConcurrencyConfig {
timeout_policy: TimeoutManagerPolicy {
min_timeout: Duration::from_secs(100),
max_timeout: Duration::from_secs(50),
..Default::default()
},
..Default::default()
};
assert!(
config.validate().is_err(),
"validate() should return an error when min_timeout > max_timeout"
);
}
#[test]
fn test_features() {
let features = ConcurrencyFeatures::all();