mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-04 12:27:43 +00:00
fix: address correctness, safety, and concurrency issues (#2327)
Co-authored-by: heihutu <heihutu@gmail.com> Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
houseme
+21
-4
@@ -20,6 +20,7 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
default-run = "rustfs"
|
||||
description = "RustFS is a high-performance, distributed file system designed for modern cloud-native applications, providing efficient data storage and retrieval with advanced features like S3 Select, IAM, and policy management."
|
||||
keywords.workspace = true
|
||||
categories.workspace = true
|
||||
@@ -30,15 +31,24 @@ documentation = "https://docs.rustfs.com/"
|
||||
name = "rustfs"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "manual-test-dial9"
|
||||
path = "tests/manual/test_dial9.rs"
|
||||
test = false
|
||||
bench = false
|
||||
required-features = ["manual-test-runners"]
|
||||
|
||||
[features]
|
||||
default = ["metrics"]
|
||||
metrics = []
|
||||
metrics-gpu = ["metrics", "rustfs-metrics/gpu"]
|
||||
default = ["direct-io"]
|
||||
metrics-gpu = ["rustfs-metrics/gpu"]
|
||||
ftps = ["rustfs-protocols/ftps"]
|
||||
swift = ["rustfs-protocols/swift"]
|
||||
webdav = ["rustfs-protocols/webdav"]
|
||||
license = []
|
||||
full = ["metrics", "metrics-gpu", "ftps", "swift", "webdav"]
|
||||
direct-io = [] # Aligned direct I/O reader support (uses aligned pread, does not set O_DIRECT)
|
||||
io-scheduler-debug = [] # Enable debug information in I/O scheduler
|
||||
full = ["metrics-gpu", "ftps", "swift", "webdav", "direct-io"]
|
||||
manual-test-runners = []
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -73,6 +83,9 @@ rustfs-targets = { workspace = true }
|
||||
rustfs-trusted-proxies = { workspace = true }
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
rustfs-zip = { workspace = true }
|
||||
rustfs-io-core = { workspace = true }
|
||||
rustfs-io-metrics = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
rustfs-scanner = { workspace = true }
|
||||
|
||||
# Async Runtime and Networking
|
||||
@@ -147,6 +160,8 @@ aes-gcm = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
# Data structures
|
||||
hashbrown = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
libsystemd.workspace = true
|
||||
@@ -154,6 +169,8 @@ libsystemd.workspace = true
|
||||
[target.'cfg(not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64")))'.dependencies]
|
||||
mimalloc = { workspace = true }
|
||||
|
||||
|
||||
|
||||
# Only enable pprof-based profiling on non-Windows targets.
|
||||
[target.'cfg(all(not(target_os = "windows"), not(all(target_os = "linux", target_env = "gnu", target_arch = "x86_64"))))'.dependencies]
|
||||
starshard = { workspace = true }
|
||||
|
||||
+1149
-686
File diff suppressed because it is too large
Load Diff
+120
-50
@@ -582,6 +582,82 @@ struct FeatureInfoJson {
|
||||
description: &'static str,
|
||||
}
|
||||
|
||||
struct FeatureSpec {
|
||||
name: &'static str,
|
||||
enabled: bool,
|
||||
description: &'static str,
|
||||
dependencies: &'static str,
|
||||
default_enabled: bool,
|
||||
}
|
||||
|
||||
fn feature_specs() -> [FeatureSpec; 9] {
|
||||
[
|
||||
FeatureSpec {
|
||||
name: "direct-io",
|
||||
enabled: cfg!(feature = "direct-io"),
|
||||
description: "Aligned pread-based direct I/O reader support",
|
||||
dependencies: "(none)",
|
||||
default_enabled: true,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "metrics-gpu",
|
||||
enabled: cfg!(feature = "metrics-gpu"),
|
||||
description: "Metrics GPU support",
|
||||
dependencies: "rustfs-metrics/gpu",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "ftps",
|
||||
enabled: cfg!(feature = "ftps"),
|
||||
description: "FTPS protocol support",
|
||||
dependencies: "rustfs-protocols/ftps",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "swift",
|
||||
enabled: cfg!(feature = "swift"),
|
||||
description: "Swift storage backend",
|
||||
dependencies: "rustfs-protocols/swift",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "webdav",
|
||||
enabled: cfg!(feature = "webdav"),
|
||||
description: "WebDAV protocol support",
|
||||
dependencies: "rustfs-protocols/webdav",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "license",
|
||||
enabled: cfg!(feature = "license"),
|
||||
description: "License validation",
|
||||
dependencies: "(none)",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "io-scheduler-debug",
|
||||
enabled: cfg!(feature = "io-scheduler-debug"),
|
||||
description: "Enable debug information in I/O scheduler",
|
||||
dependencies: "(none)",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "manual-test-runners",
|
||||
enabled: cfg!(feature = "manual-test-runners"),
|
||||
description: "Enable manual test binaries",
|
||||
dependencies: "(none)",
|
||||
default_enabled: false,
|
||||
},
|
||||
FeatureSpec {
|
||||
name: "full",
|
||||
enabled: cfg!(feature = "full"),
|
||||
description: "All features enabled",
|
||||
dependencies: "metrics-gpu + ftps + swift + webdav + direct-io",
|
||||
default_enabled: false,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Dependency information for JSON output
|
||||
#[derive(Serialize)]
|
||||
struct DepsInfoJson {
|
||||
@@ -591,38 +667,14 @@ struct DepsInfoJson {
|
||||
}
|
||||
|
||||
fn collect_deps_info_json() -> DepsInfoJson {
|
||||
let features = vec![
|
||||
FeatureInfoJson {
|
||||
name: "metrics",
|
||||
enabled: cfg!(feature = "metrics"),
|
||||
description: "Metrics collection and reporting",
|
||||
},
|
||||
FeatureInfoJson {
|
||||
name: "ftps",
|
||||
enabled: cfg!(feature = "ftps"),
|
||||
description: "FTPS protocol support",
|
||||
},
|
||||
FeatureInfoJson {
|
||||
name: "swift",
|
||||
enabled: cfg!(feature = "swift"),
|
||||
description: "Swift storage backend",
|
||||
},
|
||||
FeatureInfoJson {
|
||||
name: "webdav",
|
||||
enabled: cfg!(feature = "webdav"),
|
||||
description: "WebDAV protocol support",
|
||||
},
|
||||
FeatureInfoJson {
|
||||
name: "license",
|
||||
enabled: cfg!(feature = "license"),
|
||||
description: "License validation",
|
||||
},
|
||||
FeatureInfoJson {
|
||||
name: "full",
|
||||
enabled: cfg!(feature = "full"),
|
||||
description: "All features enabled",
|
||||
},
|
||||
];
|
||||
let features: Vec<FeatureInfoJson> = feature_specs()
|
||||
.into_iter()
|
||||
.map(|feature| FeatureInfoJson {
|
||||
name: feature.name,
|
||||
enabled: feature.enabled,
|
||||
description: feature.description,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let enabled_count = features.iter().filter(|f| f.enabled).count();
|
||||
let total_count = features.len();
|
||||
@@ -758,17 +810,9 @@ fn get_workload_profile_info() -> String {
|
||||
|
||||
/// Dependency information
|
||||
fn format_deps_info() -> String {
|
||||
// Check which features are enabled at compile time
|
||||
let features = [
|
||||
("metrics", cfg!(feature = "metrics"), "Metrics collection and reporting"),
|
||||
("ftps", cfg!(feature = "ftps"), "FTPS protocol support"),
|
||||
("swift", cfg!(feature = "swift"), "Swift storage backend"),
|
||||
("webdav", cfg!(feature = "webdav"), "WebDAV protocol support"),
|
||||
("license", cfg!(feature = "license"), "License validation"),
|
||||
("full", cfg!(feature = "full"), "All features enabled"),
|
||||
];
|
||||
let features = feature_specs();
|
||||
|
||||
let enabled_count = features.iter().filter(|(_, enabled, _)| *enabled).count();
|
||||
let enabled_count = features.iter().filter(|feature| feature.enabled).count();
|
||||
|
||||
let mut output = format!(
|
||||
"## Build Features\n\n\
|
||||
@@ -782,23 +826,24 @@ fn format_deps_info() -> String {
|
||||
output.push_str("### Feature Status\n\n");
|
||||
output.push_str("| Feature | Status | Description |\n");
|
||||
output.push_str("|---------|--------|-------------|\n");
|
||||
for (name, enabled, description) in features {
|
||||
let status = if enabled { "✓" } else { "✗" };
|
||||
output.push_str(&format!("| {} | {} | {} |\n", name, status, description));
|
||||
for feature in &features {
|
||||
let status = if feature.enabled { "✓" } else { "✗" };
|
||||
output.push_str(&format!("| {} | {} | {} |\n", feature.name, status, feature.description));
|
||||
}
|
||||
|
||||
output.push_str("\n### Default Features\n\n");
|
||||
output.push_str("| Feature | Note |\n");
|
||||
output.push_str("|---------|------|\n");
|
||||
output.push_str("| metrics | enabled by default |\n");
|
||||
for feature in features.iter().filter(|feature| feature.default_enabled) {
|
||||
output.push_str(&format!("| {} | enabled by default |\n", feature.name));
|
||||
}
|
||||
|
||||
output.push_str("\n### Feature Dependencies\n\n");
|
||||
output.push_str("| Feature | Dependencies |\n");
|
||||
output.push_str("|---------|-------------|\n");
|
||||
output.push_str("| full | metrics + ftps + swift + webdav |\n");
|
||||
output.push_str("| ftps | rustfs-protocols/ftps |\n");
|
||||
output.push_str("| swift | rustfs-protocols/swift |\n");
|
||||
output.push_str("| webdav | rustfs-protocols/webdav |\n");
|
||||
for feature in &features {
|
||||
output.push_str(&format!("| {} | {} |\n", feature.name, feature.dependencies));
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
@@ -875,4 +920,29 @@ mod tests {
|
||||
let info = RuntimeInfo::collect();
|
||||
assert!(info.process_id > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_collect_deps_info_json_matches_cargo_features() {
|
||||
let info = collect_deps_info_json();
|
||||
let feature_names: Vec<_> = info.features.iter().map(|feature| feature.name).collect();
|
||||
|
||||
assert_eq!(info.total_count, 9);
|
||||
assert_eq!(info.features.len(), 9);
|
||||
assert!(feature_names.contains(&"direct-io"));
|
||||
assert!(feature_names.contains(&"metrics-gpu"));
|
||||
assert!(feature_names.contains(&"io-scheduler-debug"));
|
||||
assert!(feature_names.contains(&"manual-test-runners"));
|
||||
assert!(!feature_names.contains(&"metrics"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_deps_info_matches_cargo_feature_output() {
|
||||
let output = format_deps_info();
|
||||
|
||||
assert!(output.contains("| metrics-gpu |"));
|
||||
assert!(output.contains("| io-scheduler-debug |"));
|
||||
assert!(output.contains("| manual-test-runners |"));
|
||||
assert!(output.contains("| direct-io | enabled by default |"));
|
||||
assert!(output.contains("| full | metrics-gpu + ftps + swift + webdav + direct-io |"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -414,6 +414,52 @@ where
|
||||
shutdown_tx
|
||||
}
|
||||
|
||||
/// Starts the auto-tuner for performance optimization if enabled via environment variable.
|
||||
///
|
||||
/// The auto-tuner reads `RUSTFS_AUTOTUNER_ENABLED` to decide whether to run.
|
||||
/// When enabled, it spawns a background task that tunes concurrency settings
|
||||
/// every 60 seconds.
|
||||
pub async fn init_auto_tuner(ctx: tokio_util::sync::CancellationToken) {
|
||||
use crate::storage::concurrency::get_concurrency_manager;
|
||||
use rustfs_io_metrics::AutoTuner;
|
||||
use rustfs_io_metrics::TunerConfig;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
let autotuner_enabled = rustfs_utils::get_env_bool("RUSTFS_AUTOTUNER_ENABLED", false);
|
||||
|
||||
if autotuner_enabled {
|
||||
info!(target: "rustfs::main::run", "Starting auto-tuner for performance optimization");
|
||||
|
||||
let config = TunerConfig::default();
|
||||
let manager = get_concurrency_manager();
|
||||
let performance_metrics = manager.performance_metrics();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut tuner = AutoTuner::with_config(config).with_metrics(performance_metrics);
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = ctx.cancelled() => {
|
||||
info!(target: "rustfs::autotuner", "Auto-tuner shutting down");
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(tokio::time::Duration::from_secs(60)) => {
|
||||
if let Err(e) = tuner.tune().await {
|
||||
error!(target: "rustfs::autotuner", "Auto-tuner iteration failed: {}", e);
|
||||
} else {
|
||||
debug!(target: "rustfs::autotuner", "Auto-tuner iteration completed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
info!(target: "rustfs::main::run", "Auto-tuner started successfully");
|
||||
} else {
|
||||
info!(target: "rustfs::main::run", "Auto-tuner disabled (set RUSTFS_AUTOTUNER_ENABLED=true to enable)");
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize the FTP system
|
||||
///
|
||||
/// This function initializes the FTP server (non-encrypted) if enabled in the configuration.
|
||||
|
||||
@@ -564,6 +564,9 @@ async fn run(config: config::Config) -> Result<()> {
|
||||
if rustfs_obs::observability_metric_enabled() {
|
||||
// Initialize metrics system
|
||||
init_metrics_system(ctx.clone());
|
||||
|
||||
// Initialize auto-tuner for performance optimization (optional)
|
||||
crate::init::init_auto_tuner(ctx.clone()).await;
|
||||
}
|
||||
|
||||
info!(
|
||||
|
||||
@@ -45,7 +45,6 @@ use std::time::Instant;
|
||||
use tokio::io::{DuplexStream, duplex};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use metrics::counter;
|
||||
|
||||
/// Backpressure pipe configuration.
|
||||
@@ -281,7 +280,6 @@ impl BackpressurePipe {
|
||||
if usage >= threshold && !self.state.load(Ordering::Relaxed) {
|
||||
self.state.store(true, Ordering::Relaxed);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.backpressure.events.total", "state" => "high_watermark").increment(1);
|
||||
|
||||
warn!(
|
||||
@@ -302,7 +300,6 @@ impl BackpressurePipe {
|
||||
if usage <= threshold && self.state.load(Ordering::Relaxed) {
|
||||
self.state.store(false, Ordering::Relaxed);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.backpressure.events.total", "state" => "normal").increment(1);
|
||||
|
||||
debug!(
|
||||
@@ -409,7 +406,6 @@ impl BackpressureMonitor {
|
||||
|
||||
if usage >= high {
|
||||
if !self.in_high_watermark.swap(true, Ordering::Relaxed) {
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.backpressure.events.total", "state" => "high_watermark").increment(1);
|
||||
|
||||
debug!(usage_percent = self.usage_percent() as u32, "Backpressure: entered high watermark");
|
||||
@@ -417,7 +413,6 @@ impl BackpressureMonitor {
|
||||
BackpressureState::HighWatermark
|
||||
} else if usage <= low {
|
||||
if self.in_high_watermark.swap(false, Ordering::Relaxed) {
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.backpressure.events.total", "state" => "normal").increment(1);
|
||||
|
||||
debug!(usage_percent = self.usage_percent() as u32, "Backpressure: returned to normal");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,12 +15,18 @@
|
||||
//! Concurrency manager for coordinating concurrent GetObject requests.
|
||||
|
||||
use super::io_schedule::{
|
||||
IoLoadLevel, IoLoadMetrics, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, IoStrategy,
|
||||
IoLoadLevel, IoLoadMetrics, IoPriority, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus, IoSchedulerConfig, IoStrategy,
|
||||
get_advanced_buffer_size,
|
||||
};
|
||||
use super::object_cache::{CacheStats, CachedGetObject, CachedObject, HotObjectCache};
|
||||
use super::object_cache::{CacheStats, CachedGetObject, TieredObjectCache, WarmupPattern};
|
||||
use super::request_guard::GetObjectGuard;
|
||||
use rustfs_concurrency::{GetObjectCacheEligibility, GetObjectQueueSnapshot};
|
||||
use rustfs_config::{KI_B, MI_B};
|
||||
use rustfs_io_core::BytesPool;
|
||||
use rustfs_io_core::io_profile::{AccessPattern, IoPatternDetector, StorageMedia, detect_storage_media};
|
||||
use rustfs_io_metrics::bandwidth::{BandwidthMonitor, BandwidthSnapshot};
|
||||
use rustfs_io_metrics::global_metrics::get_global_metrics;
|
||||
use rustfs_io_metrics::{MetricsCollector, PerformanceMetrics};
|
||||
use std::sync::{Arc, LazyLock, Mutex};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
@@ -31,8 +37,8 @@ pub(crate) static CONCURRENCY_MANAGER: LazyLock<ConcurrencyManager> = LazyLock::
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ConcurrencyManager {
|
||||
/// Hot object cache for frequently accessed objects
|
||||
cache: Arc<HotObjectCache>,
|
||||
/// Tiered object cache (L1 + L2) for frequently accessed objects
|
||||
cache: Arc<TieredObjectCache>,
|
||||
/// Semaphore to limit concurrent disk reads
|
||||
disk_read_semaphore: Arc<Semaphore>,
|
||||
/// Whether object caching is enabled (from RUSTFS_OBJECT_CACHE_ENABLE env var)
|
||||
@@ -42,6 +48,19 @@ pub struct ConcurrencyManager {
|
||||
/// I/O priority queue for request scheduling
|
||||
#[allow(dead_code)]
|
||||
priority_queue: Arc<IoPriorityQueue<()>>,
|
||||
/// Bytes pool for buffer allocation and reuse
|
||||
bytes_pool: Arc<BytesPool>,
|
||||
// Enhanced scheduler state
|
||||
/// I/O scheduler configuration (cached at initialization)
|
||||
scheduler_config: IoSchedulerConfig,
|
||||
/// Detected storage media type
|
||||
storage_media: StorageMedia,
|
||||
/// I/O pattern detector for sequential/random access tracking
|
||||
pattern_detector: Arc<Mutex<IoPatternDetector>>,
|
||||
/// Bandwidth monitor for adaptive I/O sizing
|
||||
bandwidth_monitor: Arc<Mutex<BandwidthMonitor>>,
|
||||
/// Metrics collector for I/O latency tracking (P50, P95, P99)
|
||||
metrics_collector: Arc<MetricsCollector>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ConcurrencyManager {
|
||||
@@ -52,10 +71,21 @@ impl std::fmt::Debug for ConcurrencyManager {
|
||||
} else {
|
||||
"locked".to_string()
|
||||
};
|
||||
let bandwidth_info = if let Ok(monitor) = self.bandwidth_monitor.lock() {
|
||||
format!("{:?}", monitor.snapshot())
|
||||
} else {
|
||||
"locked".to_string()
|
||||
};
|
||||
f.debug_struct("ConcurrencyManager")
|
||||
.field("active_requests", &super::io_schedule::ACTIVE_GET_REQUESTS.load(Ordering::Relaxed))
|
||||
.field(
|
||||
"active_requests",
|
||||
&crate::storage::concurrency::io_schedule::ACTIVE_GET_REQUESTS.load(Ordering::Relaxed),
|
||||
)
|
||||
.field("disk_read_permits", &self.disk_read_semaphore.available_permits())
|
||||
.field("io_metrics", &io_metrics_info)
|
||||
.field("storage_media", &self.storage_media)
|
||||
.field("bandwidth", &bandwidth_info)
|
||||
.field("bytes_pool", &self.bytes_pool)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -64,22 +94,78 @@ impl ConcurrencyManager {
|
||||
/// Create a new concurrency manager with default settings
|
||||
///
|
||||
/// Reads configuration from environment variables:
|
||||
/// - `RUSTFS_OBJECT_CACHE_ENABLE`: Enable/disable object caching (default: false)
|
||||
/// - `RUSTFS_OBJECT_CACHE_ENABLE`: Enable/disable object caching (default: true)
|
||||
/// - `RUSTFS_OBJECT_TIERED_CACHE_ENABLE`: Enable tiered L1+L2 caching (default: true)
|
||||
/// - `RUSTFS_OBJECT_MAX_CONCURRENT_DISK_READS`: Maximum concurrent disk reads (default: 64)
|
||||
pub fn new() -> Self {
|
||||
// Load scheduler configuration once at initialization
|
||||
let scheduler_config = IoSchedulerConfig::from_env();
|
||||
|
||||
let cache_enabled =
|
||||
rustfs_utils::get_env_bool(rustfs_config::ENV_OBJECT_CACHE_ENABLE, rustfs_config::DEFAULT_OBJECT_CACHE_ENABLE);
|
||||
|
||||
let max_disk_reads = rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
let tiered_cache_enabled = rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_OBJECT_TIERED_CACHE_ENABLE,
|
||||
rustfs_config::DEFAULT_OBJECT_TIERED_CACHE_ENABLE,
|
||||
);
|
||||
|
||||
let max_disk_reads = scheduler_config.max_concurrent_reads;
|
||||
|
||||
// Detect storage media
|
||||
let storage_media =
|
||||
detect_storage_media(scheduler_config.storage_detection_enabled, &scheduler_config.storage_media_override);
|
||||
|
||||
// Create tiered cache configuration
|
||||
let cache = if tiered_cache_enabled {
|
||||
Arc::new(TieredObjectCache::new())
|
||||
} else {
|
||||
// If tiered cache is disabled, create a simple tiered cache (acts as single-level)
|
||||
// For now, we always use TieredObjectCache since the configuration is now enabled by default
|
||||
Arc::new(TieredObjectCache::new())
|
||||
};
|
||||
|
||||
// Initialize I/O pattern detector
|
||||
let pattern_detector = Arc::new(Mutex::new(IoPatternDetector::new(
|
||||
scheduler_config.pattern_history_size,
|
||||
scheduler_config.sequential_step_tolerance_bytes,
|
||||
)));
|
||||
|
||||
// Initialize bandwidth monitor
|
||||
let bandwidth_monitor = Arc::new(Mutex::new(BandwidthMonitor::new(
|
||||
scheduler_config.bandwidth_ema_beta,
|
||||
scheduler_config.bandwidth_low_threshold_bps,
|
||||
scheduler_config.bandwidth_high_threshold_bps,
|
||||
)));
|
||||
|
||||
// Use global performance metrics instance for consistent metrics tracking
|
||||
// This allows AutoTuner and other components to access the same metrics data
|
||||
let performance_metrics = get_global_metrics();
|
||||
|
||||
// Initialize metrics collector for I/O latency tracking
|
||||
// Keep 1000 samples for P95/P99 calculation
|
||||
let metrics_collector = Arc::new(MetricsCollector::new(performance_metrics.clone(), 1000));
|
||||
|
||||
// Build priority queue config
|
||||
let queue_config = IoPriorityQueueConfig {
|
||||
queue_high_capacity: scheduler_config.queue_high_capacity,
|
||||
queue_normal_capacity: scheduler_config.queue_normal_capacity,
|
||||
queue_low_capacity: scheduler_config.queue_low_capacity,
|
||||
starvation_prevention_interval_ms: scheduler_config.starvation_prevention_interval_ms,
|
||||
starvation_threshold_secs: scheduler_config.starvation_threshold_secs,
|
||||
};
|
||||
|
||||
Self {
|
||||
cache: Arc::new(HotObjectCache::new()),
|
||||
cache,
|
||||
disk_read_semaphore: Arc::new(Semaphore::new(max_disk_reads)),
|
||||
cache_enabled,
|
||||
io_metrics: Arc::new(Mutex::new(IoLoadMetrics::new(100))), // Keep last 100 observations
|
||||
priority_queue: Arc::new(IoPriorityQueue::new(IoPriorityQueueConfig::default())),
|
||||
io_metrics: Arc::new(Mutex::new(IoLoadMetrics::new(scheduler_config.load_sample_window))),
|
||||
priority_queue: Arc::new(IoPriorityQueue::new(queue_config)),
|
||||
bytes_pool: Arc::new(BytesPool::new_tiered()),
|
||||
scheduler_config,
|
||||
storage_media,
|
||||
pattern_detector,
|
||||
bandwidth_monitor,
|
||||
metrics_collector,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,14 +190,25 @@ impl ConcurrencyManager {
|
||||
|
||||
/// Try to get an object from cache
|
||||
pub async fn get_cached(&self, key: &str) -> Option<Arc<Vec<u8>>> {
|
||||
self.cache.get(key).await
|
||||
self.cache.get_bytes(key).await
|
||||
}
|
||||
|
||||
/// Cache an object for future retrievals
|
||||
pub async fn cache_object(&self, key: String, data: Vec<u8>) {
|
||||
let size = data.len();
|
||||
let cached_obj = Arc::new(CachedObject::new_with_size(data, size));
|
||||
self.cache.put(key, cached_obj).await;
|
||||
let cached_data = Arc::new(data);
|
||||
self.cache.put_bytes(key, cached_data).await;
|
||||
}
|
||||
|
||||
/// Get the bytes pool for buffer allocation
|
||||
///
|
||||
/// Returns a reference to the BytesPool which can be used to acquire
|
||||
/// reusable buffers for I/O operations, reducing allocation overhead.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Arc-wrapped BytesPool instance
|
||||
pub fn bytes_pool(&self) -> Arc<BytesPool> {
|
||||
self.bytes_pool.clone()
|
||||
}
|
||||
|
||||
/// Acquire a permit to perform a disk read operation
|
||||
@@ -138,13 +235,57 @@ impl ConcurrencyManager {
|
||||
if let Ok(mut metrics) = self.io_metrics.lock() {
|
||||
metrics.record(wait_duration);
|
||||
}
|
||||
}
|
||||
|
||||
// Record histogram metric for Prometheus
|
||||
#[cfg(all(feature = "metrics", not(test)))]
|
||||
{
|
||||
use metrics::histogram;
|
||||
histogram!("rustfs.disk.permit.wait.duration.seconds").record(wait_duration.as_secs_f64());
|
||||
}
|
||||
// ============================================
|
||||
// Metrics Collection Methods
|
||||
// ============================================
|
||||
|
||||
/// Record a disk I/O operation for latency tracking.
|
||||
///
|
||||
/// This method delegates to MetricsCollector which:
|
||||
/// 1. Updates atomic counters in PerformanceMetrics
|
||||
/// 2. Records latency for P95/P99 calculation
|
||||
/// 3. Reports to metrics crate (which exports to OTEL)
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `bytes` - Number of bytes transferred
|
||||
/// * `duration` - Duration of the I/O operation
|
||||
/// * `is_read` - true for read operations, false for writes
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// let manager = get_concurrency_manager();
|
||||
/// let start = Instant::now();
|
||||
/// // ... perform disk I/O ...
|
||||
/// let duration = start.elapsed();
|
||||
/// manager.record_disk_operation(1024 * 1024, duration, true).await;
|
||||
/// ```
|
||||
pub async fn record_disk_operation(&self, bytes: u64, duration: Duration, is_read: bool) {
|
||||
self.metrics_collector.record_io_operation(bytes, duration, is_read).await;
|
||||
}
|
||||
|
||||
/// Get a reference to the metrics collector for external use.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Arc-wrapped MetricsCollector instance
|
||||
pub fn metrics_collector(&self) -> &Arc<MetricsCollector> {
|
||||
&self.metrics_collector
|
||||
}
|
||||
|
||||
/// Get the global performance metrics instance.
|
||||
///
|
||||
/// This provides access to the shared PerformanceMetrics that is used
|
||||
/// across all components, including AutoTuner.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Arc-wrapped PerformanceMetrics instance
|
||||
pub fn performance_metrics(&self) -> Arc<PerformanceMetrics> {
|
||||
get_global_metrics()
|
||||
}
|
||||
|
||||
/// Calculate an adaptive I/O strategy based on disk permit wait time.
|
||||
@@ -179,6 +320,85 @@ impl ConcurrencyManager {
|
||||
IoStrategy::from_wait_duration(permit_wait_duration, base_buffer_size)
|
||||
}
|
||||
|
||||
/// Calculate I/O strategy with enhanced multi-factor context.
|
||||
///
|
||||
/// This method integrates storage media, access patterns, bandwidth observations,
|
||||
/// and concurrent request count to provide a more sophisticated I/O strategy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `file_size` - Size of the file/object being read (-1 if unknown)
|
||||
/// * `base_buffer_size` - Base buffer size from workload configuration
|
||||
/// * `permit_wait_duration` - Time spent waiting for disk read permit
|
||||
/// * `is_sequential_hint` - Whether the access pattern is known to be sequential
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// An `IoStrategy` with optimized parameters based on all available factors.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// let strategy = manager.calculate_io_strategy_with_context(
|
||||
/// file_size,
|
||||
/// 256 * 1024,
|
||||
/// permit_wait_duration,
|
||||
/// false,
|
||||
/// );
|
||||
/// let optimal_buffer = strategy.buffer_size;
|
||||
/// let enable_readahead = strategy.enable_readahead;
|
||||
/// ```
|
||||
pub fn calculate_io_strategy_with_context(
|
||||
&self,
|
||||
file_size: i64,
|
||||
base_buffer_size: usize,
|
||||
permit_wait_duration: Duration,
|
||||
is_sequential_hint: bool,
|
||||
) -> IoStrategy {
|
||||
use crate::storage::concurrency::io_schedule::IoSchedulingContext;
|
||||
|
||||
// Record the observation for future smoothing
|
||||
self.record_permit_wait(permit_wait_duration);
|
||||
|
||||
// Get current access pattern
|
||||
let access_pattern = if let Ok(detector) = self.pattern_detector.lock() {
|
||||
detector.current_pattern()
|
||||
} else {
|
||||
AccessPattern::Unknown
|
||||
};
|
||||
|
||||
// Get current bandwidth snapshot
|
||||
let observed_bandwidth_bps = if let Ok(monitor) = self.bandwidth_monitor.lock() {
|
||||
let snapshot = monitor.snapshot();
|
||||
if snapshot.tier == rustfs_io_metrics::bandwidth::BandwidthTier::Unknown {
|
||||
None
|
||||
} else {
|
||||
Some(snapshot.bytes_per_second)
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Get concurrent request count
|
||||
let concurrent_requests =
|
||||
crate::storage::concurrency::io_schedule::ACTIVE_GET_REQUESTS.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
// Build scheduling context
|
||||
let context = IoSchedulingContext {
|
||||
file_size,
|
||||
base_buffer_size,
|
||||
permit_wait_duration,
|
||||
is_sequential_hint,
|
||||
access_pattern,
|
||||
storage_media: self.storage_media,
|
||||
observed_bandwidth_bps,
|
||||
concurrent_requests,
|
||||
};
|
||||
|
||||
// Calculate strategy using multi-factor approach
|
||||
IoStrategy::from_context_with_config(&context, &self.scheduler_config)
|
||||
}
|
||||
|
||||
/// Get the smoothed I/O load level based on recent observations.
|
||||
///
|
||||
/// This uses the rolling window of permit wait times to provide a more
|
||||
@@ -242,9 +462,78 @@ impl ConcurrencyManager {
|
||||
buffer_size.clamp(32 * KI_B, MI_B)
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Enhanced I/O Scheduling Methods
|
||||
// ============================================
|
||||
|
||||
/// Record an I/O access for pattern detection.
|
||||
///
|
||||
/// This updates the pattern detector with the offset and size of an access,
|
||||
/// allowing it to distinguish between sequential and random access patterns.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `offset` - File offset being accessed
|
||||
/// * `len` - Length of the access
|
||||
pub fn record_access(&self, offset: u64, len: u64) {
|
||||
if let Ok(mut detector) = self.pattern_detector.lock() {
|
||||
detector.record(offset, len);
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the current access pattern.
|
||||
///
|
||||
/// Returns the detected access pattern (Sequential, Random, Mixed, or Unknown).
|
||||
pub fn current_access_pattern(&self) -> AccessPattern {
|
||||
if let Ok(detector) = self.pattern_detector.lock() {
|
||||
detector.current_pattern()
|
||||
} else {
|
||||
AccessPattern::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a data transfer for bandwidth monitoring.
|
||||
///
|
||||
/// This updates the bandwidth monitor with the bytes transferred and duration,
|
||||
/// allowing it to maintain an EMA (Exponential Moving Average) of the observed bandwidth.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `bytes` - Number of bytes transferred
|
||||
/// * `duration` - Duration of the transfer
|
||||
pub fn record_transfer(&self, bytes: u64, duration: Duration) {
|
||||
if let Ok(mut monitor) = self.bandwidth_monitor.lock() {
|
||||
monitor.record_transfer(bytes, duration);
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the current bandwidth snapshot.
|
||||
///
|
||||
/// Returns a snapshot of the current bandwidth including bytes per second and tier.
|
||||
pub fn current_bandwidth_snapshot(&self) -> BandwidthSnapshot {
|
||||
if let Ok(monitor) = self.bandwidth_monitor.lock() {
|
||||
monitor.snapshot()
|
||||
} else {
|
||||
BandwidthSnapshot {
|
||||
bytes_per_second: 0,
|
||||
tier: rustfs_io_metrics::bandwidth::BandwidthTier::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the detected storage media type.
|
||||
pub fn storage_media(&self) -> StorageMedia {
|
||||
self.storage_media
|
||||
}
|
||||
|
||||
/// Get the scheduler configuration.
|
||||
pub fn scheduler_config(&self) -> &IoSchedulerConfig {
|
||||
&self.scheduler_config
|
||||
}
|
||||
|
||||
/// Get cache statistics
|
||||
pub async fn cache_stats(&self) -> CacheStats {
|
||||
self.cache.stats().await
|
||||
self.cache.stats_as_hot_cache().await
|
||||
}
|
||||
|
||||
/// Clear all cached objects
|
||||
@@ -252,6 +541,13 @@ impl ConcurrencyManager {
|
||||
self.cache.clear().await;
|
||||
}
|
||||
|
||||
/// Reset cache hit/miss metrics counters.
|
||||
///
|
||||
/// This is useful for testing to get a clean slate for hit rate calculations.
|
||||
pub fn reset_cache_metrics(&self) {
|
||||
self.cache.reset_metrics();
|
||||
}
|
||||
|
||||
/// Check if a key is cached
|
||||
pub async fn is_cached(&self, key: &str) -> bool {
|
||||
self.cache.contains(key).await
|
||||
@@ -259,17 +555,18 @@ impl ConcurrencyManager {
|
||||
|
||||
/// Get multiple cached objects in a single operation
|
||||
pub async fn get_cached_batch(&self, keys: &[String]) -> Vec<Option<Arc<Vec<u8>>>> {
|
||||
self.cache.get_batch(keys).await
|
||||
self.cache.get_batch_bytes(keys).await
|
||||
}
|
||||
|
||||
/// Remove a specific object from cache
|
||||
pub async fn remove_cached(&self, key: &str) -> bool {
|
||||
self.cache.remove(key).await
|
||||
self.cache.remove(key).await.is_some()
|
||||
}
|
||||
|
||||
/// Get the most frequently accessed keys
|
||||
pub async fn get_hot_keys(&self, limit: usize) -> Vec<(String, u64)> {
|
||||
self.cache.get_hot_keys(limit).await
|
||||
let keys = self.cache.get_hot_keys(limit).await;
|
||||
keys.into_iter().map(|(k, v)| (k, v as u64)).collect()
|
||||
}
|
||||
|
||||
/// Get cache hit rate percentage
|
||||
@@ -282,7 +579,55 @@ impl ConcurrencyManager {
|
||||
/// This can be called during server startup or maintenance windows
|
||||
/// to pre-populate the cache with known hot objects.
|
||||
pub async fn warm_cache(&self, objects: Vec<(String, Vec<u8>)>) {
|
||||
self.cache.warm(objects).await;
|
||||
if !self.cache_enabled {
|
||||
debug!("Cache is disabled, skipping warmup");
|
||||
return;
|
||||
}
|
||||
|
||||
// Cache each object
|
||||
for (key, data) in objects {
|
||||
self.cache_object(key, data).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Warm up cache with a specific pattern.
|
||||
///
|
||||
/// This method supports different warming patterns for more intelligent
|
||||
/// cache pre-population during server startup or maintenance windows.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `pattern` - The warming pattern to use
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// The number of objects successfully warmed
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// // Warm the 100 most recently accessed objects
|
||||
/// let pattern = WarmupPattern::RecentAccesses { limit: 100 };
|
||||
/// let warmed = manager.warm_cache_with_pattern(pattern).await;
|
||||
///
|
||||
/// // Warm specific keys
|
||||
/// let keys = vec!["bucket1/key1".to_string(), "bucket1/key2".to_string()];
|
||||
/// let pattern = WarmupPattern::SpecificKeys(keys);
|
||||
/// manager.warm_cache_with_pattern(pattern).await;
|
||||
/// ```
|
||||
pub async fn warm_cache_with_pattern(&self, pattern: WarmupPattern) -> usize {
|
||||
if !self.cache_enabled {
|
||||
debug!("Cache is disabled, skipping warmup");
|
||||
return 0;
|
||||
}
|
||||
|
||||
debug!("warm_cache_with_pattern called with pattern: {:?}", pattern);
|
||||
|
||||
// Delegate to the tiered cache's warm implementation
|
||||
// Note: This returns the count of keys identified for warming,
|
||||
// but actual object loading from storage would need to be implemented
|
||||
// at a higher layer (object_usecase) that has access to storage backends
|
||||
self.cache.warm(pattern).await
|
||||
}
|
||||
|
||||
/// Get optimized buffer size for a request
|
||||
@@ -459,31 +804,32 @@ impl ConcurrencyManager {
|
||||
// Unknown size, use normal priority
|
||||
IoPriority::Normal
|
||||
} else {
|
||||
IoPriority::from_size(request_size)
|
||||
// Use cached scheduler config thresholds
|
||||
IoPriority::from_size_with_thresholds(
|
||||
request_size,
|
||||
self.scheduler_config.high_priority_size_threshold,
|
||||
self.scheduler_config.low_priority_size_threshold,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if priority scheduling is enabled.
|
||||
pub fn is_priority_scheduling_enabled(&self) -> bool {
|
||||
rustfs_utils::get_env_bool(
|
||||
rustfs_config::ENV_OBJECT_PRIORITY_SCHEDULING_ENABLE,
|
||||
rustfs_config::DEFAULT_OBJECT_PRIORITY_SCHEDULING_ENABLE,
|
||||
)
|
||||
self.scheduler_config.enable_priority
|
||||
}
|
||||
|
||||
/// Get current I/O queue status for monitoring.
|
||||
///
|
||||
/// Returns information about permit usage and waiting requests.
|
||||
pub fn io_queue_status(&self) -> IoQueueStatus {
|
||||
let total_permits = rustfs_utils::get_env_usize(
|
||||
rustfs_config::ENV_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
rustfs_config::DEFAULT_OBJECT_MAX_CONCURRENT_DISK_READS,
|
||||
let snapshot = GetObjectQueueSnapshot::from_available_permits(
|
||||
self.scheduler_config.max_concurrent_reads,
|
||||
self.disk_read_semaphore.available_permits(),
|
||||
);
|
||||
let permits_in_use = total_permits.saturating_sub(self.disk_read_semaphore.available_permits());
|
||||
|
||||
IoQueueStatus {
|
||||
total_permits,
|
||||
permits_in_use,
|
||||
total_permits: snapshot.total_permits,
|
||||
permits_in_use: snapshot.permits_in_use,
|
||||
high_priority_waiting: 0, // Would need additional tracking
|
||||
normal_priority_waiting: 0,
|
||||
low_priority_waiting: 0,
|
||||
@@ -511,11 +857,7 @@ impl ConcurrencyManager {
|
||||
&self,
|
||||
priority: IoPriority,
|
||||
) -> Result<tokio::sync::SemaphorePermit<'_>, tokio::sync::AcquireError> {
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
use metrics::counter;
|
||||
counter!("rustfs.disk.read.queue.total", "priority" => priority.as_str()).increment(1);
|
||||
}
|
||||
rustfs_io_metrics::record_io_priority_assignment(priority.as_str());
|
||||
|
||||
debug!(
|
||||
priority = %priority,
|
||||
@@ -526,6 +868,26 @@ impl ConcurrencyManager {
|
||||
self.disk_read_semaphore.acquire().await
|
||||
}
|
||||
|
||||
/// Build the minimal cache eligibility decision for a GetObject response.
|
||||
pub fn get_object_cache_eligibility(
|
||||
&self,
|
||||
cache_writeback_enabled: bool,
|
||||
is_part_request: bool,
|
||||
is_range_request: bool,
|
||||
encryption_applied: bool,
|
||||
response_size: i64,
|
||||
) -> GetObjectCacheEligibility {
|
||||
GetObjectCacheEligibility {
|
||||
cache_enabled: self.is_cache_enabled(),
|
||||
cache_writeback_enabled,
|
||||
is_part_request,
|
||||
is_range_request,
|
||||
encryption_applied,
|
||||
response_size,
|
||||
max_cacheable_size: self.max_object_size(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the global concurrency manager instance.
|
||||
pub fn global() -> &'static Self {
|
||||
&CONCURRENCY_MANAGER
|
||||
@@ -714,4 +1076,177 @@ mod integration_tests {
|
||||
assert!(size1 > 0);
|
||||
assert!(size1 <= 2 * 1024 * 1024); // Not more than 2MB
|
||||
}
|
||||
|
||||
// ============================================
|
||||
// Multi-Factor Strategy Integration Tests
|
||||
// ============================================
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_nvme_optimal() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Simulate optimal conditions: Unknown/SSD + Sequential + Low load
|
||||
let file_size = 100 * 1024 * 1024; // 100MB
|
||||
let base_buffer = 256 * 1024;
|
||||
let permit_wait = Duration::from_millis(5); // Low load
|
||||
let is_sequential = true;
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(file_size, base_buffer, permit_wait, is_sequential);
|
||||
let media = manager.storage_media();
|
||||
|
||||
// Verify basic optimizations work
|
||||
assert_eq!(strategy.storage_media, media);
|
||||
assert!(strategy.buffer_size >= base_buffer * 8 / 10, "Sequential should maintain or boost buffer");
|
||||
let expected_readahead = !matches!(media, StorageMedia::Hdd);
|
||||
assert_eq!(
|
||||
strategy.enable_readahead, expected_readahead,
|
||||
"Readahead should follow storage profile preference under low load"
|
||||
);
|
||||
assert_eq!(strategy.load_level, IoLoadLevel::Low);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_access_pattern_tracking() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Record sequential accesses
|
||||
for offset in [0, 1024, 2048, 3072, 4096] {
|
||||
manager.record_access(offset, 1024);
|
||||
}
|
||||
|
||||
// Check pattern detection
|
||||
let pattern = manager.current_access_pattern();
|
||||
assert_eq!(pattern, AccessPattern::Sequential);
|
||||
|
||||
// Record random accesses
|
||||
for offset in [0, 10 * 1024, 100 * 1024, 5 * 1024 * 1024] {
|
||||
manager.record_access(offset, 1024);
|
||||
}
|
||||
|
||||
// Pattern should change to mixed or random
|
||||
let pattern_after = manager.current_access_pattern();
|
||||
assert!(!matches!(pattern_after, AccessPattern::Sequential));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_bandwidth_recording() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Simulate transfer
|
||||
let bytes = 10 * 1024 * 1024; // 10MB
|
||||
let duration = Duration::from_millis(100); // 100ms = 100MB/s
|
||||
|
||||
manager.record_transfer(bytes, duration);
|
||||
|
||||
// Check bandwidth snapshot (returns BandwidthSnapshot directly)
|
||||
let snapshot = manager.current_bandwidth_snapshot();
|
||||
assert!(snapshot.bytes_per_second > 0, "Should have bandwidth data after recording");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_compatibility() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Test that old API still works
|
||||
let old_strategy = manager.calculate_io_strategy(Duration::from_millis(50), 256 * 1024);
|
||||
|
||||
assert!(old_strategy.buffer_size > 0);
|
||||
|
||||
// New API with context should also work
|
||||
let new_strategy =
|
||||
manager.calculate_io_strategy_with_context(50 * 1024 * 1024, 256 * 1024, Duration::from_millis(50), false);
|
||||
|
||||
assert!(new_strategy.buffer_size > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_high_concurrency() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Simulate high concurrent requests by keeping guards alive
|
||||
let _guards: Vec<_> = (0..20).map(|_| GetObjectGuard::new()).collect();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(100 * 1024 * 1024, 512 * 1024, Duration::from_millis(10), true);
|
||||
|
||||
// High concurrency should reduce buffer
|
||||
assert!(strategy.concurrent_requests >= manager.scheduler_config().high_concurrency_threshold);
|
||||
assert!(strategy.buffer_size < 512 * 1024, "High concurrency should reduce buffer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_buffer_clamp() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
let media = manager.storage_media();
|
||||
let config = manager.scheduler_config();
|
||||
|
||||
// Request very large base buffer
|
||||
let large_base = 16 * 1024 * 1024; // 16MB
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
1024 * 1024, // 1GB file
|
||||
large_base,
|
||||
Duration::from_millis(1),
|
||||
true,
|
||||
);
|
||||
|
||||
let media_cap = match media {
|
||||
StorageMedia::Nvme => config.nvme_buffer_cap,
|
||||
StorageMedia::Ssd => config.ssd_buffer_cap,
|
||||
StorageMedia::Hdd => config.hdd_buffer_cap,
|
||||
StorageMedia::Unknown => config.ssd_buffer_cap,
|
||||
};
|
||||
let expected_max = media_cap.min(MI_B);
|
||||
|
||||
// Large base buffer should be constrained by storage cap first, then global clamp.
|
||||
assert_eq!(
|
||||
strategy.buffer_size, expected_max,
|
||||
"Buffer should be capped by media profile and global clamp"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_storage_media_detection() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Check storage media was detected at initialization
|
||||
let media = manager.storage_media();
|
||||
|
||||
// Should be one of the known types (not Unknown unless detection failed)
|
||||
// We accept Unknown if detection wasn't configured
|
||||
assert!(matches!(
|
||||
media,
|
||||
StorageMedia::Nvme | StorageMedia::Ssd | StorageMedia::Hdd | StorageMedia::Unknown
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_concurrency_manager_multi_factor_strategy_priority_with_context() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Test priority is correctly calculated in multi-factor strategy
|
||||
let small_file_strategy = manager.calculate_io_strategy_with_context(
|
||||
500 * 1024, // 500KB
|
||||
256 * 1024,
|
||||
Duration::from_millis(10),
|
||||
false,
|
||||
);
|
||||
|
||||
let large_file_strategy = manager.calculate_io_strategy_with_context(
|
||||
50 * 1024 * 1024, // 50MB
|
||||
256 * 1024,
|
||||
Duration::from_millis(10),
|
||||
false,
|
||||
);
|
||||
|
||||
assert_eq!(small_file_strategy.priority, IoPriority::High);
|
||||
assert_eq!(large_file_strategy.priority, IoPriority::Low);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,8 +13,28 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! Concurrency optimization module for high-performance object retrieval.
|
||||
//!
|
||||
//! This module provides concurrency management, I/O scheduling, and object caching
|
||||
//! for high-performance object retrieval operations.
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! The module is organized into several components:
|
||||
//! - **I/O Scheduling**: Adaptive buffer sizing and load management
|
||||
//! - **Object Caching**: Tiered L1/L2 cache for frequently accessed objects
|
||||
//! - **Concurrency Management**: Coordination of concurrent GetObject requests
|
||||
//! - **Request Tracking**: RAII guards for request lifecycle management
|
||||
//!
|
||||
//! # Migration Note
|
||||
//!
|
||||
//! Core algorithms have been migrated to `rustfs-io-core` and metrics to
|
||||
//! `rustfs-io-metrics`. This module maintains API compatibility while
|
||||
//! delegating to the new implementations.
|
||||
|
||||
// Sub-modules
|
||||
// pub mod bandwidth_monitor; // Migrated to rustfs-io-metrics
|
||||
// pub mod global_metrics; // Migrated to rustfs-io-metrics
|
||||
// pub mod io_profile; // Migrated to rustfs-io-core
|
||||
pub mod io_schedule;
|
||||
pub mod manager;
|
||||
pub mod object_cache;
|
||||
@@ -24,11 +44,11 @@ pub mod request_guard;
|
||||
// Public API Re-exports
|
||||
// ============================================
|
||||
|
||||
// I/O scheduling types
|
||||
// I/O scheduling types (from io_schedule.rs for backward compatibility)
|
||||
#[allow(unused_imports)]
|
||||
pub use io_schedule::{
|
||||
IO_PRIORITY_METRICS, IoLoadLevel, IoPriority, IoPriorityMetrics, IoPriorityQueue, IoPriorityQueueConfig, IoQueueStatus,
|
||||
IoStrategy, get_advanced_buffer_size, get_concurrency_aware_buffer_size,
|
||||
IoSchedulerConfig, IoStrategy, get_advanced_buffer_size, get_buffer_size_opt_in, get_concurrency_aware_buffer_size,
|
||||
};
|
||||
|
||||
// Request tracking
|
||||
@@ -41,6 +61,24 @@ pub use object_cache::{CacheHealthStatus, CacheStats, CachedGetObject};
|
||||
// Concurrency manager
|
||||
pub use manager::ConcurrencyManager;
|
||||
|
||||
// ============================================
|
||||
// New Module Re-exports (for gradual migration)
|
||||
// ============================================
|
||||
|
||||
// Re-export types from rustfs-io-core for convenience
|
||||
pub use rustfs_io_core::{
|
||||
// Backpressure types
|
||||
BackpressureMonitor,
|
||||
// Deadlock detection types
|
||||
DeadlockDetector,
|
||||
// Scheduler types
|
||||
IoScheduler,
|
||||
// Lock optimization types
|
||||
LockOptimizer,
|
||||
};
|
||||
|
||||
// Re-export types from rustfs-io-metrics for convenience
|
||||
|
||||
// ============================================
|
||||
// Helper Functions
|
||||
// ============================================
|
||||
@@ -55,3 +93,27 @@ pub fn get_concurrency_manager() -> &'static ConcurrencyManager {
|
||||
pub fn reset_active_get_requests() {
|
||||
io_schedule::ACTIVE_GET_REQUESTS.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Create a new I/O scheduler with default configuration.
|
||||
#[allow(dead_code)]
|
||||
pub fn create_io_scheduler() -> IoScheduler {
|
||||
IoScheduler::with_defaults()
|
||||
}
|
||||
|
||||
/// Create a new backpressure monitor with default configuration.
|
||||
#[allow(dead_code)]
|
||||
pub fn create_backpressure_monitor() -> BackpressureMonitor {
|
||||
BackpressureMonitor::with_defaults()
|
||||
}
|
||||
|
||||
/// Create a new deadlock detector with default configuration.
|
||||
#[allow(dead_code)]
|
||||
pub fn create_deadlock_detector() -> DeadlockDetector {
|
||||
DeadlockDetector::with_defaults()
|
||||
}
|
||||
|
||||
/// Create a new lock optimizer with default configuration.
|
||||
#[allow(dead_code)]
|
||||
pub fn create_lock_optimizer() -> LockOptimizer {
|
||||
LockOptimizer::with_defaults()
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,11 +18,14 @@ use std::sync::atomic::Ordering;
|
||||
use std::time::Instant;
|
||||
|
||||
use super::io_schedule::ACTIVE_GET_REQUESTS;
|
||||
use rustfs_io_metrics::{record_get_object_request_result, record_get_object_request_start};
|
||||
|
||||
/// RAII guard for tracking active GetObject requests.
|
||||
#[derive(Debug)]
|
||||
pub struct GetObjectGuard {
|
||||
start_time: Instant,
|
||||
/// Final status set by the caller; if None when dropped, reported as "unknown".
|
||||
result: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl GetObjectGuard {
|
||||
@@ -30,18 +33,37 @@ impl GetObjectGuard {
|
||||
pub fn new() -> Self {
|
||||
ACTIVE_GET_REQUESTS.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
#[cfg(all(feature = "metrics", not(test)))]
|
||||
if !std::thread::panicking() {
|
||||
use metrics::counter;
|
||||
counter!("rustfs.get.object.requests.started").increment(1);
|
||||
}
|
||||
// Record metrics for a started GetObject request. Capture the
|
||||
// concurrent request count AFTER increment to reflect the current
|
||||
// active requests.
|
||||
let concurrent = ACTIVE_GET_REQUESTS.load(Ordering::Relaxed);
|
||||
record_get_object_request_start(concurrent);
|
||||
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
result: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark the request as completed successfully.
|
||||
///
|
||||
/// Call this before the guard is dropped to record the correct status.
|
||||
pub fn finish_ok(&mut self) {
|
||||
self.result = Some("ok");
|
||||
}
|
||||
|
||||
/// Mark the request as failed.
|
||||
///
|
||||
/// Call this before the guard is dropped to record the correct status.
|
||||
pub fn finish_err(&mut self) {
|
||||
self.result = Some("error");
|
||||
}
|
||||
|
||||
/// Get the elapsed time since this guard was created.
|
||||
#[allow(dead_code)]
|
||||
// This helper is primarily used by unit tests to assert timing.
|
||||
// It's intentionally kept public for callers that may want to inspect
|
||||
// a guard's duration without dropping it.
|
||||
pub fn elapsed(&self) -> std::time::Duration {
|
||||
self.start_time.elapsed()
|
||||
}
|
||||
@@ -65,6 +87,15 @@ impl Default for GetObjectGuard {
|
||||
|
||||
impl Drop for GetObjectGuard {
|
||||
fn drop(&mut self) {
|
||||
// Record duration of this request before decrementing the global
|
||||
// counter. This ensures `start_time` is actually used and the
|
||||
// `elapsed()` method remains meaningful for tests and callers.
|
||||
let duration_secs = self.start_time.elapsed().as_secs_f64();
|
||||
// Use the caller-set status, or "unknown" if the result was never set
|
||||
// (e.g., the future was cancelled or the guard dropped without explicit completion).
|
||||
let status = self.result.unwrap_or("unknown");
|
||||
record_get_object_request_result(status, duration_secs);
|
||||
|
||||
if let Err(previous) =
|
||||
ACTIVE_GET_REQUESTS.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| current.checked_sub(1))
|
||||
{
|
||||
@@ -74,13 +105,6 @@ impl Drop for GetObjectGuard {
|
||||
previous
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "metrics", not(test)))]
|
||||
if !std::thread::panicking() {
|
||||
use metrics::{counter, histogram};
|
||||
counter!("rustfs.get.object.requests.completed").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(self.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -367,9 +367,14 @@ mod tests {
|
||||
async fn test_moka_cache_eviction() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Clear cache for clean test state
|
||||
manager.clear_cache().await;
|
||||
manager.reset_cache_metrics();
|
||||
|
||||
// Cache multiple objects to exceed the limit
|
||||
let object_size = 6 * MI_B; // 6MB each
|
||||
let num_objects = 20; // Total 120MB > 100MB limit
|
||||
// Tiered cache has L1 (50MB) + L2 (200MB) = 250MB total
|
||||
let object_size = 15 * MI_B; // 15MB each
|
||||
let num_objects = 20; // Total 300MB > 250MB limit
|
||||
|
||||
for i in 0..num_objects {
|
||||
let key = format!("test/object{i}");
|
||||
@@ -383,6 +388,7 @@ mod tests {
|
||||
|
||||
// Verify cache size is within limit (Moka manages this automatically)
|
||||
let stats = manager.cache_stats().await;
|
||||
eprintln!("DEBUG: size={}, max_size={}, entries={}", stats.size, stats.max_size, stats.entries);
|
||||
assert!(
|
||||
stats.size <= stats.max_size,
|
||||
"Moka should keep cache size {} within max {}",
|
||||
@@ -628,6 +634,10 @@ mod tests {
|
||||
async fn test_cache_hit_rate() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Reset metrics for clean test
|
||||
manager.reset_cache_metrics();
|
||||
manager.clear_cache().await;
|
||||
|
||||
// Cache some objects
|
||||
for i in 0..5 {
|
||||
let key = format!("hitrate/object{i}");
|
||||
@@ -637,6 +647,12 @@ mod tests {
|
||||
|
||||
sleep(Duration::from_millis(100)).await;
|
||||
|
||||
// Verify objects are cached
|
||||
for i in 0..5 {
|
||||
let key = format!("hitrate/object{i}");
|
||||
assert!(manager.is_cached(&key).await, "Object {} should be cached", key);
|
||||
}
|
||||
|
||||
// Mix of hits and misses
|
||||
for i in 0..10 {
|
||||
let key = if i < 5 {
|
||||
@@ -647,9 +663,9 @@ mod tests {
|
||||
let _ = manager.get_cached(&key).await;
|
||||
}
|
||||
|
||||
// Hit rate should be around 50%
|
||||
// Hit rate should be around 50% (0.5 on 0.0-1.0 scale)
|
||||
let hit_rate = manager.cache_hit_rate();
|
||||
assert!((40.0..=60.0).contains(&hit_rate), "Hit rate should be ~50%, got {hit_rate:.1}%");
|
||||
assert!((0.4..=0.6).contains(&hit_rate), "Hit rate should be ~50% (0.5), got {hit_rate:.3}");
|
||||
}
|
||||
|
||||
/// Test TTL expiration (Moka automatic cleanup)
|
||||
@@ -1029,6 +1045,9 @@ mod tests {
|
||||
async fn test_cache_invalidation_versioned() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Clear cache for clean test state
|
||||
manager.clear_cache().await;
|
||||
|
||||
let bucket = "bucket";
|
||||
let key = "object";
|
||||
let version_id = "v123";
|
||||
|
||||
@@ -17,6 +17,18 @@
|
||||
//! This module provides deadlock detection capabilities for diagnosing
|
||||
//! hanging requests and lock contention issues in production systems.
|
||||
//!
|
||||
//! # Migration Note
|
||||
//!
|
||||
//! This module extends `rustfs_io_core::DeadlockDetector` with request-level
|
||||
//! resource tracking (memory, file handles). For basic deadlock detection,
|
||||
//! consider using the io-core version directly:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! // Basic deadlock detection
|
||||
//! use rustfs_io_core::DeadlockDetector;
|
||||
//! let detector = DeadlockDetector::with_defaults();
|
||||
//! ```
|
||||
//!
|
||||
//! # Key Features
|
||||
//!
|
||||
//! - Request resource tracking (locks, memory, file handles)
|
||||
@@ -53,7 +65,6 @@ use std::time::{Duration, Instant};
|
||||
use tokio::sync::broadcast;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use metrics::counter;
|
||||
|
||||
/// Request identifier type.
|
||||
@@ -453,7 +464,6 @@ impl DeadlockDetector {
|
||||
if let Some(cycle) = Self::find_cycle(&wait_graph) {
|
||||
deadlocks_detected.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.deadlock.detected.total").increment(1);
|
||||
|
||||
// Log detailed deadlock information
|
||||
|
||||
@@ -187,7 +187,6 @@ pub(crate) fn get_buffer_size_opt_in(file_size: i64) -> usize {
|
||||
};
|
||||
|
||||
// Optional performance metrics collection for monitoring and optimization
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
use metrics::histogram;
|
||||
histogram!("rustfs.buffer.size.bytes").record(buffer_size as f64);
|
||||
|
||||
@@ -17,6 +17,18 @@
|
||||
//! This module provides optimized lock management for read operations,
|
||||
//! reducing lock contention by releasing locks early (after metadata read)
|
||||
//! rather than holding them for the entire data transfer duration.
|
||||
//!
|
||||
//! # Migration Note
|
||||
//!
|
||||
//! For new code, consider using `rustfs_io_core::LockOptimizer` which provides
|
||||
//! the same core functionality with better separation of concerns. This module
|
||||
//! remains for backward compatibility and storage-specific configuration.
|
||||
//!
|
||||
//! ```ignore
|
||||
//! // Recommended: Use io-core directly
|
||||
//! use rustfs_io_core::LockOptimizer;
|
||||
//! let optimizer = LockOptimizer::with_defaults();
|
||||
//! ```
|
||||
|
||||
// Allow dead_code for public API that may be used by external modules or future features
|
||||
#![allow(dead_code)]
|
||||
@@ -42,7 +54,6 @@ use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::debug;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use metrics::histogram;
|
||||
|
||||
/// Lock optimization configuration.
|
||||
@@ -216,7 +227,6 @@ impl<G> OptimizedLockGuard<G> {
|
||||
|
||||
self.stats.record_early_release(hold_time);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
histogram!("rustfs.lock.hold.duration.seconds").record(hold_time.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
@@ -241,7 +251,6 @@ impl<G> Drop for OptimizedLockGuard<G> {
|
||||
|
||||
self.stats.record_early_release(hold_time);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
histogram!("rustfs.lock.hold.duration.seconds").record(hold_time.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
|
||||
@@ -37,6 +37,8 @@ mod ecfs_extend;
|
||||
mod ecfs_test;
|
||||
pub(crate) mod head_prefix;
|
||||
#[cfg(test)]
|
||||
mod multi_factor_scheduler_integration_test;
|
||||
#[cfg(test)]
|
||||
mod sse_test;
|
||||
|
||||
pub(crate) use ecfs_extend::*;
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Integration tests for multi-factor I/O scheduler.
|
||||
//!
|
||||
//! These tests verify the enhanced scheduler behavior in realistic scenarios
|
||||
//! combining storage media, access patterns, bandwidth, and concurrency.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::storage::concurrency::ConcurrencyManager;
|
||||
use serial_test::serial;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Test scenario: NVMe sequential read with low load
|
||||
///
|
||||
/// Expected behavior: Maximum buffer size, readahead enabled
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_nvme_sequential_low_load() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
5 * 1024 * 1024, // 5MB file
|
||||
256 * 1024, // 256KB base buffer
|
||||
Duration::from_millis(5), // Low load
|
||||
true, // Sequential
|
||||
);
|
||||
|
||||
// Verify basic strategy properties
|
||||
assert!(strategy.buffer_size > 0);
|
||||
assert_eq!(strategy.load_level.level_index(), 0); // Low
|
||||
}
|
||||
|
||||
/// Test scenario: High concurrency reduces buffer
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_high_concurrency() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Low concurrency
|
||||
let low_strategy = {
|
||||
let _g1 = ConcurrencyManager::track_request();
|
||||
let _g2 = ConcurrencyManager::track_request();
|
||||
manager.calculate_io_strategy_with_context(50 * 1024 * 1024, 512 * 1024, Duration::from_millis(10), true)
|
||||
};
|
||||
|
||||
// High concurrency
|
||||
let high_strategy = {
|
||||
let _guards: Vec<_> = (0..16).map(|_| ConcurrencyManager::track_request()).collect();
|
||||
manager.calculate_io_strategy_with_context(50 * 1024 * 1024, 512 * 1024, Duration::from_millis(10), true)
|
||||
};
|
||||
|
||||
// Buffer should decrease with higher concurrency
|
||||
assert!(high_strategy.concurrent_requests >= low_strategy.concurrent_requests);
|
||||
}
|
||||
|
||||
/// Test scenario: Progressive load increase
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_progressive_load() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let file_size = 50 * 1024 * 1024;
|
||||
let base_buffer = 512 * 1024;
|
||||
|
||||
// Low load
|
||||
let low_strategy = manager.calculate_io_strategy_with_context(file_size, base_buffer, Duration::from_millis(5), true);
|
||||
|
||||
// High load
|
||||
let high_strategy = manager.calculate_io_strategy_with_context(file_size, base_buffer, Duration::from_millis(100), true);
|
||||
|
||||
// Critical load
|
||||
let critical_strategy =
|
||||
manager.calculate_io_strategy_with_context(file_size, base_buffer, Duration::from_millis(300), true);
|
||||
|
||||
// Load levels should increase
|
||||
assert!(low_strategy.load_level.level_index() < high_strategy.load_level.level_index());
|
||||
assert!(high_strategy.load_level.level_index() < critical_strategy.load_level.level_index());
|
||||
|
||||
// Readahead should be disabled at critical load
|
||||
assert!(!critical_strategy.enable_readahead);
|
||||
}
|
||||
|
||||
/// Test scenario: Small file gets high priority
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_small_file_priority() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
100 * 1024, // 100KB (small)
|
||||
256 * 1024,
|
||||
Duration::from_millis(100), // Even under high load
|
||||
false,
|
||||
);
|
||||
|
||||
// Should be high priority due to size
|
||||
assert!(strategy.priority.is_high());
|
||||
}
|
||||
|
||||
/// Test scenario: Large file gets low priority
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_large_file_priority() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
100 * 1024 * 1024, // 100MB (large)
|
||||
256 * 1024,
|
||||
Duration::from_millis(5), // Even under low load
|
||||
false,
|
||||
);
|
||||
|
||||
// Should be low priority due to size
|
||||
assert!(strategy.priority.is_low());
|
||||
}
|
||||
|
||||
/// Test scenario: Access pattern tracking
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_access_pattern_tracking() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Record sequential accesses
|
||||
for offset in [0, 1024, 2048, 3072, 4096] {
|
||||
manager.record_access(offset, 1024);
|
||||
}
|
||||
|
||||
// Should detect sequential pattern
|
||||
let pattern = manager.current_access_pattern();
|
||||
assert!(pattern.is_sequential() || pattern.is_unknown());
|
||||
}
|
||||
|
||||
/// Test scenario: Bandwidth recording
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_bandwidth_recording() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
// Record transfer
|
||||
manager.record_transfer(10 * 1024 * 1024, Duration::from_millis(100));
|
||||
|
||||
// Bandwidth snapshot should be available (returns BandwidthSnapshot directly)
|
||||
let snapshot = manager.current_bandwidth_snapshot();
|
||||
assert!(snapshot.bytes_per_second > 0);
|
||||
}
|
||||
|
||||
/// Test scenario: Sequential vs random comparison
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_scenario_sequential_vs_random() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let file_size = 50 * 1024 * 1024;
|
||||
let base_buffer = 512 * 1024;
|
||||
let wait = Duration::from_millis(20);
|
||||
|
||||
let sequential_strategy = manager.calculate_io_strategy_with_context(file_size, base_buffer, wait, true);
|
||||
|
||||
let random_strategy = manager.calculate_io_strategy_with_context(file_size, base_buffer, wait, false);
|
||||
|
||||
// Sequential should get better (or equal) treatment
|
||||
assert!(sequential_strategy.buffer_size >= random_strategy.buffer_size);
|
||||
}
|
||||
|
||||
/// Test scenario: Real-world video streaming
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_real_world_video_streaming() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
500 * 1024 * 1024, // 500MB video
|
||||
512 * 1024,
|
||||
Duration::from_millis(25),
|
||||
true, // Sequential streaming
|
||||
);
|
||||
|
||||
// Should be optimized for streaming
|
||||
assert!(strategy.buffer_size > 0);
|
||||
assert_eq!(strategy.load_level.level_index(), 1); // Medium load
|
||||
}
|
||||
|
||||
/// Test scenario: Real-world API config files
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn test_real_world_api_configs() {
|
||||
let manager = ConcurrencyManager::new();
|
||||
|
||||
let strategy = manager.calculate_io_strategy_with_context(
|
||||
100 * 1024, // 100KB JSON
|
||||
256 * 1024,
|
||||
Duration::from_millis(5),
|
||||
false, // Random access to different files
|
||||
);
|
||||
|
||||
// Should optimize for low latency
|
||||
assert!(strategy.priority.is_high());
|
||||
assert_eq!(strategy.load_level.level_index(), 0); // Low load
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,25 @@
|
||||
//!
|
||||
//! This module provides timeout protection for GetObject requests to prevent
|
||||
//! indefinite hangs caused by deadlocks, resource exhaustion, or slow I/O.
|
||||
//!
|
||||
//! # Migration Note
|
||||
//!
|
||||
//! This module extends `rustfs_io_core::RequestTimeoutWrapper` with Tokio
|
||||
//! cancellation token support. For basic timeout handling without async
|
||||
//! cancellation, consider using the io-core version:
|
||||
//!
|
||||
//! ```ignore
|
||||
//! // Basic timeout handling
|
||||
//! use rustfs_io_core::RequestTimeoutWrapper;
|
||||
//! let wrapper = RequestTimeoutWrapper::new(config);
|
||||
//! ```
|
||||
//!
|
||||
//! # Key Features
|
||||
//!
|
||||
//! - Configurable request-level timeout (default 30 seconds)
|
||||
//! - Automatic cancellation of sub-tasks on timeout
|
||||
//! - Resource cleanup on timeout (locks, memory, file handles)
|
||||
//! - Prometheus metrics for timeout monitoring
|
||||
|
||||
// Allow dead_code for public API that may be used by external modules or future features
|
||||
#![allow(dead_code)]
|
||||
@@ -47,8 +66,7 @@ use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use metrics::{counter, histogram};
|
||||
// Re-export types from rustfs_io_core for convenience
|
||||
|
||||
/// Timeout configuration for GetObject requests.
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -190,66 +208,6 @@ pub struct TimeoutInfo {
|
||||
pub progress_percent: Option<f32>,
|
||||
}
|
||||
|
||||
/// Progress tracking for long-running operations
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OperationProgress {
|
||||
/// Start time
|
||||
start_time: Instant,
|
||||
/// Last progress update time
|
||||
last_update: Instant,
|
||||
/// Bytes transferred so far
|
||||
bytes_transferred: u64,
|
||||
/// Total object size (if known)
|
||||
total_size: Option<u64>,
|
||||
/// Stale timeout - if no progress for this duration, consider stuck
|
||||
stale_timeout: Duration,
|
||||
}
|
||||
|
||||
impl OperationProgress {
|
||||
/// Create a new progress tracker
|
||||
pub fn new(total_size: Option<u64>, stale_timeout: Duration) -> Self {
|
||||
Self {
|
||||
start_time: Instant::now(),
|
||||
last_update: Instant::now(),
|
||||
bytes_transferred: 0,
|
||||
total_size,
|
||||
stale_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Update progress with new bytes transferred
|
||||
pub fn update(&mut self, bytes: u64) {
|
||||
self.bytes_transferred = bytes;
|
||||
self.last_update = Instant::now();
|
||||
}
|
||||
|
||||
/// Check if progress is stale (no updates for stale_timeout)
|
||||
pub fn is_stale(&self) -> bool {
|
||||
self.last_update.elapsed() > self.stale_timeout
|
||||
}
|
||||
|
||||
/// Get progress percentage (0-100)
|
||||
pub fn progress_percent(&self) -> Option<f32> {
|
||||
self.total_size.map(|total| {
|
||||
if total == 0 {
|
||||
100.0
|
||||
} else {
|
||||
(self.bytes_transferred as f32 / total as f32 * 100.0).min(100.0)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Get transfer rate in bytes per second
|
||||
pub fn transfer_rate(&self) -> u64 {
|
||||
let elapsed = self.start_time.elapsed().as_secs_f64();
|
||||
if elapsed > 0.0 {
|
||||
(self.bytes_transferred as f64 / elapsed) as u64
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a timed GetObject operation.
|
||||
#[derive(Debug)]
|
||||
pub enum TimedGetObjectResult<T, E> {
|
||||
@@ -405,8 +363,7 @@ impl RequestTimeoutWrapper {
|
||||
);
|
||||
|
||||
// Record start time for metrics
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.get.object.requests.started").increment(1);
|
||||
rustfs_io_metrics::record_get_object_request_started();
|
||||
|
||||
// Clone cancel_token for the operation, keep original for potential cancellation
|
||||
let cancel_token_for_op = self.cancel_token.clone();
|
||||
@@ -416,11 +373,7 @@ impl RequestTimeoutWrapper {
|
||||
// Operation completed successfully
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.requests.completed").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
request_id = %request_id,
|
||||
@@ -434,11 +387,7 @@ impl RequestTimeoutWrapper {
|
||||
// Operation failed before timeout
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.requests.failed").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
request_id = %request_id,
|
||||
@@ -455,11 +404,8 @@ impl RequestTimeoutWrapper {
|
||||
// Cancel the operation
|
||||
self.cancel_token.cancel();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.timeout.total").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64()));
|
||||
rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64());
|
||||
|
||||
warn!(
|
||||
request_id = %request_id,
|
||||
@@ -527,8 +473,7 @@ impl RequestTimeoutWrapper {
|
||||
"Starting timed operation"
|
||||
);
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
counter!("rustfs.get.object.requests.started").increment(1);
|
||||
rustfs_io_metrics::record_get_object_request_started();
|
||||
|
||||
// Clone cancel_token for the operation, keep original for potential cancellation
|
||||
let cancel_token_for_op = self.cancel_token.clone();
|
||||
@@ -537,11 +482,7 @@ impl RequestTimeoutWrapper {
|
||||
Ok(Ok(result)) => {
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.requests.completed").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_request_result("success", elapsed.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
request_id = %request_id,
|
||||
@@ -556,11 +497,7 @@ impl RequestTimeoutWrapper {
|
||||
Ok(Err(e)) => {
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.requests.failed").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_request_result("error", elapsed.as_secs_f64());
|
||||
|
||||
debug!(
|
||||
request_id = %request_id,
|
||||
@@ -576,11 +513,8 @@ impl RequestTimeoutWrapper {
|
||||
let elapsed = start_time.elapsed();
|
||||
self.cancel_token.cancel();
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
counter!("rustfs.get.object.timeout.total").increment(1);
|
||||
histogram!("rustfs.get.object.duration.seconds").record(elapsed.as_secs_f64());
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_timeout(None, Some(elapsed.as_secs_f64()));
|
||||
rustfs_io_metrics::record_get_object_request_result("timeout", elapsed.as_secs_f64());
|
||||
|
||||
warn!(
|
||||
request_id = %request_id,
|
||||
@@ -622,130 +556,6 @@ pub fn get_io_buffer_size() -> usize {
|
||||
rustfs_utils::get_env_usize(rustfs_config::ENV_OBJECT_IO_BUFFER_SIZE, rustfs_config::DEFAULT_OBJECT_IO_BUFFER_SIZE)
|
||||
}
|
||||
|
||||
/// Calculate adaptive timeout based on historical performance
|
||||
///
|
||||
/// This function adjusts timeout based on:
|
||||
/// - Historical transfer rates
|
||||
/// - Recent timeout occurrences
|
||||
/// - System load indicators
|
||||
pub fn calculate_adaptive_timeout(
|
||||
base_timeout: Duration,
|
||||
historical_rate_bps: Option<u64>,
|
||||
recent_timeout_count: u32,
|
||||
object_size: u64,
|
||||
) -> Duration {
|
||||
// If we have recent timeouts, increase timeout
|
||||
let timeout_multiplier = if recent_timeout_count > 3 {
|
||||
2.0 // Double timeout if many recent timeouts
|
||||
} else if recent_timeout_count > 1 {
|
||||
1.5 // 50% increase if some timeouts
|
||||
} else {
|
||||
1.0 // No adjustment
|
||||
};
|
||||
|
||||
// If we have historical rate data, use it for estimation
|
||||
let estimated_duration = if let Some(rate) = historical_rate_bps {
|
||||
if rate > 0 {
|
||||
let estimated_secs = (object_size as f64 / rate as f64) * 1.2; // 20% buffer
|
||||
Duration::from_secs_f64(estimated_secs)
|
||||
} else {
|
||||
base_timeout
|
||||
}
|
||||
} else {
|
||||
base_timeout
|
||||
};
|
||||
|
||||
// Apply timeout multiplier but clamp to reasonable bounds
|
||||
let adaptive_duration = Duration::from_secs_f64(estimated_duration.as_secs_f64() * timeout_multiplier);
|
||||
|
||||
// Clamp to 5 seconds minimum and 10 minutes maximum
|
||||
adaptive_duration.max(Duration::from_secs(5)).min(Duration::from_secs(600))
|
||||
}
|
||||
|
||||
/// Estimate bytes per second for timeout calculation
|
||||
///
|
||||
/// Uses a conservative estimate to avoid premature timeouts
|
||||
pub fn estimate_bytes_per_second(object_size: u64, expected_duration: Duration) -> u64 {
|
||||
let secs = expected_duration.as_secs_f64();
|
||||
if secs > 0.0 {
|
||||
(object_size as f64 / secs) as u64
|
||||
} else {
|
||||
rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod adaptive_timeout_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_calculate_adaptive_timeout_basic() {
|
||||
let base_timeout = Duration::from_secs(30);
|
||||
let adaptive = calculate_adaptive_timeout(base_timeout, None, 0, 1024 * 1024);
|
||||
|
||||
// Should return base timeout when no historical data
|
||||
assert_eq!(adaptive, base_timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_calculate_adaptive_timeout_with_history() {
|
||||
let base_timeout = Duration::from_secs(30);
|
||||
let historical_rate = 2 * 1024 * 1024; // 2 MB/s
|
||||
let object_size = 10 * 1024 * 1024; // 10 MB
|
||||
|
||||
let adaptive = calculate_adaptive_timeout(base_timeout, Some(historical_rate), 0, object_size);
|
||||
|
||||
// With 2 MB/s, 10 MB should take ~5 seconds + 20% buffer = 6 seconds
|
||||
assert!(adaptive >= Duration::from_secs(5));
|
||||
assert!(adaptive <= Duration::from_secs(10));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_calculate_adaptive_timeout_with_recent_timeouts() {
|
||||
let base_timeout = Duration::from_secs(30);
|
||||
|
||||
// No timeouts
|
||||
let adaptive1 = calculate_adaptive_timeout(base_timeout, None, 0, 1024 * 1024);
|
||||
assert_eq!(adaptive1, base_timeout);
|
||||
|
||||
// Some timeouts (2 timeouts -> 1.5x multiplier -> 30 * 1.5 = 45 seconds)
|
||||
let adaptive2 = calculate_adaptive_timeout(base_timeout, None, 2, 1024 * 1024);
|
||||
assert!(adaptive2 > base_timeout);
|
||||
assert!(adaptive2 <= Duration::from_secs(45)); // Changed from < to <=
|
||||
|
||||
// Many timeouts
|
||||
let adaptive3 = calculate_adaptive_timeout(base_timeout, None, 5, 1024 * 1024);
|
||||
assert!(adaptive3 >= base_timeout * 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_calculate_adaptive_timeout_clamping() {
|
||||
let base_timeout = Duration::from_secs(1);
|
||||
let adaptive = calculate_adaptive_timeout(base_timeout, None, 10, 1024 * 1024);
|
||||
|
||||
// Should clamp to minimum of 5 seconds
|
||||
assert!(adaptive >= Duration::from_secs(5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_bytes_per_second() {
|
||||
let object_size = 10 * 1024 * 1024; // 10 MB
|
||||
let duration = Duration::from_secs(10);
|
||||
|
||||
let bps = estimate_bytes_per_second(object_size, duration);
|
||||
assert_eq!(bps, 1024 * 1024); // 1 MB/s
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_estimate_bytes_per_second_zero_duration() {
|
||||
let object_size = 1024;
|
||||
let duration = Duration::from_secs(0);
|
||||
|
||||
let bps = estimate_bytes_per_second(object_size, duration);
|
||||
assert_eq!(bps, rustfs_config::DEFAULT_OBJECT_BYTES_PER_SECOND);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -908,32 +718,23 @@ mod tests {
|
||||
assert_eq!(timeout1, config.get_object_timeout);
|
||||
assert_eq!(timeout2, config.get_object_timeout);
|
||||
}
|
||||
|
||||
use rustfs_concurrency::OperationProgress;
|
||||
#[test]
|
||||
fn test_operation_progress_new() {
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
assert_eq!(progress.bytes_transferred, 0);
|
||||
assert_eq!(progress.total_size, Some(1000));
|
||||
assert!(!progress.is_stale());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operation_progress_update() {
|
||||
let mut progress = OperationProgress::new(Some(1000), Duration::from_secs(5));
|
||||
|
||||
assert_eq!(progress.current(), 0);
|
||||
progress.update(500);
|
||||
assert_eq!(progress.bytes_transferred, 500);
|
||||
assert_eq!(progress.current(), 500);
|
||||
assert!(!progress.is_stale());
|
||||
|
||||
// Simulate time passing
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
progress.update(1000);
|
||||
assert_eq!(progress.bytes_transferred, 1000);
|
||||
assert_eq!(progress.current(), 1000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_operation_progress_stale() {
|
||||
let mut progress = OperationProgress::new(Some(1000), Duration::from_millis(100));
|
||||
let progress = OperationProgress::new(Some(1000), Duration::from_millis(100));
|
||||
|
||||
progress.update(500);
|
||||
assert!(!progress.is_stale());
|
||||
@@ -953,7 +754,6 @@ mod tests {
|
||||
|
||||
assert_eq!(progress.progress_percent(), Some(0.0));
|
||||
|
||||
let mut progress = progress;
|
||||
progress.update(500);
|
||||
assert_eq!(progress.progress_percent(), Some(50.0));
|
||||
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# Concurrent Download Tool (tests)
|
||||
|
||||
This tool downloads multiple URLs concurrently and saves files to a target directory.
|
||||
|
||||
Saved filename format:
|
||||
|
||||
`<nanoseconds>_<index>_<original_filename>`
|
||||
|
||||
All downloaded files are written into one output directory.
|
||||
|
||||
## Environment variables
|
||||
|
||||
- `DOWNLOAD_URLS` (required): comma-separated URLs.
|
||||
- `DOWNLOAD_OUTPUT_DIR` (optional): output directory, default `target/tmp/concurrent_downloads`.
|
||||
- `DOWNLOAD_CONCURRENCY` (optional): max concurrent downloads, default `8`.
|
||||
- `DOWNLOAD_REPEAT` (optional): repeat count per URL, default `1`.
|
||||
- `DOWNLOAD_MAX_RETRIES` (optional): retry count per task after first failure, default `0`.
|
||||
- `DOWNLOAD_RETRY_BACKOFF_MS` (optional): fixed backoff between retries, default `200`.
|
||||
|
||||
## Statistics output
|
||||
|
||||
After run, the tool prints:
|
||||
|
||||
- total tasks
|
||||
- succeeded
|
||||
- failed
|
||||
- total bytes
|
||||
- elapsed ms
|
||||
- throughput bps
|
||||
- total attempts
|
||||
- retried tasks
|
||||
- retry attempts
|
||||
- latency p50 ms
|
||||
- latency p95 ms
|
||||
- failure details (`[index] url => error`) when failures exist
|
||||
|
||||
If any task fails, the test returns error after printing the summary.
|
||||
|
||||
Retry is triggered only for recoverable cases:
|
||||
|
||||
- network/request timeout/connect errors
|
||||
- HTTP `429`
|
||||
- HTTP `5xx`
|
||||
|
||||
## Compile check
|
||||
|
||||
```bash
|
||||
cargo test -p rustfs --test concurrent_download_tool --no-run
|
||||
```
|
||||
|
||||
## Manual run example
|
||||
|
||||
The commands below are for manual execution only.
|
||||
They are not part of automated test runs.
|
||||
|
||||
```bash
|
||||
DOWNLOAD_URLS="http://127.0.0.1:9001/demo/google-cloud-aiplugin-1.46.1-253.zip?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=HAXVOTZK9MLBJT8KWI4E%2F20260329%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260329T105159Z&X-Amz-Expires=86400&X-Amz-Security-Token=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJwYXJlbnQiOiJydXN0ZnNhZG1pbiIsImV4cCI6MTc3NDgyMDgyMX0.tYhQoPRcg0Ysx4KVw9ez7ZpYxsqGgqomtsP_iaeTsKzoii8EVNt74BZm2wbUjXW-FbGXc1pqEYX6wZ5Ncpk9Iw&X-Amz-Signature=15f47b19832f53b34f9e0fe1862d53d71660bbf8f1a512669bb2d041ac8d0697&X-Amz-SignedHeaders=host&x-amz-checksum-mode=ENABLED&x-id=GetObject" \
|
||||
DOWNLOAD_OUTPUT_DIR="/Users/zhi/Documents/code/rust/rustfs/rustfs/target/tmp/concurrent_downloads" \
|
||||
DOWNLOAD_CONCURRENCY="40" \
|
||||
DOWNLOAD_REPEAT="40" \
|
||||
DOWNLOAD_MAX_RETRIES="2" \
|
||||
DOWNLOAD_RETRY_BACKOFF_MS="300" \
|
||||
cargo test -p rustfs --test concurrent_download_tool -- --ignored --nocapture
|
||||
```
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use futures::stream::{self, StreamExt};
|
||||
use reqwest::{Client, Url};
|
||||
use std::env;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Instant, SystemTime, UNIX_EPOCH};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DownloadSettings {
|
||||
urls: Vec<String>,
|
||||
output_dir: PathBuf,
|
||||
concurrency: usize,
|
||||
repeat: usize,
|
||||
max_retries: usize,
|
||||
retry_backoff_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DownloadSuccess {
|
||||
path: PathBuf,
|
||||
bytes: usize,
|
||||
attempts_used: usize,
|
||||
elapsed_ms: u128,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DownloadAttemptError {
|
||||
attempts_used: usize,
|
||||
error: String,
|
||||
elapsed_ms: u128,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DownloadFailure {
|
||||
index: usize,
|
||||
url: String,
|
||||
attempts_used: usize,
|
||||
error: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DownloadSummary {
|
||||
saved_files: Vec<PathBuf>,
|
||||
total_tasks: usize,
|
||||
succeeded: usize,
|
||||
failed: usize,
|
||||
total_bytes: usize,
|
||||
elapsed_ms: u128,
|
||||
throughput_bps: f64,
|
||||
total_attempts: usize,
|
||||
retried_tasks: usize,
|
||||
retry_attempts: usize,
|
||||
latency_p50_ms: u128,
|
||||
latency_p95_ms: u128,
|
||||
failures: Vec<DownloadFailure>,
|
||||
}
|
||||
|
||||
fn should_retry_status(status: reqwest::StatusCode) -> bool {
|
||||
status.as_u16() == 429 || status.is_server_error()
|
||||
}
|
||||
|
||||
fn should_retry_reqwest_error(err: &reqwest::Error) -> bool {
|
||||
if err.is_timeout() || err.is_connect() || err.is_request() {
|
||||
return true;
|
||||
}
|
||||
|
||||
match err.status() {
|
||||
Some(status) => should_retry_status(status),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn percentile(values: &[u128], p: f64) -> u128 {
|
||||
if values.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut sorted = values.to_vec();
|
||||
sorted.sort_unstable();
|
||||
|
||||
let rank = ((sorted.len() as f64 - 1.0) * p).round() as usize;
|
||||
sorted[rank]
|
||||
}
|
||||
|
||||
impl DownloadSettings {
|
||||
fn from_env() -> Result<Self> {
|
||||
let urls_raw = env::var("DOWNLOAD_URLS").context("missing DOWNLOAD_URLS, expected comma-separated URLs")?;
|
||||
|
||||
let urls: Vec<String> = urls_raw
|
||||
.split(',')
|
||||
.map(str::trim)
|
||||
.filter(|v| !v.is_empty())
|
||||
.map(ToString::to_string)
|
||||
.collect();
|
||||
|
||||
if urls.is_empty() {
|
||||
return Err(anyhow!("DOWNLOAD_URLS is empty, expected comma-separated URLs"));
|
||||
}
|
||||
|
||||
let output_dir = env::var("DOWNLOAD_OUTPUT_DIR")
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|_| PathBuf::from("target/tmp/concurrent_downloads"));
|
||||
|
||||
let concurrency = env::var("DOWNLOAD_CONCURRENCY")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.filter(|v| *v > 0)
|
||||
.unwrap_or(8);
|
||||
|
||||
let repeat = env::var("DOWNLOAD_REPEAT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.filter(|v| *v > 0)
|
||||
.unwrap_or(1);
|
||||
|
||||
let max_retries = env::var("DOWNLOAD_MAX_RETRIES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<usize>().ok())
|
||||
.unwrap_or(0);
|
||||
|
||||
let retry_backoff_ms = env::var("DOWNLOAD_RETRY_BACKOFF_MS")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.filter(|v| *v > 0)
|
||||
.unwrap_or(200);
|
||||
|
||||
Ok(Self {
|
||||
urls,
|
||||
output_dir,
|
||||
concurrency,
|
||||
repeat,
|
||||
max_retries,
|
||||
retry_backoff_ms,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn original_filename(url: &str) -> String {
|
||||
Url::parse(url)
|
||||
.ok()
|
||||
.and_then(|parsed| {
|
||||
parsed
|
||||
.path_segments()
|
||||
.and_then(|mut segments| segments.rfind(|s| !s.is_empty()))
|
||||
.map(ToString::to_string)
|
||||
})
|
||||
.filter(|name| !name.is_empty())
|
||||
.unwrap_or_else(|| "download.bin".to_string())
|
||||
}
|
||||
|
||||
fn nanos_prefix() -> Result<u128> {
|
||||
Ok(SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.context("system clock is before UNIX_EPOCH")?
|
||||
.as_nanos())
|
||||
}
|
||||
|
||||
async fn download_one(
|
||||
client: &Client,
|
||||
output_dir: &Path,
|
||||
index: usize,
|
||||
url: String,
|
||||
max_retries: usize,
|
||||
retry_backoff_ms: u64,
|
||||
) -> std::result::Result<DownloadSuccess, DownloadAttemptError> {
|
||||
let task_started_at = Instant::now();
|
||||
let mut attempt = 0usize;
|
||||
let mut last_error = String::new();
|
||||
let mut retryable = false;
|
||||
|
||||
while attempt <= max_retries {
|
||||
attempt += 1;
|
||||
|
||||
let response = match client.get(&url).send().await {
|
||||
Ok(resp) => resp,
|
||||
Err(err) => {
|
||||
retryable = should_retry_reqwest_error(&err);
|
||||
last_error = format!("failed request: {url}, error: {err}");
|
||||
if retryable && attempt <= max_retries {
|
||||
sleep(Duration::from_millis(retry_backoff_ms)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
retryable = should_retry_status(status);
|
||||
last_error = format!("non-success status for URL: {url}, status: {status}");
|
||||
if retryable && attempt <= max_retries {
|
||||
sleep(Duration::from_millis(retry_backoff_ms)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
let body = match response.bytes().await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(err) => {
|
||||
retryable = should_retry_reqwest_error(&err);
|
||||
last_error = format!("failed to read response body: {url}, error: {err}");
|
||||
if retryable && attempt <= max_retries {
|
||||
sleep(Duration::from_millis(retry_backoff_ms)).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let source_name = original_filename(&url);
|
||||
let nanos = match nanos_prefix() {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
last_error = err.to_string();
|
||||
retryable = false;
|
||||
break;
|
||||
}
|
||||
};
|
||||
let target_name = format!("{}_{}_{}", nanos, index, source_name);
|
||||
let target_path = output_dir.join(target_name);
|
||||
|
||||
let result: Result<DownloadSuccess> = async {
|
||||
tokio::fs::write(&target_path, &body)
|
||||
.await
|
||||
.with_context(|| format!("failed to write file: {}", target_path.display()))?;
|
||||
|
||||
Ok(DownloadSuccess {
|
||||
path: target_path,
|
||||
bytes: body.len(),
|
||||
attempts_used: attempt,
|
||||
elapsed_ms: task_started_at.elapsed().as_millis(),
|
||||
})
|
||||
}
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(success) => return Ok(success),
|
||||
Err(err) => {
|
||||
last_error = err.to_string();
|
||||
retryable = false;
|
||||
if retryable && attempt <= max_retries {
|
||||
sleep(Duration::from_millis(retry_backoff_ms)).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(DownloadAttemptError {
|
||||
attempts_used: attempt,
|
||||
error: if retryable {
|
||||
last_error
|
||||
} else {
|
||||
format!("{} (non-retryable)", last_error)
|
||||
},
|
||||
elapsed_ms: task_started_at.elapsed().as_millis(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn run_concurrent_downloads(settings: DownloadSettings) -> Result<DownloadSummary> {
|
||||
let started_at = Instant::now();
|
||||
|
||||
tokio::fs::create_dir_all(&settings.output_dir)
|
||||
.await
|
||||
.with_context(|| format!("failed to create output dir: {}", settings.output_dir.display()))?;
|
||||
|
||||
let client = Client::new();
|
||||
let tasks = settings
|
||||
.urls
|
||||
.into_iter()
|
||||
.flat_map(|url| (0..settings.repeat).map(move |_| url.clone()))
|
||||
.enumerate();
|
||||
|
||||
let results = stream::iter(tasks)
|
||||
.map(|(index, url)| {
|
||||
let client = client.clone();
|
||||
let output_dir = settings.output_dir.clone();
|
||||
let max_retries = settings.max_retries;
|
||||
let retry_backoff_ms = settings.retry_backoff_ms;
|
||||
async move {
|
||||
let current_url = url.clone();
|
||||
let result = download_one(&client, &output_dir, index, url, max_retries, retry_backoff_ms).await;
|
||||
(index, current_url, result)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(settings.concurrency)
|
||||
.collect::<Vec<(usize, String, std::result::Result<DownloadSuccess, DownloadAttemptError>)>>()
|
||||
.await;
|
||||
|
||||
let mut saved_files = Vec::new();
|
||||
let mut total_bytes = 0usize;
|
||||
let mut total_attempts = 0usize;
|
||||
let mut retried_tasks = 0usize;
|
||||
let mut latencies_ms = Vec::new();
|
||||
let mut failures = Vec::new();
|
||||
|
||||
for (index, url, item) in results {
|
||||
match item {
|
||||
Ok(success) => {
|
||||
total_bytes += success.bytes;
|
||||
total_attempts += success.attempts_used;
|
||||
if success.attempts_used > 1 {
|
||||
retried_tasks += 1;
|
||||
}
|
||||
latencies_ms.push(success.elapsed_ms);
|
||||
saved_files.push(success.path);
|
||||
}
|
||||
Err(err) => {
|
||||
total_attempts += err.attempts_used;
|
||||
if err.attempts_used > 1 {
|
||||
retried_tasks += 1;
|
||||
}
|
||||
latencies_ms.push(err.elapsed_ms);
|
||||
failures.push(DownloadFailure {
|
||||
index,
|
||||
url,
|
||||
attempts_used: err.attempts_used,
|
||||
error: err.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_tasks = saved_files.len() + failures.len();
|
||||
let retry_attempts = total_attempts.saturating_sub(total_tasks);
|
||||
let elapsed_ms = started_at.elapsed().as_millis();
|
||||
let throughput_bps = if elapsed_ms == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(total_bytes as f64) / ((elapsed_ms as f64) / 1000.0)
|
||||
};
|
||||
let latency_p50_ms = percentile(&latencies_ms, 0.50);
|
||||
let latency_p95_ms = percentile(&latencies_ms, 0.95);
|
||||
|
||||
Ok(DownloadSummary {
|
||||
total_tasks,
|
||||
succeeded: saved_files.len(),
|
||||
failed: failures.len(),
|
||||
total_bytes,
|
||||
elapsed_ms,
|
||||
throughput_bps,
|
||||
total_attempts,
|
||||
retried_tasks,
|
||||
retry_attempts,
|
||||
latency_p50_ms,
|
||||
latency_p95_ms,
|
||||
saved_files,
|
||||
failures,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn concurrent_download_tool() -> Result<()> {
|
||||
let settings = DownloadSettings::from_env()?;
|
||||
let summary = run_concurrent_downloads(settings).await?;
|
||||
|
||||
for path in &summary.saved_files {
|
||||
println!("saved: {}", path.display());
|
||||
}
|
||||
|
||||
println!("download complete");
|
||||
println!("total tasks: {}", summary.total_tasks);
|
||||
println!("succeeded: {}", summary.succeeded);
|
||||
println!("failed: {}", summary.failed);
|
||||
println!("total bytes: {}", summary.total_bytes);
|
||||
println!("elapsed ms: {}", summary.elapsed_ms);
|
||||
println!("throughput bps: {:.2}", summary.throughput_bps);
|
||||
println!("total attempts: {}", summary.total_attempts);
|
||||
println!("retried tasks: {}", summary.retried_tasks);
|
||||
println!("retry attempts: {}", summary.retry_attempts);
|
||||
println!("latency p50 ms: {}", summary.latency_p50_ms);
|
||||
println!("latency p95 ms: {}", summary.latency_p95_ms);
|
||||
|
||||
if !summary.failures.is_empty() {
|
||||
println!("failure details:");
|
||||
for failure in &summary.failures {
|
||||
println!(
|
||||
" [{}] attempts={} {} => {}",
|
||||
failure.index, failure.attempts_used, failure.url, failure.error
|
||||
);
|
||||
}
|
||||
|
||||
return Err(anyhow!("download finished with {} failures", summary.failures.len()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# Manual test runners
|
||||
|
||||
Files in this directory are for manual execution only.
|
||||
They are not auto-discovered as integration tests by `cargo test`.
|
||||
|
||||
## Dial9 runner
|
||||
|
||||
Build:
|
||||
|
||||
```bash
|
||||
cargo build -p rustfs --features manual-test-runners --bin manual-test-dial9
|
||||
```
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
cargo run -p rustfs --features manual-test-runners --bin manual-test-dial9
|
||||
```
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
// Manual Dial9 integration runner.
|
||||
//
|
||||
// Run with:
|
||||
// `cargo run -p rustfs --features manual-test-runners --bin manual-test-dial9`
|
||||
//
|
||||
// This file lives under `rustfs/tests/manual` and is registered explicitly in
|
||||
// `rustfs/Cargo.toml` so it stays out of `cargo test` auto-discovery.
|
||||
use rustfs_obs::dial9::{Dial9Config, Dial9SessionGuard};
|
||||
use tokio::time::{Duration, sleep};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
println!("=== Dial9 Integration Test ===\n");
|
||||
|
||||
// Test 1: Check default dial9 configuration
|
||||
println!("Test 1: Default configuration");
|
||||
let default_config = Dial9Config::default();
|
||||
println!(" default enabled: {}", default_config.enabled);
|
||||
println!(" default output_dir: {}", default_config.output_dir);
|
||||
println!(" default file_prefix: {}", default_config.file_prefix);
|
||||
println!(" ✓ PASS: Default configuration loaded\n");
|
||||
|
||||
// Test 2: Create explicit dial9 configuration
|
||||
println!("Test 2: Explicit dial9 configuration");
|
||||
let config = Dial9Config {
|
||||
enabled: true,
|
||||
output_dir: "/tmp/rustfs-test-telemetry".to_string(),
|
||||
sampling_rate: 0.5,
|
||||
..Dial9Config::default()
|
||||
};
|
||||
println!(" config.enabled: {}", config.enabled);
|
||||
println!(" config.output_dir: {}", config.output_dir);
|
||||
println!(" config.file_prefix: {}", config.file_prefix);
|
||||
println!(" config.sampling_rate: {}", config.sampling_rate);
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.output_dir, "/tmp/rustfs-test-telemetry");
|
||||
assert_eq!(config.sampling_rate, 0.5);
|
||||
println!(" ✓ PASS: Configuration loaded correctly\n");
|
||||
|
||||
// Test 3: Initialize dial9 session
|
||||
println!("Test 3: Initialize dial9 session");
|
||||
match Dial9SessionGuard::new(config.clone()).await {
|
||||
Ok(Some(guard)) => {
|
||||
println!(" Dial9 session initialized successfully");
|
||||
println!(" guard.is_active(): {}", guard.is_active());
|
||||
println!(" ✓ PASS: Session initialized\n");
|
||||
|
||||
// Test 4: Generate some async activity
|
||||
println!("Test 4: Generate async activity for tracing");
|
||||
let handle = tokio::spawn(async {
|
||||
for i in 1..=5 {
|
||||
println!(" Task iteration {}", i);
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
});
|
||||
handle.await?;
|
||||
println!(" ✓ PASS: Async activity completed\n");
|
||||
|
||||
// Test 5: Session shutdown
|
||||
println!("Test 5: Session cleanup");
|
||||
drop(guard);
|
||||
println!(" ✓ PASS: Session cleaned up\n");
|
||||
}
|
||||
Ok(None) => {
|
||||
println!(" ⚠ SKIP: Dial9 session not created (configuration validation may have failed)\n");
|
||||
}
|
||||
Err(e) => {
|
||||
println!(" ✗ FAIL: {:?}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
if let Err(err) = tokio::fs::remove_dir_all(&config.output_dir).await {
|
||||
println!(" ⚠ SKIP: Failed to remove output directory: {}", err);
|
||||
}
|
||||
|
||||
println!("=== All Tests Passed! ===");
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user