mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-16 18:08:21 +00:00
feat(storage): harden internode data-path controls (#4224)
* fix(rio): propagate http writer shutdown errors * fix(ecstore): unify remote lock rpc deadlines * fix(storage): reject corrupt read multiple payloads * feat(rio): add internode http tuning profiles * feat(metrics): add internode baseline signals * feat(ecstore): observe shard locality topology * feat(ecstore): gate shard locality scheduling * feat(ecstore): gate batch read version rpc * feat(ecstore): observe batch processor adaptation * feat(ecstore): gate batch processor observation * docs: add get benchmark regression analysis * docs: add issue 797 execution plan status * fix(ecstore): require explicit batch rpc support * fix(ecstore): honor documented batch read gate * fix(ecstore): keep batch read gate stable per call * chore: update workspace dependencies * feat(ecstore): log batch read gate decisions * feat(ecstore): count batch read gate decisions * test(issue-797): add local internode A/B runner * test(rio): fix tuning profile spelling fixture * fix(protocols): adapt sftp channel open callbacks * fix(metrics): wrap batch processor observation args * chore(docs): keep issue notes local only * fix(storage): address internode review feedback * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. * fix(storage): address internode data-path review findings - Run the BatchReadVersion auto-mode unary fallback outside the batch RPC deadline so each read_version keeps its own per-op timeout and health accounting instead of racing the whole batch against one drive timeout. - Cap adaptive batch-processor concurrency growth at a hard multiple of the configured baseline so sustained fast batches cannot ratchet past the configured limit. - Parse RUSTFS_INTERNODE_HTTP_* tuning, RUSTFS_BATCH_PROCESSOR_ADAPTIVE, and RUSTFS_METADATA_BATCH_READ once per process instead of re-reading the environment on hot paths. - Skip shard read-cost collection in observe mode when stage metrics are disabled, and cache the local endpoint host list instead of rebuilding it on every read. - Allow --warp-extra-args values starting with -- and drop the unused warp_hosts_csv helper in the issue-797 A/B runner. Co-Authored-By: heihutu<heihutu@gmail.com> * fix(storage): align buffer clamp test with media cap * fix(ecstore): release optimized read locks before streaming --------- Co-authored-by: Zhengchao An <anzhengchao@gmail.com>
This commit is contained in:
houseme
@@ -19,17 +19,255 @@
|
||||
|
||||
use crate::disk::error::{Error, Result};
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
const BATCH_PROCESSOR_OPERATION_CUSTOM: &str = "custom";
|
||||
const BATCH_PROCESSOR_OPERATION_READ: &str = "read";
|
||||
const BATCH_PROCESSOR_OPERATION_WRITE: &str = "write";
|
||||
const BATCH_PROCESSOR_OPERATION_METADATA: &str = "metadata";
|
||||
const ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE: &str = "RUSTFS_BATCH_PROCESSOR_ADAPTIVE";
|
||||
const BATCH_PROCESSOR_ADAPTIVE_OFF: &str = "off";
|
||||
const BATCH_PROCESSOR_ADAPTIVE_OBSERVE: &str = "observe";
|
||||
const BATCH_PROCESSOR_ADAPTIVE_ON: &str = "on";
|
||||
const BATCH_SUGGESTION_REASON_IMPROVING: &str = "improving";
|
||||
const BATCH_SUGGESTION_REASON_DEGRADING: &str = "degrading";
|
||||
const BATCH_SUGGESTION_REASON_STABLE: &str = "stable";
|
||||
const BATCH_SUGGESTION_REASON_COOLDOWN: &str = "cooldown";
|
||||
const BATCH_OBSERVATION_IMPROVING_LATENCY: Duration = Duration::from_millis(50);
|
||||
const BATCH_OBSERVATION_DEGRADING_LATENCY: Duration = Duration::from_millis(250);
|
||||
const BATCH_OBSERVATION_COOLDOWN: Duration = Duration::from_secs(30);
|
||||
const BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR: usize = 4;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum BatchProcessorAdaptiveMode {
|
||||
Off,
|
||||
Observe,
|
||||
On,
|
||||
}
|
||||
|
||||
impl BatchProcessorAdaptiveMode {
|
||||
fn should_observe(self) -> bool {
|
||||
matches!(self, Self::Observe | Self::On)
|
||||
}
|
||||
|
||||
fn should_apply(self) -> bool {
|
||||
matches!(self, Self::On)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_batch_processor_adaptive_mode(raw: &str) -> BatchProcessorAdaptiveMode {
|
||||
match raw.trim() {
|
||||
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OBSERVE) => BatchProcessorAdaptiveMode::Observe,
|
||||
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_ON) => BatchProcessorAdaptiveMode::On,
|
||||
value if value.eq_ignore_ascii_case(BATCH_PROCESSOR_ADAPTIVE_OFF) => BatchProcessorAdaptiveMode::Off,
|
||||
_ => BatchProcessorAdaptiveMode::Off,
|
||||
}
|
||||
}
|
||||
|
||||
fn batch_processor_adaptive_mode_from_env() -> BatchProcessorAdaptiveMode {
|
||||
rustfs_utils::get_env_opt_str(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE)
|
||||
.as_deref()
|
||||
.map(parse_batch_processor_adaptive_mode)
|
||||
.unwrap_or(BatchProcessorAdaptiveMode::Off)
|
||||
}
|
||||
|
||||
fn batch_processor_adaptive_mode() -> BatchProcessorAdaptiveMode {
|
||||
// The gate cannot change at runtime; parse it once instead of re-reading
|
||||
// the environment on every batch.
|
||||
static MODE: std::sync::LazyLock<BatchProcessorAdaptiveMode> =
|
||||
std::sync::LazyLock::new(batch_processor_adaptive_mode_from_env);
|
||||
*MODE
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct BatchConcurrencySuggestion {
|
||||
concurrency: usize,
|
||||
reason: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct BatchObservation {
|
||||
batch_size: usize,
|
||||
success_count: usize,
|
||||
error_count: usize,
|
||||
timeout_count: usize,
|
||||
max_queue_wait: Duration,
|
||||
execution_latency: Duration,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BatchObservationState {
|
||||
last_suggestion_at: Option<Instant>,
|
||||
last_suggested_concurrency: usize,
|
||||
}
|
||||
|
||||
impl BatchObservationState {
|
||||
fn new(configured_concurrency: usize) -> Self {
|
||||
Self {
|
||||
last_suggestion_at: None,
|
||||
last_suggested_concurrency: configured_concurrency,
|
||||
}
|
||||
}
|
||||
|
||||
fn suggest(
|
||||
&mut self,
|
||||
current_concurrency: usize,
|
||||
configured_concurrency: usize,
|
||||
observation: BatchObservation,
|
||||
now: Instant,
|
||||
) -> BatchConcurrencySuggestion {
|
||||
let raw = calculate_batch_concurrency_suggestion(current_concurrency, configured_concurrency, observation);
|
||||
if raw.concurrency == current_concurrency {
|
||||
self.last_suggested_concurrency = current_concurrency;
|
||||
return raw;
|
||||
}
|
||||
|
||||
if self
|
||||
.last_suggestion_at
|
||||
.is_some_and(|last_suggestion_at| now.duration_since(last_suggestion_at) < BATCH_OBSERVATION_COOLDOWN)
|
||||
{
|
||||
return BatchConcurrencySuggestion {
|
||||
concurrency: self.last_suggested_concurrency,
|
||||
reason: BATCH_SUGGESTION_REASON_COOLDOWN,
|
||||
};
|
||||
}
|
||||
|
||||
self.last_suggestion_at = Some(now);
|
||||
self.last_suggested_concurrency = raw.concurrency;
|
||||
raw
|
||||
}
|
||||
}
|
||||
|
||||
fn calculate_batch_concurrency_suggestion(
|
||||
current_concurrency: usize,
|
||||
configured_concurrency: usize,
|
||||
observation: BatchObservation,
|
||||
) -> BatchConcurrencySuggestion {
|
||||
let current_concurrency = current_concurrency.max(1);
|
||||
let configured_concurrency = configured_concurrency.max(1);
|
||||
|
||||
if observation.batch_size == 0 {
|
||||
return BatchConcurrencySuggestion {
|
||||
concurrency: current_concurrency,
|
||||
reason: BATCH_SUGGESTION_REASON_STABLE,
|
||||
};
|
||||
}
|
||||
|
||||
if observation.timeout_count > 0
|
||||
|| observation.error_count > 0
|
||||
|| (observation.batch_size >= current_concurrency && observation.execution_latency >= BATCH_OBSERVATION_DEGRADING_LATENCY)
|
||||
{
|
||||
return BatchConcurrencySuggestion {
|
||||
concurrency: decrease_batch_concurrency(current_concurrency),
|
||||
reason: BATCH_SUGGESTION_REASON_DEGRADING,
|
||||
};
|
||||
}
|
||||
|
||||
if observation.success_count == observation.batch_size
|
||||
&& observation.batch_size >= current_concurrency
|
||||
&& observation.execution_latency <= BATCH_OBSERVATION_IMPROVING_LATENCY
|
||||
{
|
||||
return BatchConcurrencySuggestion {
|
||||
concurrency: increase_batch_concurrency(current_concurrency, configured_concurrency, observation.batch_size),
|
||||
reason: BATCH_SUGGESTION_REASON_IMPROVING,
|
||||
};
|
||||
}
|
||||
|
||||
BatchConcurrencySuggestion {
|
||||
concurrency: current_concurrency,
|
||||
reason: BATCH_SUGGESTION_REASON_STABLE,
|
||||
}
|
||||
}
|
||||
|
||||
fn decrease_batch_concurrency(current_concurrency: usize) -> usize {
|
||||
(current_concurrency.saturating_mul(3) / 4).max(1)
|
||||
}
|
||||
|
||||
fn increase_batch_concurrency(current_concurrency: usize, configured_concurrency: usize, batch_size: usize) -> usize {
|
||||
let step = (current_concurrency / 4).max(1);
|
||||
// Suggestions build on the previous suggestion, so bound the ratchet at a
|
||||
// hard multiple of the configured baseline instead of the current value.
|
||||
let upper_bound = configured_concurrency
|
||||
.saturating_mul(BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR)
|
||||
.max(current_concurrency);
|
||||
current_concurrency
|
||||
.saturating_add(step)
|
||||
.min(upper_bound)
|
||||
.min(batch_size.max(current_concurrency))
|
||||
}
|
||||
|
||||
fn is_timeout_error(err: &Error) -> bool {
|
||||
matches!(err, Error::Timeout | Error::SourceStalled)
|
||||
|| matches!(err, Error::Io(io_err) if io_err.kind() == std::io::ErrorKind::TimedOut)
|
||||
}
|
||||
|
||||
/// Batch processor that executes tasks concurrently with a semaphore
|
||||
pub struct AsyncBatchProcessor {
|
||||
max_concurrent: usize,
|
||||
operation: &'static str,
|
||||
observation_state: Mutex<BatchObservationState>,
|
||||
}
|
||||
|
||||
impl AsyncBatchProcessor {
|
||||
pub fn new(max_concurrent: usize) -> Self {
|
||||
Self { max_concurrent }
|
||||
Self::new_with_operation(max_concurrent, BATCH_PROCESSOR_OPERATION_CUSTOM)
|
||||
}
|
||||
|
||||
fn new_with_operation(max_concurrent: usize, operation: &'static str) -> Self {
|
||||
let max_concurrent = max_concurrent.max(1);
|
||||
Self {
|
||||
max_concurrent,
|
||||
operation,
|
||||
observation_state: Mutex::new(BatchObservationState::new(max_concurrent)),
|
||||
}
|
||||
}
|
||||
|
||||
fn execution_concurrency(&self, mode: BatchProcessorAdaptiveMode) -> usize {
|
||||
if !mode.should_apply() {
|
||||
return self.max_concurrent;
|
||||
}
|
||||
|
||||
self.observation_state
|
||||
.lock()
|
||||
.map(|state| state.last_suggested_concurrency.max(1))
|
||||
.unwrap_or(self.max_concurrent)
|
||||
}
|
||||
|
||||
fn observe_batch_with_mode(
|
||||
&self,
|
||||
mode: BatchProcessorAdaptiveMode,
|
||||
execution_concurrency: usize,
|
||||
observation: BatchObservation,
|
||||
) -> BatchConcurrencySuggestion {
|
||||
let execution_concurrency = execution_concurrency.max(1);
|
||||
if !mode.should_observe() {
|
||||
return BatchConcurrencySuggestion {
|
||||
concurrency: execution_concurrency,
|
||||
reason: BATCH_SUGGESTION_REASON_STABLE,
|
||||
};
|
||||
}
|
||||
|
||||
let suggestion = match self.observation_state.lock() {
|
||||
Ok(mut state) => state.suggest(execution_concurrency, self.max_concurrent, observation, Instant::now()),
|
||||
Err(_) => calculate_batch_concurrency_suggestion(execution_concurrency, self.max_concurrent, observation),
|
||||
};
|
||||
|
||||
rustfs_io_metrics::record_batch_processor_observation(rustfs_io_metrics::BatchProcessorObservation {
|
||||
operation: self.operation,
|
||||
batch_size: observation.batch_size,
|
||||
configured_concurrency: execution_concurrency,
|
||||
max_queue_wait_secs: observation.max_queue_wait.as_secs_f64(),
|
||||
execution_latency_secs: observation.execution_latency.as_secs_f64(),
|
||||
successes: observation.success_count,
|
||||
errors: observation.error_count,
|
||||
timeouts: observation.timeout_count,
|
||||
suggested_concurrency: suggestion.concurrency,
|
||||
suggestion_reason: suggestion.reason,
|
||||
});
|
||||
|
||||
suggestion
|
||||
}
|
||||
|
||||
/// Execute a batch of tasks concurrently with concurrency control
|
||||
@@ -42,27 +280,35 @@ impl AsyncBatchProcessor {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
|
||||
let batch_size = tasks.len();
|
||||
let batch_started_at = Instant::now();
|
||||
let adaptive_mode = batch_processor_adaptive_mode();
|
||||
let execution_concurrency = self.execution_concurrency(adaptive_mode);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut results = Vec::with_capacity(tasks.len());
|
||||
for _ in 0..tasks.len() {
|
||||
let mut results = Vec::with_capacity(batch_size);
|
||||
for _ in 0..batch_size {
|
||||
results.push(Err(Error::other("Not completed")));
|
||||
}
|
||||
|
||||
// Spawn all tasks with semaphore control
|
||||
for (i, task) in tasks.into_iter().enumerate() {
|
||||
let sem = semaphore.clone();
|
||||
let queued_at = Instant::now();
|
||||
join_set.spawn(async move {
|
||||
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
|
||||
let queue_wait = queued_at.elapsed();
|
||||
let result = task.await;
|
||||
Ok::<(usize, Result<T>), Error>((i, result))
|
||||
Ok::<(usize, Result<T>, Duration), Error>((i, result, queue_wait))
|
||||
});
|
||||
}
|
||||
|
||||
// Collect results
|
||||
let mut max_queue_wait = Duration::ZERO;
|
||||
while let Some(join_result) = join_set.join_next().await {
|
||||
match join_result {
|
||||
Ok(Ok((index, task_result))) => {
|
||||
Ok(Ok((index, task_result, queue_wait))) => {
|
||||
max_queue_wait = max_queue_wait.max(queue_wait);
|
||||
if index < results.len() {
|
||||
results[index] = task_result;
|
||||
}
|
||||
@@ -78,6 +324,29 @@ impl AsyncBatchProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
let mut success_count = 0;
|
||||
let mut error_count = 0;
|
||||
let mut timeout_count = 0;
|
||||
for result in &results {
|
||||
match result {
|
||||
Ok(_) => success_count += 1,
|
||||
Err(err) if is_timeout_error(err) => timeout_count += 1,
|
||||
Err(_) => error_count += 1,
|
||||
}
|
||||
}
|
||||
self.observe_batch_with_mode(
|
||||
adaptive_mode,
|
||||
execution_concurrency,
|
||||
BatchObservation {
|
||||
batch_size,
|
||||
success_count,
|
||||
error_count,
|
||||
timeout_count,
|
||||
max_queue_wait,
|
||||
execution_latency: batch_started_at.elapsed(),
|
||||
},
|
||||
);
|
||||
|
||||
results
|
||||
}
|
||||
|
||||
@@ -97,17 +366,29 @@ impl AsyncBatchProcessor {
|
||||
)));
|
||||
}
|
||||
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(self.max_concurrent));
|
||||
let batch_size = tasks.len();
|
||||
let batch_started_at = Instant::now();
|
||||
let adaptive_mode = batch_processor_adaptive_mode();
|
||||
let execution_concurrency = self.execution_concurrency(adaptive_mode);
|
||||
let semaphore = Arc::new(tokio::sync::Semaphore::new(execution_concurrency));
|
||||
let mut join_set = JoinSet::new();
|
||||
let mut successes = Vec::new();
|
||||
let mut pending_tasks = tasks.len();
|
||||
let mut pending_tasks = batch_size;
|
||||
let mut first_error = None;
|
||||
let mut error_count = 0;
|
||||
let mut timeout_count = 0;
|
||||
let mut max_queue_wait = Duration::ZERO;
|
||||
|
||||
for task in tasks {
|
||||
let sem = semaphore.clone();
|
||||
let queued_at = Instant::now();
|
||||
join_set.spawn(async move {
|
||||
let _permit = sem.acquire().await.map_err(|_| Error::other("Semaphore error"))?;
|
||||
task.await
|
||||
let _permit = sem
|
||||
.acquire()
|
||||
.await
|
||||
.map_err(|_| (Error::other("Semaphore error"), queued_at.elapsed()))?;
|
||||
let queue_wait = queued_at.elapsed();
|
||||
task.await.map(|value| (value, queue_wait)).map_err(|err| (err, queue_wait))
|
||||
});
|
||||
}
|
||||
|
||||
@@ -115,18 +396,38 @@ impl AsyncBatchProcessor {
|
||||
pending_tasks = pending_tasks.saturating_sub(1);
|
||||
|
||||
match join_result {
|
||||
Ok(Ok(value)) => {
|
||||
Ok(Ok((value, queue_wait))) => {
|
||||
max_queue_wait = max_queue_wait.max(queue_wait);
|
||||
successes.push(value);
|
||||
if successes.len() >= required_successes {
|
||||
self.observe_batch_with_mode(
|
||||
adaptive_mode,
|
||||
execution_concurrency,
|
||||
BatchObservation {
|
||||
batch_size,
|
||||
success_count: successes.len(),
|
||||
error_count,
|
||||
timeout_count,
|
||||
max_queue_wait,
|
||||
execution_latency: batch_started_at.elapsed(),
|
||||
},
|
||||
);
|
||||
return Ok(successes);
|
||||
}
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
Ok(Err((err, queue_wait))) => {
|
||||
max_queue_wait = max_queue_wait.max(queue_wait);
|
||||
if is_timeout_error(&err) {
|
||||
timeout_count += 1;
|
||||
} else {
|
||||
error_count += 1;
|
||||
}
|
||||
if first_error.is_none() {
|
||||
first_error = Some(err);
|
||||
}
|
||||
}
|
||||
Err(join_error) => {
|
||||
error_count += 1;
|
||||
if first_error.is_none() {
|
||||
first_error = Some(Error::other(format!("Task panicked in quorum batch processor: {join_error}")));
|
||||
}
|
||||
@@ -134,6 +435,18 @@ impl AsyncBatchProcessor {
|
||||
}
|
||||
|
||||
if successes.len() + pending_tasks < required_successes {
|
||||
self.observe_batch_with_mode(
|
||||
adaptive_mode,
|
||||
execution_concurrency,
|
||||
BatchObservation {
|
||||
batch_size,
|
||||
success_count: successes.len(),
|
||||
error_count,
|
||||
timeout_count,
|
||||
max_queue_wait,
|
||||
execution_latency: batch_started_at.elapsed(),
|
||||
},
|
||||
);
|
||||
return Err(first_error.unwrap_or_else(|| {
|
||||
Error::other(format!(
|
||||
"Insufficient successful results: got {}, needed {}",
|
||||
@@ -144,6 +457,18 @@ impl AsyncBatchProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
self.observe_batch_with_mode(
|
||||
adaptive_mode,
|
||||
execution_concurrency,
|
||||
BatchObservation {
|
||||
batch_size,
|
||||
success_count: successes.len(),
|
||||
error_count,
|
||||
timeout_count,
|
||||
max_queue_wait,
|
||||
execution_latency: batch_started_at.elapsed(),
|
||||
},
|
||||
);
|
||||
Err(first_error.unwrap_or_else(|| {
|
||||
Error::other(format!(
|
||||
"Insufficient successful results: got {}, needed {}",
|
||||
@@ -164,9 +489,9 @@ pub struct GlobalBatchProcessors {
|
||||
impl GlobalBatchProcessors {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
read_processor: AsyncBatchProcessor::new(16), // Higher concurrency for reads
|
||||
write_processor: AsyncBatchProcessor::new(8), // Lower concurrency for writes
|
||||
metadata_processor: AsyncBatchProcessor::new(12), // Medium concurrency for metadata
|
||||
read_processor: AsyncBatchProcessor::new_with_operation(16, BATCH_PROCESSOR_OPERATION_READ),
|
||||
write_processor: AsyncBatchProcessor::new_with_operation(8, BATCH_PROCESSOR_OPERATION_WRITE),
|
||||
metadata_processor: AsyncBatchProcessor::new_with_operation(12, BATCH_PROCESSOR_OPERATION_METADATA),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,4 +646,208 @@ mod tests {
|
||||
assert!(err.to_string().contains("first failure"));
|
||||
assert!(started.elapsed() < Duration::from_millis(120));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_processor_observe_only_keeps_configured_concurrency() {
|
||||
let processor = AsyncBatchProcessor::new(2);
|
||||
|
||||
let tasks: Vec<_> = (0..4)
|
||||
.map(|i| async move {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
Ok::<i32, Error>(i)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let results = processor.execute_batch(tasks).await;
|
||||
assert_eq!(results.len(), 4);
|
||||
assert_eq!(processor.max_concurrent, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_adaptive_mode_defaults_to_off_and_parses_supported_values() {
|
||||
assert_eq!(parse_batch_processor_adaptive_mode(""), BatchProcessorAdaptiveMode::Off);
|
||||
assert_eq!(parse_batch_processor_adaptive_mode("off"), BatchProcessorAdaptiveMode::Off);
|
||||
assert_eq!(parse_batch_processor_adaptive_mode("observe"), BatchProcessorAdaptiveMode::Observe);
|
||||
assert_eq!(parse_batch_processor_adaptive_mode("on"), BatchProcessorAdaptiveMode::On);
|
||||
assert_eq!(parse_batch_processor_adaptive_mode("unknown"), BatchProcessorAdaptiveMode::Off);
|
||||
assert!(!BatchProcessorAdaptiveMode::Off.should_observe());
|
||||
assert!(BatchProcessorAdaptiveMode::Observe.should_observe());
|
||||
assert!(BatchProcessorAdaptiveMode::On.should_observe());
|
||||
assert!(!BatchProcessorAdaptiveMode::Observe.should_apply());
|
||||
assert!(BatchProcessorAdaptiveMode::On.should_apply());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_adaptive_mode_env_gate_is_parsed_from_env() {
|
||||
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, None::<&str>, || {
|
||||
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Off);
|
||||
});
|
||||
temp_env::with_var(ENV_RUSTFS_BATCH_PROCESSOR_ADAPTIVE, Some("observe"), || {
|
||||
assert_eq!(batch_processor_adaptive_mode_from_env(), BatchProcessorAdaptiveMode::Observe);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_observation_gate_keeps_default_off_stable() {
|
||||
let processor = AsyncBatchProcessor::new(8);
|
||||
let suggestion = processor.observe_batch_with_mode(
|
||||
BatchProcessorAdaptiveMode::Off,
|
||||
processor.execution_concurrency(BatchProcessorAdaptiveMode::Off),
|
||||
BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 16,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_STABLE);
|
||||
assert_eq!(suggestion.concurrency, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_observation_gate_observe_records_suggestion() {
|
||||
let processor = AsyncBatchProcessor::new(8);
|
||||
let suggestion = processor.observe_batch_with_mode(
|
||||
BatchProcessorAdaptiveMode::Observe,
|
||||
processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe),
|
||||
BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 16,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
|
||||
assert!(suggestion.concurrency > 8);
|
||||
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::Observe), 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_adaptive_on_applies_last_suggestion_to_next_batch() {
|
||||
let processor = AsyncBatchProcessor::new(8);
|
||||
let suggestion = processor.observe_batch_with_mode(
|
||||
BatchProcessorAdaptiveMode::On,
|
||||
processor.execution_concurrency(BatchProcessorAdaptiveMode::On),
|
||||
BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 16,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
|
||||
assert!(suggestion.concurrency > 8);
|
||||
assert_eq!(processor.execution_concurrency(BatchProcessorAdaptiveMode::On), suggestion.concurrency);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_suggestion_increases_for_fast_successful_batches() {
|
||||
let suggestion = calculate_batch_concurrency_suggestion(
|
||||
8,
|
||||
8,
|
||||
BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 16,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
|
||||
assert!(suggestion.concurrency > 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_suggestion_decreases_for_timeout_batches() {
|
||||
let suggestion = calculate_batch_concurrency_suggestion(
|
||||
8,
|
||||
8,
|
||||
BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 14,
|
||||
error_count: 0,
|
||||
timeout_count: 2,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_DEGRADING);
|
||||
assert!(suggestion.concurrency < 8);
|
||||
assert!(suggestion.concurrency >= 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_suggestion_clamps_to_batch_size() {
|
||||
let suggestion = calculate_batch_concurrency_suggestion(
|
||||
64,
|
||||
64,
|
||||
BatchObservation {
|
||||
batch_size: 65,
|
||||
success_count: 65,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(suggestion.reason, BATCH_SUGGESTION_REASON_IMPROVING);
|
||||
assert_eq!(suggestion.concurrency, 65);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_suggestion_cooldown_prevents_rapid_oscillation() {
|
||||
let mut state = BatchObservationState::new(8);
|
||||
let now = Instant::now();
|
||||
let observation = BatchObservation {
|
||||
batch_size: 16,
|
||||
success_count: 16,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
};
|
||||
|
||||
let first = state.suggest(8, 8, observation, now);
|
||||
let second = state.suggest(8, 8, observation, now + Duration::from_secs(1));
|
||||
|
||||
assert_eq!(first.reason, BATCH_SUGGESTION_REASON_IMPROVING);
|
||||
assert_eq!(second.reason, BATCH_SUGGESTION_REASON_COOLDOWN);
|
||||
assert_eq!(second.concurrency, first.concurrency);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_processor_suggestion_growth_is_capped_relative_to_configured_concurrency() {
|
||||
const CONFIGURED: usize = 8;
|
||||
let mut state = BatchObservationState::new(CONFIGURED);
|
||||
let mut current = CONFIGURED;
|
||||
let mut now = Instant::now();
|
||||
let observation = BatchObservation {
|
||||
batch_size: 1024,
|
||||
success_count: 1024,
|
||||
error_count: 0,
|
||||
timeout_count: 0,
|
||||
max_queue_wait: Duration::ZERO,
|
||||
execution_latency: Duration::from_millis(20),
|
||||
};
|
||||
|
||||
for _ in 0..64 {
|
||||
current = state.suggest(current, CONFIGURED, observation, now).concurrency;
|
||||
now += BATCH_OBSERVATION_COOLDOWN;
|
||||
}
|
||||
|
||||
assert_eq!(current, CONFIGURED * BATCH_ADAPTIVE_MAX_CONCURRENCY_FACTOR);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user