gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-27 15:37:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: decouple readiness from cluster health (#3912)

Browse Source 
* fix: include foreground write pressure

* fix: split readiness and cluster health probes

* fix: publish ready on node readiness

* fix: bound cluster health collection

* test: pin health probe collector routing

* fix: qualify app storage ECStore type

* test(admin): narrow runtime capabilities topology assertion

---------

Co-authored-by: houseme <housemecn@gmail.com>
This commit is contained in:
cxymds
2026-06-26 21:55:18 +08:00
committed by GitHub
parent 6469d6ada8
commit 7820a55fdc
14 changed files with 769 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/config/src/constants/heal.rs
+7
View File
@@ -140,6 +140,10 @@ pub const ENV_HEAL_MAINLINE_THROTTLE_ENABLE: &str = "RUSTFS_HEAL_MAINLINE_THROTT
/// at which background heal work pauses starting new tasks.
pub const ENV_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT: &str = "RUSTFS_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT";
/// Environment variable that controls the foreground write utilization percentage
/// at which background heal work pauses starting new tasks.
pub const ENV_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: &str = "RUSTFS_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT";
/// Environment variable that controls how soon the heal scheduler rechecks foreground
/// pressure after delaying background work.
pub const ENV_HEAL_MAINLINE_MAX_SLEEP_MS: &str = "RUSTFS_HEAL_MAINLINE_MAX_SLEEP_MS";
@@ -168,5 +172,8 @@ pub const DEFAULT_HEAL_MAINLINE_THROTTLE_ENABLE: bool = true;
/// Default foreground read permit utilization threshold for pausing best-effort heal task starts.
pub const DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT: usize = 80;
/// Default foreground write utilization threshold for pausing best-effort heal task starts.
pub const DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT: usize = 80;
/// Default foreground pressure recheck delay for heal scheduler, in milliseconds.
pub const DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS: u64 = 250;
crates/config/src/constants/health.rs
+5
View File
@@ -22,6 +22,11 @@ pub const DEFAULT_HEALTH_ENDPOINT_ENABLE: bool = true;
pub const ENV_HEALTH_READINESS_CACHE_TTL_MS: &str = "RUSTFS_HEALTH_READINESS_CACHE_TTL_MS";
pub const DEFAULT_HEALTH_READINESS_CACHE_TTL_MS: u64 = 1000;
/// Timeout for cluster health readiness collectors (milliseconds).
/// This bounds expensive storage and lock quorum checks used by cluster probes.
pub const ENV_HEALTH_CLUSTER_TIMEOUT_MS: &str = "RUSTFS_HEALTH_CLUSTER_TIMEOUT_MS";
pub const DEFAULT_HEALTH_CLUSTER_TIMEOUT_MS: u64 = 2000;
/// Enable minimal health payload mode for GET `/health*` responses.
/// When enabled, only `status` and `ready` fields are returned.
pub const ENV_HEALTH_MINIMAL_RESPONSE_ENABLE: &str = "RUSTFS_HEALTH_MINIMAL_RESPONSE_ENABLE";
crates/ecstore/src/data_movement_backpressure.rs
+115 -37
View File
@@ -26,9 +26,11 @@ const LOG_SUBSYSTEM_DATA_MOVEMENT: &str = "data_movement";
const EVENT_DATA_MOVEMENT_BACKPRESSURE: &str = "data_movement_backpressure";
const DATA_MOVEMENT_BACKPRESSURE_ENABLE_ENV: &str = "RUSTFS_DATA_MOVEMENT_BACKPRESSURE_ENABLE";
const DATA_MOVEMENT_FOREGROUND_READ_HIGH_PERCENT_ENV: &str = "RUSTFS_DATA_MOVEMENT_FOREGROUND_READ_HIGH_PERCENT";
const DATA_MOVEMENT_FOREGROUND_WRITE_HIGH_PERCENT_ENV: &str = "RUSTFS_DATA_MOVEMENT_FOREGROUND_WRITE_HIGH_PERCENT";
const DATA_MOVEMENT_RECHECK_MS_ENV: &str = "RUSTFS_DATA_MOVEMENT_RECHECK_MS";
const DEFAULT_DATA_MOVEMENT_BACKPRESSURE_ENABLE: bool = true;
const DEFAULT_DATA_MOVEMENT_FOREGROUND_READ_HIGH_PERCENT: usize = 80;
const DEFAULT_DATA_MOVEMENT_FOREGROUND_WRITE_HIGH_PERCENT: usize = 80;
const DEFAULT_DATA_MOVEMENT_RECHECK_MS: u64 = 250;
const MIN_DATA_MOVEMENT_RECHECK_MS: u64 = 1;
@@ -51,6 +53,7 @@ impl DataMovementOperation {
struct DataMovementBackpressureConfig {
enabled: bool,
foreground_read_high_percent: usize,
foreground_write_high_percent: usize,
recheck_delay: Duration,
}
@@ -62,6 +65,10 @@ impl DataMovementBackpressureConfig {
DATA_MOVEMENT_FOREGROUND_READ_HIGH_PERCENT_ENV,
DEFAULT_DATA_MOVEMENT_FOREGROUND_READ_HIGH_PERCENT,
),
foreground_write_high_percent: rustfs_utils::get_env_usize(
DATA_MOVEMENT_FOREGROUND_WRITE_HIGH_PERCENT_ENV,
DEFAULT_DATA_MOVEMENT_FOREGROUND_WRITE_HIGH_PERCENT,
),
recheck_delay: Duration::from_millis(
rustfs_utils::get_env_u64(DATA_MOVEMENT_RECHECK_MS_ENV, DEFAULT_DATA_MOVEMENT_RECHECK_MS)
.max(MIN_DATA_MOVEMENT_RECHECK_MS),
@@ -95,24 +102,24 @@ async fn wait_for_data_movement_admission_with_provider(
if cancel_token.is_cancelled() {
return Err(Error::OperationCanceled);
}
if !config.enabled || config.foreground_read_high_percent == 0 {
if !config.enabled || (config.foreground_read_high_percent == 0 && config.foreground_write_high_percent == 0) {
return Ok(());
}
let Some(provider) = provider else {
return Ok(());
};
let mut delayed_since: Option<Instant> = None;
let mut delayed_since: Option<(Instant, ForegroundPressure)> = None;
loop {
if cancel_token.is_cancelled() {
record_delay_completion(operation, pool_index, delayed_since, "cancelled");
return Err(Error::OperationCanceled);
}
if let Some(usage_pct) = foreground_read_pressure_pct(&config, Some(provider.as_ref())) {
if let Some(pressure) = foreground_pressure(&config, Some(provider.as_ref())) {
if delayed_since.is_none() {
delayed_since = Some(Instant::now());
record_delay_start(operation, pool_index, usage_pct, &config);
delayed_since = Some((Instant::now(), pressure));
record_delay_start(operation, pool_index, pressure, &config);
}
tokio::select! {
@@ -130,44 +137,77 @@ async fn wait_for_data_movement_admission_with_provider(
}
}
fn foreground_read_pressure_pct(
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ForegroundPressure {
class: WorkloadClass,
usage_pct: usize,
threshold_pct: usize,
}
impl ForegroundPressure {
const fn reason(self) -> &'static str {
match self.class {
WorkloadClass::ForegroundRead => "foreground_read_pressure",
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
_ => "foreground_pressure",
}
}
}
fn foreground_pressure(
config: &DataMovementBackpressureConfig,
provider: Option<&(dyn WorkloadAdmissionSnapshotProvider + Send + Sync)>,
) -> Option<usize> {
if !config.enabled || config.foreground_read_high_percent == 0 {
) -> Option<ForegroundPressure> {
if !config.enabled {
return None;
}
let snapshot = provider?.workload_admission_snapshot();
let foreground_read = snapshot.get(WorkloadClass::ForegroundRead)?;
if matches!(foreground_read.state, AdmissionState::Saturated) {
return Some(100);
}
[
(WorkloadClass::ForegroundRead, config.foreground_read_high_percent),
(WorkloadClass::ForegroundWrite, config.foreground_write_high_percent),
]
.into_iter()
.filter_map(|(class, threshold_pct)| {
if threshold_pct == 0 {
return None;
}
let limit = foreground_read.limit?;
if limit == 0 {
return None;
}
let entry = snapshot.get(class)?;
let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
100
} else {
let limit = entry.limit?;
if limit == 0 {
return None;
}
entry
.active
.unwrap_or(0)
.saturating_mul(100)
.checked_div(limit)
.unwrap_or(100)
};
let usage_pct = foreground_read
.active
.unwrap_or(0)
.saturating_mul(100)
.checked_div(limit)
.unwrap_or(100);
(usage_pct >= config.foreground_read_high_percent).then_some(usage_pct)
(usage_pct >= threshold_pct).then_some(ForegroundPressure {
class,
usage_pct,
threshold_pct,
})
})
.max_by_key(|pressure| pressure.usage_pct)
}
fn record_delay_start(
operation: DataMovementOperation,
pool_index: usize,
foreground_read_usage_pct: usize,
pressure: ForegroundPressure,
config: &DataMovementBackpressureConfig,
) {
counter!(
"rustfs_data_movement_backpressure_total",
"operation" => operation.as_str().to_string(),
"reason" => "foreground_read_pressure".to_string(),
"reason" => pressure.reason().to_string(),
"result" => "delayed".to_string(),
"pool_index" => pool_index.to_string()
)
@@ -181,21 +221,22 @@ fn record_delay_start(
operation = operation.as_str(),
pool_index,
state = "delayed",
reason = "foreground_read_pressure",
foreground_read_usage_pct,
threshold_pct = config.foreground_read_high_percent,
reason = pressure.reason(),
workload_class = pressure.class.as_str(),
foreground_usage_pct = pressure.usage_pct,
threshold_pct = pressure.threshold_pct,
recheck_delay_ms = config.recheck_delay.as_millis(),
"Data movement delayed under foreground read pressure"
"Data movement delayed under foreground pressure"
);
}
fn record_delay_completion(
operation: DataMovementOperation,
pool_index: usize,
delayed_since: Option<Instant>,
delayed_since: Option<(Instant, ForegroundPressure)>,
result: &'static str,
) {
let Some(delayed_since) = delayed_since else {
let Some((delayed_since, pressure)) = delayed_since else {
return;
};
@@ -203,7 +244,7 @@ fn record_delay_completion(
counter!(
"rustfs_data_movement_backpressure_total",
"operation" => operation.as_str().to_string(),
"reason" => "foreground_read_pressure".to_string(),
"reason" => pressure.reason().to_string(),
"result" => result.to_string(),
"pool_index" => pool_index.to_string()
)
@@ -211,7 +252,7 @@ fn record_delay_completion(
histogram!(
"rustfs_data_movement_backpressure_delay_seconds",
"operation" => operation.as_str().to_string(),
"reason" => "foreground_read_pressure".to_string(),
"reason" => pressure.reason().to_string(),
"result" => result.to_string(),
"pool_index" => pool_index.to_string()
)
@@ -225,6 +266,8 @@ fn record_delay_completion(
operation = operation.as_str(),
pool_index,
state = result,
reason = pressure.reason(),
workload_class = pressure.class.as_str(),
delay_secs = delay.as_secs_f64(),
"Data movement backpressure wait completed"
);
@@ -259,6 +302,7 @@ mod tests {
DataMovementBackpressureConfig {
enabled: true,
foreground_read_high_percent: 80,
foreground_write_high_percent: 80,
recheck_delay: Duration::from_millis(1),
}
}
@@ -268,7 +312,14 @@ mod tests {
let provider =
StaticWorkloadProvider::new(WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Saturated));
assert_eq!(foreground_read_pressure_pct(&test_config(), Some(&provider)), Some(100));
assert_eq!(
foreground_pressure(&test_config(), Some(&provider)),
Some(ForegroundPressure {
class: WorkloadClass::ForegroundRead,
usage_pct: 100,
threshold_pct: 80,
})
);
}
#[test]
@@ -281,11 +332,38 @@ mod tests {
),
);
assert_eq!(foreground_read_pressure_pct(&test_config(), Some(&provider)), Some(80));
assert_eq!(
foreground_pressure(&test_config(), Some(&provider)),
Some(ForegroundPressure {
class: WorkloadClass::ForegroundRead,
usage_pct: 80,
threshold_pct: 80,
})
);
}
#[test]
fn foreground_read_pressure_ignores_open_low_usage() {
fn foreground_write_pressure_uses_active_limit_threshold() {
let provider = StaticWorkloadProvider::new(
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(
Some(9),
None,
Some(10),
),
);
assert_eq!(
foreground_pressure(&test_config(), Some(&provider)),
Some(ForegroundPressure {
class: WorkloadClass::ForegroundWrite,
usage_pct: 90,
threshold_pct: 80,
})
);
}
#[test]
fn foreground_pressure_ignores_open_low_usage() {
let provider = StaticWorkloadProvider::new(
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
Some(7),
@@ -294,7 +372,7 @@ mod tests {
),
);
assert_eq!(foreground_read_pressure_pct(&test_config(), Some(&provider)), None);
assert_eq!(foreground_pressure(&test_config(), Some(&provider)), None);
}
#[tokio::test]
crates/heal/src/heal/manager.rs
+121 -36
View File
@@ -106,6 +106,23 @@ enum QueuePushOutcome {
Merged,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ForegroundPressure {
class: WorkloadClass,
usage_pct: usize,
threshold_pct: usize,
}
impl ForegroundPressure {
const fn reason(self) -> &'static str {
match self.class {
WorkloadClass::ForegroundRead => "foreground_read_pressure",
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
_ => "foreground_pressure",
}
}
}
#[derive(Debug, Clone)]
struct CompletedHealStatus {
heal_type: HealType,
@@ -636,7 +653,9 @@ pub struct HealConfig {
pub mainline_throttle_enable: bool,
/// Foreground read permit utilization percentage that delays best-effort heal starts.
pub mainline_read_utilization_high_percent: usize,
/// Delay before rechecking foreground read pressure after delaying heal starts.
/// Foreground write utilization percentage that delays best-effort heal starts.
pub mainline_write_utilization_high_percent: usize,
/// Delay before rechecking foreground pressure after delaying heal starts.
pub mainline_max_sleep: Duration,
}
@@ -691,6 +710,11 @@ impl Default for HealConfig {
rustfs_config::DEFAULT_HEAL_MAINLINE_READ_UTILIZATION_HIGH_PERCENT,
)
.min(100);
let mainline_write_utilization_high_percent = rustfs_utils::get_env_usize(
rustfs_config::ENV_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT,
rustfs_config::DEFAULT_HEAL_MAINLINE_WRITE_UTILIZATION_HIGH_PERCENT,
)
.min(100);
let mainline_max_sleep = Duration::from_millis(rustfs_utils::get_env_u64(
rustfs_config::ENV_HEAL_MAINLINE_MAX_SLEEP_MS,
rustfs_config::DEFAULT_HEAL_MAINLINE_MAX_SLEEP_MS,
@@ -709,6 +733,7 @@ impl Default for HealConfig {
page_parallel_enable,
mainline_throttle_enable,
mainline_read_utilization_high_percent,
mainline_write_utilization_high_percent,
mainline_max_sleep,
}
}
@@ -833,29 +858,51 @@ impl HealManager {
|| matches!(request.priority, HealPriority::High | HealPriority::Urgent)
}
fn mainline_throttle_active(config: &HealConfig, provider: &Option<WorkloadSnapshotProviderRef>) -> Option<usize> {
if !config.mainline_throttle_enable || config.mainline_read_utilization_high_percent == 0 {
fn mainline_throttle_active(
config: &HealConfig,
provider: &Option<WorkloadSnapshotProviderRef>,
) -> Option<ForegroundPressure> {
if !config.mainline_throttle_enable
|| (config.mainline_read_utilization_high_percent == 0 && config.mainline_write_utilization_high_percent == 0)
{
return None;
}
let provider = provider.as_ref()?;
let snapshot = provider.workload_admission_snapshot();
let foreground_read = snapshot.get(WorkloadClass::ForegroundRead)?;
if matches!(foreground_read.state, AdmissionState::Saturated) {
return Some(100);
}
[
(WorkloadClass::ForegroundRead, config.mainline_read_utilization_high_percent),
(WorkloadClass::ForegroundWrite, config.mainline_write_utilization_high_percent),
]
.into_iter()
.filter_map(|(class, threshold_pct)| {
if threshold_pct == 0 {
return None;
}
let limit = foreground_read.limit?;
if limit == 0 {
return None;
}
let usage_pct = foreground_read
.active
.unwrap_or(0)
.saturating_mul(100)
.checked_div(limit)
.unwrap_or(100);
(usage_pct >= config.mainline_read_utilization_high_percent).then_some(usage_pct)
let entry = snapshot.get(class)?;
let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
100
} else {
let limit = entry.limit?;
if limit == 0 {
return None;
}
entry
.active
.unwrap_or(0)
.saturating_mul(100)
.checked_div(limit)
.unwrap_or(100)
};
(usage_pct >= threshold_pct).then_some(ForegroundPressure {
class,
usage_pct,
threshold_pct,
})
})
.max_by_key(|pressure| pressure.usage_pct)
}
fn schedule_mainline_throttle_recheck(notify: Arc<Notify>, delay: Duration) {
@@ -870,12 +917,12 @@ impl HealManager {
});
}
fn record_mainline_throttle_delay(usage_pct: usize, config: &HealConfig) {
fn record_mainline_throttle_delay(pressure: ForegroundPressure, config: &HealConfig) {
counter!(
"rustfs_heal_mainline_throttle_total",
"source" => "background",
"result" => "delayed",
"reason" => "foreground_read_pressure"
"reason" => pressure.reason()
)
.increment(1);
debug!(
@@ -884,10 +931,12 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "delayed",
foreground_read_usage_pct = usage_pct,
threshold_pct = config.mainline_read_utilization_high_percent,
reason = pressure.reason(),
workload_class = pressure.class.as_str(),
foreground_usage_pct = pressure.usage_pct,
threshold_pct = pressure.threshold_pct,
recheck_delay_ms = config.mainline_max_sleep.as_millis(),
"Heal scheduler delayed background work under foreground read pressure"
"Heal scheduler delayed background work under foreground pressure"
);
}
@@ -2294,7 +2343,7 @@ impl HealManager {
} = context;
let config = config.read().await;
let mainline_usage_pct = Self::mainline_throttle_active(&config, workload_provider);
let mainline_pressure = Self::mainline_throttle_active(&config, workload_provider);
let mut active_heals_guard = active_heals.lock().await;
publish_active_heal_count(&active_heals_guard);
@@ -2320,13 +2369,13 @@ impl HealManager {
let mut delayed_by_mainline_throttle = false;
for _ in 0..available_slots {
let selected_request = if config.set_bulkhead_enable || mainline_usage_pct.is_some() {
let selected_request = if config.set_bulkhead_enable || mainline_pressure.is_some() {
let max_concurrent_per_set = config.max_concurrent_per_set;
let (selected_request, skipped_sets) = queue.pop_runnable_with_skips(
|request| {
let set_allowed = !config.set_bulkhead_enable
|| can_schedule_request(request, &running_per_set, max_concurrent_per_set);
let mainline_allowed = mainline_usage_pct.is_none() || Self::request_bypasses_mainline_throttle(request);
let mainline_allowed = mainline_pressure.is_none() || Self::request_bypasses_mainline_throttle(request);
set_allowed && mainline_allowed
},
|request| heal_request_set_key(request).map(|_| heal_request_set_metric_label(request)),
@@ -2614,7 +2663,7 @@ impl HealManager {
});
tasks_started += 1;
} else {
delayed_by_mainline_throttle = mainline_usage_pct.is_some();
delayed_by_mainline_throttle = mainline_pressure.is_some();
break;
}
}
@@ -2626,8 +2675,8 @@ impl HealManager {
publish_active_heal_count(&active_heals_guard);
publish_heal_queue_length(&queue);
if delayed_by_mainline_throttle && let Some(usage_pct) = mainline_usage_pct {
Self::record_mainline_throttle_delay(usage_pct, &config);
if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure {
Self::record_mainline_throttle_delay(pressure, &config);
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep);
}
@@ -2757,6 +2806,7 @@ mod tests {
#[derive(Debug)]
struct FixedWorkloadProvider {
class: WorkloadClass,
active: usize,
limit: usize,
state: AdmissionState,
@@ -2764,13 +2814,11 @@ mod tests {
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
WorkloadAdmissionRegistrySnapshot::new(vec![
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, self.state).with_counts(
Some(self.active),
None,
Some(self.limit),
),
])
WorkloadAdmissionRegistrySnapshot::new(vec![WorkloadAdmissionSnapshot::new(self.class, self.state).with_counts(
Some(self.active),
None,
Some(self.limit),
)])
}
}
@@ -4346,6 +4394,7 @@ mod tests {
async fn test_mainline_throttle_delays_background_heal_start() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
class: WorkloadClass::ForegroundRead,
active: 8,
limit: 10,
state: AdmissionState::Open,
@@ -4356,6 +4405,40 @@ mod tests {
max_concurrent_heals: 1,
mainline_throttle_enable: true,
mainline_read_utilization_high_percent: 80,
mainline_write_utilization_high_percent: 80,
mainline_max_sleep: Duration::from_millis(1),
..HealConfig::default()
}),
Some(provider),
);
manager
.submit_heal_request(bucket_request("read-repair", HealPriority::Normal, HealRequestSource::ReadRepair))
.await
.expect("read repair request should be queued");
process_manager_queue_once(&manager).await;
assert_eq!(manager.get_queue_length().await, 1);
assert_eq!(manager.get_active_task_count().await, 0);
}
#[tokio::test]
async fn test_mainline_throttle_delays_background_heal_start_under_write_pressure() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
class: WorkloadClass::ForegroundWrite,
active: 9,
limit: 10,
state: AdmissionState::Open,
});
let manager = HealManager::new_with_workload_provider(
storage,
Some(HealConfig {
max_concurrent_heals: 1,
mainline_throttle_enable: true,
mainline_read_utilization_high_percent: 80,
mainline_write_utilization_high_percent: 80,
mainline_max_sleep: Duration::from_millis(1),
..HealConfig::default()
}),
@@ -4377,6 +4460,7 @@ mod tests {
async fn test_mainline_throttle_allows_admin_high_start() {
let storage: Arc<dyn HealStorageAPI> = Arc::new(MockStorage);
let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
class: WorkloadClass::ForegroundRead,
active: 10,
limit: 10,
state: AdmissionState::Saturated,
@@ -4387,6 +4471,7 @@ mod tests {
max_concurrent_heals: 1,
mainline_throttle_enable: true,
mainline_read_utilization_high_percent: 80,
mainline_write_utilization_high_percent: 80,
mainline_max_sleep: Duration::from_millis(1),
..HealConfig::default()
}),