mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
feat(scanner): coordinate usage and workload boundaries (#7093)
* test(scanner): wire usage and heal rebuild gates * docs(scanner): define usage authority protocol * docs(heal): clarify scanner and ecstore boundaries * refactor(scanner): split metrics from contracts * feat(scanner): use shared workload snapshots * fix(ecstore): recheck capacity before decommission drain
This commit is contained in:
@@ -73,8 +73,9 @@ hotpath-cpu = [
|
||||
hotpath.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
rustfs-common = { workspace = true }
|
||||
rustfs-concurrency = { workspace = true }
|
||||
rustfs-heal-contracts = { workspace = true }
|
||||
rustfs-scanner-contracts = { workspace = true }
|
||||
rustfs-scanner-metrics = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
tokio = { workspace = true, features = ["fs", "sync", "time", "macros", "rt-multi-thread"] }
|
||||
|
||||
@@ -70,6 +70,7 @@ mod scanner_heal_admission_baseline;
|
||||
pub mod scanner_io;
|
||||
pub mod sleeper;
|
||||
pub(crate) mod storage_api;
|
||||
mod workload_admission;
|
||||
|
||||
pub use data_usage_define::*;
|
||||
pub use error::ScannerError;
|
||||
@@ -80,7 +81,7 @@ pub use remote_scanner::{
|
||||
remote_scanner_request_matches_envelope, serve_remote_scanner_request, validate_remote_scanner_request_fence,
|
||||
};
|
||||
pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config};
|
||||
pub use rustfs_scanner_contracts::last_minute;
|
||||
pub use rustfs_scanner_metrics::last_minute;
|
||||
pub use scanner::{
|
||||
ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason,
|
||||
ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, ScannerUsageStateResetResult,
|
||||
@@ -96,6 +97,7 @@ pub use sleeper::{DynamicSleeper, SCANNER_IDLE_MODE, SCANNER_SLEEPER};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION};
|
||||
pub use workload_admission::set_scanner_workload_admission_snapshot_provider;
|
||||
|
||||
static SCANNER_ACTIVE_WORK_UNITS: AtomicU64 = AtomicU64::new(0);
|
||||
static SCANNER_RUNTIME_INSTANCES: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
@@ -28,7 +28,7 @@ use crate::{
|
||||
use hmac::{Hmac, KeyInit, Mac};
|
||||
use rustfs_credentials::try_get_rpc_token;
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use rustfs_scanner_contracts::metrics::{Metric, Metrics};
|
||||
use rustfs_scanner_metrics::metrics::{Metric, Metrics};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::Sha256;
|
||||
|
||||
@@ -52,7 +52,7 @@ use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELA
|
||||
use rustfs_data_usage::observed_data_usage_is_newer;
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_contracts::metrics::{
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
|
||||
emit_scan_cycle_complete, emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded,
|
||||
global_metrics,
|
||||
|
||||
@@ -50,7 +50,7 @@ use rustfs_heal_contracts::heal_channel::{
|
||||
HEAL_DELETE_DANGLING, HealAdmissionDropReason, HealAdmissionResult, HealChannelPriority, HealChannelRequest,
|
||||
HealRequestSource, HealScanMode, send_heal_request_with_admission,
|
||||
};
|
||||
use rustfs_scanner_contracts::metrics::{
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
|
||||
UpdateCurrentPathFn, current_path_updater, global_metrics,
|
||||
};
|
||||
|
||||
@@ -30,7 +30,7 @@ use rustfs_data_usage::{BucketTargetUsageInfo, BucketUsageInfo};
|
||||
use rustfs_filemeta::FileMeta;
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use rustfs_lock::{LockError, NamespaceLockGuard};
|
||||
use rustfs_scanner_contracts::metrics::{
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
|
||||
@@ -147,13 +147,13 @@ impl Drop for DiskBucketScanActiveGuard {
|
||||
|
||||
pub(super) struct BucketDriveFailureGuard {
|
||||
failed: bool,
|
||||
source: rustfs_scanner_contracts::metrics::ScannerWorkSource,
|
||||
source: rustfs_scanner_metrics::metrics::ScannerWorkSource,
|
||||
bucket: String,
|
||||
drive: String,
|
||||
}
|
||||
|
||||
impl BucketDriveFailureGuard {
|
||||
pub(super) fn new(source: rustfs_scanner_contracts::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
|
||||
pub(super) fn new(source: rustfs_scanner_metrics::metrics::ScannerWorkSource, bucket: &str, drive: &str) -> Self {
|
||||
Self {
|
||||
failed: true,
|
||||
source,
|
||||
@@ -245,7 +245,7 @@ pub(super) fn scanner_concurrency_limit(configured: usize, available: usize) ->
|
||||
return 0;
|
||||
}
|
||||
|
||||
if crate::current_foreground_read_activity() > 0 {
|
||||
if crate::workload_admission::foreground_workload_activity() > 0 {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -285,7 +285,7 @@ pub(super) fn scanner_task_join_error(stage: &str, err: tokio::task::JoinError)
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_scanner_contracts::metrics::{ScannerWorkSource, global_metrics};
|
||||
use rustfs_scanner_metrics::metrics::{ScannerWorkSource, global_metrics};
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
fn active_bucket_drive_count(source: ScannerWorkSource, bucket: &str, drive: &str) -> u64 {
|
||||
|
||||
@@ -161,8 +161,8 @@ impl ScannerIODisk for Disk {
|
||||
let bucket = cache.info.name.clone();
|
||||
let disk_path = self.path().to_string_lossy().to_string();
|
||||
let source = match scan_mode {
|
||||
HealScanMode::Deep => rustfs_scanner_contracts::metrics::ScannerWorkSource::Bitrot,
|
||||
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_contracts::metrics::ScannerWorkSource::Usage,
|
||||
HealScanMode::Deep => rustfs_scanner_metrics::metrics::ScannerWorkSource::Bitrot,
|
||||
HealScanMode::Normal | HealScanMode::Unknown => rustfs_scanner_metrics::metrics::ScannerWorkSource::Usage,
|
||||
};
|
||||
global_metrics().record_scan_bucket_drive_start(source, &bucket, &disk_path);
|
||||
let mut failure_guard = BucketDriveFailureGuard::new(source, &bucket, &disk_path);
|
||||
|
||||
@@ -26,12 +26,32 @@ use crate::{
|
||||
ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests,
|
||||
init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
|
||||
};
|
||||
use rustfs_concurrency::{
|
||||
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
|
||||
WorkloadClass,
|
||||
};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use serial_test::serial;
|
||||
use std::sync::Arc;
|
||||
use temp_env::with_var;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixedWorkloadProvider {
|
||||
snapshot: WorkloadAdmissionRegistrySnapshot,
|
||||
}
|
||||
|
||||
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
|
||||
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
|
||||
self.snapshot.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn install_scanner_workload_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
|
||||
crate::set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
|
||||
}
|
||||
|
||||
fn bucket_info(name: &str) -> BucketInfo {
|
||||
BucketInfo {
|
||||
name: name.to_string(),
|
||||
@@ -1079,6 +1099,7 @@ async fn bucket_cache_pending_heal_reaches_cycle_maintenance_state() {
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(0, 4), 4);
|
||||
}
|
||||
|
||||
@@ -1086,6 +1107,7 @@ fn scanner_concurrency_limit_preserves_available_when_unconfigured() {
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_caps_to_configured_value() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(2, 4), 2);
|
||||
}
|
||||
|
||||
@@ -1093,6 +1115,7 @@ fn scanner_concurrency_limit_caps_to_configured_value() {
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_never_exceeds_available_work() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(8, 4), 4);
|
||||
}
|
||||
|
||||
@@ -1100,6 +1123,7 @@ fn scanner_concurrency_limit_never_exceeds_available_work() {
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_handles_no_available_work() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
assert_eq!(scanner_concurrency_limit(2, 0), 0);
|
||||
}
|
||||
|
||||
@@ -1107,16 +1131,33 @@ fn scanner_concurrency_limit_handles_no_available_work() {
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_yields_to_foreground_reads() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
crate::set_foreground_read_activity(8);
|
||||
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||
assert_eq!(scanner_concurrency_limit(3, 4), 1);
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_yields_to_shared_foreground_pressure() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
install_scanner_workload_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(Some(2), None, Some(16)),
|
||||
]));
|
||||
|
||||
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||
assert_eq!(scanner_concurrency_limit(3, 4), 1);
|
||||
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn scanner_concurrency_limit_yields_to_streaming_reads() {
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::workload_admission::clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
let _guard = crate::ForegroundReadGuard::new();
|
||||
|
||||
assert_eq!(scanner_concurrency_limit(0, 4), 1);
|
||||
|
||||
@@ -20,7 +20,7 @@ use rustfs_config::{
|
||||
DEFAULT_SCANNER_IDLE_MODE, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_IDLE_MODE, ENV_SCANNER_SPEED,
|
||||
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||
};
|
||||
use rustfs_scanner_contracts::metrics::global_metrics;
|
||||
use rustfs_scanner_metrics::metrics::global_metrics;
|
||||
use tokio::time::Duration;
|
||||
|
||||
const MIN_SLEEP: Duration = Duration::from_millis(1);
|
||||
@@ -29,8 +29,8 @@ const SCANNER_SPEED_FAST: u8 = 1;
|
||||
const SCANNER_SPEED_DEFAULT: u8 = 2;
|
||||
const SCANNER_SPEED_SLOW: u8 = 3;
|
||||
const SCANNER_SPEED_SLOWEST: u8 = 4;
|
||||
const FOREGROUND_READ_BACKOFF_PER_REQUEST_MS: u64 = 10;
|
||||
const FOREGROUND_READ_BACKOFF_MAX_MS: u64 = 250;
|
||||
const FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS: u64 = 10;
|
||||
const FOREGROUND_WORKLOAD_BACKOFF_MAX_MS: u64 = 250;
|
||||
|
||||
static SCANNER_DEFAULT_SPEED_PRESET: AtomicU8 = AtomicU8::new(SCANNER_SPEED_DEFAULT);
|
||||
|
||||
@@ -78,15 +78,15 @@ pub(crate) fn scanner_yield_every_n_objects() -> u64 {
|
||||
rustfs_utils::get_env_u64(ENV_SCANNER_YIELD_EVERY_N_OBJECTS, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS)
|
||||
}
|
||||
|
||||
fn foreground_read_backoff_duration(active_reads: u64) -> Duration {
|
||||
if active_reads == 0 {
|
||||
fn foreground_workload_backoff_duration(active_foreground_workloads: u64) -> Duration {
|
||||
if active_foreground_workloads == 0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
|
||||
Duration::from_millis(
|
||||
active_reads
|
||||
.saturating_mul(FOREGROUND_READ_BACKOFF_PER_REQUEST_MS)
|
||||
.min(FOREGROUND_READ_BACKOFF_MAX_MS),
|
||||
active_foreground_workloads
|
||||
.saturating_mul(FOREGROUND_WORKLOAD_BACKOFF_PER_REQUEST_MS)
|
||||
.min(FOREGROUND_WORKLOAD_BACKOFF_MAX_MS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -147,14 +147,15 @@ impl DynamicSleeper {
|
||||
}
|
||||
let (factor, max_sleep) = self.read_params();
|
||||
if factor == 0.0 || max_sleep.is_zero() {
|
||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
||||
let foreground_sleep =
|
||||
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
if !foreground_sleep.is_zero() {
|
||||
tokio::time::sleep(foreground_sleep).await;
|
||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
||||
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let sleep_dur = Duration::from_secs_f64(MIN_SLEEP.as_secs_f64() * factor)
|
||||
.min(max_sleep)
|
||||
.max(foreground_sleep);
|
||||
@@ -235,7 +236,8 @@ impl SleepTimer {
|
||||
}
|
||||
let (factor, max_sleep) = self.sleeper.read_params();
|
||||
if factor == 0.0 || max_sleep.is_zero() {
|
||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
||||
let foreground_sleep =
|
||||
foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
if !foreground_sleep.is_zero() {
|
||||
tokio::time::sleep(foreground_sleep).await;
|
||||
global_metrics().record_scanner_throttle_sleep(foreground_sleep);
|
||||
@@ -243,7 +245,7 @@ impl SleepTimer {
|
||||
return;
|
||||
}
|
||||
let elapsed = self.start.elapsed();
|
||||
let foreground_sleep = foreground_read_backoff_duration(crate::current_foreground_read_activity());
|
||||
let foreground_sleep = foreground_workload_backoff_duration(crate::workload_admission::foreground_workload_activity());
|
||||
let sleep_dur = Duration::from_secs_f64(elapsed.as_secs_f64() * factor)
|
||||
.max(MIN_SLEEP)
|
||||
.min(max_sleep)
|
||||
@@ -304,10 +306,10 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_read_backoff_is_capped() {
|
||||
assert_eq!(foreground_read_backoff_duration(0), Duration::ZERO);
|
||||
assert_eq!(foreground_read_backoff_duration(1), Duration::from_millis(10));
|
||||
assert_eq!(foreground_read_backoff_duration(80), Duration::from_millis(250));
|
||||
fn foreground_workload_backoff_is_capped() {
|
||||
assert_eq!(foreground_workload_backoff_duration(0), Duration::ZERO);
|
||||
assert_eq!(foreground_workload_backoff_duration(1), Duration::from_millis(10));
|
||||
assert_eq!(foreground_workload_backoff_duration(80), Duration::from_millis(250));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshotProvider, WorkloadClass};
|
||||
|
||||
type WorkloadSnapshotProviderRef = Arc<dyn WorkloadAdmissionSnapshotProvider + Send + Sync>;
|
||||
|
||||
static SCANNER_WORKLOAD_ADMISSION_PROVIDER: LazyLock<RwLock<Option<WorkloadSnapshotProviderRef>>> =
|
||||
LazyLock::new(|| RwLock::new(None));
|
||||
|
||||
pub fn set_scanner_workload_admission_snapshot_provider(provider: WorkloadSnapshotProviderRef) {
|
||||
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||
.write()
|
||||
.unwrap_or_else(|err| err.into_inner()) = Some(provider);
|
||||
}
|
||||
|
||||
fn scanner_workload_admission_snapshot_provider() -> Option<WorkloadSnapshotProviderRef> {
|
||||
SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||
.read()
|
||||
.unwrap_or_else(|err| err.into_inner())
|
||||
.clone()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn clear_scanner_workload_admission_snapshot_provider_for_test() {
|
||||
*SCANNER_WORKLOAD_ADMISSION_PROVIDER
|
||||
.write()
|
||||
.unwrap_or_else(|err| err.into_inner()) = None;
|
||||
}
|
||||
|
||||
pub(crate) fn foreground_workload_activity() -> u64 {
|
||||
let local_activity = crate::current_foreground_read_activity();
|
||||
let Some(provider) = scanner_workload_admission_snapshot_provider() else {
|
||||
return local_activity;
|
||||
};
|
||||
|
||||
local_activity.max(foreground_activity_from_snapshot(&provider.workload_admission_snapshot()))
|
||||
}
|
||||
|
||||
fn foreground_activity_from_snapshot(snapshot: &WorkloadAdmissionRegistrySnapshot) -> u64 {
|
||||
[WorkloadClass::ForegroundRead, WorkloadClass::ForegroundWrite]
|
||||
.into_iter()
|
||||
.filter_map(|class| snapshot.get(class))
|
||||
.map(|entry| {
|
||||
entry
|
||||
.active
|
||||
.or_else(|| {
|
||||
matches!(entry.state, AdmissionState::Saturated).then(|| entry.limit.filter(|limit| *limit > 0).unwrap_or(1))
|
||||
})
|
||||
.map(usize_to_u64_saturated)
|
||||
.unwrap_or(0)
|
||||
})
|
||||
.max()
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
fn usize_to_u64_saturated(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshot};
|
||||
use serial_test::serial;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixedWorkloadProvider {
|
||||
snapshot: WorkloadAdmissionRegistrySnapshot,
|
||||
}
|
||||
|
||||
impl WorkloadAdmissionSnapshotProvider for FixedWorkloadProvider {
|
||||
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot {
|
||||
self.snapshot.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn install_provider(snapshot: WorkloadAdmissionRegistrySnapshot) {
|
||||
set_scanner_workload_admission_snapshot_provider(Arc::new(FixedWorkloadProvider { snapshot }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_workload_activity_falls_back_to_local_read_activity() {
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
crate::set_foreground_read_activity(3);
|
||||
|
||||
assert_eq!(foreground_workload_activity(), 3);
|
||||
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_workload_activity_uses_shared_provider_counts() {
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Open).with_counts(
|
||||
Some(5),
|
||||
None,
|
||||
Some(8),
|
||||
),
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Open).with_counts(
|
||||
Some(2),
|
||||
None,
|
||||
Some(4),
|
||||
),
|
||||
]));
|
||||
|
||||
assert_eq!(foreground_workload_activity(), 5);
|
||||
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_workload_activity_treats_saturation_without_counts_as_pressure() {
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundRead, AdmissionState::Saturated).with_counts(
|
||||
None,
|
||||
None,
|
||||
Some(7),
|
||||
),
|
||||
]));
|
||||
|
||||
assert_eq!(foreground_workload_activity(), 7);
|
||||
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn foreground_workload_activity_treats_zero_limit_saturation_as_pressure() {
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
crate::reset_foreground_read_activity_for_test();
|
||||
install_provider(WorkloadAdmissionRegistrySnapshot::new(vec![
|
||||
WorkloadAdmissionSnapshot::new(WorkloadClass::ForegroundWrite, AdmissionState::Saturated).with_counts(
|
||||
None,
|
||||
None,
|
||||
Some(0),
|
||||
),
|
||||
]));
|
||||
|
||||
assert_eq!(foreground_workload_activity(), 1);
|
||||
|
||||
clear_scanner_workload_admission_snapshot_provider_for_test();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user