mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 16:59:52 +00:00
feat(scanner): add adaptive pacing controls (#3315)
* feat(scanner): add adaptive pacing controls * fix(scanner): bound pacing delay status precision --------- Co-authored-by: Henry Guo <marshawcoco@users.noreply.github.com>
This commit is contained in:
@@ -65,6 +65,8 @@ Current guidance:
|
|||||||
## Scanner environment aliases
|
## Scanner environment aliases
|
||||||
|
|
||||||
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
- `RUSTFS_SCANNER_SPEED` (canonical, also accepts `MINIO_SCANNER_SPEED`)
|
||||||
|
- `RUSTFS_SCANNER_DELAY` (canonical)
|
||||||
|
- `RUSTFS_SCANNER_MAX_WAIT_SECS` (canonical)
|
||||||
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
|
- `RUSTFS_SCANNER_CYCLE` (canonical, also accepts `MINIO_SCANNER_CYCLE`)
|
||||||
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
|
- `RUSTFS_SCANNER_START_DELAY_SECS` (canonical)
|
||||||
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
|
- `RUSTFS_DATA_SCANNER_START_DELAY_SECS` (deprecated alias for compatibility)
|
||||||
|
|||||||
@@ -20,6 +20,12 @@ pub const SCANNER_SUB_SYS: &str = "scanner";
|
|||||||
/// Scanner config key selecting the speed preset.
|
/// Scanner config key selecting the speed preset.
|
||||||
pub const SCANNER_SPEED: &str = "speed";
|
pub const SCANNER_SPEED: &str = "speed";
|
||||||
|
|
||||||
|
/// Scanner config key overriding the scanner sleep multiplier.
|
||||||
|
pub const SCANNER_DELAY: &str = "delay";
|
||||||
|
|
||||||
|
/// Scanner config key overriding the maximum scanner sleep in seconds.
|
||||||
|
pub const SCANNER_MAX_WAIT: &str = "max_wait";
|
||||||
|
|
||||||
/// Scanner config key overriding the cycle interval in seconds.
|
/// Scanner config key overriding the cycle interval in seconds.
|
||||||
pub const SCANNER_CYCLE: &str = "cycle";
|
pub const SCANNER_CYCLE: &str = "cycle";
|
||||||
|
|
||||||
@@ -68,6 +74,8 @@ pub const SCANNER_ALERT_EXCESS_FOLDERS: &str = "alert_excess_folders";
|
|||||||
/// Scanner config keys supported by the admin config subsystem.
|
/// Scanner config keys supported by the admin config subsystem.
|
||||||
pub const SCANNER_KEYS: &[&str] = &[
|
pub const SCANNER_KEYS: &[&str] = &[
|
||||||
SCANNER_SPEED,
|
SCANNER_SPEED,
|
||||||
|
SCANNER_DELAY,
|
||||||
|
SCANNER_MAX_WAIT,
|
||||||
SCANNER_CYCLE,
|
SCANNER_CYCLE,
|
||||||
SCANNER_START_DELAY,
|
SCANNER_START_DELAY,
|
||||||
SCANNER_CYCLE_MAX_DURATION,
|
SCANNER_CYCLE_MAX_DURATION,
|
||||||
@@ -123,6 +131,15 @@ pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DI
|
|||||||
/// - Example: `export RUSTFS_SCANNER_SPEED=slow`
|
/// - Example: `export RUSTFS_SCANNER_SPEED=slow`
|
||||||
pub const ENV_SCANNER_SPEED: &str = "RUSTFS_SCANNER_SPEED";
|
pub const ENV_SCANNER_SPEED: &str = "RUSTFS_SCANNER_SPEED";
|
||||||
|
|
||||||
|
/// Environment variable that overrides the scanner sleep multiplier.
|
||||||
|
/// - Example: `export RUSTFS_SCANNER_DELAY=30.0`
|
||||||
|
pub const ENV_SCANNER_DELAY: &str = "RUSTFS_SCANNER_DELAY";
|
||||||
|
|
||||||
|
/// Environment variable that overrides the maximum scanner sleep in seconds.
|
||||||
|
/// - Unit: seconds (u64).
|
||||||
|
/// - Example: `export RUSTFS_SCANNER_MAX_WAIT_SECS=15`
|
||||||
|
pub const ENV_SCANNER_MAX_WAIT_SECS: &str = "RUSTFS_SCANNER_MAX_WAIT_SECS";
|
||||||
|
|
||||||
/// Default scanner speed preset.
|
/// Default scanner speed preset.
|
||||||
pub const DEFAULT_SCANNER_SPEED: &str = "default";
|
pub const DEFAULT_SCANNER_SPEED: &str = "default";
|
||||||
|
|
||||||
@@ -237,7 +254,9 @@ pub const DEFAULT_SCANNER_INLINE_HEAL_ENABLE: bool = false;
|
|||||||
/// | `slow` | 10x | 15 seconds| 1 minute |
|
/// | `slow` | 10x | 15 seconds| 1 minute |
|
||||||
/// | `slowest` | 100x | 15 seconds| 30 minutes |
|
/// | `slowest` | 100x | 15 seconds| 30 minutes |
|
||||||
///
|
///
|
||||||
/// The cycle interval can be overridden by `RUSTFS_SCANNER_CYCLE`.
|
/// The sleep factor, max sleep, and cycle interval can be overridden by
|
||||||
|
/// `RUSTFS_SCANNER_DELAY`, `RUSTFS_SCANNER_MAX_WAIT_SECS`, and
|
||||||
|
/// `RUSTFS_SCANNER_CYCLE`.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||||
pub enum ScannerSpeed {
|
pub enum ScannerSpeed {
|
||||||
Fastest,
|
Fastest,
|
||||||
|
|||||||
@@ -289,7 +289,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, DEFAULT_SCANNER_SPEED, HEAL_BITROT_CYCLE, SCANNER_CYCLE_MAX_OBJECTS,
|
DEFAULT_DELIMITER, DEFAULT_HEAL_BITROT_CYCLE_SECS, DEFAULT_SCANNER_SPEED, HEAL_BITROT_CYCLE, SCANNER_CYCLE_MAX_OBJECTS,
|
||||||
SCANNER_SPEED, SCANNER_SUB_SYS,
|
SCANNER_DELAY, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_SUB_SYS,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -318,6 +318,8 @@ mod tests {
|
|||||||
.expect("scanner defaults should exist");
|
.expect("scanner defaults should exist");
|
||||||
|
|
||||||
assert_eq!(scanner_kvs.get(SCANNER_SPEED), DEFAULT_SCANNER_SPEED);
|
assert_eq!(scanner_kvs.get(SCANNER_SPEED), DEFAULT_SCANNER_SPEED);
|
||||||
|
assert_eq!(scanner_kvs.get(SCANNER_DELAY), "");
|
||||||
|
assert_eq!(scanner_kvs.get(SCANNER_MAX_WAIT), "");
|
||||||
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
|
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
|
||||||
|
|
||||||
let heal_kvs = cfg
|
let heal_kvs = cfg
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ use rustfs_config::{
|
|||||||
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, DEFAULT_SCANNER_SPEED,
|
DEFAULT_SCANNER_MAX_CONCURRENT_DISK_SCANS, DEFAULT_SCANNER_MAX_CONCURRENT_SET_SCANS, DEFAULT_SCANNER_SPEED,
|
||||||
DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
|
DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE,
|
||||||
SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE,
|
||||||
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_IDLE_MODE,
|
SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE,
|
||||||
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_SPEED, SCANNER_START_DELAY,
|
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY,
|
||||||
SCANNER_YIELD_EVERY_N_OBJECTS,
|
SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||||
};
|
};
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
@@ -32,6 +32,16 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
|||||||
value: DEFAULT_SCANNER_SPEED.to_owned(),
|
value: DEFAULT_SCANNER_SPEED.to_owned(),
|
||||||
hidden_if_empty: false,
|
hidden_if_empty: false,
|
||||||
},
|
},
|
||||||
|
KV {
|
||||||
|
key: SCANNER_DELAY.to_owned(),
|
||||||
|
value: String::new(),
|
||||||
|
hidden_if_empty: true,
|
||||||
|
},
|
||||||
|
KV {
|
||||||
|
key: SCANNER_MAX_WAIT.to_owned(),
|
||||||
|
value: String::new(),
|
||||||
|
hidden_if_empty: true,
|
||||||
|
},
|
||||||
KV {
|
KV {
|
||||||
key: SCANNER_CYCLE.to_owned(),
|
key: SCANNER_CYCLE.to_owned(),
|
||||||
value: String::new(),
|
value: String::new(),
|
||||||
|
|||||||
@@ -22,12 +22,13 @@ use rustfs_config::{
|
|||||||
DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
DEFAULT_SCANNER_SPEED, DEFAULT_SCANNER_YIELD_EVERY_N_OBJECTS, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
||||||
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
|
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
|
||||||
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
|
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_IDLE_MODE,
|
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE,
|
||||||
ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
|
ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||||
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_ALERT_EXCESS_FOLDERS,
|
ENV_SCANNER_START_DELAY_SECS, ENV_SCANNER_YIELD_EVERY_N_OBJECTS, HEAL_BITROT_CYCLE, HEAL_SUB_SYS,
|
||||||
SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT,
|
SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE,
|
||||||
SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_IDLE_MODE,
|
SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
|
||||||
SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_SUB_SYS,
|
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS,
|
||||||
|
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_SUB_SYS,
|
||||||
SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
SCANNER_YIELD_EVERY_N_OBJECTS, ScannerSpeed,
|
||||||
};
|
};
|
||||||
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
||||||
@@ -40,6 +41,8 @@ use tracing::warn;
|
|||||||
|
|
||||||
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
|
||||||
const NO_DEFAULT_CYCLE_OVERRIDE: u64 = 0;
|
const NO_DEFAULT_CYCLE_OVERRIDE: u64 = 0;
|
||||||
|
const MAX_SCANNER_DELAY_FACTOR: f64 = 10_000.0;
|
||||||
|
const SCANNER_DELAY_RANGE_REASON: &str = "expected scanner delay between 0 and 10000";
|
||||||
|
|
||||||
static SCANNER_DEFAULT_CYCLE_SECS: AtomicU64 = AtomicU64::new(NO_DEFAULT_CYCLE_OVERRIDE);
|
static SCANNER_DEFAULT_CYCLE_SECS: AtomicU64 = AtomicU64::new(NO_DEFAULT_CYCLE_OVERRIDE);
|
||||||
|
|
||||||
@@ -55,10 +58,14 @@ pub enum ScannerRuntimeConfigSource {
|
|||||||
Default,
|
Default,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
pub(crate) struct ScannerRuntimeConfig {
|
pub(crate) struct ScannerRuntimeConfig {
|
||||||
pub(crate) speed: ScannerSpeed,
|
pub(crate) speed: ScannerSpeed,
|
||||||
pub(crate) speed_source: ScannerRuntimeConfigSource,
|
pub(crate) speed_source: ScannerRuntimeConfigSource,
|
||||||
|
pub(crate) delay: f64,
|
||||||
|
pub(crate) delay_source: ScannerRuntimeConfigSource,
|
||||||
|
pub(crate) max_wait: Duration,
|
||||||
|
pub(crate) max_wait_source: ScannerRuntimeConfigSource,
|
||||||
pub(crate) idle_mode: bool,
|
pub(crate) idle_mode: bool,
|
||||||
pub(crate) idle_mode_source: ScannerRuntimeConfigSource,
|
pub(crate) idle_mode_source: ScannerRuntimeConfigSource,
|
||||||
pub(crate) start_delay: Option<Duration>,
|
pub(crate) start_delay: Option<Duration>,
|
||||||
@@ -92,6 +99,10 @@ impl Default for ScannerRuntimeConfig {
|
|||||||
Self {
|
Self {
|
||||||
speed: scanner_default_speed(),
|
speed: scanner_default_speed(),
|
||||||
speed_source: ScannerRuntimeConfigSource::Default,
|
speed_source: ScannerRuntimeConfigSource::Default,
|
||||||
|
delay: scanner_default_speed().sleep_factor(),
|
||||||
|
delay_source: ScannerRuntimeConfigSource::Default,
|
||||||
|
max_wait: scanner_default_speed().max_sleep(),
|
||||||
|
max_wait_source: ScannerRuntimeConfigSource::Default,
|
||||||
idle_mode: DEFAULT_SCANNER_IDLE_MODE,
|
idle_mode: DEFAULT_SCANNER_IDLE_MODE,
|
||||||
idle_mode_source: ScannerRuntimeConfigSource::Default,
|
idle_mode_source: ScannerRuntimeConfigSource::Default,
|
||||||
start_delay: None,
|
start_delay: None,
|
||||||
@@ -143,6 +154,8 @@ pub struct ScannerRuntimeConfigValue<T> {
|
|||||||
#[derive(Clone, Debug, Serialize)]
|
#[derive(Clone, Debug, Serialize)]
|
||||||
pub struct ScannerRuntimeConfigStatus {
|
pub struct ScannerRuntimeConfigStatus {
|
||||||
pub speed: ScannerRuntimeConfigValue<String>,
|
pub speed: ScannerRuntimeConfigValue<String>,
|
||||||
|
pub delay: ScannerRuntimeConfigValue<f64>,
|
||||||
|
pub max_wait_seconds: ScannerRuntimeConfigValue<f64>,
|
||||||
pub idle_mode: ScannerRuntimeConfigValue<bool>,
|
pub idle_mode: ScannerRuntimeConfigValue<bool>,
|
||||||
pub start_delay_seconds: ScannerRuntimeConfigValue<Option<u64>>,
|
pub start_delay_seconds: ScannerRuntimeConfigValue<Option<u64>>,
|
||||||
pub cycle_interval_seconds: ScannerRuntimeConfigValue<u64>,
|
pub cycle_interval_seconds: ScannerRuntimeConfigValue<u64>,
|
||||||
@@ -246,6 +259,27 @@ fn parse_config_speed(value: String) -> Result<ScannerSpeed, ScannerRuntimeConfi
|
|||||||
ScannerSpeed::parse_str(&value).ok_or_else(|| invalid_value(SCANNER_SPEED, value, "expected scanner speed preset"))
|
ScannerSpeed::parse_str(&value).ok_or_else(|| invalid_value(SCANNER_SPEED, value, "expected scanner speed preset"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_config_delay(key: &'static str, value: String) -> Result<f64, ScannerRuntimeConfigError> {
|
||||||
|
let parsed = value
|
||||||
|
.parse::<f64>()
|
||||||
|
.map_err(|_| invalid_value(key, value.clone(), SCANNER_DELAY_RANGE_REASON))?;
|
||||||
|
if parsed.is_finite() && (0.0..=MAX_SCANNER_DELAY_FACTOR).contains(&parsed) {
|
||||||
|
Ok(parsed)
|
||||||
|
} else {
|
||||||
|
Err(invalid_value(key, value, SCANNER_DELAY_RANGE_REASON))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_env_delay(key: &'static str, value: String, fallback: f64) -> f64 {
|
||||||
|
match parse_config_delay(key, value.clone()) {
|
||||||
|
Ok(parsed) => parsed,
|
||||||
|
Err(_) => {
|
||||||
|
warn!(env = key, value, fallback, "Invalid scanner delay config, using derived value");
|
||||||
|
fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_config_bitrot_cycle(key: &'static str, value: String) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
|
fn parse_config_bitrot_cycle(key: &'static str, value: String) -> Result<Option<Duration>, ScannerRuntimeConfigError> {
|
||||||
match value.trim().to_ascii_lowercase().as_str() {
|
match value.trim().to_ascii_lowercase().as_str() {
|
||||||
"0" | "true" | "on" | "yes" => Ok(Some(Duration::ZERO)),
|
"0" | "true" | "on" | "yes" => Ok(Some(Duration::ZERO)),
|
||||||
@@ -304,6 +338,10 @@ fn validate_persisted_scanner_runtime_config(config: &ServerConfig) -> Result<()
|
|||||||
if let Some(value) = config_value(scanner_kvs, SCANNER_SPEED, DEFAULT_SCANNER_SPEED) {
|
if let Some(value) = config_value(scanner_kvs, SCANNER_SPEED, DEFAULT_SCANNER_SPEED) {
|
||||||
parse_config_speed(value)?;
|
parse_config_speed(value)?;
|
||||||
}
|
}
|
||||||
|
if let Some(value) = config_value(scanner_kvs, SCANNER_DELAY, "") {
|
||||||
|
parse_config_delay(SCANNER_DELAY, value)?;
|
||||||
|
}
|
||||||
|
validate_optional_config_u64(scanner_kvs, SCANNER_MAX_WAIT, "")?;
|
||||||
if let Some(value) = config_value(scanner_kvs, SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE) {
|
if let Some(value) = config_value(scanner_kvs, SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE) {
|
||||||
parse_config_bool(SCANNER_IDLE_MODE, value)?;
|
parse_config_bool(SCANNER_IDLE_MODE, value)?;
|
||||||
}
|
}
|
||||||
@@ -341,6 +379,36 @@ fn lookup_speed(kvs: Option<&KVS>) -> Result<(ScannerSpeed, ScannerRuntimeConfig
|
|||||||
Ok((scanner_default_speed(), ScannerRuntimeConfigSource::Default))
|
Ok((scanner_default_speed(), ScannerRuntimeConfigSource::Default))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn lookup_delay(
|
||||||
|
kvs: Option<&KVS>,
|
||||||
|
speed: ScannerSpeed,
|
||||||
|
speed_source: ScannerRuntimeConfigSource,
|
||||||
|
) -> Result<(f64, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||||
|
let derived = speed.sleep_factor();
|
||||||
|
if let Some(value) = rustfs_utils::get_env_opt_str(ENV_SCANNER_DELAY) {
|
||||||
|
return Ok((parse_env_delay(ENV_SCANNER_DELAY, value, derived), ScannerRuntimeConfigSource::Env));
|
||||||
|
}
|
||||||
|
if let Some(value) = config_value(kvs, SCANNER_DELAY, "") {
|
||||||
|
return parse_config_delay(SCANNER_DELAY, value).map(|delay| (delay, ScannerRuntimeConfigSource::Config));
|
||||||
|
}
|
||||||
|
Ok((derived, speed_source))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lookup_max_wait(
|
||||||
|
kvs: Option<&KVS>,
|
||||||
|
speed: ScannerSpeed,
|
||||||
|
speed_source: ScannerRuntimeConfigSource,
|
||||||
|
) -> Result<(Duration, ScannerRuntimeConfigSource), ScannerRuntimeConfigError> {
|
||||||
|
if let Some(secs) = rustfs_utils::get_env_opt_u64(ENV_SCANNER_MAX_WAIT_SECS) {
|
||||||
|
return Ok((Duration::from_secs(secs), ScannerRuntimeConfigSource::Env));
|
||||||
|
}
|
||||||
|
if let Some(value) = config_value(kvs, SCANNER_MAX_WAIT, "") {
|
||||||
|
return parse_config_u64(SCANNER_MAX_WAIT, value)
|
||||||
|
.map(|secs| (Duration::from_secs(secs), ScannerRuntimeConfigSource::Config));
|
||||||
|
}
|
||||||
|
Ok((speed.max_sleep(), speed_source))
|
||||||
|
}
|
||||||
|
|
||||||
fn lookup_optional_seconds(
|
fn lookup_optional_seconds(
|
||||||
kvs: Option<&KVS>,
|
kvs: Option<&KVS>,
|
||||||
key: &'static str,
|
key: &'static str,
|
||||||
@@ -437,6 +505,8 @@ pub(crate) fn lookup_scanner_runtime_config(
|
|||||||
let heal_kvs = heal_kvs(config);
|
let heal_kvs = heal_kvs(config);
|
||||||
let heal_kvs = heal_kvs.as_ref();
|
let heal_kvs = heal_kvs.as_ref();
|
||||||
let (speed, speed_source) = lookup_speed(scanner_kvs)?;
|
let (speed, speed_source) = lookup_speed(scanner_kvs)?;
|
||||||
|
let (delay, delay_source) = lookup_delay(scanner_kvs, speed, speed_source)?;
|
||||||
|
let (max_wait, max_wait_source) = lookup_max_wait(scanner_kvs, speed, speed_source)?;
|
||||||
let (idle_mode, idle_mode_source) =
|
let (idle_mode, idle_mode_source) =
|
||||||
lookup_bool(scanner_kvs, SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE)?;
|
lookup_bool(scanner_kvs, SCANNER_IDLE_MODE, ENV_SCANNER_IDLE_MODE, DEFAULT_SCANNER_IDLE_MODE)?;
|
||||||
let (start_delay, start_delay_source) = lookup_start_delay(scanner_kvs)?;
|
let (start_delay, start_delay_source) = lookup_start_delay(scanner_kvs)?;
|
||||||
@@ -542,6 +612,10 @@ pub(crate) fn lookup_scanner_runtime_config(
|
|||||||
Ok(ScannerRuntimeConfig {
|
Ok(ScannerRuntimeConfig {
|
||||||
speed,
|
speed,
|
||||||
speed_source,
|
speed_source,
|
||||||
|
delay,
|
||||||
|
delay_source,
|
||||||
|
max_wait,
|
||||||
|
max_wait_source,
|
||||||
idle_mode,
|
idle_mode,
|
||||||
idle_mode_source,
|
idle_mode_source,
|
||||||
start_delay,
|
start_delay,
|
||||||
@@ -572,7 +646,7 @@ pub(crate) fn lookup_scanner_runtime_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn apply_resolved_runtime_config(config: ScannerRuntimeConfig) {
|
fn apply_resolved_runtime_config(config: ScannerRuntimeConfig) {
|
||||||
SCANNER_SLEEPER.update_from_runtime_config(config.speed, config.idle_mode, config.yield_every_n_objects);
|
SCANNER_SLEEPER.update_from_runtime_config(config.delay, config.max_wait, config.idle_mode, config.yield_every_n_objects);
|
||||||
if let Ok(mut guard) = SCANNER_RUNTIME_CONFIG.write() {
|
if let Ok(mut guard) = SCANNER_RUNTIME_CONFIG.write() {
|
||||||
*guard = config;
|
*guard = config;
|
||||||
}
|
}
|
||||||
@@ -614,6 +688,14 @@ pub fn scanner_runtime_config_status() -> ScannerRuntimeConfigStatus {
|
|||||||
value: config.speed.to_string(),
|
value: config.speed.to_string(),
|
||||||
source: config.speed_source,
|
source: config.speed_source,
|
||||||
},
|
},
|
||||||
|
delay: ScannerRuntimeConfigValue {
|
||||||
|
value: config.delay,
|
||||||
|
source: config.delay_source,
|
||||||
|
},
|
||||||
|
max_wait_seconds: ScannerRuntimeConfigValue {
|
||||||
|
value: config.max_wait.as_secs_f64(),
|
||||||
|
source: config.max_wait_source,
|
||||||
|
},
|
||||||
idle_mode: ScannerRuntimeConfigValue {
|
idle_mode: ScannerRuntimeConfigValue {
|
||||||
value: config.idle_mode,
|
value: config.idle_mode,
|
||||||
source: config.idle_mode_source,
|
source: config.idle_mode_source,
|
||||||
@@ -718,9 +800,10 @@ mod tests {
|
|||||||
use super::{ScannerRuntimeConfigSource, lookup_scanner_runtime_config, validate_scanner_runtime_config};
|
use super::{ScannerRuntimeConfigSource, lookup_scanner_runtime_config, validate_scanner_runtime_config};
|
||||||
use rustfs_config::{
|
use rustfs_config::{
|
||||||
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
DEFAULT_DELIMITER, ENV_SCANNER_BITROT_CYCLE_SECS, ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE,
|
||||||
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, SCANNER_BITROT_CYCLE,
|
ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED, HEAL_BITROT_CYCLE,
|
||||||
SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
|
HEAL_SUB_SYS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||||
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS, ScannerSpeed,
|
SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY, SCANNER_IDLE_MODE, SCANNER_SPEED, SCANNER_SUB_SYS,
|
||||||
|
ScannerSpeed,
|
||||||
};
|
};
|
||||||
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
use rustfs_ecstore::config::{Config as ServerConfig, KVS};
|
||||||
use serial_test::serial;
|
use serial_test::serial;
|
||||||
@@ -864,6 +947,42 @@ mod tests {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_runtime_config_validation_rejects_invalid_persisted_delay() {
|
||||||
|
let config = server_config_with_scanner(&[(SCANNER_DELAY, "-1")]);
|
||||||
|
|
||||||
|
let err = validate_scanner_runtime_config(&config).expect_err("persisted scanner delay should be validated");
|
||||||
|
assert!(err.to_string().contains("delay"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_runtime_config_validation_rejects_excessive_persisted_delay() {
|
||||||
|
let config = server_config_with_scanner(&[(SCANNER_DELAY, "10000.1")]);
|
||||||
|
|
||||||
|
let err = validate_scanner_runtime_config(&config).expect_err("persisted scanner delay should be bounded");
|
||||||
|
assert!(err.to_string().contains("delay"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scanner_runtime_config_validation_accepts_delay_upper_bound() {
|
||||||
|
let config = server_config_with_scanner(&[(SCANNER_DELAY, "10000")]);
|
||||||
|
|
||||||
|
validate_scanner_runtime_config(&config).expect("delay upper bound should be accepted");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn scanner_runtime_config_uses_derived_delay_for_excessive_env_override() {
|
||||||
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "slow")]);
|
||||||
|
|
||||||
|
with_var(ENV_SCANNER_DELAY, Some("1e308"), || {
|
||||||
|
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||||
|
|
||||||
|
assert_eq!(resolved.delay, ScannerSpeed::Slow.sleep_factor());
|
||||||
|
assert_eq!(resolved.delay_source, ScannerRuntimeConfigSource::Env);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn scanner_runtime_config_validation_rejects_non_default_target() {
|
fn scanner_runtime_config_validation_rejects_non_default_target() {
|
||||||
let config = server_config_with_scanner_target("analytics", &[(SCANNER_SPEED, "slow")]);
|
let config = server_config_with_scanner_target("analytics", &[(SCANNER_SPEED, "slow")]);
|
||||||
@@ -892,4 +1011,67 @@ mod tests {
|
|||||||
});
|
});
|
||||||
super::refresh_scanner_runtime_config_for_tests();
|
super::refresh_scanner_runtime_config_for_tests();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn scanner_runtime_config_status_reports_persisted_pacing_overrides() {
|
||||||
|
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||||
|
|
||||||
|
with_var_unset("RUSTFS_SCANNER_DELAY", || {
|
||||||
|
with_var_unset("RUSTFS_SCANNER_MAX_WAIT_SECS", || {
|
||||||
|
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||||
|
super::apply_resolved_runtime_config(resolved);
|
||||||
|
|
||||||
|
let encoded =
|
||||||
|
serde_json::to_value(super::scanner_runtime_config_status()).expect("scanner status should serialize");
|
||||||
|
|
||||||
|
assert_eq!(encoded["delay"]["value"], 3.5);
|
||||||
|
assert_eq!(encoded["delay"]["source"], "config");
|
||||||
|
assert_eq!(encoded["max_wait_seconds"]["value"], 7.0);
|
||||||
|
assert_eq!(encoded["max_wait_seconds"]["source"], "config");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
super::refresh_scanner_runtime_config_for_tests();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn scanner_runtime_config_status_prefers_env_pacing_overrides() {
|
||||||
|
let config = server_config_with_scanner(&[("delay", "3.5"), ("max_wait", "7")]);
|
||||||
|
|
||||||
|
with_var("RUSTFS_SCANNER_DELAY", Some("1.25"), || {
|
||||||
|
with_var("RUSTFS_SCANNER_MAX_WAIT_SECS", Some("2"), || {
|
||||||
|
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||||
|
super::apply_resolved_runtime_config(resolved);
|
||||||
|
|
||||||
|
let encoded =
|
||||||
|
serde_json::to_value(super::scanner_runtime_config_status()).expect("scanner status should serialize");
|
||||||
|
|
||||||
|
assert_eq!(encoded["delay"]["value"], 1.25);
|
||||||
|
assert_eq!(encoded["delay"]["source"], "env");
|
||||||
|
assert_eq!(encoded["max_wait_seconds"]["value"], 2.0);
|
||||||
|
assert_eq!(encoded["max_wait_seconds"]["source"], "env");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
super::refresh_scanner_runtime_config_for_tests();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial]
|
||||||
|
fn scanner_runtime_config_status_preserves_subsecond_max_wait() {
|
||||||
|
let config = server_config_with_scanner(&[(SCANNER_SPEED, "fast")]);
|
||||||
|
|
||||||
|
with_var_unset(ENV_SCANNER_SPEED, || {
|
||||||
|
with_var_unset(ENV_SCANNER_MAX_WAIT_SECS, || {
|
||||||
|
let resolved = lookup_scanner_runtime_config(Some(&config)).expect("scanner runtime config");
|
||||||
|
super::apply_resolved_runtime_config(resolved);
|
||||||
|
|
||||||
|
let status = super::scanner_runtime_config_status();
|
||||||
|
|
||||||
|
assert!((status.max_wait_seconds.value - 0.1).abs() < f64::EPSILON);
|
||||||
|
assert_eq!(status.max_wait_seconds.source, ScannerRuntimeConfigSource::Config);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
super::refresh_scanner_runtime_config_for_tests();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,22 +152,32 @@ impl DynamicSleeper {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn update_params(&self, factor: f64, max_sleep: Duration) {
|
||||||
|
let mut f = self.inner.factor.write().unwrap_or_else(|e| e.into_inner());
|
||||||
|
*f = factor;
|
||||||
|
let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner());
|
||||||
|
*m = max_sleep;
|
||||||
|
}
|
||||||
|
|
||||||
/// Swap in parameters from a new speed preset (for runtime reconfiguration).
|
/// Swap in parameters from a new speed preset (for runtime reconfiguration).
|
||||||
pub fn update(&self, speed: ScannerSpeed) {
|
pub fn update(&self, speed: ScannerSpeed) {
|
||||||
let mut f = self.inner.factor.write().unwrap_or_else(|e| e.into_inner());
|
self.update_params(speed.sleep_factor(), speed.max_sleep());
|
||||||
*f = speed.sleep_factor();
|
|
||||||
let mut m = self.inner.max_sleep.write().unwrap_or_else(|e| e.into_inner());
|
|
||||||
*m = speed.max_sleep();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reload speed and idle-mode settings from the current environment.
|
/// Reload speed and idle-mode settings from the current environment.
|
||||||
pub fn refresh_from_env(&self) {
|
pub fn refresh_from_env(&self) {
|
||||||
let (speed, idle_mode) = scanner_env_config();
|
let (speed, idle_mode) = scanner_env_config();
|
||||||
self.update_from_runtime_config(speed, idle_mode, scanner_yield_every_n_objects());
|
self.update_from_runtime_config(speed.sleep_factor(), speed.max_sleep(), idle_mode, scanner_yield_every_n_objects());
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn update_from_runtime_config(&self, speed: ScannerSpeed, idle_mode: bool, yield_every_n_objects: u64) {
|
pub(crate) fn update_from_runtime_config(
|
||||||
self.update(speed);
|
&self,
|
||||||
|
sleep_factor: f64,
|
||||||
|
max_sleep: Duration,
|
||||||
|
idle_mode: bool,
|
||||||
|
yield_every_n_objects: u64,
|
||||||
|
) {
|
||||||
|
self.update_params(sleep_factor, max_sleep);
|
||||||
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
SCANNER_IDLE_MODE.store(idle_mode, Ordering::Relaxed);
|
||||||
self.record_throttle_config_with_yield(yield_every_n_objects);
|
self.record_throttle_config_with_yield(yield_every_n_objects);
|
||||||
}
|
}
|
||||||
@@ -264,6 +274,21 @@ mod tests {
|
|||||||
assert_eq!(max_sleep, Duration::from_secs(15));
|
assert_eq!(max_sleep, Duration::from_secs(15));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_update_from_runtime_config_applies_explicit_pacing_params() {
|
||||||
|
let prev_mode = SCANNER_IDLE_MODE.load(Ordering::Relaxed);
|
||||||
|
let s = DynamicSleeper::new(ScannerSpeed::Default);
|
||||||
|
|
||||||
|
s.update_from_runtime_config(3.5, Duration::from_secs(7), true, 64);
|
||||||
|
|
||||||
|
let (factor, max_sleep) = s.read_params();
|
||||||
|
assert_eq!(factor, 3.5);
|
||||||
|
assert_eq!(max_sleep, Duration::from_secs(7));
|
||||||
|
assert!(SCANNER_IDLE_MODE.load(Ordering::Relaxed));
|
||||||
|
|
||||||
|
SCANNER_IDLE_MODE.store(prev_mode, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
fn test_refresh_from_env_applies_speed_and_idle_mode_for_next_cycle() {
|
||||||
|
|||||||
@@ -52,14 +52,15 @@ use rustfs_config::{
|
|||||||
COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
COMMENT_KEY, DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, ENV_SCANNER_ALERT_EXCESS_FOLDERS,
|
||||||
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
|
ENV_SCANNER_ALERT_EXCESS_VERSION_SIZE, ENV_SCANNER_ALERT_EXCESS_VERSIONS, ENV_SCANNER_BITROT_CYCLE_SECS,
|
||||||
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
|
ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS, ENV_SCANNER_CYCLE, ENV_SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||||
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_IDLE_MODE,
|
ENV_SCANNER_CYCLE_MAX_DURATION_SECS, ENV_SCANNER_CYCLE_MAX_OBJECTS, ENV_SCANNER_DELAY, ENV_SCANNER_IDLE_MODE,
|
||||||
ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS,
|
ENV_SCANNER_MAX_CONCURRENT_DISK_SCANS, ENV_SCANNER_MAX_CONCURRENT_SET_SCANS, ENV_SCANNER_MAX_WAIT_SECS, ENV_SCANNER_SPEED,
|
||||||
ENV_SCANNER_YIELD_EVERY_N_OBJECTS, HEAL_BITROT_CYCLE, HEAL_SUB_SYS, MAX_ADMIN_REQUEST_BODY_SIZE, MQTT_BROKER,
|
ENV_SCANNER_START_DELAY_SECS, ENV_SCANNER_YIELD_EVERY_N_OBJECTS, HEAL_BITROT_CYCLE, HEAL_SUB_SYS,
|
||||||
MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC,
|
MAX_ADMIN_REQUEST_BODY_SIZE, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||||
MQTT_USERNAME, SCANNER_ALERT_EXCESS_FOLDERS, SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS,
|
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, SCANNER_ALERT_EXCESS_FOLDERS,
|
||||||
SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT, SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION,
|
SCANNER_ALERT_EXCESS_VERSION_SIZE, SCANNER_ALERT_EXCESS_VERSIONS, SCANNER_BITROT_CYCLE, SCANNER_CACHE_SAVE_TIMEOUT,
|
||||||
SCANNER_CYCLE_MAX_OBJECTS, SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS,
|
SCANNER_CYCLE, SCANNER_CYCLE_MAX_DIRECTORIES, SCANNER_CYCLE_MAX_DURATION, SCANNER_CYCLE_MAX_OBJECTS, SCANNER_DELAY,
|
||||||
SCANNER_SPEED, SCANNER_START_DELAY, SCANNER_SUB_SYS, SCANNER_YIELD_EVERY_N_OBJECTS, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE,
|
SCANNER_IDLE_MODE, SCANNER_MAX_CONCURRENT_DISK_SCANS, SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_SPEED,
|
||||||
|
SCANNER_START_DELAY, SCANNER_SUB_SYS, SCANNER_YIELD_EVERY_N_OBJECTS, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE,
|
||||||
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
|
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
|
||||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
|
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
|
||||||
};
|
};
|
||||||
@@ -192,6 +193,18 @@ const SCANNER_HELP_KEYS: &[HelpKeyMetadata] = &[
|
|||||||
description: "set scanner throttling preset",
|
description: "set scanner throttling preset",
|
||||||
optional: true,
|
optional: true,
|
||||||
},
|
},
|
||||||
|
HelpKeyMetadata {
|
||||||
|
key: SCANNER_DELAY,
|
||||||
|
type_name: "float",
|
||||||
|
description: "override scanner sleep multiplier derived from speed",
|
||||||
|
optional: true,
|
||||||
|
},
|
||||||
|
HelpKeyMetadata {
|
||||||
|
key: SCANNER_MAX_WAIT,
|
||||||
|
type_name: "seconds",
|
||||||
|
description: "override scanner maximum sleep duration derived from speed",
|
||||||
|
optional: true,
|
||||||
|
},
|
||||||
HelpKeyMetadata {
|
HelpKeyMetadata {
|
||||||
key: SCANNER_CYCLE,
|
key: SCANNER_CYCLE,
|
||||||
type_name: "seconds",
|
type_name: "seconds",
|
||||||
@@ -1322,6 +1335,8 @@ fn env_help_key(sub_system: &str, key: &str) -> String {
|
|||||||
(STORAGE_CLASS_SUB_SYS, "optimize") => OPTIMIZE_ENV.to_string(),
|
(STORAGE_CLASS_SUB_SYS, "optimize") => OPTIMIZE_ENV.to_string(),
|
||||||
(STORAGE_CLASS_SUB_SYS, "inline_block") => INLINE_BLOCK_ENV.to_string(),
|
(STORAGE_CLASS_SUB_SYS, "inline_block") => INLINE_BLOCK_ENV.to_string(),
|
||||||
(SCANNER_SUB_SYS, SCANNER_SPEED) => ENV_SCANNER_SPEED.to_string(),
|
(SCANNER_SUB_SYS, SCANNER_SPEED) => ENV_SCANNER_SPEED.to_string(),
|
||||||
|
(SCANNER_SUB_SYS, SCANNER_DELAY) => ENV_SCANNER_DELAY.to_string(),
|
||||||
|
(SCANNER_SUB_SYS, SCANNER_MAX_WAIT) => ENV_SCANNER_MAX_WAIT_SECS.to_string(),
|
||||||
(SCANNER_SUB_SYS, SCANNER_CYCLE) => ENV_SCANNER_CYCLE.to_string(),
|
(SCANNER_SUB_SYS, SCANNER_CYCLE) => ENV_SCANNER_CYCLE.to_string(),
|
||||||
(SCANNER_SUB_SYS, SCANNER_START_DELAY) => ENV_SCANNER_START_DELAY_SECS.to_string(),
|
(SCANNER_SUB_SYS, SCANNER_START_DELAY) => ENV_SCANNER_START_DELAY_SECS.to_string(),
|
||||||
(SCANNER_SUB_SYS, SCANNER_CYCLE_MAX_DURATION) => ENV_SCANNER_CYCLE_MAX_DURATION_SECS.to_string(),
|
(SCANNER_SUB_SYS, SCANNER_CYCLE_MAX_DURATION) => ENV_SCANNER_CYCLE_MAX_DURATION_SECS.to_string(),
|
||||||
@@ -1990,6 +2005,25 @@ identity_openid config_url="https://issuer.example" client_id="console""#,
|
|||||||
assert!(response.keys_help[0].description.contains("cycle"));
|
assert!(response.keys_help[0].description.contains("cycle"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_config_directives_accepts_scanner_pacing_keys() {
|
||||||
|
let input = format!("{SCANNER_SUB_SYS} {SCANNER_DELAY}=\"3.5\" {SCANNER_MAX_WAIT}=\"7\"");
|
||||||
|
let directives = parse_config_directives(&input, false).expect("parse scanner pacing directive");
|
||||||
|
|
||||||
|
validate_config_directives(&directives).expect("scanner pacing keys should be supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_help_response_reports_scanner_delay() {
|
||||||
|
let response = build_help_response(Some(SCANNER_SUB_SYS), Some(SCANNER_DELAY), false).expect("scanner help response");
|
||||||
|
|
||||||
|
assert_eq!(response.sub_sys, SCANNER_SUB_SYS);
|
||||||
|
assert_eq!(response.keys_help.len(), 1);
|
||||||
|
assert_eq!(response.keys_help[0].key, SCANNER_DELAY);
|
||||||
|
assert_eq!(response.keys_help[0].type_name, "float");
|
||||||
|
assert!(response.keys_help[0].description.contains("multiplier"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn validate_config_directives_accepts_heal_bitrot_cycle() {
|
fn validate_config_directives_accepts_heal_bitrot_cycle() {
|
||||||
let input = format!("{HEAL_SUB_SYS} {HEAL_BITROT_CYCLE}=\"off\"");
|
let input = format!("{HEAL_SUB_SYS} {HEAL_BITROT_CYCLE}=\"off\"");
|
||||||
|
|||||||
Reference in New Issue
Block a user