mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-22 04:16:38 +00:00
fix(storage): add scoped timeout policy and startup fs guardrail (#3056)
* fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends * style: fix cargo fmt formatting in disk_store.rs * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends Extend the TimeoutHealthAction introduced in #2996 to read_metadata, list_dir, and disk_info operations when RUSTFS_NETWORK_MOUNT_MODE=true. Also raises all drive operation timeouts to 60s (explicit per-operation overrides still take precedence). Closes #2790 * feat(startup): add unsupported filesystem policy guardrail * chore(deps): refresh lockfile and dependency pins * feat(ecstore): add scoped timeout health-action policy * docs(config): document drive timeout health-action policy * refactor(ecstore): cache timeout health policy per disk wrapper * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends (#2838) * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends * style: fix cargo fmt formatting in disk_store.rs * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends Extend the TimeoutHealthAction introduced in #2996 to read_metadata, list_dir, and disk_info operations when RUSTFS_NETWORK_MOUNT_MODE=true. Also raises all drive operation timeouts to 60s (explicit per-operation overrides still take precedence). Closes #2790 * fix(utils): map verified Linux filesystem magic values (#3051) * fix(utils): cover sha256 checksum validation (#3052) * fix(utils): cover sha256 checksum validation * docs: clarify sha256 checksum validation --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com> * refactor(config): replace network mount mode with timeout profile preset * fix(review): align fallback defaults and extend fs-type detection * fix(review): cache timeout profile and restore probe timeout semantics * refactor(ecstore): cache timeout health policy lookup * perf(ecstore): cache active probe timeout per monitor task --------- Co-authored-by: mistik <mistiklord4@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
@@ -83,6 +83,28 @@ Legacy compatibility fallback:
|
||||
- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
|
||||
This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
|
||||
|
||||
Drive timeout health-action policy:
|
||||
- `RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`
|
||||
- `mark_failure` (default): timeout marks failure and may transition drive runtime state.
|
||||
- `ignore_scanner`: timeout does not mark failure for scanner-sensitive operations (`walk_dir`, `read_metadata`, `list_dir`, `disk_info`).
|
||||
|
||||
Drive timeout profile preset:
|
||||
- `RUSTFS_DRIVE_TIMEOUT_PROFILE`
|
||||
- `default` (default): keep current timeout defaults.
|
||||
- `high_latency`: use 60s default timeout for scanner-sensitive operations when no per-operation timeout override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`).
|
||||
- Precedence:
|
||||
- Explicit per-operation timeout env (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) takes highest precedence.
|
||||
- Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
|
||||
- Then the profile-derived default (`default` or `high_latency`).
|
||||
|
||||
## Startup filesystem boundary policy
|
||||
|
||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||
- `warn` (default): log warning and continue startup.
|
||||
- `fail`: abort startup with an error.
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -161,6 +161,16 @@ pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
|
||||
/// provide non-default `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` values.
|
||||
pub const ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS: &str = "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS";
|
||||
|
||||
/// Environment variable controlling startup behavior when unsupported filesystem types are detected.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "warn" (default): log a warning and continue startup
|
||||
/// - "fail": abort startup with an error
|
||||
pub const ENV_RUSTFS_UNSUPPORTED_FS_POLICY: &str = "RUSTFS_UNSUPPORTED_FS_POLICY";
|
||||
pub const RUSTFS_UNSUPPORTED_FS_POLICY_WARN: &str = "warn";
|
||||
pub const RUSTFS_UNSUPPORTED_FS_POLICY_FAIL: &str = "fail";
|
||||
pub const DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY: &str = RUSTFS_UNSUPPORTED_FS_POLICY_WARN;
|
||||
|
||||
/// Environment variable for server OBS endpoint.
|
||||
pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
|
||||
|
||||
@@ -47,6 +47,16 @@ pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
|
||||
pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Timeout-to-health transition policy for drive operations.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "mark_failure" (default): timeout marks failure and may transition runtime state.
|
||||
/// - "ignore_scanner": timeout does not mark failure for scanner-sensitive operations.
|
||||
pub const ENV_DRIVE_TIMEOUT_HEALTH_ACTION: &str = "RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION";
|
||||
pub const DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE: &str = "mark_failure";
|
||||
pub const DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER: &str = "ignore_scanner";
|
||||
pub const DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION: &str = DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE;
|
||||
|
||||
/// Number of consecutive failures before a suspect drive is classified as offline.
|
||||
pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
|
||||
pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
|
||||
@@ -66,3 +76,19 @@ pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
|
||||
/// Duration in seconds after which a recovered drive is classified as long offline.
|
||||
pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
|
||||
pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;
|
||||
|
||||
/// Drive timeout profile preset.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "default": keep current timeout defaults.
|
||||
/// - "high_latency": use a higher default timeout preset for scanner-sensitive and metadata operations.
|
||||
///
|
||||
/// Explicit per-operation overrides (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) still take precedence.
|
||||
pub const ENV_DRIVE_TIMEOUT_PROFILE: &str = "RUSTFS_DRIVE_TIMEOUT_PROFILE";
|
||||
pub const DRIVE_TIMEOUT_PROFILE_DEFAULT: &str = "default";
|
||||
pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY: &str = "high_latency";
|
||||
pub const DEFAULT_DRIVE_TIMEOUT_PROFILE: &str = DRIVE_TIMEOUT_PROFILE_DEFAULT;
|
||||
|
||||
/// Timeout preset (seconds) used when `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`
|
||||
/// and no per-operation timeout override is provided.
|
||||
pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS: u64 = 60;
|
||||
|
||||
@@ -27,6 +27,8 @@ use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
|
||||
use bytes::Bytes;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
#[cfg(not(test))]
|
||||
use std::sync::OnceLock;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
@@ -50,10 +52,54 @@ enum TimeoutHealthAction {
|
||||
IgnoreFailure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TimeoutHealthPolicy {
|
||||
MarkFailure,
|
||||
IgnoreScanner,
|
||||
}
|
||||
|
||||
impl TimeoutHealthPolicy {
|
||||
fn parse(raw: &str) -> Option<Self> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE => Some(Self::MarkFailure),
|
||||
rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER => Some(Self::IgnoreScanner),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_timeout_health_action(self) -> TimeoutHealthAction {
|
||||
match self {
|
||||
Self::MarkFailure => TimeoutHealthAction::MarkFailure,
|
||||
Self::IgnoreScanner => TimeoutHealthAction::IgnoreFailure,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
|
||||
pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
|
||||
pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum DriveTimeoutProfile {
|
||||
Default,
|
||||
HighLatency,
|
||||
}
|
||||
|
||||
impl DriveTimeoutProfile {
|
||||
fn parse(raw: &str) -> Option<Self> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
rustfs_config::DRIVE_TIMEOUT_PROFILE_DEFAULT => Some(Self::Default),
|
||||
rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY => Some(Self::HighLatency),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static DRIVE_TIMEOUT_PROFILE_CACHE: OnceLock<DriveTimeoutProfile> = OnceLock::new();
|
||||
#[cfg(not(test))]
|
||||
static DRIVE_TIMEOUT_HEALTH_POLICY_CACHE: OnceLock<TimeoutHealthPolicy> = OnceLock::new();
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
|
||||
static ref TEST_BUCKET: String = ".rustfs.sys/tmp".to_string();
|
||||
@@ -66,10 +112,39 @@ pub fn get_max_timeout_duration() -> Duration {
|
||||
))
|
||||
}
|
||||
|
||||
fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
|
||||
fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
|
||||
let raw = rustfs_utils::get_env_str(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE);
|
||||
if let Some(profile) = DriveTimeoutProfile::parse(&raw) {
|
||||
return profile;
|
||||
}
|
||||
warn!(
|
||||
env = rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
value = %raw,
|
||||
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE,
|
||||
"Invalid drive timeout profile; falling back to default"
|
||||
);
|
||||
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
|
||||
}
|
||||
|
||||
fn get_drive_timeout_profile() -> DriveTimeoutProfile {
|
||||
#[cfg(test)]
|
||||
{
|
||||
resolve_drive_timeout_profile_from_env()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
*DRIVE_TIMEOUT_PROFILE_CACHE.get_or_init(resolve_drive_timeout_profile_from_env)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_drive_timeout_duration(env_key: &str, default_secs: u64, high_latency_secs: Option<u64>) -> Duration {
|
||||
let fallback_default = match (get_drive_timeout_profile(), high_latency_secs) {
|
||||
(DriveTimeoutProfile::HighLatency, Some(secs)) => secs,
|
||||
_ => default_secs,
|
||||
};
|
||||
Duration::from_secs(
|
||||
rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
|
||||
.unwrap_or(default_secs),
|
||||
.unwrap_or(fallback_default),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -77,6 +152,7 @@ pub fn get_drive_metadata_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -84,6 +160,7 @@ pub fn get_drive_disk_info_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -91,6 +168,7 @@ pub fn get_drive_list_dir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -98,6 +176,7 @@ pub fn get_drive_walkdir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -105,6 +184,7 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -122,6 +202,34 @@ pub fn get_drive_active_check_timeout() -> Duration {
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_drive_timeout_health_policy_from_env() -> TimeoutHealthPolicy {
|
||||
let raw = rustfs_utils::get_env_str(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
);
|
||||
if let Some(policy) = TimeoutHealthPolicy::parse(&raw) {
|
||||
return policy;
|
||||
}
|
||||
warn!(
|
||||
env = rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
value = %raw,
|
||||
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
"Invalid drive timeout health action policy; falling back to default"
|
||||
);
|
||||
TimeoutHealthPolicy::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION).unwrap_or(TimeoutHealthPolicy::MarkFailure)
|
||||
}
|
||||
|
||||
fn get_drive_timeout_health_policy() -> TimeoutHealthPolicy {
|
||||
#[cfg(test)]
|
||||
{
|
||||
resolve_drive_timeout_health_policy_from_env()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
*DRIVE_TIMEOUT_HEALTH_POLICY_CACHE.get_or_init(resolve_drive_timeout_health_policy_from_env)
|
||||
}
|
||||
}
|
||||
|
||||
/// DiskHealthTracker tracks the health status of a disk.
|
||||
/// Similar to Go's diskHealthTracker.
|
||||
#[derive(Debug)]
|
||||
@@ -470,6 +578,8 @@ pub struct LocalDiskWrapper {
|
||||
cancel_token: CancellationToken,
|
||||
/// Disk ID for stale checking
|
||||
disk_id: Arc<RwLock<Option<Uuid>>>,
|
||||
/// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
|
||||
timeout_health_policy: TimeoutHealthPolicy,
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
@@ -486,6 +596,7 @@ impl LocalDiskWrapper {
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
timeout_health_policy: get_drive_timeout_health_policy(),
|
||||
};
|
||||
record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
|
||||
wrapper
|
||||
@@ -511,6 +622,10 @@ impl LocalDiskWrapper {
|
||||
self.health.record_capacity_probe(total, used, free);
|
||||
}
|
||||
|
||||
fn scanner_timeout_health_action(&self) -> TimeoutHealthAction {
|
||||
self.timeout_health_policy.scanner_timeout_health_action()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
@@ -558,6 +673,7 @@ impl LocalDiskWrapper {
|
||||
/// Monitor disk writability periodically
|
||||
async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
let mut interval = time::interval(get_drive_active_check_interval());
|
||||
let active_check_timeout = get_drive_active_check_timeout();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -596,7 +712,7 @@ impl LocalDiskWrapper {
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
true,
|
||||
get_drive_active_check_timeout(),
|
||||
active_check_timeout,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
@@ -691,6 +807,7 @@ impl LocalDiskWrapper {
|
||||
/// Monitor disk status and try to bring it back online
|
||||
async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
let check_every = get_drive_returning_probe_interval();
|
||||
let active_check_timeout = get_drive_active_check_timeout();
|
||||
|
||||
let mut interval = time::interval(check_every);
|
||||
|
||||
@@ -711,7 +828,7 @@ impl LocalDiskWrapper {
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
false,
|
||||
get_drive_active_check_timeout(),
|
||||
active_check_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -893,10 +1010,11 @@ impl LocalDiskWrapper {
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
|| async { self.disk.read_metadata(volume, path).await },
|
||||
get_drive_metadata_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -973,7 +1091,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"disk_info",
|
||||
|| async {
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
@@ -987,6 +1105,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
Ok(result)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1021,7 +1140,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
get_drive_walkdir_timeout(),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1130,10 +1249,11 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"list_dir",
|
||||
|| async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
|
||||
get_drive_list_dir_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1256,14 +1376,48 @@ mod tests {
|
||||
fn drive_metadata_timeout_uses_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_high_latency_profile_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS)
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_invalid_profile_falls_back_to_default() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, Some("invalid"), || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
@@ -1398,6 +1552,58 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_writer_backpressure_timeout_does_not_mark_drive_failure() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1")),
|
||||
(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
|
||||
),
|
||||
],
|
||||
async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let bucket = "test-bucket";
|
||||
let object = "test-object";
|
||||
|
||||
wrapper.make_volume(bucket).await.expect("bucket should be created");
|
||||
|
||||
let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
|
||||
file_info.volume = bucket.to_string();
|
||||
file_info.name = object.to_string();
|
||||
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
|
||||
file_info.erasure.index = 1;
|
||||
|
||||
wrapper
|
||||
.write_metadata("", bucket, object, file_info)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
|
||||
let mut writer = PendingWriter;
|
||||
let result = wrapper
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_writer_backpressure_timeout_marks_drive_failure_by_default() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
@@ -1433,8 +1639,7 @@ mod tests {
|
||||
.await;
|
||||
|
||||
assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -1462,6 +1667,40 @@ mod tests {
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_defaults_to_mark_failure() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, || {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_respects_ignore_scanner() {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
|
||||
|| {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::IgnoreScanner);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::IgnoreFailure);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_invalid_value_falls_back_to_default() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, Some("invalid"), || {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_for_store_init_retry_clears_faulty_and_back_online() {
|
||||
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry").expect("endpoint should parse");
|
||||
|
||||
@@ -45,6 +45,7 @@ pub(crate) fn get_fs_type(fs_type: u64) -> &'static str {
|
||||
0x73717368 => "SQUASHFS",
|
||||
0xFF534D42 => "CIFS",
|
||||
0xFE534D42 => "SMB2",
|
||||
0x01021997 => "V9FS",
|
||||
0xca451a4e => "BCACHEFS",
|
||||
_ => "UNKNOWN",
|
||||
}
|
||||
@@ -71,6 +72,7 @@ mod tests {
|
||||
assert_eq!(get_fs_type(0x73717368), "SQUASHFS");
|
||||
assert_eq!(get_fs_type(0xFF534D42), "CIFS");
|
||||
assert_eq!(get_fs_type(0xFE534D42), "SMB2");
|
||||
assert_eq!(get_fs_type(0x01021997), "V9FS");
|
||||
assert_eq!(get_fs_type(0xca451a4e), "BCACHEFS");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user