diff --git a/Cargo.lock b/Cargo.lock index 10a8481bb..0d7d7a42e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4395,6 +4395,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "482ce8a491a501da4cd806bd190275363d674f2845005c6ddbd5d3e1dd54495d" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "google-cloud-auth" version = "1.10.0" @@ -6442,9 +6454,9 @@ dependencies = [ [[package]] name = "mysql_common" -version = "0.37.1" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffc2127d4035fa5a614935c663a15a4468e64e798473e0cc21c8df40a607588" +checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2" dependencies = [ "base64 0.22.1", "bitflags 2.11.1", @@ -9089,20 +9101,22 @@ dependencies = [ [[package]] name = "russh-sftp" -version = "2.1.2" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09daa0ebcf53fb18d7b16167586a68b5bf2cfa3eaad49e661a19302552a2b879" +checksum = "c826d40e310bbcb377fd8f6c5873c12eaf54da30569d8e8da56914fdac9773cd" dependencies = [ "bitflags 2.11.1", "bytes", "chrono", "dashmap", + "gloo-timers", "log", "serde", "serde_bytes", "thiserror 2.0.18", "tokio", "tokio-util", + "wasm-bindgen-futures", ] [[package]] @@ -10697,9 +10711,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -11802,13 +11816,12 @@ dependencies = [ [[package]] name = "tokio-postgres-rustls" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" dependencies = [ - "const-oid 0.9.6", - "ring", "rustls", + "sha2 0.11.0", "tokio", "tokio-postgres", "tokio-rustls", diff --git a/Cargo.toml b/Cargo.toml index 6c9669f15..ec9c62e3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,7 +158,7 @@ quick-xml = "0.40.1" rmp = { version = "0.8.15" } rmp-serde = { version = "1.3.1" } serde = { version = "1.0.228", features = ["derive"] } -serde_json = { version = "1.0.149", features = ["raw_value"] } +serde_json = { version = "1.0.150", features = ["raw_value"] } serde_urlencoded = "0.7.1" # Cryptography and Security @@ -189,7 +189,7 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros # Database deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] } tokio-postgres = { version = "0.7", default-features = false, features = ["runtime", "with-serde_json-1"] } -tokio-postgres-rustls = "0.13" +tokio-postgres-rustls = "0.14.0" # Utilities and Tools anyhow = "1.0.102" @@ -307,7 +307,7 @@ unftp-core = "0.1.0" suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] } rcgen = "0.14.8" russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" } -russh-sftp = "2.1.2" +russh-sftp = "2.2.2" # WebDAV dav-server = "0.11.0" diff --git a/crates/config/README.md b/crates/config/README.md index 560ddbd81..ea2aaa77b 100644 --- a/crates/config/README.md +++ b/crates/config/README.md @@ -83,6 +83,28 @@ Legacy compatibility fallback: - `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset. +Drive timeout health-action policy: +- `RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION` + - `mark_failure` (default): timeout marks failure and may transition drive runtime state. + - `ignore_scanner`: timeout does not mark failure for scanner-sensitive operations (`walk_dir`, `read_metadata`, `list_dir`, `disk_info`). + +Drive timeout profile preset: +- `RUSTFS_DRIVE_TIMEOUT_PROFILE` + - `default` (default): keep current timeout defaults. + - `high_latency`: use 60s default timeout for scanner-sensitive operations when no per-operation timeout override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`). +- Precedence: + - Explicit per-operation timeout env (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) takes highest precedence. + - Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback. + - Then the profile-derived default (`default` or `high_latency`). + +## Startup filesystem boundary policy + +- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary. + - `warn` (default): log warning and continue startup. + - `fail`: abort startup with an error. + +RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard. + ## 📄 License This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details. diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index b77db7edc..766146a45 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -161,6 +161,16 @@ pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE"; /// provide non-default `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` values. pub const ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS: &str = "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS"; +/// Environment variable controlling startup behavior when unsupported filesystem types are detected. +/// +/// Accepted values: +/// - "warn" (default): log a warning and continue startup +/// - "fail": abort startup with an error +pub const ENV_RUSTFS_UNSUPPORTED_FS_POLICY: &str = "RUSTFS_UNSUPPORTED_FS_POLICY"; +pub const RUSTFS_UNSUPPORTED_FS_POLICY_WARN: &str = "warn"; +pub const RUSTFS_UNSUPPORTED_FS_POLICY_FAIL: &str = "fail"; +pub const DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY: &str = RUSTFS_UNSUPPORTED_FS_POLICY_WARN; + /// Environment variable for server OBS endpoint. pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT"; diff --git a/crates/config/src/constants/drive.rs b/crates/config/src/constants/drive.rs index f3d6dcd14..49fac3a5a 100644 --- a/crates/config/src/constants/drive.rs +++ b/crates/config/src/constants/drive.rs @@ -47,6 +47,16 @@ pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15; pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS"; pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5; +/// Timeout-to-health transition policy for drive operations. +/// +/// Accepted values: +/// - "mark_failure" (default): timeout marks failure and may transition runtime state. +/// - "ignore_scanner": timeout does not mark failure for scanner-sensitive operations. +pub const ENV_DRIVE_TIMEOUT_HEALTH_ACTION: &str = "RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION"; +pub const DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE: &str = "mark_failure"; +pub const DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER: &str = "ignore_scanner"; +pub const DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION: &str = DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE; + /// Number of consecutive failures before a suspect drive is classified as offline. pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD"; pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2; @@ -66,3 +76,19 @@ pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30; /// Duration in seconds after which a recovered drive is classified as long offline. pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS"; pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800; + +/// Drive timeout profile preset. +/// +/// Accepted values: +/// - "default": keep current timeout defaults. +/// - "high_latency": use a higher default timeout preset for scanner-sensitive and metadata operations. +/// +/// Explicit per-operation overrides (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) still take precedence. +pub const ENV_DRIVE_TIMEOUT_PROFILE: &str = "RUSTFS_DRIVE_TIMEOUT_PROFILE"; +pub const DRIVE_TIMEOUT_PROFILE_DEFAULT: &str = "default"; +pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY: &str = "high_latency"; +pub const DEFAULT_DRIVE_TIMEOUT_PROFILE: &str = DRIVE_TIMEOUT_PROFILE_DEFAULT; + +/// Timeout preset (seconds) used when `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency` +/// and no per-operation timeout override is provided. +pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS: u64 = 60; diff --git a/crates/ecstore/src/disk/disk_store.rs b/crates/ecstore/src/disk/disk_store.rs index ed664e394..9c864ac8d 100644 --- a/crates/ecstore/src/disk/disk_store.rs +++ b/crates/ecstore/src/disk/disk_store.rs @@ -27,6 +27,8 @@ use crate::global::GLOBAL_LOCAL_DISK_ID_MAP; use bytes::Bytes; use metrics::counter; use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo}; +#[cfg(not(test))] +use std::sync::OnceLock; use std::{ path::PathBuf, sync::{ @@ -50,10 +52,54 @@ enum TimeoutHealthAction { IgnoreFailure, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TimeoutHealthPolicy { + MarkFailure, + IgnoreScanner, +} + +impl TimeoutHealthPolicy { + fn parse(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE => Some(Self::MarkFailure), + rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER => Some(Self::IgnoreScanner), + _ => None, + } + } + + fn scanner_timeout_health_action(self) -> TimeoutHealthAction { + match self { + Self::MarkFailure => TimeoutHealthAction::MarkFailure, + Self::IgnoreScanner => TimeoutHealthAction::IgnoreFailure, + } + } +} + pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING"; pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true; pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum DriveTimeoutProfile { + Default, + HighLatency, +} + +impl DriveTimeoutProfile { + fn parse(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + rustfs_config::DRIVE_TIMEOUT_PROFILE_DEFAULT => Some(Self::Default), + rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY => Some(Self::HighLatency), + _ => None, + } + } +} + +#[cfg(not(test))] +static DRIVE_TIMEOUT_PROFILE_CACHE: OnceLock = OnceLock::new(); +#[cfg(not(test))] +static DRIVE_TIMEOUT_HEALTH_POLICY_CACHE: OnceLock = OnceLock::new(); + lazy_static::lazy_static! { static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]); static ref TEST_BUCKET: String = ".rustfs.sys/tmp".to_string(); @@ -66,10 +112,39 @@ pub fn get_max_timeout_duration() -> Duration { )) } -fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration { +fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile { + let raw = rustfs_utils::get_env_str(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE); + if let Some(profile) = DriveTimeoutProfile::parse(&raw) { + return profile; + } + warn!( + env = rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, + value = %raw, + default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE, + "Invalid drive timeout profile; falling back to default" + ); + DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default) +} + +fn get_drive_timeout_profile() -> DriveTimeoutProfile { + #[cfg(test)] + { + resolve_drive_timeout_profile_from_env() + } + #[cfg(not(test))] + { + *DRIVE_TIMEOUT_PROFILE_CACHE.get_or_init(resolve_drive_timeout_profile_from_env) + } +} + +fn get_drive_timeout_duration(env_key: &str, default_secs: u64, high_latency_secs: Option) -> Duration { + let fallback_default = match (get_drive_timeout_profile(), high_latency_secs) { + (DriveTimeoutProfile::HighLatency, Some(secs)) => secs, + _ => default_secs, + }; Duration::from_secs( rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION]) - .unwrap_or(default_secs), + .unwrap_or(fallback_default), ) } @@ -77,6 +152,7 @@ pub fn get_drive_metadata_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS), ) } @@ -84,6 +160,7 @@ pub fn get_drive_disk_info_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS, rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS), ) } @@ -91,6 +168,7 @@ pub fn get_drive_list_dir_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS, rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS), ) } @@ -98,6 +176,7 @@ pub fn get_drive_walkdir_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS), ) } @@ -105,6 +184,7 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration { get_drive_timeout_duration( rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS), ) } @@ -122,6 +202,34 @@ pub fn get_drive_active_check_timeout() -> Duration { )) } +fn resolve_drive_timeout_health_policy_from_env() -> TimeoutHealthPolicy { + let raw = rustfs_utils::get_env_str( + rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, + rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION, + ); + if let Some(policy) = TimeoutHealthPolicy::parse(&raw) { + return policy; + } + warn!( + env = rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, + value = %raw, + default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION, + "Invalid drive timeout health action policy; falling back to default" + ); + TimeoutHealthPolicy::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION).unwrap_or(TimeoutHealthPolicy::MarkFailure) +} + +fn get_drive_timeout_health_policy() -> TimeoutHealthPolicy { + #[cfg(test)] + { + resolve_drive_timeout_health_policy_from_env() + } + #[cfg(not(test))] + { + *DRIVE_TIMEOUT_HEALTH_POLICY_CACHE.get_or_init(resolve_drive_timeout_health_policy_from_env) + } +} + /// DiskHealthTracker tracks the health status of a disk. /// Similar to Go's diskHealthTracker. #[derive(Debug)] @@ -470,6 +578,8 @@ pub struct LocalDiskWrapper { cancel_token: CancellationToken, /// Disk ID for stale checking disk_id: Arc>>, + /// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization. + timeout_health_policy: TimeoutHealthPolicy, } impl LocalDiskWrapper { @@ -486,6 +596,7 @@ impl LocalDiskWrapper { health_check: health_check && env_health_check, cancel_token: CancellationToken::new(), disk_id: Arc::new(RwLock::new(None)), + timeout_health_policy: get_drive_timeout_health_policy(), }; record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online); wrapper @@ -511,6 +622,10 @@ impl LocalDiskWrapper { self.health.record_capacity_probe(total, used, free); } + fn scanner_timeout_health_action(&self) -> TimeoutHealthAction { + self.timeout_health_policy.scanner_timeout_health_action() + } + #[cfg(test)] pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) { self.health.force_runtime_state_for_test(state); @@ -558,6 +673,7 @@ impl LocalDiskWrapper { /// Monitor disk writability periodically async fn monitor_disk_writable(disk: Arc, health: Arc, cancel_token: CancellationToken) { let mut interval = time::interval(get_drive_active_check_interval()); + let active_check_timeout = get_drive_active_check_timeout(); loop { tokio::select! { @@ -596,7 +712,7 @@ impl LocalDiskWrapper { &test_obj, &TEST_DATA, true, - get_drive_active_check_timeout(), + active_check_timeout, ) .await .is_err() @@ -691,6 +807,7 @@ impl LocalDiskWrapper { /// Monitor disk status and try to bring it back online async fn monitor_disk_status(disk: Arc, health: Arc, cancel_token: CancellationToken) { let check_every = get_drive_returning_probe_interval(); + let active_check_timeout = get_drive_active_check_timeout(); let mut interval = time::interval(check_every); @@ -711,7 +828,7 @@ impl LocalDiskWrapper { &test_obj, &TEST_DATA, false, - get_drive_active_check_timeout(), + active_check_timeout, ) .await { @@ -893,10 +1010,11 @@ impl LocalDiskWrapper { #[async_trait::async_trait] impl DiskAPI for LocalDiskWrapper { async fn read_metadata(&self, volume: &str, path: &str) -> Result { - self.track_disk_health_with_op( + self.track_disk_health_with_op_and_timeout_action( "read_metadata", || async { self.disk.read_metadata(volume, path).await }, get_drive_metadata_timeout(), + self.scanner_timeout_health_action(), ) .await } @@ -973,7 +1091,7 @@ impl DiskAPI for LocalDiskWrapper { return Err(DiskError::FaultyDisk); } - self.track_disk_health_with_op( + self.track_disk_health_with_op_and_timeout_action( "disk_info", || async { let result = self.disk.disk_info(opts).await?; @@ -987,6 +1105,7 @@ impl DiskAPI for LocalDiskWrapper { Ok(result) }, get_drive_disk_info_timeout(), + self.scanner_timeout_health_action(), ) .await } @@ -1021,7 +1140,7 @@ impl DiskAPI for LocalDiskWrapper { "walk_dir", || async { self.disk.walk_dir(opts, wr).await }, get_drive_walkdir_timeout(), - TimeoutHealthAction::IgnoreFailure, + self.scanner_timeout_health_action(), ) .await } @@ -1130,10 +1249,11 @@ impl DiskAPI for LocalDiskWrapper { } async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { - self.track_disk_health_with_op( + self.track_disk_health_with_op_and_timeout_action( "list_dir", || async { self.disk.list_dir(origvolume, volume, dir_path, count).await }, get_drive_list_dir_timeout(), + self.scanner_timeout_health_action(), ) .await } @@ -1256,14 +1376,48 @@ mod tests { fn drive_metadata_timeout_uses_default_when_unset() { temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || { temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || { - assert_eq!( - get_drive_metadata_timeout(), - Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS) + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || { + assert_eq!( + get_drive_metadata_timeout(), + Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS) + ); + }); + }); + }); + } + + #[test] + fn drive_metadata_timeout_uses_high_latency_profile_when_unset() { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || { + temp_env::with_var( + rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, + Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY), + || { + assert_eq!( + get_drive_metadata_timeout(), + Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS) + ); + }, ); }); }); } + #[test] + fn drive_metadata_timeout_invalid_profile_falls_back_to_default() { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || { + temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, Some("invalid"), || { + assert_eq!( + get_drive_metadata_timeout(), + Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS) + ); + }); + }); + }); + } + #[test] fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() { temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || { @@ -1398,6 +1552,58 @@ mod tests { #[tokio::test] async fn walk_dir_writer_backpressure_timeout_does_not_mark_drive_failure() { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1")), + ( + rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, + Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER), + ), + ], + async { + let dir = tempfile::tempdir().expect("temp dir should be created"); + let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8")) + .expect("endpoint should parse"); + let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created")); + let wrapper = LocalDiskWrapper::new(disk, false); + let bucket = "test-bucket"; + let object = "test-object"; + + wrapper.make_volume(bucket).await.expect("bucket should be created"); + + let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0); + file_info.volume = bucket.to_string(); + file_info.name = object.to_string(); + file_info.mod_time = Some(::time::OffsetDateTime::now_utc()); + file_info.erasure.index = 1; + + wrapper + .write_metadata("", bucket, object, file_info) + .await + .expect("object metadata should be written"); + + let mut writer = PendingWriter; + let result = wrapper + .walk_dir( + WalkDirOptions { + bucket: bucket.to_string(), + recursive: true, + ..Default::default() + }, + &mut writer, + ) + .await; + + assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout); + assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online); + assert!(!wrapper.health.is_faulty()); + }, + ) + .await; + } + + #[tokio::test] + async fn walk_dir_writer_backpressure_timeout_marks_drive_failure_by_default() { temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async { let dir = tempfile::tempdir().expect("temp dir should be created"); let endpoint = @@ -1433,8 +1639,7 @@ mod tests { .await; assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout); - assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online); - assert!(!wrapper.health.is_faulty()); + assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect); }) .await; } @@ -1462,6 +1667,40 @@ mod tests { assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect); } + #[test] + #[serial_test::serial] + fn drive_timeout_health_policy_defaults_to_mark_failure() { + temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, || { + let policy = get_drive_timeout_health_policy(); + assert_eq!(policy, TimeoutHealthPolicy::MarkFailure); + assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure); + }); + } + + #[test] + #[serial_test::serial] + fn drive_timeout_health_policy_respects_ignore_scanner() { + temp_env::with_var( + rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, + Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER), + || { + let policy = get_drive_timeout_health_policy(); + assert_eq!(policy, TimeoutHealthPolicy::IgnoreScanner); + assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::IgnoreFailure); + }, + ); + } + + #[test] + #[serial_test::serial] + fn drive_timeout_health_policy_invalid_value_falls_back_to_default() { + temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, Some("invalid"), || { + let policy = get_drive_timeout_health_policy(); + assert_eq!(policy, TimeoutHealthPolicy::MarkFailure); + assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure); + }); + } + #[test] fn reset_for_store_init_retry_clears_faulty_and_back_online() { let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry").expect("endpoint should parse"); diff --git a/crates/utils/src/os/fs_type.rs b/crates/utils/src/os/fs_type.rs index f5bea171f..2d162b130 100644 --- a/crates/utils/src/os/fs_type.rs +++ b/crates/utils/src/os/fs_type.rs @@ -45,6 +45,7 @@ pub(crate) fn get_fs_type(fs_type: u64) -> &'static str { 0x73717368 => "SQUASHFS", 0xFF534D42 => "CIFS", 0xFE534D42 => "SMB2", + 0x01021997 => "V9FS", 0xca451a4e => "BCACHEFS", _ => "UNKNOWN", } @@ -71,6 +72,7 @@ mod tests { assert_eq!(get_fs_type(0x73717368), "SQUASHFS"); assert_eq!(get_fs_type(0xFF534D42), "CIFS"); assert_eq!(get_fs_type(0xFE534D42), "SMB2"); + assert_eq!(get_fs_type(0x01021997), "V9FS"); assert_eq!(get_fs_type(0xca451a4e), "BCACHEFS"); } } diff --git a/helm/rustfs/templates/configmap.yaml b/helm/rustfs/templates/configmap.yaml index 39a5fb524..6573eb6f5 100644 --- a/helm/rustfs/templates/configmap.yaml +++ b/helm/rustfs/templates/configmap.yaml @@ -86,6 +86,9 @@ data: RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS: {{ .cache_save_timeout_secs | quote }} {{- end }} {{- end }} + {{- if .Values.config.rustfs.timeout_profile }} + RUSTFS_DRIVE_TIMEOUT_PROFILE: {{ .Values.config.rustfs.timeout_profile | quote }} + {{- end }} {{- if .Values.config.rustfs.kms.enabled }} {{- if eq .Values.config.rustfs.kms.type "vault" }} {{- with .Values.config.rustfs.kms.vault }} diff --git a/helm/rustfs/values.yaml b/helm/rustfs/values.yaml index 0b9840367..c3784c2c4 100644 --- a/helm/rustfs/values.yaml +++ b/helm/rustfs/values.yaml @@ -91,6 +91,10 @@ config: idle_mode: "" # Timeout for scanner cache saves in seconds (minimum 1 second) cache_save_timeout_secs: "" + # Drive timeout profile preset: default|high_latency + # - default: keep current timeout defaults. + # - high_latency: use higher timeout defaults for scanner-sensitive operations. + timeout_profile: "" obs_endpoint: enabled: false # If true, rustfs will export metrics, traces, logs and profiling data to the specified OTLP endpoints. If false, the individual settings for metrics, traces, logs and profiling endpoints will be ignored and all data will not be exported. base_endpoint: "" #Root OTLP/HTTP endpoint, e.g. http://otel-collector:4318 diff --git a/rustfs/src/embedded.rs b/rustfs/src/embedded.rs index 914a52330..4a55a1f09 100644 --- a/rustfs/src/embedded.rs +++ b/rustfs/src/embedded.rs @@ -50,6 +50,7 @@ use crate::app::context::{AppContext, init_global_app_context}; use crate::config::Config; use crate::init::{add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system}; use crate::server::{init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server, stop_audit_system}; +use crate::startup_fs_guard::enforce_unsupported_fs_policy; use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr}; use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS; use rustfs_credentials::init_global_action_credentials; @@ -350,6 +351,7 @@ impl RustFSServerBuilder { let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_addr_str.as_str(), config.volumes.clone()) .await .map_err(|e| ServerError::Init(format!("endpoints: {e}")))?; + enforce_unsupported_fs_policy(&endpoint_pools).map_err(|e| ServerError::Init(format!("unsupported fs guard: {e}")))?; set_global_endpoints(endpoint_pools.as_ref().clone()); update_erasure_type(setup_type).await; diff --git a/rustfs/src/lib.rs b/rustfs/src/lib.rs index 6be77f54b..2b186d907 100644 --- a/rustfs/src/lib.rs +++ b/rustfs/src/lib.rs @@ -67,6 +67,7 @@ pub mod profiling; #[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))] pub mod protocols; pub mod server; +pub mod startup_fs_guard; pub mod storage; pub mod update; pub mod version; diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index c306fa6ea..fefce4ff1 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -33,6 +33,7 @@ use rustfs::server::{ SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server, stop_audit_system, wait_for_shutdown, }; +use rustfs::startup_fs_guard::enforce_unsupported_fs_policy; use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr}; use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS; use rustfs_credentials::init_global_action_credentials; @@ -301,6 +302,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> { let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), config.volumes.clone()) .await .map_err(Error::other)?; + enforce_unsupported_fs_policy(&endpoint_pools)?; set_global_endpoints(endpoint_pools.as_ref().clone()); update_erasure_type(setup_type).await; diff --git a/rustfs/src/startup_fs_guard.rs b/rustfs/src/startup_fs_guard.rs new file mode 100644 index 000000000..eb4db808e --- /dev/null +++ b/rustfs/src/startup_fs_guard.rs @@ -0,0 +1,170 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use rustfs_config::{ + DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL, + RUSTFS_UNSUPPORTED_FS_POLICY_WARN, +}; +use rustfs_ecstore::endpoints::EndpointServerPools; +use std::collections::BTreeSet; +use std::io::{Error, Result}; +use tracing::warn; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum UnsupportedFsPolicy { + Warn, + Fail, +} + +impl UnsupportedFsPolicy { + fn parse(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + RUSTFS_UNSUPPORTED_FS_POLICY_WARN => Some(Self::Warn), + RUSTFS_UNSUPPORTED_FS_POLICY_FAIL => Some(Self::Fail), + _ => None, + } + } +} + +fn get_unsupported_fs_policy() -> UnsupportedFsPolicy { + let raw = rustfs_utils::get_env_str(ENV_RUSTFS_UNSUPPORTED_FS_POLICY, DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY); + if let Some(policy) = UnsupportedFsPolicy::parse(&raw) { + return policy; + } + + warn!( + env = ENV_RUSTFS_UNSUPPORTED_FS_POLICY, + value = %raw, + default = DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, + "Invalid unsupported filesystem policy; falling back to default" + ); + UnsupportedFsPolicy::parse(DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY).unwrap_or(UnsupportedFsPolicy::Warn) +} + +fn is_unsupported_fs_type(fs_type: &str) -> bool { + let normalized = fs_type.trim().to_ascii_lowercase(); + matches!( + normalized.as_str(), + "nfs" | "cifs" | "smb2" | "fuse" | "fuseblk" | "overlayfs" | "9p" | "v9fs" | "ceph" | "glusterfs" | "gfs" | "gfs2" + ) || normalized.starts_with("fuse.") +} + +fn collect_local_paths(endpoint_pools: &EndpointServerPools) -> Vec { + let mut local_paths = BTreeSet::new(); + for pool in endpoint_pools.as_ref() { + for endpoint in pool.endpoints.as_ref() { + if endpoint.is_local { + local_paths.insert(endpoint.get_file_path()); + } + } + } + local_paths.into_iter().collect() +} + +pub fn enforce_unsupported_fs_policy(endpoint_pools: &EndpointServerPools) -> Result<()> { + let local_paths = collect_local_paths(endpoint_pools); + if local_paths.is_empty() { + return Ok(()); + } + + let mut unsupported = Vec::new(); + for path in local_paths { + let info = match rustfs_utils::os::get_info(&path) { + Ok(info) => info, + Err(err) => { + warn!( + path = %path, + error = %err, + "Failed to inspect filesystem type for startup boundary guard; skipping this path" + ); + continue; + } + }; + + if is_unsupported_fs_type(&info.fstype) { + unsupported.push((path, info.fstype)); + } + } + + if unsupported.is_empty() { + return Ok(()); + } + + let detail = unsupported + .iter() + .map(|(path, fs_type)| format!("{path} ({fs_type})")) + .collect::>() + .join(", "); + + let message = format!( + "Unsupported filesystem type detected for RustFS local endpoints: {detail}. \ +RustFS only supports direct-attached local POSIX filesystems for production workloads." + ); + + match get_unsupported_fs_policy() { + UnsupportedFsPolicy::Warn => { + warn!( + env = ENV_RUSTFS_UNSUPPORTED_FS_POLICY, + expected = format!("{RUSTFS_UNSUPPORTED_FS_POLICY_WARN}|{RUSTFS_UNSUPPORTED_FS_POLICY_FAIL}"), + policy = RUSTFS_UNSUPPORTED_FS_POLICY_WARN, + "{message}" + ); + Ok(()) + } + UnsupportedFsPolicy::Fail => Err(Error::other(format!( + "{message} Startup aborted by policy: {ENV_RUSTFS_UNSUPPORTED_FS_POLICY}={RUSTFS_UNSUPPORTED_FS_POLICY_FAIL}" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_fs_type_matcher_covers_network_like_types() { + assert!(is_unsupported_fs_type("NFS")); + assert!(is_unsupported_fs_type("cifs")); + assert!(is_unsupported_fs_type("smb2")); + assert!(is_unsupported_fs_type("fuse")); + assert!(is_unsupported_fs_type("fuse.sshfs")); + assert!(is_unsupported_fs_type("v9fs")); + assert!(is_unsupported_fs_type("overlayfs")); + } + + #[test] + fn unsupported_fs_type_matcher_keeps_local_posix_types_supported() { + assert!(!is_unsupported_fs_type("XFS")); + assert!(!is_unsupported_fs_type("EXT4")); + assert!(!is_unsupported_fs_type("BTRFS")); + } + + #[test] + fn policy_parser_handles_valid_and_invalid_values() { + assert_eq!(UnsupportedFsPolicy::parse("warn"), Some(UnsupportedFsPolicy::Warn)); + assert_eq!(UnsupportedFsPolicy::parse("FAIL"), Some(UnsupportedFsPolicy::Fail)); + assert_eq!(UnsupportedFsPolicy::parse("unknown"), None); + } + + #[test] + #[serial_test::serial] + fn invalid_policy_falls_back_to_configured_default() { + temp_env::with_var(ENV_RUSTFS_UNSUPPORTED_FS_POLICY, Some("invalid"), || { + assert_eq!( + get_unsupported_fs_policy(), + UnsupportedFsPolicy::parse(DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY).unwrap_or(UnsupportedFsPolicy::Warn) + ); + }); + } +}