mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
fix(storage): add scoped timeout policy and startup fs guardrail (#3056)
* fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends * style: fix cargo fmt formatting in disk_store.rs * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends Extend the TimeoutHealthAction introduced in #2996 to read_metadata, list_dir, and disk_info operations when RUSTFS_NETWORK_MOUNT_MODE=true. Also raises all drive operation timeouts to 60s (explicit per-operation overrides still take precedence). Closes #2790 * feat(startup): add unsupported filesystem policy guardrail * chore(deps): refresh lockfile and dependency pins * feat(ecstore): add scoped timeout health-action policy * docs(config): document drive timeout health-action policy * refactor(ecstore): cache timeout health policy per disk wrapper * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends (#2838) * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends * style: fix cargo fmt formatting in disk_store.rs * fix(storage): add RUSTFS_NETWORK_MOUNT_MODE for CIFS/NFS backends Extend the TimeoutHealthAction introduced in #2996 to read_metadata, list_dir, and disk_info operations when RUSTFS_NETWORK_MOUNT_MODE=true. Also raises all drive operation timeouts to 60s (explicit per-operation overrides still take precedence). Closes #2790 * fix(utils): map verified Linux filesystem magic values (#3051) * fix(utils): cover sha256 checksum validation (#3052) * fix(utils): cover sha256 checksum validation * docs: clarify sha256 checksum validation --------- Co-authored-by: houseme <housemecn@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com> * refactor(config): replace network mount mode with timeout profile preset * fix(review): align fallback defaults and extend fs-type detection * fix(review): cache timeout profile and restore probe timeout semantics * refactor(ecstore): cache timeout health policy lookup * perf(ecstore): cache active probe timeout per monitor task --------- Co-authored-by: mistik <mistiklord4@gmail.com> Co-authored-by: 安正超 <anzhengchao@gmail.com>
This commit is contained in:
houseme
Generated
+23
-10
@@ -4395,6 +4395,18 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "gloo-timers"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "482ce8a491a501da4cd806bd190275363d674f2845005c6ddbd5d3e1dd54495d"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-auth"
|
||||
version = "1.10.0"
|
||||
@@ -6442,9 +6454,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "mysql_common"
|
||||
version = "0.37.1"
|
||||
version = "0.37.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bffc2127d4035fa5a614935c663a15a4468e64e798473e0cc21c8df40a607588"
|
||||
checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bitflags 2.11.1",
|
||||
@@ -9089,20 +9101,22 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "russh-sftp"
|
||||
version = "2.1.2"
|
||||
version = "2.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09daa0ebcf53fb18d7b16167586a68b5bf2cfa3eaad49e661a19302552a2b879"
|
||||
checksum = "c826d40e310bbcb377fd8f6c5873c12eaf54da30569d8e8da56914fdac9773cd"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dashmap",
|
||||
"gloo-timers",
|
||||
"log",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"wasm-bindgen-futures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10697,9 +10711,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.149"
|
||||
version = "1.0.150"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
|
||||
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
@@ -11802,13 +11816,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio-postgres-rustls"
|
||||
version = "0.13.0"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144"
|
||||
checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb"
|
||||
dependencies = [
|
||||
"const-oid 0.9.6",
|
||||
"ring",
|
||||
"rustls",
|
||||
"sha2 0.11.0",
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tokio-rustls",
|
||||
|
||||
+3
-3
@@ -158,7 +158,7 @@ quick-xml = "0.40.1"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.149", features = ["raw_value"] }
|
||||
serde_json = { version = "1.0.150", features = ["raw_value"] }
|
||||
serde_urlencoded = "0.7.1"
|
||||
|
||||
# Cryptography and Security
|
||||
@@ -189,7 +189,7 @@ time = { version = "0.3.47", features = ["std", "parsing", "formatting", "macros
|
||||
# Database
|
||||
deadpool-postgres = { version = "0.14", features = ["rt_tokio_1"] }
|
||||
tokio-postgres = { version = "0.7", default-features = false, features = ["runtime", "with-serde_json-1"] }
|
||||
tokio-postgres-rustls = "0.13"
|
||||
tokio-postgres-rustls = "0.14.0"
|
||||
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.102"
|
||||
@@ -307,7 +307,7 @@ unftp-core = "0.1.0"
|
||||
suppaftp = { version = "8.0.3", features = ["tokio", "tokio-rustls-aws-lc-rs"] }
|
||||
rcgen = "0.14.8"
|
||||
russh = { version = "0.60.3", git = "https://github.com/Eugeny/russh", rev = "fc6e3ab4cd4338e94ae64e17aeed2acee9335e6b" }
|
||||
russh-sftp = "2.1.2"
|
||||
russh-sftp = "2.2.2"
|
||||
|
||||
# WebDAV
|
||||
dav-server = "0.11.0"
|
||||
|
||||
@@ -83,6 +83,28 @@ Legacy compatibility fallback:
|
||||
- `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION`
|
||||
This legacy variable is treated as a deprecated fallback for the operation-specific drive timeout variables above when a canonical variable is unset.
|
||||
|
||||
Drive timeout health-action policy:
|
||||
- `RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION`
|
||||
- `mark_failure` (default): timeout marks failure and may transition drive runtime state.
|
||||
- `ignore_scanner`: timeout does not mark failure for scanner-sensitive operations (`walk_dir`, `read_metadata`, `list_dir`, `disk_info`).
|
||||
|
||||
Drive timeout profile preset:
|
||||
- `RUSTFS_DRIVE_TIMEOUT_PROFILE`
|
||||
- `default` (default): keep current timeout defaults.
|
||||
- `high_latency`: use 60s default timeout for scanner-sensitive operations when no per-operation timeout override is set (`read_metadata`, `disk_info`, `list_dir`, `walk_dir`, `walk_dir_stall`).
|
||||
- Precedence:
|
||||
- Explicit per-operation timeout env (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) takes highest precedence.
|
||||
- Then `RUSTFS_DRIVE_MAX_TIMEOUT_DURATION` legacy fallback.
|
||||
- Then the profile-derived default (`default` or `high_latency`).
|
||||
|
||||
## Startup filesystem boundary policy
|
||||
|
||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||
- `warn` (default): log warning and continue startup.
|
||||
- `fail`: abort startup with an error.
|
||||
|
||||
RustFS production guidance remains direct-attached local POSIX filesystems. Network-mounted filesystems (for example `nfs`, `cifs`, `smb2`, and `fuse.*`) are treated as unsupported by this startup guard.
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the Apache License 2.0 - see the [LICENSE](../../LICENSE) file for details.
|
||||
|
||||
@@ -161,6 +161,16 @@ pub const ENV_RUSTFS_SECRET_KEY_FILE: &str = "RUSTFS_SECRET_KEY_FILE";
|
||||
/// provide non-default `RUSTFS_ACCESS_KEY` and `RUSTFS_SECRET_KEY` values.
|
||||
pub const ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS: &str = "RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS";
|
||||
|
||||
/// Environment variable controlling startup behavior when unsupported filesystem types are detected.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "warn" (default): log a warning and continue startup
|
||||
/// - "fail": abort startup with an error
|
||||
pub const ENV_RUSTFS_UNSUPPORTED_FS_POLICY: &str = "RUSTFS_UNSUPPORTED_FS_POLICY";
|
||||
pub const RUSTFS_UNSUPPORTED_FS_POLICY_WARN: &str = "warn";
|
||||
pub const RUSTFS_UNSUPPORTED_FS_POLICY_FAIL: &str = "fail";
|
||||
pub const DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY: &str = RUSTFS_UNSUPPORTED_FS_POLICY_WARN;
|
||||
|
||||
/// Environment variable for server OBS endpoint.
|
||||
pub const ENV_RUSTFS_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
|
||||
|
||||
@@ -47,6 +47,16 @@ pub const DEFAULT_DRIVE_ACTIVE_CHECK_INTERVAL_SECS: u64 = 15;
|
||||
pub const ENV_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: &str = "RUSTFS_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS";
|
||||
pub const DEFAULT_DRIVE_ACTIVE_CHECK_TIMEOUT_SECS: u64 = 5;
|
||||
|
||||
/// Timeout-to-health transition policy for drive operations.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "mark_failure" (default): timeout marks failure and may transition runtime state.
|
||||
/// - "ignore_scanner": timeout does not mark failure for scanner-sensitive operations.
|
||||
pub const ENV_DRIVE_TIMEOUT_HEALTH_ACTION: &str = "RUSTFS_DRIVE_TIMEOUT_HEALTH_ACTION";
|
||||
pub const DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE: &str = "mark_failure";
|
||||
pub const DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER: &str = "ignore_scanner";
|
||||
pub const DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION: &str = DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE;
|
||||
|
||||
/// Number of consecutive failures before a suspect drive is classified as offline.
|
||||
pub const ENV_DRIVE_SUSPECT_FAILURE_THRESHOLD: &str = "RUSTFS_DRIVE_SUSPECT_FAILURE_THRESHOLD";
|
||||
pub const DEFAULT_DRIVE_SUSPECT_FAILURE_THRESHOLD: u64 = 2;
|
||||
@@ -66,3 +76,19 @@ pub const DEFAULT_DRIVE_OFFLINE_GRACE_PERIOD_SECS: u64 = 30;
|
||||
/// Duration in seconds after which a recovered drive is classified as long offline.
|
||||
pub const ENV_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: &str = "RUSTFS_DRIVE_LONG_OFFLINE_THRESHOLD_SECS";
|
||||
pub const DEFAULT_DRIVE_LONG_OFFLINE_THRESHOLD_SECS: u64 = 172_800;
|
||||
|
||||
/// Drive timeout profile preset.
|
||||
///
|
||||
/// Accepted values:
|
||||
/// - "default": keep current timeout defaults.
|
||||
/// - "high_latency": use a higher default timeout preset for scanner-sensitive and metadata operations.
|
||||
///
|
||||
/// Explicit per-operation overrides (`RUSTFS_DRIVE_*_TIMEOUT_SECS`) still take precedence.
|
||||
pub const ENV_DRIVE_TIMEOUT_PROFILE: &str = "RUSTFS_DRIVE_TIMEOUT_PROFILE";
|
||||
pub const DRIVE_TIMEOUT_PROFILE_DEFAULT: &str = "default";
|
||||
pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY: &str = "high_latency";
|
||||
pub const DEFAULT_DRIVE_TIMEOUT_PROFILE: &str = DRIVE_TIMEOUT_PROFILE_DEFAULT;
|
||||
|
||||
/// Timeout preset (seconds) used when `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`
|
||||
/// and no per-operation timeout override is provided.
|
||||
pub const DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS: u64 = 60;
|
||||
|
||||
@@ -27,6 +27,8 @@ use crate::global::GLOBAL_LOCAL_DISK_ID_MAP;
|
||||
use bytes::Bytes;
|
||||
use metrics::counter;
|
||||
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
|
||||
#[cfg(not(test))]
|
||||
use std::sync::OnceLock;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
sync::{
|
||||
@@ -50,10 +52,54 @@ enum TimeoutHealthAction {
|
||||
IgnoreFailure,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum TimeoutHealthPolicy {
|
||||
MarkFailure,
|
||||
IgnoreScanner,
|
||||
}
|
||||
|
||||
impl TimeoutHealthPolicy {
|
||||
fn parse(raw: &str) -> Option<Self> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_MARK_FAILURE => Some(Self::MarkFailure),
|
||||
rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER => Some(Self::IgnoreScanner),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_timeout_health_action(self) -> TimeoutHealthAction {
|
||||
match self {
|
||||
Self::MarkFailure => TimeoutHealthAction::MarkFailure,
|
||||
Self::IgnoreScanner => TimeoutHealthAction::IgnoreFailure,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub const ENV_RUSTFS_DRIVE_ACTIVE_MONITORING: &str = "RUSTFS_DRIVE_ACTIVE_MONITORING";
|
||||
pub const DEFAULT_RUSTFS_DRIVE_ACTIVE_MONITORING: bool = true;
|
||||
pub const SKIP_IF_SUCCESS_BEFORE: Duration = Duration::from_secs(5);
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum DriveTimeoutProfile {
|
||||
Default,
|
||||
HighLatency,
|
||||
}
|
||||
|
||||
impl DriveTimeoutProfile {
|
||||
fn parse(raw: &str) -> Option<Self> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
rustfs_config::DRIVE_TIMEOUT_PROFILE_DEFAULT => Some(Self::Default),
|
||||
rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY => Some(Self::HighLatency),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static DRIVE_TIMEOUT_PROFILE_CACHE: OnceLock<DriveTimeoutProfile> = OnceLock::new();
|
||||
#[cfg(not(test))]
|
||||
static DRIVE_TIMEOUT_HEALTH_POLICY_CACHE: OnceLock<TimeoutHealthPolicy> = OnceLock::new();
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref TEST_DATA: Bytes = Bytes::from(vec![42u8; 2048]);
|
||||
static ref TEST_BUCKET: String = ".rustfs.sys/tmp".to_string();
|
||||
@@ -66,10 +112,39 @@ pub fn get_max_timeout_duration() -> Duration {
|
||||
))
|
||||
}
|
||||
|
||||
fn get_drive_timeout_duration(env_key: &str, default_secs: u64) -> Duration {
|
||||
fn resolve_drive_timeout_profile_from_env() -> DriveTimeoutProfile {
|
||||
let raw = rustfs_utils::get_env_str(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE);
|
||||
if let Some(profile) = DriveTimeoutProfile::parse(&raw) {
|
||||
return profile;
|
||||
}
|
||||
warn!(
|
||||
env = rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
value = %raw,
|
||||
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE,
|
||||
"Invalid drive timeout profile; falling back to default"
|
||||
);
|
||||
DriveTimeoutProfile::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_PROFILE).unwrap_or(DriveTimeoutProfile::Default)
|
||||
}
|
||||
|
||||
fn get_drive_timeout_profile() -> DriveTimeoutProfile {
|
||||
#[cfg(test)]
|
||||
{
|
||||
resolve_drive_timeout_profile_from_env()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
*DRIVE_TIMEOUT_PROFILE_CACHE.get_or_init(resolve_drive_timeout_profile_from_env)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_drive_timeout_duration(env_key: &str, default_secs: u64, high_latency_secs: Option<u64>) -> Duration {
|
||||
let fallback_default = match (get_drive_timeout_profile(), high_latency_secs) {
|
||||
(DriveTimeoutProfile::HighLatency, Some(secs)) => secs,
|
||||
_ => default_secs,
|
||||
};
|
||||
Duration::from_secs(
|
||||
rustfs_utils::get_env_opt_u64_with_aliases(env_key, &[rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION])
|
||||
.unwrap_or(default_secs),
|
||||
.unwrap_or(fallback_default),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -77,6 +152,7 @@ pub fn get_drive_metadata_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -84,6 +160,7 @@ pub fn get_drive_disk_info_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_DISK_INFO_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -91,6 +168,7 @@ pub fn get_drive_list_dir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_LIST_DIR_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -98,6 +176,7 @@ pub fn get_drive_walkdir_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -105,6 +184,7 @@ pub fn get_drive_walkdir_stall_timeout() -> Duration {
|
||||
get_drive_timeout_duration(
|
||||
rustfs_config::ENV_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
rustfs_config::DEFAULT_DRIVE_WALKDIR_STALL_TIMEOUT_SECS,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -122,6 +202,34 @@ pub fn get_drive_active_check_timeout() -> Duration {
|
||||
))
|
||||
}
|
||||
|
||||
fn resolve_drive_timeout_health_policy_from_env() -> TimeoutHealthPolicy {
|
||||
let raw = rustfs_utils::get_env_str(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
);
|
||||
if let Some(policy) = TimeoutHealthPolicy::parse(&raw) {
|
||||
return policy;
|
||||
}
|
||||
warn!(
|
||||
env = rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
value = %raw,
|
||||
default = rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
"Invalid drive timeout health action policy; falling back to default"
|
||||
);
|
||||
TimeoutHealthPolicy::parse(rustfs_config::DEFAULT_DRIVE_TIMEOUT_HEALTH_ACTION).unwrap_or(TimeoutHealthPolicy::MarkFailure)
|
||||
}
|
||||
|
||||
fn get_drive_timeout_health_policy() -> TimeoutHealthPolicy {
|
||||
#[cfg(test)]
|
||||
{
|
||||
resolve_drive_timeout_health_policy_from_env()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
*DRIVE_TIMEOUT_HEALTH_POLICY_CACHE.get_or_init(resolve_drive_timeout_health_policy_from_env)
|
||||
}
|
||||
}
|
||||
|
||||
/// DiskHealthTracker tracks the health status of a disk.
|
||||
/// Similar to Go's diskHealthTracker.
|
||||
#[derive(Debug)]
|
||||
@@ -470,6 +578,8 @@ pub struct LocalDiskWrapper {
|
||||
cancel_token: CancellationToken,
|
||||
/// Disk ID for stale checking
|
||||
disk_id: Arc<RwLock<Option<Uuid>>>,
|
||||
/// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
|
||||
timeout_health_policy: TimeoutHealthPolicy,
|
||||
}
|
||||
|
||||
impl LocalDiskWrapper {
|
||||
@@ -486,6 +596,7 @@ impl LocalDiskWrapper {
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
timeout_health_policy: get_drive_timeout_health_policy(),
|
||||
};
|
||||
record_drive_runtime_state(&wrapper.disk.endpoint(), RuntimeDriveHealthState::Online);
|
||||
wrapper
|
||||
@@ -511,6 +622,10 @@ impl LocalDiskWrapper {
|
||||
self.health.record_capacity_probe(total, used, free);
|
||||
}
|
||||
|
||||
fn scanner_timeout_health_action(&self) -> TimeoutHealthAction {
|
||||
self.timeout_health_policy.scanner_timeout_health_action()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn force_runtime_state_for_test(&self, state: RuntimeDriveHealthState) {
|
||||
self.health.force_runtime_state_for_test(state);
|
||||
@@ -558,6 +673,7 @@ impl LocalDiskWrapper {
|
||||
/// Monitor disk writability periodically
|
||||
async fn monitor_disk_writable(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
let mut interval = time::interval(get_drive_active_check_interval());
|
||||
let active_check_timeout = get_drive_active_check_timeout();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -596,7 +712,7 @@ impl LocalDiskWrapper {
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
true,
|
||||
get_drive_active_check_timeout(),
|
||||
active_check_timeout,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
@@ -691,6 +807,7 @@ impl LocalDiskWrapper {
|
||||
/// Monitor disk status and try to bring it back online
|
||||
async fn monitor_disk_status(disk: Arc<LocalDisk>, health: Arc<DiskHealthTracker>, cancel_token: CancellationToken) {
|
||||
let check_every = get_drive_returning_probe_interval();
|
||||
let active_check_timeout = get_drive_active_check_timeout();
|
||||
|
||||
let mut interval = time::interval(check_every);
|
||||
|
||||
@@ -711,7 +828,7 @@ impl LocalDiskWrapper {
|
||||
&test_obj,
|
||||
&TEST_DATA,
|
||||
false,
|
||||
get_drive_active_check_timeout(),
|
||||
active_check_timeout,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -893,10 +1010,11 @@ impl LocalDiskWrapper {
|
||||
#[async_trait::async_trait]
|
||||
impl DiskAPI for LocalDiskWrapper {
|
||||
async fn read_metadata(&self, volume: &str, path: &str) -> Result<Bytes> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"read_metadata",
|
||||
|| async { self.disk.read_metadata(volume, path).await },
|
||||
get_drive_metadata_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -973,7 +1091,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"disk_info",
|
||||
|| async {
|
||||
let result = self.disk.disk_info(opts).await?;
|
||||
@@ -987,6 +1105,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
Ok(result)
|
||||
},
|
||||
get_drive_disk_info_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1021,7 +1140,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
"walk_dir",
|
||||
|| async { self.disk.walk_dir(opts, wr).await },
|
||||
get_drive_walkdir_timeout(),
|
||||
TimeoutHealthAction::IgnoreFailure,
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1130,10 +1249,11 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||
self.track_disk_health_with_op(
|
||||
self.track_disk_health_with_op_and_timeout_action(
|
||||
"list_dir",
|
||||
|| async { self.disk.list_dir(origvolume, volume, dir_path, count).await },
|
||||
get_drive_list_dir_timeout(),
|
||||
self.scanner_timeout_health_action(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1256,14 +1376,48 @@ mod tests {
|
||||
fn drive_metadata_timeout_uses_default_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_high_latency_profile_when_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY),
|
||||
|| {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DRIVE_TIMEOUT_PROFILE_HIGH_LATENCY_SECS)
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_invalid_profile_falls_back_to_default() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_MAX_TIMEOUT_DURATION, || {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_PROFILE, Some("invalid"), || {
|
||||
assert_eq!(
|
||||
get_drive_metadata_timeout(),
|
||||
Duration::from_secs(rustfs_config::DEFAULT_DRIVE_METADATA_TIMEOUT_SECS)
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drive_metadata_timeout_uses_legacy_fallback_when_canonical_unset() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_METADATA_TIMEOUT_SECS, || {
|
||||
@@ -1398,6 +1552,58 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_writer_backpressure_timeout_does_not_mark_drive_failure() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1")),
|
||||
(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
|
||||
),
|
||||
],
|
||||
async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("temp dir should be valid UTF-8"))
|
||||
.expect("endpoint should parse");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk should be created"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let bucket = "test-bucket";
|
||||
let object = "test-object";
|
||||
|
||||
wrapper.make_volume(bucket).await.expect("bucket should be created");
|
||||
|
||||
let mut file_info = FileInfo::new(&format!("{bucket}/{object}"), 1, 0);
|
||||
file_info.volume = bucket.to_string();
|
||||
file_info.name = object.to_string();
|
||||
file_info.mod_time = Some(::time::OffsetDateTime::now_utc());
|
||||
file_info.erasure.index = 1;
|
||||
|
||||
wrapper
|
||||
.write_metadata("", bucket, object, file_info)
|
||||
.await
|
||||
.expect("object metadata should be written");
|
||||
|
||||
let mut writer = PendingWriter;
|
||||
let result = wrapper
|
||||
.walk_dir(
|
||||
WalkDirOptions {
|
||||
bucket: bucket.to_string(),
|
||||
recursive: true,
|
||||
..Default::default()
|
||||
},
|
||||
&mut writer,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_writer_backpressure_timeout_marks_drive_failure_by_default() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_DRIVE_WALKDIR_TIMEOUT_SECS, Some("1"))], async {
|
||||
let dir = tempfile::tempdir().expect("temp dir should be created");
|
||||
let endpoint =
|
||||
@@ -1433,8 +1639,7 @@ mod tests {
|
||||
.await;
|
||||
|
||||
assert_eq!(result.expect_err("walk_dir should time out"), DiskError::Timeout);
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Online);
|
||||
assert!(!wrapper.health.is_faulty());
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -1462,6 +1667,40 @@ mod tests {
|
||||
assert_eq!(wrapper.runtime_state(), RuntimeDriveHealthState::Suspect);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_defaults_to_mark_failure() {
|
||||
temp_env::with_var_unset(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, || {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_respects_ignore_scanner() {
|
||||
temp_env::with_var(
|
||||
rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION,
|
||||
Some(rustfs_config::DRIVE_TIMEOUT_HEALTH_ACTION_IGNORE_SCANNER),
|
||||
|| {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::IgnoreScanner);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::IgnoreFailure);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn drive_timeout_health_policy_invalid_value_falls_back_to_default() {
|
||||
temp_env::with_var(rustfs_config::ENV_DRIVE_TIMEOUT_HEALTH_ACTION, Some("invalid"), || {
|
||||
let policy = get_drive_timeout_health_policy();
|
||||
assert_eq!(policy, TimeoutHealthPolicy::MarkFailure);
|
||||
assert_eq!(policy.scanner_timeout_health_action(), TimeoutHealthAction::MarkFailure);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_for_store_init_retry_clears_faulty_and_back_online() {
|
||||
let endpoint = Endpoint::try_from("/tmp/reset-store-init-retry").expect("endpoint should parse");
|
||||
|
||||
@@ -45,6 +45,7 @@ pub(crate) fn get_fs_type(fs_type: u64) -> &'static str {
|
||||
0x73717368 => "SQUASHFS",
|
||||
0xFF534D42 => "CIFS",
|
||||
0xFE534D42 => "SMB2",
|
||||
0x01021997 => "V9FS",
|
||||
0xca451a4e => "BCACHEFS",
|
||||
_ => "UNKNOWN",
|
||||
}
|
||||
@@ -71,6 +72,7 @@ mod tests {
|
||||
assert_eq!(get_fs_type(0x73717368), "SQUASHFS");
|
||||
assert_eq!(get_fs_type(0xFF534D42), "CIFS");
|
||||
assert_eq!(get_fs_type(0xFE534D42), "SMB2");
|
||||
assert_eq!(get_fs_type(0x01021997), "V9FS");
|
||||
assert_eq!(get_fs_type(0xca451a4e), "BCACHEFS");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,9 @@ data:
|
||||
RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS: {{ .cache_save_timeout_secs | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.config.rustfs.timeout_profile }}
|
||||
RUSTFS_DRIVE_TIMEOUT_PROFILE: {{ .Values.config.rustfs.timeout_profile | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.config.rustfs.kms.enabled }}
|
||||
{{- if eq .Values.config.rustfs.kms.type "vault" }}
|
||||
{{- with .Values.config.rustfs.kms.vault }}
|
||||
|
||||
@@ -91,6 +91,10 @@ config:
|
||||
idle_mode: ""
|
||||
# Timeout for scanner cache saves in seconds (minimum 1 second)
|
||||
cache_save_timeout_secs: ""
|
||||
# Drive timeout profile preset: default|high_latency
|
||||
# - default: keep current timeout defaults.
|
||||
# - high_latency: use higher timeout defaults for scanner-sensitive operations.
|
||||
timeout_profile: ""
|
||||
obs_endpoint:
|
||||
enabled: false # If true, rustfs will export metrics, traces, logs and profiling data to the specified OTLP endpoints. If false, the individual settings for metrics, traces, logs and profiling endpoints will be ignored and all data will not be exported.
|
||||
base_endpoint: "" #Root OTLP/HTTP endpoint, e.g. http://otel-collector:4318
|
||||
|
||||
@@ -50,6 +50,7 @@ use crate::app::context::{AppContext, init_global_app_context};
|
||||
use crate::config::Config;
|
||||
use crate::init::{add_bucket_notification_configuration, init_buffer_profile_system, init_kms_system};
|
||||
use crate::server::{init_event_notifier, shutdown_event_notifier, start_audit_system, start_http_server, stop_audit_system};
|
||||
use crate::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
|
||||
use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS;
|
||||
use rustfs_credentials::init_global_action_credentials;
|
||||
@@ -350,6 +351,7 @@ impl RustFSServerBuilder {
|
||||
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_addr_str.as_str(), config.volumes.clone())
|
||||
.await
|
||||
.map_err(|e| ServerError::Init(format!("endpoints: {e}")))?;
|
||||
enforce_unsupported_fs_policy(&endpoint_pools).map_err(|e| ServerError::Init(format!("unsupported fs guard: {e}")))?;
|
||||
|
||||
set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
update_erasure_type(setup_type).await;
|
||||
|
||||
@@ -67,6 +67,7 @@ pub mod profiling;
|
||||
#[cfg(any(feature = "ftps", feature = "webdav", feature = "sftp"))]
|
||||
pub mod protocols;
|
||||
pub mod server;
|
||||
pub mod startup_fs_guard;
|
||||
pub mod storage;
|
||||
pub mod update;
|
||||
pub mod version;
|
||||
|
||||
@@ -33,6 +33,7 @@ use rustfs::server::{
|
||||
SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, ShutdownSignal, init_event_notifier, shutdown_event_notifier,
|
||||
start_audit_system, start_http_server, stop_audit_system, wait_for_shutdown,
|
||||
};
|
||||
use rustfs::startup_fs_guard::enforce_unsupported_fs_policy;
|
||||
use rustfs_common::{GlobalReadiness, SystemStage, set_global_addr};
|
||||
use rustfs_config::ENV_RUSTFS_ALLOW_INSECURE_DEFAULT_CREDENTIALS;
|
||||
use rustfs_credentials::init_global_action_credentials;
|
||||
@@ -301,6 +302,7 @@ async fn run(config: rustfs::config::Config) -> Result<()> {
|
||||
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), config.volumes.clone())
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
enforce_unsupported_fs_policy(&endpoint_pools)?;
|
||||
|
||||
set_global_endpoints(endpoint_pools.as_ref().clone());
|
||||
update_erasure_type(setup_type).await;
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_config::{
|
||||
DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY, ENV_RUSTFS_UNSUPPORTED_FS_POLICY, RUSTFS_UNSUPPORTED_FS_POLICY_FAIL,
|
||||
RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
||||
};
|
||||
use rustfs_ecstore::endpoints::EndpointServerPools;
|
||||
use std::collections::BTreeSet;
|
||||
use std::io::{Error, Result};
|
||||
use tracing::warn;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum UnsupportedFsPolicy {
|
||||
Warn,
|
||||
Fail,
|
||||
}
|
||||
|
||||
impl UnsupportedFsPolicy {
|
||||
fn parse(raw: &str) -> Option<Self> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
RUSTFS_UNSUPPORTED_FS_POLICY_WARN => Some(Self::Warn),
|
||||
RUSTFS_UNSUPPORTED_FS_POLICY_FAIL => Some(Self::Fail),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_unsupported_fs_policy() -> UnsupportedFsPolicy {
|
||||
let raw = rustfs_utils::get_env_str(ENV_RUSTFS_UNSUPPORTED_FS_POLICY, DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY);
|
||||
if let Some(policy) = UnsupportedFsPolicy::parse(&raw) {
|
||||
return policy;
|
||||
}
|
||||
|
||||
warn!(
|
||||
env = ENV_RUSTFS_UNSUPPORTED_FS_POLICY,
|
||||
value = %raw,
|
||||
default = DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY,
|
||||
"Invalid unsupported filesystem policy; falling back to default"
|
||||
);
|
||||
UnsupportedFsPolicy::parse(DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY).unwrap_or(UnsupportedFsPolicy::Warn)
|
||||
}
|
||||
|
||||
fn is_unsupported_fs_type(fs_type: &str) -> bool {
|
||||
let normalized = fs_type.trim().to_ascii_lowercase();
|
||||
matches!(
|
||||
normalized.as_str(),
|
||||
"nfs" | "cifs" | "smb2" | "fuse" | "fuseblk" | "overlayfs" | "9p" | "v9fs" | "ceph" | "glusterfs" | "gfs" | "gfs2"
|
||||
) || normalized.starts_with("fuse.")
|
||||
}
|
||||
|
||||
fn collect_local_paths(endpoint_pools: &EndpointServerPools) -> Vec<String> {
|
||||
let mut local_paths = BTreeSet::new();
|
||||
for pool in endpoint_pools.as_ref() {
|
||||
for endpoint in pool.endpoints.as_ref() {
|
||||
if endpoint.is_local {
|
||||
local_paths.insert(endpoint.get_file_path());
|
||||
}
|
||||
}
|
||||
}
|
||||
local_paths.into_iter().collect()
|
||||
}
|
||||
|
||||
pub fn enforce_unsupported_fs_policy(endpoint_pools: &EndpointServerPools) -> Result<()> {
|
||||
let local_paths = collect_local_paths(endpoint_pools);
|
||||
if local_paths.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut unsupported = Vec::new();
|
||||
for path in local_paths {
|
||||
let info = match rustfs_utils::os::get_info(&path) {
|
||||
Ok(info) => info,
|
||||
Err(err) => {
|
||||
warn!(
|
||||
path = %path,
|
||||
error = %err,
|
||||
"Failed to inspect filesystem type for startup boundary guard; skipping this path"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if is_unsupported_fs_type(&info.fstype) {
|
||||
unsupported.push((path, info.fstype));
|
||||
}
|
||||
}
|
||||
|
||||
if unsupported.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let detail = unsupported
|
||||
.iter()
|
||||
.map(|(path, fs_type)| format!("{path} ({fs_type})"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
|
||||
let message = format!(
|
||||
"Unsupported filesystem type detected for RustFS local endpoints: {detail}. \
|
||||
RustFS only supports direct-attached local POSIX filesystems for production workloads."
|
||||
);
|
||||
|
||||
match get_unsupported_fs_policy() {
|
||||
UnsupportedFsPolicy::Warn => {
|
||||
warn!(
|
||||
env = ENV_RUSTFS_UNSUPPORTED_FS_POLICY,
|
||||
expected = format!("{RUSTFS_UNSUPPORTED_FS_POLICY_WARN}|{RUSTFS_UNSUPPORTED_FS_POLICY_FAIL}"),
|
||||
policy = RUSTFS_UNSUPPORTED_FS_POLICY_WARN,
|
||||
"{message}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
UnsupportedFsPolicy::Fail => Err(Error::other(format!(
|
||||
"{message} Startup aborted by policy: {ENV_RUSTFS_UNSUPPORTED_FS_POLICY}={RUSTFS_UNSUPPORTED_FS_POLICY_FAIL}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unsupported_fs_type_matcher_covers_network_like_types() {
|
||||
assert!(is_unsupported_fs_type("NFS"));
|
||||
assert!(is_unsupported_fs_type("cifs"));
|
||||
assert!(is_unsupported_fs_type("smb2"));
|
||||
assert!(is_unsupported_fs_type("fuse"));
|
||||
assert!(is_unsupported_fs_type("fuse.sshfs"));
|
||||
assert!(is_unsupported_fs_type("v9fs"));
|
||||
assert!(is_unsupported_fs_type("overlayfs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_fs_type_matcher_keeps_local_posix_types_supported() {
|
||||
assert!(!is_unsupported_fs_type("XFS"));
|
||||
assert!(!is_unsupported_fs_type("EXT4"));
|
||||
assert!(!is_unsupported_fs_type("BTRFS"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn policy_parser_handles_valid_and_invalid_values() {
|
||||
assert_eq!(UnsupportedFsPolicy::parse("warn"), Some(UnsupportedFsPolicy::Warn));
|
||||
assert_eq!(UnsupportedFsPolicy::parse("FAIL"), Some(UnsupportedFsPolicy::Fail));
|
||||
assert_eq!(UnsupportedFsPolicy::parse("unknown"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn invalid_policy_falls_back_to_configured_default() {
|
||||
temp_env::with_var(ENV_RUSTFS_UNSUPPORTED_FS_POLICY, Some("invalid"), || {
|
||||
assert_eq!(
|
||||
get_unsupported_fs_policy(),
|
||||
UnsupportedFsPolicy::parse(DEFAULT_RUSTFS_UNSUPPORTED_FS_POLICY).unwrap_or(UnsupportedFsPolicy::Warn)
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user