mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
feat(ecstore): enforce local disk topology guardrails and expose device ids (#2679)
This commit is contained in:
@@ -131,6 +131,18 @@ pub const ENV_RUSTFS_ADDRESS: &str = "RUSTFS_ADDRESS";
|
|||||||
/// Environment variable for server volumes.
|
/// Environment variable for server volumes.
|
||||||
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
|
pub const ENV_RUSTFS_VOLUMES: &str = "RUSTFS_VOLUMES";
|
||||||
|
|
||||||
|
/// Environment variable to explicitly bypass local physical disk independence checks.
|
||||||
|
pub const ENV_UNSAFE_BYPASS_DISK_CHECK: &str = "RUSTFS_UNSAFE_BYPASS_DISK_CHECK";
|
||||||
|
|
||||||
|
/// Compatibility alias used by legacy MinIO CI pipelines.
|
||||||
|
///
|
||||||
|
/// RustFS keeps this alias for backward compatibility only. Prefer
|
||||||
|
/// `ENV_UNSAFE_BYPASS_DISK_CHECK` for explicit bypass control.
|
||||||
|
pub const ENV_MINIO_CI: &str = "MINIO_CI";
|
||||||
|
|
||||||
|
/// Default flag value for bypassing local physical disk independence checks.
|
||||||
|
pub const DEFAULT_UNSAFE_BYPASS_DISK_CHECK: bool = false;
|
||||||
|
|
||||||
/// Environment variable for server access key.
|
/// Environment variable for server access key.
|
||||||
pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
|
pub const ENV_RUSTFS_ACCESS_KEY: &str = "RUSTFS_ACCESS_KEY";
|
||||||
|
|
||||||
|
|||||||
@@ -126,7 +126,7 @@ metrics = { workspace = true }
|
|||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
||||||
criterion = { workspace = true, features = ["html_reports"] }
|
criterion = { workspace = true, features = ["html_reports"] }
|
||||||
temp-env = { workspace = true }
|
temp-env = { workspace = true, features = ["async_closure"] }
|
||||||
tracing-subscriber = { workspace = true }
|
tracing-subscriber = { workspace = true }
|
||||||
serial_test = { workspace = true }
|
serial_test = { workspace = true }
|
||||||
opentelemetry_sdk = { workspace = true }
|
opentelemetry_sdk = { workspace = true }
|
||||||
|
|||||||
@@ -176,7 +176,15 @@ impl LocalDisk {
|
|||||||
let root = root_clone.clone();
|
let root = root_clone.clone();
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
match get_disk_info(root.clone()).await {
|
match get_disk_info(root.clone()).await {
|
||||||
Ok((info, root)) => {
|
Ok((info, root_disk)) => {
|
||||||
|
let physical_device_ids = match rustfs_utils::os::get_physical_device_ids(root.to_string_lossy().as_ref())
|
||||||
|
{
|
||||||
|
Ok(ids) => ids,
|
||||||
|
Err(err) => {
|
||||||
|
warn!(root = ?root, error = ?err, "failed to resolve physical device ids for disk root");
|
||||||
|
Vec::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
let disk_info = DiskInfo {
|
let disk_info = DiskInfo {
|
||||||
total: info.total,
|
total: info.total,
|
||||||
free: info.free,
|
free: info.free,
|
||||||
@@ -186,7 +194,8 @@ impl LocalDisk {
|
|||||||
major: info.major,
|
major: info.major,
|
||||||
minor: info.minor,
|
minor: info.minor,
|
||||||
fs_type: info.fstype,
|
fs_type: info.fstype,
|
||||||
root_disk: root,
|
root_disk,
|
||||||
|
physical_device_ids,
|
||||||
id: disk_id,
|
id: disk_id,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -596,6 +596,8 @@ pub struct DiskInfo {
|
|||||||
pub scanning: bool,
|
pub scanning: bool,
|
||||||
pub endpoint: String,
|
pub endpoint: String,
|
||||||
pub mount_path: String,
|
pub mount_path: String,
|
||||||
|
/// Leaf physical block devices backing this mount path when available.
|
||||||
|
pub physical_device_ids: Vec<String>,
|
||||||
pub id: Option<Uuid>,
|
pub id: Option<Uuid>,
|
||||||
pub rotational: bool,
|
pub rotational: bool,
|
||||||
pub metrics: DiskMetrics,
|
pub metrics: DiskMetrics,
|
||||||
|
|||||||
@@ -17,10 +17,11 @@ use crate::{
|
|||||||
disks_layout::DisksLayout,
|
disks_layout::DisksLayout,
|
||||||
global::global_rustfs_port,
|
global::global_rustfs_port,
|
||||||
};
|
};
|
||||||
|
use rustfs_config::{DEFAULT_UNSAFE_BYPASS_DISK_CHECK, ENV_MINIO_CI, ENV_UNSAFE_BYPASS_DISK_CHECK};
|
||||||
use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
|
use rustfs_utils::{XHost, check_local_server_addr, get_host_ip, is_local_host};
|
||||||
use std::{
|
use std::{
|
||||||
collections::{HashMap, HashSet, hash_map::Entry},
|
collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry},
|
||||||
io::{Error, Result},
|
io::{Error, ErrorKind, Result},
|
||||||
net::IpAddr,
|
net::IpAddr,
|
||||||
};
|
};
|
||||||
use tracing::{error, info, instrument, warn};
|
use tracing::{error, info, instrument, warn};
|
||||||
@@ -348,6 +349,8 @@ impl PoolEndpointList {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validate_local_physical_disk_independence(pool_endpoint_list.as_ref())?;
|
||||||
|
|
||||||
let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
|
let setup_type = match pool_endpoint_list.as_ref()[0].as_ref()[0].get_type() {
|
||||||
EndpointType::Path => SetupType::Erasure,
|
EndpointType::Path => SetupType::Erasure,
|
||||||
EndpointType::Url => match unique_args.len() {
|
EndpointType::Url => match unique_args.len() {
|
||||||
@@ -645,12 +648,107 @@ impl EndpointServerPools {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_local_physical_disk_independence(pools: &[Endpoints]) -> Result<()> {
|
||||||
|
let mut local_paths = BTreeSet::new();
|
||||||
|
for endpoints in pools {
|
||||||
|
for endpoint in endpoints.as_ref() {
|
||||||
|
if endpoint.is_local {
|
||||||
|
local_paths.insert(endpoint.get_file_path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if local_paths.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let local_paths = local_paths.into_iter().collect::<Vec<_>>();
|
||||||
|
validate_local_cross_device_mounts(&local_paths)?;
|
||||||
|
|
||||||
|
if local_paths.len() <= 1 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compatibility behavior:
|
||||||
|
// - canonical key: RUSTFS_UNSAFE_BYPASS_DISK_CHECK
|
||||||
|
// - legacy CI alias: MINIO_CI
|
||||||
|
// If both are set, `get_env_bool_with_aliases` keeps canonical key precedence.
|
||||||
|
if rustfs_utils::get_env_bool_with_aliases(ENV_UNSAFE_BYPASS_DISK_CHECK, &[ENV_MINIO_CI], DEFAULT_UNSAFE_BYPASS_DISK_CHECK) {
|
||||||
|
warn!(
|
||||||
|
env = ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||||
|
local_paths = ?local_paths,
|
||||||
|
"Skipping local physical disk independence validation due to explicit environment override",
|
||||||
|
);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut device_paths = BTreeMap::<String, BTreeSet<String>>::new();
|
||||||
|
|
||||||
|
for path in &local_paths {
|
||||||
|
let canonical = match rustfs_utils::canonicalize(path) {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(err) if err.kind() == ErrorKind::NotFound => continue,
|
||||||
|
Err(err) => {
|
||||||
|
return Err(Error::other(format!(
|
||||||
|
"failed to resolve local endpoint path '{path}' for disk validation: {err}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let canonical_path = canonical.to_string_lossy().into_owned();
|
||||||
|
let device_ids = rustfs_utils::os::get_physical_device_ids(&canonical_path).map_err(|err| {
|
||||||
|
Error::other(format!("failed to inspect physical disk for local endpoint '{canonical_path}': {err}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
for device_id in device_ids {
|
||||||
|
device_paths.entry(device_id).or_default().insert(canonical_path.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let shared_devices = device_paths
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(device_id, paths)| {
|
||||||
|
if paths.len() <= 1 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some((device_id, paths.into_iter().collect::<Vec<_>>()))
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
if shared_devices.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let details = shared_devices
|
||||||
|
.into_iter()
|
||||||
|
.map(|(device_id, paths)| format!("{device_id} => {}", paths.join(", ")))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("; ");
|
||||||
|
|
||||||
|
Err(Error::other(format!(
|
||||||
|
"local erasure endpoints must use distinct physical disks; detected shared devices [{details}]. \
|
||||||
|
Set {ENV_UNSAFE_BYPASS_DISK_CHECK}=true only for local testing or CI to bypass this safety check"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_local_cross_device_mounts(local_paths: &[String]) -> Result<()> {
|
||||||
|
rustfs_utils::os::check_cross_device_mounts(local_paths)
|
||||||
|
.map_err(|err| Error::other(format!("local endpoint cross-device mount validation failed: {err}")))
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod test {
|
mod test {
|
||||||
use rustfs_utils::must_get_local_ips;
|
use rustfs_utils::must_get_local_ips;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
use serial_test::serial;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
use temp_env::async_with_vars;
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_new_endpoints() {
|
fn test_new_endpoints() {
|
||||||
@@ -1412,4 +1510,69 @@ mod test {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[serial]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn reject_shared_local_physical_disks_by_default() {
|
||||||
|
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, None::<&str>)], async {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let disk1 = dir.path().join("disk1");
|
||||||
|
let disk2 = dir.path().join("disk2");
|
||||||
|
std::fs::create_dir_all(&disk1).unwrap();
|
||||||
|
std::fs::create_dir_all(&disk2).unwrap();
|
||||||
|
|
||||||
|
let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
|
||||||
|
let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
|
||||||
|
|
||||||
|
let err = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
|
||||||
|
let err_text = err.to_string();
|
||||||
|
assert!(err_text.contains("distinct physical disks"), "unexpected error: {err_text}");
|
||||||
|
assert!(err_text.contains(ENV_UNSAFE_BYPASS_DISK_CHECK), "unexpected error: {err_text}");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[serial]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn allow_shared_local_physical_disks_with_explicit_env_bypass() {
|
||||||
|
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], async {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let disk1 = dir.path().join("disk1");
|
||||||
|
let disk2 = dir.path().join("disk2");
|
||||||
|
std::fs::create_dir_all(&disk1).unwrap();
|
||||||
|
std::fs::create_dir_all(&disk2).unwrap();
|
||||||
|
|
||||||
|
let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
|
||||||
|
let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
|
||||||
|
|
||||||
|
let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
|
||||||
|
assert!(ret.is_ok(), "expected bypassed disk validation to succeed, got {ret:?}");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
#[serial]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn allow_shared_local_physical_disks_with_minio_ci_alias() {
|
||||||
|
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, None::<&str>), (ENV_MINIO_CI, Some("1"))], async {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let disk1 = dir.path().join("disk1");
|
||||||
|
let disk2 = dir.path().join("disk2");
|
||||||
|
std::fs::create_dir_all(&disk1).unwrap();
|
||||||
|
std::fs::create_dir_all(&disk2).unwrap();
|
||||||
|
|
||||||
|
let args = vec![disk1.to_string_lossy().into_owned(), disk2.to_string_lossy().into_owned()];
|
||||||
|
let layout = DisksLayout::from_volumes(args.as_slice()).unwrap();
|
||||||
|
|
||||||
|
let ret = EndpointServerPools::create_server_endpoints("0.0.0.0:9000", &layout).await;
|
||||||
|
assert!(ret.is_ok(), "expected MINIO_CI alias to bypass disk validation, got {ret:?}");
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4148,6 +4148,7 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
|
|||||||
total_space: res.total,
|
total_space: res.total,
|
||||||
used_space: res.used,
|
used_space: res.used,
|
||||||
available_space: res.free,
|
available_space: res.free,
|
||||||
|
physical_device_ids: (!res.physical_device_ids.is_empty()).then_some(res.physical_device_ids.clone()),
|
||||||
utilization: {
|
utilization: {
|
||||||
if res.total > 0 {
|
if res.total > 0 {
|
||||||
res.used as f64 / res.total as f64 * 100_f64
|
res.used as f64 / res.total as f64 * 100_f64
|
||||||
|
|||||||
@@ -97,6 +97,9 @@ pub struct Disk {
|
|||||||
pub runtime_state: Option<String>,
|
pub runtime_state: Option<String>,
|
||||||
#[serde(rename = "offlineDurationSeconds", default, skip_serializing_if = "Option::is_none")]
|
#[serde(rename = "offlineDurationSeconds", default, skip_serializing_if = "Option::is_none")]
|
||||||
pub offline_duration_seconds: Option<u64>,
|
pub offline_duration_seconds: Option<u64>,
|
||||||
|
/// Leaf physical block devices backing this disk path when the platform can resolve them.
|
||||||
|
#[serde(rename = "physicalDeviceIds", default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub physical_device_ids: Option<Vec<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||||
@@ -481,6 +484,7 @@ mod tests {
|
|||||||
assert!(disk.heal_info.is_none());
|
assert!(disk.heal_info.is_none());
|
||||||
assert_eq!(disk.used_inodes, 0);
|
assert_eq!(disk.used_inodes, 0);
|
||||||
assert_eq!(disk.free_inodes, 0);
|
assert_eq!(disk.free_inodes, 0);
|
||||||
|
assert!(disk.physical_device_ids.is_none());
|
||||||
assert!(!disk.local);
|
assert!(!disk.local);
|
||||||
assert_eq!(disk.pool_index, 0);
|
assert_eq!(disk.pool_index, 0);
|
||||||
assert_eq!(disk.set_index, 0);
|
assert_eq!(disk.set_index, 0);
|
||||||
@@ -514,6 +518,7 @@ mod tests {
|
|||||||
heal_info: None,
|
heal_info: None,
|
||||||
used_inodes: 1000000,
|
used_inodes: 1000000,
|
||||||
free_inodes: 9000000,
|
free_inodes: 9000000,
|
||||||
|
physical_device_ids: Some(vec!["nvme0n1".to_string()]),
|
||||||
local: true,
|
local: true,
|
||||||
pool_index: 0,
|
pool_index: 0,
|
||||||
set_index: 1,
|
set_index: 1,
|
||||||
@@ -533,6 +538,7 @@ mod tests {
|
|||||||
assert!(disk.metrics.is_some());
|
assert!(disk.metrics.is_some());
|
||||||
assert_eq!(disk.runtime_state.as_deref(), Some("online"));
|
assert_eq!(disk.runtime_state.as_deref(), Some("online"));
|
||||||
assert_eq!(disk.offline_duration_seconds, Some(0));
|
assert_eq!(disk.offline_duration_seconds, Some(0));
|
||||||
|
assert_eq!(disk.physical_device_ids, Some(vec!["nvme0n1".to_string()]));
|
||||||
assert!(disk.local);
|
assert!(disk.local);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -575,6 +581,7 @@ mod tests {
|
|||||||
assert_eq!(decoded.used_inodes, 11_125);
|
assert_eq!(decoded.used_inodes, 11_125);
|
||||||
assert_eq!(decoded.runtime_state, None);
|
assert_eq!(decoded.runtime_state, None);
|
||||||
assert_eq!(decoded.offline_duration_seconds, None);
|
assert_eq!(decoded.offline_duration_seconds, None);
|
||||||
|
assert_eq!(decoded.physical_device_ids, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -606,6 +613,7 @@ mod tests {
|
|||||||
pool_index: 1,
|
pool_index: 1,
|
||||||
set_index: 2,
|
set_index: 2,
|
||||||
disk_index: 3,
|
disk_index: 3,
|
||||||
|
physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
|
||||||
runtime_state: Some("online".to_string()),
|
runtime_state: Some("online".to_string()),
|
||||||
offline_duration_seconds: Some(0),
|
offline_duration_seconds: Some(0),
|
||||||
};
|
};
|
||||||
@@ -622,6 +630,19 @@ mod tests {
|
|||||||
assert_eq!(decoded.endpoint, "http://current-node:9000");
|
assert_eq!(decoded.endpoint, "http://current-node:9000");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_disk_serializes_physical_device_ids_when_present() {
|
||||||
|
let disk = Disk {
|
||||||
|
physical_device_ids: Some(vec!["nvme0n1".to_string(), "nvme1n1".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let json = serde_json::to_string(&disk).unwrap();
|
||||||
|
assert!(json.contains("physicalDeviceIds"));
|
||||||
|
assert!(json.contains("nvme0n1"));
|
||||||
|
assert!(json.contains("nvme1n1"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_healing_disk_default() {
|
fn test_healing_disk_default() {
|
||||||
let healing_disk = HealingDisk::default();
|
let healing_disk = HealingDisk::default();
|
||||||
|
|||||||
@@ -14,9 +14,11 @@
|
|||||||
|
|
||||||
use crate::os::{DiskInfo, IOStats};
|
use crate::os::{DiskInfo, IOStats};
|
||||||
use rustix::fs::statfs;
|
use rustix::fs::statfs;
|
||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::fs;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{self, BufRead, Error, ErrorKind};
|
use std::io::{self, BufRead, Error, ErrorKind, Read};
|
||||||
use std::path::Path;
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// Returns total and free bytes available in a directory, e.g. `/`.
|
/// Returns total and free bytes available in a directory, e.g. `/`.
|
||||||
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
|
pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
|
||||||
@@ -120,6 +122,164 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
|
|||||||
Ok(stat1.st_dev == stat2.st_dev)
|
Ok(stat1.st_dev == stat2.st_dev)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Resolve the leaf physical device identities backing a local filesystem path.
|
||||||
|
///
|
||||||
|
/// Linux block stacks such as partitions, `dm-*`, or software RAID can all
|
||||||
|
/// expose a filesystem through an intermediate device node. This helper walks
|
||||||
|
/// sysfs until it reaches the leaf backing devices so the caller can compare
|
||||||
|
/// physical failure domains instead of only filesystem device numbers.
|
||||||
|
pub fn get_physical_device_ids(disk: &str) -> std::io::Result<Vec<String>> {
|
||||||
|
let stat = rustix::fs::stat(disk)?;
|
||||||
|
let major = rustix::fs::major(stat.st_dev) as u64;
|
||||||
|
let minor = rustix::fs::minor(stat.st_dev) as u64;
|
||||||
|
let devices = resolve_block_device_ids(major, minor)?;
|
||||||
|
|
||||||
|
Ok(devices.into_iter().collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_block_device_ids(major: u64, minor: u64) -> std::io::Result<BTreeSet<String>> {
|
||||||
|
let sysfs_path = PathBuf::from(format!("/sys/dev/block/{major}:{minor}"));
|
||||||
|
let resolved = match fs::canonicalize(&sysfs_path) {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||||
|
return Ok(BTreeSet::from([format!("{major}:{minor}")]));
|
||||||
|
}
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
};
|
||||||
|
let devices = collect_block_device_ids(&resolved)?;
|
||||||
|
|
||||||
|
if devices.is_empty() {
|
||||||
|
Ok(BTreeSet::from([format!("{major}:{minor}")]))
|
||||||
|
} else {
|
||||||
|
Ok(devices)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_block_device_ids(device_path: &Path) -> std::io::Result<BTreeSet<String>> {
|
||||||
|
let mut ids = BTreeSet::new();
|
||||||
|
let slaves_dir = device_path.join("slaves");
|
||||||
|
|
||||||
|
match fs::read_dir(&slaves_dir) {
|
||||||
|
Ok(entries) => {
|
||||||
|
let mut found_slave = false;
|
||||||
|
for entry in entries {
|
||||||
|
let entry = entry?;
|
||||||
|
found_slave = true;
|
||||||
|
let resolved = fs::canonicalize(entry.path())?;
|
||||||
|
ids.extend(collect_block_device_ids(&resolved)?);
|
||||||
|
}
|
||||||
|
|
||||||
|
if found_slave {
|
||||||
|
return Ok(ids);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
}
|
||||||
|
|
||||||
|
ids.insert(normalize_block_device_name(device_path));
|
||||||
|
|
||||||
|
Ok(ids)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_block_device_name(device_path: &Path) -> String {
|
||||||
|
if device_path.join("partition").exists()
|
||||||
|
&& let Some(parent_name) = device_path.parent().and_then(|parent| parent.file_name())
|
||||||
|
{
|
||||||
|
return parent_name.to_string_lossy().into_owned();
|
||||||
|
}
|
||||||
|
|
||||||
|
device_path
|
||||||
|
.file_name()
|
||||||
|
.map(|name| name.to_string_lossy().into_owned())
|
||||||
|
.unwrap_or_else(|| device_path.display().to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Check whether any configured export path contains nested mount points.
|
||||||
|
///
|
||||||
|
/// This mirrors the intent of MinIO's cross-device mount guardrail: once an
|
||||||
|
/// export path is selected, RustFS should not silently traverse into child
|
||||||
|
/// mount points hosted by other devices.
|
||||||
|
pub fn check_cross_device_mounts(paths: &[String]) -> std::io::Result<()> {
|
||||||
|
check_cross_device_mounts_with_reader(paths, File::open("/proc/mounts")?)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse `/proc/mounts`-style content and validate each export path against it.
|
||||||
|
fn check_cross_device_mounts_with_reader(paths: &[String], mut reader: impl Read) -> std::io::Result<()> {
|
||||||
|
let mut content = String::new();
|
||||||
|
reader.read_to_string(&mut content)?;
|
||||||
|
let mount_paths = parse_mount_paths(&content);
|
||||||
|
|
||||||
|
for path in paths {
|
||||||
|
ensure_no_sub_mounts(path, &mount_paths)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract mount paths from `/proc/mounts` content, decoding escaped spaces.
|
||||||
|
fn parse_mount_paths(content: &str) -> Vec<String> {
|
||||||
|
content
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| {
|
||||||
|
let fields = line.split_whitespace().collect::<Vec<_>>();
|
||||||
|
if fields.len() != 6 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(fields[1].replace("\\040", " "))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate that `path` does not contain nested child mount points.
|
||||||
|
fn ensure_no_sub_mounts(path: &str, mount_paths: &[String]) -> std::io::Result<()> {
|
||||||
|
if !Path::new(path).is_absolute() {
|
||||||
|
return Err(Error::new(
|
||||||
|
ErrorKind::InvalidInput,
|
||||||
|
format!("Invalid argument, path ({path}) is expected to be absolute"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if path == "/" {
|
||||||
|
return Err(Error::new(
|
||||||
|
ErrorKind::InvalidInput,
|
||||||
|
"Invalid argument, path (/) cannot be the filesystem root for export validation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let base = normalize_mount_path(path);
|
||||||
|
let mut cross_mounts = Vec::new();
|
||||||
|
|
||||||
|
for mount_path in mount_paths {
|
||||||
|
let mount_base = normalize_mount_path(mount_path);
|
||||||
|
if mount_base.starts_with(&base) && mount_base != base {
|
||||||
|
cross_mounts.push(mount_path.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if cross_mounts.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
cross_mounts.sort();
|
||||||
|
cross_mounts.dedup();
|
||||||
|
|
||||||
|
Err(Error::other(format!(
|
||||||
|
"Nested mount points detected under path ({path}) at the following locations: {}. Export path should not have any sub-mounts, refusing to start.",
|
||||||
|
cross_mounts.join(", ")
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Normalize mount paths so prefix checks treat `/a/b` and `/a/b/` identically.
|
||||||
|
fn normalize_mount_path(path: &str) -> String {
|
||||||
|
let trimmed = path.trim_end_matches('/');
|
||||||
|
if trimmed.is_empty() {
|
||||||
|
"/".to_string()
|
||||||
|
} else {
|
||||||
|
format!("{trimmed}/")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_drive_stats(major: u32, minor: u32) -> std::io::Result<IOStats> {
|
pub fn get_drive_stats(major: u32, minor: u32) -> std::io::Result<IOStats> {
|
||||||
read_drive_stats(&format!("/sys/dev/block/{major}:{minor}/stat"))
|
read_drive_stats(&format!("/sys/dev/block/{major}:{minor}/stat"))
|
||||||
}
|
}
|
||||||
@@ -180,3 +340,100 @@ fn read_stat(file_name: &str) -> std::io::Result<Vec<u64>> {
|
|||||||
|
|
||||||
Ok(stats)
|
Ok(stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalize_partition_device_to_parent_disk() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let block = dir.path().join("block");
|
||||||
|
let disk = block.join("nvme0n1");
|
||||||
|
let partition = disk.join("nvme0n1p1");
|
||||||
|
let slaves = partition.join("slaves");
|
||||||
|
|
||||||
|
fs::create_dir_all(&slaves).unwrap();
|
||||||
|
fs::write(partition.join("partition"), "1").unwrap();
|
||||||
|
|
||||||
|
let ids = collect_block_device_ids(&partition).unwrap();
|
||||||
|
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec!["nvme0n1".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flatten_device_mapper_slaves_to_leaf_devices() {
|
||||||
|
let dir = tempdir().unwrap();
|
||||||
|
let block = dir.path().join("block");
|
||||||
|
let dm = block.join("dm-0");
|
||||||
|
let dm_slaves = dm.join("slaves");
|
||||||
|
let nvme0 = block.join("nvme0n1");
|
||||||
|
let nvme1 = block.join("nvme1n1");
|
||||||
|
|
||||||
|
fs::create_dir_all(&dm_slaves).unwrap();
|
||||||
|
fs::create_dir_all(&nvme0).unwrap();
|
||||||
|
fs::create_dir_all(&nvme1).unwrap();
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
std::os::unix::fs::symlink(&nvme0, dm_slaves.join("nvme0n1")).unwrap();
|
||||||
|
std::os::unix::fs::symlink(&nvme1, dm_slaves.join("nvme1n1")).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let ids = collect_block_device_ids(&dm).unwrap();
|
||||||
|
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec!["nvme0n1".to_string(), "nvme1n1".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn detect_cross_device_sub_mounts() {
|
||||||
|
let mounts = "\
|
||||||
|
/dev/root / ext4 rw 0 0
|
||||||
|
/dev/sdb1 /data ext4 rw 0 0
|
||||||
|
/dev/sdc1 /data/disk1/sub ext4 rw 0 0
|
||||||
|
";
|
||||||
|
|
||||||
|
let err = check_cross_device_mounts_with_reader(&["/data/disk1".to_string()], mounts.as_bytes()).unwrap_err();
|
||||||
|
assert!(err.to_string().contains("Nested mount points detected under path"));
|
||||||
|
assert!(err.to_string().contains("/data/disk1/sub"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn allow_mount_path_without_sub_mounts() {
|
||||||
|
let mounts = "\
|
||||||
|
/dev/root / ext4 rw 0 0
|
||||||
|
/dev/sdb1 /data/disk1 ext4 rw 0 0
|
||||||
|
";
|
||||||
|
|
||||||
|
check_cross_device_mounts_with_reader(&["/data/disk1".to_string()], mounts.as_bytes()).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_mount_paths_decodes_escaped_spaces() {
|
||||||
|
let mounts = "/dev/sdb1 /data/my\\040disk ext4 rw 0 0\n";
|
||||||
|
|
||||||
|
let paths = parse_mount_paths(mounts);
|
||||||
|
assert_eq!(paths, vec!["/data/my disk".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reject_relative_path_for_cross_device_validation() {
|
||||||
|
let err = ensure_no_sub_mounts("relative/path", &[]).unwrap_err();
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidInput);
|
||||||
|
assert!(err.to_string().contains("expected to be absolute"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reject_root_path_for_cross_device_validation() {
|
||||||
|
let err = ensure_no_sub_mounts("/", &[]).unwrap_err();
|
||||||
|
assert_eq!(err.kind(), ErrorKind::InvalidInput);
|
||||||
|
assert!(err.to_string().contains("cannot be the filesystem root"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fallback_to_major_minor_when_sysfs_link_missing() {
|
||||||
|
let major = u64::MAX;
|
||||||
|
let minor = u64::MAX;
|
||||||
|
let ids = resolve_block_device_ids(major, minor).unwrap();
|
||||||
|
assert_eq!(ids.into_iter().collect::<Vec<_>>(), vec![format!("{major}:{minor}")]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -21,13 +21,13 @@ mod unix;
|
|||||||
mod windows;
|
mod windows;
|
||||||
|
|
||||||
#[cfg(target_os = "linux")]
|
#[cfg(target_os = "linux")]
|
||||||
pub use linux::{get_drive_stats, get_info, same_disk};
|
pub use linux::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, same_disk};
|
||||||
|
|
||||||
#[cfg(all(unix, not(target_os = "linux")))]
|
#[cfg(all(unix, not(target_os = "linux")))]
|
||||||
pub use unix::{get_drive_stats, get_info, same_disk};
|
pub use unix::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, same_disk};
|
||||||
|
|
||||||
#[cfg(target_os = "windows")]
|
#[cfg(target_os = "windows")]
|
||||||
pub use windows::{get_drive_stats, get_info, same_disk};
|
pub use windows::{check_cross_device_mounts, get_drive_stats, get_info, get_physical_device_ids, same_disk};
|
||||||
|
|
||||||
#[derive(Debug, Default, PartialEq)]
|
#[derive(Debug, Default, PartialEq)]
|
||||||
pub struct IOStats {
|
pub struct IOStats {
|
||||||
|
|||||||
@@ -93,6 +93,18 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
|
|||||||
Ok(stat1.st_dev == stat2.st_dev)
|
Ok(stat1.st_dev == stat2.st_dev)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn get_physical_device_ids(disk: &str) -> std::io::Result<Vec<String>> {
|
||||||
|
let stat = rustix::fs::stat(disk)?;
|
||||||
|
let major = rustix::fs::major(stat.st_dev);
|
||||||
|
let minor = rustix::fs::minor(stat.st_dev);
|
||||||
|
|
||||||
|
Ok(vec![format!("{major}:{minor}")])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_cross_device_mounts(_paths: &[String]) -> std::io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(not(target_os = "linux"))]
|
#[cfg(not(target_os = "linux"))]
|
||||||
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
|
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
|
||||||
Ok(IOStats::default())
|
Ok(IOStats::default())
|
||||||
|
|||||||
@@ -151,6 +151,17 @@ pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result<bool> {
|
|||||||
Ok(volume1 == volume2)
|
Ok(volume1 == volume2)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn get_physical_device_ids(disk: &str) -> std::io::Result<Vec<String>> {
|
||||||
|
let path_wide = to_wide_path(Path::new(disk));
|
||||||
|
let volume = get_volume_name(&path_wide)?;
|
||||||
|
|
||||||
|
Ok(vec![String::from_utf16_lossy(&volume)])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn check_cross_device_mounts(_paths: &[String]) -> std::io::Result<()> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
|
pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
|
||||||
Ok(IOStats::default())
|
Ok(IOStats::default())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -38,6 +38,14 @@ TEST_MODE="${TEST_MODE:-single}"
|
|||||||
MAXFAIL="${MAXFAIL:-1}"
|
MAXFAIL="${MAXFAIL:-1}"
|
||||||
XDIST="${XDIST:-0}"
|
XDIST="${XDIST:-0}"
|
||||||
|
|
||||||
|
# Compatibility default for the s3-tests harness:
|
||||||
|
# this script provisions multiple local export directories on the same physical disk.
|
||||||
|
# Prefer the canonical bypass knob by default, and only honor the legacy CI alias
|
||||||
|
# when it is already provided by the environment.
|
||||||
|
if [ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ] && [ -z "${MINIO_CI+x}" ]; then
|
||||||
|
export RUSTFS_UNSAFE_BYPASS_DISK_CHECK="true"
|
||||||
|
fi
|
||||||
|
|
||||||
# Directories (define early for use in test list loading)
|
# Directories (define early for use in test list loading)
|
||||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||||
|
|||||||
Reference in New Issue
Block a user