diff --git a/Cargo.lock b/Cargo.lock index 498688f76..0e88b2ee2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9457,6 +9457,7 @@ dependencies = [ "futures", "hotpath", "http 1.5.0", + "libc", "metrics", "rustfs-common", "rustfs-concurrency", diff --git a/crates/heal/Cargo.toml b/crates/heal/Cargo.toml index 668c4af24..6e5954222 100644 --- a/crates/heal/Cargo.toml +++ b/crates/heal/Cargo.toml @@ -91,6 +91,7 @@ metrics = { workspace = true } base64 = { workspace = true } [dev-dependencies] +libc = { workspace = true } serde_json = { workspace = true, features = ["raw_value"] } rustfs-test-utils = { workspace = true } serial_test = { workspace = true } diff --git a/crates/heal/src/heal/replacement_readiness.rs b/crates/heal/src/heal/replacement_readiness.rs index f0ac9b640..e370f1849 100644 --- a/crates/heal/src/heal/replacement_readiness.rs +++ b/crates/heal/src/heal/replacement_readiness.rs @@ -157,4 +157,225 @@ mod tests { ) .await; } + + #[cfg(target_os = "linux")] + mod linux_privileged_tests { + use super::*; + use std::error::Error; + use std::ffi::CString; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; + + const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS"; + + struct MountGuard { + mounts: Vec, + } + + impl MountGuard { + fn new() -> Result> { + let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) }; + if rc != 0 { + return Err(format!("unshare(CLONE_NEWNS) failed: {}", std::io::Error::last_os_error()).into()); + } + make_mounts_private()?; + Ok(Self { mounts: Vec::new() }) + } + + fn mount_tmpfs(&mut self, target: &Path, label: &str) -> Result<(), Box> { + mount_tmpfs(target, label)?; + self.mounts.push(target.to_path_buf()); + Ok(()) + } + + fn mount_bind(&mut self, source: &Path, target: &Path) -> Result<(), Box> { + mount_bind(source, target)?; + self.mounts.push(target.to_path_buf()); + Ok(()) + } + } + + impl Drop for MountGuard { + fn drop(&mut self) { + for mount in self.mounts.iter().rev() { + if let Ok(target) = c_path(mount) { + let _ = unsafe { libc::umount2(target.as_ptr(), libc::MNT_DETACH) }; + } + } + } + } + + fn c_path(path: &Path) -> Result> { + Ok(CString::new(path.as_os_str().as_bytes())?) + } + + fn make_mounts_private() -> Result<(), Box> { + let root = CString::new("/")?; + let rc = unsafe { + libc::mount( + std::ptr::null(), + root.as_ptr(), + std::ptr::null(), + (libc::MS_REC | libc::MS_PRIVATE) as libc::c_ulong, + std::ptr::null(), + ) + }; + if rc != 0 { + return Err(format!("making the mount namespace private failed: {}", std::io::Error::last_os_error()).into()); + } + Ok(()) + } + + fn mount_tmpfs(target: &Path, label: &str) -> Result<(), Box> { + let source = CString::new(label)?; + let target = c_path(target)?; + let fstype = CString::new("tmpfs")?; + let data = CString::new("size=32m,mode=0700")?; + let rc = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + fstype.as_ptr(), + (libc::MS_NOSUID | libc::MS_NODEV) as libc::c_ulong, + data.as_ptr().cast(), + ) + }; + if rc != 0 { + return Err(format!("mount(tmpfs) failed: {}", std::io::Error::last_os_error()).into()); + } + Ok(()) + } + + fn mount_bind(source: &Path, target: &Path) -> Result<(), Box> { + let source = c_path(source)?; + let target = c_path(target)?; + let rc = unsafe { + libc::mount( + source.as_ptr(), + target.as_ptr(), + std::ptr::null(), + libc::MS_BIND as libc::c_ulong, + std::ptr::null(), + ) + }; + if rc != 0 { + return Err(format!("mount(MS_BIND) failed: {}", std::io::Error::last_os_error()).into()); + } + Ok(()) + } + + fn privileged_enabled() -> Result> { + let enabled = std::env::var(ENABLE_ENV) + .ok() + .is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")); + if !enabled { + return Ok(false); + } + if unsafe { libc::geteuid() } != 0 { + return Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN").into()); + } + Ok(true) + } + + fn run_privileged_mount_test(test: F) -> Result<(), Box> + where + F: FnOnce(MountGuard) -> Fut + Send + 'static, + Fut: std::future::Future>> + 'static, + { + if !privileged_enabled()? { + return Ok(()); + } + std::thread::spawn(move || { + let guard = MountGuard::new()?; + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + runtime.block_on(test(guard)) + }) + .join() + .map_err(|_| "privileged mount readiness test thread panicked")? + } + + #[test] + #[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"] + fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box> { + run_privileged_mount_test(|mut mounts| async move { + let temp = TempDir::new().expect("temporary replacement roots should be created"); + let target = temp.path().join("target"); + let sibling = temp.path().join("sibling"); + std::fs::create_dir(&target).expect("target mountpoint should be created"); + std::fs::create_dir(&sibling).expect("sibling mountpoint should be created"); + mounts.mount_tmpfs(&target, "rustfs-readiness-target")?; + mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?; + + let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?; + let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?; + let target_disk = new_disk( + &target_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await?; + let sibling_disk = new_disk( + &sibling_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await?; + + let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await; + assert!( + identity.is_some(), + "a separately mounted replacement target with no sibling device overlap must be admitted" + ); + Ok(()) + }) + } + + #[test] + #[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"] + fn auto_replacement_readiness_rejects_a_same_device_sibling_bind_mount() -> Result<(), Box> { + run_privileged_mount_test(|mut mounts| async move { + let temp = TempDir::new().expect("temporary replacement roots should be created"); + let source = temp.path().join("source"); + let target = temp.path().join("target"); + let sibling = temp.path().join("sibling"); + std::fs::create_dir(&source).expect("source mountpoint should be created"); + std::fs::create_dir(&target).expect("target mountpoint should be created"); + std::fs::create_dir(&sibling).expect("sibling mountpoint should be created"); + mounts.mount_tmpfs(&source, "rustfs-readiness-shared-source")?; + mounts.mount_bind(&source, &target)?; + mounts.mount_bind(&source, &sibling)?; + + let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?; + let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?; + let target_disk = new_disk( + &target_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await?; + let sibling_disk = new_disk( + &sibling_endpoint, + &DiskOption { + cleanup: false, + health_check: false, + }, + ) + .await?; + + assert!( + auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]) + .await + .is_none(), + "replacement readiness must reject a target sharing its physical device with a sibling endpoint" + ); + Ok(()) + }) + } + } }