test(heal): cover privileged mount readiness

Add Linux-only ignored replacement readiness tests for independent mount admission and same-device sibling rejection.

Co-Authored-By: heihutu <heihutu@gmail.com>

Cargo.lock

@@ -9457,6 +9457,7 @@ dependencies = [
  "futures",
  "hotpath",
  "http 1.5.0",
+ "libc",
  "metrics",
  "rustfs-common",
  "rustfs-concurrency",

crates/heal/Cargo.toml

@@ -91,6 +91,7 @@ metrics = { workspace = true }
 base64 = { workspace = true }
 
 [dev-dependencies]
+libc = { workspace = true }
 serde_json = { workspace = true, features = ["raw_value"] }
 rustfs-test-utils = { workspace = true }
 serial_test = { workspace = true }

crates/heal/src/heal/replacement_readiness.rs

@@ -157,4 +157,225 @@ mod tests {
         )
         .await;
     }
+
+    #[cfg(target_os = "linux")]
+    mod linux_privileged_tests {
+        use super::*;
+        use std::error::Error;
+        use std::ffi::CString;
+        use std::os::unix::ffi::OsStrExt;
+        use std::path::Path;
+
+        const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
+
+        struct MountGuard {
+            mounts: Vec<std::path::PathBuf>,
+        }
+
+        impl MountGuard {
+            fn new() -> Result<Self, Box<dyn Error + Send + Sync>> {
+                let rc = unsafe { libc::unshare(libc::CLONE_NEWNS) };
+                if rc != 0 {
+                    return Err(format!("unshare(CLONE_NEWNS) failed: {}", std::io::Error::last_os_error()).into());
+                }
+                make_mounts_private()?;
+                Ok(Self { mounts: Vec::new() })
+            }
+
+            fn mount_tmpfs(&mut self, target: &Path, label: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+                mount_tmpfs(target, label)?;
+                self.mounts.push(target.to_path_buf());
+                Ok(())
+            }
+
+            fn mount_bind(&mut self, source: &Path, target: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+                mount_bind(source, target)?;
+                self.mounts.push(target.to_path_buf());
+                Ok(())
+            }
+        }
+
+        impl Drop for MountGuard {
+            fn drop(&mut self) {
+                for mount in self.mounts.iter().rev() {
+                    if let Ok(target) = c_path(mount) {
+                        let _ = unsafe { libc::umount2(target.as_ptr(), libc::MNT_DETACH) };
+                    }
+                }
+            }
+        }
+
+        fn c_path(path: &Path) -> Result<CString, Box<dyn Error + Send + Sync>> {
+            Ok(CString::new(path.as_os_str().as_bytes())?)
+        }
+
+        fn make_mounts_private() -> Result<(), Box<dyn Error + Send + Sync>> {
+            let root = CString::new("/")?;
+            let rc = unsafe {
+                libc::mount(
+                    std::ptr::null(),
+                    root.as_ptr(),
+                    std::ptr::null(),
+                    (libc::MS_REC | libc::MS_PRIVATE) as libc::c_ulong,
+                    std::ptr::null(),
+                )
+            };
+            if rc != 0 {
+                return Err(format!("making the mount namespace private failed: {}", std::io::Error::last_os_error()).into());
+            }
+            Ok(())
+        }
+
+        fn mount_tmpfs(target: &Path, label: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let source = CString::new(label)?;
+            let target = c_path(target)?;
+            let fstype = CString::new("tmpfs")?;
+            let data = CString::new("size=32m,mode=0700")?;
+            let rc = unsafe {
+                libc::mount(
+                    source.as_ptr(),
+                    target.as_ptr(),
+                    fstype.as_ptr(),
+                    (libc::MS_NOSUID | libc::MS_NODEV) as libc::c_ulong,
+                    data.as_ptr().cast(),
+                )
+            };
+            if rc != 0 {
+                return Err(format!("mount(tmpfs) failed: {}", std::io::Error::last_os_error()).into());
+            }
+            Ok(())
+        }
+
+        fn mount_bind(source: &Path, target: &Path) -> Result<(), Box<dyn Error + Send + Sync>> {
+            let source = c_path(source)?;
+            let target = c_path(target)?;
+            let rc = unsafe {
+                libc::mount(
+                    source.as_ptr(),
+                    target.as_ptr(),
+                    std::ptr::null(),
+                    libc::MS_BIND as libc::c_ulong,
+                    std::ptr::null(),
+                )
+            };
+            if rc != 0 {
+                return Err(format!("mount(MS_BIND) failed: {}", std::io::Error::last_os_error()).into());
+            }
+            Ok(())
+        }
+
+        fn privileged_enabled() -> Result<bool, Box<dyn Error + Send + Sync>> {
+            let enabled = std::env::var(ENABLE_ENV)
+                .ok()
+                .is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"));
+            if !enabled {
+                return Ok(false);
+            }
+            if unsafe { libc::geteuid() } != 0 {
+                return Err(format!("{ENABLE_ENV}=1 requires root or CAP_SYS_ADMIN").into());
+            }
+            Ok(true)
+        }
+
+        fn run_privileged_mount_test<F, Fut>(test: F) -> Result<(), Box<dyn Error + Send + Sync>>
+        where
+            F: FnOnce(MountGuard) -> Fut + Send + 'static,
+            Fut: std::future::Future<Output = Result<(), Box<dyn Error + Send + Sync>>> + 'static,
+        {
+            if !privileged_enabled()? {
+                return Ok(());
+            }
+            std::thread::spawn(move || {
+                let guard = MountGuard::new()?;
+                let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?;
+                runtime.block_on(test(guard))
+            })
+            .join()
+            .map_err(|_| "privileged mount readiness test thread panicked")?
+        }
+
+        #[test]
+        #[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
+        fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
+            run_privileged_mount_test(|mut mounts| async move {
+                let temp = TempDir::new().expect("temporary replacement roots should be created");
+                let target = temp.path().join("target");
+                let sibling = temp.path().join("sibling");
+                std::fs::create_dir(&target).expect("target mountpoint should be created");
+                std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
+                mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
+                mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
+
+                let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
+                let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
+                let target_disk = new_disk(
+                    &target_endpoint,
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: false,
+                    },
+                )
+                .await?;
+                let sibling_disk = new_disk(
+                    &sibling_endpoint,
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: false,
+                    },
+                )
+                .await?;
+
+                let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
+                assert!(
+                    identity.is_some(),
+                    "a separately mounted replacement target with no sibling device overlap must be admitted"
+                );
+                Ok(())
+            })
+        }
+
+        #[test]
+        #[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
+        fn auto_replacement_readiness_rejects_a_same_device_sibling_bind_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
+            run_privileged_mount_test(|mut mounts| async move {
+                let temp = TempDir::new().expect("temporary replacement roots should be created");
+                let source = temp.path().join("source");
+                let target = temp.path().join("target");
+                let sibling = temp.path().join("sibling");
+                std::fs::create_dir(&source).expect("source mountpoint should be created");
+                std::fs::create_dir(&target).expect("target mountpoint should be created");
+                std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
+                mounts.mount_tmpfs(&source, "rustfs-readiness-shared-source")?;
+                mounts.mount_bind(&source, &target)?;
+                mounts.mount_bind(&source, &sibling)?;
+
+                let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
+                let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
+                let target_disk = new_disk(
+                    &target_endpoint,
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: false,
+                    },
+                )
+                .await?;
+                let sibling_disk = new_disk(
+                    &sibling_endpoint,
+                    &DiskOption {
+                        cleanup: false,
+                        health_check: false,
+                    },
+                )
+                .await?;
+
+                assert!(
+                    auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
+                        .await
+                        .is_none(),
+                    "replacement readiness must reject a target sharing its physical device with a sibling endpoint"
+                );
+                Ok(())
+            })
+        }
+    }
 }