// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::{DiskInfo, IOStats, fs_type::get_fs_type}; use rustix::fs::statfs; use std::collections::BTreeSet; use std::fs; use std::fs::File; use std::io::{self, BufRead, Error, ErrorKind, Read}; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use tracing::warn; static BAVAIL_GT_BFREE_WARNING_PATHS: OnceLock>> = OnceLock::new(); /// Returns total and free bytes available in a directory, e.g. `/`. pub fn get_info(p: impl AsRef) -> std::io::Result { // Use statfs on Linux to get access to f_type (filesystem magic number) let stat = statfs(p.as_ref())?; // Linux statfs: // f_bsize: Optimal transfer block size // f_blocks: Total data blocks in file system // f_frsize: Fragment size (since Linux 2.6) - unit for blocks // // If f_frsize is > 0, it is the unit for f_blocks, f_bfree, f_bavail. // Otherwise f_bsize is used. let bsize = if stat.f_frsize > 0 { stat.f_frsize as u64 } else { stat.f_bsize as u64 }; let bfree = stat.f_bfree as u64; let bavail = stat.f_bavail as u64; let blocks = stat.f_blocks as u64; let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, p.as_ref())?; let st = rustix::fs::stat(p.as_ref())?; Ok(DiskInfo { total, free, used, files: stat.f_files as u64, ffree: stat.f_ffree as u64, fstype: get_fs_type(stat.f_type as u64).to_string(), major: rustix::fs::major(st.st_dev) as u64, minor: rustix::fs::minor(st.st_dev) as u64, ..Default::default() }) } fn calculate_space_usage(blocks: u64, bfree: u64, bavail: u64, bsize: u64, path: &Path) -> std::io::Result<(u64, u64, u64)> { let available = if bfree < bavail { if should_warn_bavail_greater_than_bfree(path) { warn!( path = %path.display(), f_bfree = bfree, f_bavail = bavail, "detected f_bavail greater than f_bfree, capping available blocks to f_bfree for compatibility" ); } bfree } else { bavail }; let reserved = bfree - available; let total = match blocks.checked_sub(reserved) { Some(total) => total * bsize, None => { return Err(Error::other(format!( "detected reserved space ({reserved}) > blocks space ({blocks}), fs corruption at ({}). please run 'fsck'", path.display(), ))); } }; let free = available * bsize; let used = match total.checked_sub(free) { Some(used) => used, None => { return Err(Error::other(format!( "detected free space ({free}) > total drive space ({total}), fs corruption at ({}). please run 'fsck'", path.display(), ))); } }; Ok((total, free, used)) } fn should_warn_bavail_greater_than_bfree(path: &Path) -> bool { let warned_paths = BAVAIL_GT_BFREE_WARNING_PATHS.get_or_init(|| Mutex::new(BTreeSet::new())); let mut warned_paths = match warned_paths.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), }; if warned_paths.contains(path) { false } else { warned_paths.insert(path.to_path_buf()) } } pub fn same_disk(disk1: &str, disk2: &str) -> std::io::Result { let stat1 = rustix::fs::stat(disk1)?; let stat2 = rustix::fs::stat(disk2)?; Ok(stat1.st_dev == stat2.st_dev) } /// Resolve the leaf physical device identities backing a local filesystem path. /// /// Linux block stacks such as partitions, `dm-*`, or software RAID can all /// expose a filesystem through an intermediate device node. This helper walks /// sysfs until it reaches the leaf backing devices so the caller can compare /// physical failure domains instead of only filesystem device numbers. pub fn get_physical_device_ids(disk: &str) -> std::io::Result> { let stat = rustix::fs::stat(disk)?; let major = rustix::fs::major(stat.st_dev) as u64; let minor = rustix::fs::minor(stat.st_dev) as u64; let devices = resolve_block_device_ids(major, minor)?; Ok(devices.into_iter().collect()) } fn resolve_block_device_ids(major: u64, minor: u64) -> std::io::Result> { let sysfs_path = PathBuf::from(format!("/sys/dev/block/{major}:{minor}")); let resolved = match fs::canonicalize(&sysfs_path) { Ok(path) => path, Err(err) if err.kind() == ErrorKind::NotFound => { return Ok(BTreeSet::from([format!("{major}:{minor}")])); } Err(err) => return Err(err), }; let devices = collect_block_device_ids(&resolved)?; if devices.is_empty() { Ok(BTreeSet::from([format!("{major}:{minor}")])) } else { Ok(devices) } } fn collect_block_device_ids(device_path: &Path) -> std::io::Result> { let mut ids = BTreeSet::new(); let slaves_dir = device_path.join("slaves"); match fs::read_dir(&slaves_dir) { Ok(entries) => { let mut found_slave = false; for entry in entries { let entry = entry?; found_slave = true; let resolved = fs::canonicalize(entry.path())?; ids.extend(collect_block_device_ids(&resolved)?); } if found_slave { return Ok(ids); } } Err(err) if err.kind() == ErrorKind::NotFound => {} Err(err) => return Err(err), } ids.insert(normalize_block_device_name(device_path)); Ok(ids) } fn normalize_block_device_name(device_path: &Path) -> String { if device_path.join("partition").exists() && let Some(parent_name) = device_path.parent().and_then(|parent| parent.file_name()) { return parent_name.to_string_lossy().into_owned(); } device_path .file_name() .map(|name| name.to_string_lossy().into_owned()) .unwrap_or_else(|| device_path.display().to_string()) } /// Check whether any configured export path contains nested mount points. /// /// This mirrors the intent of MinIO's cross-device mount guardrail: once an /// export path is selected, RustFS should not silently traverse into child /// mount points hosted by other devices. pub fn check_cross_device_mounts(paths: &[String]) -> std::io::Result<()> { check_cross_device_mounts_with_reader(paths, File::open("/proc/mounts")?) } /// Parse `/proc/mounts`-style content and validate each export path against it. fn check_cross_device_mounts_with_reader(paths: &[String], mut reader: impl Read) -> std::io::Result<()> { let mut content = String::new(); reader.read_to_string(&mut content)?; let mount_paths = parse_mount_paths(&content); for path in paths { ensure_no_sub_mounts(path, &mount_paths)?; } Ok(()) } /// Extract mount paths from `/proc/mounts` content, decoding escaped spaces. fn parse_mount_paths(content: &str) -> Vec { content .lines() .filter_map(|line| { let fields = line.split_whitespace().collect::>(); if fields.len() != 6 { return None; } Some(fields[1].replace("\\040", " ")) }) .collect() } /// Validate that `path` does not contain nested child mount points. fn ensure_no_sub_mounts(path: &str, mount_paths: &[String]) -> std::io::Result<()> { if !Path::new(path).is_absolute() { return Err(Error::new( ErrorKind::InvalidInput, format!("Invalid argument, path ({path}) is expected to be absolute"), )); } if path == "/" { return Err(Error::new( ErrorKind::InvalidInput, "Invalid argument, path (/) cannot be the filesystem root for export validation", )); } let base = normalize_mount_path(path); let mut cross_mounts = Vec::new(); for mount_path in mount_paths { let mount_base = normalize_mount_path(mount_path); if mount_base.starts_with(&base) && mount_base != base { cross_mounts.push(mount_path.clone()); } } if cross_mounts.is_empty() { return Ok(()); } cross_mounts.sort(); cross_mounts.dedup(); Err(Error::other(format!( "Nested mount points detected under path ({path}) at the following locations: {}. Export path should not have any sub-mounts, refusing to start.", cross_mounts.join(", ") ))) } /// Normalize mount paths so prefix checks treat `/a/b` and `/a/b/` identically. fn normalize_mount_path(path: &str) -> String { let trimmed = path.trim_end_matches('/'); if trimmed.is_empty() { "/".to_string() } else { format!("{trimmed}/") } } pub fn get_drive_stats(major: u32, minor: u32) -> std::io::Result { read_drive_stats(&format!("/sys/dev/block/{major}:{minor}/stat")) } fn read_drive_stats(stats_file: &str) -> std::io::Result { let stats = read_stat(stats_file)?; if stats.len() < 11 { return Err(Error::new( ErrorKind::InvalidData, format!("found invalid format while reading {stats_file}"), )); } let mut io_stats = IOStats { read_ios: stats[0], read_merges: stats[1], read_sectors: stats[2], read_ticks: stats[3], write_ios: stats[4], write_merges: stats[5], write_sectors: stats[6], write_ticks: stats[7], current_ios: stats[8], total_ticks: stats[9], req_ticks: stats[10], ..Default::default() }; if stats.len() > 14 { io_stats.discard_ios = stats[11]; io_stats.discard_merges = stats[12]; io_stats.discard_sectors = stats[13]; io_stats.discard_ticks = stats[14]; } Ok(io_stats) } fn read_stat(file_name: &str) -> std::io::Result> { // Open file let path = Path::new(file_name); let file = File::open(path)?; // Create a BufReader let reader = io::BufReader::new(file); // Read first line let mut stats = Vec::new(); if let Some(line) = reader.lines().next() { let line = line?; // Split line and parse as u64 // https://rust-lang.github.io/rust-clippy/master/index.html#trim_split_whitespace for token in line.split_whitespace() { let ui64: u64 = token .parse() .map_err(|e| Error::new(ErrorKind::InvalidData, format!("failed to parse '{token}' as u64: {e}")))?; stats.push(ui64); } } Ok(stats) } #[cfg(test)] mod tests { use super::*; use tempfile::tempdir; #[test] fn normalize_partition_device_to_parent_disk() { let dir = tempdir().unwrap(); let block = dir.path().join("block"); let disk = block.join("nvme0n1"); let partition = disk.join("nvme0n1p1"); let slaves = partition.join("slaves"); fs::create_dir_all(&slaves).unwrap(); fs::write(partition.join("partition"), "1").unwrap(); let ids = collect_block_device_ids(&partition).unwrap(); assert_eq!(ids.into_iter().collect::>(), vec!["nvme0n1".to_string()]); } #[test] fn flatten_device_mapper_slaves_to_leaf_devices() { let dir = tempdir().unwrap(); let block = dir.path().join("block"); let dm = block.join("dm-0"); let dm_slaves = dm.join("slaves"); let nvme0 = block.join("nvme0n1"); let nvme1 = block.join("nvme1n1"); fs::create_dir_all(&dm_slaves).unwrap(); fs::create_dir_all(&nvme0).unwrap(); fs::create_dir_all(&nvme1).unwrap(); #[cfg(unix)] { std::os::unix::fs::symlink(&nvme0, dm_slaves.join("nvme0n1")).unwrap(); std::os::unix::fs::symlink(&nvme1, dm_slaves.join("nvme1n1")).unwrap(); } let ids = collect_block_device_ids(&dm).unwrap(); assert_eq!(ids.into_iter().collect::>(), vec!["nvme0n1".to_string(), "nvme1n1".to_string()]); } #[test] fn detect_cross_device_sub_mounts() { let mounts = "\ /dev/root / ext4 rw 0 0 /dev/sdb1 /data ext4 rw 0 0 /dev/sdc1 /data/disk1/sub ext4 rw 0 0 "; let err = check_cross_device_mounts_with_reader(&["/data/disk1".to_string()], mounts.as_bytes()).unwrap_err(); assert!(err.to_string().contains("Nested mount points detected under path")); assert!(err.to_string().contains("/data/disk1/sub")); } #[test] fn allow_mount_path_without_sub_mounts() { let mounts = "\ /dev/root / ext4 rw 0 0 /dev/sdb1 /data/disk1 ext4 rw 0 0 "; check_cross_device_mounts_with_reader(&["/data/disk1".to_string()], mounts.as_bytes()).unwrap(); } #[test] fn parse_mount_paths_decodes_escaped_spaces() { let mounts = "/dev/sdb1 /data/my\\040disk ext4 rw 0 0\n"; let paths = parse_mount_paths(mounts); assert_eq!(paths, vec!["/data/my disk".to_string()]); } #[test] fn reject_relative_path_for_cross_device_validation() { let err = ensure_no_sub_mounts("relative/path", &[]).unwrap_err(); assert_eq!(err.kind(), ErrorKind::InvalidInput); assert!(err.to_string().contains("expected to be absolute")); } #[test] fn reject_root_path_for_cross_device_validation() { let err = ensure_no_sub_mounts("/", &[]).unwrap_err(); assert_eq!(err.kind(), ErrorKind::InvalidInput); assert!(err.to_string().contains("cannot be the filesystem root")); } #[test] fn fallback_to_major_minor_when_sysfs_link_missing() { let major = u64::MAX; let minor = u64::MAX; let ids = resolve_block_device_ids(major, minor).unwrap(); assert_eq!(ids.into_iter().collect::>(), vec![format!("{major}:{minor}")]); } #[test] fn calculate_space_usage_normal_bfree_greater_than_bavail() { // Typical ext4/xfs scenario: bavail < bfree due to reserved blocks let blocks = 1_000_u64; let bfree = 900_u64; let bavail = 850_u64; let bsize = 4_096_u64; let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap(); let reserved = bfree - bavail; assert_eq!(total, (blocks - reserved) * bsize); assert_eq!(free, bavail * bsize); assert_eq!(used, total - free); } #[test] fn calculate_space_usage_bfree_equals_bavail() { // No reserved blocks: bfree == bavail (e.g. FAT/exFAT or root user) let blocks = 500_u64; let bfree = 400_u64; let bavail = 400_u64; let bsize = 4_096_u64; let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap(); assert_eq!(total, blocks * bsize); assert_eq!(free, bfree * bsize); assert_eq!(used, (blocks - bfree) * bsize); } #[test] fn calculate_space_usage_all_zero_blocks() { let (total, free, used) = calculate_space_usage(0, 0, 0, 4_096, Path::new("/data")).unwrap(); assert_eq!(total, 0); assert_eq!(free, 0); assert_eq!(used, 0); } #[test] fn calculate_space_usage_all_blocks_free() { let blocks = 1_000_u64; let bfree = 1_000_u64; let bavail = 1_000_u64; let bsize = 4_096_u64; let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap(); assert_eq!(total, blocks * bsize); assert_eq!(free, blocks * bsize); assert_eq!(used, 0); } #[test] fn calculate_space_usage_allows_bavail_greater_than_bfree() { let blocks = 1_000_u64; let bfree = 900_u64; let bavail = 920_u64; let bsize = 4_096_u64; let (total, free, used) = calculate_space_usage(blocks, bfree, bavail, bsize, Path::new("/data")).unwrap(); assert_eq!(total, blocks * bsize); assert_eq!(free, bfree * bsize); assert_eq!(used, total - free); } #[test] fn calculate_space_usage_rejects_free_greater_than_total() { let err = calculate_space_usage(100, 120, 120, 4_096, Path::new("/data")).unwrap_err(); assert!(err.to_string().contains("detected free space")); } #[test] fn bavail_greater_than_bfree_warning_is_once_per_path() { let path = Path::new("/data/rustfs-bavail-warning-once"); assert!(should_warn_bavail_greater_than_bfree(path)); assert!(!should_warn_bavail_greater_than_bfree(path)); assert!(should_warn_bavail_greater_than_bfree(Path::new("/data/rustfs-bavail-warning-other"))); } }