gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-20 19:42:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: unify path handling to use S3-standard forward slashes on all platforms (#1555)

Browse Source 
Signed-off-by: houseme <housemecn@gmail.com>
Signed-off-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: heihutu <30542132+heihutu@users.noreply.github.com>
Co-authored-by: LeonWang0735 <wlywly0735@126.com>
Co-authored-by: loverustfs <hello@rustfs.com>
Co-authored-by: 安正超 <anzhengchao@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
houseme
2026-01-26 18:49:21 +08:00
committed by GitHub
parent 172bed0ff2
commit d251b9fb35
22 changed files with 1011 additions and 853 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/Cargo.toml
-1
View File
@@ -48,7 +48,6 @@ async-trait.workspace = true
bytes.workspace = true
byteorder = { workspace = true }
chrono.workspace = true
dunce.workspace = true
glob = { workspace = true }
thiserror.workspace = true
flatbuffers.workspace = true
crates/ecstore/src/bucket/utils.rs
+3 -3
View File
@@ -15,7 +15,7 @@
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result, StorageError};
use regex::Regex;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use rustfs_utils::path::SLASH_SEPARATOR;
use s3s::xml;
use tracing::instrument;
@@ -193,7 +193,7 @@ pub fn is_valid_object_name(object: &str) -> bool {
return false;
}
if object.ends_with(SLASH_SEPARATOR_STR) {
if object.ends_with(SLASH_SEPARATOR) {
return false;
}
@@ -205,7 +205,7 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned()));
}
if object.starts_with(SLASH_SEPARATOR_STR) {
if object.starts_with(SLASH_SEPARATOR) {
return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned()));
}
crates/ecstore/src/config/com.rs
+3 -3
View File
@@ -18,7 +18,7 @@ use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap;
use rustfs_config::DEFAULT_DELIMITER;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use rustfs_utils::path::SLASH_SEPARATOR;
use std::collections::HashSet;
use std::sync::Arc;
use std::sync::LazyLock;
@@ -29,7 +29,7 @@ const CONFIG_FILE: &str = "config.json";
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{CONFIG_PREFIX}"));
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
let mut h = HashSet::new();
@@ -129,7 +129,7 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
}
fn get_config_file() -> String {
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR_STR}{CONFIG_FILE}")
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
}
/// Handle the situation where the configuration file does not exist, create and save a new configuration
crates/ecstore/src/data_usage.rs
+5 -5
View File
@@ -26,7 +26,7 @@ pub use local_snapshot::{
use rustfs_common::data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
};
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
collections::{HashMap, hash_map::Entry},
sync::{Arc, OnceLock},
@@ -37,7 +37,7 @@ use tokio::sync::RwLock;
use tracing::{debug, error, info, warn};
// Data usage storage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR;
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
@@ -61,17 +61,17 @@ fn cache_updating() -> &'static CacheUpdating {
lazy_static::lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
crate::disk::RUSTFS_META_BUCKET,
SLASH_SEPARATOR_STR,
SLASH_SEPARATOR,
crate::disk::BUCKET_META_PREFIX
);
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR_STR,
SLASH_SEPARATOR,
DATA_USAGE_OBJ_NAME
);
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR_STR,
SLASH_SEPARATOR,
DATA_USAGE_BLOOM_NAME
);
}
crates/ecstore/src/disk/local.rs
+60 -28
View File
@@ -40,8 +40,8 @@ use rustfs_filemeta::{
use rustfs_utils::HashAlgorithm;
use rustfs_utils::os::get_info;
use rustfs_utils::path::{
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR_STR, clean, decode_dir_object, encode_dir_object,
has_suffix, path_join, path_join_buf,
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
path_join, path_join_buf,
};
use std::collections::HashMap;
use std::collections::HashSet;
@@ -122,7 +122,7 @@ impl LocalDisk {
debug!("Creating local disk");
// Use optimized path resolution instead of absolutize() for better performance
// Use dunce::canonicalize instead of std::fs::canonicalize to avoid UNC paths on Windows
let root = match dunce::canonicalize(ep.get_file_path()) {
let root = match rustfs_utils::canonicalize(ep.get_file_path()) {
Ok(path) => path,
Err(e) => {
if e.kind() == ErrorKind::NotFound {
@@ -269,8 +269,22 @@ impl LocalDisk {
}
async fn cleanup_deleted_objects(root: PathBuf) -> Result<()> {
let trash = path_join(&[root, RUSTFS_META_TMP_DELETED_BUCKET.into()]);
let mut entries = fs::read_dir(&trash).await?;
#[cfg(windows)]
let trash_path = RUSTFS_META_TMP_DELETED_BUCKET.replace('/', "\\");
#[cfg(not(windows))]
let trash_path = RUSTFS_META_TMP_DELETED_BUCKET.to_string();
let trash = root.join(trash_path);
let mut entries = match fs::read_dir(&trash).await {
Ok(entries) => entries,
Err(e) => {
if e.kind() == ErrorKind::NotFound {
return Ok(());
}
return Err(e.into());
}
};
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name().to_string_lossy().to_string();
if name.is_empty() || name == "." || name == ".." {
@@ -279,7 +293,7 @@ impl LocalDisk {
let file_type = entry.file_type().await?;
let path = path_join(&[trash.clone(), name.into()]);
let path = trash.join(name);
if file_type.is_dir() {
if let Err(e) = tokio::fs::remove_dir_all(path).await
@@ -362,7 +376,14 @@ impl LocalDisk {
let abs_path = if path_ref.is_absolute() {
path_ref.to_path_buf()
} else {
self.root.join(path_ref)
#[cfg(windows)]
{
self.root.join(path_str.replace('/', "\\"))
}
#[cfg(not(windows))]
{
self.root.join(path_ref)
}
};
// Normalize path components to avoid filesystem calls
@@ -396,14 +417,22 @@ impl LocalDisk {
path_join_buf(&[bucket, key])
};
#[cfg(windows)]
let path = self.root.join(cache_key.replace('/', "\\"));
#[cfg(not(windows))]
let path = self.root.join(cache_key);
self.check_valid_path(&path)?;
Ok(path)
}
// Get the absolute path of a bucket
pub fn get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
#[cfg(windows)]
let bucket_path = self.root.join(bucket.replace('/', "\\"));
#[cfg(not(windows))]
let bucket_path = self.root.join(bucket);
self.check_valid_path(&bucket_path)?;
Ok(bucket_path)
}
@@ -440,7 +469,11 @@ impl LocalDisk {
if !cache_misses.is_empty() {
let mut new_entries = Vec::new();
for (i, _bucket, _key, cache_key) in cache_misses {
#[cfg(windows)]
let path = self.root.join(cache_key.replace('/', "\\"));
#[cfg(not(windows))]
let path = self.root.join(&cache_key);
results.push((i, path.clone()));
new_entries.push((cache_key, path));
}
@@ -555,7 +588,7 @@ impl LocalDisk {
.err()
};
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR_STR) {
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) {
let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
let _ = rename_all(
encode_dir_object(delete_path.to_string_lossy().as_ref()),
@@ -1022,16 +1055,15 @@ impl LocalDisk {
continue;
}
if entry.ends_with(SLASH_SEPARATOR_STR) {
if entry.ends_with(SLASH_SEPARATOR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
let entry =
format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR_STR);
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
dir_objes.insert(entry.clone());
*item = entry;
continue;
}
*item = entry.trim_end_matches(SLASH_SEPARATOR_STR).to_owned();
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned();
continue;
}
@@ -1043,7 +1075,7 @@ impl LocalDisk {
.await?;
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
let name = entry.trim_end_matches(SLASH_SEPARATOR_STR);
let name = entry.trim_end_matches(SLASH_SEPARATOR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
// if opts.limit > 0
@@ -1126,7 +1158,7 @@ impl LocalDisk {
Ok(res) => {
if is_dir_obj {
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
meta.name.push_str(SLASH_SEPARATOR_STR);
meta.name.push_str(SLASH_SEPARATOR);
}
meta.metadata = res.to_vec();
@@ -1144,7 +1176,7 @@ impl LocalDisk {
// NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
meta.name.push_str(SLASH_SEPARATOR_STR);
meta.name.push_str(SLASH_SEPARATOR);
dir_stack.push(meta.name);
}
}
@@ -1625,8 +1657,8 @@ impl DiskAPI for LocalDisk {
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
}
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
warn!(
@@ -1692,8 +1724,8 @@ impl DiskAPI for LocalDisk {
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
}
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) {
return Err(Error::from(DiskError::FileAccessDenied));
}
@@ -1844,7 +1876,7 @@ impl DiskAPI for LocalDisk {
}
let volume_dir = self.get_bucket_path(volume)?;
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR_STR))?;
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?;
let entries = match os::read_dir(&dir_path_abs, count).await {
Ok(res) => res,
@@ -1880,12 +1912,12 @@ impl DiskAPI for LocalDisk {
let mut objs_returned = 0;
if opts.base_dir.ends_with(SLASH_SEPARATOR_STR) {
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
if let Ok(data) = self
.read_metadata(
&opts.bucket,
path_join_buf(&[
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX).as_str(),
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(),
STORAGE_FORMAT_FILE,
])
.as_str(),
@@ -2135,7 +2167,7 @@ impl DiskAPI for LocalDisk {
let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?;
for entry in entries {
if !has_suffix(&entry, SLASH_SEPARATOR_STR) || !Self::is_valid_volname(clean(&entry).as_str()) {
if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) {
continue;
}
@@ -2357,7 +2389,7 @@ impl DiskAPI for LocalDisk {
force_del_marker: bool,
opts: DeleteOptions,
) -> Result<()> {
if path.starts_with(SLASH_SEPARATOR_STR) {
if path.starts_with(SLASH_SEPARATOR) {
return self
.delete(
volume,
@@ -2418,7 +2450,7 @@ impl DiskAPI for LocalDisk {
if !meta.versions.is_empty() {
let buf = meta.marshal_msg()?;
return self
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
.await;
}
@@ -2428,11 +2460,11 @@ impl DiskAPI for LocalDisk {
{
let src_path = path_join(&[
file_path.as_path(),
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
]);
let dst_path = path_join(&[
file_path.as_path(),
Path::new(format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str()),
Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()),
]);
return rename_all(src_path, dst_path, file_path).await;
}
@@ -2584,7 +2616,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
if root_disk_threshold > 0 {
disk_info.total <= root_disk_threshold
} else {
is_root_disk(&drive_path, SLASH_SEPARATOR_STR).unwrap_or_default()
is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default()
}
} else {
false
crates/ecstore/src/disk/os.rs
+2 -2
View File
@@ -15,7 +15,7 @@
use crate::disk::error::DiskError;
use crate::disk::error::Result;
use crate::disk::error_conv::to_file_error;
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use rustfs_utils::path::SLASH_SEPARATOR;
use std::{
io,
path::{Component, Path},
@@ -116,7 +116,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
if file_type.is_file() {
volumes.push(name);
} else if file_type.is_dir() {
volumes.push(format!("{name}{SLASH_SEPARATOR_STR}"));
volumes.push(format!("{name}{SLASH_SEPARATOR}"));
}
count -= 1;
if count == 0 {
crates/ecstore/src/pools.rs
+3 -3
View File
@@ -38,7 +38,7 @@ use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{HashReader, WarpReader};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, encode_dir_object, path_join};
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
use rustfs_workers::workers::Workers;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -451,10 +451,10 @@ pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (Strin
let trimmed_path = path
.strip_prefix(base_path)
.unwrap_or(path)
.strip_prefix(SLASH_SEPARATOR_STR)
.strip_prefix(SLASH_SEPARATOR)
.unwrap_or(path);
// Find the position of the first '/'
let Some(pos) = trimmed_path.find(SLASH_SEPARATOR_STR) else {
let Some(pos) = trimmed_path.find(SLASH_SEPARATOR) else {
return (trimmed_path.to_string(), "".to_string());
};
// Split into bucket and prefix
crates/ecstore/src/set_disk.rs
+14 -4
View File
@@ -83,7 +83,7 @@ use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX,
use rustfs_utils::{
HashAlgorithm,
crypto::hex,
path::{SLASH_SEPARATOR_STR, encode_dir_object, has_suffix, path_join_buf},
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
};
use rustfs_workers::workers::Workers;
use s3s::header::X_AMZ_RESTORE;
@@ -5232,7 +5232,7 @@ impl StorageAPI for SetDisks {
&upload_id_path,
fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(),
]),
SLASH_SEPARATOR_STR
SLASH_SEPARATOR
);
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
@@ -5370,7 +5370,7 @@ impl StorageAPI for SetDisks {
let mut populated_upload_ids = HashSet::new();
for upload_id in upload_ids.iter() {
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR_STR).to_string();
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR).to_string();
if populated_upload_ids.contains(&upload_id) {
continue;
}
@@ -6125,7 +6125,7 @@ impl StorageAPI for SetDisks {
None
};
if has_suffix(object, SLASH_SEPARATOR_STR) {
if has_suffix(object, SLASH_SEPARATOR) {
let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?;
return Ok((result, err.map(|e| e.into())));
}
@@ -6672,6 +6672,16 @@ async fn get_disks_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> Vec<ru
ret
}
async fn get_storage_info(disks: &[Option<DiskStore>], eps: &[Endpoint]) -> rustfs_madmin::StorageInfo {
// let mut disks = get_disks_info(disks, eps).await;
// disks.sort_by(|a, b| a.total_space.cmp(&b.total_space));
//
// rustfs_madmin::StorageInfo {
// disks,
// backend: rustfs_madmin::BackendInfo {
// backend_type: rustfs_madmin::BackendByte::Erasure,
// ..Default::default()
// },
// }
let mut disks = get_disks_info(disks, eps).await;
disks.sort_by(|a, b| a.total_space.cmp(&b.total_space));
crates/ecstore/src/store_list_objects.rs
+10 -10
View File
@@ -34,7 +34,7 @@ use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions,
};
use rustfs_utils::path::{self, SLASH_SEPARATOR_STR, base_dir_from_prefix};
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::broadcast::{self};
@@ -132,7 +132,7 @@ impl ListPathOptions {
return;
}
let s = SLASH_SEPARATOR_STR.chars().next().unwrap_or_default();
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default();
self.filter_prefix = {
let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s);
@@ -350,7 +350,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() {
let mut found = false;
if delimiter != SLASH_SEPARATOR_STR {
if delimiter != SLASH_SEPARATOR {
for p in prefixes.iter() {
if found {
break;
@@ -477,7 +477,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() {
let mut found = false;
if delimiter != SLASH_SEPARATOR_STR {
if delimiter != SLASH_SEPARATOR {
for p in prefixes.iter() {
if found {
break;
@@ -509,7 +509,7 @@ impl ECStore {
// warn!("list_path opt {:?}", &o);
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
// if opts.prefix.ends_with(SLASH_SEPARATOR_STR) {
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
// return Err(Error::msg("eof"));
// }
@@ -527,11 +527,11 @@ impl ECStore {
return Err(Error::Unexpected);
}
if o.prefix.starts_with(SLASH_SEPARATOR_STR) {
if o.prefix.starts_with(SLASH_SEPARATOR) {
return Err(Error::Unexpected);
}
let slash_separator = Some(SLASH_SEPARATOR_STR.to_owned());
let slash_separator = Some(SLASH_SEPARATOR.to_owned());
o.include_directories = o.separator == slash_separator;
@@ -781,8 +781,8 @@ impl ECStore {
let mut filter_prefix = {
prefix
.trim_start_matches(&path)
.trim_start_matches(SLASH_SEPARATOR_STR)
.trim_end_matches(SLASH_SEPARATOR_STR)
.trim_start_matches(SLASH_SEPARATOR)
.trim_end_matches(SLASH_SEPARATOR)
.to_owned()
};
@@ -1137,7 +1137,7 @@ async fn merge_entry_channels(
if path::clean(&best_entry.name) == path::clean(&other_entry.name) {
let dir_matches = best_entry.is_dir() && other_entry.is_dir();
let suffix_matches =
best_entry.name.ends_with(SLASH_SEPARATOR_STR) == other_entry.name.ends_with(SLASH_SEPARATOR_STR);
best_entry.name.ends_with(SLASH_SEPARATOR) == other_entry.name.ends_with(SLASH_SEPARATOR);
if dir_matches && suffix_matches {
to_merge.push(other_idx);
crates/ecstore/src/tier/tier.rs
+3 -3
View File
@@ -51,7 +51,7 @@ use crate::{
store_api::{ObjectOptions, PutObjReader},
};
use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
use s3s::S3ErrorCode;
use super::{
@@ -403,7 +403,7 @@ impl TierConfigMgr {
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await
}
@@ -483,7 +483,7 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data {
if is_err_config_not_found(&err) {
crates/ecstore/src/tier/warm_backend_s3.rs
+2 -2
View File
@@ -34,7 +34,7 @@ use crate::tier::{
tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts},
};
use rustfs_utils::path::SLASH_SEPARATOR_STR;
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>,
@@ -176,7 +176,7 @@ impl WarmBackend for WarmBackendS3 {
async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self
.core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR_STR, 1)
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
.await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
crates/iam/src/store/object.rs
+2 -2
View File
@@ -32,7 +32,7 @@ use rustfs_ecstore::{
store_api::{ObjectInfo, ObjectOptions},
};
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use serde::{Serialize, de::DeserializeOwned};
use std::sync::LazyLock;
use std::{collections::HashMap, sync::Arc};
@@ -188,7 +188,7 @@ impl ObjectStore {
} else {
info.name
};
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR_STR);
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR);
let _ = sender
.send(StringOrErr {
item: Some(name.to_owned()),
crates/scanner/src/data_usage_define.rs
+6 -6
View File
@@ -32,12 +32,12 @@ use rustfs_ecstore::{
error::{Error, Result as StorageResult, StorageError},
store_api::{ObjectInfo, ObjectOptions},
};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use tokio::time::{Duration, sleep, timeout};
use tracing::{error, warn};
// Data usage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR;
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
@@ -47,16 +47,16 @@ pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
// Data usage paths (computed at runtime)
pub static DATA_USAGE_BUCKET: LazyLock<String> =
LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{BUCKET_META_PREFIX}"));
LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{BUCKET_META_PREFIX}"));
pub static DATA_USAGE_OBJ_NAME_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR_STR}{DATA_USAGE_OBJ_NAME}"));
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJ_NAME}"));
pub static DATA_USAGE_BLOOM_NAME_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR_STR}{DATA_USAGE_BLOOM_NAME}"));
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_BLOOM_NAME}"));
pub static BACKGROUND_HEAL_INFO_PATH: LazyLock<String> =
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR_STR}.background-heal.json"));
LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}.background-heal.json"));
#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize, PartialEq)]
pub struct TierStats {
crates/scanner/src/scanner_folder.rs
+2 -2
View File
@@ -44,7 +44,7 @@ use rustfs_ecstore::pools::{path2_bucket_object, path2_bucket_object_with_base_p
use rustfs_ecstore::store_api::{ObjectInfo, ObjectToDelete};
use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams, ReplicationStatusType};
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration};
use tokio::select;
use tokio::sync::mpsc;
@@ -110,7 +110,7 @@ impl ScannerItem {
/// This converts a directory path like "bucket/dir1/dir2/file" to prefix="bucket/dir1/dir2" and object_name="file"
pub fn transform_meta_dir(&mut self) {
let prefix = self.prefix.clone(); // Clone to avoid borrow checker issues
let split: Vec<&str> = prefix.split(SLASH_SEPARATOR_STR).collect();
let split: Vec<&str> = prefix.split(SLASH_SEPARATOR).collect();
if split.len() > 1 {
let prefix_parts: Vec<&str> = split[..split.len() - 1].to_vec();
crates/scanner/src/scanner_io.rs
+2 -2
View File
@@ -22,7 +22,7 @@ use rustfs_ecstore::set_disk::SetDisks;
use rustfs_ecstore::store_api::{BucketInfo, BucketOptions, ObjectInfo};
use rustfs_ecstore::{StorageAPI, error::Result, store::ECStore};
use rustfs_filemeta::FileMeta;
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use s3s::dto::{BucketLifecycleConfiguration, ReplicationConfiguration};
use std::collections::HashMap;
use std::time::SystemTime;
@@ -469,7 +469,7 @@ impl ScannerIOCache for SetDisks {
#[async_trait::async_trait]
impl ScannerIODisk for Disk {
async fn get_size(&self, mut item: ScannerItem) -> Result<SizeSummary> {
if !item.path.ends_with(&format!("{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}")) {
if !item.path.ends_with(&format!("{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}")) {
return Err(StorageError::other("skip file".to_string()));
}
crates/utils/src/dunce.rs
+415
View File
@@ -0,0 +1,415 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Utility functions to handle Windows paths, specifically converting UNC paths to regular paths where possible.
//! This is useful for compatibility with tools that do not support UNC paths (starting with `\\?\`).
#[cfg(any(windows, test))]
use std::ffi::OsStr;
#[cfg(windows)]
use std::path::{Component, Prefix};
use std::{
fs, io,
path::{Path, PathBuf},
};
/// Takes any path, and when possible, converts Windows UNC paths to regular paths.
/// If the path can't be converted, it's returned unmodified.
///
/// On non-Windows this is a no-op.
///
/// `\\?\C:\Windows` will be converted to `C:\Windows`,
/// but `\\?\C:\COM` will be left as-is (due to a reserved filename).
///
/// Use this to pass arbitrary paths to programs that may not be UNC-aware.
///
/// It's generally safe to pass UNC paths to legacy programs, because
/// these paths contain a reserved prefix, so will gracefully fail
/// if used with legacy APIs that don't support UNC.
///
/// This function does not perform any I/O.
///
/// Currently paths with unpaired surrogates aren't converted even if they
/// could be, due to limitations of Rust's `OsStr` API.
///
/// To check if a path remained as UNC, use [`is_simplified()`] or `path.as_os_str().as_encoded_bytes().starts_with(b"\\\\")`.
///
/// # Arguments
///
/// * `path` - The path to simplify.
///
/// # Returns
///
/// A reference to the simplified path if it could be simplified, or the original path otherwise.
///
/// # Examples
///
/// ```
/// use std::path::Path;
/// use rustfs_utils::simplified;
///
/// #[cfg(windows)]
/// {
/// let unc_path = Path::new(r"\\?\C:\Windows");
/// let simple_path = simplified(unc_path);
/// assert_eq!(simple_path, Path::new(r"C:\Windows"));
/// }
/// ```
#[inline]
#[must_use]
pub fn simplified(path: &Path) -> &Path {
try_simplified(path).unwrap_or(path)
}
/// Like `std::fs::canonicalize()`, but on Windows it outputs the most
/// compatible form of a path instead of UNC.
///
/// This function first canonicalizes the path (resolving symlinks and absolute path),
/// and then attempts to simplify it using [`simplified()`] on Windows.
///
/// # Arguments
///
/// * `path` - The path to canonicalize.
///
/// # Returns
///
/// The canonicalized and simplified path, or an error if canonicalization fails.
#[inline(always)]
pub fn canonicalize<P: AsRef<Path>>(path: P) -> io::Result<PathBuf> {
let path = path.as_ref();
#[cfg(not(windows))]
{
fs::canonicalize(path)
}
#[cfg(windows)]
{
canonicalize_win(path)
}
}
#[cfg(windows)]
fn canonicalize_win(path: &Path) -> io::Result<PathBuf> {
let real_path = fs::canonicalize(path)?;
Ok(try_simplified(&real_path).map(PathBuf::from).unwrap_or(real_path))
}
/// Returns `true` if the path is relative or starts with a disk prefix (like "C:").
///
/// This checks if the path is in a "simplified" format. On Windows, this means it
/// is either a relative path, a rooted path without a prefix, or starts with a drive letter (e.g., `C:\`).
/// UNC paths (starting with `\\`) and verbatim paths (starting with `\\?\`) will return `false`.
///
/// On non-Windows platforms, this always returns `true`.
///
/// # Arguments
///
/// * `path` - The path to check.
///
/// # Returns
///
/// `true` if the path is simplified, `false` otherwise.
#[cfg_attr(not(windows), allow(unused))]
#[must_use]
pub fn is_simplified(path: &Path) -> bool {
#[cfg(windows)]
if let Some(Component::Prefix(prefix)) = path.components().next() {
return matches!(prefix.kind(), Prefix::Disk(..));
}
true
}
#[doc(hidden)]
pub use self::canonicalize as realpath;
/// Checks if the filename is valid for a regular Windows path.
///
/// This checks for:
/// - Length limit (255 chars)
/// - Invalid characters (< > : " / \ | ? * and control chars)
/// - Trailing spaces or dots
#[cfg(any(windows, test))]
fn is_valid_filename(file_name: &OsStr) -> bool {
// Convert to str first. If not valid UTF-8, it's not a valid "simplified" filename candidate
// (since we require the whole path to be UTF-8 at the end).
let s = match file_name.to_str() {
Some(s) => s,
None => return false,
};
// Windows filename limit is 255 characters (UTF-16 code units)
// We check byte length first as a fast path optimization.
if s.len() > 255 && s.encode_utf16().count() > 255 {
return false;
}
if s.is_empty() {
return false;
}
// Only ASCII subset is checked, and WTF-8/UTF-8 is safe for that
// Invalid chars: < > : " / \ | ? * and control chars (0-31)
let byte_str = s.as_bytes();
if byte_str
.iter()
.any(|&c| matches!(c, 0..=31 | b'<' | b'>' | b':' | b'"' | b'/' | b'\\' | b'|' | b'?' | b'*'))
{
return false;
}
// Filename can't end with . or space
if matches!(byte_str.last(), Some(b' ' | b'.')) {
return false;
}
true
}
/// Checks if the filename is a reserved DOS device name (e.g., CON, PRN, COM1).
///
/// Reserved names are case-insensitive and apply even if an extension is present (e.g., "CON.txt" is reserved).
#[cfg(any(windows, test))]
fn is_reserved<P: AsRef<OsStr>>(file_name: P) -> bool {
let s = match file_name.as_ref().to_str() {
Some(s) => s,
None => return false, // Reserved names are all ASCII
};
if s.is_empty() {
return false;
}
// Get the stem (part before the first dot)
// "CON.txt" -> "CON"
// "CON" -> "CON"
let stem = match s.find('.') {
Some(idx) => &s[..idx],
None => s,
};
// Trim trailing spaces (Windows ignores them, so "CON " is "CON")
let stem = stem.trim_end();
// Reserved names are 3 or 4 chars
if stem.len() != 3 && stem.len() != 4 {
return false;
}
if stem.len() == 3 {
return stem.eq_ignore_ascii_case("CON")
|| stem.eq_ignore_ascii_case("PRN")
|| stem.eq_ignore_ascii_case("AUX")
|| stem.eq_ignore_ascii_case("NUL");
}
// Check COM1-9, LPT1-9
let (base, suffix) = stem.split_at(3);
if base.eq_ignore_ascii_case("COM") || base.eq_ignore_ascii_case("LPT") {
return suffix.chars().next().map(|c| c.is_ascii_digit() && c != '0').unwrap_or(false);
}
false
}
#[cfg(not(windows))]
#[inline]
const fn try_simplified(_path: &Path) -> Option<&Path> {
None
}
#[cfg(windows)]
fn try_simplified(path: &Path) -> Option<&Path> {
let mut components = path.components();
// Check if it starts with \\?\
match components.next() {
Some(Component::Prefix(p)) => match p.kind() {
Prefix::VerbatimDisk(..) => {}
_ => return None, // Other kinds of UNC paths or regular paths
},
_ => return None, // relative or empty
}
// Validate all subsequent components
for component in components {
match component {
Component::RootDir => {}
Component::Normal(file_name) => {
if !is_valid_filename(file_name) || is_reserved(file_name) {
return None;
}
}
_ => return None, // UNC paths take things like ".." literally, which we can't simplify to
};
}
// We use to_str() here because we need to slice the string to remove the prefix.
// This implicitly requires the path to be valid UTF-8.
// If it's not, we return None, which is safe (we just don't simplify).
let s = path.to_str()?;
// Remove "\\?\" (4 chars)
let simplified = &s[4..];
// Check length limit (MAX_PATH is 260)
if simplified.len() > 260 && simplified.encode_utf16().count() > 260 {
return None;
}
Some(Path::new(simplified))
}
#[cfg(test)]
mod tests {
use super::*;
use std::ffi::OsStr;
use std::path::Path;
#[test]
fn test_reserved_names() {
let reserved = [
"CON",
"PRN",
"AUX",
"NUL",
"COM1",
"COM2",
"COM3",
"COM4",
"COM5",
"COM6",
"COM7",
"COM8",
"COM9",
"LPT1",
"LPT2",
"LPT3",
"LPT4",
"LPT5",
"LPT6",
"LPT7",
"LPT8",
"LPT9",
"con",
"Con",
"CON.txt",
"con.txt",
"CON .txt",
"con .txt",
"CON.tar.gz",
"con.",
"con .",
];
for name in reserved {
assert!(is_reserved(name), "Should be reserved: {}", name);
}
let not_reserved = [
"CON0", "COM0", "LPT0", "COM10", "LPT10", "CONsole", "PrNter", "NuLly", "not_con", "con_not", ".CON", "@CON",
"CON。", "foo.con",
];
for name in not_reserved {
assert!(!is_reserved(name), "Should NOT be reserved: {}", name);
}
}
#[test]
fn test_valid_filename() {
let invalid = [
"",
".",
"..",
" ",
" .",
"foo ",
"foo.",
"foo/bar",
"foo\\bar",
"foo:bar",
"foo*bar",
"foo?bar",
"foo\"bar",
"foo<bar",
"foo>bar",
"foo|bar",
"foo\0bar",
"foo\x1fbar",
];
for name in invalid {
assert!(!is_valid_filename(OsStr::new(name)), "Should be invalid: {:?}", name);
}
let valid = [
"foo",
"foo.bar",
".foo",
"foo.bar.baz",
"foo bar",
"foo. bar", // space inside is ok
"中文",
"😀",
];
for name in valid {
assert!(is_valid_filename(OsStr::new(name)), "Should be valid: {:?}", name);
}
// Test length limit
let long_name = "a".repeat(256);
assert!(!is_valid_filename(OsStr::new(&long_name)));
let ok_name = "a".repeat(255);
assert!(is_valid_filename(OsStr::new(&ok_name)));
}
#[test]
#[cfg(windows)]
fn test_simplified_windows() {
// Basic
assert_eq!(simplified(Path::new(r"\\?\C:\Windows")), Path::new(r"C:\Windows"));
// Reserved
assert_eq!(simplified(Path::new(r"\\?\C:\CON")), Path::new(r"\\?\C:\CON"));
assert_eq!(simplified(Path::new(r"\\?\C:\aux.txt")), Path::new(r"\\?\C:\aux.txt"));
// Long paths
let long_path = format!(r"\\?\C:\{}", "a".repeat(300));
assert_eq!(simplified(Path::new(&long_path)), Path::new(&long_path));
let ok_path = format!(r"\\?\C:\{}", "a".repeat(200));
assert_eq!(simplified(Path::new(&ok_path)), Path::new(&ok_path[4..]));
// Relative/other prefixes
assert_eq!(simplified(Path::new(r"C:\Windows")), Path::new(r"C:\Windows"));
assert_eq!(simplified(Path::new(r"\\server\share")), Path::new(r"\\server\share"));
assert_eq!(simplified(Path::new(r"\\.\C:\Windows")), Path::new(r"\\.\C:\Windows"));
}
#[test]
#[cfg(not(windows))]
fn test_simplified_unix() {
let p = Path::new("/tmp/foo");
assert_eq!(simplified(p), p);
}
#[test]
fn test_is_simplified() {
#[cfg(windows)]
{
assert!(is_simplified(Path::new(r"C:\Windows")));
assert!(is_simplified(Path::new(r"relative\path")));
assert!(!is_simplified(Path::new(r"\\?\C:\Windows")));
assert!(!is_simplified(Path::new(r"\\server\share")));
}
#[cfg(not(windows))]
{
assert!(is_simplified(Path::new("/tmp/foo")));
assert!(is_simplified(Path::new("relative/path")));
}
}
}
crates/utils/src/lib.rs
+5
View File
@@ -85,5 +85,10 @@ pub use notify::*;
#[cfg(feature = "obj")]
pub mod obj;
#[cfg(feature = "path")]
mod dunce;
#[cfg(feature = "path")]
pub use dunce::*;
mod envs;
pub use envs::*;
crates/utils/src/os/windows.rs
+1 -3
View File
@@ -44,7 +44,7 @@ pub fn get_info(p: impl AsRef<Path>) -> std::io::Result<DiskInfo> {
}
let total = total_number_of_bytes;
let free = total_number_of_free_bytes;
let free = free_bytes_available;
if free > total {
return Err(Error::other(format!(
@@ -180,8 +180,6 @@ pub fn get_drive_stats(_major: u32, _minor: u32) -> std::io::Result<IOStats> {
#[cfg(test)]
mod tests {
use crate::os::{get_info, same_disk};
#[cfg(target_os = "windows")]
#[test]
fn test_get_info_valid_path() {
crates/utils/src/path.rs
+413 -706
View File
File diff suppressed because it is too large Load Diff