Fix Windows Path Separator Handling in rustfs_utils (#1464)

Co-authored-by: reatang <tangtang1251@qq.com>
This commit is contained in:
houseme
2026-01-11 19:53:51 +08:00
committed by GitHub
parent 6b2eebee1d
commit 760cb1d734
18 changed files with 999 additions and 500 deletions
Generated
+169 -172
View File
File diff suppressed because it is too large Load Diff
+5 -4
View File
@@ -130,7 +130,7 @@ bytesize = "2.3.1"
byteorder = "1.5.0" byteorder = "1.5.0"
flatbuffers = "25.12.19" flatbuffers = "25.12.19"
form_urlencoded = "1.2.2" form_urlencoded = "1.2.2"
prost = "0.14.1" prost = "0.14.3"
quick-xml = "0.38.4" quick-xml = "0.38.4"
rmcp = { version = "0.12.0" } rmcp = { version = "0.12.0" }
rmp = { version = "0.8.15" } rmp = { version = "0.8.15" }
@@ -143,7 +143,7 @@ schemars = "1.2.0"
# Cryptography and Security # Cryptography and Security
aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] } aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
argon2 = { version = "0.6.0-rc.5" } argon2 = { version = "0.6.0-rc.5" }
blake3 = { version = "1.8.2", features = ["rayon", "mmap"] } blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
chacha20poly1305 = { version = "0.11.0-rc.2" } chacha20poly1305 = { version = "0.11.0-rc.2" }
crc-fast = "1.6.0" crc-fast = "1.6.0"
hmac = { version = "0.13.0-rc.3" } hmac = { version = "0.13.0-rc.3" }
@@ -184,6 +184,7 @@ criterion = { version = "0.8", features = ["html_reports"] }
crossbeam-queue = "0.3.12" crossbeam-queue = "0.3.12"
datafusion = "51.0.0" datafusion = "51.0.0"
derive_builder = "0.20.2" derive_builder = "0.20.2"
dunce = "1.0.5"
enumset = "1.1.10" enumset = "1.1.10"
faster-hex = "0.10.0" faster-hex = "0.10.0"
flate2 = "1.1.5" flate2 = "1.1.5"
@@ -197,7 +198,7 @@ hex-simd = "0.8.0"
highway = { version = "1.3.0" } highway = { version = "1.3.0" }
ipnetwork = { version = "0.21.1", features = ["serde"] } ipnetwork = { version = "0.21.1", features = ["serde"] }
lazy_static = "1.5.0" lazy_static = "1.5.0"
libc = "0.2.179" libc = "0.2.180"
libsystemd = "0.7.2" libsystemd = "0.7.2"
local-ip-address = "0.6.8" local-ip-address = "0.6.8"
lz4 = "1.28.1" lz4 = "1.28.1"
@@ -270,7 +271,7 @@ libunftp = "0.21.0"
russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false } russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false }
russh-sftp = "2.1.1" russh-sftp = "2.1.1"
ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] } ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] }
suppaftp = { version = "7.0.7", features = ["tokio", "tokio-rustls", "rustls"] } suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] }
rcgen = "0.14.6" rcgen = "0.14.6"
# Performance Analysis and Memory Profiling # Performance Analysis and Memory Profiling
+1 -1
View File
@@ -48,6 +48,7 @@ async-trait.workspace = true
bytes.workspace = true bytes.workspace = true
byteorder = { workspace = true } byteorder = { workspace = true }
chrono.workspace = true chrono.workspace = true
dunce.workspace = true
glob = { workspace = true } glob = { workspace = true }
thiserror.workspace = true thiserror.workspace = true
flatbuffers.workspace = true flatbuffers.workspace = true
@@ -109,7 +110,6 @@ google-cloud-auth = { workspace = true }
aws-config = { workspace = true } aws-config = { workspace = true }
faster-hex = { workspace = true } faster-hex = { workspace = true }
[dev-dependencies] [dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
criterion = { workspace = true, features = ["html_reports"] } criterion = { workspace = true, features = ["html_reports"] }
+3 -3
View File
@@ -14,7 +14,7 @@
use crate::disk::RUSTFS_META_BUCKET; use crate::disk::RUSTFS_META_BUCKET;
use crate::error::{Error, Result, StorageError}; use crate::error::{Error, Result, StorageError};
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR_STR;
use s3s::xml; use s3s::xml;
pub fn is_meta_bucketname(name: &str) -> bool { pub fn is_meta_bucketname(name: &str) -> bool {
@@ -194,7 +194,7 @@ pub fn is_valid_object_name(object: &str) -> bool {
return false; return false;
} }
if object.ends_with(SLASH_SEPARATOR) { if object.ends_with(SLASH_SEPARATOR_STR) {
return false; return false;
} }
@@ -206,7 +206,7 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned())); return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned()));
} }
if object.starts_with(SLASH_SEPARATOR) { if object.starts_with(SLASH_SEPARATOR_STR) {
return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned())); return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned()));
} }
+3 -3
View File
@@ -18,7 +18,7 @@ use crate::error::{Error, Result};
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI}; use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
use http::HeaderMap; use http::HeaderMap;
use rustfs_config::DEFAULT_DELIMITER; use rustfs_config::DEFAULT_DELIMITER;
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR_STR;
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
use std::sync::LazyLock; use std::sync::LazyLock;
@@ -29,7 +29,7 @@ const CONFIG_FILE: &str = "config.json";
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class"; pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}")); static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{CONFIG_PREFIX}"));
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| { static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
let mut h = HashSet::new(); let mut h = HashSet::new();
@@ -129,7 +129,7 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
} }
fn get_config_file() -> String { fn get_config_file() -> String {
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}") format!("{CONFIG_PREFIX}{SLASH_SEPARATOR_STR}{CONFIG_FILE}")
} }
/// Handle the situation where the configuration file does not exist, create and save a new configuration /// Handle the situation where the configuration file does not exist, create and save a new configuration
+5 -5
View File
@@ -31,14 +31,14 @@ use crate::{
use rustfs_common::data_usage::{ use rustfs_common::data_usage::{
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary, BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
}; };
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR_STR;
use tokio::fs; use tokio::fs;
use tracing::{error, info, warn}; use tracing::{error, info, warn};
use crate::error::Error; use crate::error::Error;
// Data usage storage constants // Data usage storage constants
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR; pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR;
const DATA_USAGE_OBJ_NAME: &str = ".usage.json"; const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin"; const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin"; pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
@@ -47,17 +47,17 @@ pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
lazy_static::lazy_static! { lazy_static::lazy_static! {
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}", pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
crate::disk::RUSTFS_META_BUCKET, crate::disk::RUSTFS_META_BUCKET,
SLASH_SEPARATOR, SLASH_SEPARATOR_STR,
crate::disk::BUCKET_META_PREFIX crate::disk::BUCKET_META_PREFIX
); );
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}", pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX, crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR, SLASH_SEPARATOR_STR,
DATA_USAGE_OBJ_NAME DATA_USAGE_OBJ_NAME
); );
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}", pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
crate::disk::BUCKET_META_PREFIX, crate::disk::BUCKET_META_PREFIX,
SLASH_SEPARATOR, SLASH_SEPARATOR_STR,
DATA_USAGE_BLOOM_NAME DATA_USAGE_BLOOM_NAME
); );
} }
+56 -66
View File
@@ -12,39 +12,26 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use super::error::{Error, Result};
use super::os::{is_root_disk, rename_all};
use super::{
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os,
};
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
use crate::config::storageclass::DEFAULT_INLINE_BLOCK; use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
use crate::data_usage::local_snapshot::ensure_data_usage_layout; use crate::data_usage::local_snapshot::ensure_data_usage_layout;
use crate::disk::error::FileAccessDeniedWithContext;
use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
use crate::disk::fs::{
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename,
};
use crate::disk::os::{check_path_length, is_empty_dir};
use crate::disk::{ use crate::disk::{
CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND, BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
FileReader, RUSTFS_META_TMP_DELETED_BUCKET, conv_part_err_to_int, CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq,
ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts,
VolumeInfo, WalkDirOptions, conv_part_err_to_int,
endpoint::Endpoint,
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3,
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
os,
os::{check_path_length, is_empty_dir, is_root_disk, rename_all},
}; };
use crate::disk::{FileWriter, STORAGE_FORMAT_FILE};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use rustfs_utils::path::{
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
path_join, path_join_buf,
};
use tokio::time::interval;
use crate::erasure_coding::bitrot_verify; use crate::erasure_coding::bitrot_verify;
use bytes::Bytes;
// use path_absolutize::Absolutize; // Replaced with direct path operations for better performance
use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached}; use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
use bytes::Bytes;
use parking_lot::RwLock as ParkingLotRwLock; use parking_lot::RwLock as ParkingLotRwLock;
use rustfs_filemeta::{ use rustfs_filemeta::{
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn, Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
@@ -52,6 +39,10 @@ use rustfs_filemeta::{
}; };
use rustfs_utils::HashAlgorithm; use rustfs_utils::HashAlgorithm;
use rustfs_utils::os::get_info; use rustfs_utils::os::get_info;
use rustfs_utils::path::{
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR_STR, clean, decode_dir_object, encode_dir_object,
has_suffix, path_join, path_join_buf,
};
use std::collections::HashMap; use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt::Debug; use std::fmt::Debug;
@@ -67,6 +58,7 @@ use time::OffsetDateTime;
use tokio::fs::{self, File}; use tokio::fs::{self, File};
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind}; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
use tokio::sync::RwLock; use tokio::sync::RwLock;
use tokio::time::interval;
use tracing::{debug, error, info, warn}; use tracing::{debug, error, info, warn};
use uuid::Uuid; use uuid::Uuid;
@@ -129,7 +121,8 @@ impl LocalDisk {
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> { pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
debug!("Creating local disk"); debug!("Creating local disk");
// Use optimized path resolution instead of absolutize() for better performance // Use optimized path resolution instead of absolutize() for better performance
let root = match std::fs::canonicalize(ep.get_file_path()) { // Use dunce::canonicalize instead of std::fs::canonicalize to avoid UNC paths on Windows
let root = match dunce::canonicalize(ep.get_file_path()) {
Ok(path) => path, Ok(path) => path,
Err(e) => { Err(e) => {
if e.kind() == ErrorKind::NotFound { if e.kind() == ErrorKind::NotFound {
@@ -483,7 +476,7 @@ impl LocalDisk {
// Async prefetch related files, don't block current read // Async prefetch related files, don't block current read
if let Some(parent) = file_path.parent() { if let Some(parent) = file_path.parent() {
prefetch_metadata_patterns(parent, &[super::STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await; prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
} }
// Main read logic // Main read logic
@@ -507,7 +500,7 @@ impl LocalDisk {
async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> { async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
let paths: Vec<PathBuf> = requests let paths: Vec<PathBuf> = requests
.iter() .iter()
.map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, super::STORAGE_FORMAT_FILE))) .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE)))
.collect::<Result<Vec<_>>>()?; .collect::<Result<Vec<_>>>()?;
let cache = get_global_file_cache(); let cache = get_global_file_cache();
@@ -544,7 +537,7 @@ impl LocalDisk {
// TODO: async notifications for disk space checks and trash cleanup // TODO: async notifications for disk space checks and trash cleanup
let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
// if let Some(parent) = trash_path.parent() { // if let Some(parent) = trash_path.parent() {
// if !parent.exists() { // if !parent.exists() {
// fs::create_dir_all(parent).await?; // fs::create_dir_all(parent).await?;
@@ -552,7 +545,7 @@ impl LocalDisk {
// } // }
let err = if recursive { let err = if recursive {
rename_all(delete_path, trash_path, self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?) rename_all(delete_path, trash_path, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?)
.await .await
.err() .err()
} else { } else {
@@ -562,12 +555,12 @@ impl LocalDisk {
.err() .err()
}; };
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) { if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR_STR) {
let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
let _ = rename_all( let _ = rename_all(
encode_dir_object(delete_path.to_string_lossy().as_ref()), encode_dir_object(delete_path.to_string_lossy().as_ref()),
trash_path2, trash_path2,
self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?,
) )
.await; .await;
} }
@@ -916,7 +909,7 @@ impl LocalDisk {
} }
if let Some(parent) = path.as_ref().parent() { if let Some(parent) = path.as_ref().parent() {
super::os::make_dir_all(parent, skip_parent).await?; os::make_dir_all(parent, skip_parent).await?;
} }
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?; let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
@@ -942,7 +935,7 @@ impl LocalDisk {
let meta = file.metadata().await.map_err(to_file_error)?; let meta = file.metadata().await.map_err(to_file_error)?;
let file_size = meta.len() as usize; let file_size = meta.len() as usize;
bitrot_verify(Box::new(file), file_size, part_size, algo, bytes::Bytes::copy_from_slice(sum), shard_size) bitrot_verify(Box::new(file), file_size, part_size, algo, Bytes::copy_from_slice(sum), shard_size)
.await .await
.map_err(to_file_error)?; .map_err(to_file_error)?;
@@ -1038,15 +1031,16 @@ impl LocalDisk {
continue; continue;
} }
if entry.ends_with(SLASH_SEPARATOR) { if entry.ends_with(SLASH_SEPARATOR_STR) {
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR); let entry =
format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR_STR);
dir_objes.insert(entry.clone()); dir_objes.insert(entry.clone());
*item = entry; *item = entry;
continue; continue;
} }
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned(); *item = entry.trim_end_matches(SLASH_SEPARATOR_STR).to_owned();
continue; continue;
} }
@@ -1058,7 +1052,7 @@ impl LocalDisk {
.await?; .await?;
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned(); let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
let name = entry.trim_end_matches(SLASH_SEPARATOR); let name = entry.trim_end_matches(SLASH_SEPARATOR_STR);
let name = decode_dir_object(format!("{}/{}", &current, &name).as_str()); let name = decode_dir_object(format!("{}/{}", &current, &name).as_str());
// if opts.limit > 0 // if opts.limit > 0
@@ -1141,7 +1135,7 @@ impl LocalDisk {
Ok(res) => { Ok(res) => {
if is_dir_obj { if is_dir_obj {
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned(); meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
meta.name.push_str(SLASH_SEPARATOR); meta.name.push_str(SLASH_SEPARATOR_STR);
} }
meta.metadata = res; meta.metadata = res;
@@ -1159,7 +1153,7 @@ impl LocalDisk {
// NOT an object, append to stack (with slash) // NOT an object, append to stack (with slash)
// If dirObject, but no metadata (which is unexpected) we skip it. // If dirObject, but no metadata (which is unexpected) we skip it.
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await { if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
meta.name.push_str(SLASH_SEPARATOR); meta.name.push_str(SLASH_SEPARATOR_STR);
dir_stack.push(meta.name); dir_stack.push(meta.name);
} }
} }
@@ -1234,7 +1228,7 @@ async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
fn skip_access_checks(p: impl AsRef<str>) -> bool { fn skip_access_checks(p: impl AsRef<str>) -> bool {
let vols = [ let vols = [
super::RUSTFS_META_TMP_DELETED_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET,
super::RUSTFS_META_TMP_BUCKET, super::RUSTFS_META_TMP_BUCKET,
super::RUSTFS_META_MULTIPART_BUCKET, super::RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
@@ -1628,8 +1622,8 @@ impl DiskAPI for LocalDisk {
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))? super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
} }
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir { if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
warn!( warn!(
@@ -1695,8 +1689,8 @@ impl DiskAPI for LocalDisk {
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?; .map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
} }
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR); let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR); let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) { if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) {
return Err(Error::from(DiskError::FileAccessDenied)); return Err(Error::from(DiskError::FileAccessDenied));
} }
@@ -1847,12 +1841,12 @@ impl DiskAPI for LocalDisk {
} }
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?; let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR_STR))?;
let entries = match os::read_dir(&dir_path_abs, count).await { let entries = match os::read_dir(&dir_path_abs, count).await {
Ok(res) => res, Ok(res) => res,
Err(e) => { Err(e) => {
if e.kind() == std::io::ErrorKind::NotFound if e.kind() == ErrorKind::NotFound
&& !skip_access_checks(volume) && !skip_access_checks(volume)
&& let Err(e) = access(&volume_dir).await && let Err(e) = access(&volume_dir).await
{ {
@@ -1883,11 +1877,11 @@ impl DiskAPI for LocalDisk {
let mut objs_returned = 0; let mut objs_returned = 0;
if opts.base_dir.ends_with(SLASH_SEPARATOR) { if opts.base_dir.ends_with(SLASH_SEPARATOR_STR) {
let fpath = self.get_object_path( let fpath = self.get_object_path(
&opts.bucket, &opts.bucket,
path_join_buf(&[ path_join_buf(&[
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(), format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX).as_str(),
STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE,
]) ])
.as_str(), .as_str(),
@@ -2119,7 +2113,7 @@ impl DiskAPI for LocalDisk {
let volume_dir = self.get_bucket_path(volume)?; let volume_dir = self.get_bucket_path(volume)?;
if let Err(e) = access(&volume_dir).await { if let Err(e) = access(&volume_dir).await {
if e.kind() == std::io::ErrorKind::NotFound { if e.kind() == ErrorKind::NotFound {
os::make_dir_all(&volume_dir, self.root.as_path()).await?; os::make_dir_all(&volume_dir, self.root.as_path()).await?;
return Ok(()); return Ok(());
} }
@@ -2137,7 +2131,7 @@ impl DiskAPI for LocalDisk {
let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?; let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?;
for entry in entries { for entry in entries {
if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) { if !has_suffix(&entry, SLASH_SEPARATOR_STR) || !Self::is_valid_volname(clean(&entry).as_str()) {
continue; continue;
} }
@@ -2359,7 +2353,7 @@ impl DiskAPI for LocalDisk {
force_del_marker: bool, force_del_marker: bool,
opts: DeleteOptions, opts: DeleteOptions,
) -> Result<()> { ) -> Result<()> {
if path.starts_with(SLASH_SEPARATOR) { if path.starts_with(SLASH_SEPARATOR_STR) {
return self return self
.delete( .delete(
volume, volume,
@@ -2420,7 +2414,7 @@ impl DiskAPI for LocalDisk {
if !meta.versions.is_empty() { if !meta.versions.is_empty() {
let buf = meta.marshal_msg()?; let buf = meta.marshal_msg()?;
return self return self
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true) .write_all_meta(volume, format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
.await; .await;
} }
@@ -2430,11 +2424,11 @@ impl DiskAPI for LocalDisk {
{ {
let src_path = path_join(&[ let src_path = path_join(&[
file_path.as_path(), file_path.as_path(),
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()), Path::new(format!("{old_data_dir}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
]); ]);
let dst_path = path_join(&[ let dst_path = path_join(&[
file_path.as_path(), file_path.as_path(),
Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()), Path::new(format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str()),
]); ]);
return rename_all(src_path, dst_path, file_path).await; return rename_all(src_path, dst_path, file_path).await;
} }
@@ -2563,7 +2557,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
if root_disk_threshold > 0 { if root_disk_threshold > 0 {
disk_info.total <= root_disk_threshold disk_info.total <= root_disk_threshold
} else { } else {
is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default() is_root_disk(&drive_path, SLASH_SEPARATOR_STR).unwrap_or_default()
} }
} else { } else {
false false
@@ -2581,7 +2575,7 @@ mod test {
// let arr = Vec::new(); // let arr = Vec::new();
let vols = [ let vols = [
super::super::RUSTFS_META_TMP_DELETED_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET,
super::super::RUSTFS_META_TMP_BUCKET, super::super::RUSTFS_META_TMP_BUCKET,
super::super::RUSTFS_META_MULTIPART_BUCKET, super::super::RUSTFS_META_MULTIPART_BUCKET,
RUSTFS_META_BUCKET, RUSTFS_META_BUCKET,
@@ -2609,9 +2603,7 @@ mod test {
let disk = LocalDisk::new(&ep, false).await.unwrap(); let disk = LocalDisk::new(&ep, false).await.unwrap();
let tmpp = disk let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
.unwrap();
println!("ppp :{:?}", &tmpp); println!("ppp :{:?}", &tmpp);
@@ -2639,9 +2631,7 @@ mod test {
let disk = LocalDisk::new(&ep, false).await.unwrap(); let disk = LocalDisk::new(&ep, false).await.unwrap();
let tmpp = disk let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
.unwrap();
println!("ppp :{:?}", &tmpp); println!("ppp :{:?}", &tmpp);
+2 -2
View File
@@ -19,7 +19,7 @@ use std::{
use super::error::Result; use super::error::Result;
use crate::disk::error_conv::to_file_error; use crate::disk::error_conv::to_file_error;
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR_STR;
use tokio::fs; use tokio::fs;
use tracing::warn; use tracing::warn;
@@ -118,7 +118,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
if file_type.is_file() { if file_type.is_file() {
volumes.push(name); volumes.push(name);
} else if file_type.is_dir() { } else if file_type.is_dir() {
volumes.push(format!("{name}{SLASH_SEPARATOR}")); volumes.push(format!("{name}{SLASH_SEPARATOR_STR}"));
} }
count -= 1; count -= 1;
if count == 0 { if count == 0 {
+3 -3
View File
@@ -38,7 +38,7 @@ use rustfs_common::defer;
use rustfs_common::heal_channel::HealOpts; use rustfs_common::heal_channel::HealOpts;
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
use rustfs_rio::{HashReader, WarpReader}; use rustfs_rio::{HashReader, WarpReader};
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join}; use rustfs_utils::path::{SLASH_SEPARATOR_STR, encode_dir_object, path_join};
use rustfs_workers::workers::Workers; use rustfs_workers::workers::Workers;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
@@ -451,10 +451,10 @@ fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, S
let trimmed_path = path let trimmed_path = path
.strip_prefix(base_path) .strip_prefix(base_path)
.unwrap_or(path) .unwrap_or(path)
.strip_prefix(SLASH_SEPARATOR) .strip_prefix(SLASH_SEPARATOR_STR)
.unwrap_or(path); .unwrap_or(path);
// Find the position of the first '/' // Find the position of the first '/'
let pos = trimmed_path.find(SLASH_SEPARATOR).unwrap_or(trimmed_path.len()); let pos = trimmed_path.find(SLASH_SEPARATOR_STR).unwrap_or(trimmed_path.len());
// Split into bucket and prefix // Split into bucket and prefix
let bucket = &trimmed_path[0..pos]; let bucket = &trimmed_path[0..pos];
let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists
+4 -4
View File
@@ -82,7 +82,7 @@ use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX,
use rustfs_utils::{ use rustfs_utils::{
HashAlgorithm, HashAlgorithm,
crypto::hex, crypto::hex,
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, path::{SLASH_SEPARATOR_STR, encode_dir_object, has_suffix, path_join_buf},
}; };
use rustfs_workers::workers::Workers; use rustfs_workers::workers::Workers;
use s3s::header::X_AMZ_RESTORE; use s3s::header::X_AMZ_RESTORE;
@@ -5324,7 +5324,7 @@ impl StorageAPI for SetDisks {
&upload_id_path, &upload_id_path,
fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(), fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(),
]), ]),
SLASH_SEPARATOR SLASH_SEPARATOR_STR
); );
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await { let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
@@ -5462,7 +5462,7 @@ impl StorageAPI for SetDisks {
let mut populated_upload_ids = HashSet::new(); let mut populated_upload_ids = HashSet::new();
for upload_id in upload_ids.iter() { for upload_id in upload_ids.iter() {
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR).to_string(); let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR_STR).to_string();
if populated_upload_ids.contains(&upload_id) { if populated_upload_ids.contains(&upload_id) {
continue; continue;
} }
@@ -6222,7 +6222,7 @@ impl StorageAPI for SetDisks {
None None
}; };
if has_suffix(object, SLASH_SEPARATOR) { if has_suffix(object, SLASH_SEPARATOR_STR) {
let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?; let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?;
return Ok((result, err.map(|e| e.into()))); return Ok((result, err.map(|e| e.into())));
} }
+10 -10
View File
@@ -34,7 +34,7 @@ use rustfs_filemeta::{
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams, MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
merge_file_meta_versions, merge_file_meta_versions,
}; };
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix}; use rustfs_utils::path::{self, SLASH_SEPARATOR_STR, base_dir_from_prefix};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::broadcast::{self}; use tokio::sync::broadcast::{self};
@@ -132,7 +132,7 @@ impl ListPathOptions {
return; return;
} }
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default(); let s = SLASH_SEPARATOR_STR.chars().next().unwrap_or_default();
self.filter_prefix = { self.filter_prefix = {
let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s); let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s);
@@ -346,7 +346,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter { if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() { if obj.is_dir && obj.mod_time.is_none() {
let mut found = false; let mut found = false;
if delimiter != SLASH_SEPARATOR { if delimiter != SLASH_SEPARATOR_STR {
for p in prefixes.iter() { for p in prefixes.iter() {
if found { if found {
break; break;
@@ -470,7 +470,7 @@ impl ECStore {
if let Some(delimiter) = &delimiter { if let Some(delimiter) = &delimiter {
if obj.is_dir && obj.mod_time.is_none() { if obj.is_dir && obj.mod_time.is_none() {
let mut found = false; let mut found = false;
if delimiter != SLASH_SEPARATOR { if delimiter != SLASH_SEPARATOR_STR {
for p in prefixes.iter() { for p in prefixes.iter() {
if found { if found {
break; break;
@@ -502,7 +502,7 @@ impl ECStore {
// warn!("list_path opt {:?}", &o); // warn!("list_path opt {:?}", &o);
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?; check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
// if opts.prefix.ends_with(SLASH_SEPARATOR) { // if opts.prefix.ends_with(SLASH_SEPARATOR_STR) {
// return Err(Error::msg("eof")); // return Err(Error::msg("eof"));
// } // }
@@ -520,11 +520,11 @@ impl ECStore {
return Err(Error::Unexpected); return Err(Error::Unexpected);
} }
if o.prefix.starts_with(SLASH_SEPARATOR) { if o.prefix.starts_with(SLASH_SEPARATOR_STR) {
return Err(Error::Unexpected); return Err(Error::Unexpected);
} }
let slash_separator = Some(SLASH_SEPARATOR.to_owned()); let slash_separator = Some(SLASH_SEPARATOR_STR.to_owned());
o.include_directories = o.separator == slash_separator; o.include_directories = o.separator == slash_separator;
@@ -774,8 +774,8 @@ impl ECStore {
let mut filter_prefix = { let mut filter_prefix = {
prefix prefix
.trim_start_matches(&path) .trim_start_matches(&path)
.trim_start_matches(SLASH_SEPARATOR) .trim_start_matches(SLASH_SEPARATOR_STR)
.trim_end_matches(SLASH_SEPARATOR) .trim_end_matches(SLASH_SEPARATOR_STR)
.to_owned() .to_owned()
}; };
@@ -1130,7 +1130,7 @@ async fn merge_entry_channels(
if path::clean(&best_entry.name) == path::clean(&other_entry.name) { if path::clean(&best_entry.name) == path::clean(&other_entry.name) {
let dir_matches = best_entry.is_dir() && other_entry.is_dir(); let dir_matches = best_entry.is_dir() && other_entry.is_dir();
let suffix_matches = let suffix_matches =
best_entry.name.ends_with(SLASH_SEPARATOR) == other_entry.name.ends_with(SLASH_SEPARATOR); best_entry.name.ends_with(SLASH_SEPARATOR_STR) == other_entry.name.ends_with(SLASH_SEPARATOR_STR);
if dir_matches && suffix_matches { if dir_matches && suffix_matches {
to_merge.push(other_idx); to_merge.push(other_idx);
+3 -3
View File
@@ -51,7 +51,7 @@ use crate::{
store_api::{ObjectOptions, PutObjReader}, store_api::{ObjectOptions, PutObjReader},
}; };
use rustfs_rio::HashReader; use rustfs_rio::HashReader;
use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join};
use s3s::S3ErrorCode; use s3s::S3ErrorCode;
use super::{ use super::{
@@ -403,7 +403,7 @@ impl TierConfigMgr {
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> { pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
let data = self.marshal()?; let data = self.marshal()?;
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
self.save_config(api, &config_file, data).await self.save_config(api, &config_file, data).await
} }
@@ -483,7 +483,7 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))] #[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> { async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE); let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
let data = read_config(api.clone(), config_file.as_str()).await; let data = read_config(api.clone(), config_file.as_str()).await;
if let Err(err) = data { if let Err(err) = data {
if is_err_config_not_found(&err) { if is_err_config_not_found(&err) {
+2 -4
View File
@@ -30,13 +30,11 @@ use crate::client::{
transition_api::{Options, TransitionClient, TransitionCore}, transition_api::{Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl}, transition_api::{ReadCloser, ReaderImpl},
}; };
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{ use crate::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts}, warm_backend::{WarmBackend, WarmBackendGetOpts},
}; };
use rustfs_utils::path::SLASH_SEPARATOR; use rustfs_utils::path::SLASH_SEPARATOR_STR;
pub struct WarmBackendS3 { pub struct WarmBackendS3 {
pub client: Arc<TransitionClient>, pub client: Arc<TransitionClient>,
@@ -178,7 +176,7 @@ impl WarmBackend for WarmBackendS3 {
async fn in_use(&self) -> Result<bool, std::io::Error> { async fn in_use(&self) -> Result<bool, std::io::Error> {
let result = self let result = self
.core .core
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1) .list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR_STR, 1)
.await?; .await?;
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0) Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
@@ -27,19 +27,11 @@ use aws_sdk_s3::Client;
use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::primitives::ByteStream;
use crate::client::{ use crate::client::transition_api::{ReadCloser, ReaderImpl};
api_get_options::GetObjectOptions,
api_put_object::PutObjectOptions,
api_remove::RemoveObjectOptions,
transition_api::{ReadCloser, ReaderImpl},
};
use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::tier::{ use crate::tier::{
tier_config::TierS3, tier_config::TierS3,
warm_backend::{WarmBackend, WarmBackendGetOpts}, warm_backend::{WarmBackend, WarmBackendGetOpts},
}; };
use rustfs_utils::path::SLASH_SEPARATOR;
pub struct WarmBackendS3 { pub struct WarmBackendS3 {
pub client: Arc<Client>, pub client: Arc<Client>,
+2 -2
View File
@@ -32,7 +32,7 @@ use rustfs_ecstore::{
store_api::{ObjectInfo, ObjectOptions}, store_api::{ObjectInfo, ObjectOptions},
}; };
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use serde::{Serialize, de::DeserializeOwned}; use serde::{Serialize, de::DeserializeOwned};
use std::sync::LazyLock; use std::sync::LazyLock;
use std::{collections::HashMap, sync::Arc}; use std::{collections::HashMap, sync::Arc};
@@ -182,7 +182,7 @@ impl ObjectStore {
} else { } else {
info.name info.name
}; };
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR); let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR_STR);
let _ = sender let _ = sender
.send(StringOrErr { .send(StringOrErr {
item: Some(name.to_owned()), item: Some(name.to_owned()),
+1 -1
View File
@@ -83,7 +83,7 @@ ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver
io = ["dep:tokio"] io = ["dep:tokio"]
path = [] path = [] # path manipulation features
notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
string = ["dep:regex"] string = ["dep:regex"]
+639 -117
View File
@@ -15,12 +15,36 @@
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
#[cfg(target_os = "windows")]
const SLASH_SEPARATOR: char = '\\';
#[cfg(not(target_os = "windows"))]
const SLASH_SEPARATOR: char = '/';
/// GLOBAL_DIR_SUFFIX is a special suffix used to denote directory objects
/// in object storage systems that do not have a native directory concept.
pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; pub const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
pub const SLASH_SEPARATOR: &str = "/"; /// SLASH_SEPARATOR_STR is the string representation of the path separator
/// used in the current operating system.
pub const SLASH_SEPARATOR_STR: &str = if cfg!(target_os = "windows") { "\\" } else { "/" };
/// GLOBAL_DIR_SUFFIX_WITH_SLASH is the directory suffix followed by the
/// platform-specific path separator, used to denote directory objects.
#[cfg(target_os = "windows")]
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__\\";
#[cfg(not(target_os = "windows"))]
pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/"; pub const GLOBAL_DIR_SUFFIX_WITH_SLASH: &str = "__XLDIR__/";
/// has_suffix checks if the string `s` ends with the specified `suffix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `suffix` - A string slice that holds the suffix to check for.
///
/// # Returns
/// A boolean indicating whether `s` ends with `suffix`.
///
pub fn has_suffix(s: &str, suffix: &str) -> bool { pub fn has_suffix(s: &str, suffix: &str) -> bool {
if cfg!(target_os = "windows") { if cfg!(target_os = "windows") {
s.to_lowercase().ends_with(&suffix.to_lowercase()) s.to_lowercase().ends_with(&suffix.to_lowercase())
@@ -29,19 +53,46 @@ pub fn has_suffix(s: &str, suffix: &str) -> bool {
} }
} }
/// encode_dir_object encodes a directory object by appending
/// a special suffix if it ends with a slash.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be encoded.
///
/// # Returns
/// A `String` representing the encoded directory object.
///
pub fn encode_dir_object(object: &str) -> String { pub fn encode_dir_object(object: &str) -> String {
if has_suffix(object, SLASH_SEPARATOR) { if has_suffix(object, SLASH_SEPARATOR_STR) {
format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX) format!("{}{}", object.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX)
} else { } else {
object.to_string() object.to_string()
} }
} }
/// is_dir_object checks if the given object string represents
/// a directory object by verifying if it ends with the special suffix.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be checked.
///
/// # Returns
/// A boolean indicating whether the object is a directory object.
///
pub fn is_dir_object(object: &str) -> bool { pub fn is_dir_object(object: &str) -> bool {
let obj = encode_dir_object(object); let obj = encode_dir_object(object);
obj.ends_with(GLOBAL_DIR_SUFFIX) obj.ends_with(GLOBAL_DIR_SUFFIX)
} }
/// decode_dir_object decodes a directory object by removing
/// the special suffix if it is present.
///
/// # Arguments
/// * `object` - A string slice that holds the object to be decoded.
///
/// # Returns
/// A `String` representing the decoded directory object.
///
#[allow(dead_code)] #[allow(dead_code)]
pub fn decode_dir_object(object: &str) -> String { pub fn decode_dir_object(object: &str) -> String {
if has_suffix(object, GLOBAL_DIR_SUFFIX) { if has_suffix(object, GLOBAL_DIR_SUFFIX) {
@@ -51,21 +102,50 @@ pub fn decode_dir_object(object: &str) -> String {
} }
} }
/// retain_slash ensures that the given string `s` ends with a slash.
/// If it does not, a slash is appended.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be processed.
///
/// # Returns
/// A `String` that ends with a slash.
///
pub fn retain_slash(s: &str) -> String { pub fn retain_slash(s: &str) -> String {
if s.is_empty() { if s.is_empty() {
return s.to_string(); return s.to_string();
} }
if s.ends_with(SLASH_SEPARATOR) { if s.ends_with(SLASH_SEPARATOR_STR) {
s.to_string() s.to_string()
} else { } else {
format!("{s}{SLASH_SEPARATOR}") format!("{s}{SLASH_SEPARATOR_STR}")
} }
} }
/// strings_has_prefix_fold checks if the string `s` starts with the specified `prefix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `prefix` - A string slice that holds the prefix to check for.
///
/// # Returns
/// A boolean indicating whether `s` starts with `prefix`.
///
pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool { pub fn strings_has_prefix_fold(s: &str, prefix: &str) -> bool {
s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix)) s.len() >= prefix.len() && (s[..prefix.len()] == *prefix || s[..prefix.len()].eq_ignore_ascii_case(prefix))
} }
/// has_prefix checks if the string `s` starts with the specified `prefix`,
/// performing a case-insensitive comparison on Windows platforms.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
/// * `prefix` - A string slice that holds the prefix to check for.
///
/// # Returns
/// A boolean indicating whether `s` starts with `prefix`.
///
pub fn has_prefix(s: &str, prefix: &str) -> bool { pub fn has_prefix(s: &str, prefix: &str) -> bool {
if cfg!(target_os = "windows") { if cfg!(target_os = "windows") {
return strings_has_prefix_fold(s, prefix); return strings_has_prefix_fold(s, prefix);
@@ -74,21 +154,37 @@ pub fn has_prefix(s: &str, prefix: &str) -> bool {
s.starts_with(prefix) s.starts_with(prefix)
} }
/// path_join joins multiple path elements into a single PathBuf,
/// ensuring that the resulting path is clean and properly formatted.
///
/// # Arguments
/// * `elem` - A slice of path elements to be joined.
///
/// # Returns
/// A PathBuf representing the joined path.
///
pub fn path_join<P: AsRef<Path>>(elem: &[P]) -> PathBuf { pub fn path_join<P: AsRef<Path>>(elem: &[P]) -> PathBuf {
path_join_buf( if elem.is_empty() {
elem.iter() return PathBuf::from(".");
.map(|p| p.as_ref().to_string_lossy().into_owned()) }
.collect::<Vec<String>>() // Collect components as owned Strings (lossy for non-UTF8)
.iter() let strs: Vec<String> = elem.iter().map(|p| p.as_ref().to_string_lossy().into_owned()).collect();
.map(|s| s.as_str()) // Convert to slice of &str for path_join_buf
.collect::<Vec<&str>>() let refs: Vec<&str> = strs.iter().map(|s| s.as_str()).collect();
.as_slice(), PathBuf::from(path_join_buf(&refs))
)
.into()
} }
/// path_join_buf joins multiple string path elements into a single String,
/// ensuring that the resulting path is clean and properly formatted.
///
/// # Arguments
/// * `elements` - A slice of string path elements to be joined.
///
/// # Returns
/// A String representing the joined path.
///
pub fn path_join_buf(elements: &[&str]) -> String { pub fn path_join_buf(elements: &[&str]) -> String {
let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR)); let trailing_slash = !elements.is_empty() && elements.last().is_some_and(|last| last.ends_with(SLASH_SEPARATOR_STR));
let mut dst = String::new(); let mut dst = String::new();
let mut added = 0; let mut added = 0;
@@ -96,7 +192,7 @@ pub fn path_join_buf(elements: &[&str]) -> String {
for e in elements { for e in elements {
if added > 0 || !e.is_empty() { if added > 0 || !e.is_empty() {
if added > 0 { if added > 0 {
dst.push_str(SLASH_SEPARATOR); dst.push(SLASH_SEPARATOR);
} }
dst.push_str(e); dst.push_str(e);
added += e.len(); added += e.len();
@@ -106,168 +202,420 @@ pub fn path_join_buf(elements: &[&str]) -> String {
if path_needs_clean(dst.as_bytes()) { if path_needs_clean(dst.as_bytes()) {
let mut clean_path = clean(&dst); let mut clean_path = clean(&dst);
if trailing_slash { if trailing_slash {
clean_path.push_str(SLASH_SEPARATOR); clean_path.push(SLASH_SEPARATOR);
} }
return clean_path; return clean_path;
} }
if trailing_slash { if trailing_slash {
dst.push_str(SLASH_SEPARATOR); dst.push(SLASH_SEPARATOR);
} }
dst dst
} }
/// Platform-aware separator check
fn is_sep(b: u8) -> bool {
#[cfg(target_os = "windows")]
{
b == b'/' || b == b'\\'
}
#[cfg(not(target_os = "windows"))]
{
b == b'/'
}
}
/// path_needs_clean returns whether path cleaning may change the path. /// path_needs_clean returns whether path cleaning may change the path.
/// Will detect all cases that will be cleaned, /// Will detect all cases that will be cleaned,
/// but may produce false positives on non-trivial paths. /// but may produce false positives on non-trivial paths.
///
/// # Arguments
/// * `path` - A byte slice that holds the path to be checked.
///
/// # Returns
/// A boolean indicating whether the path needs cleaning.
///
fn path_needs_clean(path: &[u8]) -> bool { fn path_needs_clean(path: &[u8]) -> bool {
if path.is_empty() { if path.is_empty() {
return true; return true;
} }
let rooted = path[0] == b'/';
let n = path.len(); let n = path.len();
let (mut r, mut w) = if rooted { (1, 1) } else { (0, 0) }; // On Windows: any forward slash indicates normalization to backslash is required.
#[cfg(target_os = "windows")]
{
if path.iter().any(|&b| b == b'/') {
return true;
}
}
while r < n { // Initialize scan index and previous-separator flag.
match path[r] { let mut i = 0usize;
b if b > 127 => { let mut prev_was_sep = false;
// Non ascii.
// Platform-aware prefix handling to avoid flagging meaningful leading sequences:
// - Windows: handle drive letter "C:" and UNC leading "\\"
// - Non-Windows: detect and flag double leading '/' (e.g. "//abc") as needing clean
if n >= 1 && is_sep(path[0]) {
#[cfg(target_os = "windows")]
{
// If starts with two separators -> UNC prefix: allow exactly two without flag
if n >= 2 && is_sep(path[1]) {
// If a third leading separator exists, that's redundant (e.g. "///...") -> needs clean
if n >= 3 && is_sep(path[2]) {
return true; return true;
} }
b'/' => { // Skip the two UNC leading separators for scanning; do not mark prev_was_sep true
// multiple / elements i = 2;
prev_was_sep = false;
} else {
// Single leading separator (rooted) -> mark as seen separator so immediate next sep is duplicate
i = 1;
prev_was_sep = true;
}
}
#[cfg(not(target_os = "windows"))]
{
// POSIX: double leading '/' is redundant and should be cleaned (e.g. "//abc" -> "/abc")
if n >= 2 && is_sep(path[1]) {
return true; return true;
} }
b'.' => { i = 1;
if r + 1 == n || path[r + 1] == b'/' { prev_was_sep = true;
// . element - assume it has to be cleaned. }
} else {
// If not starting with separator, check for Windows drive-letter prefix like "C:"
#[cfg(target_os = "windows")]
{
if n >= 2 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() {
// Position after "C:"
i = 2;
// If a separator immediately follows the drive (rooted like "C:\"),
// treat that first separator as seen; if more separators follow, it's redundant.
if i < n && is_sep(path[i]) {
i += 1; // consume the single allowed separator after drive
if i < n && is_sep(path[i]) {
// multiple separators after drive like "C:\\..." -> needs clean
return true; return true;
} }
if r + 1 < n && path[r + 1] == b'.' && (r + 2 == n || path[r + 2] == b'/') { prev_was_sep = true;
// .. element: remove to last / - assume it has to be cleaned. } else {
return true; prev_was_sep = false;
}
// Handle single dot case
if r + 1 == n {
// . element - assume it has to be cleaned.
return true;
}
// Copy the dot
w += 1;
r += 1;
}
_ => {
// real path element.
// add slash if needed
if (rooted && w != 1) || (!rooted && w != 0) {
w += 1;
}
// copy element
while r < n && path[r] != b'/' {
w += 1;
r += 1;
}
// allow one slash, not at end
if r < n - 1 && path[r] == b'/' {
r += 1;
} }
} }
} }
} }
// Turn empty string into "." // Generic scan for repeated separators and dot / dot-dot components.
if w == 0 { while i < n {
let b = path[i];
if is_sep(b) {
if prev_was_sep {
// Multiple separators (except allowed UNC prefix handled above)
return true; return true;
} }
prev_was_sep = true;
i += 1;
continue;
}
// Not a separator: parse current path element
let start = i;
while i < n && !is_sep(path[i]) {
i += 1;
}
let len = i - start;
if len == 1 && path[start] == b'.' {
// single "." element -> needs cleaning
return true;
}
if len == 2 && path[start] == b'.' && path[start + 1] == b'.' {
// ".." element -> needs cleaning
return true;
}
prev_was_sep = false;
}
// Trailing separator: if last byte is a separator and path length > 1, then usually needs cleaning,
// except when the path is a platform-specific root form (e.g. "/" on POSIX, "\\" or "C:\" on Windows).
if n > 1 && is_sep(path[n - 1]) {
#[cfg(not(target_os = "windows"))]
{
// POSIX: any trailing separator except the single-root "/" needs cleaning.
return true;
}
#[cfg(target_os = "windows")]
{
// Windows special root forms that are acceptable with trailing separator:
// - UNC root: exactly two leading separators "\" "\" (i.e. "\\") -> n == 2
if n == 2 && is_sep(path[0]) && is_sep(path[1]) {
return false;
}
// - Drive root: pattern "C:\" or "C:/" (len == 3)
if n == 3 && path[1] == b':' && (path[0] as char).is_ascii_alphabetic() && is_sep(path[2]) {
return false;
}
// Otherwise, trailing separator should be cleaned.
return true;
}
}
// No conditions triggered: assume path is already clean.
false false
} }
pub fn path_to_bucket_object_with_base_path(bash_path: &str, path: &str) -> (String, String) { /// path_to_bucket_object_with_base_path splits a given path into bucket and object components,
let path = path.trim_start_matches(bash_path).trim_start_matches(SLASH_SEPARATOR); /// considering a base path to trim from the start.
///
/// # Arguments
/// * `base_path` - A string slice that holds the base path to be trimmed.
/// * `path` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the bucket and object as `String`s.
///
pub fn path_to_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
let path = path.trim_start_matches(base_path).trim_start_matches(SLASH_SEPARATOR);
if let Some(m) = path.find(SLASH_SEPARATOR) { if let Some(m) = path.find(SLASH_SEPARATOR) {
return (path[..m].to_string(), path[m + SLASH_SEPARATOR.len()..].to_string()); return (path[..m].to_string(), path[m + SLASH_SEPARATOR_STR.len()..].to_string());
} }
(path.to_string(), "".to_string()) (path.to_string(), "".to_string())
} }
/// path_to_bucket_object splits a given path into bucket and object components.
///
/// # Arguments
/// * `s` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the bucket and object as `String`s.
///
pub fn path_to_bucket_object(s: &str) -> (String, String) { pub fn path_to_bucket_object(s: &str) -> (String, String) {
path_to_bucket_object_with_base_path("", s) path_to_bucket_object_with_base_path("", s)
} }
/// contains_any_sep_str checks if the given string contains any path separators.
///
/// # Arguments
/// * `s` - A string slice that holds the string to be checked.
///
/// # Returns
/// A boolean indicating whether the string contains any path separators.
fn contains_any_sep_str(s: &str) -> bool {
#[cfg(target_os = "windows")]
{
s.contains('/') || s.contains('\\')
}
#[cfg(not(target_os = "windows"))]
{
s.contains('/')
}
}
/// base_dir_from_prefix extracts the base directory from a given prefix.
///
/// # Arguments
/// * `prefix` - A string slice that holds the prefix to be processed.
///
/// # Returns
/// A `String` representing the base directory extracted from the prefix.
///
pub fn base_dir_from_prefix(prefix: &str) -> String { pub fn base_dir_from_prefix(prefix: &str) -> String {
let mut base_dir = dir(prefix).to_owned(); if !contains_any_sep_str(prefix) {
if base_dir == "." || base_dir == "./" || base_dir == "/" { return String::new();
base_dir = "".to_owned();
} }
if !prefix.contains('/') { let mut base_dir = dir(prefix);
base_dir = "".to_owned(); if base_dir == "." || base_dir == SLASH_SEPARATOR_STR {
base_dir.clear();
} }
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) { if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR_STR) {
base_dir.push_str(SLASH_SEPARATOR); base_dir.push_str(SLASH_SEPARATOR_STR);
} }
base_dir base_dir
} }
pub struct LazyBuf { /// clean returns the shortest path name equivalent to path
s: String, /// by purely lexical processing. It applies the following rules
buf: Option<Vec<u8>>, /// iteratively until no further processing can be done:
w: usize, ///
} /// 1. Replace multiple slashes with a single slash.
/// 2. Eliminate each . path name element (the current directory).
impl LazyBuf { /// 3. Eliminate each inner .. path name element (the parent directory)
pub fn new(s: String) -> Self { /// along with the non-.. element that precedes it.
LazyBuf { s, buf: None, w: 0 } /// 4. Eliminate .. elements that begin a rooted path,
} /// that is, replace "/.." by "/" at the beginning of a path.
///
pub fn index(&self, i: usize) -> u8 { /// If the result of this process is an empty string, clean returns the string ".".
if let Some(ref buf) = self.buf { ///
buf[i] /// This function is adapted to work cross-platform by using the appropriate path separator.
} else { /// On Windows, this function is aware of drive letters (e.g., `C:`) and UNC paths
self.s.as_bytes()[i] /// (e.g., `\\server\share`) and cleans them using the appropriate separator.
} ///
} /// # Arguments
/// * `path` - A string slice that holds the path to be cleaned.
pub fn append(&mut self, c: u8) { ///
if self.buf.is_none() { /// # Returns
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c { /// A `String` representing the cleaned path.
self.w += 1; ///
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
pub fn clean(path: &str) -> String { pub fn clean(path: &str) -> String {
if path.is_empty() { if path.is_empty() {
return ".".to_string(); return ".".to_string();
} }
#[cfg(target_os = "windows")]
{
use std::borrow::Cow;
let bytes = path.as_bytes();
let n = bytes.len();
// Windows-aware handling
let mut i = 0usize;
let mut drive: Option<char> = None;
let mut rooted = false;
let mut preserve_leading_double_sep = false;
// Drive letter detection
if n >= 2 && bytes[1] == b':' && (bytes[0] as char).is_ascii_alphabetic() {
drive = Some(bytes[0] as char);
i = 2;
// If next is separator, it's an absolute drive-root (e.g., "C:\")
if i < n && is_sep(bytes[i]) {
rooted = true;
// consume all leading separators after drive
while i < n && is_sep(bytes[i]) {
i += 1;
}
}
} else {
// UNC or absolute by separators
if n >= 2 && is_sep(bytes[0]) && is_sep(bytes[1]) {
rooted = true;
preserve_leading_double_sep = true;
i = 2;
// consume extra leading separators
while i < n && is_sep(bytes[i]) {
i += 1;
}
} else if is_sep(bytes[0]) {
rooted = true;
i = 1;
while i < n && is_sep(bytes[i]) {
i += 1;
}
}
}
// Component stack
let mut comps: Vec<Cow<'_, str>> = Vec::new();
let mut r = i;
while r < n {
// find next sep or end
let start = r;
while r < n && !is_sep(bytes[r]) {
r += 1;
}
// component bytes [start..r)
let comp = String::from_utf8_lossy(&bytes[start..r]);
if comp == "." {
// skip
} else if comp == ".." {
if !comps.is_empty() {
// pop last component
comps.pop();
} else if !rooted {
// relative path with .. at front must be kept
comps.push(Cow::Owned("..".to_string()));
} else {
// rooted and at root => ignore
}
} else {
comps.push(comp);
}
// skip separators
while r < n && is_sep(bytes[r]) {
r += 1;
}
}
// Build result
let mut out = String::new();
if let Some(d) = drive {
out.push(d);
out.push(':');
if rooted {
out.push(SLASH_SEPARATOR);
}
} else if preserve_leading_double_sep {
out.push(SLASH_SEPARATOR);
out.push(SLASH_SEPARATOR);
} else if rooted {
out.push(SLASH_SEPARATOR);
}
// Join components
for (idx, c) in comps.iter().enumerate() {
if !out.is_empty() && !out.ends_with(SLASH_SEPARATOR_STR) {
out.push(SLASH_SEPARATOR);
}
out.push_str(c);
}
// Special cases:
if out.is_empty() {
// No drive, no components -> "."
return ".".to_string();
}
// If output is just "C:" (drive without components and not rooted), keep as "C:"
if drive.is_some() {
if out.len() == 2 && out.as_bytes()[1] == b':' {
return out;
}
// If drive+colon+sep and no components, return "C:\"
if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) {
return out;
}
}
// Remove trailing separator unless it's a root form (single leading sep or drive root or UNC)
if out.len() > 1 && out.ends_with(SLASH_SEPARATOR_STR) {
// Determine if it's a root form: "/" or "\\" or "C:\"
let is_root = {
// "/" (non-drive single sep)
if out == SLASH_SEPARATOR_STR {
true
} else if out.starts_with(SLASH_SEPARATOR_STR) && out == format!("{}{}", SLASH_SEPARATOR_STR, SLASH_SEPARATOR_STR)
{
// only double separator
true
} else {
// drive root "C:\" length >=3 with pattern X:\
if out.len() == 3 && out.as_bytes()[1] == b':' && is_sep(out.as_bytes()[2]) {
true
} else {
false
}
}
};
if !is_root {
out.pop();
}
}
out
}
#[cfg(not(target_os = "windows"))]
{
// POSIX-like behavior (original implementation but simplified)
let rooted = path.starts_with('/'); let rooted = path.starts_with('/');
let n = path.len(); let n = path.len();
let mut out = LazyBuf::new(path.to_string()); let mut out = LazyBuf::new(path.to_string());
let mut r = 0; let mut r = 0usize;
let mut dotdot = 0; let mut dotdot = 0usize;
if rooted { if rooted {
out.append(b'/'); out.append(b'/');
@@ -327,11 +675,23 @@ pub fn clean(path: &str) -> String {
} }
out.string() out.string()
}
} }
/// split splits path immediately after the final slash,
/// separating it into a directory and file name component.
/// If there is no slash in path, split returns
/// ("", path).
///
/// # Arguments
/// * `path` - A string slice that holds the path to be split.
///
/// # Returns
/// A tuple containing the directory and file name as string slices.
///
pub fn split(path: &str) -> (&str, &str) { pub fn split(path: &str) -> (&str, &str) {
// Find the last occurrence of the '/' character // Find the last occurrence of the '/' character
if let Some(i) = path.rfind('/') { if let Some(i) = path.rfind(SLASH_SEPARATOR_STR) {
// Return the directory (up to and including the last '/') and the file name // Return the directory (up to and including the last '/') and the file name
return (&path[..i + 1], &path[i + 1..]); return (&path[..i + 1], &path[i + 1..]);
} }
@@ -339,19 +699,168 @@ pub fn split(path: &str) -> (&str, &str) {
(path, "") (path, "")
} }
/// dir returns all but the last element of path,
/// typically the path's directory. After dropping the final
/// element, the path is cleaned. If the path is empty,
/// dir returns ".".
///
/// # Arguments
/// * `path` - A string slice that holds the path to be processed.
///
/// # Returns
/// A `String` representing the directory part of the path.
///
pub fn dir(path: &str) -> String { pub fn dir(path: &str) -> String {
let (a, _) = split(path); let (a, _) = split(path);
clean(a) clean(a)
} }
/// trim_etag removes surrounding double quotes from an ETag string.
///
/// # Arguments
/// * `etag` - A string slice that holds the ETag to be trimmed.
///
/// # Returns
/// A `String` representing the trimmed ETag.
///
pub fn trim_etag(etag: &str) -> String { pub fn trim_etag(etag: &str) -> String {
etag.trim_matches('"').to_string() etag.trim_matches('"').to_string()
} }
/// LazyBuf is a structure that efficiently builds a byte buffer
/// from a string by delaying the allocation of the buffer until
/// a modification is necessary. It allows appending bytes and
/// retrieving the current string representation.
pub struct LazyBuf {
s: String,
buf: Option<Vec<u8>>,
w: usize,
}
impl LazyBuf {
/// Creates a new LazyBuf with the given string.
///
/// # Arguments
/// * `s` - A string to initialize the LazyBuf.
///
/// # Returns
/// A new instance of LazyBuf.
pub fn new(s: String) -> Self {
LazyBuf { s, buf: None, w: 0 }
}
pub fn index(&self, i: usize) -> u8 {
if let Some(ref buf) = self.buf {
buf[i]
} else {
self.s.as_bytes()[i]
}
}
pub fn append(&mut self, c: u8) {
if self.buf.is_none() {
if self.w < self.s.len() && self.s.as_bytes()[self.w] == c {
self.w += 1;
return;
}
let mut new_buf = vec![0; self.s.len()];
new_buf[..self.w].copy_from_slice(&self.s.as_bytes()[..self.w]);
self.buf = Some(new_buf);
}
if let Some(ref mut buf) = self.buf {
buf[self.w] = c;
self.w += 1;
}
}
pub fn string(&self) -> String {
if let Some(ref buf) = self.buf {
String::from_utf8(buf[..self.w].to_vec()).unwrap()
} else {
self.s[..self.w].to_string()
}
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn test_path_join_buf() {
#[cfg(not(target_os = "windows"))]
{
// Basic joining
assert_eq!(path_join_buf(&["a", "b"]), "a/b");
assert_eq!(path_join_buf(&["a/", "b"]), "a/b");
// Empty array input
assert_eq!(path_join_buf(&[]), ".");
// Single element
assert_eq!(path_join_buf(&["a"]), "a");
// Multiple elements
assert_eq!(path_join_buf(&["a", "b", "c"]), "a/b/c");
// Elements with trailing separators
assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/");
// Elements requiring cleaning (with "." and "..")
assert_eq!(path_join_buf(&["a", ".", "b"]), "a/b");
assert_eq!(path_join_buf(&["a", "..", "b"]), "b");
assert_eq!(path_join_buf(&["a", "b", ".."]), "a");
// Preservation of trailing slashes
assert_eq!(path_join_buf(&["a", "b/"]), "a/b/");
assert_eq!(path_join_buf(&["a/", "b/"]), "a/b/");
// Empty elements
assert_eq!(path_join_buf(&["a", "", "b"]), "a/b");
// Double slashes (cleaning)
assert_eq!(path_join_buf(&["a//", "b"]), "a/b");
}
#[cfg(target_os = "windows")]
{
// Basic joining
assert_eq!(path_join_buf(&["a", "b"]), "a\\b");
assert_eq!(path_join_buf(&["a\\", "b"]), "a\\b");
// Empty array input
assert_eq!(path_join_buf(&[]), ".");
// Single element
assert_eq!(path_join_buf(&["a"]), "a");
// Multiple elements
assert_eq!(path_join_buf(&["a", "b", "c"]), "a\\b\\c");
// Elements with trailing separators
assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\");
// Elements requiring cleaning (with "." and "..")
assert_eq!(path_join_buf(&["a", ".", "b"]), "a\\b");
assert_eq!(path_join_buf(&["a", "..", "b"]), "b");
assert_eq!(path_join_buf(&["a", "b", ".."]), "a");
// Mixed separator handling
assert_eq!(path_join_buf(&["a/b", "c"]), "a\\b\\c");
assert_eq!(path_join_buf(&["a\\", "b/c"]), "a\\b\\c");
// Preservation of trailing slashes
assert_eq!(path_join_buf(&["a", "b\\"]), "a\\b\\");
assert_eq!(path_join_buf(&["a\\", "b\\"]), "a\\b\\");
// Empty elements
assert_eq!(path_join_buf(&["a", "", "b"]), "a\\b");
// Double slashes (cleaning)
assert_eq!(path_join_buf(&["a\\\\", "b"]), "a\\b");
}
}
#[test] #[test]
fn test_trim_etag() { fn test_trim_etag() {
// Test with quoted ETag // Test with quoted ETag
@@ -383,6 +892,8 @@ mod tests {
#[test] #[test]
fn test_clean() { fn test_clean() {
#[cfg(not(target_os = "windows"))]
{
assert_eq!(clean(""), "."); assert_eq!(clean(""), ".");
assert_eq!(clean("abc"), "abc"); assert_eq!(clean("abc"), "abc");
assert_eq!(clean("abc/def"), "abc/def"); assert_eq!(clean("abc/def"), "abc/def");
@@ -422,6 +933,17 @@ mod tests {
assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno"); assert_eq!(clean("abc/def/../../../ghi/jkl/../../../mno"), "../../mno");
} }
#[cfg(target_os = "windows")]
{
assert_eq!(clean("a\\b\\..\\c"), "a\\c");
assert_eq!(clean("a\\\\b"), "a\\b");
assert_eq!(clean("C:\\"), "C:\\");
assert_eq!(clean("C:\\a\\..\\b"), "C:\\b");
assert_eq!(clean("C:a\\b\\..\\c"), "C:a\\c");
assert_eq!(clean("\\\\server\\share\\a\\\\b"), "\\\\server\\share\\a\\b");
}
}
#[test] #[test]
fn test_path_needs_clean() { fn test_path_needs_clean() {
struct PathTest { struct PathTest {
+7 -8
View File
@@ -12,11 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
use std::{
collections::HashMap,
io::{Cursor, Read as _, Write as _},
};
use crate::{ use crate::{
admin::{auth::validate_admin_request, router::Operation}, admin::{auth::validate_admin_request, router::Operation},
auth::{check_key_valid, get_session_token}, auth::{check_key_valid, get_session_token},
@@ -49,7 +44,7 @@ use rustfs_policy::policy::{
BucketPolicy, BucketPolicy,
action::{Action, AdminAction}, action::{Action, AdminAction},
}; };
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
use s3s::{ use s3s::{
Body, S3Request, S3Response, S3Result, Body, S3Request, S3Response, S3Result,
dto::{ dto::{
@@ -61,6 +56,10 @@ use s3s::{
}; };
use serde::Deserialize; use serde::Deserialize;
use serde_urlencoded::from_bytes; use serde_urlencoded::from_bytes;
use std::{
collections::HashMap,
io::{Cursor, Read as _, Write as _},
};
use time::OffsetDateTime; use time::OffsetDateTime;
use tracing::warn; use tracing::warn;
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
@@ -424,7 +423,7 @@ impl Operation for ImportBucketMetadata {
// Extract bucket names // Extract bucket names
let mut bucket_names = Vec::new(); let mut bucket_names = Vec::new();
for (file_path, _) in &file_contents { for (file_path, _) in &file_contents {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>(); let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
if file_path_split.len() < 2 { if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path); warn!("file path is invalid: {}", file_path);
@@ -463,7 +462,7 @@ impl Operation for ImportBucketMetadata {
// Second pass: process file contents // Second pass: process file contents
for (file_path, content) in file_contents { for (file_path, content) in file_contents {
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>(); let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
if file_path_split.len() < 2 { if file_path_split.len() < 2 {
warn!("file path is invalid: {}", file_path); warn!("file path is invalid: {}", file_path);