mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-05 21:07:43 +00:00
Fix Windows Path Separator Handling in rustfs_utils (#1464)
Co-authored-by: reatang <tangtang1251@qq.com>
This commit is contained in:
Generated
+169
-172
File diff suppressed because it is too large
Load Diff
+5
-4
@@ -130,7 +130,7 @@ bytesize = "2.3.1"
|
|||||||
byteorder = "1.5.0"
|
byteorder = "1.5.0"
|
||||||
flatbuffers = "25.12.19"
|
flatbuffers = "25.12.19"
|
||||||
form_urlencoded = "1.2.2"
|
form_urlencoded = "1.2.2"
|
||||||
prost = "0.14.1"
|
prost = "0.14.3"
|
||||||
quick-xml = "0.38.4"
|
quick-xml = "0.38.4"
|
||||||
rmcp = { version = "0.12.0" }
|
rmcp = { version = "0.12.0" }
|
||||||
rmp = { version = "0.8.15" }
|
rmp = { version = "0.8.15" }
|
||||||
@@ -143,7 +143,7 @@ schemars = "1.2.0"
|
|||||||
# Cryptography and Security
|
# Cryptography and Security
|
||||||
aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
|
aes-gcm = { version = "0.11.0-rc.2", features = ["rand_core"] }
|
||||||
argon2 = { version = "0.6.0-rc.5" }
|
argon2 = { version = "0.6.0-rc.5" }
|
||||||
blake3 = { version = "1.8.2", features = ["rayon", "mmap"] }
|
blake3 = { version = "1.8.3", features = ["rayon", "mmap"] }
|
||||||
chacha20poly1305 = { version = "0.11.0-rc.2" }
|
chacha20poly1305 = { version = "0.11.0-rc.2" }
|
||||||
crc-fast = "1.6.0"
|
crc-fast = "1.6.0"
|
||||||
hmac = { version = "0.13.0-rc.3" }
|
hmac = { version = "0.13.0-rc.3" }
|
||||||
@@ -184,6 +184,7 @@ criterion = { version = "0.8", features = ["html_reports"] }
|
|||||||
crossbeam-queue = "0.3.12"
|
crossbeam-queue = "0.3.12"
|
||||||
datafusion = "51.0.0"
|
datafusion = "51.0.0"
|
||||||
derive_builder = "0.20.2"
|
derive_builder = "0.20.2"
|
||||||
|
dunce = "1.0.5"
|
||||||
enumset = "1.1.10"
|
enumset = "1.1.10"
|
||||||
faster-hex = "0.10.0"
|
faster-hex = "0.10.0"
|
||||||
flate2 = "1.1.5"
|
flate2 = "1.1.5"
|
||||||
@@ -197,7 +198,7 @@ hex-simd = "0.8.0"
|
|||||||
highway = { version = "1.3.0" }
|
highway = { version = "1.3.0" }
|
||||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||||
lazy_static = "1.5.0"
|
lazy_static = "1.5.0"
|
||||||
libc = "0.2.179"
|
libc = "0.2.180"
|
||||||
libsystemd = "0.7.2"
|
libsystemd = "0.7.2"
|
||||||
local-ip-address = "0.6.8"
|
local-ip-address = "0.6.8"
|
||||||
lz4 = "1.28.1"
|
lz4 = "1.28.1"
|
||||||
@@ -270,7 +271,7 @@ libunftp = "0.21.0"
|
|||||||
russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false }
|
russh = { version = "0.56.0", features = ["aws-lc-rs", "rsa"], default-features = false }
|
||||||
russh-sftp = "2.1.1"
|
russh-sftp = "2.1.1"
|
||||||
ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] }
|
ssh-key = { version = "0.7.0-rc.4", features = ["std", "rsa", "ed25519"] }
|
||||||
suppaftp = { version = "7.0.7", features = ["tokio", "tokio-rustls", "rustls"] }
|
suppaftp = { version = "7.1.0", features = ["tokio", "tokio-rustls", "rustls"] }
|
||||||
rcgen = "0.14.6"
|
rcgen = "0.14.6"
|
||||||
|
|
||||||
# Performance Analysis and Memory Profiling
|
# Performance Analysis and Memory Profiling
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ async-trait.workspace = true
|
|||||||
bytes.workspace = true
|
bytes.workspace = true
|
||||||
byteorder = { workspace = true }
|
byteorder = { workspace = true }
|
||||||
chrono.workspace = true
|
chrono.workspace = true
|
||||||
|
dunce.workspace = true
|
||||||
glob = { workspace = true }
|
glob = { workspace = true }
|
||||||
thiserror.workspace = true
|
thiserror.workspace = true
|
||||||
flatbuffers.workspace = true
|
flatbuffers.workspace = true
|
||||||
@@ -109,7 +110,6 @@ google-cloud-auth = { workspace = true }
|
|||||||
aws-config = { workspace = true }
|
aws-config = { workspace = true }
|
||||||
faster-hex = { workspace = true }
|
faster-hex = { workspace = true }
|
||||||
|
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
|
||||||
criterion = { workspace = true, features = ["html_reports"] }
|
criterion = { workspace = true, features = ["html_reports"] }
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
use crate::disk::RUSTFS_META_BUCKET;
|
use crate::disk::RUSTFS_META_BUCKET;
|
||||||
use crate::error::{Error, Result, StorageError};
|
use crate::error::{Error, Result, StorageError};
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR_STR;
|
||||||
use s3s::xml;
|
use s3s::xml;
|
||||||
|
|
||||||
pub fn is_meta_bucketname(name: &str) -> bool {
|
pub fn is_meta_bucketname(name: &str) -> bool {
|
||||||
@@ -194,7 +194,7 @@ pub fn is_valid_object_name(object: &str) -> bool {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if object.ends_with(SLASH_SEPARATOR) {
|
if object.ends_with(SLASH_SEPARATOR_STR) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,7 +206,7 @@ pub fn check_object_name_for_length_and_slash(bucket: &str, object: &str) -> Res
|
|||||||
return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned()));
|
return Err(StorageError::ObjectNameTooLong(bucket.to_owned(), object.to_owned()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if object.starts_with(SLASH_SEPARATOR) {
|
if object.starts_with(SLASH_SEPARATOR_STR) {
|
||||||
return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned()));
|
return Err(StorageError::ObjectNamePrefixAsSlash(bucket.to_owned(), object.to_owned()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ use crate::error::{Error, Result};
|
|||||||
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
|
use crate::store_api::{ObjectInfo, ObjectOptions, PutObjReader, StorageAPI};
|
||||||
use http::HeaderMap;
|
use http::HeaderMap;
|
||||||
use rustfs_config::DEFAULT_DELIMITER;
|
use rustfs_config::DEFAULT_DELIMITER;
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR_STR;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
@@ -29,7 +29,7 @@ const CONFIG_FILE: &str = "config.json";
|
|||||||
|
|
||||||
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
|
pub const STORAGE_CLASS_SUB_SYS: &str = "storage_class";
|
||||||
|
|
||||||
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{CONFIG_PREFIX}"));
|
static CONFIG_BUCKET: LazyLock<String> = LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR_STR}{CONFIG_PREFIX}"));
|
||||||
|
|
||||||
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
static SUB_SYSTEMS_DYNAMIC: LazyLock<HashSet<String>> = LazyLock::new(|| {
|
||||||
let mut h = HashSet::new();
|
let mut h = HashSet::new();
|
||||||
@@ -129,7 +129,7 @@ async fn new_and_save_server_config<S: StorageAPI>(api: Arc<S>) -> Result<Config
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn get_config_file() -> String {
|
fn get_config_file() -> String {
|
||||||
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR}{CONFIG_FILE}")
|
format!("{CONFIG_PREFIX}{SLASH_SEPARATOR_STR}{CONFIG_FILE}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
/// Handle the situation where the configuration file does not exist, create and save a new configuration
|
||||||
|
|||||||
@@ -31,14 +31,14 @@ use crate::{
|
|||||||
use rustfs_common::data_usage::{
|
use rustfs_common::data_usage::{
|
||||||
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
|
BucketTargetUsageInfo, BucketUsageInfo, DataUsageCache, DataUsageEntry, DataUsageInfo, DiskUsageStatus, SizeSummary,
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR_STR;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
|
|
||||||
// Data usage storage constants
|
// Data usage storage constants
|
||||||
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR_STR;
|
||||||
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
|
const DATA_USAGE_OBJ_NAME: &str = ".usage.json";
|
||||||
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
|
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
|
||||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||||
@@ -47,17 +47,17 @@ pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
|||||||
lazy_static::lazy_static! {
|
lazy_static::lazy_static! {
|
||||||
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
|
pub static ref DATA_USAGE_BUCKET: String = format!("{}{}{}",
|
||||||
crate::disk::RUSTFS_META_BUCKET,
|
crate::disk::RUSTFS_META_BUCKET,
|
||||||
SLASH_SEPARATOR,
|
SLASH_SEPARATOR_STR,
|
||||||
crate::disk::BUCKET_META_PREFIX
|
crate::disk::BUCKET_META_PREFIX
|
||||||
);
|
);
|
||||||
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
|
pub static ref DATA_USAGE_OBJ_NAME_PATH: String = format!("{}{}{}",
|
||||||
crate::disk::BUCKET_META_PREFIX,
|
crate::disk::BUCKET_META_PREFIX,
|
||||||
SLASH_SEPARATOR,
|
SLASH_SEPARATOR_STR,
|
||||||
DATA_USAGE_OBJ_NAME
|
DATA_USAGE_OBJ_NAME
|
||||||
);
|
);
|
||||||
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
|
pub static ref DATA_USAGE_BLOOM_NAME_PATH: String = format!("{}{}{}",
|
||||||
crate::disk::BUCKET_META_PREFIX,
|
crate::disk::BUCKET_META_PREFIX,
|
||||||
SLASH_SEPARATOR,
|
SLASH_SEPARATOR_STR,
|
||||||
DATA_USAGE_BLOOM_NAME
|
DATA_USAGE_BLOOM_NAME
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,39 +12,26 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use super::error::{Error, Result};
|
|
||||||
use super::os::{is_root_disk, rename_all};
|
|
||||||
use super::{
|
|
||||||
BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
|
|
||||||
FileInfoVersions, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp,
|
|
||||||
STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os,
|
|
||||||
};
|
|
||||||
use super::{endpoint::Endpoint, error::DiskError, format::FormatV3};
|
|
||||||
|
|
||||||
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
|
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
|
||||||
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
|
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
|
||||||
use crate::disk::error::FileAccessDeniedWithContext;
|
|
||||||
use crate::disk::error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error};
|
|
||||||
use crate::disk::fs::{
|
|
||||||
O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename,
|
|
||||||
};
|
|
||||||
use crate::disk::os::{check_path_length, is_empty_dir};
|
|
||||||
use crate::disk::{
|
use crate::disk::{
|
||||||
CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN, CHECK_PART_VOLUME_NOT_FOUND,
|
BUCKET_META_PREFIX, CHECK_PART_FILE_CORRUPT, CHECK_PART_FILE_NOT_FOUND, CHECK_PART_SUCCESS, CHECK_PART_UNKNOWN,
|
||||||
FileReader, RUSTFS_META_TMP_DELETED_BUCKET, conv_part_err_to_int,
|
CHECK_PART_VOLUME_NOT_FOUND, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics,
|
||||||
|
FileInfoVersions, FileReader, FileWriter, RUSTFS_META_BUCKET, RUSTFS_META_TMP_DELETED_BUCKET, ReadMultipleReq,
|
||||||
|
ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts,
|
||||||
|
VolumeInfo, WalkDirOptions, conv_part_err_to_int,
|
||||||
|
endpoint::Endpoint,
|
||||||
|
error::{DiskError, Error, FileAccessDeniedWithContext, Result},
|
||||||
|
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
|
||||||
|
format::FormatV3,
|
||||||
|
fs::{O_APPEND, O_CREATE, O_RDONLY, O_TRUNC, O_WRONLY, access, lstat, lstat_std, remove, remove_all_std, remove_std, rename},
|
||||||
|
os,
|
||||||
|
os::{check_path_length, is_empty_dir, is_root_disk, rename_all},
|
||||||
};
|
};
|
||||||
use crate::disk::{FileWriter, STORAGE_FORMAT_FILE};
|
|
||||||
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
|
|
||||||
use rustfs_utils::path::{
|
|
||||||
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix,
|
|
||||||
path_join, path_join_buf,
|
|
||||||
};
|
|
||||||
use tokio::time::interval;
|
|
||||||
|
|
||||||
use crate::erasure_coding::bitrot_verify;
|
use crate::erasure_coding::bitrot_verify;
|
||||||
use bytes::Bytes;
|
|
||||||
// use path_absolutize::Absolutize; // Replaced with direct path operations for better performance
|
|
||||||
use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
|
use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
|
||||||
|
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
|
||||||
|
use bytes::Bytes;
|
||||||
use parking_lot::RwLock as ParkingLotRwLock;
|
use parking_lot::RwLock as ParkingLotRwLock;
|
||||||
use rustfs_filemeta::{
|
use rustfs_filemeta::{
|
||||||
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
|
Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
|
||||||
@@ -52,6 +39,10 @@ use rustfs_filemeta::{
|
|||||||
};
|
};
|
||||||
use rustfs_utils::HashAlgorithm;
|
use rustfs_utils::HashAlgorithm;
|
||||||
use rustfs_utils::os::get_info;
|
use rustfs_utils::os::get_info;
|
||||||
|
use rustfs_utils::path::{
|
||||||
|
GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR_STR, clean, decode_dir_object, encode_dir_object,
|
||||||
|
has_suffix, path_join, path_join_buf,
|
||||||
|
};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
use std::fmt::Debug;
|
use std::fmt::Debug;
|
||||||
@@ -67,6 +58,7 @@ use time::OffsetDateTime;
|
|||||||
use tokio::fs::{self, File};
|
use tokio::fs::{self, File};
|
||||||
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
|
use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWrite, AsyncWriteExt, ErrorKind};
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::RwLock;
|
||||||
|
use tokio::time::interval;
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
@@ -129,7 +121,8 @@ impl LocalDisk {
|
|||||||
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
|
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
|
||||||
debug!("Creating local disk");
|
debug!("Creating local disk");
|
||||||
// Use optimized path resolution instead of absolutize() for better performance
|
// Use optimized path resolution instead of absolutize() for better performance
|
||||||
let root = match std::fs::canonicalize(ep.get_file_path()) {
|
// Use dunce::canonicalize instead of std::fs::canonicalize to avoid UNC paths on Windows
|
||||||
|
let root = match dunce::canonicalize(ep.get_file_path()) {
|
||||||
Ok(path) => path,
|
Ok(path) => path,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if e.kind() == ErrorKind::NotFound {
|
if e.kind() == ErrorKind::NotFound {
|
||||||
@@ -483,7 +476,7 @@ impl LocalDisk {
|
|||||||
|
|
||||||
// Async prefetch related files, don't block current read
|
// Async prefetch related files, don't block current read
|
||||||
if let Some(parent) = file_path.parent() {
|
if let Some(parent) = file_path.parent() {
|
||||||
prefetch_metadata_patterns(parent, &[super::STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
|
prefetch_metadata_patterns(parent, &[STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Main read logic
|
// Main read logic
|
||||||
@@ -507,7 +500,7 @@ impl LocalDisk {
|
|||||||
async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
|
async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
|
||||||
let paths: Vec<PathBuf> = requests
|
let paths: Vec<PathBuf> = requests
|
||||||
.iter()
|
.iter()
|
||||||
.map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, super::STORAGE_FORMAT_FILE)))
|
.map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, STORAGE_FORMAT_FILE)))
|
||||||
.collect::<Result<Vec<_>>>()?;
|
.collect::<Result<Vec<_>>>()?;
|
||||||
|
|
||||||
let cache = get_global_file_cache();
|
let cache = get_global_file_cache();
|
||||||
@@ -544,7 +537,7 @@ impl LocalDisk {
|
|||||||
|
|
||||||
// TODO: async notifications for disk space checks and trash cleanup
|
// TODO: async notifications for disk space checks and trash cleanup
|
||||||
|
|
||||||
let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
|
let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
|
||||||
// if let Some(parent) = trash_path.parent() {
|
// if let Some(parent) = trash_path.parent() {
|
||||||
// if !parent.exists() {
|
// if !parent.exists() {
|
||||||
// fs::create_dir_all(parent).await?;
|
// fs::create_dir_all(parent).await?;
|
||||||
@@ -552,7 +545,7 @@ impl LocalDisk {
|
|||||||
// }
|
// }
|
||||||
|
|
||||||
let err = if recursive {
|
let err = if recursive {
|
||||||
rename_all(delete_path, trash_path, self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?)
|
rename_all(delete_path, trash_path, self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?)
|
||||||
.await
|
.await
|
||||||
.err()
|
.err()
|
||||||
} else {
|
} else {
|
||||||
@@ -562,12 +555,12 @@ impl LocalDisk {
|
|||||||
.err()
|
.err()
|
||||||
};
|
};
|
||||||
|
|
||||||
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR) {
|
if immediate_purge || delete_path.to_string_lossy().ends_with(SLASH_SEPARATOR_STR) {
|
||||||
let trash_path2 = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
|
let trash_path2 = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
|
||||||
let _ = rename_all(
|
let _ = rename_all(
|
||||||
encode_dir_object(delete_path.to_string_lossy().as_ref()),
|
encode_dir_object(delete_path.to_string_lossy().as_ref()),
|
||||||
trash_path2,
|
trash_path2,
|
||||||
self.get_bucket_path(super::RUSTFS_META_TMP_DELETED_BUCKET)?,
|
self.get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET)?,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -916,7 +909,7 @@ impl LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if let Some(parent) = path.as_ref().parent() {
|
if let Some(parent) = path.as_ref().parent() {
|
||||||
super::os::make_dir_all(parent, skip_parent).await?;
|
os::make_dir_all(parent, skip_parent).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
|
let f = super::fs::open_file(path.as_ref(), mode).await.map_err(to_file_error)?;
|
||||||
@@ -942,7 +935,7 @@ impl LocalDisk {
|
|||||||
let meta = file.metadata().await.map_err(to_file_error)?;
|
let meta = file.metadata().await.map_err(to_file_error)?;
|
||||||
let file_size = meta.len() as usize;
|
let file_size = meta.len() as usize;
|
||||||
|
|
||||||
bitrot_verify(Box::new(file), file_size, part_size, algo, bytes::Bytes::copy_from_slice(sum), shard_size)
|
bitrot_verify(Box::new(file), file_size, part_size, algo, Bytes::copy_from_slice(sum), shard_size)
|
||||||
.await
|
.await
|
||||||
.map_err(to_file_error)?;
|
.map_err(to_file_error)?;
|
||||||
|
|
||||||
@@ -1038,15 +1031,16 @@ impl LocalDisk {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if entry.ends_with(SLASH_SEPARATOR) {
|
if entry.ends_with(SLASH_SEPARATOR_STR) {
|
||||||
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
|
if entry.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
|
||||||
let entry = format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
|
let entry =
|
||||||
|
format!("{}{}", entry.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR_STR);
|
||||||
dir_objes.insert(entry.clone());
|
dir_objes.insert(entry.clone());
|
||||||
*item = entry;
|
*item = entry;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
*item = entry.trim_end_matches(SLASH_SEPARATOR).to_owned();
|
*item = entry.trim_end_matches(SLASH_SEPARATOR_STR).to_owned();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1058,7 +1052,7 @@ impl LocalDisk {
|
|||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
|
let entry = entry.strip_suffix(STORAGE_FORMAT_FILE).unwrap_or_default().to_owned();
|
||||||
let name = entry.trim_end_matches(SLASH_SEPARATOR);
|
let name = entry.trim_end_matches(SLASH_SEPARATOR_STR);
|
||||||
let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str());
|
let name = decode_dir_object(format!("{}/{}", ¤t, &name).as_str());
|
||||||
|
|
||||||
// if opts.limit > 0
|
// if opts.limit > 0
|
||||||
@@ -1141,7 +1135,7 @@ impl LocalDisk {
|
|||||||
Ok(res) => {
|
Ok(res) => {
|
||||||
if is_dir_obj {
|
if is_dir_obj {
|
||||||
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
|
meta.name = meta.name.trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH).to_owned();
|
||||||
meta.name.push_str(SLASH_SEPARATOR);
|
meta.name.push_str(SLASH_SEPARATOR_STR);
|
||||||
}
|
}
|
||||||
|
|
||||||
meta.metadata = res;
|
meta.metadata = res;
|
||||||
@@ -1159,7 +1153,7 @@ impl LocalDisk {
|
|||||||
// NOT an object, append to stack (with slash)
|
// NOT an object, append to stack (with slash)
|
||||||
// If dirObject, but no metadata (which is unexpected) we skip it.
|
// If dirObject, but no metadata (which is unexpected) we skip it.
|
||||||
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
|
if !is_dir_obj && !is_empty_dir(self.get_object_path(&opts.bucket, &meta.name)?).await {
|
||||||
meta.name.push_str(SLASH_SEPARATOR);
|
meta.name.push_str(SLASH_SEPARATOR_STR);
|
||||||
dir_stack.push(meta.name);
|
dir_stack.push(meta.name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1234,7 +1228,7 @@ async fn read_file_metadata(p: impl AsRef<Path>) -> Result<Metadata> {
|
|||||||
|
|
||||||
fn skip_access_checks(p: impl AsRef<str>) -> bool {
|
fn skip_access_checks(p: impl AsRef<str>) -> bool {
|
||||||
let vols = [
|
let vols = [
|
||||||
super::RUSTFS_META_TMP_DELETED_BUCKET,
|
RUSTFS_META_TMP_DELETED_BUCKET,
|
||||||
super::RUSTFS_META_TMP_BUCKET,
|
super::RUSTFS_META_TMP_BUCKET,
|
||||||
super::RUSTFS_META_MULTIPART_BUCKET,
|
super::RUSTFS_META_MULTIPART_BUCKET,
|
||||||
RUSTFS_META_BUCKET,
|
RUSTFS_META_BUCKET,
|
||||||
@@ -1628,8 +1622,8 @@ impl DiskAPI for LocalDisk {
|
|||||||
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
super::fs::access_std(&dst_volume_dir).map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?
|
||||||
}
|
}
|
||||||
|
|
||||||
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
|
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
|
||||||
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
|
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
|
||||||
|
|
||||||
if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
|
if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -1695,8 +1689,8 @@ impl DiskAPI for LocalDisk {
|
|||||||
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
.map_err(|e| to_access_error(e, DiskError::VolumeAccessDenied))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR);
|
let src_is_dir = has_suffix(src_path, SLASH_SEPARATOR_STR);
|
||||||
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR);
|
let dst_is_dir = has_suffix(dst_path, SLASH_SEPARATOR_STR);
|
||||||
if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) {
|
if (dst_is_dir || src_is_dir) && (!dst_is_dir || !src_is_dir) {
|
||||||
return Err(Error::from(DiskError::FileAccessDenied));
|
return Err(Error::from(DiskError::FileAccessDenied));
|
||||||
}
|
}
|
||||||
@@ -1847,12 +1841,12 @@ impl DiskAPI for LocalDisk {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let volume_dir = self.get_bucket_path(volume)?;
|
let volume_dir = self.get_bucket_path(volume)?;
|
||||||
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR))?;
|
let dir_path_abs = self.get_object_path(volume, dir_path.trim_start_matches(SLASH_SEPARATOR_STR))?;
|
||||||
|
|
||||||
let entries = match os::read_dir(&dir_path_abs, count).await {
|
let entries = match os::read_dir(&dir_path_abs, count).await {
|
||||||
Ok(res) => res,
|
Ok(res) => res,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if e.kind() == std::io::ErrorKind::NotFound
|
if e.kind() == ErrorKind::NotFound
|
||||||
&& !skip_access_checks(volume)
|
&& !skip_access_checks(volume)
|
||||||
&& let Err(e) = access(&volume_dir).await
|
&& let Err(e) = access(&volume_dir).await
|
||||||
{
|
{
|
||||||
@@ -1883,11 +1877,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
|
|
||||||
let mut objs_returned = 0;
|
let mut objs_returned = 0;
|
||||||
|
|
||||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
if opts.base_dir.ends_with(SLASH_SEPARATOR_STR) {
|
||||||
let fpath = self.get_object_path(
|
let fpath = self.get_object_path(
|
||||||
&opts.bucket,
|
&opts.bucket,
|
||||||
path_join_buf(&[
|
path_join_buf(&[
|
||||||
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), GLOBAL_DIR_SUFFIX).as_str(),
|
format!("{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR_STR), GLOBAL_DIR_SUFFIX).as_str(),
|
||||||
STORAGE_FORMAT_FILE,
|
STORAGE_FORMAT_FILE,
|
||||||
])
|
])
|
||||||
.as_str(),
|
.as_str(),
|
||||||
@@ -2119,7 +2113,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
let volume_dir = self.get_bucket_path(volume)?;
|
let volume_dir = self.get_bucket_path(volume)?;
|
||||||
|
|
||||||
if let Err(e) = access(&volume_dir).await {
|
if let Err(e) = access(&volume_dir).await {
|
||||||
if e.kind() == std::io::ErrorKind::NotFound {
|
if e.kind() == ErrorKind::NotFound {
|
||||||
os::make_dir_all(&volume_dir, self.root.as_path()).await?;
|
os::make_dir_all(&volume_dir, self.root.as_path()).await?;
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
@@ -2137,7 +2131,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?;
|
let entries = os::read_dir(&self.root, -1).await.map_err(to_volume_error)?;
|
||||||
|
|
||||||
for entry in entries {
|
for entry in entries {
|
||||||
if !has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(clean(&entry).as_str()) {
|
if !has_suffix(&entry, SLASH_SEPARATOR_STR) || !Self::is_valid_volname(clean(&entry).as_str()) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2359,7 +2353,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
force_del_marker: bool,
|
force_del_marker: bool,
|
||||||
opts: DeleteOptions,
|
opts: DeleteOptions,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if path.starts_with(SLASH_SEPARATOR) {
|
if path.starts_with(SLASH_SEPARATOR_STR) {
|
||||||
return self
|
return self
|
||||||
.delete(
|
.delete(
|
||||||
volume,
|
volume,
|
||||||
@@ -2420,7 +2414,7 @@ impl DiskAPI for LocalDisk {
|
|||||||
if !meta.versions.is_empty() {
|
if !meta.versions.is_empty() {
|
||||||
let buf = meta.marshal_msg()?;
|
let buf = meta.marshal_msg()?;
|
||||||
return self
|
return self
|
||||||
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
|
.write_all_meta(volume, format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str(), &buf, true)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2430,11 +2424,11 @@ impl DiskAPI for LocalDisk {
|
|||||||
{
|
{
|
||||||
let src_path = path_join(&[
|
let src_path = path_join(&[
|
||||||
file_path.as_path(),
|
file_path.as_path(),
|
||||||
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
|
Path::new(format!("{old_data_dir}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE_BACKUP}").as_str()),
|
||||||
]);
|
]);
|
||||||
let dst_path = path_join(&[
|
let dst_path = path_join(&[
|
||||||
file_path.as_path(),
|
file_path.as_path(),
|
||||||
Path::new(format!("{path}{SLASH_SEPARATOR}{STORAGE_FORMAT_FILE}").as_str()),
|
Path::new(format!("{path}{SLASH_SEPARATOR_STR}{STORAGE_FORMAT_FILE}").as_str()),
|
||||||
]);
|
]);
|
||||||
return rename_all(src_path, dst_path, file_path).await;
|
return rename_all(src_path, dst_path, file_path).await;
|
||||||
}
|
}
|
||||||
@@ -2563,7 +2557,7 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
|
|||||||
if root_disk_threshold > 0 {
|
if root_disk_threshold > 0 {
|
||||||
disk_info.total <= root_disk_threshold
|
disk_info.total <= root_disk_threshold
|
||||||
} else {
|
} else {
|
||||||
is_root_disk(&drive_path, SLASH_SEPARATOR).unwrap_or_default()
|
is_root_disk(&drive_path, SLASH_SEPARATOR_STR).unwrap_or_default()
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
false
|
false
|
||||||
@@ -2581,7 +2575,7 @@ mod test {
|
|||||||
// let arr = Vec::new();
|
// let arr = Vec::new();
|
||||||
|
|
||||||
let vols = [
|
let vols = [
|
||||||
super::super::RUSTFS_META_TMP_DELETED_BUCKET,
|
RUSTFS_META_TMP_DELETED_BUCKET,
|
||||||
super::super::RUSTFS_META_TMP_BUCKET,
|
super::super::RUSTFS_META_TMP_BUCKET,
|
||||||
super::super::RUSTFS_META_MULTIPART_BUCKET,
|
super::super::RUSTFS_META_MULTIPART_BUCKET,
|
||||||
RUSTFS_META_BUCKET,
|
RUSTFS_META_BUCKET,
|
||||||
@@ -2609,9 +2603,7 @@ mod test {
|
|||||||
|
|
||||||
let disk = LocalDisk::new(&ep, false).await.unwrap();
|
let disk = LocalDisk::new(&ep, false).await.unwrap();
|
||||||
|
|
||||||
let tmpp = disk
|
let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
|
||||||
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
println!("ppp :{:?}", &tmpp);
|
println!("ppp :{:?}", &tmpp);
|
||||||
|
|
||||||
@@ -2639,9 +2631,7 @@ mod test {
|
|||||||
|
|
||||||
let disk = LocalDisk::new(&ep, false).await.unwrap();
|
let disk = LocalDisk::new(&ep, false).await.unwrap();
|
||||||
|
|
||||||
let tmpp = disk
|
let tmpp = disk.resolve_abs_path(Path::new(RUSTFS_META_TMP_DELETED_BUCKET)).unwrap();
|
||||||
.resolve_abs_path(Path::new(super::super::RUSTFS_META_TMP_DELETED_BUCKET))
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
println!("ppp :{:?}", &tmpp);
|
println!("ppp :{:?}", &tmpp);
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use std::{
|
|||||||
|
|
||||||
use super::error::Result;
|
use super::error::Result;
|
||||||
use crate::disk::error_conv::to_file_error;
|
use crate::disk::error_conv::to_file_error;
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR_STR;
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
|
|
||||||
@@ -118,7 +118,7 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
|
|||||||
if file_type.is_file() {
|
if file_type.is_file() {
|
||||||
volumes.push(name);
|
volumes.push(name);
|
||||||
} else if file_type.is_dir() {
|
} else if file_type.is_dir() {
|
||||||
volumes.push(format!("{name}{SLASH_SEPARATOR}"));
|
volumes.push(format!("{name}{SLASH_SEPARATOR_STR}"));
|
||||||
}
|
}
|
||||||
count -= 1;
|
count -= 1;
|
||||||
if count == 0 {
|
if count == 0 {
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ use rustfs_common::defer;
|
|||||||
use rustfs_common::heal_channel::HealOpts;
|
use rustfs_common::heal_channel::HealOpts;
|
||||||
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
|
||||||
use rustfs_rio::{HashReader, WarpReader};
|
use rustfs_rio::{HashReader, WarpReader};
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join};
|
use rustfs_utils::path::{SLASH_SEPARATOR_STR, encode_dir_object, path_join};
|
||||||
use rustfs_workers::workers::Workers;
|
use rustfs_workers::workers::Workers;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -451,10 +451,10 @@ fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, S
|
|||||||
let trimmed_path = path
|
let trimmed_path = path
|
||||||
.strip_prefix(base_path)
|
.strip_prefix(base_path)
|
||||||
.unwrap_or(path)
|
.unwrap_or(path)
|
||||||
.strip_prefix(SLASH_SEPARATOR)
|
.strip_prefix(SLASH_SEPARATOR_STR)
|
||||||
.unwrap_or(path);
|
.unwrap_or(path);
|
||||||
// Find the position of the first '/'
|
// Find the position of the first '/'
|
||||||
let pos = trimmed_path.find(SLASH_SEPARATOR).unwrap_or(trimmed_path.len());
|
let pos = trimmed_path.find(SLASH_SEPARATOR_STR).unwrap_or(trimmed_path.len());
|
||||||
// Split into bucket and prefix
|
// Split into bucket and prefix
|
||||||
let bucket = &trimmed_path[0..pos];
|
let bucket = &trimmed_path[0..pos];
|
||||||
let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists
|
let prefix = &trimmed_path[pos + 1..]; // +1 to skip the '/' character if it exists
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX,
|
|||||||
use rustfs_utils::{
|
use rustfs_utils::{
|
||||||
HashAlgorithm,
|
HashAlgorithm,
|
||||||
crypto::hex,
|
crypto::hex,
|
||||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
|
path::{SLASH_SEPARATOR_STR, encode_dir_object, has_suffix, path_join_buf},
|
||||||
};
|
};
|
||||||
use rustfs_workers::workers::Workers;
|
use rustfs_workers::workers::Workers;
|
||||||
use s3s::header::X_AMZ_RESTORE;
|
use s3s::header::X_AMZ_RESTORE;
|
||||||
@@ -5324,7 +5324,7 @@ impl StorageAPI for SetDisks {
|
|||||||
&upload_id_path,
|
&upload_id_path,
|
||||||
fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(),
|
fi.data_dir.map(|v| v.to_string()).unwrap_or_default().as_str(),
|
||||||
]),
|
]),
|
||||||
SLASH_SEPARATOR
|
SLASH_SEPARATOR_STR
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
|
let mut part_numbers = match Self::list_parts(&online_disks, &part_path, read_quorum).await {
|
||||||
@@ -5462,7 +5462,7 @@ impl StorageAPI for SetDisks {
|
|||||||
let mut populated_upload_ids = HashSet::new();
|
let mut populated_upload_ids = HashSet::new();
|
||||||
|
|
||||||
for upload_id in upload_ids.iter() {
|
for upload_id in upload_ids.iter() {
|
||||||
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR).to_string();
|
let upload_id = upload_id.trim_end_matches(SLASH_SEPARATOR_STR).to_string();
|
||||||
if populated_upload_ids.contains(&upload_id) {
|
if populated_upload_ids.contains(&upload_id) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -6222,7 +6222,7 @@ impl StorageAPI for SetDisks {
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
if has_suffix(object, SLASH_SEPARATOR) {
|
if has_suffix(object, SLASH_SEPARATOR_STR) {
|
||||||
let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?;
|
let (result, err) = self.heal_object_dir_locked(bucket, object, opts.dry_run, opts.remove).await?;
|
||||||
return Ok((result, err.map(|e| e.into())));
|
return Ok((result, err.map(|e| e.into())));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ use rustfs_filemeta::{
|
|||||||
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
|
MetaCacheEntries, MetaCacheEntriesSorted, MetaCacheEntriesSortedResult, MetaCacheEntry, MetadataResolutionParams,
|
||||||
merge_file_meta_versions,
|
merge_file_meta_versions,
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::{self, SLASH_SEPARATOR, base_dir_from_prefix};
|
use rustfs_utils::path::{self, SLASH_SEPARATOR_STR, base_dir_from_prefix};
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::broadcast::{self};
|
use tokio::sync::broadcast::{self};
|
||||||
@@ -132,7 +132,7 @@ impl ListPathOptions {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let s = SLASH_SEPARATOR.chars().next().unwrap_or_default();
|
let s = SLASH_SEPARATOR_STR.chars().next().unwrap_or_default();
|
||||||
self.filter_prefix = {
|
self.filter_prefix = {
|
||||||
let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s);
|
let fp = self.prefix.trim_start_matches(&self.base_dir).trim_matches(s);
|
||||||
|
|
||||||
@@ -346,7 +346,7 @@ impl ECStore {
|
|||||||
if let Some(delimiter) = &delimiter {
|
if let Some(delimiter) = &delimiter {
|
||||||
if obj.is_dir && obj.mod_time.is_none() {
|
if obj.is_dir && obj.mod_time.is_none() {
|
||||||
let mut found = false;
|
let mut found = false;
|
||||||
if delimiter != SLASH_SEPARATOR {
|
if delimiter != SLASH_SEPARATOR_STR {
|
||||||
for p in prefixes.iter() {
|
for p in prefixes.iter() {
|
||||||
if found {
|
if found {
|
||||||
break;
|
break;
|
||||||
@@ -470,7 +470,7 @@ impl ECStore {
|
|||||||
if let Some(delimiter) = &delimiter {
|
if let Some(delimiter) = &delimiter {
|
||||||
if obj.is_dir && obj.mod_time.is_none() {
|
if obj.is_dir && obj.mod_time.is_none() {
|
||||||
let mut found = false;
|
let mut found = false;
|
||||||
if delimiter != SLASH_SEPARATOR {
|
if delimiter != SLASH_SEPARATOR_STR {
|
||||||
for p in prefixes.iter() {
|
for p in prefixes.iter() {
|
||||||
if found {
|
if found {
|
||||||
break;
|
break;
|
||||||
@@ -502,7 +502,7 @@ impl ECStore {
|
|||||||
// warn!("list_path opt {:?}", &o);
|
// warn!("list_path opt {:?}", &o);
|
||||||
|
|
||||||
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
|
check_list_objs_args(&o.bucket, &o.prefix, &o.marker)?;
|
||||||
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
|
// if opts.prefix.ends_with(SLASH_SEPARATOR_STR) {
|
||||||
// return Err(Error::msg("eof"));
|
// return Err(Error::msg("eof"));
|
||||||
// }
|
// }
|
||||||
|
|
||||||
@@ -520,11 +520,11 @@ impl ECStore {
|
|||||||
return Err(Error::Unexpected);
|
return Err(Error::Unexpected);
|
||||||
}
|
}
|
||||||
|
|
||||||
if o.prefix.starts_with(SLASH_SEPARATOR) {
|
if o.prefix.starts_with(SLASH_SEPARATOR_STR) {
|
||||||
return Err(Error::Unexpected);
|
return Err(Error::Unexpected);
|
||||||
}
|
}
|
||||||
|
|
||||||
let slash_separator = Some(SLASH_SEPARATOR.to_owned());
|
let slash_separator = Some(SLASH_SEPARATOR_STR.to_owned());
|
||||||
|
|
||||||
o.include_directories = o.separator == slash_separator;
|
o.include_directories = o.separator == slash_separator;
|
||||||
|
|
||||||
@@ -774,8 +774,8 @@ impl ECStore {
|
|||||||
let mut filter_prefix = {
|
let mut filter_prefix = {
|
||||||
prefix
|
prefix
|
||||||
.trim_start_matches(&path)
|
.trim_start_matches(&path)
|
||||||
.trim_start_matches(SLASH_SEPARATOR)
|
.trim_start_matches(SLASH_SEPARATOR_STR)
|
||||||
.trim_end_matches(SLASH_SEPARATOR)
|
.trim_end_matches(SLASH_SEPARATOR_STR)
|
||||||
.to_owned()
|
.to_owned()
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1130,7 +1130,7 @@ async fn merge_entry_channels(
|
|||||||
if path::clean(&best_entry.name) == path::clean(&other_entry.name) {
|
if path::clean(&best_entry.name) == path::clean(&other_entry.name) {
|
||||||
let dir_matches = best_entry.is_dir() && other_entry.is_dir();
|
let dir_matches = best_entry.is_dir() && other_entry.is_dir();
|
||||||
let suffix_matches =
|
let suffix_matches =
|
||||||
best_entry.name.ends_with(SLASH_SEPARATOR) == other_entry.name.ends_with(SLASH_SEPARATOR);
|
best_entry.name.ends_with(SLASH_SEPARATOR_STR) == other_entry.name.ends_with(SLASH_SEPARATOR_STR);
|
||||||
|
|
||||||
if dir_matches && suffix_matches {
|
if dir_matches && suffix_matches {
|
||||||
to_merge.push(other_idx);
|
to_merge.push(other_idx);
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ use crate::{
|
|||||||
store_api::{ObjectOptions, PutObjReader},
|
store_api::{ObjectOptions, PutObjReader},
|
||||||
};
|
};
|
||||||
use rustfs_rio::HashReader;
|
use rustfs_rio::HashReader;
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join};
|
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join};
|
||||||
use s3s::S3ErrorCode;
|
use s3s::S3ErrorCode;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
@@ -403,7 +403,7 @@ impl TierConfigMgr {
|
|||||||
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
|
pub async fn save_tiering_config<S: StorageAPI>(&self, api: Arc<S>) -> std::result::Result<(), std::io::Error> {
|
||||||
let data = self.marshal()?;
|
let data = self.marshal()?;
|
||||||
|
|
||||||
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
|
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
|
||||||
|
|
||||||
self.save_config(api, &config_file, data).await
|
self.save_config(api, &config_file, data).await
|
||||||
}
|
}
|
||||||
@@ -483,7 +483,7 @@ async fn new_and_save_tiering_config<S: StorageAPI>(api: Arc<S>) -> Result<TierC
|
|||||||
|
|
||||||
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
|
#[tracing::instrument(level = "debug", name = "load_tier_config", skip(api))]
|
||||||
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
|
async fn load_tier_config(api: Arc<ECStore>) -> std::result::Result<TierConfigMgr, std::io::Error> {
|
||||||
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR, TIER_CONFIG_FILE);
|
let config_file = format!("{}{}{}", CONFIG_PREFIX, SLASH_SEPARATOR_STR, TIER_CONFIG_FILE);
|
||||||
let data = read_config(api.clone(), config_file.as_str()).await;
|
let data = read_config(api.clone(), config_file.as_str()).await;
|
||||||
if let Err(err) = data {
|
if let Err(err) = data {
|
||||||
if is_err_config_not_found(&err) {
|
if is_err_config_not_found(&err) {
|
||||||
|
|||||||
@@ -30,13 +30,11 @@ use crate::client::{
|
|||||||
transition_api::{Options, TransitionClient, TransitionCore},
|
transition_api::{Options, TransitionClient, TransitionCore},
|
||||||
transition_api::{ReadCloser, ReaderImpl},
|
transition_api::{ReadCloser, ReaderImpl},
|
||||||
};
|
};
|
||||||
use crate::error::ErrorResponse;
|
|
||||||
use crate::error::error_resp_to_object_err;
|
|
||||||
use crate::tier::{
|
use crate::tier::{
|
||||||
tier_config::TierS3,
|
tier_config::TierS3,
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
use rustfs_utils::path::SLASH_SEPARATOR_STR;
|
||||||
|
|
||||||
pub struct WarmBackendS3 {
|
pub struct WarmBackendS3 {
|
||||||
pub client: Arc<TransitionClient>,
|
pub client: Arc<TransitionClient>,
|
||||||
@@ -178,7 +176,7 @@ impl WarmBackend for WarmBackendS3 {
|
|||||||
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
async fn in_use(&self) -> Result<bool, std::io::Error> {
|
||||||
let result = self
|
let result = self
|
||||||
.core
|
.core
|
||||||
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR, 1)
|
.list_objects_v2(&self.bucket, &self.prefix, "", "", SLASH_SEPARATOR_STR, 1)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
|
||||||
|
|||||||
@@ -27,19 +27,11 @@ use aws_sdk_s3::Client;
|
|||||||
use aws_sdk_s3::config::{Credentials, Region};
|
use aws_sdk_s3::config::{Credentials, Region};
|
||||||
use aws_sdk_s3::primitives::ByteStream;
|
use aws_sdk_s3::primitives::ByteStream;
|
||||||
|
|
||||||
use crate::client::{
|
use crate::client::transition_api::{ReadCloser, ReaderImpl};
|
||||||
api_get_options::GetObjectOptions,
|
|
||||||
api_put_object::PutObjectOptions,
|
|
||||||
api_remove::RemoveObjectOptions,
|
|
||||||
transition_api::{ReadCloser, ReaderImpl},
|
|
||||||
};
|
|
||||||
use crate::error::ErrorResponse;
|
|
||||||
use crate::error::error_resp_to_object_err;
|
|
||||||
use crate::tier::{
|
use crate::tier::{
|
||||||
tier_config::TierS3,
|
tier_config::TierS3,
|
||||||
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
warm_backend::{WarmBackend, WarmBackendGetOpts},
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
|
||||||
|
|
||||||
pub struct WarmBackendS3 {
|
pub struct WarmBackendS3 {
|
||||||
pub client: Arc<Client>,
|
pub client: Arc<Client>,
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ use rustfs_ecstore::{
|
|||||||
store_api::{ObjectInfo, ObjectOptions},
|
store_api::{ObjectInfo, ObjectOptions},
|
||||||
};
|
};
|
||||||
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
|
use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc};
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
|
||||||
use serde::{Serialize, de::DeserializeOwned};
|
use serde::{Serialize, de::DeserializeOwned};
|
||||||
use std::sync::LazyLock;
|
use std::sync::LazyLock;
|
||||||
use std::{collections::HashMap, sync::Arc};
|
use std::{collections::HashMap, sync::Arc};
|
||||||
@@ -182,7 +182,7 @@ impl ObjectStore {
|
|||||||
} else {
|
} else {
|
||||||
info.name
|
info.name
|
||||||
};
|
};
|
||||||
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR);
|
let name = object_name.trim_start_matches(&prefix).trim_end_matches(SLASH_SEPARATOR_STR);
|
||||||
let _ = sender
|
let _ = sender
|
||||||
.send(StringOrErr {
|
.send(StringOrErr {
|
||||||
item: Some(name.to_owned()),
|
item: Some(name.to_owned()),
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ ip = ["dep:local-ip-address"] # ip characteristics and their dependencies
|
|||||||
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
|
tls = ["dep:rustls", "dep:rustls-pemfile", "dep:rustls-pki-types"] # tls characteristics and their dependencies
|
||||||
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver
|
net = ["ip", "dep:url", "dep:netif", "dep:futures", "dep:transform-stream", "dep:bytes", "dep:s3s", "dep:hyper", "dep:thiserror", "dep:tokio"] # network features with DNS resolver
|
||||||
io = ["dep:tokio"]
|
io = ["dep:tokio"]
|
||||||
path = []
|
path = [] # path manipulation features
|
||||||
notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features
|
notify = ["dep:hyper", "dep:s3s", "dep:hashbrown", "dep:thiserror", "dep:serde", "dep:libc", "dep:url", "dep:regex"] # file system notification features
|
||||||
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
|
compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"]
|
||||||
string = ["dep:regex"]
|
string = ["dep:regex"]
|
||||||
|
|||||||
+722
-200
File diff suppressed because it is too large
Load Diff
@@ -12,11 +12,6 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
use std::{
|
|
||||||
collections::HashMap,
|
|
||||||
io::{Cursor, Read as _, Write as _},
|
|
||||||
};
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
admin::{auth::validate_admin_request, router::Operation},
|
admin::{auth::validate_admin_request, router::Operation},
|
||||||
auth::{check_key_valid, get_session_token},
|
auth::{check_key_valid, get_session_token},
|
||||||
@@ -49,7 +44,7 @@ use rustfs_policy::policy::{
|
|||||||
BucketPolicy,
|
BucketPolicy,
|
||||||
action::{Action, AdminAction},
|
action::{Action, AdminAction},
|
||||||
};
|
};
|
||||||
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
use rustfs_utils::path::{SLASH_SEPARATOR_STR, path_join_buf};
|
||||||
use s3s::{
|
use s3s::{
|
||||||
Body, S3Request, S3Response, S3Result,
|
Body, S3Request, S3Response, S3Result,
|
||||||
dto::{
|
dto::{
|
||||||
@@ -61,6 +56,10 @@ use s3s::{
|
|||||||
};
|
};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_urlencoded::from_bytes;
|
use serde_urlencoded::from_bytes;
|
||||||
|
use std::{
|
||||||
|
collections::HashMap,
|
||||||
|
io::{Cursor, Read as _, Write as _},
|
||||||
|
};
|
||||||
use time::OffsetDateTime;
|
use time::OffsetDateTime;
|
||||||
use tracing::warn;
|
use tracing::warn;
|
||||||
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
|
use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions};
|
||||||
@@ -424,7 +423,7 @@ impl Operation for ImportBucketMetadata {
|
|||||||
// Extract bucket names
|
// Extract bucket names
|
||||||
let mut bucket_names = Vec::new();
|
let mut bucket_names = Vec::new();
|
||||||
for (file_path, _) in &file_contents {
|
for (file_path, _) in &file_contents {
|
||||||
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
|
let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
|
||||||
|
|
||||||
if file_path_split.len() < 2 {
|
if file_path_split.len() < 2 {
|
||||||
warn!("file path is invalid: {}", file_path);
|
warn!("file path is invalid: {}", file_path);
|
||||||
@@ -463,7 +462,7 @@ impl Operation for ImportBucketMetadata {
|
|||||||
|
|
||||||
// Second pass: process file contents
|
// Second pass: process file contents
|
||||||
for (file_path, content) in file_contents {
|
for (file_path, content) in file_contents {
|
||||||
let file_path_split = file_path.split(SLASH_SEPARATOR).collect::<Vec<&str>>();
|
let file_path_split = file_path.split(SLASH_SEPARATOR_STR).collect::<Vec<&str>>();
|
||||||
|
|
||||||
if file_path_split.len() < 2 {
|
if file_path_split.len() < 2 {
|
||||||
warn!("file path is invalid: {}", file_path);
|
warn!("file path is invalid: {}", file_path);
|
||||||
|
|||||||
Reference in New Issue
Block a user