Performance: improve (#514)

* Performance: improve

Signed-off-by: junxiang Mu <1948535941@qq.com>

* remove dirty

Signed-off-by: junxiang Mu <1948535941@qq.com>

* fix some err

Signed-off-by: junxiang Mu <1948535941@qq.com>

---------

Signed-off-by: junxiang Mu <1948535941@qq.com>

crates/ecstore/src/disk/fs.rs

@@ -12,13 +12,45 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::{fs::Metadata, path::Path};
+use std::{
+    fs::Metadata,
+    path::Path,
+    sync::{Arc, OnceLock},
+};
 
 use tokio::{
     fs::{self, File},
     io,
 };
 
+static READONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
+static WRITEONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
+static READWRITE_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
+
+fn get_readonly_options() -> &'static Arc<fs::OpenOptions> {
+    READONLY_OPTIONS.get_or_init(|| {
+        let mut opts = fs::OpenOptions::new();
+        opts.read(true);
+        Arc::new(opts)
+    })
+}
+
+fn get_writeonly_options() -> &'static Arc<fs::OpenOptions> {
+    WRITEONLY_OPTIONS.get_or_init(|| {
+        let mut opts = fs::OpenOptions::new();
+        opts.write(true);
+        Arc::new(opts)
+    })
+}
+
+fn get_readwrite_options() -> &'static Arc<fs::OpenOptions> {
+    READWRITE_OPTIONS.get_or_init(|| {
+        let mut opts = fs::OpenOptions::new();
+        opts.read(true).write(true);
+        Arc::new(opts)
+    })
+}
+
 #[cfg(not(windows))]
 pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool {
     use std::os::unix::fs::MetadataExt;
@@ -84,35 +116,28 @@ pub const O_APPEND: FileMode = 0x00400;
 //     create_new: bool,
 
 pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
-    let mut opts = fs::OpenOptions::new();
-
-    match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
-        O_RDONLY => {
-            opts.read(true);
-        }
-        O_WRONLY => {
-            opts.write(true);
-        }
-        O_RDWR => {
-            opts.read(true);
-            opts.write(true);
-        }
-        _ => (),
+    let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
+        O_RDONLY => get_readonly_options(),
+        O_WRONLY => get_writeonly_options(),
+        O_RDWR => get_readwrite_options(),
+        _ => get_readonly_options(),
     };
 
-    if mode & O_CREATE != 0 {
-        opts.create(true);
+    if (mode & (O_CREATE | O_APPEND | O_TRUNC)) != 0 {
+        let mut opts = (**base_opts).clone();
+        if mode & O_CREATE != 0 {
+            opts.create(true);
+        }
+        if mode & O_APPEND != 0 {
+            opts.append(true);
+        }
+        if mode & O_TRUNC != 0 {
+            opts.truncate(true);
+        }
+        opts.open(path.as_ref()).await
+    } else {
+        base_opts.open(path.as_ref()).await
     }
-
-    if mode & O_APPEND != 0 {
-        opts.append(true);
-    }
-
-    if mode & O_TRUNC != 0 {
-        opts.truncate(true);
-    }
-
-    opts.open(path.as_ref()).await
 }
 
 pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
@@ -121,7 +146,7 @@ pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
 }
 
 pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
-    tokio::task::block_in_place(|| std::fs::metadata(path))?;
+    std::fs::metadata(path)?;
     Ok(())
 }
 
@@ -130,7 +155,7 @@ pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
 }
 
 pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
-    tokio::task::block_in_place(|| std::fs::metadata(path))
+    std::fs::metadata(path)
 }
 
 pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
@@ -159,26 +184,22 @@ pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
 #[tracing::instrument(level = "debug", skip_all)]
 pub fn remove_std(path: impl AsRef<Path>) -> io::Result<()> {
     let path = path.as_ref();
-    tokio::task::block_in_place(|| {
-        let meta = std::fs::metadata(path)?;
-        if meta.is_dir() {
-            std::fs::remove_dir(path)
-        } else {
-            std::fs::remove_file(path)
-        }
-    })
+    let meta = std::fs::metadata(path)?;
+    if meta.is_dir() {
+        std::fs::remove_dir(path)
+    } else {
+        std::fs::remove_file(path)
+    }
 }
 
 pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
     let path = path.as_ref();
-    tokio::task::block_in_place(|| {
-        let meta = std::fs::metadata(path)?;
-        if meta.is_dir() {
-            std::fs::remove_dir_all(path)
-        } else {
-            std::fs::remove_file(path)
-        }
-    })
+    let meta = std::fs::metadata(path)?;
+    if meta.is_dir() {
+        std::fs::remove_dir_all(path)
+    } else {
+        std::fs::remove_file(path)
+    }
 }
 
 pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
@@ -190,7 +211,7 @@ pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<
 }
 
 pub fn rename_std(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
-    tokio::task::block_in_place(|| std::fs::rename(from, to))
+    std::fs::rename(from, to)
 }
 
 #[tracing::instrument(level = "debug", skip_all)]

crates/ecstore/src/disk/local.rs

@@ -41,18 +41,21 @@ use tokio::time::interval;
 
 use crate::erasure_coding::bitrot_verify;
 use bytes::Bytes;
-use path_absolutize::Absolutize;
+// use path_absolutize::Absolutize;  // Replaced with direct path operations for better performance
+use crate::file_cache::{get_global_file_cache, prefetch_metadata_patterns, read_metadata_cached};
+use parking_lot::RwLock as ParkingLotRwLock;
 use rustfs_filemeta::{
     Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, ObjectPartInfo, Opts, RawFileInfo, UpdateFn,
     get_file_info, read_xl_meta_no_data,
 };
 use rustfs_utils::HashAlgorithm;
 use rustfs_utils::os::get_info;
+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::fmt::Debug;
 use std::io::SeekFrom;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 use std::{
     fs::Metadata,
@@ -101,6 +104,9 @@ pub struct LocalDisk {
     pub major: u64,
     pub minor: u64,
     pub nrrequests: u64,
+    // Performance optimization fields
+    path_cache: Arc<ParkingLotRwLock<HashMap<String, PathBuf>>>,
+    current_dir: Arc<OnceLock<PathBuf>>,
     // pub id: Mutex<Option<Uuid>>,
     // pub format_data: Mutex<Vec<u8>>,
     // pub format_file_info: Mutex<Option<Metadata>>,
@@ -130,8 +136,9 @@ impl Debug for LocalDisk {
 impl LocalDisk {
     pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
         debug!("Creating local disk");
-        let root = match PathBuf::from(ep.get_file_path()).absolutize() {
-            Ok(path) => path.into_owned(),
+        // Use optimized path resolution instead of absolutize() for better performance
+        let root = match std::fs::canonicalize(ep.get_file_path()) {
+            Ok(path) => path,
             Err(e) => {
                 if e.kind() == ErrorKind::NotFound {
                     return Err(DiskError::VolumeNotFound);
@@ -144,10 +151,8 @@ impl LocalDisk {
             // TODO: 删除 tmp 数据
         }
 
-        let format_path = Path::new(RUSTFS_META_BUCKET)
-            .join(Path::new(super::FORMAT_CONFIG_FILE))
-            .absolutize_virtually(&root)?
-            .into_owned();
+        // Use optimized path resolution instead of absolutize_virtually
+        let format_path = root.join(RUSTFS_META_BUCKET).join(super::FORMAT_CONFIG_FILE);
         debug!("format_path: {:?}", format_path);
         let (format_data, format_meta) = read_file_exists(&format_path).await?;
 
@@ -227,6 +232,8 @@ impl LocalDisk {
             // format_file_info: Mutex::new(format_meta),
             // format_data: Mutex::new(format_data),
             // format_last_check: Mutex::new(format_last_check),
+            path_cache: Arc::new(ParkingLotRwLock::new(HashMap::with_capacity(2048))),
+            current_dir: Arc::new(OnceLock::new()),
             exit_signal: None,
         };
         let (info, _root) = get_disk_info(root).await?;
@@ -351,19 +358,178 @@ impl LocalDisk {
         self.make_volumes(defaults).await
     }
 
+    // Optimized path resolution with caching
     pub fn resolve_abs_path(&self, path: impl AsRef<Path>) -> Result<PathBuf> {
-        Ok(path.as_ref().absolutize_virtually(&self.root)?.into_owned())
+        let path_ref = path.as_ref();
+        let path_str = path_ref.to_string_lossy();
+
+        // Fast cache read
+        {
+            let cache = self.path_cache.read();
+            if let Some(cached_path) = cache.get(path_str.as_ref()) {
+                return Ok(cached_path.clone());
+            }
+        }
+
+        // Calculate absolute path without using path_absolutize for better performance
+        let abs_path = if path_ref.is_absolute() {
+            path_ref.to_path_buf()
+        } else {
+            self.root.join(path_ref)
+        };
+
+        // Normalize path components to avoid filesystem calls
+        let normalized = self.normalize_path_components(&abs_path);
+
+        // Cache the result
+        {
+            let mut cache = self.path_cache.write();
+
+            // Simple cache size control
+            if cache.len() >= 4096 {
+                // Clear half the cache - simple eviction strategy
+                let keys_to_remove: Vec<_> = cache.keys().take(cache.len() / 2).cloned().collect();
+                for key in keys_to_remove {
+                    cache.remove(&key);
+                }
+            }
+
+            cache.insert(path_str.into_owned(), normalized.clone());
+        }
+
+        Ok(normalized)
     }
 
+    // Lightweight path normalization without filesystem calls
+    fn normalize_path_components(&self, path: &Path) -> PathBuf {
+        let mut result = PathBuf::new();
+
+        for component in path.components() {
+            match component {
+                std::path::Component::Normal(name) => {
+                    result.push(name);
+                }
+                std::path::Component::ParentDir => {
+                    result.pop();
+                }
+                std::path::Component::CurDir => {
+                    // Ignore current directory components
+                }
+                std::path::Component::RootDir => {
+                    result.push(component);
+                }
+                std::path::Component::Prefix(_prefix) => {
+                    result.push(component);
+                }
+            }
+        }
+
+        result
+    }
+
+    // Highly optimized object path generation
     pub fn get_object_path(&self, bucket: &str, key: &str) -> Result<PathBuf> {
-        let dir = Path::new(&bucket);
-        let file_path = Path::new(&key);
-        self.resolve_abs_path(dir.join(file_path))
+        // For high-frequency paths, use faster string concatenation
+        let cache_key = if key.is_empty() {
+            bucket.to_string()
+        } else {
+            // Use with_capacity to pre-allocate, reducing memory reallocations
+            let mut path_str = String::with_capacity(bucket.len() + key.len() + 1);
+            path_str.push_str(bucket);
+            path_str.push('/');
+            path_str.push_str(key);
+            path_str
+        };
+
+        // Fast path: directly calculate based on root, avoiding cache lookup overhead for simple cases
+        Ok(self.root.join(&cache_key))
     }
 
     pub fn get_bucket_path(&self, bucket: &str) -> Result<PathBuf> {
-        let dir = Path::new(&bucket);
-        self.resolve_abs_path(dir)
+        Ok(self.root.join(bucket))
+    }
+
+    // Batch path generation with single lock acquisition
+    pub fn get_object_paths_batch(&self, requests: &[(String, String)]) -> Result<Vec<PathBuf>> {
+        let mut results = Vec::with_capacity(requests.len());
+        let mut cache_misses = Vec::new();
+
+        // First attempt to get all paths from cache
+        {
+            let cache = self.path_cache.read();
+            for (i, (bucket, key)) in requests.iter().enumerate() {
+                let cache_key = format!("{}/{}", bucket, key);
+                if let Some(cached_path) = cache.get(&cache_key) {
+                    results.push((i, cached_path.clone()));
+                } else {
+                    cache_misses.push((i, bucket, key, cache_key));
+                }
+            }
+        }
+
+        // Handle cache misses
+        if !cache_misses.is_empty() {
+            let mut new_entries = Vec::new();
+            for (i, _bucket, _key, cache_key) in cache_misses {
+                let path = self.root.join(&cache_key);
+                results.push((i, path.clone()));
+                new_entries.push((cache_key, path));
+            }
+
+            // Batch update cache
+            {
+                let mut cache = self.path_cache.write();
+                for (key, path) in new_entries {
+                    cache.insert(key, path);
+                }
+            }
+        }
+
+        // Sort results back to original order
+        results.sort_by_key(|(i, _)| *i);
+        Ok(results.into_iter().map(|(_, path)| path).collect())
+    }
+
+    // Optimized metadata reading with caching
+    pub async fn read_metadata_cached(&self, path: PathBuf) -> Result<Arc<FileMeta>> {
+        read_metadata_cached(path).await
+    }
+
+    // Smart prefetching for related files
+    pub async fn read_version_with_prefetch(
+        &self,
+        volume: &str,
+        path: &str,
+        version_id: &str,
+        opts: &ReadOptions,
+    ) -> Result<FileInfo> {
+        let file_path = self.get_object_path(volume, path)?;
+
+        // Async prefetch related files, don't block current read
+        if let Some(parent) = file_path.parent() {
+            prefetch_metadata_patterns(parent, &[super::STORAGE_FORMAT_FILE, "part.1", "part.2", "part.meta"]).await;
+        }
+
+        // Main read logic
+        let file_dir = self.get_bucket_path(volume)?;
+        let (data, _) = self.read_raw(volume, file_dir, file_path, opts.read_data).await?;
+
+        get_file_info(&data, volume, path, version_id, FileInfoOpts { data: opts.read_data })
+            .await
+            .map_err(|_e| DiskError::Unexpected)
+    }
+
+    // Batch metadata reading for multiple objects
+    pub async fn read_metadata_batch(&self, requests: Vec<(String, String)>) -> Result<Vec<Option<Arc<FileMeta>>>> {
+        let paths: Vec<PathBuf> = requests
+            .iter()
+            .map(|(bucket, key)| self.get_object_path(bucket, &format!("{}/{}", key, super::STORAGE_FORMAT_FILE)))
+            .collect::<Result<Vec<_>>>()?;
+
+        let cache = get_global_file_cache();
+        let results = cache.get_metadata_batch(paths).await;
+
+        Ok(results.into_iter().map(|r| r.ok()).collect())
     }
 
     // /// Write to the filesystem atomically.
@@ -549,7 +715,15 @@ impl LocalDisk {
     }
 
     async fn read_metadata(&self, file_path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        // TODO: support timeout
+        // Try to use cached file content reading for better performance, with safe fallback
+        let path = file_path.as_ref().to_path_buf();
+
+        // First, try the cache
+        if let Ok(bytes) = get_global_file_cache().get_file_content(path.clone()).await {
+            return Ok(bytes.to_vec());
+        }
+
+        // Fallback to direct read if cache fails
         let (data, _) = self.read_metadata_with_dmtime(file_path.as_ref()).await?;
         Ok(data)
     }

crates/ecstore/src/disk/mod.rs

@@ -668,7 +668,7 @@ pub struct VolumeInfo {
     pub created: Option<OffsetDateTime>,
 }
 
-#[derive(Deserialize, Serialize, Debug, Default)]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
 pub struct ReadOptions {
     pub incl_free_versions: bool,
     pub read_data: bool,