todo:list_object

.vscode/launch.json

@@ -19,6 +19,9 @@
                     "kind": "bin"
                 }
             },
+            "env": {
+                "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug"
+            },
             "args": [
                 "--access-key",
                 "AKEXAMPLERUSTFS",

TODO.md

@@ -5,7 +5,7 @@
  - [ ] 上传同名文件时，删除旧版本文件
  - [ ] EC可用读写数量判断
  - [ ] 小文件存储到metafile, inlinedata
- - [ ] 错误类型判断
+ - [ ] 错误类型判断,程序中判断错误类型，如何统一错误
  - [ ] 优化并发执行
  - [ ] 抽象出metafile存储
  - [ ] 代码优化
@@ -13,14 +13,21 @@
 ## 基础功能
 
  - [ ] 桶操作
-   - [x] 创建
+   - [x] 创建 CreateBucket
-   - [x] 列表
+   - [x] 列表 ListBuckets
-   - [x] 详情
+     - [ ] 桶下面的文件列表 ListObjects
+   - [x] 详情 HeadBucket
    - [ ] 删除
 - [ ] 文件操作
-  - [x] 上传
+  - [x] 上传 PutObject
-  - [x] 大文件上传
+  - [ ] 大文件上传
-  - [x] 下载
+    - [ ] 创建分片上传 CreateMultipartUpload
+    - [x] 上传分片 PubObjectPart
+    - [x] 提交完成 CompleteMultipartUpload
+    - [ ] 取消上传
+  - [x] 下载 GetObject
+  - [ ] 复制 CopyObject
+  - [ ] 详情 HeadObject
   - [ ] 删除 
 
 ## 扩展功能

ecstore/src/disk.rs

@@ -1,6 +1,6 @@
 use std::{
     fs::Metadata,
-    io::SeekFrom,
+    io::{self, SeekFrom},
     os::unix::ffi::OsStringExt,
     path::{Path, PathBuf},
     sync::Arc,
@@ -218,8 +218,14 @@ impl LocalDisk {
 
         if recursive {
             let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?;
-            fs::create_dir_all(&trash_path).await?;
+            // fs::create_dir_all(&trash_path).await?;
-            fs::rename(&delete_path, &trash_path).await?;
+            fs::rename(&delete_path, &trash_path).await.map_err(|err| {
+                // 使用文件路径自定义错误信息
+                io::Error::new(
+                    err.kind(),
+                    format!("Failed to rename file '{:?}' to '{:?}': {}", &delete_path, &trash_path, err),
+                )
+            })?;
 
             // TODO: immediate
 
@@ -526,6 +532,40 @@ impl DiskAPI for LocalDisk {
 
         // Ok((buffer, bytes_read))
     }
+    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: usize) -> Result<Vec<String>> {
+        let p = self.get_bucket_path(&volume)?;
+
+        let mut entries = fs::read_dir(&p).await?;
+
+        let mut volumes = Vec::new();
+
+        while let Some(entry) = entries.next_entry().await? {
+            if let Ok(metadata) = entry.metadata().await {
+                let vec = entry.file_name().into_vec();
+
+                // if !metadata.is_dir() {
+                //     continue;
+                // }
+
+                let name = match String::from_utf8(vec) {
+                    Ok(s) => s,
+                    Err(_) => return Err(Error::msg("Not supported utf8 file name on this platform")),
+                };
+
+                // let created = match metadata.created() {
+                //     Ok(md) => OffsetDateTime::from(md),
+                //     Err(_) => return Err(Error::msg("Not supported created on this platform")),
+                // };
+
+                volumes.push(name);
+            }
+        }
+
+        Ok(volumes)
+    }
+    async fn walk_dir(&self) -> Result<Vec<FileInfo>> {
+        unimplemented!()
+    }
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -648,9 +688,9 @@ impl DiskAPI for LocalDisk {
             if let Ok(metadata) = entry.metadata().await {
                 let vec = entry.file_name().into_vec();
 
-                if !metadata.is_dir() {
+                // if !metadata.is_dir() {
-                    continue;
+                //     continue;
-                }
+                // }
 
                 let name = match String::from_utf8(vec) {
                     Ok(s) => s,

ecstore/src/disk_api.rs

@@ -26,6 +26,10 @@ pub trait DiskAPI: Debug + Send + Sync + 'static {
     async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result<FileWriter>;
     async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter>;
     async fn read_file(&self, volume: &str, path: &str) -> Result<FileReader>;
+    // 读目录下的所有文件、目录
+    async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: usize) -> Result<Vec<String>>;
+    // 读目录下的所有xl.meta
+    async fn walk_dir(&self) -> Result<Vec<FileInfo>>;
     async fn rename_data(
         &self,
         src_volume: &str,
@@ -206,8 +210,6 @@ pub enum DiskError {
 
 impl DiskError {
     pub fn check_disk_fatal_errs(errs: &Vec<Option<Error>>) -> Result<()> {
-        println!("errs: {:?}", errs);
-
         if Self::count_errs(errs, &DiskError::UnsupportedDisk) == errs.len() {
             return Err(Error::new(DiskError::UnsupportedDisk));
         }

ecstore/src/erasure.rs

@@ -29,6 +29,10 @@ pub struct Erasure {
 
 impl Erasure {
     pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self {
+        warn!(
+            "Erasure new data_shards {},parity_shards {} block_size {} ",
+            data_shards, parity_shards, block_size
+        );
         Erasure {
             data_shards,
             parity_shards,

ecstore/src/store_init.rs

@@ -1,4 +1,5 @@
 use futures::future::join_all;
+use tracing::warn;
 use uuid::Uuid;
 
 use crate::{
@@ -99,6 +100,7 @@ fn get_format_file_in_quorum(formats: &Vec<Option<FormatV3>>) -> Result<FormatV3
     let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0));
 
     if *max_drives == 0 || *max_count < formats.len() / 2 {
+        warn!("*max_drives == 0 || *max_count < formats.len() / 2");
         return Err(Error::new(ErasureError::ErasureReadQuorum));
     }
 

scripts/run.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
+mkdir -p ./target/volume/test
 mkdir -p ./target/volume/test{0..4}
 
+# DATA_DIR="./target/volume/test"
 DATA_DIR="./target/volume/test{0...4}"
 
 if [ -n "$1" ]; then
@@ -9,7 +11,7 @@ if [ -n "$1" ]; then
 fi
 
 if [ -z "$RUST_LOG" ]; then
-    export RUST_LOG="rustfs=debug,ecstore=info,s3s=debug"
+    export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug"
 fi
 
 cargo run \