merge main

rustfs/Cargo.toml

@@ -10,6 +10,25 @@ rust-version.workspace = true
 
 [dependencies]
 async-trait.workspace = true
+bytes.workspace = true
+clap.workspace = true
+ecstore.workspace = true
+flatbuffers.workspace = true
+futures.workspace = true
+futures-util.workspace = true
+hyper.workspace = true
+hyper-util.workspace = true
+http.workspace = true
+http-body.workspace = true
+mime.workspace = true
+netif.workspace = true
+pin-project-lite.workspace = true
+prost.workspace = true
+prost-types.workspace = true
+protos.workspace = true
+protobuf.workspace = true
+s3s.workspace = true
+serde_json.workspace = true
 tracing.workspace = true
 time = { workspace = true, features = ["parsing", "formatting"] }
 tokio = { workspace = true, features = [
@@ -18,12 +37,22 @@ tokio = { workspace = true, features = [
     "net",
     "signal",
 ] }
+tonic = { version = "0.12.1", features = ["gzip"] }
+tonic-reflection.workspace = true
+tower.workspace = true
 tracing-error.workspace = true
+tracing-subscriber.workspace = true
+transform-stream.workspace = true
+uuid = "1.10.0"
+
+[build-dependencies]
+prost-build.workspace = true
+tonic-build.workspace = true
 http.workspace = true
 bytes.workspace = true
 futures.workspace = true
 futures-util.workspace = true
-uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] }
+# uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] }
 ecstore = { path = "../ecstore" }
 s3s = "0.10.0"
 clap = { version = "4.5.7", features = ["derive"] }

rustfs/src/grpc.rs

@@ -0,0 +1,816 @@
+use ecstore::{
+    disk::{DeleteOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, WalkDirOptions},
+    erasure::{ReadAt, Write},
+    peer::{LocalPeerS3Client, PeerS3Client},
+    store::{all_local_disk_path, find_local_disk},
+    store_api::{BucketOptions, FileInfo, MakeBucketOptions},
+};
+use tonic::{Request, Response, Status};
+use tracing::{debug, error, info};
+
+use protos::{
+    models::{PingBody, PingBodyBuilder},
+    proto_gen::node_service::{
+        node_service_server::{NodeService as Node, NodeServiceServer as NodeServer},
+        DeleteBucketRequest, DeleteBucketResponse, DeleteRequest, DeleteResponse, DeleteVersionsRequest, DeleteVersionsResponse,
+        DeleteVolumeRequest, DeleteVolumeResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest,
+        ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest,
+        MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest,
+        PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse,
+        ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse,
+        RenameFileRequst, RenameFileResponse, StatVolumeRequest, StatVolumeResponse, WalkDirRequest, WalkDirResponse,
+        WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse,
+    },
+};
+
+#[derive(Debug)]
+struct NodeService {
+    pub local_peer: LocalPeerS3Client,
+}
+
+pub fn make_server() -> NodeServer<impl Node> {
+    // let local_disks = all_local_disk().await;
+    let local_peer = LocalPeerS3Client::new(None, None);
+    NodeServer::new(NodeService { local_peer })
+}
+
+impl NodeService {
+    async fn find_disk(&self, disk_path: &String) -> Option<DiskStore> {
+        find_local_disk(disk_path).await
+        // let disk_path = match fs::canonicalize(disk_path).await {
+        //     Ok(disk_path) => disk_path,
+        //     Err(_) => return None,
+        // };
+        // self.local_peer.local_disks.iter().find(|&x| x.path() == disk_path).cloned()
+    }
+
+    async fn all_disk(&self) -> Vec<String> {
+        all_local_disk_path().await
+        // self.local_peer
+        //     .local_disks
+        //     .iter()
+        //     .map(|disk| disk.path().to_string_lossy().to_string())
+        //     .collect()
+    }
+}
+
+#[tonic::async_trait]
+impl Node for NodeService {
+    async fn ping(&self, request: Request<PingRequest>) -> Result<Response<PingResponse>, Status> {
+        debug!("PING");
+
+        let ping_req = request.into_inner();
+        let ping_body = flatbuffers::root::<PingBody>(&ping_req.body);
+        if let Err(e) = ping_body {
+            error!("{}", e);
+        } else {
+            info!("ping_req:body(flatbuffer): {:?}", ping_body);
+        }
+
+        let mut fbb = flatbuffers::FlatBufferBuilder::new();
+        let payload = fbb.create_vector(b"hello, caller");
+
+        let mut builder = PingBodyBuilder::new(&mut fbb);
+        builder.add_payload(payload);
+        let root = builder.finish();
+        fbb.finish(root, None);
+
+        let finished_data = fbb.finished_data();
+
+        Ok(tonic::Response::new(PingResponse {
+            version: 1,
+            body: finished_data.to_vec(),
+        }))
+    }
+
+    async fn list_bucket(&self, request: Request<ListBucketRequest>) -> Result<Response<ListBucketResponse>, Status> {
+        debug!("list bucket");
+
+        let request = request.into_inner();
+        let options = match serde_json::from_str::<BucketOptions>(&request.options) {
+            Ok(options) => options,
+            Err(err) => {
+                return Ok(tonic::Response::new(ListBucketResponse {
+                    success: false,
+                    bucket_infos: Vec::new(),
+                    error_info: Some(format!("decode BucketOptions failed: {}", err.to_string())),
+                }))
+            }
+        };
+        match self.local_peer.list_bucket(&options).await {
+            Ok(bucket_infos) => {
+                let bucket_infos = bucket_infos
+                    .into_iter()
+                    .filter_map(|bucket_info| serde_json::to_string(&bucket_info).ok())
+                    .collect();
+                Ok(tonic::Response::new(ListBucketResponse {
+                    success: true,
+                    bucket_infos,
+                    error_info: None,
+                }))
+            }
+
+            Err(err) => Ok(tonic::Response::new(ListBucketResponse {
+                success: false,
+                bucket_infos: Vec::new(),
+                error_info: Some(format!("make failed: {}", err.to_string())),
+            })),
+        }
+    }
+
+    async fn make_bucket(&self, request: Request<MakeBucketRequest>) -> Result<Response<MakeBucketResponse>, Status> {
+        debug!("make bucket");
+
+        let request = request.into_inner();
+        let options = match serde_json::from_str::<MakeBucketOptions>(&request.options) {
+            Ok(options) => options,
+            Err(err) => {
+                return Ok(tonic::Response::new(MakeBucketResponse {
+                    success: false,
+                    error_info: Some(format!("decode MakeBucketOptions failed: {}", err.to_string())),
+                }))
+            }
+        };
+        match self.local_peer.make_bucket(&request.name, &options).await {
+            Ok(_) => Ok(tonic::Response::new(MakeBucketResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(tonic::Response::new(MakeBucketResponse {
+                success: false,
+                error_info: Some(format!("make failed: {}", err.to_string())),
+            })),
+        }
+    }
+
+    async fn get_bucket_info(&self, request: Request<GetBucketInfoRequest>) -> Result<Response<GetBucketInfoResponse>, Status> {
+        debug!("get bucket info");
+
+        let request = request.into_inner();
+        let options = match serde_json::from_str::<BucketOptions>(&request.options) {
+            Ok(options) => options,
+            Err(err) => {
+                return Ok(tonic::Response::new(GetBucketInfoResponse {
+                    success: false,
+                    bucket_info: String::new(),
+                    error_info: Some(format!("decode BucketOptions failed: {}", err.to_string())),
+                }))
+            }
+        };
+        match self.local_peer.get_bucket_info(&request.bucket, &options).await {
+            Ok(bucket_info) => {
+                let bucket_info = match serde_json::to_string(&bucket_info) {
+                    Ok(bucket_info) => bucket_info,
+                    Err(err) => {
+                        return Ok(tonic::Response::new(GetBucketInfoResponse {
+                            success: false,
+                            bucket_info: String::new(),
+                            error_info: Some(format!("encode BucketInfo failed: {}", err.to_string())),
+                        }));
+                    }
+                };
+                Ok(tonic::Response::new(GetBucketInfoResponse {
+                    success: true,
+                    bucket_info,
+                    error_info: None,
+                }))
+            }
+
+            Err(err) => Ok(tonic::Response::new(GetBucketInfoResponse {
+                success: false,
+                bucket_info: String::new(),
+                error_info: Some(format!("make failed: {}", err.to_string())),
+            })),
+        }
+    }
+
+    async fn delete_bucket(&self, request: Request<DeleteBucketRequest>) -> Result<Response<DeleteBucketResponse>, Status> {
+        debug!("make bucket");
+
+        let request = request.into_inner();
+        match self.local_peer.delete_bucket(&request.bucket).await {
+            Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse {
+                success: true,
+                error_info: None,
+            })),
+            Err(err) => Ok(tonic::Response::new(DeleteBucketResponse {
+                success: false,
+                error_info: Some(format!("make failed: {}", err.to_string())),
+            })),
+        }
+    }
+
+    async fn read_all(&self, request: Request<ReadAllRequest>) -> Result<Response<ReadAllResponse>, Status> {
+        debug!("read all");
+
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.read_all(&request.volume, &request.path).await {
+                Ok(data) => Ok(tonic::Response::new(ReadAllResponse {
+                    success: true,
+                    data: data.to_vec(),
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(ReadAllResponse {
+                    success: false,
+                    data: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ReadAllResponse {
+                success: false,
+                data: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn write_all(&self, request: Request<WriteAllRequest>) -> Result<Response<WriteAllResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.write_all(&request.volume, &request.path, request.data).await {
+                Ok(_) => Ok(tonic::Response::new(WriteAllResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(WriteAllResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(WriteAllResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn delete(&self, request: Request<DeleteRequest>) -> Result<Response<DeleteResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let options = match serde_json::from_str::<DeleteOptions>(&request.options) {
+                Ok(options) => options,
+                Err(_) => {
+                    return Ok(tonic::Response::new(DeleteResponse {
+                        success: false,
+                        error_info: Some("can not decode DeleteOptions".to_string()),
+                    }));
+                }
+            };
+            match disk.delete(&request.volume, &request.path, options).await {
+                Ok(_) => Ok(tonic::Response::new(DeleteResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(DeleteResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(DeleteResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn rename_file(&self, request: Request<RenameFileRequst>) -> Result<Response<RenameFileResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk
+                .rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path)
+                .await
+            {
+                Ok(_) => Ok(tonic::Response::new(RenameFileResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(RenameFileResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(RenameFileResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn write(&self, request: Request<WriteRequest>) -> Result<Response<WriteResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let file_writer = if request.is_append {
+                disk.append_file(&request.volume, &request.path).await
+            } else {
+                disk.create_file("", &request.volume, &request.path, 0).await
+            };
+
+            match file_writer {
+                Ok(mut file_writer) => match file_writer.write(&request.data).await {
+                    Ok(_) => Ok(tonic::Response::new(WriteResponse {
+                        success: true,
+                        error_info: None,
+                    })),
+                    Err(err) => Ok(tonic::Response::new(WriteResponse {
+                        success: false,
+                        error_info: Some(err.to_string()),
+                    })),
+                },
+                Err(err) => Ok(tonic::Response::new(WriteResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(WriteResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn read_at(&self, request: Request<ReadAtRequest>) -> Result<Response<ReadAtResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.read_file(&request.volume, &request.path).await {
+                Ok(mut file_reader) => {
+                    match file_reader
+                        .read_at(request.offset.try_into().unwrap(), request.length.try_into().unwrap())
+                        .await
+                    {
+                        Ok((data, read_size)) => Ok(tonic::Response::new(ReadAtResponse {
+                            success: true,
+                            data,
+                            read_size: read_size.try_into().unwrap(),
+                            error_info: None,
+                        })),
+                        Err(err) => Ok(tonic::Response::new(ReadAtResponse {
+                            success: false,
+                            data: Vec::new(),
+                            read_size: -1,
+                            error_info: Some(err.to_string()),
+                        })),
+                    }
+                }
+                Err(err) => Ok(tonic::Response::new(ReadAtResponse {
+                    success: false,
+                    data: Vec::new(),
+                    read_size: -1,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ReadAtResponse {
+                success: false,
+                data: Vec::new(),
+                read_size: -1,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.list_dir("", &request.volume, "", 0).await {
+                Ok(volumes) => Ok(tonic::Response::new(ListDirResponse {
+                    success: true,
+                    volumes,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(ListDirResponse {
+                    success: false,
+                    volumes: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ListDirResponse {
+                success: false,
+                volumes: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn walk_dir(&self, request: Request<WalkDirRequest>) -> Result<Response<WalkDirResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let opts = match serde_json::from_str::<WalkDirOptions>(&request.walk_dir_options) {
+                Ok(options) => options,
+                Err(_) => {
+                    return Ok(tonic::Response::new(WalkDirResponse {
+                        success: false,
+                        meta_cache_entry: Vec::new(),
+                        error_info: Some("can not decode DeleteOptions".to_string()),
+                    }));
+                }
+            };
+            match disk.walk_dir(opts).await {
+                Ok(entries) => {
+                    let entries = entries
+                        .into_iter()
+                        .filter_map(|entry| serde_json::to_string(&entry).ok())
+                        .collect();
+                    Ok(tonic::Response::new(WalkDirResponse {
+                        success: true,
+                        meta_cache_entry: entries,
+                        error_info: None,
+                    }))
+                }
+                Err(err) => Ok(tonic::Response::new(WalkDirResponse {
+                    success: false,
+                    meta_cache_entry: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(WalkDirResponse {
+                success: false,
+                meta_cache_entry: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn rename_data(&self, request: Request<RenameDataRequest>) -> Result<Response<RenameDataResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let file_info = match serde_json::from_str::<FileInfo>(&request.file_info) {
+                Ok(file_info) => file_info,
+                Err(_) => {
+                    return Ok(tonic::Response::new(RenameDataResponse {
+                        success: false,
+                        rename_data_resp: String::new(),
+                        error_info: Some("can not decode DeleteOptions".to_string()),
+                    }));
+                }
+            };
+            match disk
+                .rename_data(&request.src_volume, &request.src_path, file_info, &request.dst_volume, &request.dst_path)
+                .await
+            {
+                Ok(rename_data_resp) => {
+                    let rename_data_resp = match serde_json::to_string(&rename_data_resp) {
+                        Ok(file_info) => file_info,
+                        Err(_) => {
+                            return Ok(tonic::Response::new(RenameDataResponse {
+                                success: false,
+                                rename_data_resp: String::new(),
+                                error_info: Some("can not encode RenameDataResp".to_string()),
+                            }));
+                        }
+                    };
+                    Ok(tonic::Response::new(RenameDataResponse {
+                        success: true,
+                        rename_data_resp,
+                        error_info: None,
+                    }))
+                }
+                Err(err) => Ok(tonic::Response::new(RenameDataResponse {
+                    success: false,
+                    rename_data_resp: String::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(RenameDataResponse {
+                success: false,
+                rename_data_resp: String::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn make_volumes(&self, request: Request<MakeVolumesRequest>) -> Result<Response<MakeVolumesResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.make_volumes(request.volumes.iter().map(|s| &**s).collect()).await {
+                Ok(_) => Ok(tonic::Response::new(MakeVolumesResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(MakeVolumesResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(MakeVolumesResponse {
+                success: false,
+                error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk().await)),
+            }))
+        }
+    }
+
+    async fn make_volume(&self, request: Request<MakeVolumeRequest>) -> Result<Response<MakeVolumeResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.make_volume(&request.volume).await {
+                Ok(_) => Ok(tonic::Response::new(MakeVolumeResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(MakeVolumeResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(MakeVolumeResponse {
+                success: false,
+                error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk().await)),
+            }))
+        }
+    }
+
+    async fn list_volumes(&self, request: Request<ListVolumesRequest>) -> Result<Response<ListVolumesResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.list_volumes().await {
+                Ok(volume_infos) => {
+                    let volume_infos = volume_infos
+                        .into_iter()
+                        .filter_map(|volume_info| serde_json::to_string(&volume_info).ok())
+                        .collect();
+                    Ok(tonic::Response::new(ListVolumesResponse {
+                        success: true,
+                        volume_infos,
+                        error_info: None,
+                    }))
+                }
+                Err(err) => Ok(tonic::Response::new(ListVolumesResponse {
+                    success: false,
+                    volume_infos: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ListVolumesResponse {
+                success: false,
+                volume_infos: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn stat_volume(&self, request: Request<StatVolumeRequest>) -> Result<Response<StatVolumeResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.stat_volume(&request.volume).await {
+                Ok(volume_info) => match serde_json::to_string(&volume_info) {
+                    Ok(volume_info) => Ok(tonic::Response::new(StatVolumeResponse {
+                        success: true,
+                        volume_info,
+                        error_info: None,
+                    })),
+                    Err(err) => Ok(tonic::Response::new(StatVolumeResponse {
+                        success: false,
+                        volume_info: String::new(),
+                        error_info: Some(format!("encode VolumeInfo failed, {}", err.to_string())),
+                    })),
+                },
+                Err(err) => Ok(tonic::Response::new(StatVolumeResponse {
+                    success: false,
+                    volume_info: String::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(StatVolumeResponse {
+                success: false,
+                volume_info: String::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn write_metadata(&self, request: Request<WriteMetadataRequest>) -> Result<Response<WriteMetadataResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let file_info = match serde_json::from_str::<FileInfo>(&request.file_info) {
+                Ok(file_info) => file_info,
+                Err(err) => {
+                    return Ok(tonic::Response::new(WriteMetadataResponse {
+                        success: false,
+                        error_info: Some(format!("decode FileInfo failed, {}", err.to_string())),
+                    }));
+                }
+            };
+            match disk.write_metadata("", &request.volume, &request.path, file_info).await {
+                Ok(_) => Ok(tonic::Response::new(WriteMetadataResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(WriteMetadataResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(WriteMetadataResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn read_version(&self, request: Request<ReadVersionRequest>) -> Result<Response<ReadVersionResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let opts = match serde_json::from_str::<ReadOptions>(&request.opts) {
+                Ok(options) => options,
+                Err(_) => {
+                    return Ok(tonic::Response::new(ReadVersionResponse {
+                        success: false,
+                        file_info: String::new(),
+                        error_info: Some("can not decode DeleteOptions".to_string()),
+                    }));
+                }
+            };
+            match disk
+                .read_version("", &request.volume, &request.path, &request.version_id, &opts)
+                .await
+            {
+                Ok(file_info) => match serde_json::to_string(&file_info) {
+                    Ok(file_info) => Ok(tonic::Response::new(ReadVersionResponse {
+                        success: true,
+                        file_info,
+                        error_info: None,
+                    })),
+                    Err(err) => Ok(tonic::Response::new(ReadVersionResponse {
+                        success: false,
+                        file_info: String::new(),
+                        error_info: Some(format!("encode VolumeInfo failed, {}", err.to_string())),
+                    })),
+                },
+                Err(err) => Ok(tonic::Response::new(ReadVersionResponse {
+                    success: false,
+                    file_info: String::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ReadVersionResponse {
+                success: false,
+                file_info: String::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn read_xl(&self, request: Request<ReadXlRequest>) -> Result<Response<ReadXlResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.read_xl(&request.volume, &request.path, request.read_data).await {
+                Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) {
+                    Ok(raw_file_info) => Ok(tonic::Response::new(ReadXlResponse {
+                        success: true,
+                        raw_file_info,
+                        error_info: None,
+                    })),
+                    Err(err) => Ok(tonic::Response::new(ReadXlResponse {
+                        success: false,
+                        raw_file_info: String::new(),
+                        error_info: Some(format!("encode RawFileInfo failed, {}", err.to_string())),
+                    })),
+                },
+                Err(err) => Ok(tonic::Response::new(ReadXlResponse {
+                    success: false,
+                    raw_file_info: String::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ReadXlResponse {
+                success: false,
+                raw_file_info: String::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn delete_versions(&self, request: Request<DeleteVersionsRequest>) -> Result<Response<DeleteVersionsResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let mut versions = Vec::with_capacity(request.versions.len());
+            for version in request.versions.iter() {
+                match serde_json::from_str::<FileInfoVersions>(&version) {
+                    Ok(version) => versions.push(version),
+                    Err(_) => {
+                        return Ok(tonic::Response::new(DeleteVersionsResponse {
+                            success: false,
+                            errors: Vec::new(),
+                            error_info: Some("can not decode FileInfoVersions".to_string()),
+                        }));
+                    }
+                };
+            }
+            let opts = match serde_json::from_str::<DeleteOptions>(&request.opts) {
+                Ok(opts) => opts,
+                Err(_) => {
+                    return Ok(tonic::Response::new(DeleteVersionsResponse {
+                        success: false,
+                        errors: Vec::new(),
+                        error_info: Some("can not decode DeleteOptions".to_string()),
+                    }));
+                }
+            };
+            match disk.delete_versions(&request.volume, versions, opts).await {
+                Ok(errors) => {
+                    let errors = errors
+                        .into_iter()
+                        .map(|error| match error {
+                            Some(e) => e.to_string(),
+                            None => "".to_string(),
+                        })
+                        .collect();
+
+                    Ok(tonic::Response::new(DeleteVersionsResponse {
+                        success: true,
+                        errors,
+                        error_info: None,
+                    }))
+                }
+                Err(err) => Ok(tonic::Response::new(DeleteVersionsResponse {
+                    success: false,
+                    errors: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(DeleteVersionsResponse {
+                success: false,
+                errors: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn read_multiple(&self, request: Request<ReadMultipleRequest>) -> Result<Response<ReadMultipleResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            let read_multiple_req = match serde_json::from_str::<ReadMultipleReq>(&request.read_multiple_req) {
+                Ok(read_multiple_req) => read_multiple_req,
+                Err(_) => {
+                    return Ok(tonic::Response::new(ReadMultipleResponse {
+                        success: false,
+                        read_multiple_resps: Vec::new(),
+                        error_info: Some("can not decode ReadMultipleReq".to_string()),
+                    }));
+                }
+            };
+            match disk.read_multiple(read_multiple_req).await {
+                Ok(read_multiple_resps) => {
+                    let read_multiple_resps = read_multiple_resps
+                        .into_iter()
+                        .filter_map(|read_multiple_resp| serde_json::to_string(&read_multiple_resp).ok())
+                        .collect();
+
+                    Ok(tonic::Response::new(ReadMultipleResponse {
+                        success: true,
+                        read_multiple_resps,
+                        error_info: None,
+                    }))
+                }
+                Err(err) => Ok(tonic::Response::new(ReadMultipleResponse {
+                    success: false,
+                    read_multiple_resps: Vec::new(),
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(ReadMultipleResponse {
+                success: false,
+                read_multiple_resps: Vec::new(),
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+
+    async fn delete_volume(&self, request: Request<DeleteVolumeRequest>) -> Result<Response<DeleteVolumeResponse>, Status> {
+        let request = request.into_inner();
+        if let Some(disk) = self.find_disk(&request.disk).await {
+            match disk.delete_volume(&request.volume).await {
+                Ok(_) => Ok(tonic::Response::new(DeleteVolumeResponse {
+                    success: true,
+                    error_info: None,
+                })),
+                Err(err) => Ok(tonic::Response::new(DeleteVolumeResponse {
+                    success: false,
+                    error_info: Some(err.to_string()),
+                })),
+            }
+        } else {
+            Ok(tonic::Response::new(DeleteVolumeResponse {
+                success: false,
+                error_info: Some("can not find disk".to_string()),
+            }))
+        }
+    }
+}

rustfs/src/main.rs

@@ -1,16 +1,25 @@
 mod config;
+mod grpc;
+mod service;
 mod storage;
 
 use clap::Parser;
-use ecstore::error::Result;
+use ecstore::{
+    endpoints::EndpointServerPools,
+    error::Result,
+    store::{init_local_disks, update_erasure_type, ECStore},
+};
+use grpc::make_server;
 use hyper_util::{
     rt::{TokioExecutor, TokioIo},
     server::conn::auto::Builder as ConnBuilder,
+    service::TowerToHyperService,
 };
 use s3s::{auth::SimpleAuth, service::S3ServiceBuilder};
+use service::hybrid;
 use std::{io::IsTerminal, net::SocketAddr, str::FromStr};
 use tokio::net::TcpListener;
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 use tracing_error::ErrorLayer;
 use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt};
 
@@ -66,10 +75,19 @@ async fn run(opt: config::Opt) -> Result<()> {
     //         })
     // };
 
+    // 用于rpc
+    let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?;
+
+    update_erasure_type(setup_type).await;
+
+    // 初始化本地磁盘
+    init_local_disks(endpoint_pools.clone()).await?;
+
     // Setup S3 service
     // 本项目使用s3s库来实现s3服务
     let service = {
-        let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), opt.volumes.clone()).await?);
+        // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?);
+        let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new());
         //设置AK和SK
         //其中部份内容从config配置文件中读取
         let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap();
@@ -101,44 +119,59 @@ async fn run(opt: config::Opt) -> Result<()> {
         b.build()
     };
 
-    let hyper_service = service.into_shared();
+    let rpc_service = make_server();
 
-    let http_server = ConnBuilder::new(TokioExecutor::new());
-    let graceful = hyper_util::server::graceful::GracefulShutdown::new();
+    tokio::spawn(async move {
+        let hyper_service = service.into_shared();
 
-    let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
+        let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, rpc_service));
 
-    info!("server is running at http://{local_addr}");
+        let http_server = ConnBuilder::new(TokioExecutor::new());
+        let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
+        let graceful = hyper_util::server::graceful::GracefulShutdown::new();
+        info!("server is running at http://{local_addr}");
 
-    loop {
-        let (socket, _) = tokio::select! {
-            res =  listener.accept() => {
-                match res {
-                    Ok(conn) => conn,
-                    Err(err) => {
-                        tracing::error!("error accepting connection: {err}");
-                        continue;
+        loop {
+            let (socket, _) = tokio::select! {
+                res =  listener.accept() => {
+                    match res {
+                        Ok(conn) => conn,
+                        Err(err) => {
+                            tracing::error!("error accepting connection: {err}");
+                            continue;
+                        }
                     }
                 }
-            }
-            _ = ctrl_c.as_mut() => {
-                break;
-            }
-        };
+                _ = ctrl_c.as_mut() => {
+                    break;
+                }
+            };
 
-        let conn = http_server.serve_connection(TokioIo::new(socket), hyper_service.clone());
-        let conn = graceful.watch(conn.into_owned());
-        tokio::spawn(async move {
-            let _ = conn.await;
-        });
-    }
+            let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone());
+            let conn = graceful.watch(conn.into_owned());
+            tokio::spawn(async move {
+                let _ = conn.await;
+            });
+        }
+
+        tokio::select! {
+            () = graceful.shutdown() => {
+                 tracing::debug!("Gracefully shutdown!");
+            },
+            () = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
+                 tracing::debug!("Waited 10 seconds for graceful shutdown, aborting...");
+            }
+        }
+    });
+
+    warn!(" init store");
+    // init store
+    ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?;
+    warn!(" init store success!");
 
     tokio::select! {
-        () = graceful.shutdown() => {
-             tracing::debug!("Gracefully shutdown!");
-        },
-        () = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
-             tracing::debug!("Waited 10 seconds for graceful shutdown, aborting...");
+        _ = tokio::signal::ctrl_c() => {
+
         }
     }
 

rustfs/src/service.rs

@@ -0,0 +1,150 @@
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use futures::Future;
+use http_body::Frame;
+use hyper::body::Incoming;
+use hyper::{Request, Response};
+use pin_project_lite::pin_project;
+use tower::Service;
+
+type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
+
+/// Generate a [`HybridService`]
+pub(crate) fn hybrid<MakeRest, Grpc>(make_rest: MakeRest, grpc: Grpc) -> HybridService<MakeRest, Grpc> {
+    HybridService { rest: make_rest, grpc }
+}
+
+/// The service that can serve both gRPC and REST HTTP Requests
+#[derive(Clone)]
+pub struct HybridService<Rest, Grpc> {
+    rest: Rest,
+    grpc: Grpc,
+}
+
+impl<Rest, Grpc, RestBody, GrpcBody> Service<Request<Incoming>> for HybridService<Rest, Grpc>
+where
+    Rest: Service<Request<Incoming>, Response = Response<RestBody>>,
+    Grpc: Service<Request<Incoming>, Response = Response<GrpcBody>>,
+    Rest::Error: Into<BoxError>,
+    Grpc::Error: Into<BoxError>,
+{
+    type Response = Response<HybridBody<RestBody, GrpcBody>>;
+    type Error = BoxError;
+    type Future = HybridFuture<Rest::Future, Grpc::Future>;
+
+    fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
+        match self.rest.poll_ready(cx) {
+            Poll::Ready(Ok(())) => match self.grpc.poll_ready(cx) {
+                Poll::Ready(Ok(())) => Poll::Ready(Ok(())),
+                Poll::Ready(Err(e)) => Poll::Ready(Err(e.into())),
+                Poll::Pending => Poll::Pending,
+            },
+            Poll::Ready(Err(e)) => Poll::Ready(Err(e.into())),
+            Poll::Pending => Poll::Pending,
+        }
+    }
+
+    /// When calling the service, gRPC is served if the HTTP request version is HTTP/2
+    /// and if the Content-Type is "application/grpc"; otherwise, the request is served
+    /// as a REST request
+    fn call(&mut self, req: Request<Incoming>) -> Self::Future {
+        match (req.version(), req.headers().get(hyper::header::CONTENT_TYPE)) {
+            (hyper::Version::HTTP_2, Some(hv)) if hv.as_bytes().starts_with(b"application/grpc") => HybridFuture::Grpc {
+                grpc_future: self.grpc.call(req),
+            },
+            _ => HybridFuture::Rest {
+                rest_future: self.rest.call(req),
+            },
+        }
+    }
+}
+
+pin_project! {
+    /// A hybrid HTTP body that will be used in the response type for the
+    /// [`HybridFuture`], i.e., the output of the [`HybridService`]
+    #[project = HybridBodyProj]
+    pub enum HybridBody<RestBody, GrpcBody> {
+        Rest {
+            #[pin]
+            rest_body: RestBody
+        },
+        Grpc {
+            #[pin]
+            grpc_body: GrpcBody
+        },
+    }
+}
+
+impl<RestBody, GrpcBody> http_body::Body for HybridBody<RestBody, GrpcBody>
+where
+    RestBody: http_body::Body + Send + Unpin,
+    GrpcBody: http_body::Body<Data = RestBody::Data> + Send + Unpin,
+    RestBody::Error: Into<BoxError>,
+    GrpcBody::Error: Into<BoxError>,
+{
+    type Data = RestBody::Data;
+    type Error = BoxError;
+
+    fn is_end_stream(&self) -> bool {
+        match self {
+            Self::Rest { rest_body } => rest_body.is_end_stream(),
+            Self::Grpc { grpc_body } => grpc_body.is_end_stream(),
+        }
+    }
+
+    fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
+        match self.project() {
+            HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into),
+            HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into),
+        }
+    }
+
+    fn size_hint(&self) -> http_body::SizeHint {
+        match self {
+            Self::Rest { rest_body } => rest_body.size_hint(),
+            Self::Grpc { grpc_body } => grpc_body.size_hint(),
+        }
+    }
+}
+
+pin_project! {
+    /// A future that accepts an HTTP request as input and returns an HTTP
+    /// response as output for the [`HybridService`]
+    #[project = HybridFutureProj]
+    pub enum HybridFuture<RestFuture, GrpcFuture> {
+        Rest {
+            #[pin]
+            rest_future: RestFuture,
+        },
+        Grpc {
+            #[pin]
+            grpc_future: GrpcFuture,
+        }
+    }
+}
+
+impl<RestFuture, GrpcFuture, RestBody, GrpcBody, RestError, GrpcError> Future for HybridFuture<RestFuture, GrpcFuture>
+where
+    RestFuture: Future<Output = Result<Response<RestBody>, RestError>>,
+    GrpcFuture: Future<Output = Result<Response<GrpcBody>, GrpcError>>,
+    RestError: Into<BoxError>,
+    GrpcError: Into<BoxError>,
+{
+    type Output = Result<Response<HybridBody<RestBody, GrpcBody>>, BoxError>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        match self.project() {
+            HybridFutureProj::Rest { rest_future } => match rest_future.poll(cx) {
+                Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|rest_body| HybridBody::Rest { rest_body }))),
+                Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
+                Poll::Pending => Poll::Pending,
+            },
+            HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) {
+                Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))),
+                Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())),
+                Poll::Pending => Poll::Pending,
+            },
+        }
+    }
+}

rustfs/src/storage/ecfs.rs

@@ -1,5 +1,6 @@
 use bytes::Bytes;
 use ecstore::disk::error::DiskError;
+use ecstore::store::new_object_layer_fn;
 use ecstore::store_api::BucketOptions;
 use ecstore::store_api::CompletePart;
 use ecstore::store_api::DeleteBucketOptions;
@@ -22,12 +23,10 @@ use s3s::S3;
 use s3s::{S3Request, S3Response};
 use std::fmt::Debug;
 use std::str::FromStr;
-use tracing::warn;
 use transform_stream::AsyncTryStream;
 use uuid::Uuid;
 
 use ecstore::error::Result;
-use ecstore::store::ECStore;
 use tracing::debug;
 
 macro_rules! try_ {
@@ -43,13 +42,13 @@ macro_rules! try_ {
 
 #[derive(Debug)]
 pub struct FS {
-    pub store: ECStore,
+    // pub store: ECStore,
 }
 
 impl FS {
-    pub async fn new(address: String, endpoints: Vec<String>) -> Result<Self> {
-        let store: ECStore = ECStore::new(address, endpoints).await?;
-        Ok(Self { store })
+    pub fn new() -> Self {
+        // let store: ECStore = ECStore::new(address, endpoint_pools).await?;
+        Self {}
     }
 }
 #[async_trait::async_trait]
@@ -62,8 +61,15 @@ impl S3 for FS {
     async fn create_bucket(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
         let input = req.input;
 
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         try_!(
-            self.store
+            store
                 .make_bucket(&input.bucket, &MakeBucketOptions { force_create: true })
                 .await
         );
@@ -88,8 +94,14 @@ impl S3 for FS {
     async fn delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
         let input = req.input;
         // TODO: DeleteBucketInput 没有force参数？
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
         try_!(
-            self.store
+            store
                 .delete_bucket(&input.bucket, &DeleteBucketOptions { force: false })
                 .await
         );
@@ -117,7 +129,13 @@ impl S3 for FS {
 
         let objects: Vec<ObjectToDelete> = vec![dobj];
 
-        let (dobjs, _errs) = try_!(self.store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+        let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
 
         // TODO: let errors;
 
@@ -178,8 +196,15 @@ impl S3 for FS {
             })
             .collect();
 
-        let (dobjs, errs) = try_!(self.store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
-        warn!("delete_objects res {:?} {:?}", &dobjs, errs);
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
+        // info!("delete_objects res {:?} {:?}", &dobjs, errs);
 
         let deleted = dobjs
             .iter()
@@ -212,7 +237,14 @@ impl S3 for FS {
         // mc get  1
         let input = req.input;
 
-        if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
             if DiskError::VolumeNotFound.is(&e) {
                 return Err(s3_error!(NoSuchBucket));
             } else {
@@ -248,11 +280,14 @@ impl S3 for FS {
         let h = HeaderMap::new();
         let opts = &ObjectOptions::default();
 
-        let reader = try_!(
-            self.store
-                .get_object_reader(bucket.as_str(), key.as_str(), range, h, opts)
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let reader = try_!(store.get_object_reader(bucket.as_str(), key.as_str(), range, h, opts).await);
 
         let info = reader.object_info;
 
@@ -275,7 +310,14 @@ impl S3 for FS {
     async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
         let input = req.input;
 
-        if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
             if DiskError::VolumeNotFound.is(&e) {
                 return Err(s3_error!(NoSuchBucket));
             } else {
@@ -292,7 +334,14 @@ impl S3 for FS {
         // mc get 2
         let HeadObjectInput { bucket, key, .. } = req.input;
 
-        let info = try_!(self.store.get_object_info(&bucket, &key, &ObjectOptions::default()).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let info = try_!(store.get_object_info(&bucket, &key, &ObjectOptions::default()).await);
         debug!("info {:?}", info);
 
         let content_type = try_!(ContentType::from_str("application/x-msdownload"));
@@ -312,7 +361,14 @@ impl S3 for FS {
     async fn list_buckets(&self, _: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
         // mc ls
 
-        let bucket_infos = try_!(self.store.list_bucket(&BucketOptions {}).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let bucket_infos = try_!(store.list_bucket(&BucketOptions {}).await);
 
         let buckets: Vec<Bucket> = bucket_infos
             .iter()
@@ -360,8 +416,15 @@ impl S3 for FS {
         let prefix = prefix.unwrap_or_default();
         let delimiter = delimiter.unwrap_or_default();
 
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         let object_infos = try_!(
-            self.store
+            store
                 .list_objects_v2(
                     &bucket,
                     &prefix,
@@ -440,9 +503,16 @@ impl S3 for FS {
 
         let reader = PutObjReader::new(body, content_length as usize);
 
-        try_!(self.store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
 
-        // self.store.put_object(bucket, object, data, opts);
+        try_!(store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await);
+
+        // store.put_object(bucket, object, data, opts);
 
         let output = PutObjectOutput { ..Default::default() };
         Ok(S3Response::new(output))
@@ -461,11 +531,15 @@ impl S3 for FS {
 
         debug!("create_multipart_upload meta {:?}", &metadata);
 
-        let MultipartUploadResult { upload_id, .. } = try_!(
-            self.store
-                .new_multipart_upload(&bucket, &key, &ObjectOptions::default())
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let MultipartUploadResult { upload_id, .. } =
+            try_!(store.new_multipart_upload(&bucket, &key, &ObjectOptions::default()).await);
 
         let output = CreateMultipartUploadOutput {
             bucket: Some(bucket),
@@ -500,11 +574,14 @@ impl S3 for FS {
         let data = PutObjReader::new(body, content_length as usize);
         let opts = ObjectOptions::default();
 
-        try_!(
-            self.store
-                .put_object_part(&bucket, &key, &upload_id, part_id, data, &opts)
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        try_!(store.put_object_part(&bucket, &key, &upload_id, part_id, data, &opts).await);
 
         let output = UploadPartOutput { ..Default::default() };
         Ok(S3Response::new(output))
@@ -559,8 +636,15 @@ impl S3 for FS {
             uploaded_parts.push(CompletePart::from(part));
         }
 
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         try_!(
-            self.store
+            store
                 .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
                 .await
         );
@@ -582,9 +666,16 @@ impl S3 for FS {
             bucket, key, upload_id, ..
         } = req.input;
 
+        let layer = new_object_layer_fn();
+        let lock = layer.read().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         let opts = &ObjectOptions::default();
         try_!(
-            self.store
+            store
                 .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts)
                 .await
         );