refactor: Restructure project layout and clean up dependencies (#30)

This commit introduces a significant reorganization of the project structure to improve maintainability and clarity.

Key changes include:
- Adjusted the directory layout for a more logical module organization.
- Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times.
- Updated import paths and configuration files to reflect the structural changes.

crates/e2e_test/src/lib.rs

@@ -0,0 +1,15 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod reliant;

crates/e2e_test/src/reliant/README.md

@@ -0,0 +1 @@
+The test cases in this dir need to run the cluster

crates/e2e_test/src/reliant/lock.rs

@@ -0,0 +1,88 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use rustfs_lock::{
+    drwmutex::Options,
+    lock_args::LockArgs,
+    namespace_lock::{NsLockMap, new_nslock},
+    new_lock_api,
+};
+use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
+use std::{error::Error, sync::Arc, time::Duration};
+use tokio::sync::RwLock;
+use tonic::Request;
+
+const CLUSTER_ADDR: &str = "http://localhost:9000";
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_lock_unlock_rpc() -> Result<(), Box<dyn Error>> {
+    let args = LockArgs {
+        uid: "1111".to_string(),
+        resources: vec!["dandan".to_string()],
+        owner: "dd".to_string(),
+        source: "".to_string(),
+        quorum: 3,
+    };
+    let args = serde_json::to_string(&args)?;
+
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    println!("got client");
+    let request = Request::new(GenerallyLockRequest { args: args.clone() });
+
+    println!("start request");
+    let response = client.lock(request).await?.into_inner();
+    println!("request ended");
+    if let Some(error_info) = response.error_info {
+        panic!("can not get lock: {error_info}");
+    }
+
+    let request = Request::new(GenerallyLockRequest { args });
+    let response = client.un_lock(request).await?.into_inner();
+    if let Some(error_info) = response.error_info {
+        panic!("can not get un_lock: {error_info}");
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn test_lock_unlock_ns_lock() -> Result<(), Box<dyn Error>> {
+    let url = url::Url::parse("http://127.0.0.1:9000/data")?;
+    let locker = new_lock_api(false, Some(url));
+    let ns_mutex = Arc::new(RwLock::new(NsLockMap::new(true)));
+    let ns = new_nslock(
+        Arc::clone(&ns_mutex),
+        "local".to_string(),
+        "dandan".to_string(),
+        vec!["foo".to_string()],
+        vec![locker],
+    )
+    .await;
+    assert!(
+        ns.0.write()
+            .await
+            .get_lock(&Options {
+                timeout: Duration::from_secs(5),
+                retry_interval: Duration::from_secs(1),
+            })
+            .await
+            .unwrap()
+    );
+
+    ns.0.write().await.un_lock().await.unwrap();
+    Ok(())
+}

crates/e2e_test/src/reliant/mod.rs

@@ -0,0 +1,16 @@
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod lock;
+mod node_interact_test;

crates/e2e_test/src/reliant/node_interact_test.rs

@@ -0,0 +1,205 @@
+#![cfg(test)]
+// Copyright 2024 RustFS Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use futures::future::join_all;
+use rmp_serde::{Deserializer, Serializer};
+use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
+use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
+use rustfs_protos::proto_gen::node_service::WalkDirRequest;
+use rustfs_protos::{
+    models::{PingBody, PingBodyBuilder},
+    node_service_time_out_client,
+    proto_gen::node_service::{
+        ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
+    },
+};
+use serde::{Deserialize, Serialize};
+use std::error::Error;
+use std::io::Cursor;
+use tokio::spawn;
+use tonic::Request;
+use tonic::codegen::tokio_stream::StreamExt;
+
+const CLUSTER_ADDR: &str = "http://localhost:9000";
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn ping() -> Result<(), Box<dyn Error>> {
+    let mut fbb = flatbuffers::FlatBufferBuilder::new();
+    let payload = fbb.create_vector(b"hello world");
+
+    let mut builder = PingBodyBuilder::new(&mut fbb);
+    builder.add_payload(payload);
+    let root = builder.finish();
+    fbb.finish(root, None);
+
+    let finished_data = fbb.finished_data();
+
+    let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
+    assert!(decoded_payload.is_ok());
+
+    // Create client
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+
+    // Construct PingRequest
+    let request = Request::new(PingRequest {
+        version: 1,
+        body: bytes::Bytes::copy_from_slice(finished_data),
+    });
+
+    // Send request and get response
+    let response: PingResponse = client.ping(request).await?.into_inner();
+
+    // Print response
+    let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
+    if let Err(e) = ping_response_body {
+        eprintln!("{e}");
+    } else {
+        println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn make_volume() -> Result<(), Box<dyn Error>> {
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    let request = Request::new(MakeVolumeRequest {
+        disk: "data".to_string(),
+        volume: "dandan".to_string(),
+    });
+
+    let response = client.make_volume(request).await?.into_inner();
+    if response.success {
+        println!("success");
+    } else {
+        println!("failed: {:?}", response.error);
+    }
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn list_volumes() -> Result<(), Box<dyn Error>> {
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    let request = Request::new(ListVolumesRequest {
+        disk: "data".to_string(),
+    });
+
+    let response = client.list_volumes(request).await?.into_inner();
+    let volume_infos: Vec<VolumeInfo> = response
+        .volume_infos
+        .into_iter()
+        .filter_map(|json_str| serde_json::from_str::<VolumeInfo>(&json_str).ok())
+        .collect();
+
+    println!("{volume_infos:?}");
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn walk_dir() -> Result<(), Box<dyn Error>> {
+    println!("walk_dir");
+    // TODO: use writer
+    let opts = WalkDirOptions {
+        bucket: "dandan".to_owned(),
+        base_dir: "".to_owned(),
+        recursive: true,
+        ..Default::default()
+    };
+    let (rd, mut wr) = tokio::io::duplex(1024);
+    let mut buf = Vec::new();
+    opts.serialize(&mut Serializer::new(&mut buf))?;
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    let request = Request::new(WalkDirRequest {
+        disk: "/home/dandan/code/rust/s3-rustfs/target/debug/data".to_string(),
+        walk_dir_options: buf.into(),
+    });
+    let mut response = client.walk_dir(request).await?.into_inner();
+
+    let job1 = spawn(async move {
+        let mut out = MetacacheWriter::new(&mut wr);
+        loop {
+            match response.next().await {
+                Some(Ok(resp)) => {
+                    if !resp.success {
+                        println!("{}", resp.error_info.unwrap_or("".to_string()));
+                    }
+                    let entry = serde_json::from_str::<MetaCacheEntry>(&resp.meta_cache_entry)
+                        .map_err(|_e| std::io::Error::other(format!("Unexpected response: {response:?}")))
+                        .unwrap();
+                    out.write_obj(&entry).await.unwrap();
+                }
+                None => {
+                    let _ = out.close().await;
+                    break;
+                }
+                _ => {
+                    println!("Unexpected response: {response:?}");
+                    let _ = out.close().await;
+                    break;
+                }
+            }
+        }
+    });
+    let job2 = spawn(async move {
+        let mut reader = MetacacheReader::new(rd);
+        while let Ok(Some(entry)) = reader.peek().await {
+            println!("{entry:?}");
+        }
+    });
+
+    join_all(vec![job1, job2]).await;
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn read_all() -> Result<(), Box<dyn Error>> {
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    let request = Request::new(ReadAllRequest {
+        disk: "data".to_string(),
+        volume: "ff".to_string(),
+        path: "format.json".to_string(),
+    });
+
+    let response = client.read_all(request).await?.into_inner();
+    let volume_infos = response.data;
+
+    println!("{}", response.success);
+    println!("{volume_infos:?}");
+    Ok(())
+}
+
+#[tokio::test]
+#[ignore = "requires running RustFS server at localhost:9000"]
+async fn storage_info() -> Result<(), Box<dyn Error>> {
+    let mut client = node_service_time_out_client(&CLUSTER_ADDR.to_string()).await?;
+    let request = Request::new(LocalStorageInfoRequest { metrics: true });
+
+    let response = client.local_storage_info(request).await?.into_inner();
+    if !response.success {
+        println!("{:?}", response.error_info);
+        return Ok(());
+    }
+    let info = response.storage_info;
+
+    let mut buf = Deserializer::new(Cursor::new(info));
+    let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
+    println!("{storage_info:?}");
+    Ok(())
+}