merge global_disks

rustfs/Cargo.toml

@@ -43,7 +43,26 @@ tower.workspace = true
 tracing-error.workspace = true
 tracing-subscriber.workspace = true
 transform-stream.workspace = true
+uuid = "1.10.0"
 
 [build-dependencies]
 prost-build.workspace = true
-tonic-build.workspace = true
+tonic-build.workspace = true
+http.workspace = true
+bytes.workspace = true
+futures.workspace = true
+futures-util.workspace = true
+# uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] }
+ecstore = { path = "../ecstore" }
+s3s = "0.10.0"
+clap = { version = "4.5.7", features = ["derive"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] }
+hyper-util = { version = "0.1.5", features = [
+    "tokio",
+    "server-auto",
+    "server-graceful",
+] }
+mime = "0.3.17"
+transform-stream = "0.3.0"
+netif = "0.1.6"
+# pin-utils = "0.1.0"

rustfs/src/grpc.rs

@@ -1,5 +1,6 @@
 use ecstore::{
     disk::{DeleteOptions, DiskStore, ReadMultipleReq, ReadOptions, WalkDirOptions},
+    endpoints::EndpointServerPools,
     erasure::{ReadAt, Write},
     peer::{LocalPeerS3Client, PeerS3Client},
     store_api::{BucketOptions, FileInfo, MakeBucketOptions},
@@ -28,7 +29,9 @@ struct NodeService {
     pub local_peer: LocalPeerS3Client,
 }
 
-pub fn make_server(local_disks: Vec<DiskStore>) -> NodeServer<impl Node> {
+pub fn make_server(endpoint_pools: EndpointServerPools) -> NodeServer<impl Node> {
+    // TODO: 参考rustfs创建 https://github.com/rustfs/s3-rustfs/issues/30#issuecomment-2339664516
+    let local_disks = Vec::new();
     let local_peer = LocalPeerS3Client::new(local_disks, None, None);
     NodeServer::new(NodeService { local_peer })
 }

rustfs/src/main.rs

@@ -4,7 +4,7 @@ mod service;
 mod storage;
 
 use clap::Parser;
-use ecstore::{error::Result, store::ECStore};
+use ecstore::{endpoints::EndpointServerPools, error::Result, store::ECStore};
 use grpc::make_server;
 use hyper_util::{
     rt::{TokioExecutor, TokioIo},
@@ -15,7 +15,7 @@ use s3s::{auth::SimpleAuth, service::S3ServiceBuilder};
 use service::hybrid;
 use std::{io::IsTerminal, net::SocketAddr, str::FromStr};
 use tokio::net::TcpListener;
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 use tracing_error::ErrorLayer;
 use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt};
 
@@ -36,10 +36,13 @@ fn setup_tracing() {
 }
 
 fn main() -> Result<()> {
+    //解析获得到的参数
     let opt = config::Opt::parse();
 
+    //设置trace
     setup_tracing();
 
+    //运行参数
     run(opt)
 }
 
@@ -47,7 +50,9 @@ fn main() -> Result<()> {
 async fn run(opt: config::Opt) -> Result<()> {
     debug!("opt: {:?}", &opt);
 
+    //监听地址,端口从参数中获取
     let listener = TcpListener::bind(opt.address.clone()).await?;
+    //获取监听地址
     let local_addr: SocketAddr = listener.local_addr()?;
 
     // let mut domain_name = {
@@ -66,13 +71,15 @@ async fn run(opt: config::Opt) -> Result<()> {
     //         })
     // };
 
-    let store: ECStore = ECStore::new(opt.address.clone(), opt.volumes.clone()).await?;
-    let local_disks = store.local_disks();
-
+    // 用于rpc
+    let (endpoint_pools, _) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?;
     // Setup S3 service
+    // 本项目使用s3s库来实现s3服务
     let service = {
-        let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(store).await?);
-
+        // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?);
+        let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new());
+        //设置AK和SK
+        //其中部份内容从config配置文件中读取
         let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap();
         let mut secret_key = String::from_str(config::DEFAULT_SECRET_KEY).unwrap();
 
@@ -81,7 +88,7 @@ async fn run(opt: config::Opt) -> Result<()> {
             access_key = ak;
             secret_key = sk;
         }
-
+        //显示info信息
         info!("authentication is enabled {}, {}", &access_key, &secret_key);
         b.set_auth(SimpleAuth::from_single(access_key, secret_key));
 
@@ -101,47 +108,58 @@ async fn run(opt: config::Opt) -> Result<()> {
 
         b.build()
     };
+    let rpc_service = make_server(endpoint_pools.clone());
 
-    let hyper_service = service.into_shared();
+    tokio::spawn(async move {
+        let hyper_service = service.into_shared();
 
-    let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, make_server(local_disks)));
+        let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, rpc_service));
 
-    let http_server = ConnBuilder::new(TokioExecutor::new());
-    let graceful = hyper_util::server::graceful::GracefulShutdown::new();
+        let http_server = ConnBuilder::new(TokioExecutor::new());
+        let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
+        let graceful = hyper_util::server::graceful::GracefulShutdown::new();
+        info!("server is running at http://{local_addr}");
 
-    let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
-
-    info!("server is running at http://{local_addr}");
-
-    loop {
-        let (socket, _) = tokio::select! {
-            res =  listener.accept() => {
-                match res {
-                    Ok(conn) => conn,
-                    Err(err) => {
-                        tracing::error!("error accepting connection: {err}");
-                        continue;
+        loop {
+            let (socket, _) = tokio::select! {
+                res =  listener.accept() => {
+                    match res {
+                        Ok(conn) => conn,
+                        Err(err) => {
+                            tracing::error!("error accepting connection: {err}");
+                            continue;
+                        }
                     }
                 }
-            }
-            _ = ctrl_c.as_mut() => {
-                break;
-            }
-        };
+                _ = ctrl_c.as_mut() => {
+                    break;
+                }
+            };
 
-        let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone());
-        let conn = graceful.watch(conn.into_owned());
-        tokio::spawn(async move {
-            let _ = conn.await;
-        });
-    }
+            let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone());
+            let conn = graceful.watch(conn.into_owned());
+            tokio::spawn(async move {
+                let _ = conn.await;
+            });
+        }
+
+        tokio::select! {
+            () = graceful.shutdown() => {
+                 tracing::debug!("Gracefully shutdown!");
+            },
+            () = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
+                 tracing::debug!("Waited 10 seconds for graceful shutdown, aborting...");
+            }
+        }
+    });
+
+    warn!(" init store");
+    // init store
+    ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?;
 
     tokio::select! {
-        () = graceful.shutdown() => {
-             tracing::debug!("Gracefully shutdown!");
-        },
-        () = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
-             tracing::debug!("Waited 10 seconds for graceful shutdown, aborting...");
+        _ = tokio::signal::ctrl_c() => {
+
         }
     }
 

rustfs/src/storage/ecfs.rs

@@ -1,11 +1,13 @@
 use bytes::Bytes;
 use ecstore::disk::error::DiskError;
+use ecstore::store::new_object_layer_fn;
 use ecstore::store_api::BucketOptions;
 use ecstore::store_api::CompletePart;
 use ecstore::store_api::HTTPRangeSpec;
 use ecstore::store_api::MakeBucketOptions;
 use ecstore::store_api::MultipartUploadResult;
 use ecstore::store_api::ObjectOptions;
+use ecstore::store_api::ObjectToDelete;
 use ecstore::store_api::PutObjReader;
 use ecstore::store_api::StorageAPI;
 use futures::pin_mut;
@@ -21,9 +23,9 @@ use s3s::{S3Request, S3Response};
 use std::fmt::Debug;
 use std::str::FromStr;
 use transform_stream::AsyncTryStream;
+use uuid::Uuid;
 
 use ecstore::error::Result;
-use ecstore::store::ECStore;
 use tracing::debug;
 
 macro_rules! try_ {
@@ -39,12 +41,13 @@ macro_rules! try_ {
 
 #[derive(Debug)]
 pub struct FS {
-    pub store: ECStore,
+    // pub store: ECStore,
 }
 
 impl FS {
-    pub async fn new(store: ECStore) -> Result<Self> {
-        Ok(Self { store })
+    pub fn new() -> Self {
+        // let store: ECStore = ECStore::new(address, endpoint_pools).await?;
+        Self {}
     }
 }
 #[async_trait::async_trait]
@@ -57,8 +60,15 @@ impl S3 for FS {
     async fn create_bucket(&self, req: S3Request<CreateBucketInput>) -> S3Result<S3Response<CreateBucketOutput>> {
         let input = req.input;
 
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         try_!(
-            self.store
+            store
                 .make_bucket(&input.bucket, &MakeBucketOptions { force_create: true })
                 .await
         );
@@ -83,24 +93,137 @@ impl S3 for FS {
     async fn delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
         let input = req.input;
 
-        try_!(self.store.delete_bucket(&input.bucket).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+        try_!(store.delete_bucket(&input.bucket).await);
 
         Ok(S3Response::new(DeleteBucketOutput {}))
     }
 
     #[tracing::instrument(level = "debug", skip(self, req))]
     async fn delete_object(&self, req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
-        let _input = req.input;
+        let DeleteObjectInput {
+            bucket, key, version_id, ..
+        } = req.input;
 
-        let output = DeleteObjectOutput::default();
+        let version_id = version_id
+            .as_ref()
+            .map(|v| match Uuid::parse_str(v) {
+                Ok(id) => Some(id),
+                Err(_) => None,
+            })
+            .unwrap_or_default();
+        let dobj = ObjectToDelete {
+            object_name: key,
+            version_id,
+        };
+
+        let objects: Vec<ObjectToDelete> = vec![dobj];
+
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+        let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
+
+        // TODO: let errors;
+
+        let (delete_marker, version_id) = {
+            if let Some((a, b)) = dobjs
+                .iter()
+                .map(|v| {
+                    let delete_marker = {
+                        if v.delete_marker {
+                            Some(true)
+                        } else {
+                            None
+                        }
+                    };
+
+                    let version_id = v.version_id.clone();
+
+                    (delete_marker, version_id)
+                })
+                .next()
+            {
+                (a, b)
+            } else {
+                (None, None)
+            }
+        };
+
+        let output = DeleteObjectOutput {
+            delete_marker,
+            version_id,
+            ..Default::default()
+        };
         Ok(S3Response::new(output))
     }
 
     #[tracing::instrument(level = "debug", skip(self, req))]
     async fn delete_objects(&self, req: S3Request<DeleteObjectsInput>) -> S3Result<S3Response<DeleteObjectsOutput>> {
-        let _input = req.input;
+        // info!("delete_objects args {:?}", req.input);
 
-        let output = DeleteObjectsOutput { ..Default::default() };
+        let DeleteObjectsInput { bucket, delete, .. } = req.input;
+
+        let objects: Vec<ObjectToDelete> = delete
+            .objects
+            .iter()
+            .map(|v| {
+                let version_id = v
+                    .version_id
+                    .as_ref()
+                    .map(|v| match Uuid::parse_str(v) {
+                        Ok(id) => Some(id),
+                        Err(_) => None,
+                    })
+                    .unwrap_or_default();
+                ObjectToDelete {
+                    object_name: v.key.clone(),
+                    version_id: version_id,
+                }
+            })
+            .collect();
+
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await);
+        // info!("delete_objects res {:?} {:?}", &dobjs, errs);
+
+        let deleted = dobjs
+            .iter()
+            .map(|v| DeletedObject {
+                delete_marker: {
+                    if v.delete_marker {
+                        Some(true)
+                    } else {
+                        None
+                    }
+                },
+                delete_marker_version_id: v.delete_marker_version_id.clone(),
+                key: Some(v.object_name.clone()),
+                version_id: v.version_id.clone(),
+            })
+            .collect();
+
+        // TODO: let errors;
+
+        let output = DeleteObjectsOutput {
+            deleted: Some(deleted),
+            // errors,
+            ..Default::default()
+        };
         Ok(S3Response::new(output))
     }
 
@@ -109,7 +232,14 @@ impl S3 for FS {
         // mc get  1
         let input = req.input;
 
-        if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
             if DiskError::VolumeNotFound.is(&e) {
                 return Err(s3_error!(NoSuchBucket));
             } else {
@@ -145,11 +275,14 @@ impl S3 for FS {
         let h = HeaderMap::new();
         let opts = &ObjectOptions::default();
 
-        let reader = try_!(
-            self.store
-                .get_object_reader(bucket.as_str(), key.as_str(), range, h, opts)
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let reader = try_!(store.get_object_reader(bucket.as_str(), key.as_str(), range, h, opts).await);
 
         let info = reader.object_info;
 
@@ -172,7 +305,14 @@ impl S3 for FS {
     async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
         let input = req.input;
 
-        if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await {
             if DiskError::VolumeNotFound.is(&e) {
                 return Err(s3_error!(NoSuchBucket));
             } else {
@@ -189,7 +329,14 @@ impl S3 for FS {
         // mc get 2
         let HeadObjectInput { bucket, key, .. } = req.input;
 
-        let info = try_!(self.store.get_object_info(&bucket, &key, &ObjectOptions::default()).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let info = try_!(store.get_object_info(&bucket, &key, &ObjectOptions::default()).await);
         debug!("info {:?}", info);
 
         let content_type = try_!(ContentType::from_str("application/x-msdownload"));
@@ -209,7 +356,14 @@ impl S3 for FS {
     async fn list_buckets(&self, _: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
         // mc ls
 
-        let bucket_infos = try_!(self.store.list_bucket(&BucketOptions {}).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let bucket_infos = try_!(store.list_bucket(&BucketOptions {}).await);
 
         let buckets: Vec<Bucket> = bucket_infos
             .iter()
@@ -257,8 +411,15 @@ impl S3 for FS {
         let prefix = prefix.unwrap_or_default();
         let delimiter = delimiter.unwrap_or_default();
 
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         let object_infos = try_!(
-            self.store
+            store
                 .list_objects_v2(
                     &bucket,
                     &prefix,
@@ -337,9 +498,16 @@ impl S3 for FS {
 
         let reader = PutObjReader::new(body, content_length as usize);
 
-        try_!(self.store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await);
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
 
-        // self.store.put_object(bucket, object, data, opts);
+        try_!(store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await);
+
+        // store.put_object(bucket, object, data, opts);
 
         let output = PutObjectOutput { ..Default::default() };
         Ok(S3Response::new(output))
@@ -358,11 +526,15 @@ impl S3 for FS {
 
         debug!("create_multipart_upload meta {:?}", &metadata);
 
-        let MultipartUploadResult { upload_id, .. } = try_!(
-            self.store
-                .new_multipart_upload(&bucket, &key, &ObjectOptions::default())
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        let MultipartUploadResult { upload_id, .. } =
+            try_!(store.new_multipart_upload(&bucket, &key, &ObjectOptions::default()).await);
 
         let output = CreateMultipartUploadOutput {
             bucket: Some(bucket),
@@ -397,11 +569,14 @@ impl S3 for FS {
         let data = PutObjReader::new(body, content_length as usize);
         let opts = ObjectOptions::default();
 
-        try_!(
-            self.store
-                .put_object_part(&bucket, &key, &upload_id, part_id, data, &opts)
-                .await
-        );
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
+        try_!(store.put_object_part(&bucket, &key, &upload_id, part_id, data, &opts).await);
 
         let output = UploadPartOutput { ..Default::default() };
         Ok(S3Response::new(output))
@@ -456,8 +631,15 @@ impl S3 for FS {
             uploaded_parts.push(CompletePart::from(part));
         }
 
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         try_!(
-            self.store
+            store
                 .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts)
                 .await
         );
@@ -479,9 +661,16 @@ impl S3 for FS {
             bucket, key, upload_id, ..
         } = req.input;
 
+        let layer = new_object_layer_fn();
+        let lock = layer.lock().await;
+        let store = match lock.as_ref() {
+            Some(s) => s,
+            None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))),
+        };
+
         let opts = &ObjectOptions::default();
         try_!(
-            self.store
+            store
                 .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts)
                 .await
         );