From 39c3a72cc00c6884b42f245d06763cd23b760a16 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 30 Aug 2024 16:18:24 +0800 Subject: [PATCH 01/70] todo: quorum err check --- ecstore/src/lib.rs | 1 + ecstore/src/quorum.rs | 90 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 ecstore/src/quorum.rs diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 1371d1f14..fe803a916 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -7,6 +7,7 @@ mod erasure; pub mod error; mod file_meta; mod peer; +mod quorum; pub mod set_disk; mod sets; pub mod store; diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs new file mode 100644 index 000000000..8b56baa25 --- /dev/null +++ b/ecstore/src/quorum.rs @@ -0,0 +1,90 @@ +use crate::{disk::error::DiskError, error::Error}; +use std::collections::HashMap; + +type CheckErrorFn = fn(e: &Error) -> bool; + +#[derive(Debug, thiserror::Error)] +enum QuorumError { + #[error("Read quorum not met")] + Read, + #[error("disk not found")] + Write, +} + +fn is_file_not_found(e: &Error) -> bool { + DiskError::FileNotFound.is(e) +} + +// 用于检查错误是否被忽略的函数 +fn is_err_ignored(err: &Error, ignored_errs: &[CheckErrorFn]) -> bool { + ignored_errs.iter().any(|&ignored_err| ignored_err(err)) +} + +// 减少错误数量并返回出现次数最多的错误 +fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usize, Option) { + let mut error_counts: HashMap = HashMap::new(); + let mut error_map: HashMap = HashMap::new(); + let nil = "nil".to_string(); + for operr in errs.iter() { + if operr.is_none() { + *error_counts.entry(nil.clone()).or_insert(0) += 1; + continue; + } + + let err = operr.as_ref().unwrap(); + + if is_err_ignored(err, &ignored_errs) { + continue; + } + + let errstr = err.to_string(); + + *error_map.entry(errstr.clone()).or_insert(err); + *error_counts.entry(errstr.clone()).or_insert(0) += 1; + } + + let mut max = 0; + let mut max_err = nil.clone(); + for (&ref err, &count) in error_counts.iter() { + if count > max || (count == max && *err == nil) { + max = count; + max_err = err.clone(); + } + } + + if let Some(c) = error_counts.get(&max_err) { + if let Some(&err) = error_map.get(&max_err) { + // return (*c, Some(err.clone())); + return (*c, None); + } + + return (*c, None); + } + + (0, None) +} + +// 根据quorum验证错误数量 +fn reduce_quorum_errs<'a>( + errs: &'a Vec>, + ignored_errs: &[CheckErrorFn], + quorum: usize, + quorum_err: Error, +) -> Option { + let (max_count, max_err) = reduce_errs(errs, ignored_errs); + if max_count >= quorum { + max_err + } else { + Some(quorum_err) + } +} + +// 根据读quorum验证错误数量 +fn reduce_read_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], read_quorum: usize) -> Option { + reduce_quorum_errs(errs, ignored_errs, read_quorum, Error::new(QuorumError::Read)) +} + +// 根据写quorum验证错误数量 +fn reduce_write_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], write_quorum: usize) -> Option { + reduce_quorum_errs(errs, ignored_errs, write_quorum, Error::new(QuorumError::Write)) +} From 8c24e38ee112fb44b317ae33ab5d6627f99a15be Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 30 Aug 2024 16:18:47 +0800 Subject: [PATCH 02/70] todo: quorum err check --- ecstore/src/quorum.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 8b56baa25..0ab57be71 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -54,8 +54,7 @@ fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usi if let Some(c) = error_counts.get(&max_err) { if let Some(&err) = error_map.get(&max_err) { - // return (*c, Some(err.clone())); - return (*c, None); + return (*c, Some(err.clone())); } return (*c, None); From da627f24a3d735ef7df412c3f2fcdf81db29db36 Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 2 Sep 2024 17:05:22 +0800 Subject: [PATCH 03/70] fix:reduce_write_quorum_errs --- ecstore/src/quorum.rs | 55 +++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 0ab57be71..e50f0c82a 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -1,7 +1,7 @@ use crate::{disk::error::DiskError, error::Error}; use std::collections::HashMap; -type CheckErrorFn = fn(e: &Error) -> bool; +pub type CheckErrorFn = fn(e: &Error) -> bool; #[derive(Debug, thiserror::Error)] enum QuorumError { @@ -11,7 +11,7 @@ enum QuorumError { Write, } -fn is_file_not_found(e: &Error) -> bool { +pub fn is_file_not_found(e: &Error) -> bool { DiskError::FileNotFound.is(e) } @@ -21,13 +21,14 @@ fn is_err_ignored(err: &Error, ignored_errs: &[CheckErrorFn]) -> bool { } // 减少错误数量并返回出现次数最多的错误 -fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usize, Option) { +fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usize, Option) { let mut error_counts: HashMap = HashMap::new(); - let mut error_map: HashMap = HashMap::new(); + let mut error_map: HashMap = HashMap::new(); // 存err位置 let nil = "nil".to_string(); - for operr in errs.iter() { + for (i, operr) in errs.iter().enumerate() { if operr.is_none() { *error_counts.entry(nil.clone()).or_insert(0) += 1; + let _ = *error_map.entry(nil.clone()).or_insert(i); continue; } @@ -39,7 +40,7 @@ fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usi let errstr = err.to_string(); - *error_map.entry(errstr.clone()).or_insert(err); + let _ = *error_map.entry(errstr.clone()).or_insert(i); *error_counts.entry(errstr.clone()).or_insert(0) += 1; } @@ -52,38 +53,52 @@ fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usi } } - if let Some(c) = error_counts.get(&max_err) { + if let Some(&c) = error_counts.get(&max_err) { if let Some(&err) = error_map.get(&max_err) { - return (*c, Some(err.clone())); + return (c, Some(err)); } - return (*c, None); + return (c, None); } (0, None) } // 根据quorum验证错误数量 -fn reduce_quorum_errs<'a>( - errs: &'a Vec>, - ignored_errs: &[CheckErrorFn], - quorum: usize, - quorum_err: Error, -) -> Option { +fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], quorum: usize) -> Option { let (max_count, max_err) = reduce_errs(errs, ignored_errs); if max_count >= quorum { max_err } else { - Some(quorum_err) + None } } // 根据读quorum验证错误数量 -fn reduce_read_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], read_quorum: usize) -> Option { - reduce_quorum_errs(errs, ignored_errs, read_quorum, Error::new(QuorumError::Read)) +pub fn reduce_read_quorum_errs( + errs: &Vec>, + ignored_errs: &[CheckErrorFn], + read_quorum: usize, +) -> Result { + let idx = reduce_quorum_errs(errs, ignored_errs, read_quorum); + if idx.is_none() { + return Err(Error::new(QuorumError::Read)); + } + + Ok(idx.unwrap()) } // 根据写quorum验证错误数量 -fn reduce_write_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], write_quorum: usize) -> Option { - reduce_quorum_errs(errs, ignored_errs, write_quorum, Error::new(QuorumError::Write)) +pub fn reduce_write_quorum_errs( + errs: &Vec>, + ignored_errs: &[CheckErrorFn], + write_quorum: usize, +) -> Result { + let idx = reduce_quorum_errs(errs, ignored_errs, write_quorum); + + if idx.is_none() { + return Err(Error::new(QuorumError::Write)); + } + + Ok(idx.unwrap()) } From ced17f6deb3d937ff16d315a9539d6b6a7114301 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 3 Sep 2024 09:12:32 +0800 Subject: [PATCH 04/70] fix:reduce_write_quorum_errs --- ecstore/src/erasure.rs | 17 ++++++++--------- ecstore/src/quorum.rs | 6 +++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 6c23842af..660849913 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,4 +1,5 @@ use crate::error::{Error, Result, StdError}; +use crate::quorum::{object_ignored_errs, reduce_write_quorum_errs}; use bytes::Bytes; use futures::future::join_all; use futures::{Stream, StreamExt}; @@ -49,7 +50,7 @@ impl Erasure { writers: &mut [W], // block_size: usize, total_size: usize, - _write_quorum: usize, + write_quorum: usize, ) -> Result where S: Stream> + Send + Sync + 'static, @@ -87,17 +88,15 @@ impl Erasure { for (i, w) in writers.iter_mut().enumerate() { match w.write_all(blocks[i].as_ref()).await { Ok(_) => errs.push(None), - Err(e) => errs.push(Some(e)), + Err(e) => errs.push(Some(Error::new(e))), } } - // debug!("{} encode_data write errs:{:?}", self.id, errs); - // // TODO: reduceWriteQuorumErrs - // for err in errs.iter() { - // if err.is_some() { - // return Err(Error::msg("message")); - // } - // } + let err_idx = reduce_write_quorum_errs(&errs, object_ignored_errs().as_slice(), write_quorum)?; + if errs[err_idx].is_some() { + let err = errs[err_idx].take().unwrap(); + return Err(err); + } } Err(e) => return Err(Error::from_std_error(e)), } diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index e50f0c82a..124ccfcba 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -15,6 +15,10 @@ pub fn is_file_not_found(e: &Error) -> bool { DiskError::FileNotFound.is(e) } +pub fn object_ignored_errs() -> Vec { + vec![is_file_not_found] +} + // 用于检查错误是否被忽略的函数 fn is_err_ignored(err: &Error, ignored_errs: &[CheckErrorFn]) -> bool { ignored_errs.iter().any(|&ignored_err| ignored_err(err)) @@ -76,7 +80,7 @@ fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], // 根据读quorum验证错误数量 pub fn reduce_read_quorum_errs( - errs: &Vec>, + errs: &mut Vec>, ignored_errs: &[CheckErrorFn], read_quorum: usize, ) -> Result { From 29359455853ec1478722d4fec903abbbd99344aa Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 3 Sep 2024 13:34:03 +0800 Subject: [PATCH 05/70] fix:reduce_write_quorum_errs --- ecstore/src/disk/local.rs | 12 ++++++++++-- ecstore/src/erasure.rs | 2 ++ ecstore/src/lib.rs | 1 + ecstore/src/storage_class.rs | 28 +++++++++++++++++++++++++++ ecstore/src/store.rs | 5 ++++- ecstore/src/store_init.rs | 9 --------- ecstore/src/writer.rs | 37 +++++------------------------------- scripts/run.sh | 10 +++++----- 8 files changed, 55 insertions(+), 49 deletions(-) create mode 100644 ecstore/src/storage_class.rs diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 3e8524aa7..ebcc1267a 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -182,7 +182,9 @@ impl LocalDisk { self.move_to_trash(delete_path, recursive, immediate_purge).await?; } else { if delete_path.is_dir() { + debug!("delete_file remove_dir {:?}", &delete_path); if let Err(err) = fs::remove_dir(&delete_path).await { + debug!("delete_file remove_dir err {:?} err: {:?}", &delete_path, err); match err.kind() { ErrorKind::NotFound => (), // ErrorKind::DirectoryNotEmpty => (), @@ -194,6 +196,7 @@ impl LocalDisk { } } } + debug!("delete_file remove_dir done {:?}", &delete_path); } else { if let Err(err) = fs::remove_file(&delete_path).await { match err.kind() { @@ -483,6 +486,13 @@ impl DiskAPI for LocalDisk { } async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { + let volpath = self.get_bucket_path(&volume)?; + // check exists + fs::metadata(&volpath).await.map_err(|e| match e.kind() { + ErrorKind::NotFound => Error::new(DiskError::VolumeNotFound), + _ => Error::new(e), + })?; + let fpath = self.get_object_path(volume, path)?; debug!("CreateFile fpath: {:?}", fpath); @@ -678,8 +688,6 @@ impl DiskAPI for LocalDisk { } } - warn!("get meta {:?}", &meta); - let mut skip_parent = dst_volume_path.clone(); if !dst_buf.is_empty() { skip_parent = PathBuf::from(&dst_file_path.parent().unwrap_or(Path::new("/"))); diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 660849913..dfff7a2b8 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -92,6 +92,8 @@ impl Erasure { } } + warn!("Erasure encode errs {:?}", errs); + let err_idx = reduce_write_quorum_errs(&errs, object_ignored_errs().as_slice(), write_quorum)?; if errs[err_idx].is_some() { let err = errs[err_idx].take().unwrap(); diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index fe803a916..87e117fb3 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -10,6 +10,7 @@ mod peer; mod quorum; pub mod set_disk; mod sets; +mod storage_class; pub mod store; pub mod store_api; mod store_init; diff --git a/ecstore/src/storage_class.rs b/ecstore/src/storage_class.rs new file mode 100644 index 000000000..71cbb4b8c --- /dev/null +++ b/ecstore/src/storage_class.rs @@ -0,0 +1,28 @@ +// use crate::error::{Error, Result}; + +// default_partiy_count 默认配置，根据磁盘总数分配校验磁盘数量 +pub fn default_partiy_count(drive: usize) -> usize { + match drive { + 1 => 0, + 2 | 3 => 1, + 4 | 5 => 2, + 6 | 7 => 3, + _ => 4, + } +} + +// Define the minimum number of parity drives required. +// const MIN_PARITY_DRIVES: usize = 0; + +// // ValidateParity validates standard storage class parity. +// pub fn validate_parity(ss_parity: usize, set_drive_count: usize) -> Result<()> { +// // if ss_parity > 0 && ss_parity < MIN_PARITY_DRIVES { +// // return Err(Error::msg(format!("parity {} 应该大于等于 {}", ss_parity, MIN_PARITY_DRIVES))); +// // } + +// if ss_parity > set_drive_count / 2 { +// return Err(Error::msg(format!("parity {} 应该小于等于 {}", ss_parity, set_drive_count / 2))); +// } + +// Ok(()) +// } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 8fdcc2fbd..bc8343b26 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -6,6 +6,7 @@ use crate::{ error::{Error, Result}, peer::{PeerS3Client, S3PeerSys}, sets::Sets, + storage_class::default_partiy_count, store_api::{ BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, @@ -48,7 +49,9 @@ impl ECStore { for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() { // TODO: read from config parseStorageClass - let partiy_count = store_init::default_partiy_count(pool_eps.drives_per_set); + let partiy_count = default_partiy_count(pool_eps.drives_per_set); + + // validate_parity(partiy_count, pool_eps.drives_per_set)?; let (disks, errs) = crate::store_init::init_disks( &pool_eps.endpoints, diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index a1178d888..76538a126 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -175,15 +175,6 @@ fn check_format_erasure_value(format: &FormatV3) -> Result<()> { Ok(()) } -pub fn default_partiy_count(drive: usize) -> usize { - match drive { - 1 => 0, - 2 | 3 => 1, - 4 | 5 => 2, - 6 | 7 => 3, - _ => 4, - } -} // read_format_file_all 读取所有foramt.json async fn read_format_file_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); diff --git a/ecstore/src/writer.rs b/ecstore/src/writer.rs index 48257dc05..a10b55cb4 100644 --- a/ecstore/src/writer.rs +++ b/ecstore/src/writer.rs @@ -39,38 +39,11 @@ impl<'a> AsyncWrite for AppendWriter<'a> { let mut fut = Box::pin(self.async_write(buf)); debug!("AsyncWrite poll_write {}, buf:{}", self.disk.id(), buf.len()); - // while let Poll::Ready(e) = fut.as_mut().poll(cx) { - // let a = match e { - // Ok(_) => { - // debug!("Ready ok {}", self.disk.id()); - // Poll::Ready(Ok(buf.len())) - // } - // Err(e) => { - // debug!("Ready err {}", self.disk.id()); - // Poll::Ready(Err(e)) - // } - // }; - - // return a; - // } - - // Poll::Pending - - match fut.as_mut().poll(cx) { - Poll::Pending => { - debug!("Pending {}", self.disk.id()); - Poll::Pending - } - Poll::Ready(e) => match e { - Ok(_) => { - debug!("Ready ok {}", self.disk.id()); - Poll::Ready(Ok(buf.len())) - } - Err(e) => { - debug!("Ready err {}", self.disk.id()); - Poll::Ready(Err(e)) - } - }, + let mut fut = self.get_mut().async_write(buf); + match futures::future::poll_fn(|cx| fut.as_mut().poll(cx)).start(cx) { + Ready(Ok(n)) => Ready(Ok(n)), + Ready(Err(e)) => Ready(Err(e)), + Pending => Pending, } } diff --git a/scripts/run.sh b/scripts/run.sh index f3a1c2c7a..b48b9abc7 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -3,8 +3,8 @@ mkdir -p ./target/volume/test mkdir -p ./target/volume/test{0..4} -DATA_DIR="./target/volume/test" -# DATA_DIR="./target/volume/test{0...4}" +# DATA_DIR="./target/volume/test" +DATA_DIR="./target/volume/test{0...4}" if [ -n "$1" ]; then DATA_DIR="$1" @@ -14,11 +14,11 @@ if [ -z "$RUST_LOG" ]; then export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug" fi -cargo run "$DATA_DIR" +# cargo run "$DATA_DIR" # -- --access-key AKEXAMPLERUSTFS \ # --secret-key SKEXAMPLERUSTFS \ # --address 0.0.0.0:9010 \ # --domain-name 127.0.0.1:9010 \ - "$DATA_DIR" + # "$DATA_DIR" -# cargo run "$DATA_DIR" \ No newline at end of file +cargo run "$DATA_DIR" \ No newline at end of file From 7f35685b355cf36c45bd1206d4a86b58e893dbfe Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 13 Sep 2024 13:30:10 +0800 Subject: [PATCH 06/70] add delete_bucket opts --- ecstore/src/disk/error.rs | 3 +++ ecstore/src/disk/local.rs | 15 ++++++++++++++- ecstore/src/peer.rs | 36 ++++++++++++++++++++++++++++-------- ecstore/src/sets.rs | 7 ++++--- ecstore/src/store.rs | 10 +++++----- ecstore/src/store_api.rs | 6 +++++- rustfs/src/storage/ecfs.rs | 9 +++++++-- 7 files changed, 66 insertions(+), 20 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 0b5e36301..c8c0c84b1 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -31,6 +31,9 @@ pub enum DiskError { #[error("volume not found")] VolumeNotFound, + + #[error("volume not empty")] + VolumeNotEmpty, } impl DiskError { diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 24f54e0a2..80c53948b 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -972,7 +972,20 @@ impl DiskAPI for LocalDisk { let p = self.get_bucket_path(volume)?; // TODO: 不能用递归删除，如果目录下面有文件，返回errVolumeNotEmpty - fs::remove_dir_all(&p).await?; + + if let Err(err) = fs::remove_dir_all(&p).await { + match err.kind() { + ErrorKind::NotFound => (), + // ErrorKind::DirectoryNotEmpty => (), + kind => { + if kind.to_string() == "directory not empty" { + return Err(Error::new(DiskError::VolumeNotEmpty)); + } + + return Err(Error::from(err)); + } + } + } Ok(()) } diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 7329923cb..878eb8287 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -8,7 +8,7 @@ use crate::{ disk::{self, error::DiskError, DiskStore, VolumeInfo}, endpoints::{EndpointServerPools, Node}, error::{Error, Result}, - store_api::{BucketInfo, BucketOptions, MakeBucketOptions}, + store_api::{BucketInfo, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, }; type Client = Arc>; @@ -17,7 +17,7 @@ type Client = Arc>; pub trait PeerS3Client: Debug + Sync + Send + 'static { async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn list_bucket(&self, opts: &BucketOptions) -> Result>; - async fn delete_bucket(&self, bucket: &str) -> Result<()>; + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; fn get_pools(&self) -> Vec; } @@ -140,10 +140,10 @@ impl PeerS3Client for S3PeerSys { Ok(buckets) } - async fn delete_bucket(&self, bucket: &str) -> Result<()> { + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { - futures.push(cli.delete_bucket(bucket)); + futures.push(cli.delete_bucket(bucket, &opts)); } let mut errors = Vec::with_capacity(self.clients.len()); @@ -350,7 +350,7 @@ impl PeerS3Client for LocalPeerS3Client { .ok_or(Error::new(DiskError::VolumeNotFound)) } - async fn delete_bucket(&self, bucket: &str) -> Result<()> { + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.local_disks.len()); for disk in self.local_disks.iter() { @@ -361,14 +361,34 @@ impl PeerS3Client for LocalPeerS3Client { let mut errs = Vec::new(); + let mut recreate = false; + for res in results { match res { Ok(_) => errs.push(None), - Err(e) => errs.push(Some(e)), + Err(e) => { + if DiskError::VolumeNotEmpty.is(&e) { + recreate = true; + } + errs.push(Some(e)) + } } } - // TODO: errVolumeNotEmpty 不删除，把已经删除的重新创建 + // errVolumeNotEmpty 不删除，把已经删除的重新创建 + + let mut idx = 0; + for err in errs { + if err.is_none() && recreate { + let _ = self.local_disks[idx].make_volume(bucket).await; + } + + idx += 1; + } + + if recreate { + return Err(Error::new(DiskError::VolumeNotEmpty)); + } // TODO: reduceWriteQuorumErrs @@ -404,7 +424,7 @@ impl PeerS3Client for RemotePeerS3Client { unimplemented!() } - async fn delete_bucket(&self, _bucket: &str) -> Result<()> { + async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } } diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 63815089d..112ae61f9 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -13,8 +13,9 @@ use crate::{ error::{Error, Result}, set_disk::SetDisks, store_api::{ - BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsV2Info, - MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, + BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, + ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, + PutObjReader, StorageAPI, }, utils::hash, }; @@ -297,7 +298,7 @@ impl StorageAPI for Sets { .await } - async fn delete_bucket(&self, _bucket: &str) -> Result<()> { + async fn delete_bucket(&self, _bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 8fdcc2fbd..3a147d444 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -7,9 +7,9 @@ use crate::{ peer::{PeerS3Client, S3PeerSys}, sets::Sets, store_api::{ - BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsInfo, - ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, - PutObjReader, StorageAPI, + BucketInfo, BucketOptions, CompletePart, DeleteBucketOptions, DeletedObject, GetObjectReader, HTTPRangeSpec, + ListObjectsInfo, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, + PartInfo, PutObjReader, StorageAPI, }, store_init, utils, }; @@ -630,8 +630,8 @@ impl StorageAPI for ECStore { unimplemented!() } - async fn delete_bucket(&self, bucket: &str) -> Result<()> { - self.peer_sys.delete_bucket(bucket).await?; + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { + self.peer_sys.delete_bucket(bucket, opts).await?; // 删除meta self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str()) diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index a330f3e30..213d829eb 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -243,6 +243,10 @@ pub struct MakeBucketOptions { pub force_create: bool, } +pub struct DeleteBucketOptions { + pub force: bool, // Force deletion +} + #[derive(Debug)] pub struct PutObjReader { pub stream: StreamingBlob, @@ -488,7 +492,7 @@ pub struct DeletedObject { #[async_trait::async_trait] pub trait StorageAPI { async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; - async fn delete_bucket(&self, bucket: &str) -> Result<()>; + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()>; async fn list_bucket(&self, opts: &BucketOptions) -> Result>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result; diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index f822c5a04..f7fb1d8e5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -2,6 +2,7 @@ use bytes::Bytes; use ecstore::disk::error::DiskError; use ecstore::store_api::BucketOptions; use ecstore::store_api::CompletePart; +use ecstore::store_api::DeleteBucketOptions; use ecstore::store_api::HTTPRangeSpec; use ecstore::store_api::MakeBucketOptions; use ecstore::store_api::MultipartUploadResult; @@ -86,8 +87,12 @@ impl S3 for FS { #[tracing::instrument(level = "debug", skip(self, req))] async fn delete_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; - - try_!(self.store.delete_bucket(&input.bucket).await); + // TODO: DeleteBucketInput 没有force参数？ + try_!( + self.store + .delete_bucket(&input.bucket, &DeleteBucketOptions { force: false }) + .await + ); Ok(S3Response::new(DeleteBucketOutput {})) } From 210d7a4d5975cf6ed5794f244c6cf92364c65e40 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 13 Sep 2024 14:15:09 +0800 Subject: [PATCH 07/70] fix:delete_bucket skip when volume not empty --- ecstore/src/peer.rs | 4 ++-- ecstore/src/store.rs | 18 +++++++++--------- rustfs/src/grpc.rs | 8 ++++++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 79dc6aec3..343817b79 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -146,7 +146,7 @@ impl S3PeerSys { Ok(buckets) } - async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { + pub async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.delete_bucket(bucket, &opts)); @@ -391,7 +391,7 @@ impl PeerS3Client for LocalPeerS3Client { let mut idx = 0; for err in errs { if err.is_none() && recreate { - let _ = self.local_disks[idx].make_volume(bucket).await; + let _ = local_disks[idx].make_volume(bucket).await; } idx += 1; diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index bc4dca208..44473c3b0 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -488,6 +488,15 @@ impl StorageAPI for ECStore { Ok(buckets) } + + async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { + self.peer_sys.delete_bucket(bucket, opts).await?; + + // 删除meta + self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str()) + .await?; + Ok(()) + } async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { // TODO: check valid bucket name @@ -781,13 +790,4 @@ impl StorageAPI for ECStore { } unimplemented!() } - - async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { - self.peer_sys.delete_bucket(bucket, opts).await?; - - // 删除meta - self.delete_all(RUSTFS_META_BUCKET, format!("{}/{}", BUCKET_META_PREFIX, bucket).as_str()) - .await?; - Ok(()) - } } diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 7afc67000..6e4747fe3 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -3,7 +3,7 @@ use ecstore::{ erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, - store_api::{BucketOptions, FileInfo, MakeBucketOptions}, + store_api::{BucketOptions, DeleteBucketOptions, FileInfo, MakeBucketOptions}, }; use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; @@ -188,7 +188,11 @@ impl Node for NodeService { debug!("make bucket"); let request = request.into_inner(); - match self.local_peer.delete_bucket(&request.bucket).await { + match self + .local_peer + .delete_bucket(&request.bucket, &DeleteBucketOptions { force: false }) + .await + { Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse { success: true, error_info: None, From f4fea92cea7795dfbcde835bc2a730760f0db63b Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 13 Sep 2024 15:09:16 +0800 Subject: [PATCH 08/70] update todo.md --- TODO.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index f19a90c3d..15472d274 100644 --- a/TODO.md +++ b/TODO.md @@ -5,10 +5,13 @@ - [ ] EC可用读写数量判断 Read/WriteQuorum - [ ] 优化并发执行，边读边取，可中断 - [ ] 小文件存储到metafile, inlinedata +- [ ] 完善bucketmeta +- [ ] 对象锁 - [ ] 代码优化 使用范型？ - [ ] 抽象出metafile存储 +- [ ] 边读写边hash +- [x] 远程rpc - [x] 错误类型判断,程序中判断错误类型，如何统一错误 -- [x] 上传同名文件时，删除旧版本文件 - [x] 优化xlmeta, 自定义msg数据结构 ## 基础功能 From 61cf6865278cac706064b6bec1a84eb76a762cec Mon Sep 17 00:00:00 2001 From: bestgopher <84328409@qq.com> Date: Mon, 26 Aug 2024 23:29:31 +0800 Subject: [PATCH 09/70] feat: support bucket tagging Closes #41 Signed-off-by: bestgopher <84328409@qq.com> --- Cargo.lock | 5 +- Cargo.toml | 3 +- ecstore/src/bucket_meta.rs | 9 +++ ecstore/src/lib.rs | 2 +- rustfs/Cargo.toml | 1 + rustfs/src/storage/ecfs.rs | 128 +++++++++++++++++++++++++++++++++++++ 6 files changed, 144 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a30a9562f..22b19bba1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -957,9 +957,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" @@ -1582,6 +1582,7 @@ dependencies = [ "http-body", "hyper", "hyper-util", + "log", "mime", "netif", "pin-project-lite", diff --git a/Cargo.toml b/Cargo.toml index 003fce5d0..8d9bcdfdd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,4 +54,5 @@ tower = { version = "0.4.13", features = ["timeout"] } tracing = "0.1.40" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } -transform-stream = "0.3.0" \ No newline at end of file +transform-stream = "0.3.0" +log = "0.4.22" diff --git a/ecstore/src/bucket_meta.rs b/ecstore/src/bucket_meta.rs index aedeb9cf7..6ad9d9ccd 100644 --- a/ecstore/src/bucket_meta.rs +++ b/ecstore/src/bucket_meta.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use rmp_serde::Serializer; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; @@ -15,6 +17,9 @@ pub struct BucketMetadata { format: u16, version: u16, pub name: String, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub tagging: Option>, + #[serde(skip_serializing_if = "Option::is_none", default)] pub created: Option, } @@ -54,4 +59,8 @@ impl BucketMetadata { Ok(buf) } + + pub fn unmarshal_from(buffer: &[u8]) -> Result { + Ok(rmp_serde::from_slice(buffer)?) + } } diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 85e1c9017..5ac3eaa81 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -1,4 +1,4 @@ -mod bucket_meta; +pub mod bucket_meta; mod chunk_stream; pub mod disk; pub mod disks_layout; diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index d2d1a989c..f1d4025dc 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +log.workspace = true async-trait.workspace = true bytes.workspace = true clap.workspace = true diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index fd8bcc9a6..da44a221c 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -1,5 +1,8 @@ +use bytes::BufMut; use bytes::Bytes; +use ecstore::bucket_meta::BucketMetadata; use ecstore::disk::error::DiskError; +use ecstore::disk::RUSTFS_META_BUCKET; use ecstore::store::new_object_layer_fn; use ecstore::store_api::BucketOptions; use ecstore::store_api::CompletePart; @@ -16,6 +19,7 @@ use futures::{Stream, StreamExt}; use http::HeaderMap; use s3s::dto::*; use s3s::s3_error; +use s3s::Body; use s3s::S3Error; use s3s::S3ErrorCode; use s3s::S3Result; @@ -681,6 +685,130 @@ impl S3 for FS { ); Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })) } + + #[tracing::instrument(level = "debug", skip(self))] + async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { + let PutBucketTaggingInput { bucket, tagging, .. } = req.input; + log::debug!("bucket: {bucket}, tagging: {tagging:?}"); + + // check bucket exists. + let _bucket = self + .head_bucket(S3Request::new(HeadBucketInput { + bucket: bucket.clone(), + expected_bucket_owner: None, + })) + .await?; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let meta_obj = try_!( + store + .get_object_reader( + RUSTFS_META_BUCKET, + BucketMetadata::new(bucket.as_str()).save_file_path().as_str(), + HTTPRangeSpec::nil(), + Default::default(), + &ObjectOptions::default(), + ) + .await + ); + + let stream = meta_obj.stream; + + let mut data = vec![]; + pin_mut!(stream); + + while let Some(x) = stream.next().await { + let x = try_!(x); + data.put_slice(&x[..]); + } + + let mut meta = try_!(BucketMetadata::unmarshal_from(&data[..])); + if tagging.tag_set.is_empty() { + meta.tagging = None; + } else { + meta.tagging = Some(tagging.tag_set.into_iter().map(|x| (x.key, x.value)).collect()) + } + + let data = try_!(meta.marshal_msg()); + let len = data.len(); + try_!( + store + .put_object( + RUSTFS_META_BUCKET, + BucketMetadata::new(bucket.as_str()).save_file_path().as_str(), + PutObjReader::new(StreamingBlob::from(Body::from(data)), len), + &ObjectOptions::default(), + ) + .await + ); + + Ok(S3Response::new(Default::default())) + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { + let GetBucketTaggingInput { bucket, .. } = req.input; + // check bucket exists. + let _bucket = self + .head_bucket(S3Request::new(HeadBucketInput { + bucket: bucket.clone(), + expected_bucket_owner: None, + })) + .await?; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let meta_obj = try_!( + store + .get_object_reader( + RUSTFS_META_BUCKET, + BucketMetadata::new(bucket.as_str()).save_file_path().as_str(), + HTTPRangeSpec::nil(), + Default::default(), + &ObjectOptions::default(), + ) + .await + ); + + let stream = meta_obj.stream; + + let mut data = vec![]; + pin_mut!(stream); + + while let Some(x) = stream.next().await { + let x = try_!(x); + data.put_slice(&x[..]); + } + + let meta = try_!(BucketMetadata::unmarshal_from(&data[..])); + if meta.tagging.is_none() { + return Err({ + let mut err = S3Error::with_message(S3ErrorCode::Custom("NoSuchTagSet".into()), "The TagSet does not exist"); + err.set_status_code("404".try_into().unwrap()); + err + }); + } + + Ok(S3Response::new(GetBucketTaggingOutput { + tag_set: meta + .tagging + .unwrap() + .into_iter() + .map(|(key, value)| Tag { key, value }) + .collect(), + })) + } } #[allow(dead_code)] From b715eb530a4135cc0d84b21328581a384a946cb4 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 18 Sep 2024 13:53:13 +0800 Subject: [PATCH 10/70] Fix bug:#45 #43 #42 --- ecstore/src/disk/local.rs | 5 ++++- ecstore/src/peer.rs | 2 +- ecstore/src/sets.rs | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 3f23819f2..5dfcb92d2 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -439,6 +439,7 @@ impl DiskAPI for LocalDisk { } async fn get_disk_id(&self) -> Option { + warn!("local get_disk_id"); // TODO: check format file let format_info = self.format_info.lock().await; @@ -446,8 +447,10 @@ impl DiskAPI for LocalDisk { // TODO: 判断源文件id,是否有效 } - async fn set_disk_id(&self, _id: Option) -> Result<()> { + async fn set_disk_id(&self, id: Option) -> Result<()> { // 本地不需要设置 + let mut format_info = self.format_info.lock().await; + format_info.id = id; Ok(()) } diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 343817b79..241280c01 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -360,7 +360,7 @@ impl PeerS3Client for LocalPeerS3Client { .ok_or(Error::new(DiskError::VolumeNotFound)) } - async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { + async fn delete_bucket(&self, bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { let local_disks = all_local_disk().await; let mut futures = Vec::with_capacity(local_disks.len()); diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index d7a937463..8698b2103 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use futures::future::join_all; use http::HeaderMap; +use tracing::warn; use uuid::Uuid; use crate::{ @@ -55,6 +56,7 @@ impl Sets { let idx = i * set_drive_count + j; let mut disk = disks[idx].clone(); if disk.is_none() { + warn!("sets new set_drive {}-{} is none", i, j); set_drive.push(None); continue; } @@ -66,6 +68,7 @@ impl Sets { }; if local_disk.is_none() { + warn!("sets new set_drive {}-{} local_disk is none", i, j); set_drive.push(None); continue; } @@ -78,10 +81,13 @@ impl Sets { if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await { set_drive.push(disk); } else { + warn!("sets new set_drive {}-{} get_disk_id is none", i, j); set_drive.push(None); } } + warn!("sets new set_drive {:?}", &set_drive); + let set_disks = SetDisks { disks: set_drive, set_drive_count, @@ -320,7 +326,7 @@ impl StorageAPI for Sets { .await } - async fn delete_bucket(&self, _bucket: &str, opts: &DeleteBucketOptions) -> Result<()> { + async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> Result<()> { unimplemented!() } } From 175482371c4ed31d3653df1b769f7f90f9687fb3 Mon Sep 17 00:00:00 2001 From: bestgopher <84328409@qq.com> Date: Wed, 18 Sep 2024 16:29:47 +0800 Subject: [PATCH 11/70] feat: remove bucket tagging --- rustfs/src/storage/ecfs.rs | 63 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index da44a221c..68787c7e7 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -260,6 +260,7 @@ impl S3 for FS { Ok(S3Response::new(output)) } + #[tracing::instrument(level = "debug", skip(self))] async fn get_object_lock_configuration( &self, _req: S3Request, @@ -686,7 +687,7 @@ impl S3 for FS { Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })) } - #[tracing::instrument(level = "debug", skip(self))] + #[tracing::instrument(level = "debug", skip(self, req))] async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { let PutBucketTaggingInput { bucket, tagging, .. } = req.input; log::debug!("bucket: {bucket}, tagging: {tagging:?}"); @@ -751,7 +752,7 @@ impl S3 for FS { Ok(S3Response::new(Default::default())) } - #[tracing::instrument(level = "debug", skip(self))] + #[tracing::instrument(level = "debug", skip(self, req))] async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { let GetBucketTaggingInput { bucket, .. } = req.input; // check bucket exists. @@ -809,6 +810,64 @@ impl S3 for FS { .collect(), })) } + + #[tracing::instrument(level = "debug", skip(self, req))] + async fn delete_bucket_tagging(&self, req: S3Request) -> S3Result> { + let DeleteBucketTaggingInput { bucket, .. } = req.input; + // check bucket exists. + let _bucket = self + .head_bucket(S3Request::new(HeadBucketInput { + bucket: bucket.clone(), + expected_bucket_owner: None, + })) + .await?; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let meta_obj = try_!( + store + .get_object_reader( + RUSTFS_META_BUCKET, + BucketMetadata::new(bucket.as_str()).save_file_path().as_str(), + HTTPRangeSpec::nil(), + Default::default(), + &ObjectOptions::default(), + ) + .await + ); + + let stream = meta_obj.stream; + + let mut data = vec![]; + pin_mut!(stream); + + while let Some(x) = stream.next().await { + let x = try_!(x); + data.put_slice(&x[..]); + } + + let mut meta = try_!(BucketMetadata::unmarshal_from(&data[..])); + meta.tagging = None; + let data = try_!(meta.marshal_msg()); + let len = data.len(); + try_!( + store + .put_object( + RUSTFS_META_BUCKET, + BucketMetadata::new(bucket.as_str()).save_file_path().as_str(), + PutObjReader::new(StreamingBlob::from(Body::from(data)), len), + &ObjectOptions::default(), + ) + .await + ); + + Ok(S3Response::new(DeleteBucketTaggingOutput {})) + } } #[allow(dead_code)] From 2b3f03eae90cb013ae2e160b79c899e756d193a2 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 18 Sep 2024 17:04:42 +0800 Subject: [PATCH 12/70] add log --- ecstore/src/store_init.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 6e6eb2fc1..91d580f31 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -65,8 +65,9 @@ pub async fn connect_load_init_formats( // new format and save let fms = init_format_erasure(disks, set_count, set_drive_count, deployment_id); - let _errs = save_format_file_all(disks, &fms).await; + let errs = save_format_file_all(disks, &fms).await; + warn!("save_format_file_all errs {:?}", &errs); // TODO: check quorum // reduceWriteQuorumErrs(&errs)?; From 60d02f3afdaefeae206bb633b4f5b099237f32ae Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 18 Sep 2024 22:57:08 +0800 Subject: [PATCH 13/70] fix get_disk_id --- ecstore/src/disk/error.rs | 3 ++ ecstore/src/disk/local.rs | 85 ++++++++++++++++++++++++++++++++++----- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index c8c0c84b1..1b879d7ab 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -34,6 +34,9 @@ pub enum DiskError { #[error("volume not empty")] VolumeNotEmpty, + + #[error("corrupted backend")] + CorruptedBackend } impl DiskError { diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 5dfcb92d2..3c34f3368 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -26,17 +26,23 @@ use uuid::Uuid; #[derive(Debug)] pub struct FormatInfo { pub id: Option, - pub _data: Vec, - pub _file_info: Option, - pub _last_check: Option, + pub data: Vec, + pub file_info: Option, + pub last_check: Option, } -impl FormatInfo {} +impl FormatInfo { + pub fn last_check_valid(&self)->bool{ + let now = OffsetDateTime::now_utc(); + self.file_info.is_some() && self.id.is_some() && self.last_check.is_some() && (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <=1) + } + +} #[derive(Debug)] pub struct LocalDisk { pub root: PathBuf, - pub _format_path: PathBuf, + pub format_path: PathBuf, pub format_info: Mutex, // pub id: Mutex>, // pub format_data: Mutex>, @@ -79,14 +85,14 @@ impl LocalDisk { let format_info = FormatInfo { id, - _data: format_data, - _file_info: format_meta, - _last_check: format_last_check, + data: format_data, + file_info: format_meta, + last_check: format_last_check, }; let disk = Self { root, - _format_path: format_path, + format_path: format_path, format_info: Mutex::new(format_info), // // format_legacy, // format_file_info: Mutex::new(format_meta), @@ -99,6 +105,17 @@ impl LocalDisk { Ok(disk) } + async fn check_format_json(&self) ->Result{ + let p = self.format_path; + let md = fs::metadata(&p).await.map_err(|e|match e.kind(){ + ErrorKind::NotFound => DiskError::DiskNotFound, + ErrorKind::PermissionDenied => DiskError::FileAccessDenied, + _ => { + warn!("check_format_json err {:?}",e); + DiskError::CorruptedBackend}, + })?; + Ok(md) + } async fn make_meta_volumes(&self) -> Result<()> { let buckets = format!("{}/{}", super::RUSTFS_META_BUCKET, super::BUCKET_META_PREFIX); let multipart = format!("{}/{}", super::RUSTFS_META_BUCKET, "multipart"); @@ -441,9 +458,55 @@ impl DiskAPI for LocalDisk { async fn get_disk_id(&self) -> Option { warn!("local get_disk_id"); // TODO: check format file - let format_info = self.format_info.lock().await; + let mut format_info = self.format_info.lock().await; - format_info.id.clone() + let id = format_info.id.clone(); + + if format_info.last_check_valid(){ + return id + } + + let file_meta = self.check_format_json().await?; + + if let Some(file_info) = format_info.file_info{ + if file_meta == file_info{ + + format_info.last_check = Some(OffsetDateTime::now_utc()); + + return id + } + } + + + let b = fs::read(&format_info.file_path).await.map_err(|e|match e.kind(){ + ErrorKind::NotFound => DiskError::DiskNotFound, + ErrorKind::PermissionDenied => DiskError::FileAccessDenied, + _ => { + warn!("check_format_json err {:?}",e); + DiskError::CorruptedBackend}, + })?; + + + let fm = FormatV3::try_from(b.as_slice()).map_err(|e|{ + warn!("decode format.json err {:?}",e); + DiskError::CorruptedBackend + })?; + + let (m,n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; + + let disk_id = fm.erasure.this; + + if m != self.endpoint.set_idx || n != self.endpoint.disk_idx { + return DiskError::InconsistentDisk; + } + + format_info.id = Some(disk_id); + format_info.file_info= Some(file_meta); + format_info.data = b; + format_info.last_check = Some(OffsetDateTime::now_utc()); +; + +Some(disk_id) // TODO: 判断源文件id,是否有效 } From 1f1e73ce083d8508f41addab3153403d01892b4d Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 19 Sep 2024 09:41:29 +0800 Subject: [PATCH 14/70] fix get_disk_id --- ecstore/src/disk/local.rs | 106 ++++++++++++++++++++----------------- ecstore/src/disk/mod.rs | 2 +- ecstore/src/disk/remote.rs | 4 +- ecstore/src/sets.rs | 2 +- ecstore/src/utils/fs.rs | 24 +++++++++ ecstore/src/utils/mod.rs | 1 + 6 files changed, 85 insertions(+), 54 deletions(-) create mode 100644 ecstore/src/utils/fs.rs diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 3c34f3368..e0a1679ac 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -32,11 +32,13 @@ pub struct FormatInfo { } impl FormatInfo { - pub fn last_check_valid(&self)->bool{ + pub fn last_check_valid(&self) -> bool { let now = OffsetDateTime::now_utc(); - self.file_info.is_some() && self.id.is_some() && self.last_check.is_some() && (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <=1) + self.file_info.is_some() + && self.id.is_some() + && self.last_check.is_some() + && (now.unix_timestamp() - self.last_check.unwrap().unix_timestamp() <= 1) } - } #[derive(Debug)] @@ -44,6 +46,7 @@ pub struct LocalDisk { pub root: PathBuf, pub format_path: PathBuf, pub format_info: Mutex, + pub endpoint: Endpoint, // pub id: Mutex>, // pub format_data: Mutex>, // pub format_file_info: Mutex>, @@ -92,6 +95,7 @@ impl LocalDisk { let disk = Self { root, + endpoint: ep.clone(), format_path: format_path, format_info: Mutex::new(format_info), // // format_legacy, @@ -105,16 +109,16 @@ impl LocalDisk { Ok(disk) } - async fn check_format_json(&self) ->Result{ - let p = self.format_path; - let md = fs::metadata(&p).await.map_err(|e|match e.kind(){ - ErrorKind::NotFound => DiskError::DiskNotFound, - ErrorKind::PermissionDenied => DiskError::FileAccessDenied, - _ => { - warn!("check_format_json err {:?}",e); - DiskError::CorruptedBackend}, - })?; - Ok(md) + async fn check_format_json(&self) -> Result { + let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() { + ErrorKind::NotFound => DiskError::DiskNotFound, + ErrorKind::PermissionDenied => DiskError::FileAccessDenied, + _ => { + warn!("check_format_json err {:?}", e); + DiskError::CorruptedBackend + } + })?; + Ok(md) } async fn make_meta_volumes(&self) -> Result<()> { let buckets = format!("{}/{}", super::RUSTFS_META_BUCKET, super::BUCKET_META_PREFIX); @@ -455,58 +459,60 @@ impl DiskAPI for LocalDisk { self.root.clone() } - async fn get_disk_id(&self) -> Option { + async fn get_disk_id(&self) -> Result> { warn!("local get_disk_id"); // TODO: check format file let mut format_info = self.format_info.lock().await; - let id = format_info.id.clone(); + let id = format_info.id.clone(); - if format_info.last_check_valid(){ - return id - } - - let file_meta = self.check_format_json().await?; - - if let Some(file_info) = format_info.file_info{ - if file_meta == file_info{ - - format_info.last_check = Some(OffsetDateTime::now_utc()); - - return id + if format_info.last_check_valid() { + return Ok(id); } - } + let file_meta = self.check_format_json().await?; - let b = fs::read(&format_info.file_path).await.map_err(|e|match e.kind(){ - ErrorKind::NotFound => DiskError::DiskNotFound, - ErrorKind::PermissionDenied => DiskError::FileAccessDenied, - _ => { - warn!("check_format_json err {:?}",e); - DiskError::CorruptedBackend}, - })?; + if let Some(file_info) = &format_info.file_info { + if utils::fs::same_file(&file_meta, file_info) { + format_info.last_check = Some(OffsetDateTime::now_utc()); + return Ok(id); + } + } - let fm = FormatV3::try_from(b.as_slice()).map_err(|e|{ - warn!("decode format.json err {:?}",e); + let b = fs::read(&self.format_path).await.map_err(|e| match e.kind() { + ErrorKind::NotFound => DiskError::DiskNotFound, + ErrorKind::PermissionDenied => DiskError::FileAccessDenied, + _ => { + warn!("check_format_json err {:?}", e); + DiskError::CorruptedBackend + } + })?; + + let fm = FormatV3::try_from(b.as_slice()).map_err(|e| { + warn!("decode format.json err {:?}", e); DiskError::CorruptedBackend - })?; - - let (m,n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; + })?; - let disk_id = fm.erasure.this; + let (m, n) = fm.find_disk_index_by_disk_id(fm.erasure.this)?; - if m != self.endpoint.set_idx || n != self.endpoint.disk_idx { - return DiskError::InconsistentDisk; - } + let disk_id = fm.erasure.this; - format_info.id = Some(disk_id); - format_info.file_info= Some(file_meta); - format_info.data = b; - format_info.last_check = Some(OffsetDateTime::now_utc()); -; + match (self.endpoint.set_idx, self.endpoint.disk_idx) { + (Some(set_idx), Some(disk_idx)) => { + if m != set_idx || n != disk_idx { + return Err(Error::new(DiskError::InconsistentDisk)); + } + } + _ => return Err(Error::new(DiskError::InconsistentDisk)), + } -Some(disk_id) + format_info.id = Some(disk_id); + format_info.file_info = Some(file_meta); + format_info.data = b; + format_info.last_check = Some(OffsetDateTime::now_utc()); + + Ok(Some(disk_id)) // TODO: 判断源文件id,是否有效 } diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index d2f63c4f9..487edd405 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -47,7 +47,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { fn is_local(&self) -> bool; fn path(&self) -> PathBuf; async fn close(&self) -> Result<()>; - async fn get_disk_id(&self) -> Option; + async fn get_disk_id(&self) -> Result>; async fn set_disk_id(&self, id: Option) -> Result<()>; async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index b0d5a8a0c..d9e47a908 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -105,8 +105,8 @@ impl DiskAPI for RemoteDisk { self.root.clone() } - async fn get_disk_id(&self) -> Option { - self.id.lock().await.clone() + async fn get_disk_id(&self) -> Result> { + Ok(self.id.lock().await.clone()) } async fn set_disk_id(&self, id: Option) -> Result<()> { let mut lock = self.id.lock().await; diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 8698b2103..c9e049d56 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -78,7 +78,7 @@ impl Sets { disk = local_disk; } - if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await { + if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await? { set_drive.push(disk); } else { warn!("sets new set_drive {}-{} get_disk_id is none", i, j); diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs new file mode 100644 index 000000000..730047855 --- /dev/null +++ b/ecstore/src/utils/fs.rs @@ -0,0 +1,24 @@ +use std::{fs::Metadata, os::unix::fs::MetadataExt}; + +pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { + if f1.dev() != f2.dev() { + return false; + } + + if f1.ino() != f2.ino() { + return false; + } + + if f1.size() != f2.size() { + return false; + } + if f1.permissions() != f2.permissions() { + return false; + } + + if f1.mtime() != f2.mtime() { + return false; + } + + true +} diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index 2b78d9e75..90e120d08 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -1,5 +1,6 @@ pub mod crypto; pub mod ellipses; +pub mod fs; pub mod hash; pub mod net; pub mod path; From c71e86b8d1eaaf688743403576801d7f657950ad Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 19 Sep 2024 13:12:32 +0800 Subject: [PATCH 15/70] fix: use option writer when disk is none --- ecstore/src/erasure.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 13cb5af18..a2261d97e 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -46,7 +46,7 @@ impl Erasure { pub async fn encode( &self, body: S, - writers: &mut [FileWriter], + writers: &mut [Option], // block_size: usize, total_size: usize, _write_quorum: usize, @@ -84,7 +84,10 @@ impl Erasure { let mut errs = Vec::new(); for (i, w) in writers.iter_mut().enumerate() { - match w.write(blocks[i].as_ref()).await { + if w.is_none() { + continue; + } + match w.as_mut().unwrap().write(blocks[i].as_ref()).await { Ok(_) => errs.push(None), Err(e) => errs.push(Some(e)), } From bc9868f926782746be8cfc47dec2a78d5c564e35 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 19 Sep 2024 18:06:44 +0800 Subject: [PATCH 16/70] add: monitor_and_connect_endpoints --- ecstore/src/disk/local.rs | 15 ++++- ecstore/src/disk/mod.rs | 14 +++++ ecstore/src/disk/remote.rs | 31 ++++++++--- ecstore/src/sets.rs | 75 ++++++++++++++++++++----- ecstore/src/store.rs | 110 +++++++++++++++++-------------------- 5 files changed, 163 insertions(+), 82 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index e0a1679ac..816ed6aa3 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,7 +1,7 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ - DeleteOptions, DiskAPI, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, - ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions, + DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, + ReadMultipleResp, ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions, }; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::{ @@ -452,6 +452,9 @@ impl DiskAPI for LocalDisk { fn is_local(&self) -> bool { true } + async fn is_online(&self) -> bool { + true + } async fn close(&self) -> Result<()> { Ok(()) } @@ -459,6 +462,14 @@ impl DiskAPI for LocalDisk { self.root.clone() } + fn get_location(&self) -> DiskLocation { + DiskLocation { + pool_idx: self.endpoint.pool_idx, + set_idx: self.endpoint.set_idx, + disk_idx: self.endpoint.pool_idx, + } + } + async fn get_disk_id(&self) -> Result> { warn!("local get_disk_id"); // TODO: check format file diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 487edd405..8c089c8ed 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -45,10 +45,12 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result bool; + async fn is_online(&self) -> bool; fn path(&self) -> PathBuf; async fn close(&self) -> Result<()>; async fn get_disk_id(&self) -> Result>; async fn set_disk_id(&self, id: Option) -> Result<()>; + fn get_location(&self) -> DiskLocation; async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; @@ -95,6 +97,18 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; } +pub struct DiskLocation { + pub pool_idx: Option, + pub set_idx: Option, + pub disk_idx: Option, +} + +impl DiskLocation { + pub fn valid(&self) -> bool { + self.pool_idx.is_some() && self.set_idx.is_some() && self.disk_idx.is_some() + } +} + #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index d9e47a908..f8f4eff7e 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -28,9 +28,9 @@ use crate::{ }; use super::{ - endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, - ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo, - WalkDirOptions, + endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, + MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, + VolumeInfo, WalkDirOptions, }; #[derive(Debug)] @@ -39,6 +39,7 @@ pub struct RemoteDisk { channel: Arc>>, url: url::Url, pub root: PathBuf, + endpoint: Endpoint, } impl RemoteDisk { @@ -50,6 +51,7 @@ impl RemoteDisk { url: ep.url.clone(), root, id: Mutex::new(None), + endpoint: ep.clone(), }) } @@ -98,6 +100,13 @@ impl DiskAPI for RemoteDisk { fn is_local(&self) -> bool { false } + async fn is_online(&self) -> bool { + // TODO: 连接状态 + if let Ok(_) = self.get_client_v2().await { + return true; + } + false + } async fn close(&self) -> Result<()> { Ok(()) } @@ -105,6 +114,14 @@ impl DiskAPI for RemoteDisk { self.root.clone() } + fn get_location(&self) -> DiskLocation { + DiskLocation { + pool_idx: self.endpoint.pool_idx, + set_idx: self.endpoint.set_idx, + disk_idx: self.endpoint.pool_idx, + } + } + async fn get_disk_id(&self) -> Result> { Ok(self.id.lock().await.clone()) } @@ -253,11 +270,11 @@ impl DiskAPI for RemoteDisk { }); let response = client.walk_dir(request).await?.into_inner(); - + if !response.success { return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); } - + let entries = response .meta_cache_entry .into_iter() @@ -288,7 +305,7 @@ impl DiskAPI for RemoteDisk { }); let response = client.rename_data(request).await?.into_inner(); - + if !response.success { return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); } @@ -363,7 +380,7 @@ impl DiskAPI for RemoteDisk { }); let response = client.stat_volume(request).await?.into_inner(); - + if !response.success { return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); } diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index c9e049d56..dc6dd3dda 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,9 +1,4 @@ -use std::collections::HashMap; - -use futures::future::join_all; -use http::HeaderMap; -use tracing::warn; -use uuid::Uuid; +use std::{collections::HashMap, sync::Arc, time::Duration}; use crate::{ disk::{ @@ -21,13 +16,19 @@ use crate::{ }, utils::hash, }; +use futures::future::join_all; +use http::HeaderMap; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; +use tracing::{debug, warn}; +use uuid::Uuid; #[derive(Debug)] pub struct Sets { pub id: Uuid, // pub sets: Vec, // pub disk_set: Vec>>, // [set_count_idx][set_drive_count_idx] = disk_idx - pub disk_set: Vec, // [set_count_idx][set_drive_count_idx] = disk_idx + pub disk_set: Vec>, // [set_count_idx][set_drive_count_idx] = disk_idx pub pool_idx: usize, pub endpoints: PoolEndpoints, pub format: FormatV3, @@ -35,6 +36,7 @@ pub struct Sets { pub set_count: usize, pub set_drive_count: usize, pub distribution_algo: DistributionAlgoVersion, + ctx: CancellationToken, } impl Sets { @@ -44,7 +46,7 @@ impl Sets { fm: &FormatV3, pool_idx: usize, partiy_count: usize, - ) -> Result { + ) -> Result> { let set_count = fm.erasure.sets.len(); let set_drive_count = fm.erasure.sets[0].len(); @@ -52,9 +54,14 @@ impl Sets { for i in 0..set_count { let mut set_drive = Vec::with_capacity(set_drive_count); + let mut set_endpoints = Vec::with_capacity(set_drive_count); for j in 0..set_drive_count { let idx = i * set_drive_count + j; let mut disk = disks[idx].clone(); + + let endpoint = endpoints.endpoints.as_ref().get(idx).cloned(); + set_endpoints.push(endpoint); + if disk.is_none() { warn!("sets new set_drive {}-{} is none", i, j); set_drive.push(None); @@ -89,17 +96,18 @@ impl Sets { warn!("sets new set_drive {:?}", &set_drive); let set_disks = SetDisks { - disks: set_drive, + disks: RwLock::new(set_drive), set_drive_count, parity_count: partiy_count, set_index: i, pool_index: pool_idx, + set_endpoints, }; - disk_set.push(set_disks); + disk_set.push(Arc::new(set_disks)); } - let sets = Self { + let sets = Arc::new(Self { id: fm.id, // sets: todo!(), disk_set, @@ -110,15 +118,54 @@ impl Sets { set_count, set_drive_count, distribution_algo: fm.erasure.distribution_algo.clone(), - }; + ctx: CancellationToken::new(), + }); + + let asets = sets.clone(); + + tokio::spawn(async move { asets.monitor_and_connect_endpoints().await }); Ok(sets) } - pub fn get_disks(&self, set_idx: usize) -> SetDisks { + + pub async fn monitor_and_connect_endpoints(&self) { + tokio::time::sleep(Duration::from_secs(5)).await; + + self.connect_disks().await; + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3)); + let cloned_token = self.ctx.clone(); + loop { + tokio::select! { + _= interval.tick()=>{ + debug!("tick..."); + self.connect_disks().await; + + interval.reset(); + }, + + _ = cloned_token.cancelled() => { + warn!("ctx cancelled"); + break; + } + } + } + + warn!("monitor_and_connect_endpoints exit"); + } + + async fn connect_disks(&self) { + debug!("start connect_disks ..."); + for set in self.disk_set.iter() { + set.connect_disks().await; + } + } + + pub fn get_disks(&self, set_idx: usize) -> Arc { self.disk_set[set_idx].clone() } - pub fn get_disks_by_key(&self, key: &str) -> SetDisks { + pub fn get_disks_by_key(&self, key: &str) -> Arc { self.get_disks(self.get_hashed_set_index(key)) } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 44473c3b0..d3a510595 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,8 +1,6 @@ use crate::{ bucket_meta::BucketMetadata, - disk::{ - error::DiskError, new_disk, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, - }, + disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, endpoints::{EndpointServerPools, SetupType}, error::{Error, Result}, peer::S3PeerSys, @@ -25,7 +23,7 @@ use std::{ }; use time::OffsetDateTime; use tokio::{fs, sync::RwLock}; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use uuid::Uuid; use lazy_static::lazy_static; @@ -150,7 +148,7 @@ pub struct ECStore { pub id: uuid::Uuid, // pub disks: Vec, pub disk_map: HashMap>>, - pub pools: Vec, + pub pools: Vec>, pub peer_sys: S3PeerSys, // pub local_disks: Vec, } @@ -223,7 +221,6 @@ impl ECStore { } let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count).await?; - pools.push(sets); disk_map.insert(i, disks); @@ -285,62 +282,54 @@ impl ECStore { for sets in self.pools.iter() { for set in sets.disk_set.iter() { - for disk in set.disks.iter() { - if disk.is_none() { - continue; - } - - let disk = disk.as_ref().unwrap(); - let opts = opts.clone(); - // let mut wr = &mut wr; - futures.push(disk.walk_dir(opts)); - // tokio::spawn(async move { disk.walk_dir(opts, wr).await }); - } + futures.push(set.walk_dir(&opts)); } } let results = join_all(futures).await; - let mut errs = Vec::new(); + // let mut errs = Vec::new(); let mut ress = Vec::new(); let mut uniq = HashSet::new(); - for res in results { - match res { - Ok(entrys) => { - for entry in entrys { - if !uniq.contains(&entry.name) { - uniq.insert(entry.name.clone()); - // TODO: 过滤 - if opts.limit > 0 && ress.len() as i32 >= opts.limit { - return Ok(ress); - } + for (disks_ress, _disks_errs) in results { + for (_i, disks_res) in disks_ress.iter().enumerate() { + if disks_res.is_none() { + // TODO handle errs + continue; + } + let entrys = disks_res.as_ref().unwrap(); - if entry.is_object() { - let fi = entry.to_fileinfo(&opts.bucket)?; - if fi.is_some() { - ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false)); - } - continue; - } + for entry in entrys { + if !uniq.contains(&entry.name) { + uniq.insert(entry.name.clone()); + // TODO: 过滤 + if opts.limit > 0 && ress.len() as i32 >= opts.limit { + return Ok(ress); + } - if entry.is_dir() { - ress.push(ObjectInfo { - is_dir: true, - bucket: opts.bucket.clone(), - name: entry.name, - ..Default::default() - }); + if entry.is_object() { + let fi = entry.to_fileinfo(&opts.bucket)?; + if fi.is_some() { + ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false)); } + continue; + } + + if entry.is_dir() { + ress.push(ObjectInfo { + is_dir: true, + bucket: opts.bucket.clone(), + name: entry.name.clone(), + ..Default::default() + }); } } - errs.push(None); } - Err(e) => errs.push(Some(e)), } } - warn!("list_merged errs {:?}", errs); + // warn!("list_merged errs {:?}", errs); Ok(ress) } @@ -349,21 +338,24 @@ impl ECStore { let mut futures = Vec::new(); for sets in self.pools.iter() { for set in sets.disk_set.iter() { - for disk in set.disks.iter() { - if disk.is_none() { - continue; - } + futures.push(set.delete_all(bucket, prefix)); + // let disks = set.disks.read().await; + // let dd = disks.clone(); + // for disk in dd { + // if disk.is_none() { + // continue; + // } - let disk = disk.as_ref().unwrap(); - futures.push(disk.delete( - bucket, - prefix, - DeleteOptions { - recursive: true, - immediate: false, - }, - )); - } + // // let disk = disk.as_ref().unwrap().clone(); + // // futures.push(disk.delete( + // // bucket, + // // prefix, + // // DeleteOptions { + // // recursive: true, + // // immediate: false, + // // }, + // // )); + // } } } let results = join_all(futures).await; From ef870a6ac26d65dcbe5978d8a203d148519ab601 Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:24:50 +0800 Subject: [PATCH 17/70] Create Audit.yml --- .github/workflows/Audit.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/Audit.yml diff --git a/.github/workflows/Audit.yml b/.github/workflows/Audit.yml new file mode 100644 index 000000000..7cd4eb54d --- /dev/null +++ b/.github/workflows/Audit.yml @@ -0,0 +1,27 @@ +name: Audit + +on: + push: + branches: + - main + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + branches: + - main + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + schedule: + - cron: '0 0 * * 0' # at midnight of each sunday + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + - run: cargo audit -D warnings From 5b0342526dd44a69a863db23f97d6e796adf8c56 Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:26:44 +0800 Subject: [PATCH 18/70] Create rust.yml --- .github/workflows/rust.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/rust.yml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 000000000..9fd45e090 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,22 @@ +name: Rust + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose From bfa73f630d2ac35a4661b69252b04a00ea73038c Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:27:51 +0800 Subject: [PATCH 19/70] Create audit.yml --- .github/workflows/audit.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/audit.yml diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 000000000..7cd4eb54d --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,27 @@ +name: Audit + +on: + push: + branches: + - main + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + pull_request: + branches: + - main + paths: + - '**/Cargo.toml' + - '**/Cargo.lock' + schedule: + - cron: '0 0 * * 0' # at midnight of each sunday + +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: taiki-e/install-action@v2 + with: + tool: cargo-audit + - run: cargo audit -D warnings From b88c9a5ae106585d0bfb22d2d8e8aa33da075436 Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:28:03 +0800 Subject: [PATCH 20/70] Delete .github/workflows/Audit.yml --- .github/workflows/Audit.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/workflows/Audit.yml diff --git a/.github/workflows/Audit.yml b/.github/workflows/Audit.yml deleted file mode 100644 index 7cd4eb54d..000000000 --- a/.github/workflows/Audit.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Audit - -on: - push: - branches: - - main - paths: - - '**/Cargo.toml' - - '**/Cargo.lock' - pull_request: - branches: - - main - paths: - - '**/Cargo.toml' - - '**/Cargo.lock' - schedule: - - cron: '0 0 * * 0' # at midnight of each sunday - -jobs: - audit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: taiki-e/install-action@v2 - with: - tool: cargo-audit - - run: cargo audit -D warnings From 1a4498a58c783d9b96f930a7295abd156150807d Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:38:13 +0800 Subject: [PATCH 21/70] Create dependabot.yml --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..e522013ad --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "cargo" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "monthly" + groups: + dependencies: + patterns: + - "*" From 3e34718bfecf6aaae5d55095702678eac0efc3f7 Mon Sep 17 00:00:00 2001 From: bestgopher <84328409@qq.com> Date: Thu, 19 Sep 2024 18:32:39 +0800 Subject: [PATCH 22/70] feat: object tagging Closes: #54 Signed-off-by: bestgopher <84328409@qq.com> --- ecstore/src/sets.rs | 7 +++ ecstore/src/store.rs | 11 ++++ ecstore/src/store_api.rs | 9 ++++ rustfs/src/storage/ecfs.rs | 105 +++++++++++++++++++++++++++++++------ 4 files changed, 116 insertions(+), 16 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 8698b2103..9d1decf59 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -275,6 +275,13 @@ impl StorageAPI for Sets { async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).get_object_info(bucket, object, opts).await } + + async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()> { + self.get_disks_by_key(object) + .put_object_info(bucket, object, info, opts) + .await + } + async fn get_object_reader( &self, bucket: &str, diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 44473c3b0..b70e381e1 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -717,6 +717,17 @@ impl StorageAPI for ECStore { unimplemented!() } + + async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()> { + let object = utils::path::encode_dir_object(object); + + if self.single_pool() { + return self.pools[0].put_object_info(bucket, object.as_str(), info, opts).await; + } + + unimplemented!() + } + async fn get_object_reader( &self, bucket: &str, diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 0638ec198..49bc7dd2f 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use crate::error::{Error, Result}; use http::HeaderMap; use rmp_serde::Serializer; @@ -25,6 +27,8 @@ pub struct FileInfo { pub fresh: bool, // indicates this is a first time call to write FileInfo. pub parts: Vec, pub is_latest: bool, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub tags: Option>, } // impl Default for FileInfo { @@ -150,6 +154,7 @@ impl FileInfo { size: self.size, parts: self.parts.clone(), is_latest: self.is_latest, + tags: self.tags.clone(), } } // to_part_offset 取offset 所在的part index, 返回part index, offset @@ -429,6 +434,7 @@ pub struct ObjectInfo { pub delete_marker: bool, pub parts: Vec, pub is_latest: bool, + pub tags: Option>, } #[derive(Debug, Default)] @@ -516,6 +522,9 @@ pub trait StorageAPI { start_after: &str, ) -> Result; async fn get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result; + + async fn put_object_info(&self, bucket: &str, object: &str, info: ObjectInfo, opts: &ObjectOptions) -> Result<()>; + async fn get_object_reader( &self, bucket: &str, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 68787c7e7..ae44da1ee 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -687,7 +687,7 @@ impl S3 for FS { Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() })) } - #[tracing::instrument(level = "debug", skip(self, req))] + #[tracing::instrument(level = "debug", skip(self))] async fn put_bucket_tagging(&self, req: S3Request) -> S3Result> { let PutBucketTaggingInput { bucket, tagging, .. } = req.input; log::debug!("bucket: {bucket}, tagging: {tagging:?}"); @@ -702,10 +702,9 @@ impl S3 for FS { let layer = new_object_layer_fn(); let lock = layer.read().await; - let store = match lock.as_ref() { - Some(s) => s, - None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), - }; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; let meta_obj = try_!( store @@ -752,7 +751,7 @@ impl S3 for FS { Ok(S3Response::new(Default::default())) } - #[tracing::instrument(level = "debug", skip(self, req))] + #[tracing::instrument(level = "debug", skip(self))] async fn get_bucket_tagging(&self, req: S3Request) -> S3Result> { let GetBucketTaggingInput { bucket, .. } = req.input; // check bucket exists. @@ -765,10 +764,9 @@ impl S3 for FS { let layer = new_object_layer_fn(); let lock = layer.read().await; - let store = match lock.as_ref() { - Some(s) => s, - None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), - }; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; let meta_obj = try_!( store @@ -811,8 +809,11 @@ impl S3 for FS { })) } - #[tracing::instrument(level = "debug", skip(self, req))] - async fn delete_bucket_tagging(&self, req: S3Request) -> S3Result> { + #[tracing::instrument(level = "debug", skip(self))] + async fn delete_bucket_tagging( + &self, + req: S3Request, + ) -> S3Result> { let DeleteBucketTaggingInput { bucket, .. } = req.input; // check bucket exists. let _bucket = self @@ -824,10 +825,9 @@ impl S3 for FS { let layer = new_object_layer_fn(); let lock = layer.read().await; - let store = match lock.as_ref() { - Some(s) => s, - None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), - }; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; let meta_obj = try_!( store @@ -868,6 +868,79 @@ impl S3 for FS { Ok(S3Response::new(DeleteBucketTaggingOutput {})) } + + #[tracing::instrument(level = "debug", skip(self))] + async fn put_object_tagging(&self, req: S3Request) -> S3Result> { + let PutObjectTaggingInput { + bucket, + key: object, + tagging, + .. + } = req.input; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; + + let mut object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await); + object_info.tags = Some(tagging.tag_set.into_iter().map(|Tag { key, value }| (key, value)).collect()); + + try_!( + store + .put_object_info(&bucket, &object, object_info, &ObjectOptions::default()) + .await + ); + + Ok(S3Response::new(PutObjectTaggingOutput { version_id: None })) + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn get_object_tagging(&self, req: S3Request) -> S3Result> { + let GetObjectTaggingInput { bucket, key: object, .. } = req.input; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; + + let object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await); + + Ok(S3Response::new(GetObjectTaggingOutput { + tag_set: object_info + .tags + .map(|tags| tags.into_iter().map(|(key, value)| Tag { key, value }).collect()) + .unwrap_or_else(|| vec![]), + version_id: None, + })) + } + + #[tracing::instrument(level = "debug", skip(self))] + async fn delete_object_tagging( + &self, + req: S3Request, + ) -> S3Result> { + let DeleteObjectTaggingInput { bucket, key: object, .. } = req.input; + + let layer = new_object_layer_fn(); + let lock = layer.read().await; + let store = lock + .as_ref() + .ok_or_else(|| S3Error::with_message(S3ErrorCode::InternalError, "Not init"))?; + + let mut object_info = try_!(store.get_object_info(&bucket, &object, &ObjectOptions::default()).await); + object_info.tags = None; + + try_!( + store + .put_object_info(&bucket, &object, object_info, &ObjectOptions::default()) + .await + ); + + Ok(S3Response::new(DeleteObjectTaggingOutput { version_id: None })) + } } #[allow(dead_code)] From 83590cb7425a7bfdfadcc6884eeec1fe8d9da19a Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 20 Sep 2024 16:45:17 +0800 Subject: [PATCH 23/70] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=AE=9A?= =?UTF-8?q?=E6=97=B6=E6=A3=80=E6=B5=8B=E9=87=8D=E8=BF=9Edisk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecstore/src/disk/format.rs | 49 ++++++++++++++++++++++++++++++++++++++ ecstore/src/disk/mod.rs | 1 + ecstore/src/sets.rs | 2 ++ ecstore/src/store_init.rs | 22 ++++++++--------- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/ecstore/src/disk/format.rs b/ecstore/src/disk/format.rs index 0c37e0862..dcae122be 100644 --- a/ecstore/src/disk/format.rs +++ b/ecstore/src/disk/format.rs @@ -182,6 +182,55 @@ impl FormatV3 { Err(Error::msg(format!("disk id not found {}", disk_id))) } + + pub fn check_other(&self, other: &FormatV3) -> Result<()> { + let mut tmp = other.clone(); + let this = tmp.erasure.this; + tmp.erasure.this = Uuid::nil(); + + if self.erasure.sets.len() != other.erasure.sets.len() { + return Err(Error::from_string(format!( + "Expected number of sets {}, got {}", + self.erasure.sets.len(), + other.erasure.sets.len() + ))); + } + + for i in 0..self.erasure.sets.len() { + if self.erasure.sets[i].len() != other.erasure.sets[i].len() { + return Err(Error::from_string(format!( + "Each set should be of same size, expected {}, got {}", + self.erasure.sets[i].len(), + other.erasure.sets[i].len() + ))); + } + + for j in 0..self.erasure.sets[i].len() { + if self.erasure.sets[i][j] != self.erasure.sets[i][j] { + return Err(Error::from_string(format!( + "UUID on positions {}:{} do not match with, expected {:?} got {:?}: (%w)", + i, + j, + self.erasure.sets[i][j].to_string(), + other.erasure.sets[i][j].to_string(), + ))); + } + } + } + + for i in 0..tmp.erasure.sets.len() { + for j in 0..tmp.erasure.sets[i].len() { + if this == tmp.erasure.sets[i][j] { + return Ok(()); + } + } + } + + Err(Error::msg(format!( + "DriveID {:?} not found in any drive sets {:?}", + this, other.erasure.sets + ))) + } } #[cfg(test)] diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 8c089c8ed..e5d88fc34 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -222,6 +222,7 @@ impl MetaCacheEntry { } } +#[derive(Debug, Default)] pub struct DiskOption { pub cleanup: bool, pub health_check: bool, diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index dc6dd3dda..1da14a235 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -102,6 +102,7 @@ impl Sets { set_index: i, pool_index: pool_idx, set_endpoints, + format: fm.clone(), }; disk_set.push(Arc::new(set_disks)); @@ -159,6 +160,7 @@ impl Sets { for set in self.disk_set.iter() { set.connect_disks().await; } + debug!("done connect_disks ..."); } pub fn get_disks(&self, set_idx: usize) -> Arc { diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 91d580f31..eb37a20fd 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -202,14 +202,19 @@ pub fn default_partiy_count(drive: usize) -> usize { // load_format_erasure_all 读取所有foramt.json async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); - - for disk in disks.iter() { - futures.push(read_format_file(disk, heal)); - } - let mut datas = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); + for disk in disks.iter() { + if disk.is_none() { + datas.push(None); + errors.push(Some(Error::new(DiskError::DiskNotFound))); + } + + let disk = disk.as_ref().unwrap(); + futures.push(load_format_erasure(disk, heal)); + } + let results = join_all(futures).await; let mut i = 0; for result in results { @@ -234,12 +239,7 @@ async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Ve (datas, errors) } -async fn read_format_file(disk: &Option, _heal: bool) -> Result { - if disk.is_none() { - return Err(Error::new(DiskError::DiskNotFound)); - } - let disk = disk.as_ref().unwrap(); - +pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result { let data = disk .read_all(RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE) .await From f1004a1324d972b34c06155c9c242044e376aa8a Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 11:46:48 +0800 Subject: [PATCH 24/70] add all StorageErr --- ecstore/src/disk/error.rs | 103 ++++++++++++++++++++++++++++++++------ ecstore/src/erasure.rs | 2 +- ecstore/src/quorum.rs | 32 ++++++++---- 3 files changed, 111 insertions(+), 26 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 0b5e36301..de6009866 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -1,36 +1,103 @@ -use crate::error::{Error, Result}; +use crate::{ + error::{Error, Result}, + quorum::CheckErrorFn, +}; +// DiskError == StorageErr #[derive(Debug, thiserror::Error)] pub enum DiskError { + #[error("maximum versions exceeded, please delete few versions to proceed")] + MaxVersionsExceeded, + + #[error("unexpected error")] + Unexpected, + + #[error("corrupted format")] + CorruptedFormat, + + #[error("corrupted backend")] + CorruptedBackend, + + #[error("unformatted disk error")] + UnformattedDisk, + + #[error("inconsistent drive found")] + InconsistentDisk, + + #[error("drive does not support O_DIRECT")] + UnsupportedDisk, + + #[error("drive path full")] + DiskFull, + + #[error("disk not a dir")] + DiskNotDir, + + #[error("disk not found")] + DiskNotFound, + + #[error("drive still did not complete the request")] + DiskOngoingReq, + + #[error("drive is part of root drive, will not be used")] + DriveIsRoot, + + #[error("remote drive is faulty")] + FaultyRemoteDisk, + + #[error("drive is faulty")] + FaultyDisk, + + #[error("drive access denied")] + DiskAccessDenied, + #[error("file not found")] FileNotFound, #[error("file version not found")] FileVersionNotFound, - #[error("disk not found")] - DiskNotFound, + #[error("too many open files, please increase 'ulimit -n'")] + TooManyOpenFiles, - #[error("disk access denied")] - FileAccessDenied, - - #[error("InconsistentDisk")] - InconsistentDisk, + #[error("file name too long")] + FileNameTooLong, #[error("volume already exists")] VolumeExists, - #[error("unformatted disk error")] - UnformattedDisk, + #[error("not of regular file type")] + IsNotRegular, - #[error("unsupport disk")] - UnsupportedDisk, - - #[error("disk not a dir")] - DiskNotDir, + #[error("path not found")] + PathNotFound, #[error("volume not found")] VolumeNotFound, + + #[error("volume is not empty")] + VolumeNotEmpty, + + #[error("volume access denied")] + VolumeAccessDenied, + + #[error("disk access denied")] + FileAccessDenied, + + #[error("file is corrupted")] + FileCorrupt, + + #[error("bit-rot hash algorithm is invalid")] + BitrotHashAlgoInvalid, + + #[error("Rename across devices not allowed, please fix your backend configuration")] + CrossDeviceLink, + + #[error("less data available than what was requested")] + LessData, + + #[error("more data was sent than what was advertised")] + MoreData, } impl DiskError { @@ -93,3 +160,9 @@ impl PartialEq for DiskError { core::mem::discriminant(self) == core::mem::discriminant(other) } } + +impl CheckErrorFn for DiskError { + fn is(&self, e: &Error) -> bool { + self.is(e) + } +} diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index dfff7a2b8..82a5ec3d0 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -94,7 +94,7 @@ impl Erasure { warn!("Erasure encode errs {:?}", errs); - let err_idx = reduce_write_quorum_errs(&errs, object_ignored_errs().as_slice(), write_quorum)?; + let err_idx = reduce_write_quorum_errs(&errs, object_ignored_errs().as_ref(), write_quorum)?; if errs[err_idx].is_some() { let err = errs[err_idx].take().unwrap(); return Err(err); diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 124ccfcba..1fd80b36d 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -1,7 +1,11 @@ use crate::{disk::error::DiskError, error::Error}; -use std::collections::HashMap; +use std::{collections::HashMap, fmt::Debug}; -pub type CheckErrorFn = fn(e: &Error) -> bool; +// pub type CheckErrorFn = fn(e: &Error) -> bool; + +pub trait CheckErrorFn: Debug + Send + Sync + 'static { + fn is(&self, e: &Error) -> bool; +} #[derive(Debug, thiserror::Error)] enum QuorumError { @@ -15,17 +19,25 @@ pub fn is_file_not_found(e: &Error) -> bool { DiskError::FileNotFound.is(e) } -pub fn object_ignored_errs() -> Vec { - vec![is_file_not_found] +pub fn base_ignored_errs() -> Vec> { + vec![ + Box::new(DiskError::DiskNotFound), + Box::new(DiskError::FaultyDisk), + Box::new(DiskError::FaultyRemoteDisk), + ] +} + +pub fn object_ignored_errs() -> Vec> { + vec![Box::new(DiskError::FileNotFound)] } // 用于检查错误是否被忽略的函数 -fn is_err_ignored(err: &Error, ignored_errs: &[CheckErrorFn]) -> bool { - ignored_errs.iter().any(|&ignored_err| ignored_err(err)) +fn is_err_ignored(err: &Error, ignored_errs: &Vec>) -> bool { + ignored_errs.iter().any(|ignored_err| ignored_err.is(err)) } // 减少错误数量并返回出现次数最多的错误 -fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usize, Option) { +fn reduce_errs(errs: &Vec>, ignored_errs: &Vec>) -> (usize, Option) { let mut error_counts: HashMap = HashMap::new(); let mut error_map: HashMap = HashMap::new(); // 存err位置 let nil = "nil".to_string(); @@ -69,7 +81,7 @@ fn reduce_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn]) -> (usi } // 根据quorum验证错误数量 -fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], quorum: usize) -> Option { +fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &Vec>, quorum: usize) -> Option { let (max_count, max_err) = reduce_errs(errs, ignored_errs); if max_count >= quorum { max_err @@ -81,7 +93,7 @@ fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &[CheckErrorFn], // 根据读quorum验证错误数量 pub fn reduce_read_quorum_errs( errs: &mut Vec>, - ignored_errs: &[CheckErrorFn], + ignored_errs: &Vec>, read_quorum: usize, ) -> Result { let idx = reduce_quorum_errs(errs, ignored_errs, read_quorum); @@ -95,7 +107,7 @@ pub fn reduce_read_quorum_errs( // 根据写quorum验证错误数量 pub fn reduce_write_quorum_errs( errs: &Vec>, - ignored_errs: &[CheckErrorFn], + ignored_errs: &Vec>, write_quorum: usize, ) -> Result { let idx = reduce_quorum_errs(errs, ignored_errs, write_quorum); From a920204f3eadeceadb117756bff376322a70bcf4 Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 14:32:48 +0800 Subject: [PATCH 25/70] test rename_data --- ecstore/src/disk/local.rs | 11 +++++++++++ ecstore/src/disk/mod.rs | 12 ++++++++++++ ecstore/src/erasure.rs | 4 ++-- ecstore/src/quorum.rs | 14 ++++++++++++-- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index ebcc1267a..90b6e15c1 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -742,6 +742,7 @@ impl DiskAPI for LocalDisk { Ok(RenameDataResp { old_data_dir: old_data_dir, + sign: None, // TODO: }) } @@ -872,6 +873,16 @@ impl DiskAPI for LocalDisk { Ok(RawFileInfo { buf }) } + async fn delete_version( + &self, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result { + unimplemented!() + } async fn delete_versions( &self, volume: &str, diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 717014676..02c064b64 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -79,6 +79,14 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { opts: &ReadOptions, ) -> Result; async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; + async fn delete_version( + &self, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, + ) -> Result; async fn delete_versions( &self, volume: &str, @@ -206,14 +214,18 @@ pub struct DiskOption { pub health_check: bool, } +#[derive(Debug, Default)] pub struct RenameDataResp { pub old_data_dir: Option, + pub sign: Option>, } #[derive(Debug, Clone, Default)] pub struct DeleteOptions { pub recursive: bool, pub immediate: bool, + pub undo_write: bool, + pub old_data_dir: Option, } #[derive(Debug, Clone)] diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 82a5ec3d0..ae88ca8c3 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -1,5 +1,5 @@ use crate::error::{Error, Result, StdError}; -use crate::quorum::{object_ignored_errs, reduce_write_quorum_errs}; +use crate::quorum::{object_op_ignored_errs, reduce_write_quorum_errs}; use bytes::Bytes; use futures::future::join_all; use futures::{Stream, StreamExt}; @@ -94,7 +94,7 @@ impl Erasure { warn!("Erasure encode errs {:?}", errs); - let err_idx = reduce_write_quorum_errs(&errs, object_ignored_errs().as_ref(), write_quorum)?; + let err_idx = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum)?; if errs[err_idx].is_some() { let err = errs[err_idx].take().unwrap(); return Err(err); diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 1fd80b36d..aabfb7994 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -27,8 +27,16 @@ pub fn base_ignored_errs() -> Vec> { ] } -pub fn object_ignored_errs() -> Vec> { - vec![Box::new(DiskError::FileNotFound)] +// object_op_ignored_errs +pub fn object_op_ignored_errs() -> Vec> { + vec![ + Box::new(DiskError::DiskNotFound), + Box::new(DiskError::FaultyDisk), + Box::new(DiskError::FaultyRemoteDisk), + Box::new(DiskError::DiskAccessDenied), + Box::new(DiskError::UnformattedDisk), + Box::new(DiskError::DiskOngoingReq), + ] } // 用于检查错误是否被忽略的函数 @@ -91,6 +99,7 @@ fn reduce_quorum_errs(errs: &Vec>, ignored_errs: &Vec>, ignored_errs: &Vec>, @@ -105,6 +114,7 @@ pub fn reduce_read_quorum_errs( } // 根据写quorum验证错误数量 +// 返回最大错误数量的下标，或QuorumError pub fn reduce_write_quorum_errs( errs: &Vec>, ignored_errs: &Vec>, From 91dd4703e214ff54b9b3290c52ef920704f6b5bf Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 15:26:57 +0800 Subject: [PATCH 26/70] clone_err --- ecstore/src/disk/error.rs | 40 +++++++++++++++++++++++++++++++++++++++ ecstore/src/store.rs | 1 + 2 files changed, 41 insertions(+) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index de6009866..de6079616 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -166,3 +166,43 @@ impl CheckErrorFn for DiskError { self.is(e) } } + +pub fn clone_err(err: &Error) -> Error { + if let Some(e) = err.downcast_ref::() { + match e { + DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded), + DiskError::Unexpected => Error::new(DiskError::Unexpected), + DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat), + DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend), + DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk), + DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk), + DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk), + DiskError::DiskFull => Error::new(DiskError::DiskFull), + DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir), + DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound), + DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq), + DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot), + DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk), + DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk), + DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied), + DiskError::FileNotFound => Error::new(DiskError::FileNotFound), + DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound), + DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles), + DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong), + DiskError::VolumeExists => Error::new(DiskError::VolumeExists), + DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular), + DiskError::PathNotFound => Error::new(DiskError::PathNotFound), + DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound), + DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty), + DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied), + DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied), + DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt), + DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid), + DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), + DiskError::LessData => Error::new(DiskError::LessData), + DiskError::MoreData => Error::new(DiskError::MoreData), + } + } else { + Error::msg(err.to_string()) + } +} diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index bc8343b26..ba14afcf4 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -212,6 +212,7 @@ impl ECStore { DeleteOptions { recursive: true, immediate: false, + ..Default::default() }, )); } From 48d2285d36b76fadeea12826302e50c12f64514b Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 15:32:48 +0800 Subject: [PATCH 27/70] =?UTF-8?q?chore:=20=E6=B7=BB=E5=8A=A0=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=B9=B6=E5=8F=91=E9=99=90=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 1 + ecstore/Cargo.toml | 3 +- ecstore/src/sets.rs | 2 +- ecstore/src/store.rs | 150 +++++++++++++++++++++++++------------------ 4 files changed, 92 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22b19bba1..b8c76b3e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -447,6 +447,7 @@ dependencies = [ "http", "lazy_static", "netif", + "num_cpus", "openssl", "path-absolutize", "path-clean", diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index ada0fc7a5..75f503ef4 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -38,13 +38,14 @@ base64-simd = "0.8.0" sha2 = "0.10.8" hex-simd = "0.8.0" path-clean = "1.0.1" -tokio = { workspace = true, features = ["io-util"] } +tokio = { workspace = true, features = ["io-util", "sync"] } tokio-stream = "0.1.15" tonic.workspace = true tower.workspace = true rmp = "0.8.14" byteorder = "1.5.0" xxhash-rust = { version = "0.8.12", features = ["xxh64"] } +num_cpus = "1.16" [target.'cfg(not(windows))'.dependencies] openssl = "0.10.66" diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 8698b2103..9bbe781a9 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -22,7 +22,7 @@ use crate::{ utils::hash, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Sets { pub id: Uuid, // pub sets: Vec, diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 44473c3b0..74a4f0c45 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -24,7 +24,10 @@ use std::{ time::Duration, }; use time::OffsetDateTime; -use tokio::{fs, sync::RwLock}; +use tokio::{ + fs, + sync::{RwLock, Semaphore}, +}; use tracing::{debug, info, warn}; use uuid::Uuid; @@ -391,62 +394,71 @@ impl ECStore { object: &str, opts: &ObjectOptions, ) -> Result<(PoolObjInfo, Vec)> { - let mut futures = Vec::new(); - - for pool in self.pools.iter() { - futures.push(pool.get_object_info(bucket, object, opts)); - } - - let results = join_all(futures).await; - - let mut ress = Vec::new(); - - let mut i = 0; - - // join_all结果跟输入顺序一致 - for res in results { - let index = i; - - match res { - Ok(r) => { - ress.push(PoolObjInfo { - index, - object_info: r, - err: None, - }); - } - Err(e) => { - ress.push(PoolObjInfo { - index, - err: Some(e), - ..Default::default() - }); - } - } - i += 1; - } - - ress.sort_by(|a, b| { - let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH); - let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH); - - at.cmp(&bt) - }); - - for res in ress { - // check - if res.err.is_none() { - // TODO: let errs = self.poolsWithObject() - return Ok((res, Vec::new())); - } - } - - let ret = PoolObjInfo::default(); - - Ok((ret, Vec::new())) + internal_get_pool_info_existing_with_opts(&self.pools, bucket, object, opts).await } } +async fn internal_get_pool_info_existing_with_opts( + pools: &[Sets], + bucket: &str, + object: &str, + opts: &ObjectOptions, +) -> Result<(PoolObjInfo, Vec)> { + let mut futures = Vec::new(); + + for pool in pools.iter() { + futures.push(pool.get_object_info(bucket, object, opts)); + } + + let results = join_all(futures).await; + + let mut ress = Vec::new(); + + let mut i = 0; + + // join_all结果跟输入顺序一致 + for res in results { + let index = i; + + match res { + Ok(r) => { + ress.push(PoolObjInfo { + index, + object_info: r, + err: None, + }); + } + Err(e) => { + ress.push(PoolObjInfo { + index, + err: Some(e), + ..Default::default() + }); + } + } + i += 1; + } + + ress.sort_by(|a, b| { + let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH); + let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH); + + at.cmp(&bt) + }); + + for res in ress { + // check + if res.err.is_none() { + // TODO: let errs = self.poolsWithObject() + return Ok((res, Vec::new())); + } + } + + let ret = PoolObjInfo::default(); + + Ok((ret, Vec::new())) +} + #[derive(Debug, Default)] pub struct PoolObjInfo { pub index: usize, @@ -559,15 +571,29 @@ impl StorageAPI for ECStore { del_errs.push(None) } - // TODO: limte 限制并发数量 - let opt = ObjectOptions::default(); - // 取所有poolObjInfo - let mut futures = Vec::new(); - for obj in objects.iter() { - futures.push(self.get_pool_info_existing_with_opts(bucket, &obj.object_name, &opt)); - } + let mut jhs = Vec::new(); + let semaphore = Arc::new(Semaphore::new(num_cpus::get())); + let pools = Arc::new(self.pools.clone()); - let results = join_all(futures).await; + for obj in objects.iter() { + let (semaphore, pools, bucket, object_name, opt) = ( + semaphore.clone(), + pools.clone(), + bucket.to_string(), + obj.object_name.to_string(), + ObjectOptions::default(), + ); + + let jh = tokio::spawn(async move { + let _permit = semaphore.acquire().await.unwrap(); + internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt).await + }); + jhs.push(jh); + } + let mut results = Vec::new(); + for jh in jhs { + results.push(jh.await.unwrap()); + } // 记录pool Index 对应的objects pool_idx -> objects idx let mut pool_index_objects = HashMap::new(); From 629f2f000f4a88e368fee9e92c2ce5088b9cda6f Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 15:33:48 +0800 Subject: [PATCH 28/70] clone_err From 184d593d27ded9111822c1cf081f2db6d4653126 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 15:38:00 +0800 Subject: [PATCH 29/70] make clippy happy --- ecstore/src/sets.rs | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 9bbe781a9..c9f77e8ee 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -205,10 +205,10 @@ impl StorageAPI for Sets { let mut set_obj_map = HashMap::new(); // hash key - let mut i = 0; - for obj in objects.iter() { + for (i, obj) in objects.iter().enumerate() { let idx = self.get_hashed_set_index(obj.object_name.as_str()); + #[allow(clippy::map_entry)] if !set_obj_map.contains_key(&idx) { set_obj_map.insert( idx, @@ -218,17 +218,13 @@ impl StorageAPI for Sets { obj: obj.clone(), }], ); - } else { - if let Some(val) = set_obj_map.get_mut(&idx) { - val.push(DelObj { - // set_idx: idx, - orig_idx: i, - obj: obj.clone(), - }); - } + } else if let Some(val) = set_obj_map.get_mut(&idx) { + val.push(DelObj { + // set_idx: idx, + orig_idx: i, + obj: obj.clone(), + }); } - - i += 1; } // TODO: 并发 @@ -237,15 +233,12 @@ impl StorageAPI for Sets { let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?; - let mut i = 0; - for err in errs { + for (i, err) in errs.into_iter().enumerate() { let obj = v.get(i).unwrap(); del_errs[obj.orig_idx] = err; del_objects[obj.orig_idx] = dobjects.get(i).unwrap().clone(); - - i += 1; } } From 3bc98eec161ce4359b5f23fd56d2094c838d38d2 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 16:07:02 +0800 Subject: [PATCH 30/70] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=B9=B6=E5=8F=91?= =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=AF=B9=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecstore/src/sets.rs | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index c9f77e8ee..819cbd1e0 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,7 +1,8 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use futures::future::join_all; use http::HeaderMap; +use tokio::sync::Semaphore; use tracing::warn; use uuid::Uuid; @@ -227,19 +228,31 @@ impl StorageAPI for Sets { } } - // TODO: 并发 + let semaphore = Arc::new(Semaphore::new(num_cpus::get())); + let mut jhs = Vec::with_capacity(semaphore.available_permits()); + for (k, v) in set_obj_map { let disks = self.get_disks(k); - let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); - let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?; + let semaphore = semaphore.clone(); + let opts = opts.clone(); + let bucket = bucket.to_string(); - for (i, err) in errs.into_iter().enumerate() { - let obj = v.get(i).unwrap(); + let jh = tokio::spawn(async move { + let _permit = semaphore.acquire().await.unwrap(); + let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); + disks.delete_objects(&bucket, objs, opts).await + }); + jhs.push(jh); + } - del_errs[obj.orig_idx] = err; + let mut results = Vec::with_capacity(jhs.len()); + for jh in jhs { + results.push(jh.await?.unwrap()); + } - del_objects[obj.orig_idx] = dobjects.get(i).unwrap().clone(); - } + for (dobjects, errs) in results { + del_objects.extend(dobjects); + del_errs.extend(errs); } Ok((del_objects, del_errs)) From 20ba112bd854b88ea60b281f90a95c267acd01df Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 16:27:33 +0800 Subject: [PATCH 31/70] make clippy happy --- ecstore/src/sets.rs | 2 +- ecstore/src/store.rs | 41 +++++++++++++---------------------------- 2 files changed, 14 insertions(+), 29 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 819cbd1e0..329d03f85 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,3 +1,4 @@ +#![allow(clippy::map_entry)] use std::{collections::HashMap, sync::Arc}; use futures::future::join_all; @@ -209,7 +210,6 @@ impl StorageAPI for Sets { for (i, obj) in objects.iter().enumerate() { let idx = self.get_hashed_set_index(obj.object_name.as_str()); - #[allow(clippy::map_entry)] if !set_obj_map.contains_key(&idx) { set_obj_map.insert( idx, diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 74a4f0c45..5c270ba88 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,3 +1,4 @@ +#![allow(clippy::map_entry)] use crate::{ bucket_meta::BucketMetadata, disk::{ @@ -54,10 +55,11 @@ pub async fn update_erasure_type(setup_type: SetupType) { *is_erasure_sd = setup_type == SetupType::ErasureSD; } +type TypeLocalDiskSetDrives = Vec>>>; + lazy_static! { pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); - pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc>>>>> = - Arc::new(RwLock::new(Vec::new())); + pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc> = Arc::new(RwLock::new(Vec::new())); } pub async fn find_local_disk(disk_path: &String) -> Option { @@ -79,7 +81,7 @@ pub async fn find_local_disk(disk_path: &String) -> Option { pub async fn all_local_disk_path() -> Vec { let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; - disk_map.keys().map(|v| v.clone()).collect() + disk_map.keys().cloned().collect() } pub async fn all_local_disk() -> Vec { @@ -159,6 +161,7 @@ pub struct ECStore { } impl ECStore { + #[allow(clippy::new_ret_no_self)] pub async fn new(_address: String, endpoint_pools: EndpointServerPools) -> Result<()> { // let layouts = DisksLayout::try_from(endpoints.as_slice())?; @@ -321,8 +324,8 @@ impl ECStore { if entry.is_object() { let fi = entry.to_fileinfo(&opts.bucket)?; - if fi.is_some() { - ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false)); + if let Some(f) = fi { + ress.push(f.into_object_info(&opts.bucket, &entry.name, false)); } continue; } @@ -414,10 +417,8 @@ async fn internal_get_pool_info_existing_with_opts( let mut ress = Vec::new(); - let mut i = 0; - // join_all结果跟输入顺序一致 - for res in results { + for (i, res) in results.into_iter().enumerate() { let index = i; match res { @@ -436,7 +437,6 @@ async fn internal_get_pool_info_existing_with_opts( }); } } - i += 1; } ress.sort_by(|a, b| { @@ -598,8 +598,7 @@ impl StorageAPI for ECStore { // 记录pool Index 对应的objects pool_idx -> objects idx let mut pool_index_objects = HashMap::new(); - let mut i = 0; - for res in results { + for (i, res) in results.into_iter().enumerate() { match res { Ok((pinfo, _)) => { if pinfo.object_info.delete_marker && opts.version_id.is_empty() { @@ -627,8 +626,6 @@ impl StorageAPI for ECStore { del_errs[i] = Some(e) } } - - i += 1; } if !pool_index_objects.is_empty() { @@ -641,16 +638,7 @@ impl StorageAPI for ECStore { let obj_idxs = vals.unwrap(); // 取对应obj,理论上不会none - let objs: Vec = obj_idxs - .iter() - .filter_map(|&idx| { - if let Some(obj) = objects.get(idx) { - Some(obj.clone()) - } else { - None - } - }) - .collect(); + let objs: Vec = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect(); if objs.is_empty() { continue; @@ -659,8 +647,7 @@ impl StorageAPI for ECStore { let (pdel_objs, perrs) = sets.delete_objects(bucket, objs, opts.clone()).await?; // perrs的顺序理论上跟obj_idxs顺序一致 - let mut i = 0; - for err in perrs { + for (i, err) in perrs.into_iter().enumerate() { let obj_idx = obj_idxs[i]; if err.is_some() { @@ -671,8 +658,6 @@ impl StorageAPI for ECStore { dobj.object_name = utils::path::decode_dir_object(&dobj.object_name); del_objects[obj_idx] = dobj; - - i += 1; } } } @@ -681,7 +666,7 @@ impl StorageAPI for ECStore { } async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result { if opts.delete_prefix { - self.delete_prefix(bucket, &object).await?; + self.delete_prefix(bucket, object).await?; return Ok(ObjectInfo::default()); } From ef8995006d48296ee52348bbc9c954220cc67b44 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Fri, 20 Sep 2024 16:11:49 +0800 Subject: [PATCH 32/70] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E7=8E=AF=E5=A2=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 39 +++++++++++++++++++++++++++++++++++++++ Makefile | 18 ++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..2208355a5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM m.daocloud.io/docker.io/library/ubuntu:22.04 + +ENV LANG C.UTF-8 + +RUN sed -i s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g /etc/apt/sources.list + +RUN apt-get clean && apt-get update && apt-get install wget git curl unzip gcc pkg-config libssl-dev -y + +# install protoc +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip \ + && unzip protoc-27.0-linux-x86_64.zip -d protoc3 \ + && mv protoc3/bin/* /usr/local/bin/ && chmod +x /usr/local/bin/protoc && mv protoc3/include/* /usr/local/include/ && rm -rf protoc-27.0-linux-x86_64.zip protoc3 + +# install flatc +RUN wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip \ + && unzip Linux.flatc.binary.g++-13.zip \ + && mv flatc /usr/local/bin/ && chmod +x /usr/local/bin/flatc && rm -rf Linux.flatc.binary.g++-13.zip + +# install rust +ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" +ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" +RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ + && sh rustup-init.sh -y && rm -rf rustup-init.sh + +RUN echo "[source.crates-io]" > /root/.cargo/config.toml \ +&& echo "registry = \"https://github.com/rust-lang/crates.io-index\"" >> /root/.cargo/config.toml \ +&& echo "replace-with = \"rsproxy-sparse\"" >> /root/.cargo/config.toml \ +&& echo "[source.rsproxy]" >> /root/.cargo/config.toml \ +&& echo "registry = \"https://rsproxy.cn/crates.io-index\"" >> /root/.cargo/config.toml \ +&& echo "[source.rsproxy-sparse]" >> /root/.cargo/config.toml \ +&& echo "registry = \"sparse+https://rsproxy.cn/index/\"" >> /root/.cargo/config.toml \ +&& echo "[registries.rsproxy]" >> /root/.cargo/config.toml \ +&& echo "index = \"https://rsproxy.cn/crates.io-index\"" >> /root/.cargo/config.toml \ +&& echo "[net]" >> /root/.cargo/config.toml \ +&& echo "git-fetch-with-cli = true" >> /root/.cargo/config.toml + +WORKDIR /root/s3-rustfs + +CMD [ "bash", "-c", "while true; do sleep 1; done" ] diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..d96c37069 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +########### +# 远程开发，需要 VSCode 安装 Dev Containers, Remote SSH, Remote Explorer +# https://code.visualstudio.com/docs/remote/containers +########### +DOCKER_CLI ?= docker +IMAGE_NAME ?= rustfs:v1.0.0 +CONTAINER_NAME ?= rustfs-dev + +.PHONY: init +init: + $(DOCKER_CLI) build -t $(IMAGE_NAME) . + $(DOCKER_CLI) stop $(CONTAINER_NAME) + $(DOCKER_CLI) rm $(CONTAINER_NAME) + $(DOCKER_CLI) run -d --name $(CONTAINER_NAME) -p 9010:9010 -v $(shell pwd):/root/s3-rustfs -it $(IMAGE_NAME) + +.PHONY: start +start: + $(DOCKER_CLI) start $(CONTAINER_NAME) From f6be1cecfe62945e5d2044d4dab0a623213ae09a Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Fri, 20 Sep 2024 16:29:11 +0800 Subject: [PATCH 33/70] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E7=8E=AF=E5=A2=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d96c37069..6b3ef8eac 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ init: $(DOCKER_CLI) build -t $(IMAGE_NAME) . $(DOCKER_CLI) stop $(CONTAINER_NAME) $(DOCKER_CLI) rm $(CONTAINER_NAME) - $(DOCKER_CLI) run -d --name $(CONTAINER_NAME) -p 9010:9010 -v $(shell pwd):/root/s3-rustfs -it $(IMAGE_NAME) + $(DOCKER_CLI) run -d --name $(CONTAINER_NAME) -p 9010:9010 -p 9000:9000 -v $(shell pwd):/root/s3-rustfs -it $(IMAGE_NAME) .PHONY: start start: From 8381eb96d5e223d133855b86f1ed9788291e9c37 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 16:48:53 +0800 Subject: [PATCH 34/70] =?UTF-8?q?=E8=AE=BE=E7=BD=AEci=E6=B5=81=E6=B0=B4?= =?UTF-8?q?=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile => .docker/Dockerfile.devenv | 12 +----- .docker/cargo.config.toml | 13 +++++++ .github/workflows/rust.yml | 50 +++++++++++++++++++++---- Makefile | 11 ++++-- 4 files changed, 65 insertions(+), 21 deletions(-) rename Dockerfile => .docker/Dockerfile.devenv (60%) create mode 100644 .docker/cargo.config.toml diff --git a/Dockerfile b/.docker/Dockerfile.devenv similarity index 60% rename from Dockerfile rename to .docker/Dockerfile.devenv index 2208355a5..e95027d2e 100644 --- a/Dockerfile +++ b/.docker/Dockerfile.devenv @@ -22,17 +22,7 @@ ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ && sh rustup-init.sh -y && rm -rf rustup-init.sh -RUN echo "[source.crates-io]" > /root/.cargo/config.toml \ -&& echo "registry = \"https://github.com/rust-lang/crates.io-index\"" >> /root/.cargo/config.toml \ -&& echo "replace-with = \"rsproxy-sparse\"" >> /root/.cargo/config.toml \ -&& echo "[source.rsproxy]" >> /root/.cargo/config.toml \ -&& echo "registry = \"https://rsproxy.cn/crates.io-index\"" >> /root/.cargo/config.toml \ -&& echo "[source.rsproxy-sparse]" >> /root/.cargo/config.toml \ -&& echo "registry = \"sparse+https://rsproxy.cn/index/\"" >> /root/.cargo/config.toml \ -&& echo "[registries.rsproxy]" >> /root/.cargo/config.toml \ -&& echo "index = \"https://rsproxy.cn/crates.io-index\"" >> /root/.cargo/config.toml \ -&& echo "[net]" >> /root/.cargo/config.toml \ -&& echo "git-fetch-with-cli = true" >> /root/.cargo/config.toml +COPY .docker/cargo.config.toml /root/.cargo/config.toml WORKDIR /root/s3-rustfs diff --git a/.docker/cargo.config.toml b/.docker/cargo.config.toml new file mode 100644 index 000000000..ef2fa863f --- /dev/null +++ b/.docker/cargo.config.toml @@ -0,0 +1,13 @@ +[source.crates-io] +registry = "https://github.com/rust-lang/crates.io-index" +replace-with = 'rsproxy-sparse' + +[source.rsproxy] +registry = "https://rsproxy.cn/crates.io-index" +[registries.rsproxy] +index = "https://rsproxy.cn/crates.io-index" +[source.rsproxy-sparse] +registry = "sparse+https://rsproxy.cn/index/" + +[net] +git-fetch-with-cli = true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9fd45e090..aa949c986 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,8 +1,8 @@ name: Rust on: + workflow_dispatch: push: - branches: [ "main" ] pull_request: branches: [ "main" ] @@ -11,12 +11,48 @@ env: jobs: build: - runs-on: ubuntu-latest + strategy: + matrix: + rust: + - stable + - beta + - nightly steps: - - uses: actions/checkout@v4 - - name: Build - run: cargo build --verbose - - name: Run tests - run: cargo test --verbose + - name: cache protoc bin + id: cache-protoc-action + uses: actions/cache@v3 + env: + cache-name: cache-protoc-action-bin + with: + path: /usr/local/bin/protoc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install protoc + if: steps.cache-protoc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip + unzip protoc-27.0-linux-x86_64.zip -d protoc3 + mv protoc3/bin/* /usr/local/bin/ + chmod +x /usr/local/bin/protoc + rm -rf protoc-27.0-linux-x86_64.zip protoc3 + + - uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + components: rustfmt, clippy + + - uses: actions-rs/cargo@v1 + with: + command: build + args: --verbose + + - uses: actions-rs/cargo@v1 + with: + command: test + args: --all --verbose diff --git a/Makefile b/Makefile index 6b3ef8eac..6f05e92b5 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,11 @@ DOCKER_CLI ?= docker IMAGE_NAME ?= rustfs:v1.0.0 CONTAINER_NAME ?= rustfs-dev +DOCKERFILE ?= $(shell pwd)/.docker/Dockerfile.devenv -.PHONY: init -init: - $(DOCKER_CLI) build -t $(IMAGE_NAME) . +.PHONY: init-devenv +init-devenv: + $(DOCKER_CLI) build -t $(IMAGE_NAME) -f $(DOCKERFILE) . $(DOCKER_CLI) stop $(CONTAINER_NAME) $(DOCKER_CLI) rm $(CONTAINER_NAME) $(DOCKER_CLI) run -d --name $(CONTAINER_NAME) -p 9010:9010 -p 9000:9000 -v $(shell pwd):/root/s3-rustfs -it $(IMAGE_NAME) @@ -16,3 +17,7 @@ init: .PHONY: start start: $(DOCKER_CLI) start $(CONTAINER_NAME) + +.PHONY: stop +stop: + $(DOCKER_CLI) stop $(CONTAINER_NAME) From 5417965e38b6b8d4b20fbf0dc3380be4ec15f858 Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 17:56:30 +0800 Subject: [PATCH 35/70] review disk api --- ecstore/src/disk/local.rs | 20 +++++++- ecstore/src/disk/mod.rs | 98 ++++++++++++++++++++++++-------------- ecstore/src/disk/remote.rs | 20 +++++++- 3 files changed, 100 insertions(+), 38 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 5ac8a9350..78ab0175f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,7 +1,7 @@ use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, - ReadMultipleResp, ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions, + ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::{ @@ -451,12 +451,23 @@ fn skip_access_checks(p: impl AsRef) -> bool { #[async_trait::async_trait] impl DiskAPI for LocalDisk { + fn to_string(&self) -> String { + self.root.to_string_lossy().to_string() + } fn is_local(&self) -> bool { true } + fn host_name(&self) -> String { + self.endpoint.host_port() + } async fn is_online(&self) -> bool { true } + + fn endpoint(&self) -> Endpoint { + self.endpoint.clone() + } + async fn close(&self) -> Result<()> { Ok(()) } @@ -959,7 +970,12 @@ impl DiskAPI for LocalDisk { created: modtime, }) } - + async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + unimplemented!() + } + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) { + unimplemented!() + } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index efee53ee1..fd46396c0 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -19,6 +19,7 @@ use crate::{ store_api::{FileInfo, RawFileInfo}, }; use bytes::Bytes; +use endpoint::Endpoint; use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest}; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc}; @@ -44,50 +45,34 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result bool; + fn to_string(&self) -> String; async fn is_online(&self) -> bool; - fn path(&self) -> PathBuf; + fn is_local(&self) -> bool; + // LastConn + fn host_name(&self) -> String; + fn endpoint(&self) -> Endpoint; async fn close(&self) -> Result<()>; async fn get_disk_id(&self) -> Result>; async fn set_disk_id(&self, id: Option) -> Result<()>; + + fn path(&self) -> PathBuf; fn get_location(&self) -> DiskLocation; - async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; - async fn read_all(&self, volume: &str, path: &str) -> Result; - async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; - async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result; - async fn append_file(&self, volume: &str, path: &str) -> Result; - async fn read_file(&self, volume: &str, path: &str) -> Result; - // 读目录下的所有文件、目录 - async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result>; + // Healing + // DiskInfo + // NSScanner + + // Volume operations. + async fn make_volume(&self, volume: &str) -> Result<()>; + async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>; + async fn list_volumes(&self) -> Result>; + async fn stat_volume(&self, volume: &str) -> Result; + async fn delete_volume(&self, volume: &str) -> Result<()>; + // 并发边读边写 TODO: wr io.Writer async fn walk_dir(&self, opts: WalkDirOptions) -> Result>; - async fn rename_data( - &self, - src_volume: &str, - src_path: &str, - file_info: FileInfo, - dst_volume: &str, - dst_path: &str, - ) -> Result; - async fn make_volumes(&self, volume: Vec<&str>) -> Result<()>; - async fn delete_volume(&self, volume: &str) -> Result<()>; - async fn list_volumes(&self) -> Result>; - async fn make_volume(&self, volume: &str) -> Result<()>; - async fn stat_volume(&self, volume: &str) -> Result; - - async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>; - async fn read_version( - &self, - org_volume: &str, - volume: &str, - path: &str, - version_id: &str, - opts: &ReadOptions, - ) -> Result; - async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; + // Metadata operations async fn delete_version( &self, volume: &str, @@ -102,7 +87,50 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { versions: Vec, opts: DeleteOptions, ) -> Result>>; + async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()>; + async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>; + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts); + async fn read_version( + &self, + org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> Result; + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result; + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + file_info: FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result; + + // File operations. + // 读目录下的所有文件、目录 + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result>; + async fn read_file(&self, volume: &str, path: &str) -> Result; + async fn append_file(&self, volume: &str, path: &str) -> Result; + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result; + // ReadFileStream + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; + // RenamePart + // CheckParts + async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; + // VerifyFile + // StatInfoFile + // ReadParts async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; + // CleanAbandonedData + async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; + async fn read_all(&self, volume: &str, path: &str) -> Result; + // GetDiskLoc +} + +pub struct UpdateMetadataOpts { + pub no_persistence: bool, } pub struct DiskLocation { diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 17a359445..5eed152c2 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -30,7 +30,7 @@ use crate::{ use super::{ endpoint::Endpoint, DeleteOptions, DiskAPI, DiskLocation, DiskOption, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, - VolumeInfo, WalkDirOptions, + UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; #[derive(Debug)] @@ -97,9 +97,17 @@ impl RemoteDisk { // TODO: all api need to handle errors #[async_trait::async_trait] impl DiskAPI for RemoteDisk { + fn to_string(&self) -> String { + self.endpoint.to_string() + } + fn is_local(&self) -> bool { false } + + fn host_name(&self) -> String { + self.endpoint.host_port() + } async fn is_online(&self) -> bool { // TODO: 连接状态 if let Ok(_) = self.get_client_v2().await { @@ -107,6 +115,9 @@ impl DiskAPI for RemoteDisk { } false } + fn endpoint(&self) -> Endpoint { + self.endpoint.clone() + } async fn close(&self) -> Result<()> { Ok(()) } @@ -390,6 +401,13 @@ impl DiskAPI for RemoteDisk { Ok(volume_info) } + async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + unimplemented!() + } + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) { + unimplemented!() + } + async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { info!("write_metadata"); let file_info = serde_json::to_string(&fi)?; From 6a3e01b3ccd6edbc834c4a1c2926cb505cf122cd Mon Sep 17 00:00:00 2001 From: weisd Date: Mon, 23 Sep 2024 18:02:03 +0800 Subject: [PATCH 36/70] rm unuse log --- ecstore/src/disk/local.rs | 1 - ecstore/src/sets.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 78ab0175f..586ed0772 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -484,7 +484,6 @@ impl DiskAPI for LocalDisk { } async fn get_disk_id(&self) -> Result> { - warn!("local get_disk_id"); // TODO: check format file let mut format_info = self.format_info.lock().await; diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 256456a0e..319a04c3e 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -93,7 +93,7 @@ impl Sets { } } - warn!("sets new set_drive {:?}", &set_drive); + // warn!("sets new set_drive {:?}", &set_drive); let set_disks = SetDisks { disks: RwLock::new(set_drive), From ed6fde5e4d43bd44a55e9cd895a71b9383202e52 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 18:05:18 +0800 Subject: [PATCH 37/70] print protoc version --- .github/workflows/rust.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index aa949c986..cd5d532c4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -37,7 +37,10 @@ jobs: mv protoc3/bin/* /usr/local/bin/ chmod +x /usr/local/bin/protoc rm -rf protoc-27.0-linux-x86_64.zip protoc3 - + + - name: print protoc version + run: protoc --version + - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From 54101378e3420c8f9f4483a4971f81d2557c3dba Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 18:11:08 +0800 Subject: [PATCH 38/70] install flatc --- .github/workflows/rust.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index cd5d532c4..83f4d280a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -40,7 +40,25 @@ jobs: - name: print protoc version run: protoc --version - + + - name: cache flatc bin + id: cache-flatc-action + uses: actions/cache@v3 + env: + cache-name: cache-flatc-action-bin + with: + path: /usr/local/bin/flatc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install flatc + if: steps.cache-flatc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip + unzip Linux.flatc.binary.g++-13.zip + mv flatc /usr/local/bin/ + chmod +x /usr/local/bin/flatc + rm -rf Linux.flatc.binary.g++-13.zip + - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From 8cce210d640351585bab191482f3693813fd5d17 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Mon, 23 Sep 2024 18:44:55 +0800 Subject: [PATCH 39/70] install flatc --- .github/workflows/rust.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 83f4d280a..6dd5d7024 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -71,9 +71,8 @@ jobs: - uses: actions-rs/cargo@v1 with: command: build - args: --verbose - uses: actions-rs/cargo@v1 with: command: test - args: --all --verbose + args: --all From 29100d525011c8e458edfb87af30c3d160a1d039 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 24 Sep 2024 00:00:44 +0800 Subject: [PATCH 40/70] review disk --- ecstore/src/disk/error.rs | 58 +++++++++++++++++++++ ecstore/src/disk/local.rs | 101 ++++++++++++++++++++++++------------- ecstore/src/disk/mod.rs | 5 +- ecstore/src/disk/os.rs | 93 ++++++++++++++++++++++++++++++++++ ecstore/src/disk/remote.rs | 2 +- ecstore/src/quorum.rs | 20 ++++---- ecstore/src/utils/fs.rs | 33 +++++++++++- ecstore/src/utils/path.rs | 2 +- 8 files changed, 265 insertions(+), 49 deletions(-) create mode 100644 ecstore/src/disk/os.rs diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index de6079616..7b69463ac 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -1,3 +1,5 @@ +use std::io::{self, ErrorKind}; + use crate::{ error::{Error, Result}, quorum::CheckErrorFn, @@ -98,6 +100,9 @@ pub enum DiskError { #[error("more data was sent than what was advertised")] MoreData, + + #[error("other io err {0}")] + IoError(io::Error), } impl DiskError { @@ -201,8 +206,61 @@ pub fn clone_err(err: &Error) -> Error { DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), DiskError::LessData => Error::new(DiskError::LessData), DiskError::MoreData => Error::new(DiskError::MoreData), + DiskError::IoError(ioerr) => Error::msg(ioerr.to_string()), } } else { Error::msg(err.to_string()) } } + + +pub fn ioerr_to_diskerr(e:io::Error)->DiskError{ +match e.kind(){ + io::ErrorKind::NotFound => DiskError::FileNotFound, + io::ErrorKind::PermissionDenied => DiskError::FileAccessDenied, + // io::ErrorKind::ConnectionRefused => todo!(), + // io::ErrorKind::ConnectionReset => todo!(), + // io::ErrorKind::HostUnreachable => todo!(), + // io::ErrorKind::NetworkUnreachable => todo!(), + // io::ErrorKind::ConnectionAborted => todo!(), + // io::ErrorKind::NotConnected => todo!(), + // io::ErrorKind::AddrInUse => todo!(), + // io::ErrorKind::AddrNotAvailable => todo!(), + // io::ErrorKind::NetworkDown => todo!(), + // io::ErrorKind::BrokenPipe => todo!(), + // io::ErrorKind::AlreadyExists => todo!(), + // io::ErrorKind::WouldBlock => todo!(), + // io::ErrorKind::NotADirectory => DiskError::FileNotFound, + // io::ErrorKind::IsADirectory => DiskError::FileNotFound, + // io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty, + // io::ErrorKind::ReadOnlyFilesystem => todo!(), + // io::ErrorKind::FilesystemLoop => todo!(), + // io::ErrorKind::StaleNetworkFileHandle => todo!(), + // io::ErrorKind::InvalidInput => todo!(), + // io::ErrorKind::InvalidData => todo!(), + // io::ErrorKind::TimedOut => todo!(), + // io::ErrorKind::WriteZero => todo!(), + // io::ErrorKind::StorageFull => DiskError::DiskFull, + // io::ErrorKind::NotSeekable => todo!(), + // io::ErrorKind::FilesystemQuotaExceeded => todo!(), + // io::ErrorKind::FileTooLarge => todo!(), + // io::ErrorKind::ResourceBusy => todo!(), + // io::ErrorKind::ExecutableFileBusy => todo!(), + // io::ErrorKind::Deadlock => todo!(), + // io::ErrorKind::CrossesDevices => todo!(), + // io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles, + // io::ErrorKind::InvalidFilename => todo!(), + // io::ErrorKind::ArgumentListTooLong => todo!(), + // io::ErrorKind::Interrupted => todo!(), + // io::ErrorKind::Unsupported => todo!(), + // io::ErrorKind::UnexpectedEof => todo!(), + // io::ErrorKind::OutOfMemory => todo!(), + // io::ErrorKind::Other => todo!(), + // TODO: 把不支持的king用字符串处理 + _ => DiskError::IoError(e), +} +} + +pub fn os_is_not_exist(e:io::Error) -> bool{ + e.kind() == ErrorKind::NotFound +} \ No newline at end of file diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 586ed0772..fd0e5d388 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,9 +1,11 @@ +use super::error::{ioerr_to_diskerr, os_is_not_exist}; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ - DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, + os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; +use crate::utils::path::SLASH_SEPARATOR; use crate::{ error::{Error, Result}, file_meta::FileMeta, @@ -93,6 +95,8 @@ impl LocalDisk { last_check: format_last_check, }; + // TODO: DIRECT suport + // TODD: DiskInfo let disk = Self { root, endpoint: ep.clone(), @@ -109,6 +113,36 @@ impl LocalDisk { Ok(disk) } + fn check_path_length(path_name: &str) -> Result<()> { + unimplemented!() + } + + fn is_valid_volname(volname: &str) -> bool { + if volname.len() < 3 { + return false; + } + + if cfg!(target_os = "windows") { + // 在 Windows 上，卷名不应该包含保留字符。 + // 这个正则表达式匹配了不允许的字符。 + if volname.contains('|') + || volname.contains('<') + || volname.contains('>') + || volname.contains('?') + || volname.contains('*') + || volname.contains(':') + || volname.contains('"') + || volname.contains('\\') + { + return false; + } + } else { + // 对于非 Windows 系统，可能需要其他的验证逻辑。 + } + + true + } + async fn check_format_json(&self) -> Result { let md = fs::metadata(&self.format_path).await.map_err(|e| match e.kind() { ErrorKind::NotFound => DiskError::DiskNotFound, @@ -475,7 +509,7 @@ impl DiskAPI for LocalDisk { self.root.clone() } - fn get_location(&self) -> DiskLocation { + fn get_disk_location(&self) -> DiskLocation { DiskLocation { pool_idx: self.endpoint.pool_idx, set_idx: self.endpoint.set_idx, @@ -900,54 +934,51 @@ impl DiskAPI for LocalDisk { async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { for vol in volumes { if let Err(e) = self.make_volume(vol).await { - match &e.downcast_ref::() { - Some(DiskError::VolumeExists) => Ok(()), - Some(_) => Err(e), - None => Err(e), - }?; + if !DiskError::VolumeExists.is(&e) { + return Err(e); + } } // TODO: health check } Ok(()) } async fn make_volume(&self, volume: &str) -> Result<()> { + if !Self::is_valid_volname(volume) { + return Err(Error::msg("Invalid arguments specified")); + } + let p = self.get_bucket_path(volume)?; - match File::open(&p).await { - Ok(_) => (), - Err(e) => match e.kind() { - ErrorKind::NotFound => { - fs::create_dir_all(&p).await?; - return Ok(()); - } - _ => return Err(Error::from(e)), - }, + + if let Err(err) = utils::fs::access(&p).await { + if os_is_not_exist(err) { + os::make_dir_all(&p).await?; + } } Err(Error::from(DiskError::VolumeExists)) } async fn list_volumes(&self) -> Result> { - let mut entries = fs::read_dir(&self.root).await?; - let mut volumes = Vec::new(); - while let Some(entry) = entries.next_entry().await? { - if let Ok(metadata) = entry.metadata().await { - // if !metadata.is_dir() { - // continue; - // } - - let name = entry.file_name().to_string_lossy().to_string(); - - let created = match metadata.created() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => { - warn!("Not supported created on this platform"); - None - } - }; - - volumes.push(VolumeInfo { name, created }); + let entries = os::read_dir(&self.root, 0).await.map_err(|e| { + if DiskError::FileAccessDenied.is(&e) { + Error::new(DiskError::DiskAccessDenied) + } else if DiskError::FileNotFound.is(&e) { + Error::new(DiskError::DiskAccessDenied) + } else { + e } + })?; + + for entry in entries { + if utils::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(&entry) { + continue; + } + + volumes.push(VolumeInfo { + name: entry, + created: None, + }); } Ok(volumes) diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index fd46396c0..3ec7f52df 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -3,6 +3,7 @@ pub mod error; pub mod format; mod local; mod remote; +pub mod os; pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; @@ -56,7 +57,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn set_disk_id(&self, id: Option) -> Result<()>; fn path(&self) -> PathBuf; - fn get_location(&self) -> DiskLocation; + fn get_disk_location(&self) -> DiskLocation; // Healing // DiskInfo @@ -126,7 +127,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; - // GetDiskLoc + } pub struct UpdateMetadataOpts { diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs new file mode 100644 index 000000000..4fa9336b3 --- /dev/null +++ b/ecstore/src/disk/os.rs @@ -0,0 +1,93 @@ +use std::path::Path; + +use futures::TryFutureExt; +use tokio::fs; + +use crate::{ + error::{Error, Result}, + utils, +}; + +use super::error::{ioerr_to_diskerr, DiskError}; + +fn check_path_length(path_name: &str) -> Result<()> { + // Apple OS X path length is limited to 1016 + if cfg!(target_os = "macos") && path_name.len() > 1016 { + return Err(Error::new(DiskError::FileNameTooLong)); + } + + // Disallow more than 1024 characters on windows, there + // are no known name_max limits on Windows. + if cfg!(target_os = "windows") && path_name.len() > 1024 { + return Err(Error::new(DiskError::FileNameTooLong)); + } + + // On Unix we reject paths if they are just '.', '..' or '/' + let invalid_paths = [".", "..", "/"]; + if invalid_paths.contains(&path_name) { + return Err(Error::new(DiskError::FileAccessDenied)); + } + + // Check each path segment length is > 255 on all Unix + // platforms, look for this value as NAME_MAX in + // /usr/include/linux/limits.h + let mut count = 0usize; + for c in path_name.chars() { + match c { + '/' | '\\' if cfg!(target_os = "windows") => count = 0, // Reset + _ => { + count += 1; + if count > 255 { + return Err(Error::new(DiskError::FileNameTooLong)); + } + } + } + } + + // Success. + Ok(()) +} + +pub async fn make_dir_all(path: impl AsRef) -> Result<()> { + check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; + + utils::fs::make_dir_all(path.as_ref()).map_err(ioerr_to_diskerr).await?; + + Ok(()) +} + +// read_dir count read limit. when count == 0 unlimit. +pub async fn read_dir(path: impl AsRef, count: usize) -> Result> { + let mut entries = fs::read_dir(path.as_ref()).await?; + + let mut volumes = Vec::new(); + + let mut count: i32 = { + if count == 0 { + -1 + } else { + count as i32 + } + }; + + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().to_string_lossy().to_string(); + + if name == "" || name == "." || name == ".." { + continue; + } + + let file_type = entry.file_type().await?; + + if file_type.is_dir() { + count -= 1; + volumes.push(format!("{}{}", name, utils::path::SLASH_SEPARATOR)); + + if count == 0 { + break; + } + } + } + + Ok(volumes) +} diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 5eed152c2..0034235f8 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -125,7 +125,7 @@ impl DiskAPI for RemoteDisk { self.root.clone() } - fn get_location(&self) -> DiskLocation { + fn get_disk_location(&self) -> DiskLocation { DiskLocation { pool_idx: self.endpoint.pool_idx, set_idx: self.endpoint.set_idx, diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index aabfb7994..13544632f 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -15,10 +15,6 @@ enum QuorumError { Write, } -pub fn is_file_not_found(e: &Error) -> bool { - DiskError::FileNotFound.is(e) -} - pub fn base_ignored_errs() -> Vec> { vec![ Box::new(DiskError::DiskNotFound), @@ -29,14 +25,20 @@ pub fn base_ignored_errs() -> Vec> { // object_op_ignored_errs pub fn object_op_ignored_errs() -> Vec> { - vec![ - Box::new(DiskError::DiskNotFound), - Box::new(DiskError::FaultyDisk), - Box::new(DiskError::FaultyRemoteDisk), + let mut base = base_ignored_errs(); + + + let ext:Vec> = vec![ + // Box::new(DiskError::DiskNotFound), + // Box::new(DiskError::FaultyDisk), + // Box::new(DiskError::FaultyRemoteDisk), Box::new(DiskError::DiskAccessDenied), Box::new(DiskError::UnformattedDisk), Box::new(DiskError::DiskOngoingReq), - ] + ]; + + base.extend(ext); + base } // 用于检查错误是否被忽略的函数 diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 730047855..67405d4d8 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -1,6 +1,11 @@ -use std::{fs::Metadata, os::unix::fs::MetadataExt}; +use std::{fs::Metadata, path::Path}; +use tokio::{fs, io}; + +#[cfg(target_os = "linux")] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { + use os::unix::fs::MetadataExt; + if f1.dev() != f2.dev() { return false; } @@ -22,3 +27,29 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { true } + +#[cfg(target_os = "windows")] +pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { + if f1.permissions() != f2.permissions() { + return false; + } + + + if f1.file_type() != f2.file_type() { + return false; + } + + if f1.len() != f2.len() { + return false; + } + true +} + +pub async fn access(path: impl AsRef) -> io::Result<()>{ + fs::metadata(path).await?; + Ok(()) +} + +pub async fn make_dir_all(path: impl AsRef) -> io::Result<()>{ + fs::create_dir_all(path.as_ref()).await +} \ No newline at end of file diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 0f799b41e..79bfc6ced 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -1,6 +1,6 @@ const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; -const SLASH_SEPARATOR: &str = "/"; +pub const SLASH_SEPARATOR: &str = "/"; pub fn has_suffix(s: &str, suffix: &str) -> bool { if cfg!(target_os = "windows") { From dbb6980e96893bd45eb1b66e6d6dcfd0ca4c35f1 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 24 Sep 2024 10:39:12 +0800 Subject: [PATCH 41/70] fix quorum err --- .gitignore | 1 + ecstore/src/disk/error.rs | 171 ++++++++++++++++++------------------- ecstore/src/disk/local.rs | 4 +- ecstore/src/disk/os.rs | 4 +- ecstore/src/disk/remote.rs | 6 +- ecstore/src/erasure.rs | 11 ++- ecstore/src/error.rs | 17 ++++ ecstore/src/peer.rs | 1 + ecstore/src/quorum.rs | 44 +++++----- ecstore/src/utils/fs.rs | 17 ++-- scripts/run.sh | 21 +++-- 11 files changed, 156 insertions(+), 141 deletions(-) diff --git a/.gitignore b/.gitignore index 0dc3785f1..3f19f03be 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .DS_Store .idea .vscode +/test \ No newline at end of file diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 7b69463ac..24864213e 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -100,9 +100,6 @@ pub enum DiskError { #[error("more data was sent than what was advertised")] MoreData, - - #[error("other io err {0}")] - IoError(io::Error), } impl DiskError { @@ -172,95 +169,89 @@ impl CheckErrorFn for DiskError { } } -pub fn clone_err(err: &Error) -> Error { - if let Some(e) = err.downcast_ref::() { - match e { - DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded), - DiskError::Unexpected => Error::new(DiskError::Unexpected), - DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat), - DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend), - DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk), - DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk), - DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk), - DiskError::DiskFull => Error::new(DiskError::DiskFull), - DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir), - DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound), - DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq), - DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot), - DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk), - DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk), - DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied), - DiskError::FileNotFound => Error::new(DiskError::FileNotFound), - DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound), - DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles), - DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong), - DiskError::VolumeExists => Error::new(DiskError::VolumeExists), - DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular), - DiskError::PathNotFound => Error::new(DiskError::PathNotFound), - DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound), - DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty), - DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied), - DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied), - DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt), - DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid), - DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), - DiskError::LessData => Error::new(DiskError::LessData), - DiskError::MoreData => Error::new(DiskError::MoreData), - DiskError::IoError(ioerr) => Error::msg(ioerr.to_string()), - } - } else { - Error::msg(err.to_string()) +pub fn clone_disk_err(e: &DiskError) -> Error { + match e { + DiskError::MaxVersionsExceeded => Error::new(DiskError::MaxVersionsExceeded), + DiskError::Unexpected => Error::new(DiskError::Unexpected), + DiskError::CorruptedFormat => Error::new(DiskError::CorruptedFormat), + DiskError::CorruptedBackend => Error::new(DiskError::CorruptedBackend), + DiskError::UnformattedDisk => Error::new(DiskError::UnformattedDisk), + DiskError::InconsistentDisk => Error::new(DiskError::InconsistentDisk), + DiskError::UnsupportedDisk => Error::new(DiskError::UnsupportedDisk), + DiskError::DiskFull => Error::new(DiskError::DiskFull), + DiskError::DiskNotDir => Error::new(DiskError::DiskNotDir), + DiskError::DiskNotFound => Error::new(DiskError::DiskNotFound), + DiskError::DiskOngoingReq => Error::new(DiskError::DiskOngoingReq), + DiskError::DriveIsRoot => Error::new(DiskError::DriveIsRoot), + DiskError::FaultyRemoteDisk => Error::new(DiskError::FaultyRemoteDisk), + DiskError::FaultyDisk => Error::new(DiskError::FaultyDisk), + DiskError::DiskAccessDenied => Error::new(DiskError::DiskAccessDenied), + DiskError::FileNotFound => Error::new(DiskError::FileNotFound), + DiskError::FileVersionNotFound => Error::new(DiskError::FileVersionNotFound), + DiskError::TooManyOpenFiles => Error::new(DiskError::TooManyOpenFiles), + DiskError::FileNameTooLong => Error::new(DiskError::FileNameTooLong), + DiskError::VolumeExists => Error::new(DiskError::VolumeExists), + DiskError::IsNotRegular => Error::new(DiskError::IsNotRegular), + DiskError::PathNotFound => Error::new(DiskError::PathNotFound), + DiskError::VolumeNotFound => Error::new(DiskError::VolumeNotFound), + DiskError::VolumeNotEmpty => Error::new(DiskError::VolumeNotEmpty), + DiskError::VolumeAccessDenied => Error::new(DiskError::VolumeAccessDenied), + DiskError::FileAccessDenied => Error::new(DiskError::FileAccessDenied), + DiskError::FileCorrupt => Error::new(DiskError::FileCorrupt), + DiskError::BitrotHashAlgoInvalid => Error::new(DiskError::BitrotHashAlgoInvalid), + DiskError::CrossDeviceLink => Error::new(DiskError::CrossDeviceLink), + DiskError::LessData => Error::new(DiskError::LessData), + DiskError::MoreData => Error::new(DiskError::MoreData), } } - -pub fn ioerr_to_diskerr(e:io::Error)->DiskError{ -match e.kind(){ - io::ErrorKind::NotFound => DiskError::FileNotFound, - io::ErrorKind::PermissionDenied => DiskError::FileAccessDenied, - // io::ErrorKind::ConnectionRefused => todo!(), - // io::ErrorKind::ConnectionReset => todo!(), - // io::ErrorKind::HostUnreachable => todo!(), - // io::ErrorKind::NetworkUnreachable => todo!(), - // io::ErrorKind::ConnectionAborted => todo!(), - // io::ErrorKind::NotConnected => todo!(), - // io::ErrorKind::AddrInUse => todo!(), - // io::ErrorKind::AddrNotAvailable => todo!(), - // io::ErrorKind::NetworkDown => todo!(), - // io::ErrorKind::BrokenPipe => todo!(), - // io::ErrorKind::AlreadyExists => todo!(), - // io::ErrorKind::WouldBlock => todo!(), - // io::ErrorKind::NotADirectory => DiskError::FileNotFound, - // io::ErrorKind::IsADirectory => DiskError::FileNotFound, - // io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty, - // io::ErrorKind::ReadOnlyFilesystem => todo!(), - // io::ErrorKind::FilesystemLoop => todo!(), - // io::ErrorKind::StaleNetworkFileHandle => todo!(), - // io::ErrorKind::InvalidInput => todo!(), - // io::ErrorKind::InvalidData => todo!(), - // io::ErrorKind::TimedOut => todo!(), - // io::ErrorKind::WriteZero => todo!(), - // io::ErrorKind::StorageFull => DiskError::DiskFull, - // io::ErrorKind::NotSeekable => todo!(), - // io::ErrorKind::FilesystemQuotaExceeded => todo!(), - // io::ErrorKind::FileTooLarge => todo!(), - // io::ErrorKind::ResourceBusy => todo!(), - // io::ErrorKind::ExecutableFileBusy => todo!(), - // io::ErrorKind::Deadlock => todo!(), - // io::ErrorKind::CrossesDevices => todo!(), - // io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles, - // io::ErrorKind::InvalidFilename => todo!(), - // io::ErrorKind::ArgumentListTooLong => todo!(), - // io::ErrorKind::Interrupted => todo!(), - // io::ErrorKind::Unsupported => todo!(), - // io::ErrorKind::UnexpectedEof => todo!(), - // io::ErrorKind::OutOfMemory => todo!(), - // io::ErrorKind::Other => todo!(), - // TODO: 把不支持的king用字符串处理 - _ => DiskError::IoError(e), -} +pub fn ioerr_to_err(e: io::Error) -> Error { + match e.kind() { + io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound), + io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied), + // io::ErrorKind::ConnectionRefused => todo!(), + // io::ErrorKind::ConnectionReset => todo!(), + // io::ErrorKind::HostUnreachable => todo!(), + // io::ErrorKind::NetworkUnreachable => todo!(), + // io::ErrorKind::ConnectionAborted => todo!(), + // io::ErrorKind::NotConnected => todo!(), + // io::ErrorKind::AddrInUse => todo!(), + // io::ErrorKind::AddrNotAvailable => todo!(), + // io::ErrorKind::NetworkDown => todo!(), + // io::ErrorKind::BrokenPipe => todo!(), + // io::ErrorKind::AlreadyExists => todo!(), + // io::ErrorKind::WouldBlock => todo!(), + // io::ErrorKind::NotADirectory => DiskError::FileNotFound, + // io::ErrorKind::IsADirectory => DiskError::FileNotFound, + // io::ErrorKind::DirectoryNotEmpty => DiskError::VolumeNotEmpty, + // io::ErrorKind::ReadOnlyFilesystem => todo!(), + // io::ErrorKind::FilesystemLoop => todo!(), + // io::ErrorKind::StaleNetworkFileHandle => todo!(), + // io::ErrorKind::InvalidInput => todo!(), + // io::ErrorKind::InvalidData => todo!(), + // io::ErrorKind::TimedOut => todo!(), + // io::ErrorKind::WriteZero => todo!(), + // io::ErrorKind::StorageFull => DiskError::DiskFull, + // io::ErrorKind::NotSeekable => todo!(), + // io::ErrorKind::FilesystemQuotaExceeded => todo!(), + // io::ErrorKind::FileTooLarge => todo!(), + // io::ErrorKind::ResourceBusy => todo!(), + // io::ErrorKind::ExecutableFileBusy => todo!(), + // io::ErrorKind::Deadlock => todo!(), + // io::ErrorKind::CrossesDevices => todo!(), + // io::ErrorKind::TooManyLinks =>DiskError::TooManyOpenFiles, + // io::ErrorKind::InvalidFilename => todo!(), + // io::ErrorKind::ArgumentListTooLong => todo!(), + // io::ErrorKind::Interrupted => todo!(), + // io::ErrorKind::Unsupported => todo!(), + // io::ErrorKind::UnexpectedEof => todo!(), + // io::ErrorKind::OutOfMemory => todo!(), + // io::ErrorKind::Other => todo!(), + // TODO: 把不支持的king用字符串处理 + _ => Error::new(e), + } } -pub fn os_is_not_exist(e:io::Error) -> bool{ - e.kind() == ErrorKind::NotFound -} \ No newline at end of file +pub fn os_is_not_exist(e: io::Error) -> bool { + e.kind() == ErrorKind::NotFound +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index fd0e5d388..9cd6a70fd 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,4 +1,4 @@ -use super::error::{ioerr_to_diskerr, os_is_not_exist}; +use super::error::os_is_not_exist; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, @@ -971,7 +971,7 @@ impl DiskAPI for LocalDisk { })?; for entry in entries { - if utils::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(&entry) { + if !utils::path::has_suffix(&entry, SLASH_SEPARATOR) || !Self::is_valid_volname(&entry) { continue; } diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 4fa9336b3..016e5e3c6 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -8,7 +8,7 @@ use crate::{ utils, }; -use super::error::{ioerr_to_diskerr, DiskError}; +use super::error::{ioerr_to_err, DiskError}; fn check_path_length(path_name: &str) -> Result<()> { // Apple OS X path length is limited to 1016 @@ -51,7 +51,7 @@ fn check_path_length(path_name: &str) -> Result<()> { pub async fn make_dir_all(path: impl AsRef) -> Result<()> { check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; - utils::fs::make_dir_all(path.as_ref()).map_err(ioerr_to_diskerr).await?; + utils::fs::make_dir_all(path.as_ref()).map_err(ioerr_to_err).await?; Ok(()) } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 0034235f8..82723e1f3 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -401,10 +401,12 @@ impl DiskAPI for RemoteDisk { Ok(volume_info) } - async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + async fn delete_paths(&self, _volume: &str, _paths: &[&str]) -> Result<()> { + // TODO: unimplemented!() } - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) { + async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) { + // TODO: unimplemented!() } diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index cd2d227a3..da245116d 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -94,11 +94,14 @@ impl Erasure { } } - warn!("Erasure encode errs {:?}", errs); + warn!("Erasure encode errs {:?}", &errs); - let err_idx = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum)?; - if errs[err_idx].is_some() { - let err = errs[err_idx].take().unwrap(); + let none_count = errs.iter().filter(|&x| x.is_none()).count(); + if none_count >= write_quorum { + continue; + } + + if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { return Err(err); } } diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index 24d2936b7..aa513fc58 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -1,5 +1,9 @@ +use std::io; + use tracing_error::{SpanTrace, SpanTraceStatus}; +use crate::disk::error::{clone_disk_err, DiskError}; + pub type StdError = Box; pub type Result = std::result::Result; @@ -80,3 +84,16 @@ impl std::fmt::Display for Error { Ok(()) } } + +impl Clone for Error { + fn clone(&self) -> Self { + if let Some(e) = self.downcast_ref::() { + clone_disk_err(e) + } else if let Some(e) = self.downcast_ref::() { + Error::new(io::Error::new(e.kind(), e.to_string())) + } else { + // TODO: 优化其他类型 + Error::msg(self.to_string()) + } + } +} diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 241280c01..89eea2c00 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -262,6 +262,7 @@ impl PeerS3Client for LocalPeerS3Client { } } + warn!("list_bucket ress {:?}", &ress); warn!("list_bucket errs {:?}", &errs); let mut uniq_map: HashMap<&String, &VolumeInfo> = HashMap::new(); diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 13544632f..72e331096 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -1,4 +1,7 @@ -use crate::{disk::error::DiskError, error::Error}; +use crate::{ + disk::error::DiskError, + error::{Error, Result}, +}; use std::{collections::HashMap, fmt::Debug}; // pub type CheckErrorFn = fn(e: &Error) -> bool; @@ -27,8 +30,7 @@ pub fn base_ignored_errs() -> Vec> { pub fn object_op_ignored_errs() -> Vec> { let mut base = base_ignored_errs(); - - let ext:Vec> = vec![ + let ext: Vec> = vec![ // Box::new(DiskError::DiskNotFound), // Box::new(DiskError::FaultyDisk), // Box::new(DiskError::FaultyRemoteDisk), @@ -47,7 +49,7 @@ fn is_err_ignored(err: &Error, ignored_errs: &Vec>) -> boo } // 减少错误数量并返回出现次数最多的错误 -fn reduce_errs(errs: &Vec>, ignored_errs: &Vec>) -> (usize, Option) { +fn reduce_errs(errs: &Vec>, ignored_errs: &Vec>) -> (usize, Option) { let mut error_counts: HashMap = HashMap::new(); let mut error_map: HashMap = HashMap::new(); // 存err位置 let nil = "nil".to_string(); @@ -80,8 +82,10 @@ fn reduce_errs(errs: &Vec>, ignored_errs: &Vec>, ignored_errs: &Vec>, ignored_errs: &Vec>, quorum: usize) -> Option { +fn reduce_quorum_errs( + errs: &Vec>, + ignored_errs: &Vec>, + quorum: usize, + quorum_err: QuorumError, +) -> Option { let (max_count, max_err) = reduce_errs(errs, ignored_errs); if max_count >= quorum { max_err } else { - None + Some(Error::new(quorum_err)) } } @@ -106,13 +115,8 @@ pub fn reduce_read_quorum_errs( errs: &mut Vec>, ignored_errs: &Vec>, read_quorum: usize, -) -> Result { - let idx = reduce_quorum_errs(errs, ignored_errs, read_quorum); - if idx.is_none() { - return Err(Error::new(QuorumError::Read)); - } - - Ok(idx.unwrap()) +) -> Option { + reduce_quorum_errs(errs, ignored_errs, read_quorum, QuorumError::Read) } // 根据写quorum验证错误数量 @@ -121,12 +125,6 @@ pub fn reduce_write_quorum_errs( errs: &Vec>, ignored_errs: &Vec>, write_quorum: usize, -) -> Result { - let idx = reduce_quorum_errs(errs, ignored_errs, write_quorum); - - if idx.is_none() { - return Err(Error::new(QuorumError::Write)); - } - - Ok(idx.unwrap()) +) -> Option { + reduce_quorum_errs(errs, ignored_errs, write_quorum, QuorumError::Write) } diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 67405d4d8..1e1cae40b 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -1,10 +1,10 @@ -use std::{fs::Metadata, path::Path}; +use std::{fs::Metadata, path::Path}; use tokio::{fs, io}; -#[cfg(target_os = "linux")] +#[cfg(not(target_os = "windows"))] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { - use os::unix::fs::MetadataExt; + use std::os::unix::fs::MetadataExt; if f1.dev() != f2.dev() { return false; @@ -31,12 +31,11 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { #[cfg(target_os = "windows")] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { if f1.permissions() != f2.permissions() { - return false; + return false; } - if f1.file_type() != f2.file_type() { - return false; + return false; } if f1.len() != f2.len() { @@ -45,11 +44,11 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { true } -pub async fn access(path: impl AsRef) -> io::Result<()>{ +pub async fn access(path: impl AsRef) -> io::Result<()> { fs::metadata(path).await?; Ok(()) } -pub async fn make_dir_all(path: impl AsRef) -> io::Result<()>{ +pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { fs::create_dir_all(path.as_ref()).await -} \ No newline at end of file +} diff --git a/scripts/run.sh b/scripts/run.sh index b48b9abc7..1a7b2073a 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,24 +1,27 @@ #!/bin/bash +current_dir=$(pwd) + mkdir -p ./target/volume/test mkdir -p ./target/volume/test{0..4} -# DATA_DIR="./target/volume/test" -DATA_DIR="./target/volume/test{0...4}" - -if [ -n "$1" ]; then - DATA_DIR="$1" -fi if [ -z "$RUST_LOG" ]; then export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug" fi -# cargo run "$DATA_DIR" +DATA_DIR_ARG="./target/volume/test{0...4}" + +if [ -n "$1" ]; then + DATA_DIR_ARG="$1" +fi + + +# cargo run "$DATA_DIR_ARG" # -- --access-key AKEXAMPLERUSTFS \ # --secret-key SKEXAMPLERUSTFS \ # --address 0.0.0.0:9010 \ # --domain-name 127.0.0.1:9010 \ - # "$DATA_DIR" + # "$DATA_DIR_ARG" -cargo run "$DATA_DIR" \ No newline at end of file +cargo run "$DATA_DIR_ARG" \ No newline at end of file From 8d1fb577e224949e235efbefc0cdbb3be9413029 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:07:57 +0800 Subject: [PATCH 42/70] add e2e test ci --- .github/workflows/e2e.yml | 76 ++++++++++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 8 ++-- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 000000000..f92ce3c2b --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,76 @@ +name: e2e + +on: + push: + pull_request: + branches: [ "main" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - stable + - beta + - nightly + + steps: + - name: cache protoc bin + id: cache-protoc-action + uses: actions/cache@v3 + env: + cache-name: cache-protoc-action-bin + with: + path: /usr/local/bin/protoc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install protoc + if: steps.cache-protoc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip + unzip protoc-27.0-linux-x86_64.zip -d protoc3 + mv protoc3/bin/* /usr/local/bin/ + chmod +x /usr/local/bin/protoc + rm -rf protoc-27.0-linux-x86_64.zip protoc3 + + - name: print protoc version + run: protoc --version + + - name: cache flatc bin + id: cache-flatc-action + uses: actions/cache@v3 + env: + cache-name: cache-flatc-action-bin + with: + path: /usr/local/bin/flatc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install flatc + if: steps.cache-flatc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip + unzip Linux.flatc.binary.g++-13.zip + mv flatc /usr/local/bin/ + chmod +x /usr/local/bin/flatc + rm -rf Linux.flatc.binary.g++-13.zip + + - name: checkout + uses: actions/checkout@v2 + + - name: run fs + working-directory: packages/api + run: | + cargo run & + env: + PORT: 9000 + + - name: e2e test + uses: actions-rs/cargo@v1 + with: + command: test + args: -p e2e_test --lib diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6dd5d7024..a46b7217d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -68,11 +68,13 @@ jobs: override: true components: rustfmt, clippy - - uses: actions-rs/cargo@v1 + - name: cargo build + uses: actions-rs/cargo@v1 with: command: build - - uses: actions-rs/cargo@v1 + - name: cargo test + uses: actions-rs/cargo@v1 with: command: test - args: --all + args: --all --exclude e2e_test From 33c3ceb3614ff5890a092c658c9b4c8482dcffc1 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:12:17 +0800 Subject: [PATCH 43/70] add e2e test ci --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f92ce3c2b..20cc1c62e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v2 - name: run fs - working-directory: packages/api + working-directory: . run: | cargo run & env: From 1f40e97dd83cdc05794c4d7eaa8bdcfab0160197 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:33:41 +0800 Subject: [PATCH 44/70] add e2e test ci --- .github/workflows/e2e.yml | 2 +- Makefile | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 20cc1c62e..6595cc8d3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,7 +65,7 @@ jobs: - name: run fs working-directory: . run: | - cargo run & + make e2e-server & env: PORT: 9000 diff --git a/Makefile b/Makefile index 6f05e92b5..5c69f4a61 100644 --- a/Makefile +++ b/Makefile @@ -21,3 +21,7 @@ start: .PHONY: stop stop: $(DOCKER_CLI) stop $(CONTAINER_NAME) + +.PHONY: e2e +e2e-server: + sh $(shell pwd)/scripts/run.sh From d0f52a2b782b878eebfd9f02d3455ecc68d235d8 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:50:12 +0800 Subject: [PATCH 45/70] add cargo cache --- .github/workflows/e2e.yml | 8 ++++++++ .github/workflows/rust.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6595cc8d3..3c5836f3e 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -59,6 +59,14 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + - name: checkout uses: actions/checkout@v2 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a46b7217d..8083e47df 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,6 +59,14 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From ed85e4e24cbbb68a1f760e2a695de5b3e48e2e3b Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:00:59 +0800 Subject: [PATCH 46/70] add probe --- .github/workflows/e2e.yml | 3 ++- Makefile | 8 ++++++-- scripts/probe.sh | 12 ++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 scripts/probe.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3c5836f3e..00b784ba8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -73,7 +73,8 @@ jobs: - name: run fs working-directory: . run: | - make e2e-server & + make e2e-server > /dev/null & + make probe-e2e env: PORT: 9000 diff --git a/Makefile b/Makefile index 5c69f4a61..131a4f1c7 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ start: stop: $(DOCKER_CLI) stop $(CONTAINER_NAME) -.PHONY: e2e +.PHONY: e2e-server e2e-server: - sh $(shell pwd)/scripts/run.sh + sh $(shell pwd)/scripts/run.sh + +.PHONY: probe-e2e +probe-e2e: + sh $(shell pwd)/scripts/probe.sh diff --git a/scripts/probe.sh b/scripts/probe.sh new file mode 100644 index 000000000..c85de2f8e --- /dev/null +++ b/scripts/probe.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +IP=127.0.0.1 +PORT=9000 + +while true; do + nc -zv ${IP} ${PORT} + if [[ "$?" == "0" ]]; then + exit 0 + fi + sleep 2 +done From 180905ecba43d2e8b29240fd00052b4a4054e073 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:09:31 +0800 Subject: [PATCH 47/70] POSIX Shell --- scripts/probe.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/probe.sh b/scripts/probe.sh index c85de2f8e..150ca6364 100644 --- a/scripts/probe.sh +++ b/scripts/probe.sh @@ -1,11 +1,14 @@ -#!/bin/bash +#!/bin/sh + +# Please use POSIX Shell +# https://www.grymoire.com/Unix/Sh.html IP=127.0.0.1 PORT=9000 while true; do nc -zv ${IP} ${PORT} - if [[ "$?" == "0" ]]; then + if [ "$?" == "0" ]; then exit 0 fi sleep 2 From 2252ac959209a2b1ada77ead9e263535676e77d7 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:12:16 +0800 Subject: [PATCH 48/70] POSIX Shell --- scripts/probe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/probe.sh b/scripts/probe.sh index 150ca6364..2cda95912 100644 --- a/scripts/probe.sh +++ b/scripts/probe.sh @@ -8,7 +8,7 @@ PORT=9000 while true; do nc -zv ${IP} ${PORT} - if [ "$?" == "0" ]; then + if [ "$?" -eq "0" ]; then exit 0 fi sleep 2 From b54458c4332a3d470e20b6cbed1527a52644c1f6 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:20:04 +0800 Subject: [PATCH 49/70] fix cargo cache --- .github/workflows/e2e.yml | 16 ++++++++-------- .github/workflows/rust.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 00b784ba8..3a900a7ad 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -59,14 +59,6 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip - - name: cache cargo - uses: actions/cache@v3 - env: - cache-name: cache-cargo - with: - path: ~/.cargo - key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 - - name: checkout uses: actions/checkout@v2 @@ -83,3 +75,11 @@ jobs: with: command: test args: -p e2e_test --lib + + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8083e47df..00eb6f8b6 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,14 +59,6 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip - - name: cache cargo - uses: actions/cache@v3 - env: - cache-name: cache-cargo - with: - path: ~/.cargo - key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 - - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -86,3 +78,11 @@ jobs: with: command: test args: --all --exclude e2e_test + + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 From 0fa13379e4f230b17bc1dd44bb07366b5a483486 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 16:17:31 +0800 Subject: [PATCH 50/70] fix --- .github/workflows/e2e.yml | 14 +++++++------- .github/workflows/rust.yml | 23 ++++++++--------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3a900a7ad..0b15d245f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,7 +16,6 @@ jobs: matrix: rust: - stable - - beta - nightly steps: @@ -59,6 +58,12 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.rust }} + # components: rustfmt, clippy + - name: checkout uses: actions/checkout@v2 @@ -67,14 +72,9 @@ jobs: run: | make e2e-server > /dev/null & make probe-e2e - env: - PORT: 9000 - name: e2e test - uses: actions-rs/cargo@v1 - with: - command: test - args: -p e2e_test --lib + run: cargo test -p e2e_test --lib - name: cache cargo uses: actions/cache@v3 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 00eb6f8b6..7f86763d7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,7 +16,6 @@ jobs: matrix: rust: - stable - - beta - nightly steps: @@ -59,25 +58,19 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.rust }} + # components: rustfmt, clippy + - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true - components: rustfmt, clippy - - name: cargo build - uses: actions-rs/cargo@v1 - with: - command: build + run: cargo build - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all --exclude e2e_test + run: cargo test --all --exclude e2e_test - name: cache cargo uses: actions/cache@v3 From 84b1ebd2c8dc68729c712aa92a31d9653a21aa48 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 24 Sep 2024 16:21:02 +0800 Subject: [PATCH 51/70] fix: #23 need test reduce_read_quorum_errs --- ecstore/src/disk/local.rs | 20 ++++++++++---------- ecstore/src/disk/remote.rs | 10 +++++----- ecstore/src/quorum.rs | 9 +++------ ecstore/src/sets.rs | 4 ++-- ecstore/src/store.rs | 4 ++-- ecstore/src/store_api.rs | 14 ++++++++++++-- ecstore/src/utils/fs.rs | 4 ++-- 7 files changed, 36 insertions(+), 29 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9cd6a70fd..7bf5cb5e6 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -113,9 +113,9 @@ impl LocalDisk { Ok(disk) } - fn check_path_length(path_name: &str) -> Result<()> { - unimplemented!() - } + // fn check_path_length(_path_name: &str) -> Result<()> { + // unimplemented!() + // } fn is_valid_volname(volname: &str) -> bool { if volname.len() < 3 { @@ -1000,10 +1000,10 @@ impl DiskAPI for LocalDisk { created: modtime, }) } - async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + async fn delete_paths(&self, _volume: &str, _paths: &[&str]) -> Result<()> { unimplemented!() } - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) { + async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) { unimplemented!() } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { @@ -1062,11 +1062,11 @@ impl DiskAPI for LocalDisk { } async fn delete_version( &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, + _volume: &str, + _path: &str, + _fi: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, ) -> Result { unimplemented!() } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 82723e1f3..b3360f13d 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -482,11 +482,11 @@ impl DiskAPI for RemoteDisk { } async fn delete_version( &self, - volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, + _volume: &str, + _path: &str, + _fi: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, ) -> Result { unimplemented!() } diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs index 72e331096..a3850077b 100644 --- a/ecstore/src/quorum.rs +++ b/ecstore/src/quorum.rs @@ -1,7 +1,4 @@ -use crate::{ - disk::error::DiskError, - error::{Error, Result}, -}; +use crate::{disk::error::DiskError, error::Error}; use std::{collections::HashMap, fmt::Debug}; // pub type CheckErrorFn = fn(e: &Error) -> bool; @@ -11,7 +8,7 @@ pub trait CheckErrorFn: Debug + Send + Sync + 'static { } #[derive(Debug, thiserror::Error)] -enum QuorumError { +pub enum QuorumError { #[error("Read quorum not met")] Read, #[error("disk not found")] @@ -112,7 +109,7 @@ fn reduce_quorum_errs( // 根据读quorum验证错误数量 // 返回最大错误数量的下标，或QuorumError pub fn reduce_read_quorum_errs( - errs: &mut Vec>, + errs: &Vec>, ignored_errs: &Vec>, read_quorum: usize, ) -> Option { diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 319a04c3e..7853c5df4 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -98,7 +98,7 @@ impl Sets { let set_disks = SetDisks { disks: RwLock::new(set_drive), set_drive_count, - parity_count: partiy_count, + default_parity_count: partiy_count, set_index: i, pool_index: pool_idx, set_endpoints, @@ -343,7 +343,7 @@ impl StorageAPI for Sets { .get_object_reader(bucket, object, range, h, opts) .await } - async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> { + async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result { self.get_disks_by_key(object).put_object(bucket, object, data, opts).await } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index c99c7ba37..caa21174d 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -314,7 +314,7 @@ impl ECStore { if entry.is_object() { let fi = entry.to_fileinfo(&opts.bucket)?; if fi.is_some() { - ress.push(fi.unwrap().into_object_info(&opts.bucket, &entry.name, false)); + ress.push(fi.unwrap().to_object_info(&opts.bucket, &entry.name, false)); } continue; } @@ -738,7 +738,7 @@ impl StorageAPI for ECStore { unimplemented!() } - async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()> { + async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result { // checkPutObjectArgs let object = utils::path::encode_dir_object(object); diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 49bc7dd2f..127107059 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -29,6 +29,8 @@ pub struct FileInfo { pub is_latest: bool, #[serde(skip_serializing_if = "Option::is_none", default)] pub tags: Option>, + pub metadata: Option>, + pub num_versions: usize, } // impl Default for FileInfo { @@ -96,6 +98,14 @@ impl FileInfo { false } + pub fn get_etag(&self) -> Option { + if let Some(meta) = &self.metadata { + meta.get("etag").cloned() + } else { + None + } + } + pub fn write_quorum(&self, quorum: usize) -> usize { if self.deleted { return quorum; @@ -141,7 +151,7 @@ impl FileInfo { self.parts.sort_by(|a, b| a.number.cmp(&b.number)); } - pub fn into_object_info(&self, bucket: &str, object: &str, _versioned: bool) -> ObjectInfo { + pub fn to_object_info(&self, bucket: &str, object: &str, _versioned: bool) -> ObjectInfo { ObjectInfo { bucket: bucket.to_string(), name: object.to_string(), @@ -533,7 +543,7 @@ pub trait StorageAPI { h: HeaderMap, opts: &ObjectOptions, ) -> Result; - async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result<()>; + async fn put_object(&self, bucket: &str, object: &str, data: PutObjReader, opts: &ObjectOptions) -> Result; async fn put_object_part( &self, bucket: &str, diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 1e1cae40b..41c7e4867 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -2,7 +2,7 @@ use std::{fs::Metadata, path::Path}; use tokio::{fs, io}; -#[cfg(not(target_os = "windows"))] +#[cfg(not(windows))] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { use std::os::unix::fs::MetadataExt; @@ -28,7 +28,7 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { true } -#[cfg(target_os = "windows")] +#[cfg(windows)] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { if f1.permissions() != f2.permissions() { return false; From 6bd0da059f4e4293a1beb14aebd2963ea2b275e6 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 24 Sep 2024 17:58:36 +0800 Subject: [PATCH 52/70] fix fileinfo mod_time bug From 6329eee92acd94ce951faadbaf156b6bb0b512c2 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 24 Sep 2024 22:51:21 +0800 Subject: [PATCH 53/70] fix fileinfo serialize --- ecstore/src/erasure.rs | 2 +- ecstore/src/store_api.rs | 2 +- rustfs/src/storage/ecfs.rs | 1 + scripts/run.sh | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index da245116d..76da879c5 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -94,7 +94,7 @@ impl Erasure { } } - warn!("Erasure encode errs {:?}", &errs); + debug!("Erasure encode errs {:?}", &errs); let none_count = errs.iter().filter(|&x| x.is_none()).count(); if none_count >= write_quorum { diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 127107059..bcf9e0a8e 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -27,7 +27,7 @@ pub struct FileInfo { pub fresh: bool, // indicates this is a first time call to write FileInfo. pub parts: Vec, pub is_latest: bool, - #[serde(skip_serializing_if = "Option::is_none", default)] + // #[serde(skip_serializing_if = "Option::is_none", default)] pub tags: Option>, pub metadata: Option>, pub num_versions: usize, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index ae44da1ee..090afb316 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -629,6 +629,7 @@ impl S3 for FS { .. } = req.input; + // error!("complete_multipart_upload {:?}", multipart_upload); // mc cp step 5 let Some(multipart_upload) = multipart_upload else { return Err(s3_error!(InvalidPart)) }; diff --git a/scripts/run.sh b/scripts/run.sh index 1a7b2073a..248d286b2 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -7,7 +7,7 @@ mkdir -p ./target/volume/test{0..4} if [ -z "$RUST_LOG" ]; then - export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug" + export RUST_LOG="rustfs=debug,ecstore=info,s3s=debug" fi DATA_DIR_ARG="./target/volume/test{0...4}" From ad4a32762dec27690c22c6650dcf1ecf8154cb35 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 11:49:12 +0800 Subject: [PATCH 54/70] add rename_part --- ecstore/src/disk/error.rs | 138 ++++++++++++++++++++++++++++++++++++- ecstore/src/disk/local.rs | 76 +++++++++++++++++--- ecstore/src/disk/mod.rs | 5 +- ecstore/src/disk/os.rs | 117 ++++++++++++++++++++++++++++++- ecstore/src/disk/remote.rs | 11 ++- ecstore/src/utils/fs.rs | 21 ++++++ ecstore/src/utils/path.rs | 6 ++ 7 files changed, 358 insertions(+), 16 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 24864213e..145b4d195 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -252,6 +252,142 @@ pub fn ioerr_to_err(e: io::Error) -> Error { } } -pub fn os_is_not_exist(e: io::Error) -> bool { +pub fn is_sys_err_no_space(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 28; + } + false +} + +pub fn is_sys_err_invalid_arg(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 22; + } + false +} + +pub fn is_sys_err_io(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 5; + } + false +} + +pub fn is_sys_err_is_dir(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 21; + } + false +} + +pub fn is_sys_err_not_dir(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 20; + } + false +} + +pub fn is_sys_err_too_long(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 63; + } + false +} + +pub fn is_sys_err_too_many_symlinks(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 62; + } + false +} + +pub fn is_sys_err_not_empty(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + if no == 66 { + return true; + } + + if cfg!(target_os = "solaris") && no == 17 { + return true; + } + + if cfg!(target_os = "windows") && no == 145 { + return true; + } + } + false +} + +pub fn is_sys_err_path_not_found(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + if cfg!(target_os = "windows") { + if no == 3 { + return true; + } + } else { + if no == 2 { + return true; + } + } + } + false +} + +pub fn is_sys_err_handle_invalid(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + if cfg!(target_os = "windows") { + if no == 6 { + return true; + } + } else { + return false; + } + } + false +} + +pub fn is_sys_err_cross_device(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 18; + } + false +} + +pub fn is_sys_err_too_many_files(e: &io::Error) -> bool { + if let Some(no) = e.raw_os_error() { + return no == 23 || no == 24; + } + false +} + +pub fn os_is_not_exist(e: &io::Error) -> bool { e.kind() == ErrorKind::NotFound } + +pub fn os_is_permission(e: &io::Error) -> bool { + if e.kind() == ErrorKind::PermissionDenied { + return true; + } + if let Some(no) = e.raw_os_error() { + if no == 30 { + return true; + } + } + + false +} + +pub fn os_is_exist(e: &io::Error) -> bool { + e.kind() == ErrorKind::AlreadyExists +} + +// map_err_not_exists +pub fn map_err_not_exists(e: io::Error) -> Error { + if os_is_not_exist(&e) { + return Error::new(DiskError::VolumeNotEmpty); + } else if is_sys_err_io(&e) { + return Error::new(DiskError::FaultyDisk); + } + + Error::new(e) +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9a85464dc..768e3ac75 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,11 +1,14 @@ -use super::error::os_is_not_exist; +use super::error::{is_sys_err_io, is_sys_err_not_empty, os_is_not_exist}; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; +use crate::disk::error::{is_sys_err_not_dir, map_err_not_exists}; +use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; -use crate::utils::path::SLASH_SEPARATOR; +use crate::utils::fs::lstat; +use crate::utils::path::{has_suffix, SLASH_SEPARATOR}; use crate::{ error::{Error, Result}, file_meta::FileMeta, @@ -22,6 +25,7 @@ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::ErrorKind; use tokio::sync::Mutex; +use tower::layer::util; use tracing::{debug, warn}; use uuid::Uuid; @@ -620,15 +624,71 @@ impl DiskAPI for LocalDisk { Ok(()) } - - async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - let src_volume_path = self.get_bucket_path(src_volume)?; + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { + let src_volume_dir = self.get_bucket_path(src_volume)?; + let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { - check_volume_exists(&src_volume_path).await?; + utils::fs::access(&src_volume_dir).await.map_err(map_err_not_exists)? } if !skip_access_checks(dst_volume) { - let vol_path = self.get_bucket_path(dst_volume)?; - check_volume_exists(&vol_path).await?; + utils::fs::access(&dst_volume_dir).await.map_err(map_err_not_exists)? + } + + let src_is_dir = has_suffix(&src_path, SLASH_SEPARATOR); + let dst_is_dir = has_suffix(&dst_path, SLASH_SEPARATOR); + + if !(src_is_dir && dst_is_dir || !src_is_dir && !dst_is_dir) { + return Err(Error::from(DiskError::FileAccessDenied)); + } + + let src_file_path = src_volume_dir.join(Path::new(src_path)); + let dst_file_path = dst_volume_dir.join(Path::new(dst_path)); + + check_path_length(&src_file_path.to_string_lossy().to_string())?; + check_path_length(&dst_file_path.to_string_lossy().to_string())?; + + if src_is_dir { + let meta_op = match lstat(&src_file_path).await { + Ok(meta) => Some(meta), + Err(e) => { + if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } + + if !os_is_not_exist(&e) { + return Err(Error::new(e)); + } + None + } + }; + + if let Some(meta) = meta_op { + if !meta.is_dir() { + return Err(Error::new(DiskError::FileAccessDenied)); + } + } + + if let Err(e) = utils::fs::remove(&dst_file_path).await { + if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } else if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } + + return Err(Error::new(e)); + } + } + + unimplemented!() + } + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + let src_volume_path = self.get_bucket_path(src_volume)?; + let dst_volume_path = self.get_bucket_path(dst_volume)?; + if !skip_access_checks(src_volume) { + utils::fs::access(&src_volume_path).await.map_err(map_err_not_exists)?; + } + if !skip_access_checks(dst_volume) { + utils::fs::access(&dst_volume_path).await.map_err(map_err_not_exists)?; } let srcp = self.get_object_path(src_volume, src_path)?; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 3ec7f52df..cd8e9af7f 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -2,8 +2,8 @@ pub mod endpoint; pub mod error; pub mod format; mod local; -mod remote; pub mod os; +mod remote; pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; @@ -117,7 +117,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result; // ReadFileStream async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()>; - // RenamePart + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()>; // CheckParts async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; // VerifyFile @@ -127,7 +127,6 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; - } pub struct UpdateMetadataOpts { diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 016e5e3c6..e9a100bb6 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -1,16 +1,20 @@ -use std::path::Path; +use std::{ + io, + path::{Component, Path}, +}; use futures::TryFutureExt; use tokio::fs; use crate::{ + disk::error::{is_sys_err_not_dir, is_sys_err_path_not_found, os_is_not_exist}, error::{Error, Result}, utils, }; -use super::error::{ioerr_to_err, DiskError}; +use super::error::{ioerr_to_err, os_is_exist, DiskError}; -fn check_path_length(path_name: &str) -> Result<()> { +pub fn check_path_length(path_name: &str) -> Result<()> { // Apple OS X path length is limited to 1016 if cfg!(target_os = "macos") && path_name.len() > 1016 { return Err(Error::new(DiskError::FileNameTooLong)); @@ -91,3 +95,110 @@ pub async fn read_dir(path: impl AsRef, count: usize) -> Result, + dst_file_path: impl AsRef, + base_dir: impl AsRef, +) -> Result<()> { + reliable_rename(src_file_path, dst_file_path, base_dir).await.map_err(|e| { + if is_sys_err_not_dir(&e) || !os_is_not_exist(&e) { + Error::new(DiskError::FileAccessDenied) + } else if is_sys_err_path_not_found(&e) { + Error::new(DiskError::FileAccessDenied) + } else if os_is_not_exist(&e) { + Error::new(DiskError::FileNotFound) + } else if os_is_exist(&e) { + Error::new(DiskError::IsNotRegular) + } else { + Error::new(e) + } + })?; + + Ok(()) +} + +pub async fn reliable_rename( + src_file_path: impl AsRef, + dst_file_path: impl AsRef, + base_dir: impl AsRef, +) -> io::Result<()> { + if let Some(parent) = dst_file_path.as_ref().parent() { + reliable_mkdir_all(parent, base_dir.as_ref()).await?; + } + + let mut i = 0; + loop { + if let Err(e) = utils::fs::rename(src_file_path.as_ref(), dst_file_path.as_ref()).await { + if os_is_not_exist(&e) && i == 0 { + i += 1; + continue; + } + + return Err(e); + } + + break; + } + + Ok(()) +} + +pub async fn reliable_mkdir_all(path: impl AsRef, base_dir: impl AsRef) -> io::Result<()> { + let mut i = 0; + + let mut base_dir = base_dir.as_ref().clone(); + loop { + if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await { + if os_is_not_exist(&e) && i == 0 { + i += 1; + + if let Some(base_parent) = base_dir.parent() { + if let Some(c) = base_parent.components().next() { + if c != Component::RootDir { + base_dir = base_parent + } + } + } + continue; + } + + return Err(e); + } + + break; + } + + Ok(()) +} + +pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef) -> io::Result<()> { + if !base_dir.as_ref().to_string_lossy().is_empty() { + if base_dir.as_ref().starts_with(dir_path.as_ref()) { + return Ok(()); + } + } + + if let Some(parent) = dir_path.as_ref().parent() { + Box::pin(os_mkdir_all(parent, base_dir)).await?; + } + + if let Err(e) = utils::fs::mkdir(dir_path.as_ref()).await { + if os_is_exist(&e) { + return Ok(()); + } + + return Err(e); + } + + Ok(()) +} + +#[cfg(test)] +mod test { + + use super::*; + + #[tokio::test] + async fn test_make_dir() {} +} diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index b3360f13d..51763c4ae 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -201,7 +201,16 @@ impl DiskAPI for RemoteDisk { Ok(()) } - + async fn rename_part( + &self, + _src_volume: &str, + _src_path: &str, + _dst_volume: &str, + _dst_path: &str, + _meta: Vec, + ) -> Result<()> { + unimplemented!() + } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { info!("rename_file"); let mut client = self.get_client_v2().await?; diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 41c7e4867..0ff3b320a 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -49,6 +49,27 @@ pub async fn access(path: impl AsRef) -> io::Result<()> { Ok(()) } +pub async fn lstat(path: impl AsRef) -> io::Result { + fs::metadata(path).await +} + pub async fn make_dir_all(path: impl AsRef) -> io::Result<()> { fs::create_dir_all(path.as_ref()).await } + +pub async fn remove(path: impl AsRef) -> io::Result<()> { + let meta = fs::metadata(path.as_ref()).await?; + if meta.is_dir() { + fs::remove_dir(path.as_ref()).await + } else { + fs::remove_file(path.as_ref()).await + } +} + +pub async fn mkdir(path: impl AsRef) -> io::Result<()> { + fs::create_dir(path.as_ref()).await +} + +pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result<()> { + fs::rename(from, to).await +} diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 79bfc6ced..49a7dc9e4 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -1,3 +1,5 @@ +use std::path::Path; + const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; pub const SLASH_SEPARATOR: &str = "/"; @@ -37,3 +39,7 @@ pub fn retain_slash(s: &str) -> String { format!("{}{}", s, SLASH_SEPARATOR) } } + +pub fn join(p1: &str, p2: &str) -> String { + Path::new(p1).join(Path::new(p2)).to_string_lossy().to_string() +} From 4f1e31999d1f1a39c559578725f7cb43e23e6288 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 15:27:23 +0800 Subject: [PATCH 55/70] update stock api --- ecstore/src/disk/error.rs | 2 +- ecstore/src/disk/local.rs | 33 ++++++++++++++++++++++++++++----- ecstore/src/disk/os.rs | 16 ++++++++++++---- ecstore/src/error.rs | 8 ++++++++ ecstore/src/utils/path.rs | 6 ------ 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 145b4d195..535273c2f 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -205,7 +205,7 @@ pub fn clone_disk_err(e: &DiskError) -> Error { } } -pub fn ioerr_to_err(e: io::Error) -> Error { +pub fn os_err_to_file_err(e: io::Error) -> Error { match e.kind() { io::ErrorKind::NotFound => Error::new(DiskError::FileNotFound), io::ErrorKind::PermissionDenied => Error::new(DiskError::FileAccessDenied), diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 768e3ac75..c1a9900d4 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -4,7 +4,7 @@ use super::{ os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; -use crate::disk::error::{is_sys_err_not_dir, map_err_not_exists}; +use crate::disk::error::{is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err}; use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::utils::fs::lstat; @@ -25,7 +25,6 @@ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::ErrorKind; use tokio::sync::Mutex; -use tower::layer::util; use tracing::{debug, warn}; use uuid::Uuid; @@ -679,7 +678,31 @@ impl DiskAPI for LocalDisk { } } - unimplemented!() + if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await { + if let Some(e) = err.to_io_err() { + if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } + + return Err(os_err_to_file_err(e)); + } + + return Err(err); + } + + if let Err(err) = self.write_all(&dst_volume, format!("{}.meta", dst_path).as_str(), meta).await { + if let Some(e) = err.to_io_err() { + return Err(os_err_to_file_err(e)); + } + + return Err(err); + } + + if let Some(parent) = src_file_path.parent() { + self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await?; + } + + Ok(()) } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { let src_volume_path = self.get_bucket_path(src_volume)?; @@ -1009,8 +1032,8 @@ impl DiskAPI for LocalDisk { let p = self.get_bucket_path(volume)?; - if let Err(err) = utils::fs::access(&p).await { - if os_is_not_exist(err) { + if let Err(e) = utils::fs::access(&p).await { + if os_is_not_exist(&e) { os::make_dir_all(&p).await?; } } diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index e9a100bb6..3bf4d10e4 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -12,7 +12,7 @@ use crate::{ utils, }; -use super::error::{ioerr_to_err, os_is_exist, DiskError}; +use super::error::{os_err_to_file_err, os_is_exist, DiskError}; pub fn check_path_length(path_name: &str) -> Result<()> { // Apple OS X path length is limited to 1016 @@ -55,7 +55,7 @@ pub fn check_path_length(path_name: &str) -> Result<()> { pub async fn make_dir_all(path: impl AsRef) -> Result<()> { check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; - utils::fs::make_dir_all(path.as_ref()).map_err(ioerr_to_err).await?; + utils::fs::make_dir_all(path.as_ref()).map_err(os_err_to_file_err).await?; Ok(()) } @@ -147,7 +147,7 @@ pub async fn reliable_rename( pub async fn reliable_mkdir_all(path: impl AsRef, base_dir: impl AsRef) -> io::Result<()> { let mut i = 0; - let mut base_dir = base_dir.as_ref().clone(); + let mut base_dir = base_dir.as_ref(); loop { if let Err(e) = os_mkdir_all(path.as_ref(), base_dir).await { if os_is_not_exist(&e) && i == 0 { @@ -180,7 +180,15 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef } if let Some(parent) = dir_path.as_ref().parent() { - Box::pin(os_mkdir_all(parent, base_dir)).await?; + // 不支持递归，直接create_dir_all了 + if let Err(e) = fs::create_dir_all(&parent).await { + if os_is_exist(&e) { + return Ok(()); + } + + return Err(e); + } + // Box::pin(os_mkdir_all(&parent, &base_dir)).await?; } if let Err(e) = utils::fs::mkdir(dir_path.as_ref()).await { diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index aa513fc58..1fc8dfabf 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -65,6 +65,14 @@ impl Error { pub fn downcast_mut(&mut self) -> Option<&mut T> { self.inner.downcast_mut() } + + pub fn to_io_err(&self) -> Option { + if let Some(e) = self.downcast_ref::() { + Some(io::Error::new(e.kind(), e.to_string())) + } else { + None + } + } } impl From for Error { diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 49a7dc9e4..79bfc6ced 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -1,5 +1,3 @@ -use std::path::Path; - const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; pub const SLASH_SEPARATOR: &str = "/"; @@ -39,7 +37,3 @@ pub fn retain_slash(s: &str) -> String { format!("{}{}", s, SLASH_SEPARATOR) } } - -pub fn join(p1: &str, p2: &str) -> String { - Path::new(p1).join(Path::new(p2)).to_string_lossy().to_string() -} From f46c53b77e1fbcc87b1f697033d0d484c05db0e8 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 16:21:21 +0800 Subject: [PATCH 56/70] done read/write quorum, need test --- ecstore/src/disk/local.rs | 28 ++++++++++++++-------------- ecstore/src/erasure.rs | 31 +++++++++++++++---------------- ecstore/src/sets.rs | 11 +++++++---- ecstore/src/store.rs | 2 +- ecstore/src/store_init.rs | 19 +++++-------------- rustfs/src/main.rs | 3 +-- scripts/run.sh | 2 +- 7 files changed, 44 insertions(+), 52 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index c1a9900d4..a798f7d17 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -204,10 +204,10 @@ impl LocalDisk { fs::create_dir_all(dst_data_path.parent().unwrap_or(Path::new("/"))).await?; } - debug!( - "rename_all from \n {:?} \n to \n {:?} \n skip:{:?}", - &src_data_path, &dst_data_path, &skip - ); + // debug!( + // "rename_all from \n {:?} \n to \n {:?} \n skip:{:?}", + // &src_data_path, &dst_data_path, &skip + // ); fs::rename(&src_data_path, &dst_data_path).await?; @@ -221,7 +221,7 @@ impl LocalDisk { fs::create_dir_all(parent).await?; } } - debug!("move_to_trash from:{:?} to {:?}", &delete_path, &trash_path); + // debug!("move_to_trash from:{:?} to {:?}", &delete_path, &trash_path); // TODO: 清空回收站 if let Err(err) = fs::rename(&delete_path, &trash_path).await { match err.kind() { @@ -263,15 +263,15 @@ impl LocalDisk { recursive: bool, immediate_purge: bool, ) -> Result<()> { - debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path); + // debug!("delete_file {:?}\n base_path:{:?}", &delete_path, &base_path); if is_root_path(base_path) || is_root_path(delete_path) { - debug!("delete_file skip {:?}", &delete_path); + // debug!("delete_file skip {:?}", &delete_path); return Ok(()); } if !delete_path.starts_with(base_path) || base_path == delete_path { - debug!("delete_file skip {:?}", &delete_path); + // debug!("delete_file skip {:?}", &delete_path); return Ok(()); } @@ -279,9 +279,9 @@ impl LocalDisk { self.move_to_trash(delete_path, recursive, immediate_purge).await?; } else { if delete_path.is_dir() { - debug!("delete_file remove_dir {:?}", &delete_path); + // debug!("delete_file remove_dir {:?}", &delete_path); if let Err(err) = fs::remove_dir(&delete_path).await { - debug!("remove_dir err {:?} when {:?}", &err, &delete_path); + // debug!("remove_dir err {:?} when {:?}", &err, &delete_path); match err.kind() { ErrorKind::NotFound => (), // ErrorKind::DirectoryNotEmpty => (), @@ -293,10 +293,10 @@ impl LocalDisk { } } } - debug!("delete_file remove_dir done {:?}", &delete_path); + // debug!("delete_file remove_dir done {:?}", &delete_path); } else { if let Err(err) = fs::remove_file(&delete_path).await { - debug!("remove_file err {:?} when {:?}", &err, &delete_path); + // debug!("remove_file err {:?} when {:?}", &err, &delete_path); match err.kind() { ErrorKind::NotFound => (), _ => { @@ -312,7 +312,7 @@ impl LocalDisk { Box::pin(self.delete_file(base_path, &PathBuf::from(dir_path), false, false)).await?; } - debug!("delete_file done {:?}", &delete_path); + // debug!("delete_file done {:?}", &delete_path); Ok(()) } @@ -811,7 +811,7 @@ impl DiskAPI for LocalDisk { async fn read_file(&self, volume: &str, path: &str) -> Result { let p = self.get_object_path(volume, path)?; - debug!("read_file {:?}", &p); + // debug!("read_file {:?}", &p); let file = File::options().read(true).open(&p).await?; Ok(FileReader::Local(LocalFileReader::new(file))) diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 76da879c5..c6ff3c190 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -26,7 +26,7 @@ pub struct Erasure { impl Erasure { pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self { - warn!( + debug!( "Erasure new data_shards {},parity_shards {} block_size {} ", data_shards, parity_shards, block_size ); @@ -57,7 +57,7 @@ impl Erasure { { let mut stream = ChunkedStream::new(body, total_size, self.block_size, false); let mut total: usize = 0; - let mut idx = 0; + // let mut idx = 0; while let Some(result) = stream.next().await { match result { Ok(data) => { @@ -68,19 +68,19 @@ impl Erasure { break; } - idx += 1; - debug!("encode {} get data {}", idx, data.len()); + // idx += 1; + // debug!("encode {} get data {}", idx, data.len()); let blocks = self.encode_data(data.as_ref())?; - debug!( - "encode shard {} size: {}/{} from block_size {}, total_size {} ", - idx, - blocks[0].len(), - blocks.len(), - data.len(), - total_size - ); + // debug!( + // "encode shard {} size: {}/{} from block_size {}, total_size {} ", + // idx, + // blocks[0].len(), + // blocks.len(), + // data.len(), + // total_size + // ); let mut errs = Vec::new(); @@ -94,14 +94,13 @@ impl Erasure { } } - debug!("Erasure encode errs {:?}", &errs); - let none_count = errs.iter().filter(|&x| x.is_none()).count(); if none_count >= write_quorum { continue; } if let Some(err) = reduce_write_quorum_errs(&errs, object_op_ignored_errs().as_ref(), write_quorum) { + warn!("Erasure encode errs {:?}", &errs); return Err(err); } } @@ -161,7 +160,7 @@ impl Erasure { break; } - debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length); + // debug!("decode {} block_offset {},block_length {} ", block_idx, block_offset, block_length); let mut bufs = reader.read().await?; @@ -175,7 +174,7 @@ impl Erasure { bytes_writed += writed_n; - debug!("decode {} writed_n {}, total_writed: {} ", block_idx, writed_n, bytes_writed); + // debug!("decode {} writed_n {}, total_writed: {} ", block_idx, writed_n, bytes_writed); } if bytes_writed != length { diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 95526ab73..5ae8f62aa 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -23,7 +23,7 @@ use tokio::sync::RwLock; use tokio::sync::Semaphore; use tokio::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; #[derive(Debug, Clone)] @@ -135,14 +135,17 @@ impl Sets { pub async fn monitor_and_connect_endpoints(&self) { tokio::time::sleep(Duration::from_secs(5)).await; + info!("start monitor_and_connect_endpoints"); + self.connect_disks().await; + // TODO: config interval let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(15 * 3)); let cloned_token = self.ctx.clone(); loop { tokio::select! { _= interval.tick()=>{ - debug!("tick..."); + // debug!("tick..."); self.connect_disks().await; interval.reset(); @@ -159,11 +162,11 @@ impl Sets { } async fn connect_disks(&self) { - debug!("start connect_disks ..."); + // debug!("start connect_disks ..."); for set in self.disk_set.iter() { set.connect_disks().await; } - debug!("done connect_disks ..."); + // debug!("done connect_disks ..."); } pub fn get_disks(&self, set_idx: usize) -> Arc { diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 1454832d8..a92177a6e 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -173,7 +173,7 @@ impl ECStore { let mut local_disks = Vec::new(); - info!("endpoint_pools: {:?}", endpoint_pools); + debug!("endpoint_pools: {:?}", endpoint_pools); for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() { // TODO: read from config parseStorageClass diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 35292b2ed..965037466 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -13,7 +13,7 @@ use std::{ fmt::Debug, }; -use tracing::warn; +use tracing::{debug, info, warn}; use uuid::Uuid; pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec>, Vec>) { @@ -50,13 +50,11 @@ pub async fn connect_load_init_formats( set_drive_count: usize, deployment_id: Option, ) -> Result { - warn!("connect_load_init_formats id: {:?}, first_disk: {}", deployment_id, first_disk); - let (formats, errs) = load_format_erasure_all(disks, false).await; - DiskError::check_disk_fatal_errs(&errs)?; + debug!("load_format_erasure_all errs {:?}", &errs); - warn!("load_format_erasure_all errs {:?}", &errs); + DiskError::check_disk_fatal_errs(&errs)?; check_format_erasure_values(&formats, set_drive_count)?; @@ -67,7 +65,7 @@ pub async fn connect_load_init_formats( let errs = save_format_file_all(disks, &fms).await; - warn!("save_format_file_all errs {:?}", &errs); + debug!("save_format_file_all errs {:?}", &errs); // TODO: check quorum // reduceWriteQuorumErrs(&errs)?; @@ -132,15 +130,8 @@ fn get_format_erasure_in_quorum(formats: &[Option]) -> Result Result<()> { } }); - warn!(" init store"); // init store ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?; - warn!(" init store success!"); + info!(" init store success!"); tokio::select! { _ = tokio::signal::ctrl_c() => { diff --git a/scripts/run.sh b/scripts/run.sh index 248d286b2..1a7b2073a 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -7,7 +7,7 @@ mkdir -p ./target/volume/test{0..4} if [ -z "$RUST_LOG" ]; then - export RUST_LOG="rustfs=debug,ecstore=info,s3s=debug" + export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug" fi DATA_DIR_ARG="./target/volume/test{0...4}" From d91856d8242080634e255abbd2331fff4336c856 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 16:28:07 +0800 Subject: [PATCH 57/70] done read/write quorum, need test --- ecstore/src/sets.rs | 2 +- ecstore/src/store_init.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 5ae8f62aa..72ca7ad47 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -23,7 +23,7 @@ use tokio::sync::RwLock; use tokio::sync::Semaphore; use tokio::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; use uuid::Uuid; #[derive(Debug, Clone)] diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 965037466..941aa4adf 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -13,7 +13,7 @@ use std::{ fmt::Debug, }; -use tracing::{debug, info, warn}; +use tracing::{debug, warn}; use uuid::Uuid; pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec>, Vec>) { From 72201049f0f0854e61361fb1f81abbe874c5e4b0 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 16:54:48 +0800 Subject: [PATCH 58/70] fix: delete_objects --- ecstore/src/sets.rs | 53 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index c0754384c..81aaefcad 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -228,31 +228,46 @@ impl StorageAPI for Sets { } } - let semaphore = Arc::new(Semaphore::new(num_cpus::get())); - let mut jhs = Vec::with_capacity(semaphore.available_permits()); + // let semaphore = Arc::new(Semaphore::new(num_cpus::get())); + // let mut jhs = Vec::with_capacity(semaphore.available_permits()); + // for (k, v) in set_obj_map { + // let disks = self.get_disks(k); + // let semaphore = semaphore.clone(); + // let opts = opts.clone(); + // let bucket = bucket.to_string(); + + // let jh = tokio::spawn(async move { + // let _permit = semaphore.acquire().await.unwrap(); + // let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); + // disks.delete_objects(&bucket, objs, opts).await + // }); + // jhs.push(jh); + // } + + // let mut results = Vec::with_capacity(jhs.len()); + // for jh in jhs { + // results.push(jh.await?.unwrap()); + // } + + // for (dobjects, errs) in results { + // del_objects.extend(dobjects); + // del_errs.extend(errs); + // } + + // TODO: 并发 for (k, v) in set_obj_map { let disks = self.get_disks(k); - let semaphore = semaphore.clone(); - let opts = opts.clone(); - let bucket = bucket.to_string(); + let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); + let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?; - let jh = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); - let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); - disks.delete_objects(&bucket, objs, opts).await - }); - jhs.push(jh); - } + for (i, err) in errs.into_iter().enumerate() { + let obj = v.get(i).unwrap(); - let mut results = Vec::with_capacity(jhs.len()); - for jh in jhs { - results.push(jh.await?.unwrap()); - } + del_errs[obj.orig_idx] = err; - for (dobjects, errs) in results { - del_objects.extend(dobjects); - del_errs.extend(errs); + del_objects[obj.orig_idx] = dobjects.get(i).unwrap().clone(); + } } Ok((del_objects, del_errs)) From 0ce707538a0529876212d5a28672c6619e1b7346 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 16:56:10 +0800 Subject: [PATCH 59/70] fix: delete_objects --- ecstore/src/sets.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 81aaefcad..7132bfac8 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -1,9 +1,8 @@ #![allow(clippy::map_entry)] -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; use futures::future::join_all; use http::HeaderMap; -use tokio::sync::Semaphore; use tracing::warn; use uuid::Uuid; From d759eb8b7c6d2204e1cebafd06cfc5576a03ecaf Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 17:31:46 +0800 Subject: [PATCH 60/70] stash --- ecstore/src/disk/error.rs | 12 ++++++++++ ecstore/src/disk/local.rs | 45 +++++++++++++++++++++++++++++++++----- ecstore/src/disk/mod.rs | 2 +- ecstore/src/disk/remote.rs | 2 +- 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/ecstore/src/disk/error.rs b/ecstore/src/disk/error.rs index 535273c2f..1b24ffc7a 100644 --- a/ecstore/src/disk/error.rs +++ b/ecstore/src/disk/error.rs @@ -391,3 +391,15 @@ pub fn map_err_not_exists(e: io::Error) -> Error { Error::new(e) } + +pub fn convert_access_error(e: io::Error, per_err: DiskError) -> Error { + if os_is_not_exist(&e) { + return Error::new(DiskError::VolumeNotEmpty); + } else if is_sys_err_io(&e) { + return Error::new(DiskError::FaultyDisk); + } else if os_is_permission(&e) { + return Error::new(per_err); + } + + Error::new(e) +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index a798f7d17..d0949c30f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -4,7 +4,7 @@ use super::{ os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; -use crate::disk::error::{is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err}; +use crate::disk::error::{convert_access_error, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err}; use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::utils::fs::lstat; @@ -585,6 +585,7 @@ impl DiskAPI for LocalDisk { #[must_use] async fn read_all(&self, volume: &str, path: &str) -> Result { + // TOFIX: let p = self.get_object_path(volume, path)?; let (data, _) = read_file_all(&p).await?; @@ -1083,11 +1084,45 @@ impl DiskAPI for LocalDisk { created: modtime, }) } - async fn delete_paths(&self, _volume: &str, _paths: &[&str]) -> Result<()> { - unimplemented!() + async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + let volume_dir = self.get_bucket_path(volume)?; + if !skip_access_checks(volume) { + utils::fs::access(&volume_dir) + .await + .map_err(|e| convert_access_error(e, DiskError::VolumeAccessDenied))? + } + + for path in paths.iter() { + let file_path = volume_dir.join(Path::new(path)); + + check_path_length(&file_path.to_string_lossy().to_string())?; + + self.move_to_trash(&file_path, false, false).await?; + } + + Ok(()) } - async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) { - unimplemented!() + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, _opts: UpdateMetadataOpts) -> Result<()> { + if let Some(metadata) = fi.metadata { + let volume_dir = self.get_bucket_path(&volume)?; + let file_path = volume_dir.join(Path::new(&path)); + + check_path_length(&file_path.to_string_lossy().to_string())?; + + self.read_all(&volume, format!("{}/{}", &path, super::STORAGE_FORMAT_FILE).as_str()) + .await + .map_err(|e| { + if DiskError::FileNotFound.is(&e) && fi.version_id.is_some() { + Error::new(DiskError::FileVersionNotFound) + } else { + e + } + })?; + + // FIXME: + } + + Err(Error::msg("Invalid Argument")) } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index cd8e9af7f..a59d720d5 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -90,7 +90,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { ) -> Result>>; async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()>; async fn write_metadata(&self, org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()>; - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts); + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()>; async fn read_version( &self, org_volume: &str, diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 51763c4ae..e035ad9c5 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -414,7 +414,7 @@ impl DiskAPI for RemoteDisk { // TODO: unimplemented!() } - async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) { + async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) -> Result<()> { // TODO: unimplemented!() } From e2b89e5ae2d07b4ec8b9bbbfe23f178e886b687b Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Wed, 25 Sep 2024 18:23:42 +0800 Subject: [PATCH 61/70] add docker builder --- .docker/Dockerfile.rockylinux9.3 | 35 ++++++++++++++++++++++++++++++++ .docker/Dockerfile.ubuntu22.04 | 27 ++++++++++++++++++++++++ Makefile | 18 ++++++++++++++-- 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 .docker/Dockerfile.rockylinux9.3 create mode 100644 .docker/Dockerfile.ubuntu22.04 diff --git a/.docker/Dockerfile.rockylinux9.3 b/.docker/Dockerfile.rockylinux9.3 new file mode 100644 index 000000000..340b84191 --- /dev/null +++ b/.docker/Dockerfile.rockylinux9.3 @@ -0,0 +1,35 @@ +FROM m.daocloud.io/docker.io/library/rockylinux:9.3 AS builder + +ENV LANG C.UTF-8 + +RUN sed -e 's|^mirrorlist=|#mirrorlist=|g' \ + -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.ustc.edu.cn/rocky|g' \ + -i.bak \ + /etc/yum.repos.d/rocky-extras.repo \ + /etc/yum.repos.d/rocky.repo + +RUN dnf makecache + +RUN yum install wget git unzip gcc openssl-devel pkgconf-pkg-config -y + +# install protoc +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip \ + && unzip protoc-27.0-linux-x86_64.zip -d protoc3 \ + && mv protoc3/bin/* /usr/local/bin/ && chmod +x /usr/local/bin/protoc \ + && rm -rf protoc-27.0-linux-x86_64.zip protoc3 + +# install flatc +RUN wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip \ + && unzip Linux.flatc.binary.g++-13.zip \ + && mv flatc /usr/local/bin/ && chmod +x /usr/local/bin/flatc \ + && rm -rf Linux.flatc.binary.g++-13.zip + +# install rust +ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" +ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" +RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ + && sh rustup-init.sh -y && rm -rf rustup-init.sh + +COPY .docker/cargo.config.toml /root/.cargo/config.toml + +WORKDIR /root/s3-rustfs diff --git a/.docker/Dockerfile.ubuntu22.04 b/.docker/Dockerfile.ubuntu22.04 new file mode 100644 index 000000000..546b16b7b --- /dev/null +++ b/.docker/Dockerfile.ubuntu22.04 @@ -0,0 +1,27 @@ +FROM m.daocloud.io/docker.io/library/ubuntu:22.04 + +ENV LANG C.UTF-8 + +RUN sed -i s@http://.*archive.ubuntu.com@http://repo.huaweicloud.com@g /etc/apt/sources.list + +RUN apt-get clean && apt-get update && apt-get install wget git curl unzip gcc pkg-config libssl-dev -y + +# install protoc +RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip \ + && unzip protoc-27.0-linux-x86_64.zip -d protoc3 \ + && mv protoc3/bin/* /usr/local/bin/ && chmod +x /usr/local/bin/protoc && mv protoc3/include/* /usr/local/include/ && rm -rf protoc-27.0-linux-x86_64.zip protoc3 + +# install flatc +RUN wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip \ + && unzip Linux.flatc.binary.g++-13.zip \ + && mv flatc /usr/local/bin/ && chmod +x /usr/local/bin/flatc && rm -rf Linux.flatc.binary.g++-13.zip + +# install rust +ENV RUSTUP_DIST_SERVER="https://rsproxy.cn" +ENV RUSTUP_UPDATE_ROOT="https://rsproxy.cn/rustup" +RUN curl -o rustup-init.sh --proto '=https' --tlsv1.2 -sSf https://rsproxy.cn/rustup-init.sh \ + && sh rustup-init.sh -y && rm -rf rustup-init.sh + +COPY .docker/cargo.config.toml /root/.cargo/config.toml + +WORKDIR /root/s3-rustfs diff --git a/Makefile b/Makefile index 131a4f1c7..b3b2e83a2 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,11 @@ DOCKER_CLI ?= docker IMAGE_NAME ?= rustfs:v1.0.0 CONTAINER_NAME ?= rustfs-dev -DOCKERFILE ?= $(shell pwd)/.docker/Dockerfile.devenv +DOCKERFILE_PATH = $(shell pwd)/.docker .PHONY: init-devenv init-devenv: - $(DOCKER_CLI) build -t $(IMAGE_NAME) -f $(DOCKERFILE) . + $(DOCKER_CLI) build -t $(IMAGE_NAME) -f $(DOCKERFILE_PATH)/Dockerfile.devenv . $(DOCKER_CLI) stop $(CONTAINER_NAME) $(DOCKER_CLI) rm $(CONTAINER_NAME) $(DOCKER_CLI) run -d --name $(CONTAINER_NAME) -p 9010:9010 -p 9000:9000 -v $(shell pwd):/root/s3-rustfs -it $(IMAGE_NAME) @@ -29,3 +29,17 @@ e2e-server: .PHONY: probe-e2e probe-e2e: sh $(shell pwd)/scripts/probe.sh + +# make BUILD_OS=ubuntu22.04 build +# in target/ubuntu22.04/release/rustfs + +# make BUILD_OS=rockylinux9.3 build +# in target/rockylinux9.3/release/rustfs +BUILD_OS ?= rockylinux9.3 +.PHONY: build +build: ROCKYLINUX_BUILD_IMAGE_NAME = $(BUILD_OS):v1 +build: ROCKYLINUX_BUILD_CONTAINER_NAME = rustfs-$(BUILD_OS)-build +build: BUILD_CMD = /root/.cargo/bin/cargo build --release --target-dir /root/s3-rustfs/target/$(BUILD_OS) +build: + $(DOCKER_CLI) build -t $(ROCKYLINUX_BUILD_IMAGE_NAME) -f $(DOCKERFILE_PATH)/Dockerfile.$(BUILD_OS) . + $(DOCKER_CLI) run --rm --name $(ROCKYLINUX_BUILD_CONTAINER_NAME) -v $(shell pwd):/root/s3-rustfs -it $(ROCKYLINUX_BUILD_IMAGE_NAME) $(BUILD_CMD) From e29ef738f26281e656aeaf945680af94fe56f61c Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 01:52:52 +0800 Subject: [PATCH 62/70] review localdisk --- ecstore/src/disk/local.rs | 117 +++++++++++++++++++++++++++++++------ ecstore/src/disk/mod.rs | 2 +- ecstore/src/disk/os.rs | 14 ++++- ecstore/src/disk/remote.rs | 4 +- ecstore/src/store_init.rs | 2 +- ecstore/src/utils/fs.rs | 50 +++++++++++++++- 6 files changed, 163 insertions(+), 26 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index d0949c30f..fced05f36 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -17,6 +17,7 @@ use crate::{ }; use bytes::Bytes; use path_absolutize::Absolutize; +use std::io::Read; use std::{ fs::Metadata, path::{Path, PathBuf}, @@ -405,6 +406,65 @@ impl LocalDisk { Ok(()) } + + async fn write_all_meta(&self, volume: &str, path: &str, buf: &[u8], sync: bool) -> Result<()> { + let volume_dir = self.get_bucket_path(&volume)?; + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(&file_path.to_string_lossy().to_string())?; + + let tmp_volume_dir = self.get_bucket_path(super::RUSTFS_META_TMP_BUCKET)?; + let tmp_file_path = tmp_volume_dir.join(Path::new(Uuid::new_v4().to_string().as_str())); + + self.write_all_internal(&tmp_file_path, buf, sync, tmp_volume_dir).await?; + + os::rename_all(tmp_file_path, file_path, volume_dir).await + } + + // write_all_private + pub async fn write_all_private( + &self, + volume: &str, + path: &str, + buf: &[u8], + sync: bool, + skip_parent: impl AsRef, + ) -> Result<()> { + let volume_dir = self.get_bucket_path(&volume)?; + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(&file_path.to_string_lossy().to_string())?; + + self.write_all_internal(file_path, buf, sync, skip_parent).await + } + + pub async fn write_all_internal( + &self, + p: impl AsRef, + data: impl AsRef<[u8]>, + sync: bool, + base_dir: impl AsRef, + ) -> Result<()> { + if sync { + } else { + } + // create top dir if not exists + fs::create_dir_all(&p.as_ref().parent().unwrap_or_else(|| Path::new("."))).await?; + + fs::write(&p, data).await?; + Ok(()) + } + + async fn open_file(&self, path: impl AsRef, mode: usize, skip_parent: impl AsRef) -> Result { + let mut skip_parent = skip_parent.as_ref(); + if skip_parent.as_os_str().is_empty() { + skip_parent = self.root.as_path(); + } + + if let Some(parent) = path.as_ref().parent() { + os::make_dir_all(parent, skip_parent).await?; + } + + unimplemented!() + } } fn is_root_path(path: impl AsRef) -> bool { @@ -433,14 +493,6 @@ pub async fn read_file_exists(path: impl AsRef) -> Result<(Vec, Option Ok((data, meta)) } -pub async fn write_all_internal(p: impl AsRef, data: impl AsRef<[u8]>) -> Result<()> { - // create top dir if not exists - fs::create_dir_all(&p.as_ref().parent().unwrap_or_else(|| Path::new("."))).await?; - - fs::write(&p, data).await?; - Ok(()) -} - pub async fn read_file_all(path: impl AsRef) -> Result<(Vec, Metadata)> { let p = path.as_ref(); let meta = read_file_metadata(&path).await?; @@ -584,18 +636,25 @@ impl DiskAPI for LocalDisk { } #[must_use] - async fn read_all(&self, volume: &str, path: &str) -> Result { + async fn read_all(&self, volume: &str, path: &str) -> Result> { // TOFIX: let p = self.get_object_path(volume, path)?; let (data, _) = read_file_all(&p).await?; - Ok(Bytes::from(data)) + Ok(data) } async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - let p = self.get_object_path(volume, path)?; + if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { + let mut format_info = self.format_info.lock().await; + format_info.data = data.clone(); + } - write_all_internal(p, data).await?; + let volume_dir = self.get_bucket_path(&volume)?; + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(&file_path.to_string_lossy().to_string())?; + + self.write_all_internal(file_path, data, true, volume_dir).await?; Ok(()) } @@ -978,7 +1037,8 @@ impl DiskAPI for LocalDisk { let fm_data = meta.marshal_msg()?; // 写入xl.meta - write_all_internal(&src_file_path, fm_data).await?; + self.write_all(src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str(), fm_data) + .await?; let no_inline = src_data_path.has_root() && fi.data.is_none() && fi.size > 0; if no_inline { @@ -1035,7 +1095,7 @@ impl DiskAPI for LocalDisk { if let Err(e) = utils::fs::access(&p).await { if os_is_not_exist(&e) { - os::make_dir_all(&p).await?; + os::make_dir_all(&p, self.root.as_path()).await?; } } @@ -1102,14 +1162,15 @@ impl DiskAPI for LocalDisk { Ok(()) } - async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, _opts: UpdateMetadataOpts) -> Result<()> { - if let Some(metadata) = fi.metadata { + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()> { + if let Some(metadata) = &fi.metadata { let volume_dir = self.get_bucket_path(&volume)?; let file_path = volume_dir.join(Path::new(&path)); check_path_length(&file_path.to_string_lossy().to_string())?; - self.read_all(&volume, format!("{}/{}", &path, super::STORAGE_FORMAT_FILE).as_str()) + let buf = self + .read_all(&volume, format!("{}/{}", &path, super::STORAGE_FORMAT_FILE).as_str()) .await .map_err(|e| { if DiskError::FileNotFound.is(&e) && fi.version_id.is_some() { @@ -1119,6 +1180,25 @@ impl DiskAPI for LocalDisk { } })?; + if !FileMeta::is_xl_format(buf.as_slice()) { + return Err(Error::new(DiskError::FileVersionNotFound)); + } + + let xl_meta = FileMeta::load(buf.as_slice())?; + + xl_meta.update_object_version(fi)?; + + let wbuf = xl_meta.marshal_msg()?; + + return self + .write_all_meta( + &volume, + format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), + &wbuf, + !opts.no_persistence, + ) + .await; + // FIXME: } @@ -1144,7 +1224,8 @@ impl DiskAPI for LocalDisk { let fm_data = meta.marshal_msg()?; - write_all_internal(p, fm_data).await?; + self.write_all(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), fm_data) + .await?; return Ok(()); } diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index a59d720d5..9d12f6e65 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -126,7 +126,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; // CleanAbandonedData async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()>; - async fn read_all(&self, volume: &str, path: &str) -> Result; + async fn read_all(&self, volume: &str, path: &str) -> Result>; } pub struct UpdateMetadataOpts { diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 3bf4d10e4..680428cb8 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -52,10 +52,18 @@ pub fn check_path_length(path_name: &str) -> Result<()> { Ok(()) } -pub async fn make_dir_all(path: impl AsRef) -> Result<()> { +pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> Result<()> { check_path_length(path.as_ref().to_string_lossy().to_string().as_str())?; - utils::fs::make_dir_all(path.as_ref()).map_err(os_err_to_file_err).await?; + if let Err(e) = reliable_mkdir_all(path.as_ref(), base_dir.as_ref()).await { + if is_sys_err_not_dir(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } else if is_sys_err_path_not_found(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } + + return Err(os_err_to_file_err(e)); + } Ok(()) } @@ -181,7 +189,7 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef if let Some(parent) = dir_path.as_ref().parent() { // 不支持递归，直接create_dir_all了 - if let Err(e) = fs::create_dir_all(&parent).await { + if let Err(e) = utils::fs::make_dir_all(&parent).await { if os_is_exist(&e) { return Ok(()); } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index e035ad9c5..7cd11d9e6 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -143,7 +143,7 @@ impl DiskAPI for RemoteDisk { Ok(()) } - async fn read_all(&self, volume: &str, path: &str) -> Result { + async fn read_all(&self, volume: &str, path: &str) -> Result> { info!("read_all"); let mut client = self.get_client_v2().await?; let request = Request::new(ReadAllRequest { @@ -160,7 +160,7 @@ impl DiskAPI for RemoteDisk { return Err(DiskError::FileNotFound.into()); } - Ok(Bytes::from(response.data)) + Ok(response.data) } async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 941aa4adf..ebd28c881 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -232,7 +232,7 @@ pub async fn load_format_erasure(disk: &DiskStore, _heal: bool) -> Result e, })?; - let fm = FormatV3::try_from(data.as_ref())?; + let fm = FormatV3::try_from(data.as_slice())?; // TODO: heal diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 0ff3b320a..9d9f54c2b 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -1,6 +1,9 @@ use std::{fs::Metadata, path::Path}; -use tokio::{fs, io}; +use tokio::{ + fs::{self, File}, + io, +}; #[cfg(not(windows))] pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { @@ -44,6 +47,51 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { true } +type FileMode = usize; + +const O_RDONLY: FileMode = 0x00000; +const O_WRONLY: FileMode = 0x00001; +const O_RDWR: FileMode = 0x00002; +const O_CREAT: FileMode = 0x00040; +const O_EXCL: FileMode = 0x00080; +const O_NOCTTY: FileMode = 0x00100; +const O_TRUNC: FileMode = 0x00200; +const O_NONBLOCK: FileMode = 0x00800; +const O_APPEND: FileMode = 0x00400; +const O_SYNC: FileMode = 0x01000; +const O_ASYNC: FileMode = 0x02000; +const O_CLOEXEC: FileMode = 0x80000; + +pub async fn open_file(path: impl AsRef, mode: FileMode) -> io::Result { + let mut opts = fs::OpenOptions::new(); + + match mode & (O_RDONLY | O_WRONLY | O_RDWR) { + O_RDONLY => { + opts.read(true); + } + O_WRONLY => { + opts.write(true); + } + O_RDWR => { + opts.read(true); + opts.write(true); + } + _ => (), + }; + + if mode & O_CREAT != 0 { + opts.write(true); + } + + if mode & O_APPEND != 0 { + opts.write(true); + } + + // FIXME: TODO + + opts.open(path.as_ref()).await +} + pub async fn access(path: impl AsRef) -> io::Result<()> { fs::metadata(path).await?; Ok(()) From b0d3641b2986d2835ab1083afa737984decdb2f8 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 14:12:47 +0800 Subject: [PATCH 63/70] add xl_meta.update_object_version --- ecstore/src/disk/local.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index fced05f36..b758b9e9e 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1163,7 +1163,7 @@ impl DiskAPI for LocalDisk { Ok(()) } async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()> { - if let Some(metadata) = &fi.metadata { + if let Some(_) = &fi.metadata { let volume_dir = self.get_bucket_path(&volume)?; let file_path = volume_dir.join(Path::new(&path)); @@ -1184,7 +1184,7 @@ impl DiskAPI for LocalDisk { return Err(Error::new(DiskError::FileVersionNotFound)); } - let xl_meta = FileMeta::load(buf.as_slice())?; + let mut xl_meta = FileMeta::load(buf.as_slice())?; xl_meta.update_object_version(fi)?; @@ -1198,8 +1198,6 @@ impl DiskAPI for LocalDisk { !opts.no_persistence, ) .await; - - // FIXME: } Err(Error::msg("Invalid Argument")) From 4a11fa61ed233d160c33157245760bd21a623ae2 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 26 Sep 2024 15:08:17 +0800 Subject: [PATCH 64/70] add remote support: delete_version,rename_part,delete_paths,update_metadata Signed-off-by: junxiang Mu <1948535941@qq.com> --- .../generated/flatbuffers_generated/models.rs | 207 +-- .../src/generated/proto_gen/node_service.rs | 1469 ++++++----------- common/protos/src/node.proto | 57 + ecstore/src/disk/mod.rs | 1 + ecstore/src/disk/remote.rs | 114 +- rustfs/src/grpc.rs | 166 +- 6 files changed, 933 insertions(+), 1081 deletions(-) diff --git a/common/protos/src/generated/flatbuffers_generated/models.rs b/common/protos/src/generated/flatbuffers_generated/models.rs index aa1f6ae2f..e4949fdcf 100644 --- a/common/protos/src/generated/flatbuffers_generated/models.rs +++ b/common/protos/src/generated/flatbuffers_generated/models.rs @@ -1,10 +1,9 @@ // automatically generated by the FlatBuffers compiler, do not modify - // @generated -use core::mem; use core::cmp::Ordering; +use core::mem; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -12,112 +11,114 @@ use self::flatbuffers::{EndianScalar, Follow}; #[allow(unused_imports, dead_code)] pub mod models { - use core::mem; - use core::cmp::Ordering; + use core::cmp::Ordering; + use core::mem; - extern crate flatbuffers; - use self::flatbuffers::{EndianScalar, Follow}; + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; -pub enum PingBodyOffset {} -#[derive(Copy, Clone, PartialEq)] + pub enum PingBodyOffset {} + #[derive(Copy, Clone, PartialEq)] -pub struct PingBody<'a> { - pub _tab: flatbuffers::Table<'a>, -} - -impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { - type Inner = PingBody<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { _tab: flatbuffers::Table::new(buf, loc) } - } -} - -impl<'a> PingBody<'a> { - pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; - - pub const fn get_fully_qualified_name() -> &'static str { - "models.PingBody" - } - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - PingBody { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args PingBodyArgs<'args> - ) -> flatbuffers::WIPOffset> { - let mut builder = PingBodyBuilder::new(_fbb); - if let Some(x) = args.payload { builder.add_payload(x); } - builder.finish() - } - - - #[inline] - pub fn payload(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { self._tab.get::>>(PingBody::VT_PAYLOAD, None)} - } -} - -impl flatbuffers::Verifiable for PingBody<'_> { - #[inline] - fn run_verifier( - v: &mut flatbuffers::Verifier, pos: usize - ) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>>("payload", Self::VT_PAYLOAD, false)? - .finish(); - Ok(()) - } -} -pub struct PingBodyArgs<'a> { - pub payload: Option>>, -} -impl<'a> Default for PingBodyArgs<'a> { - #[inline] - fn default() -> Self { - PingBodyArgs { - payload: None, + pub struct PingBody<'a> { + pub _tab: flatbuffers::Table<'a>, } - } -} -pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, -} -impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { - #[inline] - pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { - self.fbb_.push_slot_always::>(PingBody::VT_PAYLOAD, payload); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - PingBodyBuilder { - fbb_: _fbb, - start_: start, + impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { + type Inner = PingBody<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table::new(buf, loc), + } + } } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } -} -impl core::fmt::Debug for PingBody<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("PingBody"); - ds.field("payload", &self.payload()); - ds.finish() - } -} -} // pub mod models + impl<'a> PingBody<'a> { + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + pub const fn get_fully_qualified_name() -> &'static str { + "models.PingBody" + } + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PingBody { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args PingBodyArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = PingBodyBuilder::new(_fbb); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.finish() + } + + #[inline] + pub fn payload(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>(PingBody::VT_PAYLOAD, None) + } + } + } + + impl flatbuffers::Verifiable for PingBody<'_> { + #[inline] + fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>("payload", Self::VT_PAYLOAD, false)? + .finish(); + Ok(()) + } + } + pub struct PingBodyArgs<'a> { + pub payload: Option>>, + } + impl<'a> Default for PingBodyArgs<'a> { + #[inline] + fn default() -> Self { + PingBodyArgs { payload: None } + } + } + + pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { + #[inline] + pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(PingBody::VT_PAYLOAD, payload); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + PingBodyBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl core::fmt::Debug for PingBody<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("PingBody"); + ds.field("payload", &self.payload()); + ds.finish() + } + } +} // pub mod models diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index b44e1c1d8..1910283c4 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -145,6 +145,30 @@ pub struct DeleteResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenamePartRequst { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub src_volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub src_path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub dst_volume: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub dst_path: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "6")] + pub meta: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenamePartResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RenameFileRequst { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -352,6 +376,46 @@ pub struct StatVolumeResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeletePathsRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "3")] + pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeletePathsResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateMetadataRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub opts: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateMetadataResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct WriteMetadataRequest { /// indicate which one in the disks #[prost(string, tag = "1")] @@ -419,6 +483,32 @@ pub struct ReadXlResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVersionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, + #[prost(bool, tag = "5")] + pub force_del_marker: bool, + #[prost(string, tag = "6")] + pub opts: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVersionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub raw_file_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct DeleteVersionsRequest { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -476,8 +566,8 @@ pub struct DeleteVolumeResponse { /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] - use tonic::codegen::*; use tonic::codegen::http::Uri; + use tonic::codegen::*; #[derive(Debug, Clone)] pub struct NodeServiceClient { inner: tonic::client::Grpc, @@ -508,22 +598,15 @@ pub mod node_service_client { let inner = tonic::client::Grpc::with_origin(inner, origin); Self { inner } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> NodeServiceClient> + pub fn with_interceptor(inner: T, interceptor: F) -> NodeServiceClient> where F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< http::Request, - Response = http::Response< - >::ResponseBody, - >, + Response = http::Response<>::ResponseBody>, >, - , - >>::Error: Into + Send + Sync, + >>::Error: Into + Send + Sync, { NodeServiceClient::new(InterceptedService::new(inner, interceptor)) } @@ -566,16 +649,9 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/Ping", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Ping"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Ping")); @@ -584,23 +660,13 @@ pub mod node_service_client { pub async fn list_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ListBucket", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListBucket"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListBucket")); @@ -609,23 +675,13 @@ pub mod node_service_client { pub async fn make_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/MakeBucket", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeBucket"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); @@ -634,23 +690,13 @@ pub mod node_service_client { pub async fn get_bucket_info( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/GetBucketInfo", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetBucketInfo"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "GetBucketInfo")); @@ -659,23 +705,13 @@ pub mod node_service_client { pub async fn delete_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/DeleteBucket", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteBucket"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteBucket")); @@ -684,23 +720,13 @@ pub mod node_service_client { pub async fn read_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ReadAll", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAll"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAll")); @@ -709,23 +735,13 @@ pub mod node_service_client { pub async fn write_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/WriteAll", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteAll"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteAll")); @@ -738,41 +754,39 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/Delete", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Delete"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Delete")); self.inner.unary(req, path, codec).await } - pub async fn rename_file( + pub async fn rename_part( &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/RenameFile", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenamePart"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RenamePart")); + self.inner.unary(req, path, codec).await + } + pub async fn rename_file( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameFile"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameFile")); @@ -785,16 +799,9 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/Write", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Write"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Write")); @@ -808,16 +815,9 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ReadAt", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAt"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAt")); @@ -826,23 +826,13 @@ pub mod node_service_client { pub async fn list_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ListDir", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListDir"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListDir")); @@ -851,23 +841,13 @@ pub mod node_service_client { pub async fn walk_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/WalkDir", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WalkDir"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WalkDir")); @@ -876,23 +856,13 @@ pub mod node_service_client { pub async fn rename_data( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/RenameData", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameData"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); @@ -901,23 +871,13 @@ pub mod node_service_client { pub async fn make_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/MakeVolumes", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolumes"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolumes")); @@ -926,23 +886,13 @@ pub mod node_service_client { pub async fn make_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/MakeVolume", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolume"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolume")); @@ -951,23 +901,13 @@ pub mod node_service_client { pub async fn list_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ListVolumes", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListVolumes"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListVolumes")); @@ -976,48 +916,58 @@ pub mod node_service_client { pub async fn stat_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/StatVolume", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StatVolume"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "StatVolume")); self.inner.unary(req, path, codec).await } - pub async fn write_metadata( + pub async fn delete_paths( &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/WriteMetadata", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeletePaths"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DeletePaths")); + self.inner.unary(req, path, codec).await + } + pub async fn update_metadata( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/UpdateMetadata"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "UpdateMetadata")); + self.inner.unary(req, path, codec).await + } + pub async fn write_metadata( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteMetadata"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteMetadata")); @@ -1026,23 +976,13 @@ pub mod node_service_client { pub async fn read_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ReadVersion", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadVersion"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); @@ -1055,41 +995,39 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ReadXL", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadXL"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); self.inner.unary(req, path, codec).await } - pub async fn delete_versions( + pub async fn delete_version( &mut self, - request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/DeleteVersions", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersion"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersion")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_versions( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersions"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersions")); @@ -1098,23 +1036,13 @@ pub mod node_service_client { pub async fn read_multiple( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/ReadMultiple", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadMultiple"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadMultiple")); @@ -1123,23 +1051,13 @@ pub mod node_service_client { pub async fn delete_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - > { + ) -> std::result::Result, tonic::Status> { self.inner .ready() .await - .map_err(|e| { - tonic::Status::new( - tonic::Code::Unknown, - format!("Service was not ready: {}", e.into()), - ) - })?; + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static( - "/node_service.NodeService/DeleteVolume", - ); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVolume"); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVolume")); @@ -1162,31 +1080,19 @@ pub mod node_service_server { async fn list_bucket( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn make_bucket( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn get_bucket_info( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn delete_bucket( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn read_all( &self, request: tonic::Request, @@ -1194,21 +1100,19 @@ pub mod node_service_server { async fn write_all( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn delete( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn rename_part( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn rename_file( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn write( &self, request: tonic::Request, @@ -1229,77 +1133,59 @@ pub mod node_service_server { async fn rename_data( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn make_volumes( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn make_volume( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn list_volumes( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn stat_volume( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; + async fn delete_paths( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn update_metadata( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn write_metadata( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn read_version( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn read_xl( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn delete_version( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn delete_versions( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn read_multiple( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; async fn delete_volume( &self, request: tonic::Request, - ) -> std::result::Result< - tonic::Response, - tonic::Status, - >; + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct NodeServiceServer { @@ -1322,10 +1208,7 @@ pub mod node_service_server { max_encoding_message_size: None, } } - pub fn with_interceptor( - inner: T, - interceptor: F, - ) -> InterceptedService + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1369,10 +1252,7 @@ pub mod node_service_server { type Response = http::Response; type Error = std::convert::Infallible; type Future = BoxFuture; - fn poll_ready( - &mut self, - _cx: &mut Context<'_>, - ) -> Poll> { + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { @@ -1380,21 +1260,12 @@ pub mod node_service_server { "/node_service.NodeService/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); - impl tonic::server::UnaryService - for PingSvc { + impl tonic::server::UnaryService for PingSvc { type Response = super::PingResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::ping(&inner, request).await - }; + let fut = async move { ::ping(&inner, request).await }; Box::pin(fut) } } @@ -1407,14 +1278,8 @@ pub mod node_service_server { let method = PingSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1423,23 +1288,12 @@ pub mod node_service_server { "/node_service.NodeService/ListBucket" => { #[allow(non_camel_case_types)] struct ListBucketSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ListBucketSvc { + impl tonic::server::UnaryService for ListBucketSvc { type Response = super::ListBucketResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::list_bucket(&inner, request).await - }; + let fut = async move { ::list_bucket(&inner, request).await }; Box::pin(fut) } } @@ -1452,14 +1306,8 @@ pub mod node_service_server { let method = ListBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1468,23 +1316,12 @@ pub mod node_service_server { "/node_service.NodeService/MakeBucket" => { #[allow(non_camel_case_types)] struct MakeBucketSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for MakeBucketSvc { + impl tonic::server::UnaryService for MakeBucketSvc { type Response = super::MakeBucketResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::make_bucket(&inner, request).await - }; + let fut = async move { ::make_bucket(&inner, request).await }; Box::pin(fut) } } @@ -1497,14 +1334,8 @@ pub mod node_service_server { let method = MakeBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1513,23 +1344,12 @@ pub mod node_service_server { "/node_service.NodeService/GetBucketInfo" => { #[allow(non_camel_case_types)] struct GetBucketInfoSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for GetBucketInfoSvc { + impl tonic::server::UnaryService for GetBucketInfoSvc { type Response = super::GetBucketInfoResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::get_bucket_info(&inner, request).await - }; + let fut = async move { ::get_bucket_info(&inner, request).await }; Box::pin(fut) } } @@ -1542,14 +1362,8 @@ pub mod node_service_server { let method = GetBucketInfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1558,23 +1372,12 @@ pub mod node_service_server { "/node_service.NodeService/DeleteBucket" => { #[allow(non_camel_case_types)] struct DeleteBucketSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for DeleteBucketSvc { + impl tonic::server::UnaryService for DeleteBucketSvc { type Response = super::DeleteBucketResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::delete_bucket(&inner, request).await - }; + let fut = async move { ::delete_bucket(&inner, request).await }; Box::pin(fut) } } @@ -1587,14 +1390,8 @@ pub mod node_service_server { let method = DeleteBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1603,23 +1400,12 @@ pub mod node_service_server { "/node_service.NodeService/ReadAll" => { #[allow(non_camel_case_types)] struct ReadAllSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ReadAllSvc { + impl tonic::server::UnaryService for ReadAllSvc { type Response = super::ReadAllResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::read_all(&inner, request).await - }; + let fut = async move { ::read_all(&inner, request).await }; Box::pin(fut) } } @@ -1632,14 +1418,8 @@ pub mod node_service_server { let method = ReadAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1648,23 +1428,12 @@ pub mod node_service_server { "/node_service.NodeService/WriteAll" => { #[allow(non_camel_case_types)] struct WriteAllSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for WriteAllSvc { + impl tonic::server::UnaryService for WriteAllSvc { type Response = super::WriteAllResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::write_all(&inner, request).await - }; + let fut = async move { ::write_all(&inner, request).await }; Box::pin(fut) } } @@ -1677,14 +1446,8 @@ pub mod node_service_server { let method = WriteAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1693,23 +1456,12 @@ pub mod node_service_server { "/node_service.NodeService/Delete" => { #[allow(non_camel_case_types)] struct DeleteSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for DeleteSvc { + impl tonic::server::UnaryService for DeleteSvc { type Response = super::DeleteResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::delete(&inner, request).await - }; + let fut = async move { ::delete(&inner, request).await }; Box::pin(fut) } } @@ -1722,14 +1474,36 @@ pub mod node_service_server { let method = DeleteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/RenamePart" => { + #[allow(non_camel_case_types)] + struct RenamePartSvc(pub Arc); + impl tonic::server::UnaryService for RenamePartSvc { + type Response = super::RenamePartResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::rename_part(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RenamePartSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1738,23 +1512,12 @@ pub mod node_service_server { "/node_service.NodeService/RenameFile" => { #[allow(non_camel_case_types)] struct RenameFileSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for RenameFileSvc { + impl tonic::server::UnaryService for RenameFileSvc { type Response = super::RenameFileResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::rename_file(&inner, request).await - }; + let fut = async move { ::rename_file(&inner, request).await }; Box::pin(fut) } } @@ -1767,14 +1530,8 @@ pub mod node_service_server { let method = RenameFileSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1783,21 +1540,12 @@ pub mod node_service_server { "/node_service.NodeService/Write" => { #[allow(non_camel_case_types)] struct WriteSvc(pub Arc); - impl tonic::server::UnaryService - for WriteSvc { + impl tonic::server::UnaryService for WriteSvc { type Response = super::WriteResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::write(&inner, request).await - }; + let fut = async move { ::write(&inner, request).await }; Box::pin(fut) } } @@ -1810,14 +1558,8 @@ pub mod node_service_server { let method = WriteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1826,23 +1568,12 @@ pub mod node_service_server { "/node_service.NodeService/ReadAt" => { #[allow(non_camel_case_types)] struct ReadAtSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ReadAtSvc { + impl tonic::server::UnaryService for ReadAtSvc { type Response = super::ReadAtResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::read_at(&inner, request).await - }; + let fut = async move { ::read_at(&inner, request).await }; Box::pin(fut) } } @@ -1855,14 +1586,8 @@ pub mod node_service_server { let method = ReadAtSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1871,23 +1596,12 @@ pub mod node_service_server { "/node_service.NodeService/ListDir" => { #[allow(non_camel_case_types)] struct ListDirSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ListDirSvc { + impl tonic::server::UnaryService for ListDirSvc { type Response = super::ListDirResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::list_dir(&inner, request).await - }; + let fut = async move { ::list_dir(&inner, request).await }; Box::pin(fut) } } @@ -1900,14 +1614,8 @@ pub mod node_service_server { let method = ListDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1916,23 +1624,12 @@ pub mod node_service_server { "/node_service.NodeService/WalkDir" => { #[allow(non_camel_case_types)] struct WalkDirSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for WalkDirSvc { + impl tonic::server::UnaryService for WalkDirSvc { type Response = super::WalkDirResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::walk_dir(&inner, request).await - }; + let fut = async move { ::walk_dir(&inner, request).await }; Box::pin(fut) } } @@ -1945,14 +1642,8 @@ pub mod node_service_server { let method = WalkDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1961,23 +1652,12 @@ pub mod node_service_server { "/node_service.NodeService/RenameData" => { #[allow(non_camel_case_types)] struct RenameDataSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for RenameDataSvc { + impl tonic::server::UnaryService for RenameDataSvc { type Response = super::RenameDataResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::rename_data(&inner, request).await - }; + let fut = async move { ::rename_data(&inner, request).await }; Box::pin(fut) } } @@ -1990,14 +1670,8 @@ pub mod node_service_server { let method = RenameDataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2006,23 +1680,12 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolumes" => { #[allow(non_camel_case_types)] struct MakeVolumesSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for MakeVolumesSvc { + impl tonic::server::UnaryService for MakeVolumesSvc { type Response = super::MakeVolumesResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::make_volumes(&inner, request).await - }; + let fut = async move { ::make_volumes(&inner, request).await }; Box::pin(fut) } } @@ -2035,14 +1698,8 @@ pub mod node_service_server { let method = MakeVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2051,23 +1708,12 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolume" => { #[allow(non_camel_case_types)] struct MakeVolumeSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for MakeVolumeSvc { + impl tonic::server::UnaryService for MakeVolumeSvc { type Response = super::MakeVolumeResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::make_volume(&inner, request).await - }; + let fut = async move { ::make_volume(&inner, request).await }; Box::pin(fut) } } @@ -2080,14 +1726,8 @@ pub mod node_service_server { let method = MakeVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2096,23 +1736,12 @@ pub mod node_service_server { "/node_service.NodeService/ListVolumes" => { #[allow(non_camel_case_types)] struct ListVolumesSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ListVolumesSvc { + impl tonic::server::UnaryService for ListVolumesSvc { type Response = super::ListVolumesResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::list_volumes(&inner, request).await - }; + let fut = async move { ::list_volumes(&inner, request).await }; Box::pin(fut) } } @@ -2125,14 +1754,8 @@ pub mod node_service_server { let method = ListVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2141,23 +1764,12 @@ pub mod node_service_server { "/node_service.NodeService/StatVolume" => { #[allow(non_camel_case_types)] struct StatVolumeSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for StatVolumeSvc { + impl tonic::server::UnaryService for StatVolumeSvc { type Response = super::StatVolumeResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::stat_volume(&inner, request).await - }; + let fut = async move { ::stat_volume(&inner, request).await }; Box::pin(fut) } } @@ -2170,14 +1782,64 @@ pub mod node_service_server { let method = StatVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/DeletePaths" => { + #[allow(non_camel_case_types)] + struct DeletePathsSvc(pub Arc); + impl tonic::server::UnaryService for DeletePathsSvc { + type Response = super::DeletePathsResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::delete_paths(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeletePathsSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/UpdateMetadata" => { + #[allow(non_camel_case_types)] + struct UpdateMetadataSvc(pub Arc); + impl tonic::server::UnaryService for UpdateMetadataSvc { + type Response = super::UpdateMetadataResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::update_metadata(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = UpdateMetadataSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2186,23 +1848,12 @@ pub mod node_service_server { "/node_service.NodeService/WriteMetadata" => { #[allow(non_camel_case_types)] struct WriteMetadataSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for WriteMetadataSvc { + impl tonic::server::UnaryService for WriteMetadataSvc { type Response = super::WriteMetadataResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::write_metadata(&inner, request).await - }; + let fut = async move { ::write_metadata(&inner, request).await }; Box::pin(fut) } } @@ -2215,14 +1866,8 @@ pub mod node_service_server { let method = WriteMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2231,23 +1876,12 @@ pub mod node_service_server { "/node_service.NodeService/ReadVersion" => { #[allow(non_camel_case_types)] struct ReadVersionSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ReadVersionSvc { + impl tonic::server::UnaryService for ReadVersionSvc { type Response = super::ReadVersionResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::read_version(&inner, request).await - }; + let fut = async move { ::read_version(&inner, request).await }; Box::pin(fut) } } @@ -2260,14 +1894,8 @@ pub mod node_service_server { let method = ReadVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2276,23 +1904,12 @@ pub mod node_service_server { "/node_service.NodeService/ReadXL" => { #[allow(non_camel_case_types)] struct ReadXLSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ReadXLSvc { + impl tonic::server::UnaryService for ReadXLSvc { type Response = super::ReadXlResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::read_xl(&inner, request).await - }; + let fut = async move { ::read_xl(&inner, request).await }; Box::pin(fut) } } @@ -2305,14 +1922,36 @@ pub mod node_service_server { let method = ReadXLSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/DeleteVersion" => { + #[allow(non_camel_case_types)] + struct DeleteVersionSvc(pub Arc); + impl tonic::server::UnaryService for DeleteVersionSvc { + type Response = super::DeleteVersionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::delete_version(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteVersionSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2321,23 +1960,12 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVersions" => { #[allow(non_camel_case_types)] struct DeleteVersionsSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for DeleteVersionsSvc { + impl tonic::server::UnaryService for DeleteVersionsSvc { type Response = super::DeleteVersionsResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::delete_versions(&inner, request).await - }; + let fut = async move { ::delete_versions(&inner, request).await }; Box::pin(fut) } } @@ -2350,14 +1978,8 @@ pub mod node_service_server { let method = DeleteVersionsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2366,23 +1988,12 @@ pub mod node_service_server { "/node_service.NodeService/ReadMultiple" => { #[allow(non_camel_case_types)] struct ReadMultipleSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for ReadMultipleSvc { + impl tonic::server::UnaryService for ReadMultipleSvc { type Response = super::ReadMultipleResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::read_multiple(&inner, request).await - }; + let fut = async move { ::read_multiple(&inner, request).await }; Box::pin(fut) } } @@ -2395,14 +2006,8 @@ pub mod node_service_server { let method = ReadMultipleSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2411,23 +2016,12 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVolume" => { #[allow(non_camel_case_types)] struct DeleteVolumeSvc(pub Arc); - impl< - T: NodeService, - > tonic::server::UnaryService - for DeleteVolumeSvc { + impl tonic::server::UnaryService for DeleteVolumeSvc { type Response = super::DeleteVolumeResponse; - type Future = BoxFuture< - tonic::Response, - tonic::Status, - >; - fn call( - &mut self, - request: tonic::Request, - ) -> Self::Future { + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { - ::delete_volume(&inner, request).await - }; + let fut = async move { ::delete_volume(&inner, request).await }; Box::pin(fut) } } @@ -2440,34 +2034,21 @@ pub mod node_service_server { let method = DeleteVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config( - accept_compression_encodings, - send_compression_encodings, - ) - .apply_max_message_size_config( - max_decoding_message_size, - max_encoding_message_size, - ); + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); let res = grpc.unary(method, req).await; Ok(res) }; Box::pin(fut) } - _ => { - Box::pin(async move { - Ok( - http::Response::builder() - .status(200) - .header("grpc-status", tonic::Code::Unimplemented as i32) - .header( - http::header::CONTENT_TYPE, - tonic::metadata::GRPC_CONTENT_TYPE, - ) - .body(empty_body()) - .unwrap(), - ) - }) - } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", tonic::Code::Unimplemented as i32) + .header(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE) + .body(empty_body()) + .unwrap()) + }), } } } diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index a7e64b0a8..bdeb77360 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -88,6 +88,20 @@ message DeleteResponse { optional string error_info = 2; } +message RenamePartRequst { + string disk = 1; + string src_volume = 2; + string src_path = 3; + string dst_volume = 4; + string dst_path = 5; + bytes meta = 6; +} + +message RenamePartResponse { + bool success = 1; + optional string error_info = 2; +} + message RenameFileRequst { string disk = 1; string src_volume = 2; @@ -219,6 +233,30 @@ message StatVolumeResponse { optional string error_info = 3; } +message DeletePathsRequest { + string disk = 1; + string volume = 2; + repeated string paths = 3; +} + +message DeletePathsResponse { + bool success = 1; + optional string error_info = 2; +} + +message UpdateMetadataRequest { + string disk = 1; + string volume = 2; + string path = 3; + string file_info = 4; + string opts = 5; +} + +message UpdateMetadataResponse { + bool success = 1; + optional string error_info = 2; +} + message WriteMetadataRequest { string disk = 1; // indicate which one in the disks string volume = 2; @@ -258,6 +296,21 @@ message ReadXLResponse { optional string error_info = 3; } +message DeleteVersionRequest { + string disk = 1; + string volume = 2; + string path = 3; + string file_info = 4; + bool force_del_marker = 5; + string opts = 6; +} + +message DeleteVersionResponse { + bool success = 1; + string raw_file_info = 2; + optional string error_info = 3; +} + message DeleteVersionsRequest { string disk = 1; string volume = 2; @@ -307,6 +360,7 @@ service NodeService { rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; rpc Delete(DeleteRequest) returns (DeleteResponse) {}; + rpc RenamePart(RenamePartRequst) returns (RenamePartResponse) {}; rpc RenameFile(RenameFileRequst) returns (RenameFileResponse) {}; rpc Write(WriteRequest) returns (WriteResponse) {}; // rpc Append(AppendRequest) returns (AppendResponse) {}; @@ -318,9 +372,12 @@ service NodeService { rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {}; rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {}; rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {}; + rpc DeletePaths(DeletePathsRequest) returns (DeletePathsResponse) {}; + rpc UpdateMetadata(UpdateMetadataRequest) returns (UpdateMetadataResponse) {}; rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; + rpc DeleteVersion(DeleteVersionRequest) returns (DeleteVersionResponse) {}; rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 9d12f6e65..9aaeaa392 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -129,6 +129,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn read_all(&self, volume: &str, path: &str) -> Result>; } +#[derive(Debug, Serialize, Deserialize)] pub struct UpdateMetadataOpts { pub no_persistence: bool, } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 7cd11d9e6..3186a52bc 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -5,10 +5,10 @@ use futures::lock::Mutex; use protos::{ node_service_time_out_client, proto_gen::node_service::{ - node_service_client::NodeServiceClient, DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest, - ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, - ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, - WriteMetadataRequest, + node_service_client::NodeServiceClient, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, + DeleteVolumeRequest, ListDirRequest, ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, + ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, RenameDataRequest, RenameFileRequst, RenamePartRequst, + StatVolumeRequest, UpdateMetadataRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, }, DEFAULT_GRPC_SERVER_MESSAGE_LEN, }; @@ -201,15 +201,25 @@ impl DiskAPI for RemoteDisk { Ok(()) } - async fn rename_part( - &self, - _src_volume: &str, - _src_path: &str, - _dst_volume: &str, - _dst_path: &str, - _meta: Vec, - ) -> Result<()> { - unimplemented!() + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { + info!("rename_part"); + let mut client = self.get_client_v2().await?; + let request = Request::new(RenamePartRequst { + disk: self.root.to_string_lossy().to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + meta, + }); + + let response = client.rename_part(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + Ok(()) } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { info!("rename_file"); @@ -410,13 +420,45 @@ impl DiskAPI for RemoteDisk { Ok(volume_info) } - async fn delete_paths(&self, _volume: &str, _paths: &[&str]) -> Result<()> { - // TODO: - unimplemented!() + async fn delete_paths(&self, volume: &str, paths: &[&str]) -> Result<()> { + info!("delete_paths"); + let paths = paths.iter().map(|s| s.to_string()).collect::>(); + let mut client = self.get_client_v2().await?; + let request = Request::new(DeletePathsRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + paths, + }); + + let response = client.delete_paths(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + Ok(()) } - async fn update_metadata(&self, _volume: &str, _path: &str, _fi: FileInfo, _opts: UpdateMetadataOpts) -> Result<()> { - // TODO: - unimplemented!() + async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: UpdateMetadataOpts) -> Result<()> { + info!("update_metadata"); + let file_info = serde_json::to_string(&fi)?; + let opts = serde_json::to_string(&opts)?; + + let mut client = self.get_client_v2().await?; + let request = Request::new(UpdateMetadataRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + file_info, + opts, + }); + + let response = client.update_metadata(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + Ok(()) } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { @@ -491,13 +533,35 @@ impl DiskAPI for RemoteDisk { } async fn delete_version( &self, - _volume: &str, - _path: &str, - _fi: FileInfo, - _force_del_marker: bool, - _opts: DeleteOptions, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, ) -> Result { - unimplemented!() + info!("delete_version"); + let file_info = serde_json::to_string(&fi)?; + let opts = serde_json::to_string(&opts)?; + + let mut client = self.get_client_v2().await?; + let request = Request::new(DeleteVersionRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + file_info, + force_del_marker, + opts, + }); + + let response = client.delete_version(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + + let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; + + Ok(raw_file_info) } async fn delete_versions( &self, diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 6e4747fe3..f47f81854 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,5 +1,5 @@ use ecstore::{ - disk::{DeleteOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, WalkDirOptions}, + disk::{DeleteOptions, DiskStore, FileInfoVersions, ReadMultipleReq, ReadOptions, UpdateMetadataOpts, WalkDirOptions}, erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, store::{all_local_disk_path, find_local_disk}, @@ -12,14 +12,16 @@ use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ node_service_server::{NodeService as Node, NodeServiceServer as NodeServer}, - DeleteBucketRequest, DeleteBucketResponse, DeleteRequest, DeleteResponse, DeleteVersionsRequest, DeleteVersionsResponse, - DeleteVolumeRequest, DeleteVolumeResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, - ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, - MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, - PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, - ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, - RenameFileRequst, RenameFileResponse, StatVolumeRequest, StatVolumeResponse, WalkDirRequest, WalkDirResponse, - WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, WriteRequest, WriteResponse, + DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, DeletePathsResponse, DeleteRequest, DeleteResponse, + DeleteVersionRequest, DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, + DeleteVolumeResponse, GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, + ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, + MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, + ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, + ReadXlRequest, ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, + RenamePartRequst, RenamePartResponse, StatVolumeRequest, StatVolumeResponse, UpdateMetadataRequest, + UpdateMetadataResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, + WriteMetadataResponse, WriteRequest, WriteResponse, }, }; @@ -281,6 +283,36 @@ impl Node for NodeService { } } + async fn rename_part(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk + .rename_part( + &request.src_volume, + &request.src_path, + &request.dst_volume, + &request.dst_path, + request.meta, + ) + .await + { + Ok(_) => Ok(tonic::Response::new(RenamePartResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(RenamePartResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(RenamePartResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn rename_file(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { @@ -594,6 +626,68 @@ impl Node for NodeService { } } + async fn delete_paths(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let paths = request.paths.iter().map(|s| s.as_str()).collect::>(); + match disk.delete_paths(&request.volume, &paths).await { + Ok(_) => Ok(tonic::Response::new(DeletePathsResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DeletePathsResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DeletePathsResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn update_metadata(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(UpdateMetadataResponse { + success: false, + error_info: Some("can not decode FileInfoVersions".to_string()), + })); + } + }; + let opts = match serde_json::from_str::(&request.opts) { + Ok(opts) => opts, + Err(_) => { + return Ok(tonic::Response::new(UpdateMetadataResponse { + success: false, + error_info: Some("can not decode UpdateMetadataOpts".to_string()), + })); + } + }; + + match disk.update_metadata(&request.volume, &request.path, file_info, opts).await { + Ok(_) => Ok(tonic::Response::new(UpdateMetadataResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(UpdateMetadataResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(UpdateMetadataResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn write_metadata(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { @@ -699,6 +793,60 @@ impl Node for NodeService { } } + async fn delete_version(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error_info: Some("can not decode FileInfoVersions".to_string()), + })); + } + }; + let opts = match serde_json::from_str::(&request.opts) { + Ok(opts) => opts, + Err(_) => { + return Ok(tonic::Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk + .delete_version(&request.volume, &request.path, file_info, request.force_del_marker, opts) + .await + { + Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) { + Ok(raw_file_info) => Ok(tonic::Response::new(DeleteVersionResponse { + success: true, + raw_file_info, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error_info: Some(err.to_string()), + })), + }, + Err(err) => Ok(tonic::Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DeleteVersionResponse { + success: false, + raw_file_info: "".to_string(), + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn delete_versions(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { From 5d46bba216c7c92f8db3d1f194692a0009bad7f4 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 15:51:44 +0800 Subject: [PATCH 65/70] opt:read/write_all --- ecstore/src/disk/local.rs | 231 ++++++++++++++++++++++++++------------ ecstore/src/utils/fs.rs | 41 ++++--- 2 files changed, 184 insertions(+), 88 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index b758b9e9e..80f2344ce 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1,13 +1,16 @@ -use super::error::{is_sys_err_io, is_sys_err_not_empty, os_is_not_exist}; +use super::error::{is_sys_err_io, is_sys_err_not_empty, is_sys_err_too_many_files, os_is_not_exist, os_is_permission}; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; use super::{ os, DeleteOptions, DiskAPI, DiskLocation, FileInfoVersions, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, }; -use crate::disk::error::{convert_access_error, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err}; +use crate::disk::error::{ + convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, + map_err_not_exists, os_err_to_file_err, +}; use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; -use crate::utils::fs::lstat; +use crate::utils::fs::{lstat, O_RDONLY}; use crate::utils::path::{has_suffix, SLASH_SEPARATOR}; use crate::{ error::{Error, Result}, @@ -22,9 +25,9 @@ use std::{ fs::Metadata, path::{Path, PathBuf}, }; -use time::OffsetDateTime; +use time::{util, OffsetDateTime}; use tokio::fs::{self, File}; -use tokio::io::ErrorKind; +use tokio::io::{self, AsyncReadExt, AsyncWriteExt, ErrorKind}; use tokio::sync::Mutex; use tracing::{debug, warn}; use uuid::Uuid; @@ -117,10 +120,6 @@ impl LocalDisk { Ok(disk) } - // fn check_path_length(_path_name: &str) -> Result<()> { - // unimplemented!() - // } - fn is_valid_volname(volname: &str) -> bool { if volname.len() < 3 { return false; @@ -324,54 +323,108 @@ impl LocalDisk { volume_dir: impl AsRef, path: impl AsRef, read_data: bool, - ) -> Result<(Vec, OffsetDateTime)> { + ) -> Result<(Vec, Option)> { let meta_path = path.as_ref().join(Path::new(super::STORAGE_FORMAT_FILE)); if read_data { - self.read_all_data(bucket, volume_dir, meta_path).await + self.read_all_data_with_dmtime(bucket, volume_dir, meta_path).await } else { - self.read_metadata_with_dmtime(meta_path).await + self.read_all_data_with_dmtime(bucket, volume_dir, meta_path).await + // FIXME: read_metadata only suport + // self.read_metadata_with_dmtime(meta_path).await } } - async fn read_metadata_with_dmtime(&self, path: impl AsRef) -> Result<(Vec, OffsetDateTime)> { - let (data, meta) = read_file_all(path).await?; + async fn read_metadata_with_dmtime(&self, file_path: impl AsRef) -> Result<(Vec, Option)> { + check_path_length(&file_path.as_ref().to_string_lossy().to_string())?; - let modtime = match meta.modified() { - Ok(md) => OffsetDateTime::from(md), - Err(_) => return Err(Error::msg("Not supported modified on this platform")), - }; + let f = utils::fs::open_file(file_path, O_RDONLY).await?; - Ok((data, modtime)) + let meta = f.metadata().await?; + + if meta.is_dir() { + return Err(Error::new(DiskError::FileNotFound)); + } + + // FIXME: + unimplemented!() } - async fn read_all_data( - &self, - _bucket: &str, - _volume_dir: impl AsRef, - path: impl AsRef, - ) -> Result<(Vec, OffsetDateTime)> { - let (data, meta) = read_file_all(path).await?; + async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef, file_path: impl AsRef) -> Result> { + // TODO: timeout suport + let (data, _) = self.read_all_data_with_dmtime(volume, volume_dir, file_path).await?; + Ok(data) + } - let modtime = match meta.modified() { - Ok(md) => OffsetDateTime::from(md), - Err(_) => return Err(Error::msg("Not supported modified on this platform")), + async fn read_all_data_with_dmtime( + &self, + volume: &str, + volume_dir: impl AsRef, + file_path: impl AsRef, + ) -> Result<(Vec, Option)> { + let mut f = match utils::fs::open_file(file_path.as_ref(), utils::fs::O_RDONLY).await { + Ok(f) => f, + Err(e) => { + if os_is_not_exist(&e) { + if !skip_access_checks(volume) { + if let Err(er) = utils::fs::access(volume_dir.as_ref()).await { + if os_is_not_exist(&er) { + return Err(Error::new(DiskError::VolumeNotFound)); + } + } + } + + return Err(Error::new(DiskError::FileNotFound)); + } else if os_is_permission(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } else if is_sys_err_not_dir(&e) || is_sys_err_is_dir(&e) { + return Err(Error::new(DiskError::FileNotFound)); + } else if is_sys_err_handle_invalid(&e) { + return Err(Error::new(DiskError::FileNotFound)); + } else if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } else if is_sys_err_too_many_files(&e) { + return Err(Error::new(DiskError::TooManyOpenFiles)); + } else if is_sys_err_invalid_arg(&e) { + if let Ok(meta) = utils::fs::lstat(file_path.as_ref()).await { + if meta.is_dir() { + return Err(Error::new(DiskError::FileNotFound)); + } + } + return Err(Error::new(DiskError::UnsupportedDisk)); + } + + return Err(Error::new(e)); + } }; - Ok((data, modtime)) + let meta = f.metadata().await.map_err(os_err_to_file_err)?; + + if meta.is_dir() { + return Err(Error::new(DiskError::FileNotFound)); + } + + let size = meta.len() as usize; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size)?; + + f.read_to_end(&mut bytes).await.map_err(os_err_to_file_err)?; + + let modtime = match meta.modified() { + Ok(md) => Some(OffsetDateTime::from(md)), + Err(_) => None, + }; + + Ok((bytes, modtime)) } async fn delete_versions_internal(&self, volume: &str, path: &str, fis: &Vec) -> Result<()> { let volume_dir = self.get_bucket_path(volume)?; let xlpath = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; - let (data, _) = match self.read_all_data(volume, volume_dir.as_path(), &xlpath).await { - Ok(res) => res, - Err(_err) => { - // TODO: check if not found return err - - (Vec::new(), OffsetDateTime::UNIX_EPOCH) - } - }; + let data = self + .read_all_data(volume, volume_dir.as_path(), &xlpath) + .await + .unwrap_or_default(); if data.is_empty() { return Err(Error::new(DiskError::FileNotFound)); @@ -401,8 +454,16 @@ impl LocalDisk { // 更新xl.meta let buf = fm.marshal_msg()?; - self.write_all(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), buf) - .await?; + let volume_dir = self.get_bucket_path(&volume)?; + + self.write_all_private( + volume, + format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str(), + &buf, + true, + volume_dir, + ) + .await?; Ok(()) } @@ -420,7 +481,21 @@ impl LocalDisk { os::rename_all(tmp_file_path, file_path, volume_dir).await } - // write_all_private + // write_all_public for trail + async fn write_all_public(&self, volume: &str, path: &str, data: Vec) -> Result<()> { + if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { + let mut format_info = self.format_info.lock().await; + format_info.data = data.clone(); + } + + let volume_dir = self.get_bucket_path(&volume)?; + + self.write_all_private(&volume, &path, &data, true, volume_dir).await?; + + Ok(()) + } + + // write_all_private with check_path_length pub async fn write_all_private( &self, volume: &str, @@ -435,21 +510,27 @@ impl LocalDisk { self.write_all_internal(file_path, buf, sync, skip_parent).await } - + // write_all_internal do write file pub async fn write_all_internal( &self, - p: impl AsRef, + file_path: impl AsRef, data: impl AsRef<[u8]>, sync: bool, - base_dir: impl AsRef, + skip_parent: impl AsRef, ) -> Result<()> { - if sync { - } else { - } - // create top dir if not exists - fs::create_dir_all(&p.as_ref().parent().unwrap_or_else(|| Path::new("."))).await?; + let flags = utils::fs::O_CREATE | utils::fs::O_WRONLY | utils::fs::O_TRUNC; + + let mut f = { + if sync { + // TODO: suport sync + self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? + } else { + self.open_file(file_path.as_ref(), flags, skip_parent.as_ref()).await? + } + }; + + f.write_all(data.as_ref()).await?; - fs::write(&p, data).await?; Ok(()) } @@ -463,7 +544,23 @@ impl LocalDisk { os::make_dir_all(parent, skip_parent).await?; } - unimplemented!() + let f = utils::fs::open_file(path.as_ref(), mode).await.map_err(|e| { + if is_sys_err_io(&e) { + Error::new(DiskError::IsNotRegular) + } else if os_is_permission(&e) { + Error::new(DiskError::FileAccessDenied) + } else if is_sys_err_not_dir(&e) { + Error::new(DiskError::FileAccessDenied) + } else if is_sys_err_io(&e) { + Error::new(DiskError::FaultyDisk) + } else if is_sys_err_too_many_files(&e) { + Error::new(DiskError::TooManyOpenFiles) + } else { + Error::new(e) + } + })?; + + Ok(f) } } @@ -645,18 +742,7 @@ impl DiskAPI for LocalDisk { } async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { - let mut format_info = self.format_info.lock().await; - format_info.data = data.clone(); - } - - let volume_dir = self.get_bucket_path(&volume)?; - let file_path = volume_dir.join(Path::new(&path)); - check_path_length(&file_path.to_string_lossy().to_string())?; - - self.write_all_internal(file_path, data, true, volume_dir).await?; - - Ok(()) + self.write_all_public(volume, path, data).await } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { @@ -947,10 +1033,7 @@ impl DiskAPI for LocalDisk { let (fdata, _) = match self.read_metadata_with_dmtime(&fpath).await { Ok(res) => res, - Err(_) => { - // TODO: check err - (Vec::new(), OffsetDateTime::UNIX_EPOCH) - } + Err(_) => (Vec::new(), None), }; meta.metadata = fdata; @@ -1259,12 +1342,16 @@ impl DiskAPI for LocalDisk { } async fn delete_version( &self, - _volume: &str, - _path: &str, - _fi: FileInfo, - _force_del_marker: bool, - _opts: DeleteOptions, + volume: &str, + path: &str, + fi: FileInfo, + force_del_marker: bool, + opts: DeleteOptions, ) -> Result { + let volume_dir = self.get_bucket_path(&volume)?; + + // self.read_all_data(bucket, volume_dir, path); + unimplemented!() } async fn delete_versions( diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index 9d9f54c2b..afc13c5a3 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -49,18 +49,25 @@ pub fn same_file(f1: &Metadata, f2: &Metadata) -> bool { type FileMode = usize; -const O_RDONLY: FileMode = 0x00000; -const O_WRONLY: FileMode = 0x00001; -const O_RDWR: FileMode = 0x00002; -const O_CREAT: FileMode = 0x00040; -const O_EXCL: FileMode = 0x00080; -const O_NOCTTY: FileMode = 0x00100; -const O_TRUNC: FileMode = 0x00200; -const O_NONBLOCK: FileMode = 0x00800; -const O_APPEND: FileMode = 0x00400; -const O_SYNC: FileMode = 0x01000; -const O_ASYNC: FileMode = 0x02000; -const O_CLOEXEC: FileMode = 0x80000; +pub const O_RDONLY: FileMode = 0x00000; +pub const O_WRONLY: FileMode = 0x00001; +pub const O_RDWR: FileMode = 0x00002; +pub const O_CREATE: FileMode = 0x00040; +// pub const O_EXCL: FileMode = 0x00080; +// pub const O_NOCTTY: FileMode = 0x00100; +pub const O_TRUNC: FileMode = 0x00200; +// pub const O_NONBLOCK: FileMode = 0x00800; +pub const O_APPEND: FileMode = 0x00400; +// pub const O_SYNC: FileMode = 0x01000; +// pub const O_ASYNC: FileMode = 0x02000; +// pub const O_CLOEXEC: FileMode = 0x80000; + +// read: bool, +// write: bool, +// append: bool, +// truncate: bool, +// create: bool, +// create_new: bool, pub async fn open_file(path: impl AsRef, mode: FileMode) -> io::Result { let mut opts = fs::OpenOptions::new(); @@ -79,15 +86,17 @@ pub async fn open_file(path: impl AsRef, mode: FileMode) -> io::Result (), }; - if mode & O_CREAT != 0 { - opts.write(true); + if mode & O_CREATE != 0 { + opts.create(true); } if mode & O_APPEND != 0 { - opts.write(true); + opts.append(true); } - // FIXME: TODO + if mode & O_TRUNC != 0 { + opts.truncate(true); + } opts.open(path.as_ref()).await } From 98f48ba067fc4f349f58b26f9cb8a4b1f82746f2 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 16:04:20 +0800 Subject: [PATCH 66/70] merge remote support --- ecstore/src/disk/local.rs | 16 +++++++--------- ecstore/src/disk/mod.rs | 2 +- ecstore/src/disk/os.rs | 1 - ecstore/src/disk/remote.rs | 1 - 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 80f2344ce..adbec6939 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -18,16 +18,14 @@ use crate::{ store_api::{FileInfo, RawFileInfo}, utils, }; -use bytes::Bytes; use path_absolutize::Absolutize; -use std::io::Read; use std::{ fs::Metadata, path::{Path, PathBuf}, }; -use time::{util, OffsetDateTime}; +use time::OffsetDateTime; use tokio::fs::{self, File}; -use tokio::io::{self, AsyncReadExt, AsyncWriteExt, ErrorKind}; +use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; use tokio::sync::Mutex; use tracing::{debug, warn}; use uuid::Uuid; @@ -1343,12 +1341,12 @@ impl DiskAPI for LocalDisk { async fn delete_version( &self, volume: &str, - path: &str, - fi: FileInfo, - force_del_marker: bool, - opts: DeleteOptions, + _path: &str, + _fi: FileInfo, + _force_del_marker: bool, + _opts: DeleteOptions, ) -> Result { - let volume_dir = self.get_bucket_path(&volume)?; + let _volume_dir = self.get_bucket_path(&volume)?; // self.read_all_data(bucket, volume_dir, path); diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 9aaeaa392..b691afff6 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -19,7 +19,7 @@ use crate::{ file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, }; -use bytes::Bytes; + use endpoint::Endpoint; use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest}; use serde::{Deserialize, Serialize}; diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 680428cb8..78d0ada5c 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -3,7 +3,6 @@ use std::{ path::{Component, Path}, }; -use futures::TryFutureExt; use tokio::fs; use crate::{ diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 3186a52bc..47165da51 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -1,6 +1,5 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; -use bytes::Bytes; use futures::lock::Mutex; use protos::{ node_service_time_out_client, From 053cbc1b354478f7adbfc811a4016e8ab8e353db Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 16:27:01 +0800 Subject: [PATCH 67/70] fix:read_metadata --- ecstore/src/disk/local.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index adbec6939..9eb470bae 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -332,10 +332,11 @@ impl LocalDisk { } } + // FIXME: read_metadata only suport async fn read_metadata_with_dmtime(&self, file_path: impl AsRef) -> Result<(Vec, Option)> { check_path_length(&file_path.as_ref().to_string_lossy().to_string())?; - let f = utils::fs::open_file(file_path, O_RDONLY).await?; + let mut f = utils::fs::open_file(file_path, O_RDONLY).await?; let meta = f.metadata().await?; @@ -343,8 +344,24 @@ impl LocalDisk { return Err(Error::new(DiskError::FileNotFound)); } - // FIXME: - unimplemented!() + let meta = f.metadata().await.map_err(os_err_to_file_err)?; + + if meta.is_dir() { + return Err(Error::new(DiskError::FileNotFound)); + } + + let size = meta.len() as usize; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size)?; + + f.read_to_end(&mut bytes).await.map_err(os_err_to_file_err)?; + + let modtime = match meta.modified() { + Ok(md) => Some(OffsetDateTime::from(md)), + Err(_) => None, + }; + + Ok((bytes, modtime)) } async fn read_all_data(&self, volume: &str, volume_dir: impl AsRef, file_path: impl AsRef) -> Result> { From 03e3c8fb700a02b3f599a42de1b0df52ac78ba51 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 26 Sep 2024 23:45:29 +0800 Subject: [PATCH 68/70] opt: rename_data --- TODO.md | 1 + .../generated/flatbuffers_generated/models.rs | 207 ++- .../src/generated/proto_gen/node_service.rs | 1379 +++++++++++++---- ecstore/src/disk/local.rs | 583 ++++--- ecstore/src/disk/os.rs | 10 +- ecstore/src/utils/fs.rs | 4 + 6 files changed, 1561 insertions(+), 623 deletions(-) diff --git a/TODO.md b/TODO.md index 15472d274..cbcab874a 100644 --- a/TODO.md +++ b/TODO.md @@ -13,6 +13,7 @@ - [x] 远程rpc - [x] 错误类型判断,程序中判断错误类型，如何统一错误 - [x] 优化xlmeta, 自定义msg数据结构 +- [x] appendFile, createFile, readFile, walk_dir sync io ## 基础功能 diff --git a/common/protos/src/generated/flatbuffers_generated/models.rs b/common/protos/src/generated/flatbuffers_generated/models.rs index e4949fdcf..aa1f6ae2f 100644 --- a/common/protos/src/generated/flatbuffers_generated/models.rs +++ b/common/protos/src/generated/flatbuffers_generated/models.rs @@ -1,9 +1,10 @@ // automatically generated by the FlatBuffers compiler, do not modify + // @generated -use core::cmp::Ordering; use core::mem; +use core::cmp::Ordering; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow}; #[allow(unused_imports, dead_code)] pub mod models { - use core::cmp::Ordering; - use core::mem; + use core::mem; + use core::cmp::Ordering; - extern crate flatbuffers; - use self::flatbuffers::{EndianScalar, Follow}; + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; - pub enum PingBodyOffset {} - #[derive(Copy, Clone, PartialEq)] +pub enum PingBodyOffset {} +#[derive(Copy, Clone, PartialEq)] - pub struct PingBody<'a> { - pub _tab: flatbuffers::Table<'a>, +pub struct PingBody<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { + type Inner = PingBody<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { _tab: flatbuffers::Table::new(buf, loc) } + } +} + +impl<'a> PingBody<'a> { + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + pub const fn get_fully_qualified_name() -> &'static str { + "models.PingBody" + } + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PingBody { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args PingBodyArgs<'args> + ) -> flatbuffers::WIPOffset> { + let mut builder = PingBodyBuilder::new(_fbb); + if let Some(x) = args.payload { builder.add_payload(x); } + builder.finish() + } + + + #[inline] + pub fn payload(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::>>(PingBody::VT_PAYLOAD, None)} + } +} + +impl flatbuffers::Verifiable for PingBody<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, pos: usize + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>("payload", Self::VT_PAYLOAD, false)? + .finish(); + Ok(()) + } +} +pub struct PingBodyArgs<'a> { + pub payload: Option>>, +} +impl<'a> Default for PingBodyArgs<'a> { + #[inline] + fn default() -> Self { + PingBodyArgs { + payload: None, } + } +} - impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { - type Inner = PingBody<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { - _tab: flatbuffers::Table::new(buf, loc), - } - } +pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { + #[inline] + pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(PingBody::VT_PAYLOAD, payload); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + PingBodyBuilder { + fbb_: _fbb, + start_: start, } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} - impl<'a> PingBody<'a> { - pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; +impl core::fmt::Debug for PingBody<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("PingBody"); + ds.field("payload", &self.payload()); + ds.finish() + } +} +} // pub mod models - pub const fn get_fully_qualified_name() -> &'static str { - "models.PingBody" - } - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - PingBody { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args PingBodyArgs<'args>, - ) -> flatbuffers::WIPOffset> { - let mut builder = PingBodyBuilder::new(_fbb); - if let Some(x) = args.payload { - builder.add_payload(x); - } - builder.finish() - } - - #[inline] - pub fn payload(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { - self._tab - .get::>>(PingBody::VT_PAYLOAD, None) - } - } - } - - impl flatbuffers::Verifiable for PingBody<'_> { - #[inline] - fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>>("payload", Self::VT_PAYLOAD, false)? - .finish(); - Ok(()) - } - } - pub struct PingBodyArgs<'a> { - pub payload: Option>>, - } - impl<'a> Default for PingBodyArgs<'a> { - #[inline] - fn default() -> Self { - PingBodyArgs { payload: None } - } - } - - pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, - } - impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { - #[inline] - pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { - self.fbb_ - .push_slot_always::>(PingBody::VT_PAYLOAD, payload); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - PingBodyBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } - } - - impl core::fmt::Debug for PingBody<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("PingBody"); - ds.field("payload", &self.payload()); - ds.finish() - } - } -} // pub mod models diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index 1910283c4..76a704908 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -566,8 +566,8 @@ pub struct DeleteVolumeResponse { /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] - use tonic::codegen::http::Uri; use tonic::codegen::*; + use tonic::codegen::http::Uri; #[derive(Debug, Clone)] pub struct NodeServiceClient { inner: tonic::client::Grpc, @@ -598,15 +598,22 @@ pub mod node_service_client { let inner = tonic::client::Grpc::with_origin(inner, origin); Self { inner } } - pub fn with_interceptor(inner: T, interceptor: F) -> NodeServiceClient> + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> NodeServiceClient> where F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< http::Request, - Response = http::Response<>::ResponseBody>, + Response = http::Response< + >::ResponseBody, + >, >, - >>::Error: Into + Send + Sync, + , + >>::Error: Into + Send + Sync, { NodeServiceClient::new(InterceptedService::new(inner, interceptor)) } @@ -649,9 +656,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Ping"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Ping", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Ping")); @@ -660,13 +674,23 @@ pub mod node_service_client { pub async fn list_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListBucket")); @@ -675,13 +699,23 @@ pub mod node_service_client { pub async fn make_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); @@ -690,13 +724,23 @@ pub mod node_service_client { pub async fn get_bucket_info( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetBucketInfo"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/GetBucketInfo", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "GetBucketInfo")); @@ -705,13 +749,23 @@ pub mod node_service_client { pub async fn delete_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteBucket")); @@ -720,13 +774,23 @@ pub mod node_service_client { pub async fn read_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAll")); @@ -735,13 +799,23 @@ pub mod node_service_client { pub async fn write_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteAll")); @@ -754,9 +828,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Delete"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Delete", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Delete")); @@ -765,13 +846,23 @@ pub mod node_service_client { pub async fn rename_part( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenamePart"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenamePart", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenamePart")); @@ -780,13 +871,23 @@ pub mod node_service_client { pub async fn rename_file( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameFile"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameFile", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameFile")); @@ -799,9 +900,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Write"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Write", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Write")); @@ -815,9 +923,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAt"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAt", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAt")); @@ -826,13 +941,23 @@ pub mod node_service_client { pub async fn list_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListDir")); @@ -841,13 +966,23 @@ pub mod node_service_client { pub async fn walk_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WalkDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WalkDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WalkDir")); @@ -856,13 +991,23 @@ pub mod node_service_client { pub async fn rename_data( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameData"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameData", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); @@ -871,13 +1016,23 @@ pub mod node_service_client { pub async fn make_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolumes")); @@ -886,13 +1041,23 @@ pub mod node_service_client { pub async fn make_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolume")); @@ -901,13 +1066,23 @@ pub mod node_service_client { pub async fn list_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListVolumes")); @@ -916,13 +1091,23 @@ pub mod node_service_client { pub async fn stat_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StatVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/StatVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "StatVolume")); @@ -931,13 +1116,23 @@ pub mod node_service_client { pub async fn delete_paths( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeletePaths"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeletePaths", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeletePaths")); @@ -946,13 +1141,23 @@ pub mod node_service_client { pub async fn update_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/UpdateMetadata"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/UpdateMetadata", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "UpdateMetadata")); @@ -961,13 +1166,23 @@ pub mod node_service_client { pub async fn write_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteMetadata"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteMetadata", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteMetadata")); @@ -976,13 +1191,23 @@ pub mod node_service_client { pub async fn read_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadVersion"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadVersion", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); @@ -995,9 +1220,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadXL"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadXL", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); @@ -1006,13 +1238,23 @@ pub mod node_service_client { pub async fn delete_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersion"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVersion", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersion")); @@ -1021,13 +1263,23 @@ pub mod node_service_client { pub async fn delete_versions( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVersions"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVersions", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersions")); @@ -1036,13 +1288,23 @@ pub mod node_service_client { pub async fn read_multiple( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadMultiple"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadMultiple", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadMultiple")); @@ -1051,13 +1313,23 @@ pub mod node_service_client { pub async fn delete_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVolume")); @@ -1080,19 +1352,31 @@ pub mod node_service_server { async fn list_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn get_bucket_info( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_all( &self, request: tonic::Request, @@ -1100,7 +1384,10 @@ pub mod node_service_server { async fn write_all( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete( &self, request: tonic::Request, @@ -1108,11 +1395,17 @@ pub mod node_service_server { async fn rename_part( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn rename_file( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write( &self, request: tonic::Request, @@ -1133,39 +1426,66 @@ pub mod node_service_server { async fn rename_data( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn list_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn stat_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_paths( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn update_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_version( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_xl( &self, request: tonic::Request, @@ -1173,19 +1493,31 @@ pub mod node_service_server { async fn delete_version( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_versions( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_multiple( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct NodeServiceServer { @@ -1208,7 +1540,10 @@ pub mod node_service_server { max_encoding_message_size: None, } } - pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1252,7 +1587,10 @@ pub mod node_service_server { type Response = http::Response; type Error = std::convert::Infallible; type Future = BoxFuture; - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { @@ -1260,12 +1598,21 @@ pub mod node_service_server { "/node_service.NodeService/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); - impl tonic::server::UnaryService for PingSvc { + impl tonic::server::UnaryService + for PingSvc { type Response = super::PingResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::ping(&inner, request).await }; + let fut = async move { + ::ping(&inner, request).await + }; Box::pin(fut) } } @@ -1278,8 +1625,14 @@ pub mod node_service_server { let method = PingSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1288,12 +1641,23 @@ pub mod node_service_server { "/node_service.NodeService/ListBucket" => { #[allow(non_camel_case_types)] struct ListBucketSvc(pub Arc); - impl tonic::server::UnaryService for ListBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListBucketSvc { type Response = super::ListBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_bucket(&inner, request).await }; + let fut = async move { + ::list_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1306,8 +1670,14 @@ pub mod node_service_server { let method = ListBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1316,12 +1686,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeBucket" => { #[allow(non_camel_case_types)] struct MakeBucketSvc(pub Arc); - impl tonic::server::UnaryService for MakeBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeBucketSvc { type Response = super::MakeBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_bucket(&inner, request).await }; + let fut = async move { + ::make_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1334,8 +1715,14 @@ pub mod node_service_server { let method = MakeBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1344,12 +1731,23 @@ pub mod node_service_server { "/node_service.NodeService/GetBucketInfo" => { #[allow(non_camel_case_types)] struct GetBucketInfoSvc(pub Arc); - impl tonic::server::UnaryService for GetBucketInfoSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for GetBucketInfoSvc { type Response = super::GetBucketInfoResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::get_bucket_info(&inner, request).await }; + let fut = async move { + ::get_bucket_info(&inner, request).await + }; Box::pin(fut) } } @@ -1362,8 +1760,14 @@ pub mod node_service_server { let method = GetBucketInfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1372,12 +1776,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteBucket" => { #[allow(non_camel_case_types)] struct DeleteBucketSvc(pub Arc); - impl tonic::server::UnaryService for DeleteBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteBucketSvc { type Response = super::DeleteBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_bucket(&inner, request).await }; + let fut = async move { + ::delete_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1390,8 +1805,14 @@ pub mod node_service_server { let method = DeleteBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1400,12 +1821,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadAll" => { #[allow(non_camel_case_types)] struct ReadAllSvc(pub Arc); - impl tonic::server::UnaryService for ReadAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadAllSvc { type Response = super::ReadAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_all(&inner, request).await }; + let fut = async move { + ::read_all(&inner, request).await + }; Box::pin(fut) } } @@ -1418,8 +1850,14 @@ pub mod node_service_server { let method = ReadAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1428,12 +1866,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteAll" => { #[allow(non_camel_case_types)] struct WriteAllSvc(pub Arc); - impl tonic::server::UnaryService for WriteAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteAllSvc { type Response = super::WriteAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_all(&inner, request).await }; + let fut = async move { + ::write_all(&inner, request).await + }; Box::pin(fut) } } @@ -1446,8 +1895,14 @@ pub mod node_service_server { let method = WriteAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1456,12 +1911,23 @@ pub mod node_service_server { "/node_service.NodeService/Delete" => { #[allow(non_camel_case_types)] struct DeleteSvc(pub Arc); - impl tonic::server::UnaryService for DeleteSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteSvc { type Response = super::DeleteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete(&inner, request).await }; + let fut = async move { + ::delete(&inner, request).await + }; Box::pin(fut) } } @@ -1474,8 +1940,14 @@ pub mod node_service_server { let method = DeleteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1484,12 +1956,23 @@ pub mod node_service_server { "/node_service.NodeService/RenamePart" => { #[allow(non_camel_case_types)] struct RenamePartSvc(pub Arc); - impl tonic::server::UnaryService for RenamePartSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenamePartSvc { type Response = super::RenamePartResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_part(&inner, request).await }; + let fut = async move { + ::rename_part(&inner, request).await + }; Box::pin(fut) } } @@ -1502,8 +1985,14 @@ pub mod node_service_server { let method = RenamePartSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1512,12 +2001,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameFile" => { #[allow(non_camel_case_types)] struct RenameFileSvc(pub Arc); - impl tonic::server::UnaryService for RenameFileSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameFileSvc { type Response = super::RenameFileResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_file(&inner, request).await }; + let fut = async move { + ::rename_file(&inner, request).await + }; Box::pin(fut) } } @@ -1530,8 +2030,14 @@ pub mod node_service_server { let method = RenameFileSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1540,12 +2046,21 @@ pub mod node_service_server { "/node_service.NodeService/Write" => { #[allow(non_camel_case_types)] struct WriteSvc(pub Arc); - impl tonic::server::UnaryService for WriteSvc { + impl tonic::server::UnaryService + for WriteSvc { type Response = super::WriteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write(&inner, request).await }; + let fut = async move { + ::write(&inner, request).await + }; Box::pin(fut) } } @@ -1558,8 +2073,14 @@ pub mod node_service_server { let method = WriteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1568,12 +2089,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadAt" => { #[allow(non_camel_case_types)] struct ReadAtSvc(pub Arc); - impl tonic::server::UnaryService for ReadAtSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadAtSvc { type Response = super::ReadAtResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_at(&inner, request).await }; + let fut = async move { + ::read_at(&inner, request).await + }; Box::pin(fut) } } @@ -1586,8 +2118,14 @@ pub mod node_service_server { let method = ReadAtSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1596,12 +2134,23 @@ pub mod node_service_server { "/node_service.NodeService/ListDir" => { #[allow(non_camel_case_types)] struct ListDirSvc(pub Arc); - impl tonic::server::UnaryService for ListDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListDirSvc { type Response = super::ListDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_dir(&inner, request).await }; + let fut = async move { + ::list_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1614,8 +2163,14 @@ pub mod node_service_server { let method = ListDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1624,12 +2179,23 @@ pub mod node_service_server { "/node_service.NodeService/WalkDir" => { #[allow(non_camel_case_types)] struct WalkDirSvc(pub Arc); - impl tonic::server::UnaryService for WalkDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WalkDirSvc { type Response = super::WalkDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::walk_dir(&inner, request).await }; + let fut = async move { + ::walk_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1642,8 +2208,14 @@ pub mod node_service_server { let method = WalkDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1652,12 +2224,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameData" => { #[allow(non_camel_case_types)] struct RenameDataSvc(pub Arc); - impl tonic::server::UnaryService for RenameDataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameDataSvc { type Response = super::RenameDataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_data(&inner, request).await }; + let fut = async move { + ::rename_data(&inner, request).await + }; Box::pin(fut) } } @@ -1670,8 +2253,14 @@ pub mod node_service_server { let method = RenameDataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1680,12 +2269,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolumes" => { #[allow(non_camel_case_types)] struct MakeVolumesSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumesSvc { type Response = super::MakeVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volumes(&inner, request).await }; + let fut = async move { + ::make_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1698,8 +2298,14 @@ pub mod node_service_server { let method = MakeVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1708,12 +2314,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolume" => { #[allow(non_camel_case_types)] struct MakeVolumeSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumeSvc { type Response = super::MakeVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volume(&inner, request).await }; + let fut = async move { + ::make_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1726,8 +2343,14 @@ pub mod node_service_server { let method = MakeVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1736,12 +2359,23 @@ pub mod node_service_server { "/node_service.NodeService/ListVolumes" => { #[allow(non_camel_case_types)] struct ListVolumesSvc(pub Arc); - impl tonic::server::UnaryService for ListVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListVolumesSvc { type Response = super::ListVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_volumes(&inner, request).await }; + let fut = async move { + ::list_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1754,8 +2388,14 @@ pub mod node_service_server { let method = ListVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1764,12 +2404,23 @@ pub mod node_service_server { "/node_service.NodeService/StatVolume" => { #[allow(non_camel_case_types)] struct StatVolumeSvc(pub Arc); - impl tonic::server::UnaryService for StatVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for StatVolumeSvc { type Response = super::StatVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::stat_volume(&inner, request).await }; + let fut = async move { + ::stat_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1782,8 +2433,14 @@ pub mod node_service_server { let method = StatVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1792,12 +2449,23 @@ pub mod node_service_server { "/node_service.NodeService/DeletePaths" => { #[allow(non_camel_case_types)] struct DeletePathsSvc(pub Arc); - impl tonic::server::UnaryService for DeletePathsSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeletePathsSvc { type Response = super::DeletePathsResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_paths(&inner, request).await }; + let fut = async move { + ::delete_paths(&inner, request).await + }; Box::pin(fut) } } @@ -1810,8 +2478,14 @@ pub mod node_service_server { let method = DeletePathsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1820,12 +2494,23 @@ pub mod node_service_server { "/node_service.NodeService/UpdateMetadata" => { #[allow(non_camel_case_types)] struct UpdateMetadataSvc(pub Arc); - impl tonic::server::UnaryService for UpdateMetadataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for UpdateMetadataSvc { type Response = super::UpdateMetadataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::update_metadata(&inner, request).await }; + let fut = async move { + ::update_metadata(&inner, request).await + }; Box::pin(fut) } } @@ -1838,8 +2523,14 @@ pub mod node_service_server { let method = UpdateMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1848,12 +2539,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteMetadata" => { #[allow(non_camel_case_types)] struct WriteMetadataSvc(pub Arc); - impl tonic::server::UnaryService for WriteMetadataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteMetadataSvc { type Response = super::WriteMetadataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_metadata(&inner, request).await }; + let fut = async move { + ::write_metadata(&inner, request).await + }; Box::pin(fut) } } @@ -1866,8 +2568,14 @@ pub mod node_service_server { let method = WriteMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1876,12 +2584,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadVersion" => { #[allow(non_camel_case_types)] struct ReadVersionSvc(pub Arc); - impl tonic::server::UnaryService for ReadVersionSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadVersionSvc { type Response = super::ReadVersionResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_version(&inner, request).await }; + let fut = async move { + ::read_version(&inner, request).await + }; Box::pin(fut) } } @@ -1894,8 +2613,14 @@ pub mod node_service_server { let method = ReadVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1904,12 +2629,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadXL" => { #[allow(non_camel_case_types)] struct ReadXLSvc(pub Arc); - impl tonic::server::UnaryService for ReadXLSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadXLSvc { type Response = super::ReadXlResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_xl(&inner, request).await }; + let fut = async move { + ::read_xl(&inner, request).await + }; Box::pin(fut) } } @@ -1922,8 +2658,14 @@ pub mod node_service_server { let method = ReadXLSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1932,12 +2674,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVersion" => { #[allow(non_camel_case_types)] struct DeleteVersionSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVersionSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVersionSvc { type Response = super::DeleteVersionResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_version(&inner, request).await }; + let fut = async move { + ::delete_version(&inner, request).await + }; Box::pin(fut) } } @@ -1950,8 +2703,14 @@ pub mod node_service_server { let method = DeleteVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1960,12 +2719,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVersions" => { #[allow(non_camel_case_types)] struct DeleteVersionsSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVersionsSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVersionsSvc { type Response = super::DeleteVersionsResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_versions(&inner, request).await }; + let fut = async move { + ::delete_versions(&inner, request).await + }; Box::pin(fut) } } @@ -1978,8 +2748,14 @@ pub mod node_service_server { let method = DeleteVersionsSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1988,12 +2764,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadMultiple" => { #[allow(non_camel_case_types)] struct ReadMultipleSvc(pub Arc); - impl tonic::server::UnaryService for ReadMultipleSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadMultipleSvc { type Response = super::ReadMultipleResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_multiple(&inner, request).await }; + let fut = async move { + ::read_multiple(&inner, request).await + }; Box::pin(fut) } } @@ -2006,8 +2793,14 @@ pub mod node_service_server { let method = ReadMultipleSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -2016,12 +2809,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVolume" => { #[allow(non_camel_case_types)] struct DeleteVolumeSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVolumeSvc { type Response = super::DeleteVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_volume(&inner, request).await }; + let fut = async move { + ::delete_volume(&inner, request).await + }; Box::pin(fut) } } @@ -2034,21 +2838,34 @@ pub mod node_service_server { let method = DeleteVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; Box::pin(fut) } - _ => Box::pin(async move { - Ok(http::Response::builder() - .status(200) - .header("grpc-status", tonic::Code::Unimplemented as i32) - .header(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE) - .body(empty_body()) - .unwrap()) - }), + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", tonic::Code::Unimplemented as i32) + .header( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ) + .body(empty_body()) + .unwrap(), + ) + }) + } } } } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9eb470bae..b25ec2fd9 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -10,7 +10,7 @@ use crate::disk::error::{ }; use crate::disk::os::check_path_length; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; -use crate::utils::fs::{lstat, O_RDONLY}; +use crate::utils::fs::{lstat, O_APPEND, O_CREATE, O_RDONLY, O_WRONLY}; use crate::utils::path::{has_suffix, SLASH_SEPARATOR}; use crate::{ error::{Error, Result}, @@ -27,7 +27,7 @@ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; use tokio::sync::Mutex; -use tracing::{debug, warn}; +use tracing::{error, warn}; use uuid::Uuid; #[derive(Debug)] @@ -197,21 +197,6 @@ impl LocalDisk { // }) // } - pub async fn rename_all(&self, src_data_path: &PathBuf, dst_data_path: &PathBuf, skip: &PathBuf) -> Result<()> { - if !skip.starts_with(src_data_path) { - fs::create_dir_all(dst_data_path.parent().unwrap_or(Path::new("/"))).await?; - } - - // debug!( - // "rename_all from \n {:?} \n to \n {:?} \n skip:{:?}", - // &src_data_path, &dst_data_path, &skip - // ); - - fs::rename(&src_data_path, &dst_data_path).await?; - - Ok(()) - } - pub async fn move_to_trash(&self, delete_path: &PathBuf, _recursive: bool, _immediate_purge: bool) -> Result<()> { let trash_path = self.get_object_path(super::RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; if let Some(parent) = trash_path.parent() { @@ -332,6 +317,12 @@ impl LocalDisk { } } + async fn read_metadata(&self, file_path: impl AsRef) -> Result> { + // TODO: suport timeout + let (data, _) = self.read_metadata_with_dmtime(file_path.as_ref()).await?; + Ok(data) + } + // FIXME: read_metadata only suport async fn read_metadata_with_dmtime(&self, file_path: impl AsRef) -> Result<(Vec, Option)> { check_path_length(&file_path.as_ref().to_string_lossy().to_string())?; @@ -624,15 +615,6 @@ pub async fn read_file_metadata(p: impl AsRef) -> Result { Ok(meta) } -pub async fn check_volume_exists(p: impl AsRef) -> Result<()> { - fs::metadata(&p).await.map_err(|e| match e.kind() { - ErrorKind::NotFound => Error::from(DiskError::VolumeNotFound), - ErrorKind::PermissionDenied => Error::from(DiskError::FileAccessDenied), - _ => Error::from(e), - })?; - Ok(()) -} - fn skip_access_checks(p: impl AsRef) -> bool { let vols = [ super::RUSTFS_META_TMP_DELETED_BUCKET, @@ -761,29 +743,22 @@ impl DiskAPI for LocalDisk { } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { - let vol_path = self.get_bucket_path(volume)?; + let volume_dir = self.get_bucket_path(volume)?; if !skip_access_checks(volume) { - check_volume_exists(&vol_path).await?; + if let Err(e) = utils::fs::access(&volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } } - let fpath = self.get_object_path(volume, path)?; + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(file_path.to_string_lossy().to_string().as_str())?; - self.delete_file(&vol_path, &fpath, opt.recursive, opt.immediate).await?; - - // if opt.recursive { - // let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - // fs::create_dir_all(&trash_path).await?; - // fs::rename(&fpath, &trash_path).await?; - - // // TODO: immediate - - // return Ok(()); - // } - - // fs::remove_file(fpath).await?; + self.delete_file(&volume_dir, &file_path, opt.recursive, opt.immediate) + .await?; Ok(()) } + async fn rename_part(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str, meta: Vec) -> Result<()> { let src_volume_dir = self.get_bucket_path(src_volume)?; let dst_volume_dir = self.get_bucket_path(dst_volume)?; @@ -866,156 +841,220 @@ impl DiskAPI for LocalDisk { Ok(()) } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - let src_volume_path = self.get_bucket_path(src_volume)?; - let dst_volume_path = self.get_bucket_path(dst_volume)?; + let src_volume_dir = self.get_bucket_path(src_volume)?; + let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(src_volume) { - utils::fs::access(&src_volume_path).await.map_err(map_err_not_exists)?; + if let Err(e) = utils::fs::access(&src_volume_dir).await { + if os_is_not_exist(&e) { + return Err(Error::from(DiskError::VolumeNotFound)); + } else if is_sys_err_io(&e) { + return Err(Error::from(DiskError::FaultyDisk)); + } + + return Err(Error::new(e)); + } } if !skip_access_checks(dst_volume) { - utils::fs::access(&dst_volume_path).await.map_err(map_err_not_exists)?; + if let Err(e) = utils::fs::access(&dst_volume_dir).await { + if os_is_not_exist(&e) { + return Err(Error::from(DiskError::VolumeNotFound)); + } else if is_sys_err_io(&e) { + return Err(Error::from(DiskError::FaultyDisk)); + } + + return Err(Error::new(e)); + } } - let srcp = self.get_object_path(src_volume, src_path)?; - let dstp = self.get_object_path(dst_volume, dst_path)?; - - let src_is_dir = srcp.is_dir(); - let dst_is_dir = dstp.is_dir(); - if !src_is_dir && dst_is_dir || src_is_dir && !dst_is_dir { + let src_is_dir = has_suffix(&src_path, SLASH_SEPARATOR); + let dst_is_dir = has_suffix(&dst_path, SLASH_SEPARATOR); + if !(src_is_dir && dst_is_dir || !src_is_dir && !dst_is_dir) { return Err(Error::from(DiskError::FileAccessDenied)); } - // TODO: check path length + let src_file_path = src_volume_dir.join(Path::new(&src_path)); + check_path_length(src_file_path.to_string_lossy().to_string().as_str())?; + + let dst_file_path = dst_volume_dir.join(Path::new(&dst_path)); + check_path_length(dst_file_path.to_string_lossy().to_string().as_str())?; if src_is_dir { - // TODO: remove dst_dir - } + let meta_op = match lstat(&src_file_path).await { + Ok(meta) => Some(meta), + Err(e) => { + if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } - fs::create_dir_all(dstp.parent().unwrap_or_else(|| Path::new("."))).await?; - - let mut idx = 0; - loop { - if let Err(e) = fs::rename(&srcp, &dstp).await { - if e.kind() == ErrorKind::NotFound && idx == 0 { - idx += 1; - continue; + if !os_is_not_exist(&e) { + return Err(Error::new(e)); + } + None } }; - break; + if let Some(meta) = meta_op { + if !meta.is_dir() { + return Err(Error::new(DiskError::FileAccessDenied)); + } + } + + if let Err(e) = utils::fs::remove(&dst_file_path).await { + if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } else if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } + + return Err(Error::new(e)); + } } - if let Some(dir_path) = srcp.parent() { - self.delete_file(&src_volume_path, &PathBuf::from(dir_path), false, false) - .await?; + if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &dst_volume_dir).await { + if let Some(e) = err.to_io_err() { + if is_sys_err_not_empty(&e) || is_sys_err_not_dir(&e) { + return Err(Error::new(DiskError::FileAccessDenied)); + } + + return Err(os_err_to_file_err(e)); + } + + return Err(err); + } + + if let Some(parent) = src_file_path.parent() { + let _ = self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await; } Ok(()) } - async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { - let volpath = self.get_bucket_path(&volume)?; - // check exists - fs::metadata(&volpath).await.map_err(|e| match e.kind() { - ErrorKind::NotFound => Error::new(DiskError::VolumeNotFound), - _ => Error::new(e), - })?; - - let fpath = self.get_object_path(volume, path)?; - - debug!("CreateFile fpath: {:?}", fpath); - - if let Some(_dir_path) = fpath.parent() { - fs::create_dir_all(&_dir_path).await?; + // TODO: use io.reader + async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { + if !origvolume.is_empty() { + let origvolume_dir = self.get_bucket_path(&origvolume)?; + if !skip_access_checks(origvolume) { + if let Err(e) = utils::fs::access(origvolume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } + } } - let file = File::create(&fpath).await?; + let volume_dir = self.get_bucket_path(&volume)?; + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(file_path.to_string_lossy().to_string().as_str())?; - Ok(FileWriter::Local(LocalFileWriter::new(file))) - // Ok(FileWriter::new(file)) + // TODO: writeAllDirect io.copy + if let Some(parent) = file_path.parent() { + os::make_dir_all(parent, &volume_dir).await?; + } + let f = utils::fs::open_file(&file_path, O_CREATE | O_WRONLY) + .await + .map_err(os_err_to_file_err)?; - // let mut writer = BufWriter::new(file); - - // io::copy(&mut r, &mut writer).await?; + Ok(FileWriter::Local(LocalFileWriter::new(f))) // Ok(()) } // async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result { async fn append_file(&self, volume: &str, path: &str) -> Result { - let p = self.get_object_path(volume, path)?; - - if let Some(dir_path) = p.parent() { - fs::create_dir_all(&dir_path).await?; - } - - // debug!("append_file open {} {:?}", self.id(), &p); - - let file = File::options() - .read(true) - .create(true) - .write(true) - .append(true) - .open(&p) - .await?; - - Ok(FileWriter::Local(LocalFileWriter::new(file))) - // Ok(FileWriter::new(file)) - - // let mut writer = BufWriter::new(file); - - // io::copy(&mut r, &mut writer).await?; - - // debug!("append_file end {} {}", self.id(), path); - // io::copy(&mut r, &mut file).await?; - - // Ok(()) - } - async fn read_file(&self, volume: &str, path: &str) -> Result { - let p = self.get_object_path(volume, path)?; - - // debug!("read_file {:?}", &p); - let file = File::options().read(true).open(&p).await?; - - Ok(FileReader::Local(LocalFileReader::new(file))) - - // file.seek(SeekFrom::Start(offset as u64)).await?; - - // let mut buffer = vec![0; length]; - - // let bytes_read = file.read(&mut buffer).await?; - - // buffer.truncate(bytes_read); - - // Ok((buffer, bytes_read)) - } - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - let p = self.get_bucket_path(volume)?; - - let mut entries = fs::read_dir(&p).await?; - - let mut volumes = Vec::new(); - - while let Some(entry) = entries.next_entry().await? { - if let Ok(_metadata) = entry.metadata().await { - // if !metadata.is_dir() { - // continue; - // } - - let name = entry.file_name().to_string_lossy().to_string(); - - // let created = match metadata.created() { - // Ok(md) => OffsetDateTime::from(md), - // Err(_) => return Err(Error::msg("Not supported created on this platform")), - // }; - - volumes.push(name); + let volume_dir = self.get_bucket_path(&volume)?; + if !skip_access_checks(volume) { + if let Err(e) = utils::fs::access(&volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); } } - Ok(volumes) + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(file_path.to_string_lossy().to_string().as_str())?; + + let f = self.open_file(file_path, O_CREATE | O_APPEND | O_WRONLY, volume_dir).await?; + + Ok(FileWriter::Local(LocalFileWriter::new(f))) } + // TODO: io verifier + async fn read_file(&self, volume: &str, path: &str) -> Result { + let volume_dir = self.get_bucket_path(&volume)?; + if !skip_access_checks(volume) { + if let Err(e) = utils::fs::access(&volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } + } + + let file_path = volume_dir.join(Path::new(&path)); + check_path_length(file_path.to_string_lossy().to_string().as_str())?; + + let f = self.open_file(file_path, O_RDONLY, volume_dir).await.map_err(|err| { + if let Some(e) = err.to_io_err() { + if os_is_not_exist(&e) { + Error::new(DiskError::FileNotFound) + } else if os_is_permission(&e) { + Error::new(DiskError::FileAccessDenied) + } else if is_sys_err_not_dir(&e) { + Error::new(DiskError::FileAccessDenied) + } else if is_sys_err_io(&e) { + Error::new(DiskError::FaultyDisk) + } else if is_sys_err_too_many_files(&e) { + Error::new(DiskError::TooManyOpenFiles) + } else { + Error::new(e) + } + } else { + err + } + })?; + + Ok(FileReader::Local(LocalFileReader::new(f))) + } + + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { + if !origvolume.is_empty() { + let origvolume_dir = self.get_bucket_path(&origvolume)?; + if !skip_access_checks(origvolume) { + if let Err(e) = utils::fs::access(origvolume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } + } + } + + let volume_dir = self.get_bucket_path(&volume)?; + let dir_path_abs = volume_dir.join(Path::new(&dir_path)); + + let entries = match os::read_dir(&dir_path_abs, count).await { + Ok(res) => res, + Err(e) => { + if DiskError::FileNotFound.is(&e) && !skip_access_checks(&volume) { + if let Err(e) = utils::fs::access(&volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } + } + + return Err(e); + } + }; + Ok(entries) + } + + // TODO: io.writer async fn walk_dir(&self, opts: WalkDirOptions) -> Result> { - let mut entries = self.list_dir("", &opts.bucket, &opts.base_dir, -1).await?; + let mut entries = match self.list_dir("", &opts.bucket, &opts.base_dir, -1).await { + Ok(res) => res, + Err(e) => { + if !DiskError::VolumeNotFound.is(&e) && !DiskError::FileNotFound.is(&e) { + error!("list_dir err {:?}", &e); + } + + if opts.report_notfound && DiskError::FileNotFound.is(&e) { + return Err(Error::new(DiskError::FileNotFound)); + } + return Ok(Vec::new()); + } + }; + + if entries.is_empty() { + return Ok(Vec::new()); + } entries.sort(); @@ -1046,12 +1085,7 @@ impl DiskAPI for LocalDisk { let fpath = self.get_object_path(bucket, format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE).as_str())?; - let (fdata, _) = match self.read_metadata_with_dmtime(&fpath).await { - Ok(res) => res, - Err(_) => (Vec::new(), None), - }; - - meta.metadata = fdata; + meta.metadata = self.read_metadata(&fpath).await.unwrap_or_default(); metas.push(meta); } @@ -1068,107 +1102,196 @@ impl DiskAPI for LocalDisk { dst_volume: &str, dst_path: &str, ) -> Result { - let src_volume_path = self.get_bucket_path(src_volume)?; + let src_volume_dir = self.get_bucket_path(src_volume)?; if !skip_access_checks(src_volume) { - check_volume_exists(&src_volume_path).await?; + if let Err(e) = utils::fs::access(&src_volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } } - let dst_volume_path = self.get_bucket_path(dst_volume)?; + let dst_volume_dir = self.get_bucket_path(dst_volume)?; if !skip_access_checks(dst_volume) { - check_volume_exists(&dst_volume_path).await?; + if let Err(e) = utils::fs::access(&dst_volume_dir).await { + return Err(convert_access_error(e, DiskError::VolumeAccessDenied)); + } } // xl.meta路径 - let src_file_path = self.get_object_path(src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str())?; - let dst_file_path = self.get_object_path(dst_volume, format!("{}/{}", &dst_path, super::STORAGE_FORMAT_FILE).as_str())?; + let src_file_path = src_volume_dir.join(Path::new(format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str())); + let dst_file_path = dst_volume_dir.join(Path::new(format!("{}/{}", &dst_path, super::STORAGE_FORMAT_FILE).as_str())); // data_dir 路径 - let (src_data_path, dst_data_path) = { - let mut data_dir = String::new(); - if !fi.is_remote() { - data_dir = utils::path::retain_slash(fi.data_dir.unwrap_or(Uuid::nil()).to_string().as_str()); - } + let has_data_dir_path = { + let has_data_dir = { + if !fi.is_remote() { + if let Some(dir) = fi.data_dir { + Some(utils::path::retain_slash(dir.to_string().as_str())) + } else { + None + } + } else { + None + } + }; - if !data_dir.is_empty() { - let src_data_path = self.get_object_path( - src_volume, + if let Some(data_dir) = has_data_dir { + let src_data_path = src_volume_dir.join(Path::new( utils::path::retain_slash(format!("{}/{}", &src_path, data_dir).as_str()).as_str(), - )?; - let dst_data_path = self.get_object_path(dst_volume, format!("{}/{}", &dst_path, data_dir).as_str())?; + )); + let dst_data_path = dst_volume_dir.join(Path::new( + utils::path::retain_slash(format!("{}/{}", &dst_path, data_dir).as_str()).as_str(), + )); - (src_data_path, dst_data_path) + Some((src_data_path, dst_data_path)) } else { - (PathBuf::new(), PathBuf::new()) + None } }; - // 读旧xl.meta - let mut meta = FileMeta::new(); + check_path_length(&src_file_path.to_string_lossy().to_string().as_str())?; + check_path_length(&dst_file_path.to_string_lossy().to_string().as_str())?; - let (dst_buf, _) = read_file_exists(&dst_file_path).await?; - if !dst_buf.is_empty() { - // 有旧文件，加载 - match meta.unmarshal_msg(&dst_buf) { - Ok(_) => {} - Err(_) => meta = FileMeta::new(), + // 读旧xl.meta + + let has_dst_buf = match utils::fs::read_file(&dst_file_path).await { + Ok(res) => Some(res), + Err(e) => { + if is_sys_err_not_dir(&e) && !cfg!(target_os = "windows") { + return Err(Error::new(DiskError::FileAccessDenied)); + } + + if !os_is_not_exist(&e) { + return Err(os_err_to_file_err(e)); + } + + None + } + }; + + // let current_data_path = dst_volume_dir.join(Path::new(&dst_path)); + + let mut xlmeta = FileMeta::new(); + + if let Some(dst_buf) = has_dst_buf.as_ref() { + if FileMeta::is_xl_format(&dst_buf) { + if let Ok(nmeta) = FileMeta::load(&dst_buf) { + xlmeta = nmeta + } } } - let mut skip_parent = dst_volume_path.clone(); - if !dst_buf.is_empty() { - skip_parent = PathBuf::from(&dst_file_path.parent().unwrap_or(Path::new("/"))); + let mut skip_parent = dst_volume_dir.clone(); + if has_dst_buf.as_ref().is_some() { + if let Some(parent) = dst_file_path.parent() { + skip_parent = parent.to_path_buf(); + } } - // 查找版本是否已存在 - let old_data_dir = meta - .find_version(fi.version_id) - .map(|(_, version)| { - version - .get_data_dir() - .filter(|data_dir| meta.shard_data_dir_count(&fi.version_id, &Some(data_dir.clone())) == 0) - }) - .unwrap_or_default(); + // TODO: Healing - // 添加版本，写入xl.meta文件 - meta.add_version(fi.clone())?; + let has_old_data_dir = { + if let Ok((_, ver)) = xlmeta.find_version(fi.version_id) { + let has_data_dir = ver.get_data_dir(); + if let Some(data_dir) = has_data_dir { + if xlmeta.shard_data_dir_count(&fi.version_id, &Some(data_dir.clone())) == 0 { + // TODO: Healing + // remove inlinedata\ + Some(data_dir) + } else { + None + } + } else { + None + } + } else { + None + } + }; - let fm_data = meta.marshal_msg()?; + xlmeta.add_version(fi.clone())?; - // 写入xl.meta - self.write_all(src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str(), fm_data) - .await?; - - let no_inline = src_data_path.has_root() && fi.data.is_none() && fi.size > 0; - if no_inline { - self.rename_all(&src_data_path, &dst_data_path, &skip_parent).await?; + if xlmeta.versions.len() <= 10 { + // TODO: Sign } - // warn!("old_data_dir {:?}", old_data_dir); - // 有旧目录，把old xl.meta存到旧目录里 - if old_data_dir.is_some() { - self.write_all( - &dst_volume, - format!("{}/{}/{}", &dst_path, &old_data_dir.unwrap().to_string(), super::STORAGE_FORMAT_FILE).as_str(), - dst_buf, - ) - .await?; + let new_dst_buf = xlmeta.marshal_msg()?; + + self.write_all(&src_volume, format!("{}/{}", &src_path, super::STORAGE_FORMAT_FILE).as_str(), new_dst_buf) + .await + .map_err(|err| { + if let Some(e) = err.to_io_err() { + os_err_to_file_err(e) + } else { + err + } + })?; + + if let Some((src_data_path, dst_data_path)) = has_data_dir_path.as_ref() { + let no_inline = fi.data.is_none() && fi.size > 0; + if no_inline { + if let Err(err) = os::rename_all(&src_data_path, &dst_data_path, &skip_parent).await { + let _ = self.delete_file(&dst_volume_dir, &dst_data_path, false, false).await; + + return Err({ + if let Some(e) = err.to_io_err() { + os_err_to_file_err(e) + } else { + err + } + }); + } + } } - if let Err(e) = self.rename_all(&src_file_path, &dst_file_path, &skip_parent).await { - // 如果 失败删除目标目录 - let _ = self.delete_file(&dst_volume_path, &dst_data_path, false, false).await; - return Err(e); + if let Some(old_data_dir) = has_old_data_dir { + // preserve current xl.meta inside the oldDataDir. + if let Some(dst_buf) = has_dst_buf { + if let Err(err) = self + .write_all_private( + &dst_volume, + format!("{}/{}/{}", &dst_path, &old_data_dir.to_string(), super::STORAGE_FORMAT_FILE).as_str(), + &dst_buf, + true, + &skip_parent, + ) + .await + { + return Err({ + if let Some(e) = err.to_io_err() { + os_err_to_file_err(e) + } else { + err + } + }); + } + } } - if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { - fs::remove_dir(&src_file_path.parent().unwrap()).await?; - } else { - self.delete_file(&src_volume_path, &PathBuf::from(src_file_path.parent().unwrap()), true, false) - .await?; + if let Err(err) = os::rename_all(&src_file_path, &dst_file_path, &skip_parent).await { + if let Some((_, dst_data_path)) = has_data_dir_path.as_ref() { + let _ = self.delete_file(&dst_volume_dir, &dst_data_path, false, false).await; + } + return Err({ + if let Some(e) = err.to_io_err() { + os_err_to_file_err(e) + } else { + err + } + }); + } + + if let Some(src_file_path_parent) = src_file_path.parent() { + if src_volume != super::RUSTFS_META_MULTIPART_BUCKET { + let _ = utils::fs::remove(src_file_path_parent).await; + } else { + let _ = self + .delete_file(&dst_volume_dir, &src_file_path_parent.to_path_buf(), true, false) + .await; + } } Ok(RenameDataResp { - old_data_dir: old_data_dir, + old_data_dir: has_old_data_dir, sign: None, // TODO: }) } @@ -1202,7 +1325,7 @@ impl DiskAPI for LocalDisk { async fn list_volumes(&self) -> Result> { let mut volumes = Vec::new(); - let entries = os::read_dir(&self.root, 0).await.map_err(|e| { + let entries = os::read_dir(&self.root, -1).await.map_err(|e| { if DiskError::FileAccessDenied.is(&e) { Error::new(DiskError::DiskAccessDenied) } else if DiskError::FileNotFound.is(&e) { diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 78d0ada5c..c00f4affe 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -68,18 +68,12 @@ pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> } // read_dir count read limit. when count == 0 unlimit. -pub async fn read_dir(path: impl AsRef, count: usize) -> Result> { +pub async fn read_dir(path: impl AsRef, count: i32) -> Result> { let mut entries = fs::read_dir(path.as_ref()).await?; let mut volumes = Vec::new(); - let mut count: i32 = { - if count == 0 { - -1 - } else { - count as i32 - } - }; + let mut count = count; while let Some(entry) = entries.next_entry().await? { let name = entry.file_name().to_string_lossy().to_string(); diff --git a/ecstore/src/utils/fs.rs b/ecstore/src/utils/fs.rs index afc13c5a3..a3012ea28 100644 --- a/ecstore/src/utils/fs.rs +++ b/ecstore/src/utils/fs.rs @@ -130,3 +130,7 @@ pub async fn mkdir(path: impl AsRef) -> io::Result<()> { pub async fn rename(from: impl AsRef, to: impl AsRef) -> io::Result<()> { fs::rename(from, to).await } + +pub async fn read_file(path: impl AsRef) -> io::Result> { + fs::read(path.as_ref()).await +} From 96c69ef2243e5832f50883987e14412ed0269655 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 27 Sep 2024 00:06:34 +0800 Subject: [PATCH 69/70] opt: rename_data --- ecstore/src/disk/local.rs | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index b25ec2fd9..6a3891fb0 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -1312,12 +1312,19 @@ impl DiskAPI for LocalDisk { return Err(Error::msg("Invalid arguments specified")); } - let p = self.get_bucket_path(volume)?; + let volume_dir = self.get_bucket_path(volume)?; - if let Err(e) = utils::fs::access(&p).await { + if let Err(e) = utils::fs::access(&volume_dir).await { if os_is_not_exist(&e) { - os::make_dir_all(&p, self.root.as_path()).await?; + os::make_dir_all(&volume_dir, self.root.as_path()).await?; } + if os_is_permission(&e) { + return Err(Error::new(DiskError::DiskAccessDenied)); + } else if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } + + return Err(Error::new(e)); } Err(Error::from(DiskError::VolumeExists)) @@ -1349,17 +1356,27 @@ impl DiskAPI for LocalDisk { Ok(volumes) } async fn stat_volume(&self, volume: &str) -> Result { - let p = self.get_bucket_path(volume)?; - - let m = read_file_metadata(&p).await?; - let modtime = match m.modified() { - Ok(md) => Some(OffsetDateTime::from(md)), - Err(_) => { - warn!("Not supported modified on this platform"); - None + let volume_dir = self.get_bucket_path(volume)?; + let meta = match utils::fs::lstat(&volume_dir).await { + Ok(res) => res, + Err(e) => { + if os_is_not_exist(&e) { + return Err(Error::new(DiskError::VolumeNotFound)); + } else if os_is_permission(&e) { + return Err(Error::new(DiskError::DiskAccessDenied)); + } else if is_sys_err_io(&e) { + return Err(Error::new(DiskError::FaultyDisk)); + } else { + return Err(Error::new(e)); + } } }; + let modtime = match meta.modified() { + Ok(md) => Some(OffsetDateTime::from(md)), + Err(_) => None, + }; + Ok(VolumeInfo { name: volume.to_string(), created: modtime, From 3d6172c544de976247e09108ffcd12c019a0c557 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 27 Sep 2024 09:05:38 +0800 Subject: [PATCH 70/70] opt --- ecstore/src/disk/local.rs | 21 +++++++++++++-------- ecstore/src/disk/os.rs | 9 --------- ecstore/src/error.rs | 6 +++++- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9eb470bae..73139387a 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -26,7 +26,7 @@ use std::{ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::{AsyncReadExt, AsyncWriteExt, ErrorKind}; -use tokio::sync::Mutex; +use tokio::sync::RwLock; use tracing::{debug, warn}; use uuid::Uuid; @@ -52,7 +52,7 @@ impl FormatInfo { pub struct LocalDisk { pub root: PathBuf, pub format_path: PathBuf, - pub format_info: Mutex, + pub format_info: RwLock, pub endpoint: Endpoint, // pub id: Mutex>, // pub format_data: Mutex>, @@ -106,7 +106,7 @@ impl LocalDisk { root, endpoint: ep.clone(), format_path: format_path, - format_info: Mutex::new(format_info), + format_info: RwLock::new(format_info), // // format_legacy, // format_file_info: Mutex::new(format_meta), // format_data: Mutex::new(format_data), @@ -499,7 +499,7 @@ impl LocalDisk { // write_all_public for trail async fn write_all_public(&self, volume: &str, path: &str, data: Vec) -> Result<()> { if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { - let mut format_info = self.format_info.lock().await; + let mut format_info = self.format_info.write().await; format_info.data = data.clone(); } @@ -685,8 +685,7 @@ impl DiskAPI for LocalDisk { } async fn get_disk_id(&self) -> Result> { - // TODO: check format file - let mut format_info = self.format_info.lock().await; + let mut format_info = self.format_info.write().await; let id = format_info.id.clone(); @@ -737,18 +736,24 @@ impl DiskAPI for LocalDisk { format_info.last_check = Some(OffsetDateTime::now_utc()); Ok(Some(disk_id)) - // TODO: 判断源文件id,是否有效 } async fn set_disk_id(&self, id: Option) -> Result<()> { // 本地不需要设置 - let mut format_info = self.format_info.lock().await; + // TODO: add check_id_store + let mut format_info = self.format_info.write().await; format_info.id = id; Ok(()) } #[must_use] async fn read_all(&self, volume: &str, path: &str) -> Result> { + if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE { + let format_info = self.format_info.read().await; + if format_info.data.len() > 0 { + return Ok(format_info.data.clone()); + } + } // TOFIX: let p = self.get_object_path(volume, path)?; let (data, _) = read_file_all(&p).await?; diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 78d0ada5c..eea5e0fcd 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -208,12 +208,3 @@ pub async fn os_mkdir_all(dir_path: impl AsRef, base_dir: impl AsRef Ok(()) } - -#[cfg(test)] -mod test { - - use super::*; - - #[tokio::test] - async fn test_make_dir() {} -} diff --git a/ecstore/src/error.rs b/ecstore/src/error.rs index 1fc8dfabf..db5b27561 100644 --- a/ecstore/src/error.rs +++ b/ecstore/src/error.rs @@ -98,7 +98,11 @@ impl Clone for Error { if let Some(e) = self.downcast_ref::() { clone_disk_err(e) } else if let Some(e) = self.downcast_ref::() { - Error::new(io::Error::new(e.kind(), e.to_string())) + if let Some(code) = e.raw_os_error() { + Error::new(io::Error::from_raw_os_error(code)) + } else { + Error::new(io::Error::new(e.kind(), e.to_string())) + } } else { // TODO: 优化其他类型 Error::msg(self.to_string())