diff --git a/Cargo.lock b/Cargo.lock index 8e444fe12..de4e2e711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,6 +509,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "e2e_test" version = "0.0.1" @@ -982,13 +993,142 @@ dependencies = [ ] [[package]] -name = "idna" -version = "0.5.0" +name = "icu_collections" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1075,6 +1215,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock" version = "0.0.1" @@ -2163,6 +2309,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -2177,9 +2329,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.86" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -2198,6 +2350,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.13.0" @@ -2213,18 +2376,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.66" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", @@ -2273,20 +2436,15 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.8.0" +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" dependencies = [ - "tinyvec_macros", + "displaydoc", + "zerovec", ] -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" version = "1.41.0" @@ -2553,27 +2711,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "untrusted" version = "0.9.0" @@ -2582,9 +2725,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", @@ -2597,6 +2740,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2824,12 +2979,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xxhash-rust" version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -2851,8 +3042,51 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 5ce87bc7f..a8e1f3dc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "c41ac1d30a0ae3d serde = { version = "1.0.214", features = ["derive"] } serde_json = "1.0.132" tempfile = "3.13.0" -thiserror = "1.0.66" +thiserror = "1.0.68" time = { version = "0.3.36", features = [ "std", "parsing", @@ -75,7 +75,7 @@ tracing = "0.1.40" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } transform-stream = "0.3.0" -url = "2.5.2" +url = "2.5.3" uuid = { version = "1.11.0", features = [ "v4", "fast-rng", diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index d77f1322d..622a345db 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -14,7 +14,7 @@ use crate::disk::error::{ convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir, map_err_not_exists, os_err_to_file_err, }; -use crate::disk::os::check_path_length; +use crate::disk::os::{check_path_length, is_empty_dir}; use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::error::{Error, Result}; use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold}; @@ -530,7 +530,7 @@ impl LocalDisk { for fi in fis { let data_dir = fm.delete_version(fi)?; - warn!("删除版本号 对应data_dir {:?}", &data_dir); + if data_dir.is_some() { let dir_path = self.get_object_path(volume, format!("{}/{}", path, data_dir.unwrap()).as_str())?; self.move_to_trash(&dir_path, true, false).await?; @@ -539,8 +539,6 @@ impl LocalDisk { // 没有版本了，删除xl.meta if fm.versions.is_empty() { - warn!("没有版本了，删除xl.meta"); - self.delete_file(&volume_dir, &xlpath, true, false).await?; return Ok(()); } @@ -1232,6 +1230,7 @@ impl DiskAPI for LocalDisk { Ok(FileReader::Local(LocalFileReader::new(f))) } + #[tracing::instrument(level = "debug", skip(self))] async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { if !origvolume.is_empty() { let origvolume_dir = self.get_bucket_path(origvolume)?; @@ -1262,6 +1261,26 @@ impl DiskAPI for LocalDisk { // TODO: io.writer async fn walk_dir(&self, opts: WalkDirOptions) -> Result> { + // warn!("walk_dir opts {:?}", &opts); + + let mut metas = Vec::new(); + + if opts.base_dir.ends_with(SLASH_SEPARATOR) { + let fpath = self.get_object_path( + &opts.bucket, + format!("{}/{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), STORAGE_FORMAT_FILE).as_str(), + )?; + if let Ok(data) = self.read_metadata(fpath).await { + let meta = MetaCacheEntry { + name: opts.base_dir.clone(), + metadata: data, + ..Default::default() + }; + metas.push(meta); + return Ok(metas); + } + } + let mut entries = match self.list_dir("", &opts.bucket, &opts.base_dir, -1).await { Ok(res) => res, Err(e) => { @@ -1287,12 +1306,12 @@ impl DiskAPI for LocalDisk { let bucket = opts.bucket.as_str(); - let mut metas = Vec::new(); - let mut dir_objes = HashSet::new(); // 第一层过滤 for entry in entries.iter() { + // warn!("walk_dir get entry {:?}", &entry); + // check limit if opts.limit > 0 && objs_returned >= opts.limit { return Ok(metas); @@ -1302,17 +1321,16 @@ impl DiskAPI for LocalDisk { continue; } - // warn!("walk_dir entry {}", entry); - let mut meta = MetaCacheEntry { ..Default::default() }; - let fpath = self.get_object_path(bucket, format!("{}/{}", &entry, STORAGE_FORMAT_FILE).as_str())?; + let mut name = { + if opts.base_dir.is_empty() { + entry.clone() + } else { + format!("{}{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), SLASH_SEPARATOR, entry) + } + }; - if let Ok(data) = self.read_metadata(&fpath).await { - meta.metadata = data; - } - - let mut name = entry.clone(); if name.ends_with(SLASH_SEPARATOR) { if name.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) { name = format!("{}{}", name.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR); @@ -1323,6 +1341,18 @@ impl DiskAPI for LocalDisk { } meta.name = name; + let fpath = self.get_object_path(bucket, format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE).as_str())?; + + if let Ok(data) = self.read_metadata(&fpath).await { + meta.metadata = data; + } else { + let fpath = self.get_object_path(bucket, &meta.name)?; + + if !is_empty_dir(fpath).await { + meta.name = format!("{}{}", &meta.name, SLASH_SEPARATOR); + } + } + metas.push(meta); } @@ -1674,8 +1704,6 @@ impl DiskAPI for LocalDisk { async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?; - warn!("write_metadata {:?} {:?}", &p, &fi); - let mut meta = FileMeta::new(); if !fi.fresh { let (buf, _) = read_file_exists(&p).await?; diff --git a/ecstore/src/disk/os.rs b/ecstore/src/disk/os.rs index 01d8ffecc..8375d608a 100644 --- a/ecstore/src/disk/os.rs +++ b/ecstore/src/disk/os.rs @@ -73,6 +73,10 @@ pub async fn make_dir_all(path: impl AsRef, base_dir: impl AsRef) -> Ok(()) } +pub async fn is_empty_dir(path: impl AsRef) -> bool { + read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty()) +} + // read_dir count read limit. when count == 0 unlimit. pub async fn read_dir(path: impl AsRef, count: i32) -> Result> { let mut entries = fs::read_dir(path.as_ref()).await?; diff --git a/ecstore/src/options.rs b/ecstore/src/options.rs index 8664afd0b..0751b3d9b 100644 --- a/ecstore/src/options.rs +++ b/ecstore/src/options.rs @@ -1,4 +1,3 @@ -use crate::bucket::metadata; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::error::{Error, Result}; use crate::store_api::ObjectOptions; @@ -7,7 +6,6 @@ use crate::utils::path::is_dir_object; use http::{HeaderMap, HeaderValue}; use lazy_static::lazy_static; use std::collections::HashMap; -use tracing::warn; use uuid::Uuid; pub async fn put_opts( @@ -70,9 +68,6 @@ fn get_default_opts( metadata: HashMap, _copy_source: bool, ) -> Result { - warn!("get headers: {:?}", &headers); - warn!("get metadata: {:?}", &metadata); - Ok(ObjectOptions { user_defined: metadata.clone(), ..Default::default() diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 871dfdafb..065327b8e 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -21,7 +21,7 @@ use crate::store_err::{ }; use crate::store_init::ec_drives_no_config; use crate::utils::crypto::base64_decode; -use crate::utils::path::{decode_dir_object, encode_dir_object, SLASH_SEPARATOR}; +use crate::utils::path::{base_dir_from_prefix, decode_dir_object, encode_dir_object, SLASH_SEPARATOR}; use crate::{ bucket::metadata::BucketMetadata, disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, @@ -55,7 +55,7 @@ use time::OffsetDateTime; use tokio::fs; use tokio::sync::{RwLock, Semaphore}; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use uuid::Uuid; const MAX_UPLOADS_LIST: usize = 10000; @@ -218,8 +218,18 @@ impl ECStore { self.pools.len() == 1 } - async fn list_path(&self, opts: &ListPathOptions) -> Result { - let objects = self.list_merged(opts).await?; + async fn list_path(&self, opts: &ListPathOptions, delimiter: &str) -> Result { + // if opts.prefix.ends_with(SLASH_SEPARATOR) { + // return Err(Error::msg("eof")); + // } + + let mut opts = opts.clone(); + + if opts.base_dir.is_empty() { + opts.base_dir = base_dir_from_prefix(&opts.prefix); + } + + let objects = self.list_merged(&opts, delimiter).await?; let info = ListObjectsInfo { objects, @@ -229,9 +239,10 @@ impl ECStore { } // 读所有 - async fn list_merged(&self, opts: &ListPathOptions) -> Result> { - let opts = WalkDirOptions { + async fn list_merged(&self, opts: &ListPathOptions, delimiter: &str) -> Result> { + let walk_opts = WalkDirOptions { bucket: opts.bucket.clone(), + base_dir: opts.base_dir.clone(), ..Default::default() }; @@ -241,7 +252,7 @@ impl ECStore { for sets in self.pools.iter() { for set in sets.disk_set.iter() { - futures.push(set.walk_dir(&opts)); + futures.push(set.walk_dir(&walk_opts)); } } @@ -260,14 +271,25 @@ impl ECStore { let entrys = disks_res.as_ref().unwrap(); for entry in entrys { + // warn!("lst_merged entry---- {}", &entry.name); + + if !opts.prefix.is_empty() && !entry.name.starts_with(&opts.prefix) { + continue; + } + if !uniq.contains(&entry.name) { uniq.insert(entry.name.clone()); // TODO: 过滤 + if opts.limit > 0 && ress.len() as i32 >= opts.limit { return Ok(ress); } if entry.is_object() { + if !delimiter.is_empty() { + // entry.name.trim_start_matches(pat) + } + let fi = entry.to_fileinfo(&opts.bucket)?; if let Some(f) = fi { ress.push(f.to_object_info(&opts.bucket, &entry.name, false)); @@ -703,7 +725,7 @@ pub struct PoolObjInfo { pub err: Option, } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct ListPathOptions { pub id: String, @@ -1108,9 +1130,9 @@ impl StorageAPI for ECStore { async fn list_objects_v2( &self, bucket: &str, - _prefix: &str, + prefix: &str, continuation_token: &str, - _delimiter: &str, + delimiter: &str, max_keys: i32, _fetch_owner: bool, _start_after: &str, @@ -1118,10 +1140,11 @@ impl StorageAPI for ECStore { let opts = ListPathOptions { bucket: bucket.to_string(), limit: max_keys, + prefix: prefix.to_owned(), ..Default::default() }; - let info = self.list_path(&opts).await?; + let info = self.list_path(&opts, delimiter).await?; // warn!("list_objects_v2 info {:?}", info); diff --git a/ecstore/src/utils/path.rs b/ecstore/src/utils/path.rs index 0da2ff7ec..573e75e2f 100644 --- a/ecstore/src/utils/path.rs +++ b/ecstore/src/utils/path.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__"; @@ -69,6 +69,20 @@ pub fn path_join(elem: &[PathBuf]) -> PathBuf { joined_path } +pub fn base_dir_from_prefix(prefix: &str) -> String { + let mut base_dir = dir(prefix).to_owned(); + if base_dir == "." || base_dir == "./" || base_dir == "/" { + base_dir = "".to_owned(); + } + if !prefix.contains('/') { + base_dir = "".to_owned(); + } + if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) { + base_dir.push_str(SLASH_SEPARATOR); + } + base_dir +} + pub struct LazyBuf { s: String, buf: Option>, @@ -184,10 +198,33 @@ pub fn clean(path: &str) -> String { out.string() } + +pub fn split(path: &str) -> (&str, &str) { + // Find the last occurrence of the '/' character + if let Some(i) = path.rfind('/') { + // Return the directory (up to and including the last '/') and the file name + return (&path[..i + 1], &path[i + 1..]); + } + // If no '/' is found, return an empty string for the directory and the whole path as the file name + (path, "") +} + +pub fn dir(path: &str) -> &str { + let (a, _) = split(path); + a +} #[cfg(test)] mod tests { use super::*; + #[test] + fn test_base_dir_from_prefix() { + let a = "da/"; + println!("---- in {}", a); + let a = base_dir_from_prefix(a); + println!("---- out {}", a); + } + #[test] fn test_clean() { assert_eq!(clean(""), "."); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 51fd48968..5ae401e43 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -16,7 +16,9 @@ use ecstore::bucket::policy_sys::PolicySys; use ecstore::bucket::tagging::decode_tags; use ecstore::bucket::tagging::encode_tags; use ecstore::bucket::versioning_sys::BucketVersioningSys; +use ecstore::disk::error::is_err_file_not_found; use ecstore::disk::error::DiskError; +use ecstore::error::Error as EcError; use ecstore::new_object_layer_fn; use ecstore::options::extract_metadata; use ecstore::options::put_opts; @@ -69,6 +71,15 @@ lazy_static! { id: Some("c19050dbcee97fda828689dda99097a6321af2248fa760517237346e5d9c8a66".to_owned()), }; } + +fn to_s3_error(err: EcError) -> S3Error { + if is_err_file_not_found(&err) { + return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err)); + } + + S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err)) +} + #[derive(Debug, Clone)] pub struct FS { // pub store: ECStore, @@ -392,7 +403,10 @@ impl S3 for FS { None => return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())), }; - let info = try_!(store.get_object_info(&bucket, &key, &ObjectOptions::default()).await); + let info = store + .get_object_info(&bucket, &key, &ObjectOptions::default()) + .await + .map_err(to_s3_error)?; debug!("info {:?}", info); let content_type = { @@ -468,6 +482,8 @@ impl S3 for FS { #[tracing::instrument(level = "debug", skip(self, req))] async fn list_objects_v2(&self, req: S3Request) -> S3Result> { + // warn!("list_objects_v2 input {:?}", &req.input); + let ListObjectsV2Input { bucket, continuation_token,