mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 08:38:58 +00:00
Merge branch 'main' into bool-api
This commit is contained in:
weisd
Generated
+272
-38
@@ -509,6 +509,17 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "0.0.1"
|
||||
@@ -982,13 +993,142 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
name = "icu_collections"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_locid_transform_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"write16",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_locid_transform",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_provider_macros",
|
||||
"stable_deref_trait",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_macros"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
|
||||
dependencies = [
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1075,6 +1215,12 @@ version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
|
||||
|
||||
[[package]]
|
||||
name = "lock"
|
||||
version = "0.0.1"
|
||||
@@ -2163,6 +2309,12 @@ version = "0.9.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
@@ -2177,9 +2329,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.86"
|
||||
version = "2.0.87"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e89275301d38033efb81a6e60e3497e734dfcc62571f2854bf4b16690398824c"
|
||||
checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2198,6 +2350,17 @@ version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.13.0"
|
||||
@@ -2213,18 +2376,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d171f59dbaa811dbbb1aee1e73db92ec2b122911a48e1390dfe327a821ddede"
|
||||
checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.66"
|
||||
version = "1.0.68"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b08be0f17bd307950653ce45db00cd31200d82b624b36e181337d9c7d92765b5"
|
||||
checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -2273,20 +2436,15 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.8.0"
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
|
||||
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.41.0"
|
||||
@@ -2553,27 +2711,12 @@ version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
@@ -2582,9 +2725,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.2"
|
||||
version = "2.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
|
||||
checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
@@ -2597,6 +2740,18 @@ version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf16_iter"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
@@ -2824,12 +2979,48 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
|
||||
|
||||
[[package]]
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.35"
|
||||
@@ -2851,8 +3042,51 @@ dependencies = [
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.10.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
+2
-2
@@ -57,7 +57,7 @@ s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "c41ac1d30a0ae3d
|
||||
serde = { version = "1.0.214", features = ["derive"] }
|
||||
serde_json = "1.0.132"
|
||||
tempfile = "3.13.0"
|
||||
thiserror = "1.0.66"
|
||||
thiserror = "1.0.68"
|
||||
time = { version = "0.3.36", features = [
|
||||
"std",
|
||||
"parsing",
|
||||
@@ -75,7 +75,7 @@ tracing = "0.1.40"
|
||||
tracing-error = "0.2.0"
|
||||
tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] }
|
||||
transform-stream = "0.3.0"
|
||||
url = "2.5.2"
|
||||
url = "2.5.3"
|
||||
uuid = { version = "1.11.0", features = [
|
||||
"v4",
|
||||
"fast-rng",
|
||||
|
||||
+44
-16
@@ -14,7 +14,7 @@ use crate::disk::error::{
|
||||
convert_access_error, is_sys_err_handle_invalid, is_sys_err_invalid_arg, is_sys_err_is_dir, is_sys_err_not_dir,
|
||||
map_err_not_exists, os_err_to_file_err,
|
||||
};
|
||||
use crate::disk::os::check_path_length;
|
||||
use crate::disk::os::{check_path_length, is_empty_dir};
|
||||
use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::global::{GLOBAL_IsErasureSD, GLOBAL_RootDiskThreshold};
|
||||
@@ -530,7 +530,7 @@ impl LocalDisk {
|
||||
|
||||
for fi in fis {
|
||||
let data_dir = fm.delete_version(fi)?;
|
||||
warn!("删除版本号 对应data_dir {:?}", &data_dir);
|
||||
|
||||
if data_dir.is_some() {
|
||||
let dir_path = self.get_object_path(volume, format!("{}/{}", path, data_dir.unwrap()).as_str())?;
|
||||
self.move_to_trash(&dir_path, true, false).await?;
|
||||
@@ -539,8 +539,6 @@ impl LocalDisk {
|
||||
|
||||
// 没有版本了，删除xl.meta
|
||||
if fm.versions.is_empty() {
|
||||
warn!("没有版本了，删除xl.meta");
|
||||
|
||||
self.delete_file(&volume_dir, &xlpath, true, false).await?;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1232,6 +1230,7 @@ impl DiskAPI for LocalDisk {
|
||||
Ok(FileReader::Local(LocalFileReader::new(f)))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
|
||||
if !origvolume.is_empty() {
|
||||
let origvolume_dir = self.get_bucket_path(origvolume)?;
|
||||
@@ -1262,6 +1261,26 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
// TODO: io.writer
|
||||
async fn walk_dir(&self, opts: WalkDirOptions) -> Result<Vec<MetaCacheEntry>> {
|
||||
// warn!("walk_dir opts {:?}", &opts);
|
||||
|
||||
let mut metas = Vec::new();
|
||||
|
||||
if opts.base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
let fpath = self.get_object_path(
|
||||
&opts.bucket,
|
||||
format!("{}/{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), STORAGE_FORMAT_FILE).as_str(),
|
||||
)?;
|
||||
if let Ok(data) = self.read_metadata(fpath).await {
|
||||
let meta = MetaCacheEntry {
|
||||
name: opts.base_dir.clone(),
|
||||
metadata: data,
|
||||
..Default::default()
|
||||
};
|
||||
metas.push(meta);
|
||||
return Ok(metas);
|
||||
}
|
||||
}
|
||||
|
||||
let mut entries = match self.list_dir("", &opts.bucket, &opts.base_dir, -1).await {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
@@ -1287,12 +1306,12 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
let bucket = opts.bucket.as_str();
|
||||
|
||||
let mut metas = Vec::new();
|
||||
|
||||
let mut dir_objes = HashSet::new();
|
||||
|
||||
// 第一层过滤
|
||||
for entry in entries.iter() {
|
||||
// warn!("walk_dir get entry {:?}", &entry);
|
||||
|
||||
// check limit
|
||||
if opts.limit > 0 && objs_returned >= opts.limit {
|
||||
return Ok(metas);
|
||||
@@ -1302,17 +1321,16 @@ impl DiskAPI for LocalDisk {
|
||||
continue;
|
||||
}
|
||||
|
||||
// warn!("walk_dir entry {}", entry);
|
||||
|
||||
let mut meta = MetaCacheEntry { ..Default::default() };
|
||||
|
||||
let fpath = self.get_object_path(bucket, format!("{}/{}", &entry, STORAGE_FORMAT_FILE).as_str())?;
|
||||
let mut name = {
|
||||
if opts.base_dir.is_empty() {
|
||||
entry.clone()
|
||||
} else {
|
||||
format!("{}{}{}", opts.base_dir.trim_end_matches(SLASH_SEPARATOR), SLASH_SEPARATOR, entry)
|
||||
}
|
||||
};
|
||||
|
||||
if let Ok(data) = self.read_metadata(&fpath).await {
|
||||
meta.metadata = data;
|
||||
}
|
||||
|
||||
let mut name = entry.clone();
|
||||
if name.ends_with(SLASH_SEPARATOR) {
|
||||
if name.ends_with(GLOBAL_DIR_SUFFIX_WITH_SLASH) {
|
||||
name = format!("{}{}", name.as_str().trim_end_matches(GLOBAL_DIR_SUFFIX_WITH_SLASH), SLASH_SEPARATOR);
|
||||
@@ -1323,6 +1341,18 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
meta.name = name;
|
||||
|
||||
let fpath = self.get_object_path(bucket, format!("{}/{}", &meta.name, STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
if let Ok(data) = self.read_metadata(&fpath).await {
|
||||
meta.metadata = data;
|
||||
} else {
|
||||
let fpath = self.get_object_path(bucket, &meta.name)?;
|
||||
|
||||
if !is_empty_dir(fpath).await {
|
||||
meta.name = format!("{}{}", &meta.name, SLASH_SEPARATOR);
|
||||
}
|
||||
}
|
||||
|
||||
metas.push(meta);
|
||||
}
|
||||
|
||||
@@ -1674,8 +1704,6 @@ impl DiskAPI for LocalDisk {
|
||||
async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> {
|
||||
let p = self.get_object_path(volume, format!("{}/{}", path, super::STORAGE_FORMAT_FILE).as_str())?;
|
||||
|
||||
warn!("write_metadata {:?} {:?}", &p, &fi);
|
||||
|
||||
let mut meta = FileMeta::new();
|
||||
if !fi.fresh {
|
||||
let (buf, _) = read_file_exists(&p).await?;
|
||||
|
||||
@@ -73,6 +73,10 @@ pub async fn make_dir_all(path: impl AsRef<Path>, base_dir: impl AsRef<Path>) ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn is_empty_dir(path: impl AsRef<Path>) -> bool {
|
||||
read_dir(path.as_ref(), 1).await.is_ok_and(|v| v.is_empty())
|
||||
}
|
||||
|
||||
// read_dir count read limit. when count == 0 unlimit.
|
||||
pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> Result<Vec<String>> {
|
||||
let mut entries = fs::read_dir(path.as_ref()).await?;
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use crate::bucket::metadata;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store_api::ObjectOptions;
|
||||
@@ -7,7 +6,6 @@ use crate::utils::path::is_dir_object;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use lazy_static::lazy_static;
|
||||
use std::collections::HashMap;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub async fn put_opts(
|
||||
@@ -70,9 +68,6 @@ fn get_default_opts(
|
||||
metadata: HashMap<String, String>,
|
||||
_copy_source: bool,
|
||||
) -> Result<ObjectOptions> {
|
||||
warn!("get headers: {:?}", &headers);
|
||||
warn!("get metadata: {:?}", &metadata);
|
||||
|
||||
Ok(ObjectOptions {
|
||||
user_defined: metadata.clone(),
|
||||
..Default::default()
|
||||
|
||||
+34
-11
@@ -21,7 +21,7 @@ use crate::store_err::{
|
||||
};
|
||||
use crate::store_init::ec_drives_no_config;
|
||||
use crate::utils::crypto::base64_decode;
|
||||
use crate::utils::path::{decode_dir_object, encode_dir_object, SLASH_SEPARATOR};
|
||||
use crate::utils::path::{base_dir_from_prefix, decode_dir_object, encode_dir_object, SLASH_SEPARATOR};
|
||||
use crate::{
|
||||
bucket::metadata::BucketMetadata,
|
||||
disk::{error::DiskError, new_disk, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET},
|
||||
@@ -55,7 +55,7 @@ use time::OffsetDateTime;
|
||||
use tokio::fs;
|
||||
use tokio::sync::{RwLock, Semaphore};
|
||||
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
const MAX_UPLOADS_LIST: usize = 10000;
|
||||
@@ -218,8 +218,18 @@ impl ECStore {
|
||||
self.pools.len() == 1
|
||||
}
|
||||
|
||||
async fn list_path(&self, opts: &ListPathOptions) -> Result<ListObjectsInfo> {
|
||||
let objects = self.list_merged(opts).await?;
|
||||
async fn list_path(&self, opts: &ListPathOptions, delimiter: &str) -> Result<ListObjectsInfo> {
|
||||
// if opts.prefix.ends_with(SLASH_SEPARATOR) {
|
||||
// return Err(Error::msg("eof"));
|
||||
// }
|
||||
|
||||
let mut opts = opts.clone();
|
||||
|
||||
if opts.base_dir.is_empty() {
|
||||
opts.base_dir = base_dir_from_prefix(&opts.prefix);
|
||||
}
|
||||
|
||||
let objects = self.list_merged(&opts, delimiter).await?;
|
||||
|
||||
let info = ListObjectsInfo {
|
||||
objects,
|
||||
@@ -229,9 +239,10 @@ impl ECStore {
|
||||
}
|
||||
|
||||
// 读所有
|
||||
async fn list_merged(&self, opts: &ListPathOptions) -> Result<Vec<ObjectInfo>> {
|
||||
let opts = WalkDirOptions {
|
||||
async fn list_merged(&self, opts: &ListPathOptions, delimiter: &str) -> Result<Vec<ObjectInfo>> {
|
||||
let walk_opts = WalkDirOptions {
|
||||
bucket: opts.bucket.clone(),
|
||||
base_dir: opts.base_dir.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -241,7 +252,7 @@ impl ECStore {
|
||||
|
||||
for sets in self.pools.iter() {
|
||||
for set in sets.disk_set.iter() {
|
||||
futures.push(set.walk_dir(&opts));
|
||||
futures.push(set.walk_dir(&walk_opts));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -260,14 +271,25 @@ impl ECStore {
|
||||
let entrys = disks_res.as_ref().unwrap();
|
||||
|
||||
for entry in entrys {
|
||||
// warn!("lst_merged entry---- {}", &entry.name);
|
||||
|
||||
if !opts.prefix.is_empty() && !entry.name.starts_with(&opts.prefix) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !uniq.contains(&entry.name) {
|
||||
uniq.insert(entry.name.clone());
|
||||
// TODO: 过滤
|
||||
|
||||
if opts.limit > 0 && ress.len() as i32 >= opts.limit {
|
||||
return Ok(ress);
|
||||
}
|
||||
|
||||
if entry.is_object() {
|
||||
if !delimiter.is_empty() {
|
||||
// entry.name.trim_start_matches(pat)
|
||||
}
|
||||
|
||||
let fi = entry.to_fileinfo(&opts.bucket)?;
|
||||
if let Some(f) = fi {
|
||||
ress.push(f.to_object_info(&opts.bucket, &entry.name, false));
|
||||
@@ -703,7 +725,7 @@ pub struct PoolObjInfo {
|
||||
pub err: Option<Error>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct ListPathOptions {
|
||||
pub id: String,
|
||||
|
||||
@@ -1108,9 +1130,9 @@ impl StorageAPI for ECStore {
|
||||
async fn list_objects_v2(
|
||||
&self,
|
||||
bucket: &str,
|
||||
_prefix: &str,
|
||||
prefix: &str,
|
||||
continuation_token: &str,
|
||||
_delimiter: &str,
|
||||
delimiter: &str,
|
||||
max_keys: i32,
|
||||
_fetch_owner: bool,
|
||||
_start_after: &str,
|
||||
@@ -1118,10 +1140,11 @@ impl StorageAPI for ECStore {
|
||||
let opts = ListPathOptions {
|
||||
bucket: bucket.to_string(),
|
||||
limit: max_keys,
|
||||
prefix: prefix.to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let info = self.list_path(&opts).await?;
|
||||
let info = self.list_path(&opts, delimiter).await?;
|
||||
|
||||
// warn!("list_objects_v2 info {:?}", info);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const GLOBAL_DIR_SUFFIX: &str = "__XLDIR__";
|
||||
|
||||
@@ -69,6 +69,20 @@ pub fn path_join(elem: &[PathBuf]) -> PathBuf {
|
||||
joined_path
|
||||
}
|
||||
|
||||
pub fn base_dir_from_prefix(prefix: &str) -> String {
|
||||
let mut base_dir = dir(prefix).to_owned();
|
||||
if base_dir == "." || base_dir == "./" || base_dir == "/" {
|
||||
base_dir = "".to_owned();
|
||||
}
|
||||
if !prefix.contains('/') {
|
||||
base_dir = "".to_owned();
|
||||
}
|
||||
if !base_dir.is_empty() && !base_dir.ends_with(SLASH_SEPARATOR) {
|
||||
base_dir.push_str(SLASH_SEPARATOR);
|
||||
}
|
||||
base_dir
|
||||
}
|
||||
|
||||
pub struct LazyBuf {
|
||||
s: String,
|
||||
buf: Option<Vec<u8>>,
|
||||
@@ -184,10 +198,33 @@ pub fn clean(path: &str) -> String {
|
||||
|
||||
out.string()
|
||||
}
|
||||
|
||||
pub fn split(path: &str) -> (&str, &str) {
|
||||
// Find the last occurrence of the '/' character
|
||||
if let Some(i) = path.rfind('/') {
|
||||
// Return the directory (up to and including the last '/') and the file name
|
||||
return (&path[..i + 1], &path[i + 1..]);
|
||||
}
|
||||
// If no '/' is found, return an empty string for the directory and the whole path as the file name
|
||||
(path, "")
|
||||
}
|
||||
|
||||
pub fn dir(path: &str) -> &str {
|
||||
let (a, _) = split(path);
|
||||
a
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_base_dir_from_prefix() {
|
||||
let a = "da/";
|
||||
println!("---- in {}", a);
|
||||
let a = base_dir_from_prefix(a);
|
||||
println!("---- out {}", a);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clean() {
|
||||
assert_eq!(clean(""), ".");
|
||||
|
||||
@@ -16,7 +16,9 @@ use ecstore::bucket::policy_sys::PolicySys;
|
||||
use ecstore::bucket::tagging::decode_tags;
|
||||
use ecstore::bucket::tagging::encode_tags;
|
||||
use ecstore::bucket::versioning_sys::BucketVersioningSys;
|
||||
use ecstore::disk::error::is_err_file_not_found;
|
||||
use ecstore::disk::error::DiskError;
|
||||
use ecstore::error::Error as EcError;
|
||||
use ecstore::new_object_layer_fn;
|
||||
use ecstore::options::extract_metadata;
|
||||
use ecstore::options::put_opts;
|
||||
@@ -69,6 +71,15 @@ lazy_static! {
|
||||
id: Some("c19050dbcee97fda828689dda99097a6321af2248fa760517237346e5d9c8a66".to_owned()),
|
||||
};
|
||||
}
|
||||
|
||||
fn to_s3_error(err: EcError) -> S3Error {
|
||||
if is_err_file_not_found(&err) {
|
||||
return S3Error::with_message(S3ErrorCode::NoSuchKey, format!(" ec err {}", err));
|
||||
}
|
||||
|
||||
S3Error::with_message(S3ErrorCode::InternalError, format!(" ec err {}", err))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FS {
|
||||
// pub store: ECStore,
|
||||
@@ -392,7 +403,10 @@ impl S3 for FS {
|
||||
None => return Err(S3Error::with_message(S3ErrorCode::InternalError, "Not init".to_string())),
|
||||
};
|
||||
|
||||
let info = try_!(store.get_object_info(&bucket, &key, &ObjectOptions::default()).await);
|
||||
let info = store
|
||||
.get_object_info(&bucket, &key, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(to_s3_error)?;
|
||||
debug!("info {:?}", info);
|
||||
|
||||
let content_type = {
|
||||
@@ -468,6 +482,8 @@ impl S3 for FS {
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
async fn list_objects_v2(&self, req: S3Request<ListObjectsV2Input>) -> S3Result<S3Response<ListObjectsV2Output>> {
|
||||
// warn!("list_objects_v2 input {:?}", &req.input);
|
||||
|
||||
let ListObjectsV2Input {
|
||||
bucket,
|
||||
continuation_token,
|
||||
|
||||
Reference in New Issue
Block a user