From bbc76302a276749a1801e219426a5a38a69bebf6 Mon Sep 17 00:00:00 2001 From: weisd Date: Fri, 29 Nov 2024 14:58:40 +0800 Subject: [PATCH] sync s3s rustfs branch, fix versiong config --- Cargo.lock | 51 +++++++++++++++++----- Cargo.toml | 6 +-- ecstore/src/bucket/versioning/mod.rs | 64 ++++++++++++++++++++-------- ecstore/src/bucket/versioning_sys.rs | 21 +++++---- ecstore/src/pools.rs | 10 ++++- 5 files changed, 112 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a65c7ab5f..953b6e14f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -678,7 +678,7 @@ dependencies = [ "sha2", "siphasher", "tempfile", - "thiserror", + "thiserror 1.0.68", "time", "tokio", "tokio-stream", @@ -1976,7 +1976,7 @@ checksum = "a3a7c64d9bf75b1b8d981124c14c179074e8caa7dfe7b6a12e6222ddcd0c8f72" dependencies = [ "once_cell", "protobuf-support", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -1985,7 +1985,7 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b088fd20b938a875ea00843b6faf48579462630015c3788d397ad6a786663252" dependencies = [ - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -2018,7 +2018,7 @@ dependencies = [ "num_cpus", "once_cell", "platforms", - "thiserror", + "thiserror 1.0.68", "unescape", ] @@ -2082,7 +2082,7 @@ dependencies = [ "pin-project-lite", "s3s", "sha2", - "thiserror", + "thiserror 1.0.68", "tokio", "tracing", ] @@ -2332,7 +2332,7 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=207170f526c75a8190e8f9afadf961909fd01d34#207170f526c75a8190e8f9afadf961909fd01d34" +source = "git+https://github.com/Nugine/s3s.git?rev=3291c0ca0284971569499cbe75bd69ef7bde8321#3291c0ca0284971569499cbe75bd69ef7bde8321" dependencies = [ "arrayvec", "async-trait", @@ -2364,8 +2364,9 @@ dependencies = [ "sha1", "sha2", "smallvec", + "std-next", "sync_wrapper 1.0.1", - "thiserror", + "thiserror 2.0.3", "time", "tokio", "tower 0.5.1", @@ -2378,12 +2379,12 @@ dependencies = [ [[package]] name = "s3s-policy" version = "0.11.0-dev" -source = "git+https://github.com/Nugine/s3s.git?rev=207170f526c75a8190e8f9afadf961909fd01d34#207170f526c75a8190e8f9afadf961909fd01d34" +source = "git+https://github.com/Nugine/s3s.git?rev=3291c0ca0284971569499cbe75bd69ef7bde8321#3291c0ca0284971569499cbe75bd69ef7bde8321" dependencies = [ "indexmap 2.6.0", "serde", "serde_json", - "thiserror", + "thiserror 2.0.3", ] [[package]] @@ -2560,6 +2561,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "std-next" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87854dde78837ff867561b22d873c53c25938d4b152afba2fa6fc48cb406c099" +dependencies = [ + "simdutf8", + "thiserror 2.0.3", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2625,7 +2636,16 @@ version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.68", +] + +[[package]] +name = "thiserror" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +dependencies = [ + "thiserror-impl 2.0.3", ] [[package]] @@ -2639,6 +2659,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.1.8" diff --git a/Cargo.toml b/Cargo.toml index 08539fe2c..f5f283948 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ "common/protos", "api/admin", "reader", - "common/workers", + "common/workers", ] [workspace.package] @@ -55,10 +55,10 @@ protos = { path = "./common/protos" } rand = "0.8.5" rmp = "0.8.14" rmp-serde = "1.3.0" -s3s = { git = "https://github.com/Nugine/s3s.git", rev = "207170f526c75a8190e8f9afadf961909fd01d34", default-features = true, features = [ +s3s = { git = "https://github.com/Nugine/s3s.git", rev = "3291c0ca0284971569499cbe75bd69ef7bde8321", default-features = true, features = [ "tower", ] } -s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "207170f526c75a8190e8f9afadf961909fd01d34" } +s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "3291c0ca0284971569499cbe75bd69ef7bde8321" } serde = { version = "1.0.214", features = ["derive"] } serde_json = "1.0.132" tempfile = "3.13.0" diff --git a/ecstore/src/bucket/versioning/mod.rs b/ecstore/src/bucket/versioning/mod.rs index 6ad2e4014..3444034e4 100644 --- a/ecstore/src/bucket/versioning/mod.rs +++ b/ecstore/src/bucket/versioning/mod.rs @@ -1,24 +1,25 @@ use s3s::dto::{BucketVersioningStatus, VersioningConfiguration}; +use crate::utils::wildcard; + pub trait VersioningApi { fn enabled(&self) -> bool; fn prefix_enabled(&self, prefix: &str) -> bool; fn prefix_suspended(&self, prefix: &str) -> bool; + fn versioned(&self, prefix: &str) -> bool; + fn suspended(&self) -> bool; } impl VersioningApi for VersioningConfiguration { fn enabled(&self) -> bool { - self.status - .as_ref() - .is_some_and(|v| v.as_str() == BucketVersioningStatus::ENABLED) + self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) + } + fn suspended(&self) -> bool { + self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) } fn prefix_enabled(&self, prefix: &str) -> bool { - if !self - .status - .as_ref() - .is_some_and(|v| v.as_str() == BucketVersioningStatus::ENABLED) - { + if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) { return false; } @@ -26,27 +27,56 @@ impl VersioningApi for VersioningConfiguration { return true; } - // TODO: ExcludeFolders + if let Some(exclude_folders) = self.exclude_folders { + if exclude_folders && prefix.ends_with('/') { + return false; + } + } + + if let Some(ref excluded_prefixes) = self.excluded_prefixes { + for p in excluded_prefixes.iter() { + if let Some(ref sprefix) = p.prefix { + let pattern = format!("{}*", sprefix); + if wildcard::match_simple(&pattern, prefix) { + return false; + } + } + } + } true } fn prefix_suspended(&self, prefix: &str) -> bool { - if self - .status - .as_ref() - .is_some_and(|v| v.as_str() == BucketVersioningStatus::SUSPENDED) - { + if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)) { return true; } - if let Some(status) = self.status.as_ref() { - if status.as_str() == BucketVersioningStatus::ENABLED && prefix.is_empty() { + if self.status == Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)) { + if prefix.is_empty() { return false; + } - // TODO: ExcludeFolders + if let Some(exclude_folders) = self.exclude_folders { + if exclude_folders && prefix.ends_with('/') { + return true; + } + } + + if let Some(ref excluded_prefixes) = self.excluded_prefixes { + for p in excluded_prefixes.iter() { + if let Some(ref sprefix) = p.prefix { + let pattern = format!("{}*", sprefix); + if wildcard::match_simple(&pattern, prefix) { + return true; + } + } + } } } false } + fn versioned(&self, prefix: &str) -> bool { + self.prefix_enabled(prefix) || self.prefix_suspended(prefix) + } } diff --git a/ecstore/src/bucket/versioning_sys.rs b/ecstore/src/bucket/versioning_sys.rs index b464a1606..c62a829c9 100644 --- a/ecstore/src/bucket/versioning_sys.rs +++ b/ecstore/src/bucket/versioning_sys.rs @@ -7,6 +7,9 @@ use tracing::warn; pub struct BucketVersioningSys {} impl BucketVersioningSys { + pub fn new() -> Self { + Self {} + } pub async fn enabled(bucket: &str) -> bool { match Self::get(bucket).await { Ok(res) => res.enabled(), @@ -27,15 +30,15 @@ impl BucketVersioningSys { } } - // pub async fn suspended(bucket: &str) -> bool { - // match Self::get(bucket).await { - // Ok(res) => res.suspended(), - // Err(err) => { - // warn!("{:?}", err); - // false - // } - // } - // } + pub async fn suspended(bucket: &str) -> bool { + match Self::get(bucket).await { + Ok(res) => res.suspended(), + Err(err) => { + warn!("{:?}", err); + false + } + } + } pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool { match Self::get(bucket).await { diff --git a/ecstore/src/pools.rs b/ecstore/src/pools.rs index ba607edf0..38f928e30 100644 --- a/ecstore/src/pools.rs +++ b/ecstore/src/pools.rs @@ -1,3 +1,4 @@ +use crate::bucket::versioning_sys::BucketVersioningSys; use crate::config::common::{read_config, save_config, CONFIG_PREFIX}; use crate::config::error::ConfigError; use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET}; @@ -477,7 +478,14 @@ impl ECStore { Ok(()) } - async fn decommission_pool(&self, _idx: usize, _pool: Arc, _bucket: DecomBucketInfo) -> Result<()> { + async fn decommission_pool(&self, _idx: usize, _pool: Arc, bi: DecomBucketInfo) -> Result<()> { + let mut _vc = None; + + if &bi.name == RUSTFS_META_BUCKET { + let versioning = BucketVersioningSys::get(&bi.name).await?; + _vc = Some(versioning); + } + // FIXME: unimplemented!() }