merge versioning, fix bug todo

ecstore/src/bucket/metadata.rs

@@ -1,5 +1,8 @@
+use super::policy::bucket_policy::BucketPolicy;
 use super::{quota::BucketQuota, target::BucketTargets};
 
+use super::object_lock::ObjectLockApi;
+use super::versioning::VersioningApi;
 use byteorder::{BigEndian, ByteOrder, LittleEndian};
 use rmp_serde::Serializer as rmpSerializer;
 use s3s::dto::{
@@ -7,7 +10,6 @@ use s3s::dto::{
     ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
 use s3s::xml;
-use s3s_policy::model::Policy;
 use serde::Serializer;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -70,7 +72,7 @@ pub struct BucketMetadata {
     pub new_field_updated_at: OffsetDateTime,
 
     #[serde(skip)]
-    pub policy_config: Option<Policy>,
+    pub policy_config: Option<BucketPolicy>,
     #[serde(skip)]
     pub notification_config: Option<NotificationConfiguration>,
     #[serde(skip)]
@@ -149,9 +151,15 @@ impl BucketMetadata {
         format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE)
     }
 
-    // fn msg_size(&self) -> usize {
-    //     unimplemented!()
-    // }
+    pub fn versioning(&self) -> bool {
+        self.lock_enabled
+            || (self.object_lock_config.as_ref().is_some_and(|v| v.enabled())
+                || self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
+    }
+
+    pub fn object_locking(&self) -> bool {
+        self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled()))
+    }
 
     pub fn marshal_msg(&self) -> Result<Vec<u8>> {
         let mut buf = Vec::new();
@@ -278,6 +286,10 @@ impl BucketMetadata {
         Ok(updated)
     }
 
+    pub fn set_created(&mut self, created: OffsetDateTime) {
+        self.created = created
+    }
+
     pub async fn save(&mut self, api: &ECStore) -> Result<()> {
         self.parse_all_configs(api)?;
 

ecstore/src/bucket/metadata_sys.rs

@@ -15,12 +15,12 @@ use s3s::dto::{
     BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
     ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
 };
-use s3s_policy::model::Policy;
 use time::OffsetDateTime;
 use tokio::sync::RwLock;
 use tracing::{error, warn};
 
 use super::metadata::{deserialize, load_bucket_metadata, BucketMetadata};
+use super::policy::bucket_policy::BucketPolicy;
 use super::quota::BucketQuota;
 use super::target::BucketTargets;
 
@@ -44,6 +44,11 @@ pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) {
     sys.set(bucket, Arc::new(bm)).await
 }
 
+pub(crate) async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
+    let sys = GLOBAL_BucketMetadataSys.write().await;
+    sys.get(bucket).await
+}
+
 pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
     let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
     let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
@@ -114,6 +119,13 @@ pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
     bucket_meta_sys.get_config_from_disk(bucket).await
 }
 
+pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
+    let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
+    let bucket_meta_sys = bucket_meta_sys_lock.read().await;
+
+    bucket_meta_sys.created_at(bucket).await
+}
+
 #[derive(Debug, Default)]
 pub struct BucketMetadataSys {
     metadata_map: RwLock<HashMap<String, Arc<BucketMetadata>>>,
@@ -373,7 +385,7 @@ impl BucketMetadataSys {
         }
     }
 
-    pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(Policy, OffsetDateTime)> {
+    pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
         let bm = match self.get_config(bucket).await {
             Ok((res, _)) => res,
             Err(err) => {

ecstore/src/bucket/mod.rs

@@ -1,9 +1,12 @@
 pub mod error;
 pub mod metadata;
 pub mod metadata_sys;
+pub mod object_lock;
 pub mod policy;
 pub mod policy_sys;
 mod quota;
+pub mod tagging;
 mod target;
 pub mod utils;
+pub mod versioning;
 pub mod versioning_sys;

ecstore/src/bucket/object_lock/mod.rs

@@ -0,0 +1,13 @@
+use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
+
+pub trait ObjectLockApi {
+    fn enabled(&self) -> bool;
+}
+
+impl ObjectLockApi for ObjectLockConfiguration {
+    fn enabled(&self) -> bool {
+        self.object_lock_enabled
+            .as_ref()
+            .is_some_and(|v| v.as_str() == ObjectLockEnabled::ENABLED)
+    }
+}

ecstore/src/bucket/policy/action.rs

@@ -41,6 +41,7 @@ impl AsRef<HashSet<Action>> for ActionSet {
     }
 }
 
+// TODO:: 使用字符串
 // 定义Action枚举类型
 #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, Default, Hash)]
 pub enum Action {
@@ -281,7 +282,7 @@ impl Action {
         }
     }
 
-    fn _from_str(s: &str) -> Option<Self> {
+    pub fn from_str(s: &str) -> Option<Self> {
         match s {
             "s3:AbortMultipartUpload" => Some(Action::AbortMultipartUpload),
             "s3:CreateBucket" => Some(Action::CreateBucket),

ecstore/src/bucket/policy/condition/function.rs

@@ -243,8 +243,11 @@ impl<'de> Deserialize<'de> for Functions {
     {
         // Instantiate our Visitor and ask the Deserializer to drive
         // it over the input data, resulting in an instance of MyMap.
-        let _map = deserializer.deserialize_map(MyMapVisitor::new())?;
+        let map = deserializer.deserialize_map(MyMapVisitor::new())?;
 
+        for (key, vals) in map.iter() {
+            println!("functions key {}, vals {:?}", key, vals);
+        }
         // TODO: FIXME: create functions from name
 
         Ok(Functions(Vec::new()))

ecstore/src/bucket/policy/mod.rs

@@ -1,6 +1,103 @@
+// use std::collections::HashMap;
+
+// use action::Action;
+// use s3s_policy::model::{Effect, Policy, Principal, PrincipalRule, Statement};
+// use serde::{Deserialize, Serialize};
+// use tower::ready_cache::cache::Equivalent;
+
+// use crate::utils::wildcard;
+
 pub mod action;
 pub mod bucket_policy;
 pub mod condition;
 pub mod effect;
 pub mod principal;
 pub mod resource;
+
+// #[derive(Debug, Deserialize, Serialize, Default, Clone)]
+// pub struct BucketPolicyArgs {
+//     pub account_name: String,
+//     pub groups: Vec<String>,
+//     pub action: Action,
+//     pub bucket_name: String,
+//     pub condition_values: HashMap<String, Vec<String>>,
+//     pub is_owner: bool,
+//     pub object_name: String,
+// }
+
+// pub trait AllowApi {
+//     fn is_allowed(&self, args: &BucketPolicyArgs) -> bool;
+// }
+
+// pub trait MatchApi {
+//     fn is_match(&self, found: &str) -> bool;
+// }
+
+// impl AllowApi for Policy {
+//     fn is_allowed(&self, args: &BucketPolicyArgs) -> bool {
+//         for statement in self.statement.as_slice().iter() {
+//             if statement.effect == Effect::Deny {
+//                 if !statement.is_allowed(args) {
+//                     return false;
+//                 }
+//             }
+//         }
+//         false
+//     }
+// }
+
+// impl AllowApi for Statement {
+//     fn is_allowed(&self, args: &BucketPolicyArgs) -> bool {
+//         let check = || -> bool {
+//             if let Some(principal) = &self.principal {
+//                 if !principal.is_match(&args.account_name) {
+//                     return false;
+//                 }
+//             }
+
+//             false
+//         };
+
+//         self.effect.is_allowed(check())
+//     }
+// }
+
+// impl MatchApi for PrincipalRule {
+//     fn is_match(&self, found: &str) -> bool {
+//         match self {
+//             PrincipalRule::Principal(principal) => match principal {
+//                 Principal::Wildcard => return true,
+//                 Principal::Map(index_map) => {
+//                     if let Some(keys) = index_map.get("AWS") {
+//                         for key in keys.as_slice() {
+//                             if wildcard::match_simple(key, found) {
+//                                 return true;
+//                             }
+//                         }
+//                     }
+//                     return false;
+//                 }
+//             },
+//             PrincipalRule::NotPrincipal(principal) => match principal {
+//                 Principal::Wildcard => return true,
+//                 Principal::Map(index_map) => todo!(),
+//             },
+//         }
+
+//         false
+//     }
+// }
+
+// trait EffectApi {
+//     fn is_allowed(&self, b: bool) -> bool;
+// }
+
+// impl EffectApi for Effect {
+//     fn is_allowed(&self, b: bool) -> bool {
+//         if self == &Effect::Allow {
+//             b
+//         } else {
+//             !b
+//         }
+//     }
+// }

ecstore/src/bucket/policy_sys.rs

@@ -1,23 +1,27 @@
-use super::metadata_sys::get_bucket_metadata_sys;
+use super::{
+    error::BucketMetadataError,
+    metadata_sys::get_bucket_metadata_sys,
+    policy::bucket_policy::{BucketPolicy, BucketPolicyArgs},
+};
 use crate::error::Result;
-use s3s_policy::model::Policy;
+use tracing::warn;
 
 pub struct PolicySys {}
 
 impl PolicySys {
-    // pub async fn is_allowed(args: &BucketPolicyArgs) -> bool {
-    //     match Self::get(&args.bucket_name).await {
-    //         Ok(cfg) => return cfg.is_allowed(args),
-    //         Err(err) => {
-    //             if !BucketMetadataError::BucketPolicyNotFound.is(&err) {
-    //                 warn!("config get err {:?}", err);
-    //             }
-    //         }
-    //     }
+    pub async fn is_allowed(args: &BucketPolicyArgs) -> bool {
+        match Self::get(&args.bucket_name).await {
+            Ok(cfg) => return cfg.is_allowed(args),
+            Err(err) => {
+                if !BucketMetadataError::BucketPolicyNotFound.is(&err) {
+                    warn!("config get err {:?}", err);
+                }
+            }
+        }
 
-    //     args.is_owner
-    // }
-    pub async fn get(bucket: &str) -> Result<Policy> {
+        args.is_owner
+    }
+    pub async fn get(bucket: &str) -> Result<BucketPolicy> {
         let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
         let bucket_meta_sys = bucket_meta_sys_lock.write().await;
 
@@ -26,13 +30,3 @@ impl PolicySys {
         Ok(cfg)
     }
 }
-
-// trait PolicyApi {
-//     fn is_allowed(&self) -> bool;
-// }
-
-// impl PolicyApi for Policy {
-//     fn is_allowed(&self) -> bool {
-//         todo!()
-//     }
-// }

ecstore/src/bucket/tagging/mod.rs

@@ -0,0 +1,27 @@
+use s3s::dto::Tag;
+use url::form_urlencoded;
+
+pub fn decode_tags(tags: &str) -> Vec<Tag> {
+    let values = form_urlencoded::parse(tags.as_bytes());
+
+    let mut list = Vec::new();
+
+    for (k, v) in values {
+        list.push(Tag {
+            key: k.to_string(),
+            value: v.to_string(),
+        });
+    }
+
+    list
+}
+
+pub fn encode_tags(tags: Vec<Tag>) -> String {
+    let mut encoded = form_urlencoded::Serializer::new(String::new());
+
+    for tag in tags.iter() {
+        encoded.append_pair(tag.key.as_str(), tag.value.as_str());
+    }
+
+    encoded.finish()
+}

ecstore/src/bucket/utils.rs

@@ -1,5 +1,73 @@
-use crate::disk::RUSTFS_META_BUCKET;
+use crate::{disk::RUSTFS_META_BUCKET, error::Error};
 
 pub fn is_meta_bucketname(name: &str) -> bool {
     name.starts_with(RUSTFS_META_BUCKET)
 }
+
+use regex::Regex;
+
+lazy_static::lazy_static! {
+    static ref VALID_BUCKET_NAME: Regex = Regex::new(r"^[A-Za-z0-9][A-Za-z0-9\.\-\_\:]{1,61}[A-Za-z0-9]$").unwrap();
+    static ref VALID_BUCKET_NAME_STRICT: Regex = Regex::new(r"^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$").unwrap();
+    static ref IP_ADDRESS: Regex = Regex::new(r"^(\d+\.){3}\d+$").unwrap();
+}
+
+pub fn check_bucket_name_common(bucket_name: &str, strict: bool) -> Result<(), Error> {
+    let bucket_name_trimmed = bucket_name.trim();
+
+    if bucket_name_trimmed.is_empty() {
+        return Err(Error::msg("Bucket name cannot be empty"));
+    }
+    if bucket_name_trimmed.len() < 3 {
+        return Err(Error::msg("Bucket name cannot be shorter than 3 characters"));
+    }
+    if bucket_name_trimmed.len() > 63 {
+        return Err(Error::msg("Bucket name cannot be longer than 63 characters"));
+    }
+
+    if bucket_name_trimmed == "rustfs" {
+        return Err(Error::msg("Bucket name cannot be rustfs"));
+    }
+
+    if IP_ADDRESS.is_match(bucket_name_trimmed) {
+        return Err(Error::msg("Bucket name cannot be an IP address"));
+    }
+    if bucket_name_trimmed.contains("..") || bucket_name_trimmed.contains(".-") || bucket_name_trimmed.contains("-.") {
+        return Err(Error::msg("Bucket name contains invalid characters"));
+    }
+    if strict {
+        if !VALID_BUCKET_NAME_STRICT.is_match(bucket_name_trimmed) {
+            return Err(Error::msg("Bucket name contains invalid characters"));
+        }
+    } else {
+        if !VALID_BUCKET_NAME.is_match(bucket_name_trimmed) {
+            return Err(Error::msg("Bucket name contains invalid characters"));
+        }
+    }
+    Ok(())
+}
+
+pub fn check_valid_bucket_name(bucket_name: &str) -> Result<(), Error> {
+    check_bucket_name_common(bucket_name, false)
+}
+
+pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<(), Error> {
+    check_bucket_name_common(bucket_name, true)
+}
+
+pub fn check_valid_object_name_prefix(object_name: &str) -> Result<(), Error> {
+    if object_name.len() > 1024 {
+        return Err(Error::msg("Object name cannot be longer than 1024 characters"));
+    }
+    if !object_name.is_ascii() {
+        return Err(Error::msg("Object name with non-UTF-8 strings are not supported"));
+    }
+    Ok(())
+}
+
+pub fn check_valid_object_name(object_name: &str) -> Result<(), Error> {
+    if object_name.trim().is_empty() {
+        return Err(Error::msg("Object name cannot be empty"));
+    }
+    check_valid_object_name_prefix(object_name)
+}

ecstore/src/bucket/versioning/mod.rs

@@ -0,0 +1,54 @@
+use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
+
+pub trait VersioningApi {
+    fn enabled(&self) -> bool;
+    fn prefix_enabled(&self, prefix: &str) -> bool;
+    fn prefix_suspended(&self, prefix: &str) -> bool;
+}
+
+impl VersioningApi for VersioningConfiguration {
+    fn enabled(&self) -> bool {
+        self.status
+            .as_ref()
+            .is_some_and(|v| v.as_str() == BucketVersioningStatus::ENABLED)
+    }
+
+    fn prefix_enabled(&self, prefix: &str) -> bool {
+        if !self
+            .status
+            .as_ref()
+            .is_some_and(|v| v.as_str() == BucketVersioningStatus::ENABLED)
+        {
+            return false;
+        }
+
+        if prefix.is_empty() {
+            return true;
+        }
+
+        // TODO: ExcludeFolders
+
+        true
+    }
+    fn prefix_suspended(&self, prefix: &str) -> bool {
+        if self
+            .status
+            .as_ref()
+            .is_some_and(|v| v.as_str() == BucketVersioningStatus::SUSPENDED)
+        {
+            return true;
+        }
+
+        if let Some(status) = self.status.as_ref() {
+            if status.as_str() == BucketVersioningStatus::ENABLED {
+                if prefix.is_empty() {
+                    return false;
+                }
+
+                // TODO: ExcludeFolders
+            }
+        }
+
+        false
+    }
+}

ecstore/src/bucket/versioning_sys.rs

@@ -1,7 +1,7 @@
-use super::metadata_sys::get_bucket_metadata_sys;
+use super::{metadata_sys::get_bucket_metadata_sys, versioning::VersioningApi};
 use crate::disk::RUSTFS_META_BUCKET;
 use crate::error::Result;
-use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
+use s3s::dto::VersioningConfiguration;
 use tracing::warn;
 
 pub struct BucketVersioningSys {}
@@ -17,15 +17,15 @@ impl BucketVersioningSys {
         }
     }
 
-    // pub async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
-    //     match Self::get(bucket).await {
-    //         Ok(res) => res.prefix_enabled(prefix),
-    //         Err(err) => {
-    //             warn!("{:?}", err);
-    //             false
-    //         }
-    //     }
-    // }
+    pub async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.prefix_enabled(prefix),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
 
     // pub async fn suspended(bucket: &str) -> bool {
     //     match Self::get(bucket).await {
@@ -37,15 +37,15 @@ impl BucketVersioningSys {
     //     }
     // }
 
-    // pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
-    //     match Self::get(bucket).await {
-    //         Ok(res) => res.prefix_suspended(prefix),
-    //         Err(err) => {
-    //             warn!("{:?}", err);
-    //             false
-    //         }
-    //     }
-    // }
+    pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
+        match Self::get(bucket).await {
+            Ok(res) => res.prefix_suspended(prefix),
+            Err(err) => {
+                warn!("{:?}", err);
+                false
+            }
+        }
+    }
 
     pub async fn get(bucket: &str) -> Result<VersioningConfiguration> {
         if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
@@ -60,16 +60,3 @@ impl BucketVersioningSys {
         Ok(cfg)
     }
 }
-
-trait VersioningApi {
-    fn enabled(&self) -> bool;
-}
-
-impl VersioningApi for VersioningConfiguration {
-    fn enabled(&self) -> bool {
-        self.status
-            .as_ref()
-            .map(|v| v.as_str() == BucketVersioningStatus::ENABLED)
-            .is_some_and(|v| v)
-    }
-}