gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-06 13:27:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix:#38 implement the basic storage functions of bucketmeta config use s3s struct define

Browse Source
This commit is contained in:
weisd
2024-10-14 14:58:32 +08:00
parent 0f39887cdb
commit 12114dce16
40 changed files with 920 additions and 1234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ecstore/src/bucket/encryption/mod.rs
-59
View File
@@ -1,59 +0,0 @@
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
// 定义Algorithm枚举类型
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum Algorithm {
AES256,
AWSKms,
}
// 实现从字符串到Algorithm的转换
impl std::str::FromStr for Algorithm {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"AES256" => Ok(Algorithm::AES256),
"aws:kms" => Ok(Algorithm::AWSKms),
_ => Err(format!("未知的 SSE 算法: {}", s)),
}
}
}
// 定义EncryptionAction结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct EncryptionAction {
algorithm: Option<Algorithm>,
master_key_id: Option<String>,
}
// 定义Rule结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Rule {
default_encryption_action: EncryptionAction,
}
// 定义BucketSSEConfig结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct BucketSSEConfig {
xml_ns: String,
xml_name: String,
rules: Vec<Rule>,
}
impl BucketSSEConfig {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: BucketSSEConfig = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/event/mod.rs
-87
View File
@@ -1,87 +0,0 @@
mod name;
use crate::error::Result;
use name::Name;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
// 定义common结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
struct Common {
pub id: String,
pub filter: S3Key,
pub events: Vec<Name>,
}
// 定义Queue结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
struct Queue {
pub common: Common,
pub arn: Arn,
}
// 定义ARN结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Arn {
pub target_id: TargetID,
pub region: String,
}
// 定义TargetID结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct TargetID {
pub id: String,
pub name: String,
}
// 定义FilterRule结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct FilterRule {
pub name: String,
pub value: String,
}
// 定义FilterRuleList结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct FilterRuleList {
pub rules: Vec<FilterRule>,
}
// 定义S3Key结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct S3Key {
pub rule_list: FilterRuleList,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Lambda {
arn: String,
}
// 定义Topic结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Topic {
arn: String,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Config {
queue_list: Vec<Queue>,
lambda_list: Vec<Lambda>,
topic_list: Vec<Topic>,
}
impl Config {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Config = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/event/name.rs
-98
View File
@@ -1,98 +0,0 @@
use std::fmt::Display;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum Name {
ObjectAccessedGet = 1,
ObjectAccessedGetRetention,
ObjectAccessedGetLegalHold,
ObjectAccessedHead,
ObjectAccessedAttributes,
ObjectCreatedCompleteMultipartUpload,
ObjectCreatedCopy,
ObjectCreatedPost,
ObjectCreatedPut,
ObjectCreatedPutRetention,
ObjectCreatedPutLegalHold,
ObjectCreatedPutTagging,
ObjectCreatedDeleteTagging,
ObjectRemovedDelete,
ObjectRemovedDeleteMarkerCreated,
ObjectRemovedDeleteAllVersions,
ObjectRemovedNoOP,
BucketCreated,
BucketRemoved,
ObjectReplicationFailed,
ObjectReplicationComplete,
ObjectReplicationMissedThreshold,
ObjectReplicationReplicatedAfterThreshold,
ObjectReplicationNotTracked,
ObjectRestorePost,
ObjectRestoreCompleted,
ObjectTransitionFailed,
ObjectTransitionComplete,
ObjectManyVersions,
ObjectLargeVersions,
PrefixManyFolders,
ILMDelMarkerExpirationDelete,
ObjectAccessedAll,
ObjectCreatedAll,
ObjectRemovedAll,
ObjectReplicationAll,
ObjectRestoreAll,
ObjectTransitionAll,
ObjectScannerAll,
Everything,
}
impl Display for Name {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
match *self {
Name::ObjectAccessedGet => "s3:ObjectAccessed:Get",
Name::ObjectAccessedGetRetention => "s3:ObjectAccessed:GetRetention",
Name::ObjectAccessedGetLegalHold => "s3:ObjectAccessed:GetLegalHold",
Name::ObjectAccessedHead => "s3:ObjectAccessed:Head",
Name::ObjectAccessedAttributes => "s3:ObjectAccessed:Attributes",
Name::ObjectCreatedCompleteMultipartUpload => "s3:ObjectCreated:CompleteMultipartUpload",
Name::ObjectCreatedCopy => "s3:ObjectCreated:Copy",
Name::ObjectCreatedPost => "s3:ObjectCreated:Post",
Name::ObjectCreatedPut => "s3:ObjectCreated:Put",
Name::ObjectCreatedPutRetention => "s3:ObjectCreated:PutRetention",
Name::ObjectCreatedPutLegalHold => "s3:ObjectCreated:PutLegalHold",
Name::ObjectCreatedPutTagging => "s3:ObjectCreated:PutTagging",
Name::ObjectCreatedDeleteTagging => "s3:ObjectCreated:DeleteTagging",
Name::ObjectRemovedDelete => "s3:ObjectRemoved:Delete",
Name::ObjectRemovedDeleteMarkerCreated => "s3:ObjectRemoved:DeleteMarkerCreated",
Name::ObjectRemovedDeleteAllVersions => "s3:ObjectRemoved:DeleteAllVersions",
Name::ObjectRemovedNoOP => "s3:ObjectRemoved:NoOP",
Name::BucketCreated => "s3:BucketCreated:*",
Name::BucketRemoved => "s3:BucketRemoved:*",
Name::ObjectReplicationFailed => "s3:Replication:OperationFailedReplication",
Name::ObjectReplicationComplete => "s3:Replication:OperationCompletedReplication",
Name::ObjectReplicationMissedThreshold => "s3:Replication:OperationMissedThreshold",
Name::ObjectReplicationReplicatedAfterThreshold => "s3:Replication:OperationReplicatedAfterThreshold",
Name::ObjectReplicationNotTracked => "s3:Replication:OperationNotTracked",
Name::ObjectRestorePost => "s3:ObjectRestore:Post",
Name::ObjectRestoreCompleted => "s3:ObjectRestore:Completed",
Name::ObjectTransitionFailed => "s3:ObjectTransition:Failed",
Name::ObjectTransitionComplete => "s3:ObjectTransition:Complete",
Name::ObjectManyVersions => "s3:Scanner:ManyVersions",
Name::ObjectLargeVersions => "s3:Scanner:LargeVersions",
Name::PrefixManyFolders => "s3:Scanner:BigPrefix",
Name::ILMDelMarkerExpirationDelete => "s3:LifecycleDelMarkerExpiration:Delete",
Name::ObjectAccessedAll => "s3:ObjectAccessed:*",
Name::ObjectCreatedAll => "s3:ObjectCreated:*",
Name::ObjectRemovedAll => "s3:ObjectRemoved:*",
Name::ObjectReplicationAll => "s3:Replication:*",
Name::ObjectRestoreAll => "s3:ObjectRestore:*",
Name::ObjectTransitionAll => "s3:ObjectTransition:*",
Name::ObjectScannerAll => "s3:Scanner:*",
Name::Everything => "*",
}
)
}
}
ecstore/src/bucket/lifecycle/and.rs
-9
View File
@@ -1,9 +0,0 @@
use super::{prefix::Prefix, tag::Tag};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct And {
pub object_size_greater_than: i64,
pub object_size_less_than: i64,
pub prefix: Prefix,
pub tags: Vec<Tag>,
}
ecstore/src/bucket/lifecycle/delmarker.rs
-6
View File
@@ -1,6 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct DelMarkerExpiration {
pub days: usize,
}
ecstore/src/bucket/lifecycle/expiration.rs
-27
View File
@@ -1,27 +0,0 @@
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
// ExpirationDays is a type alias to unmarshal Days in Expiration
pub type ExpirationDays = usize;
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct ExpirationDate(Option<OffsetDateTime>);
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct ExpireDeleteMarker {
pub marker: Boolean,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Boolean {
pub val: bool,
pub set: bool,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Expiration {
pub days: Option<ExpirationDays>,
pub date: Option<ExpirationDate>,
pub delete_marker: ExpireDeleteMarker,
pub delete_all: Boolean,
pub set: bool,
}
ecstore/src/bucket/lifecycle/fileter.rs
-23
View File
@@ -1,23 +0,0 @@
use std::collections::HashMap;
use super::{and::And, prefix::Prefix, tag::Tag};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Filter {
pub set: bool,
pub prefix: Prefix,
pub object_size_greater_than: Option<i64>,
pub object_size_less_than: Option<i64>,
pub and_condition: And,
pub and_set: bool,
pub tag: Tag,
pub tag_set: bool,
// 使用HashMap存储缓存的标签
pub cached_tags: HashMap<String, String>,
}
ecstore/src/bucket/lifecycle/lifecycle.rs
-26
View File
@@ -1,26 +0,0 @@
use super::rule::Rule;
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Lifecycle {
pub rules: Vec<Rule>,
pub expiry_updated_at: Option<OffsetDateTime>,
}
impl Lifecycle {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Lifecycle = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/lifecycle/mod.rs
-10
View File
@@ -1,10 +0,0 @@
mod and;
mod delmarker;
mod expiration;
mod fileter;
pub(crate) mod lifecycle;
mod noncurrentversion;
mod prefix;
mod rule;
mod tag;
mod transition;
ecstore/src/bucket/lifecycle/noncurrentversion.rs
-16
View File
@@ -1,16 +0,0 @@
use super::{expiration::ExpirationDays, transition::TransitionDays};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct NoncurrentVersionExpiration {
pub noncurrent_days: ExpirationDays,
pub newer_noncurrent_versions: usize,
set: bool,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct NoncurrentVersionTransition {
pub noncurrent_days: TransitionDays,
pub storage_class: String,
set: bool,
}
ecstore/src/bucket/lifecycle/prefix.rs
-7
View File
@@ -1,7 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Prefix {
pub val: String,
pub set: bool,
}
ecstore/src/bucket/lifecycle/rule.rs
-29
View File
@@ -1,29 +0,0 @@
use super::{
delmarker::DelMarkerExpiration,
expiration::Expiration,
fileter::Filter,
noncurrentversion::{NoncurrentVersionExpiration, NoncurrentVersionTransition},
prefix::Prefix,
transition::Transition,
};
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub enum Status {
#[default]
Enabled,
Disabled,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Rule {
pub id: String,
pub status: Status,
pub filter: Filter,
pub prefix: Prefix,
pub pxpiration: Expiration,
pub transition: Transition,
pub del_marker_expiration: DelMarkerExpiration,
pub noncurrent_version_expiration: NoncurrentVersionExpiration,
pub noncurrent_version_transition: NoncurrentVersionTransition,
}
ecstore/src/bucket/lifecycle/tag.rs
-7
View File
@@ -1,7 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Tag {
pub key: String,
pub value: String,
}
ecstore/src/bucket/lifecycle/transition.rs
-16
View File
@@ -1,16 +0,0 @@
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
pub type TransitionDays = usize;
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct TransitionDate(Option<OffsetDateTime>);
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Transition {
pub days: Option<TransitionDays>,
pub date: Option<TransitionDate>,
pub storage_class: String,
pub set: bool,
}
ecstore/src/bucket/metadata.rs
+52 -22
View File
@@ -1,17 +1,19 @@
use super::{
encryption::BucketSSEConfig, event, lifecycle::lifecycle::Lifecycle, objectlock, policy::bucket_policy::BucketPolicy,
quota::BucketQuota, replication, tags::Tags, target::BucketTargets, versioning::Versioning,
};
use super::{quota::BucketQuota, target::BucketTargets};
use byteorder::{BigEndian, ByteOrder, LittleEndian};
use rmp_serde::Serializer as rmpSerializer;
use s3s::dto::{
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
};
use s3s::xml;
use s3s_policy::model::Policy;
use serde::Serializer;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use time::OffsetDateTime;
use tracing::{error, warn};
use crate::bucket::tags;
use crate::config;
use crate::config::common::{read_config, save_config};
use crate::error::{Error, Result};
@@ -34,7 +36,7 @@ pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml";
pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml";
pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json";
#[derive(Debug, Deserialize, Serialize, Clone)]
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "PascalCase", default)]
pub struct BucketMetadata {
pub name: String,
@@ -68,23 +70,23 @@ pub struct BucketMetadata {
pub new_field_updated_at: OffsetDateTime,
#[serde(skip)]
pub policy_config: Option<BucketPolicy>,
pub policy_config: Option<Policy>,
#[serde(skip)]
pub notification_config: Option<event::Config>,
pub notification_config: Option<NotificationConfiguration>,
#[serde(skip)]
pub lifecycle_config: Option<Lifecycle>,
pub lifecycle_config: Option<BucketLifecycleConfiguration>,
#[serde(skip)]
pub object_lock_config: Option<objectlock::Config>,
pub object_lock_config: Option<ObjectLockConfiguration>,
#[serde(skip)]
pub versioning_config: Option<Versioning>,
pub versioning_config: Option<VersioningConfiguration>,
#[serde(skip)]
pub sse_config: Option<BucketSSEConfig>,
pub sse_config: Option<ServerSideEncryptionConfiguration>,
#[serde(skip)]
pub tagging_config: Option<Tags>,
pub tagging_config: Option<Tagging>,
#[serde(skip)]
pub quota_config: Option<BucketQuota>,
#[serde(skip)]
pub replication_config: Option<replication::Config>,
pub replication_config: Option<ReplicationConfiguration>,
#[serde(skip)]
pub bucket_target_config: Option<BucketTargets>,
#[serde(skip)]
@@ -296,32 +298,32 @@ impl BucketMetadata {
fn parse_all_configs(&mut self, _api: &ECStore) -> Result<()> {
if !self.policy_config_json.is_empty() {
self.policy_config = Some(BucketPolicy::unmarshal(&self.policy_config_json)?);
self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?);
}
if !self.notification_config_xml.is_empty() {
self.notification_config = Some(event::Config::unmarshal(&self.notification_config_xml)?);
self.notification_config = Some(deserialize::<NotificationConfiguration>(&self.notification_config_xml)?);
}
if !self.lifecycle_config_xml.is_empty() {
self.lifecycle_config = Some(Lifecycle::unmarshal(&self.lifecycle_config_xml)?);
self.lifecycle_config = Some(deserialize::<BucketLifecycleConfiguration>(&self.lifecycle_config_xml)?);
}
if !self.object_lock_config_xml.is_empty() {
self.object_lock_config = Some(objectlock::Config::unmarshal(&self.object_lock_config_xml)?);
self.object_lock_config = Some(deserialize::<ObjectLockConfiguration>(&self.object_lock_config_xml)?);
}
if !self.versioning_config_xml.is_empty() {
self.versioning_config = Some(Versioning::unmarshal(&self.versioning_config_xml)?);
self.versioning_config = Some(deserialize::<VersioningConfiguration>(&self.versioning_config_xml)?);
}
if !self.encryption_config_xml.is_empty() {
self.sse_config = Some(BucketSSEConfig::unmarshal(&self.encryption_config_xml)?);
self.sse_config = Some(deserialize::<ServerSideEncryptionConfiguration>(&self.encryption_config_xml)?);
}
if !self.tagging_config_xml.is_empty() {
self.tagging_config = Some(tags::Tags::unmarshal(&self.tagging_config_xml)?);
self.tagging_config = Some(deserialize::<Tagging>(&self.tagging_config_xml)?);
}
if !self.quota_config_json.is_empty() {
self.quota_config = Some(BucketQuota::unmarshal(&self.quota_config_json)?);
}
if !self.replication_config_xml.is_empty() {
self.replication_config = Some(replication::Config::unmarshal(&self.replication_config_xml)?);
self.replication_config = Some(deserialize::<ReplicationConfiguration>(&self.replication_config_xml)?);
}
if !self.bucket_targets_config_json.is_empty() {
self.bucket_target_config = Some(BucketTargets::unmarshal(&self.bucket_targets_config_json)?);
@@ -413,3 +415,31 @@ mod test {
assert_eq!(bm.name, new.name);
}
}
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
where
T: for<'xml> xml::Deserialize<'xml>,
{
let mut d = xml::Deserializer::new(input);
let ans = T::deserialize(&mut d)?;
d.expect_eof()?;
Ok(ans)
}
pub fn serialize_content<T: xml::SerializeContent>(val: &T) -> xml::SerResult<String> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize_content(&mut ser)?;
}
Ok(String::from_utf8(buf).unwrap())
}
pub fn serialize<T: xml::Serialize>(val: &T) -> xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
{
let mut ser = xml::Serializer::new(&mut buf);
val.serialize(&mut ser)?;
}
Ok(buf)
}
ecstore/src/bucket/metadata_sys.rs
+126 -46
View File
@@ -11,18 +11,20 @@ use crate::error::{Error, Result};
use crate::global::{is_dist_erasure, is_erasure, new_object_layer_fn, GLOBAL_Endpoints};
use crate::store::ECStore;
use futures::future::join_all;
use lazy_static::lazy_static;
use s3s::dto::{
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
};
use s3s_policy::model::Policy;
use time::OffsetDateTime;
use tokio::sync::RwLock;
use tracing::{error, warn};
use super::encryption::BucketSSEConfig;
use super::lifecycle::lifecycle::Lifecycle;
use super::metadata::{load_bucket_metadata, BucketMetadata};
use super::policy::bucket_policy::BucketPolicy;
use super::metadata::{deserialize, load_bucket_metadata, BucketMetadata};
use super::quota::BucketQuota;
use super::target::BucketTargets;
use super::{event, objectlock, replication, tags, versioning};
use lazy_static::lazy_static;
lazy_static! {
static ref GLOBAL_BucketMetadataSys: Arc<RwLock<BucketMetadataSys>> = Arc::new(RwLock::new(BucketMetadataSys::new()));
@@ -32,24 +34,95 @@ pub async fn init_bucket_metadata_sys(api: ECStore, buckets: Vec<String>) {
let mut sys = GLOBAL_BucketMetadataSys.write().await;
sys.init(api, buckets).await
}
pub async fn get_bucket_metadata_sys() -> Arc<RwLock<BucketMetadataSys>> {
pub(super) async fn get_bucket_metadata_sys() -> Arc<RwLock<BucketMetadataSys>> {
GLOBAL_BucketMetadataSys.clone()
}
pub async fn bucket_metadata_sys_set(bucket: String, bm: BucketMetadata) {
pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) {
let sys = GLOBAL_BucketMetadataSys.write().await;
sys.set(bucket, bm).await
sys.set(bucket, Arc::new(bm)).await
}
pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
bucket_meta_sys.update(bucket, config_file, data).await
}
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
bucket_meta_sys.delete(&bucket, config_file).await
}
pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_tagging_config(bucket).await
}
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_lifecycle_config(bucket).await
}
pub async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_sse_config(bucket).await
}
pub async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_object_lock_config(bucket).await
}
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_replication_config(bucket).await
}
pub async fn get_notification_config(bucket: &str) -> Result<Option<NotificationConfiguration>> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_notification_config(bucket).await
}
pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_versioning_config(bucket).await
}
pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_config_from_disk(bucket).await
}
#[derive(Debug, Default)]
pub struct BucketMetadataSys {
metadata_map: RwLock<HashMap<String, BucketMetadata>>,
metadata_map: RwLock<HashMap<String, Arc<BucketMetadata>>>,
api: Option<ECStore>,
initialized: RwLock<bool>,
}
impl BucketMetadataSys {
fn new() -> Self {
pub fn new() -> Self {
Self::default()
}
@@ -108,7 +181,7 @@ impl BucketMetadataSys {
match res {
Ok(res) => {
if let Some(bucket) = buckets.get(idx) {
mp.insert(bucket.clone(), res);
mp.insert(bucket.clone(), Arc::new(res));
}
}
Err(e) => {
@@ -123,7 +196,7 @@ impl BucketMetadataSys {
}
}
pub async fn get(&self, bucket: &str) -> Result<BucketMetadata> {
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
if is_meta_bucketname(bucket) {
return Err(Error::new(ConfigError::NotFound));
}
@@ -136,7 +209,7 @@ impl BucketMetadataSys {
}
}
pub async fn set(&self, bucket: String, bm: BucketMetadata) {
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
if !is_meta_bucketname(&bucket) {
let mut map = self.metadata_map.write().await;
map.insert(bucket, bm);
@@ -166,7 +239,7 @@ impl BucketMetadataSys {
};
if !meta.lifecycle_config_xml.is_empty() {
let cfg = Lifecycle::unmarshal(&meta.lifecycle_config_xml)?;
let cfg = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml)?;
// TODO: FIXME:
// for _v in cfg.rules.iter() {
// break;
@@ -205,12 +278,12 @@ impl BucketMetadataSys {
let updated = bm.update_config(config_file, data)?;
self.save(&mut bm).await?;
self.save(bm).await?;
Ok(updated)
}
async fn save(&self, bm: &mut BucketMetadata) -> Result<()> {
async fn save(&self, bm: BucketMetadata) -> Result<()> {
let layer = new_object_layer_fn();
let lock = layer.read().await;
let store = match lock.as_ref() {
@@ -222,9 +295,11 @@ impl BucketMetadataSys {
return Err(Error::msg("errInvalidArgument"));
}
let mut bm = bm;
bm.save(store).await?;
self.set(bm.name.clone(), bm.clone()).await;
self.set(bm.name.clone(), Arc::new(bm)).await;
Ok(())
}
@@ -245,7 +320,7 @@ impl BucketMetadataSys {
}
}
pub async fn get_config(&self, bucket: &str) -> Result<(BucketMetadata, bool)> {
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
if let Some(api) = self.api.as_ref() {
let has_bm = {
let map = self.metadata_map.read().await;
@@ -268,6 +343,7 @@ impl BucketMetadataSys {
let mut map = self.metadata_map.write().await;
let bm = Arc::new(bm);
map.insert(bucket.to_string(), bm.clone());
Ok((bm, true))
@@ -277,23 +353,27 @@ impl BucketMetadataSys {
}
}
pub async fn get_versioning_config(&self, bucket: &str) -> Result<(versioning::Versioning, OffsetDateTime)> {
pub async fn get_versioning_config(&self, bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
warn!("get_versioning_config err {:?}", &err);
if config::error::is_not_found(&err) {
return Ok((versioning::Versioning::default(), OffsetDateTime::UNIX_EPOCH));
return Ok((VersioningConfiguration::default(), OffsetDateTime::UNIX_EPOCH));
} else {
return Err(err);
}
}
};
Ok((bm.versioning_config.unwrap_or_default(), bm.versioning_config_updated_at))
if let Some(config) = &bm.versioning_config {
Ok((config.clone(), bm.versioning_config_updated_at))
} else {
Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
}
}
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(Policy, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
@@ -306,14 +386,14 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.policy_config {
Ok((config, bm.policy_config_updated_at))
if let Some(config) = &bm.policy_config {
Ok((config.clone(), bm.policy_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::BucketPolicyNotFound))
}
}
pub async fn get_tagging_config(&self, bucket: &str) -> Result<(tags::Tags, OffsetDateTime)> {
pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
@@ -326,14 +406,14 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.tagging_config {
Ok((config, bm.tagging_config_updated_at))
if let Some(config) = &bm.tagging_config {
Ok((config.clone(), bm.tagging_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::TaggingNotFound))
}
}
pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(objectlock::Config, OffsetDateTime)> {
pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
@@ -346,14 +426,14 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.object_lock_config {
Ok((config, bm.object_lock_config_updated_at))
if let Some(config) = &bm.object_lock_config {
Ok((config.clone(), bm.object_lock_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::BucketObjectLockConfigNotFound))
}
}
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(Lifecycle, OffsetDateTime)> {
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
@@ -366,20 +446,20 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.lifecycle_config {
if let Some(config) = &bm.lifecycle_config {
if config.rules.is_empty() {
Err(Error::new(BucketMetadataError::BucketLifecycleNotFound))
} else {
Ok((config, bm.lifecycle_config_updated_at))
Ok((config.clone(), bm.lifecycle_config_updated_at))
}
} else {
Err(Error::new(BucketMetadataError::BucketLifecycleNotFound))
}
}
pub async fn get_notification_config(&self, bucket: &str) -> Result<Option<event::Config>> {
pub async fn get_notification_config(&self, bucket: &str) -> Result<Option<NotificationConfiguration>> {
let bm = match self.get_config(bucket).await {
Ok((bm, _)) => bm.notification_config,
Ok((bm, _)) => bm.notification_config.clone(),
Err(err) => {
warn!("get_notification_config err {:?}", &err);
if config::error::is_not_found(&err) {
@@ -393,7 +473,7 @@ impl BucketMetadataSys {
Ok(bm)
}
pub async fn get_sse_config(&self, bucket: &str) -> Result<(BucketSSEConfig, OffsetDateTime)> {
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
@@ -406,8 +486,8 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.sse_config {
Ok((config, bm.encryption_config_updated_at))
if let Some(config) = &bm.sse_config {
Ok((config.clone(), bm.encryption_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::BucketSSEConfigNotFound))
}
@@ -437,14 +517,14 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.quota_config {
Ok((config, bm.quota_config_updated_at))
if let Some(config) = &bm.quota_config {
Ok((config.clone(), bm.quota_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::BucketQuotaConfigNotFound))
}
}
pub async fn get_replication_config(&self, bucket: &str) -> Result<(replication::Config, OffsetDateTime)> {
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let (bm, reload) = match self.get_config(bucket).await {
Ok(res) => res,
Err(err) => {
@@ -457,12 +537,12 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.replication_config {
if let Some(config) = &bm.replication_config {
if reload {
// TODO: globalBucketTargetSys
}
Ok((config, bm.replication_config_updated_at))
Ok((config.clone(), bm.replication_config_updated_at))
} else {
Err(Error::new(BucketMetadataError::BucketReplicationConfigNotFound))
}
@@ -481,12 +561,12 @@ impl BucketMetadataSys {
}
};
if let Some(config) = bm.bucket_target_config {
if let Some(config) = &bm.bucket_target_config {
if reload {
// TODO: globalBucketTargetSys
}
Ok(config)
Ok(config.clone())
} else {
Err(Error::new(BucketMetadataError::BucketRemoteTargetNotFound))
}
ecstore/src/bucket/mod.rs
+1 -10
View File
@@ -1,18 +1,9 @@
mod encryption;
pub mod error;
mod event;
mod lifecycle;
pub mod metadata;
mod metadata_sys;
mod objectlock;
pub mod metadata_sys;
pub mod policy;
pub mod policy_sys;
mod quota;
mod replication;
pub mod tags;
mod target;
pub mod utils;
pub mod versioning;
pub mod versioning_sys;
pub use metadata_sys::{bucket_metadata_sys_set, get_bucket_metadata_sys, init_bucket_metadata_sys};
ecstore/src/bucket/objectlock/mod.rs
-56
View File
@@ -1,56 +0,0 @@
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, PartialEq, Eq, Hash, Clone)]
pub enum RetMode {
#[default]
Govenance,
Compliance,
}
// 为RetMode实现FromStr trait，方便从字符串创建枚举实例
impl std::str::FromStr for RetMode {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"GOVERNANCE" => Ok(RetMode::Govenance),
"COMPLIANCE" => Ok(RetMode::Compliance),
_ => Err(format!("Invalid RetMode: {}", s)),
}
}
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct DefaultRetention {
pub mode: RetMode,
pub days: Option<usize>,
pub years: Option<usize>,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Rule {
pub default_retention: DefaultRetention,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Config {
pub object_lock_enabled: String,
pub rule: Option<Rule>,
}
impl Config {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Config = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/policy/bucket_policy.rs
+1 -1
View File
@@ -126,7 +126,7 @@ impl BPStatement {
let mut resource = args.bucket_name.clone();
if !args.object_name.is_empty() {
if !args.object_name.starts_with("/") {
if !args.object_name.starts_with('/') {
resource.push('/');
}
ecstore/src/bucket/policy/resource.rs
+3 -3
View File
@@ -61,7 +61,7 @@ impl Resource {
if self.rtype == ResourceARNType::UnknownARN {
return false;
}
if self.is_s3() && self.pattern.starts_with("/") {
if self.is_s3() && self.pattern.starts_with('/') {
return false;
}
if self.is_kms() && self.pattern.as_bytes().iter().any(|&v| v == b'/' || v == b'\\' || v == b'.') {
@@ -77,10 +77,10 @@ impl Resource {
self.rtype == ResourceARNType::ResourceARNKMS
}
pub fn is_bucket_pattern(&self) -> bool {
!self.pattern.contains("/") || self.pattern.eq("*")
!self.pattern.contains('/') || self.pattern.eq("*")
}
pub fn is_object_pattern(&self) -> bool {
self.pattern.contains("/") || self.pattern.contains("*")
self.pattern.contains('/') || self.pattern.contains('*')
}
pub fn is_match(&self, res: &str, condition_values: &HashMap<String, Vec<String>>) -> bool {
let mut pattern = res.to_string();
ecstore/src/bucket/policy_sys.rs
+24 -18
View File
@@ -1,27 +1,23 @@
use super::{
error::BucketMetadataError,
get_bucket_metadata_sys,
policy::bucket_policy::{BucketPolicy, BucketPolicyArgs},
};
use super::metadata_sys::get_bucket_metadata_sys;
use crate::error::Result;
use tracing::warn;
use s3s_policy::model::Policy;
pub struct PolicySys {}
impl PolicySys {
pub async fn is_allowed(args: &BucketPolicyArgs) -> bool {
match Self::get(&args.bucket_name).await {
Ok(cfg) => return cfg.is_allowed(args),
Err(err) => {
if !BucketMetadataError::BucketPolicyNotFound.is(&err) {
warn!("config get err {:?}", err);
}
}
}
// pub async fn is_allowed(args: &BucketPolicyArgs) -> bool {
// match Self::get(&args.bucket_name).await {
// Ok(cfg) => return cfg.is_allowed(args),
// Err(err) => {
// if !BucketMetadataError::BucketPolicyNotFound.is(&err) {
// warn!("config get err {:?}", err);
// }
// }
// }
args.is_owner
}
pub async fn get(bucket: &str) -> Result<BucketPolicy> {
// args.is_owner
// }
pub async fn get(bucket: &str) -> Result<Policy> {
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
let bucket_meta_sys = bucket_meta_sys_lock.write().await;
@@ -30,3 +26,13 @@ impl PolicySys {
Ok(cfg)
}
}
// trait PolicyApi {
// fn is_allowed(&self) -> bool;
// }
// impl PolicyApi for Policy {
// fn is_allowed(&self) -> bool {
// todo!()
// }
// }
ecstore/src/bucket/replication/and.rs
-9
View File
@@ -1,9 +0,0 @@
use super::tag::Tag;
use serde::{Deserialize, Serialize};
// 定义And结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct And {
prefix: Option<String>,
tags: Option<Vec<Tag>>,
}
ecstore/src/bucket/replication/filter.rs
-12
View File
@@ -1,12 +0,0 @@
use super::and::And;
use super::tag::Tag;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Filter {
prefix: String,
and: And,
tag: Tag,
cached_tags: HashMap<String, String>,
}
ecstore/src/bucket/replication/mod.rs
-30
View File
@@ -1,30 +0,0 @@
mod and;
mod filter;
mod rule;
mod tag;
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use rule::Rule;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Config {
rules: Vec<Rule>,
role_arn: String,
}
impl Config {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Config = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/replication/rule.rs
-56
View File
@@ -1,56 +0,0 @@
use super::filter::Filter;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub enum Status {
#[default]
Enabled,
Disabled,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct DeleteMarkerReplication {
pub status: Status,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct DeleteReplication {
pub status: Status,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct ExistingObjectReplication {
pub status: Status,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Destination {
pub bucket: String,
pub storage_class: String,
pub arn: String,
}
// 定义ReplicaModifications结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct ReplicaModifications {
status: Status,
}
// 定义SourceSelectionCriteria结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct SourceSelectionCriteria {
replica_modifications: ReplicaModifications,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Rule {
pub id: String,
pub status: Status,
pub priority: usize,
pub delete_marker_replication: DeleteMarkerReplication,
pub delete_replication: DeleteReplication,
pub destination: Destination,
pub source_selection_criteria: SourceSelectionCriteria,
pub filter: Filter,
pub existing_object_replication: ExistingObjectReplication,
}
ecstore/src/bucket/replication/tag.rs
-7
View File
@@ -1,7 +0,0 @@
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Tag {
pub key: Option<String>,
pub value: Option<String>,
}
ecstore/src/bucket/tags/mod.rs
-37
View File
@@ -1,37 +0,0 @@
use crate::error::Result;
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
// 定义tagSet结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct TagSet {
pub tag_map: HashMap<String, String>,
pub is_object: bool,
}
// 定义tagging结构体
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Tags {
pub tag_set: TagSet,
}
impl Tags {
pub fn new(tag_map: HashMap<String, String>, is_object: bool) -> Self {
Self {
tag_set: TagSet { tag_map, is_object },
}
}
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Tags = rmp_serde::from_slice(buf)?;
Ok(t)
}
}
ecstore/src/bucket/versioning/mod.rs
-140
View File
@@ -1,140 +0,0 @@
use crate::{
error::{Error, Result},
utils,
};
use rmp_serde::Serializer as rmpSerializer;
use serde::{Deserialize, Serialize};
#[derive(Debug, thiserror::Error)]
pub enum VersioningErr {
#[error("too many excluded prefixes")]
TooManyExcludedPrefixes,
#[error("excluded prefixes extension supported only when versioning is enabled")]
ExcludedPrefixNotSupported,
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Deserialize, Serialize)]
pub enum State {
#[default]
Suspended,
Enabled,
}
// 实现Display trait用于打印
impl std::fmt::Display for State {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
"{}",
match *self {
State::Enabled => "Enabled",
State::Suspended => "Suspended",
}
)
}
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct ExcludedPrefix {
pub prefix: String,
}
#[derive(Debug, Deserialize, Serialize, Default, Clone)]
pub struct Versioning {
pub status: State,
pub excluded_prefixes: Vec<ExcludedPrefix>,
pub exclude_folders: bool,
}
impl Versioning {
pub fn marshal_msg(&self) -> Result<Vec<u8>> {
let mut buf = Vec::new();
self.serialize(&mut rmpSerializer::new(&mut buf).with_struct_map())?;
Ok(buf)
}
pub fn unmarshal(buf: &[u8]) -> Result<Self> {
let t: Versioning = rmp_serde::from_slice(buf)?;
Ok(t)
}
pub fn validate(&self) -> Result<()> {
match self.status {
State::Suspended => {
if !self.excluded_prefixes.is_empty() {
return Err(Error::new(VersioningErr::ExcludedPrefixNotSupported));
}
}
State::Enabled => {
if self.excluded_prefixes.len() > 10 {
return Err(Error::new(VersioningErr::TooManyExcludedPrefixes));
}
}
}
Ok(())
}
pub fn enabled(&self) -> bool {
self.status == State::Enabled
}
pub fn versioned(&self, prefix: &str) -> bool {
self.prefix_enabled(prefix) || self.prefix_suspended(prefix)
}
pub fn prefix_enabled(&self, prefix: &str) -> bool {
if self.status != State::Enabled {
return false;
}
if prefix.is_empty() {
return true;
}
if self.exclude_folders && prefix.ends_with("/") {
return false;
}
for sprefix in self.excluded_prefixes.iter() {
let full_prefix = format!("{}*", sprefix.prefix);
if utils::wildcard::match_simple(&full_prefix, prefix) {
return false;
}
}
true
}
pub fn suspended(&self) -> bool {
self.status == State::Suspended
}
pub fn prefix_suspended(&self, prefix: &str) -> bool {
if self.status == State::Suspended {
return true;
}
if self.status == State::Enabled {
if prefix.is_empty() {
return false;
}
if self.exclude_folders && prefix.starts_with("/") {
return true;
}
for sprefix in self.excluded_prefixes.iter() {
let full_prefix = format!("{}*", sprefix.prefix);
if utils::wildcard::match_simple(&full_prefix, prefix) {
return true;
}
}
}
false
}
pub fn prefixes_excluded(&self) -> bool {
!self.excluded_prefixes.is_empty() || self.exclude_folders
}
}
ecstore/src/bucket/versioning_sys.rs
+44 -31
View File
@@ -1,7 +1,7 @@
use super::get_bucket_metadata_sys;
use super::versioning::Versioning;
use super::metadata_sys::get_bucket_metadata_sys;
use crate::disk::RUSTFS_META_BUCKET;
use crate::error::Result;
use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
use tracing::warn;
pub struct BucketVersioningSys {}
@@ -17,39 +17,39 @@ impl BucketVersioningSys {
}
}
pub async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
match Self::get(bucket).await {
Ok(res) => res.prefix_enabled(prefix),
Err(err) => {
warn!("{:?}", err);
false
}
}
}
// pub async fn prefix_enabled(bucket: &str, prefix: &str) -> bool {
// match Self::get(bucket).await {
// Ok(res) => res.prefix_enabled(prefix),
// Err(err) => {
// warn!("{:?}", err);
// false
// }
// }
// }
pub async fn suspended(bucket: &str) -> bool {
match Self::get(bucket).await {
Ok(res) => res.suspended(),
Err(err) => {
warn!("{:?}", err);
false
}
}
}
// pub async fn suspended(bucket: &str) -> bool {
// match Self::get(bucket).await {
// Ok(res) => res.suspended(),
// Err(err) => {
// warn!("{:?}", err);
// false
// }
// }
// }
pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
match Self::get(bucket).await {
Ok(res) => res.prefix_suspended(prefix),
Err(err) => {
warn!("{:?}", err);
false
}
}
}
// pub async fn prefix_suspended(bucket: &str, prefix: &str) -> bool {
// match Self::get(bucket).await {
// Ok(res) => res.prefix_suspended(prefix),
// Err(err) => {
// warn!("{:?}", err);
// false
// }
// }
// }
pub async fn get(bucket: &str) -> Result<Versioning> {
pub async fn get(bucket: &str) -> Result<VersioningConfiguration> {
if bucket == RUSTFS_META_BUCKET || bucket.starts_with(RUSTFS_META_BUCKET) {
return Ok(Versioning::default());
return Ok(VersioningConfiguration::default());
}
let bucket_meta_sys_lock = get_bucket_metadata_sys().await;
@@ -60,3 +60,16 @@ impl BucketVersioningSys {
Ok(cfg)
}
}
trait VersioningApi {
fn enabled(&self) -> bool;
}
impl VersioningApi for VersioningConfiguration {
fn enabled(&self) -> bool {
self.status
.as_ref()
.map(|v| v.as_str() == BucketVersioningStatus::ENABLED)
.is_some_and(|v| v)
}
}
ecstore/src/disk/local.rs
+1 -1
View File
@@ -485,7 +485,7 @@ impl LocalDisk {
async fn write_all_public(&self, volume: &str, path: &str, data: Vec<u8>) -> Result<()> {
if volume == super::RUSTFS_META_BUCKET && path == super::FORMAT_CONFIG_FILE {
let mut format_info = self.format_info.write().await;
format_info.data = data.clone();
format_info.data.clone_from(&data);
}
let volume_dir = self.get_bucket_path(volume)?;
ecstore/src/disk/mod.rs
+1 -1
View File
@@ -222,7 +222,7 @@ impl MetaCacheEntry {
Ok(wr)
}
pub fn is_dir(&self) -> bool {
self.metadata.is_empty() && self.name.ends_with("/")
self.metadata.is_empty() && self.name.ends_with('/')
}
pub fn is_object(&self) -> bool {
!self.metadata.is_empty()
ecstore/src/quorum.rs
+1 -1
View File
@@ -74,7 +74,7 @@ fn reduce_errs(errs: &[Option<Error>], ignored_errs: &[Box<dyn CheckErrorFn>]) -
for (err, &count) in error_counts.iter() {
if count > max || (count == max && *err == nil) {
max = count;
max_err = err.clone();
max_err.clone_from(err);
}
}
ecstore/src/store.rs
+21 -3
View File
@@ -1,5 +1,7 @@
#![allow(clippy::map_entry)]
use crate::bucket::bucket_metadata_sys_set;
use crate::bucket::metadata;
use crate::bucket::metadata_sys::set_bucket_metadata;
use crate::disk::endpoint::EndpointType;
use crate::global::{is_dist_erasure, set_object_layer, GLOBAL_LOCAL_DISK_MAP, GLOBAL_LOCAL_DISK_SET_DRIVES};
use crate::store_api::ObjectIO;
@@ -22,6 +24,7 @@ use backon::{ExponentialBuilder, Retryable};
use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port};
use futures::future::join_all;
use http::HeaderMap;
use s3s::dto::{ObjectLockConfiguration, ObjectLockEnabled};
use std::{
collections::{HashMap, HashSet},
sync::Arc,
@@ -31,7 +34,7 @@ use time::OffsetDateTime;
use tokio::fs;
use tokio::sync::Semaphore;
use tracing::{debug, info};
use tracing::{debug, info, warn};
use uuid::Uuid;
#[derive(Debug, Clone)]
@@ -508,9 +511,24 @@ impl StorageAPI for ECStore {
self.peer_sys.make_bucket(bucket, opts).await?;
let mut meta = BucketMetadata::new(bucket);
warn!("make bucket opsts {:?}", &opts);
if opts.lock_enabled {
let cfg = ObjectLockConfiguration {
object_lock_enabled: Some(ObjectLockEnabled::from_static(ObjectLockEnabled::ENABLED)),
..Default::default()
};
meta.object_lock_config_xml = metadata::serialize::<ObjectLockConfiguration>(&cfg)?;
warn!("make bucket add object_lock_config_xml {:?}", &meta.object_lock_config_xml);
// FIXME: version config
}
meta.save(self).await?;
bucket_metadata_sys_set(bucket.to_string(), meta).await;
set_bucket_metadata(bucket.to_string(), meta).await;
// TODO: toObjectErr
ecstore/src/store_api.rs
+7 -2
View File
@@ -156,7 +156,7 @@ impl FileInfo {
ObjectInfo {
bucket: bucket.to_string(),
name: object.to_string(),
is_dir: object.starts_with("/"),
is_dir: object.starts_with('/'),
parity_blocks: self.erasure.parity_blocks,
data_blocks: self.erasure.data_blocks,
version_id: self.version_id,
@@ -258,7 +258,11 @@ pub enum BitrotAlgorithm {
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct MakeBucketOptions {
pub force_create: bool,
pub lock_enabled: bool,
pub versioning_enabled: bool,
pub force_create: bool, // Create buckets even if they are already created.
pub created_at: Option<OffsetDateTime>, // only for site replication
pub no_lock: bool,
}
pub struct DeleteBucketOptions {
@@ -554,6 +558,7 @@ pub trait StorageAPI: ObjectIO {
objects: Vec<ObjectToDelete>,
opts: ObjectOptions,
) -> Result<(Vec<DeletedObject>, Vec<Option<Error>>)>;
#[warn(clippy::too_many_arguments)]
async fn list_objects_v2(
&self,
bucket: &str,