gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-17 18:27:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: move ecstore rpc metadata modules (#3933)

Browse Source
This commit is contained in:
Zhengchao An
2026-06-27 07:19:45 +08:00
committed by GitHub
parent c6ecfae39e
commit 27bb9c75dc
20 changed files with 115 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ecstore/src/metadata/bucket.rs
+1335
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/metadata/bucket_sys.rs
+707
View File
@@ -0,0 +1,707 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::metadata::{BucketMetadata, load_bucket_metadata};
use super::quota::BucketQuota;
use super::target::BucketTargets;
use crate::bucket::bucket_target_sys::BucketTargetSys;
use crate::bucket::metadata::{BUCKET_LIFECYCLE_CONFIG, load_bucket_metadata_parse};
use crate::bucket::utils::{deserialize, is_meta_bucketname};
use crate::error::{Error, Result, is_err_bucket_not_found};
use crate::runtime_sources;
use crate::storage_api_contracts::heal::HealOperations as _;
use crate::store::ECStore;
use futures::future::join_all;
use lazy_static::lazy_static;
use rustfs_common::heal_channel::HealOpts;
use rustfs_policy::policy::BucketPolicy;
use s3s::dto::ReplicationConfiguration;
use s3s::dto::{
AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration,
ObjectLockConfiguration, PublicAccessBlockConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration,
Tagging, VersioningConfiguration, WebsiteConfiguration,
};
use std::collections::HashSet;
use std::sync::OnceLock;
use std::time::Duration;
use std::{collections::HashMap, sync::Arc};
use time::OffsetDateTime;
use tokio::sync::RwLock;
use tokio::time::sleep;
use tracing::error;
lazy_static! {
pub static ref GLOBAL_BucketMetadataSys: OnceLock<Arc<RwLock<BucketMetadataSys>>> = OnceLock::new();
}
pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
let mut sys = BucketMetadataSys::new(api);
sys.init(buckets).await;
let sys = Arc::new(RwLock::new(sys));
GLOBAL_BucketMetadataSys.set(sys).unwrap();
}
pub fn get_global_bucket_metadata_sys() -> Option<Arc<RwLock<BucketMetadataSys>>> {
GLOBAL_BucketMetadataSys.get().cloned()
}
// panic if not init
pub(super) fn get_bucket_metadata_sys() -> Result<Arc<RwLock<BucketMetadataSys>>> {
if let Some(sys) = GLOBAL_BucketMetadataSys.get() {
Ok(sys.clone())
} else {
Err(Error::other("GLOBAL_BucketMetadataSys not init"))
}
}
pub async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> Result<()> {
let sys = get_bucket_metadata_sys()?;
let lock = sys.write().await;
lock.set(bucket, Arc::new(bm)).await;
Ok(())
}
pub async fn get(bucket: &str) -> Result<Arc<BucketMetadata>> {
let sys = get_bucket_metadata_sys()?;
let lock = sys.read().await;
lock.get(bucket).await
}
pub async fn update(bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
bucket_meta_sys.update(bucket, config_file, data).await
}
pub async fn delete(bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let mut bucket_meta_sys = bucket_meta_sys_lock.write().await;
bucket_meta_sys.delete(bucket, config_file).await
}
pub async fn get_bucket_policy(bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_bucket_policy(bucket).await
}
/// Returns the raw JSON string of the bucket policy as originally stored.
/// This preserves the exact format of the policy document as it was PUT.
pub async fn get_bucket_policy_raw(bucket: &str) -> Result<(String, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_bucket_policy_raw(bucket).await
}
pub async fn get_bucket_acl_config(bucket: &str) -> Result<(String, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_bucket_acl_config(bucket).await
}
pub async fn get_quota_config(bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_quota_config(bucket).await
}
pub async fn get_bucket_targets_config(bucket: &str) -> Result<BucketTargets> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_bucket_targets_config(bucket).await
}
pub async fn get_cors_config(bucket: &str) -> Result<(CORSConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_cors_config(bucket).await
}
pub async fn get_tagging_config(bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_tagging_config(bucket).await
}
pub async fn get_public_access_block_config(bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_public_access_block_config(bucket).await
}
pub async fn get_lifecycle_config(bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_lifecycle_config(bucket).await
}
pub async fn get_sse_config(bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_sse_config(bucket).await
}
pub async fn get_object_lock_config(bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_object_lock_config(bucket).await
}
pub async fn get_replication_config(bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_replication_config(bucket).await
}
pub async fn get_notification_config(bucket: &str) -> Result<Option<NotificationConfiguration>> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_notification_config(bucket).await
}
pub async fn get_versioning_config(bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_versioning_config(bucket).await
}
pub async fn get_website_config(bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_website_config(bucket).await
}
pub async fn get_logging_config(bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_logging_config(bucket).await
}
pub async fn get_accelerate_config(bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_accelerate_config(bucket).await
}
pub async fn get_request_payment_config(bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_request_payment_config(bucket).await
}
pub async fn get_config_from_disk(bucket: &str) -> Result<BucketMetadata> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_config_from_disk(bucket).await
}
pub async fn created_at(bucket: &str) -> Result<OffsetDateTime> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.created_at(bucket).await
}
pub async fn list_bucket_targets(bucket: &str) -> Result<BucketTargets> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
bucket_meta_sys.get_bucket_targets_config(bucket).await
}
#[derive(Debug)]
pub struct BucketMetadataSys {
metadata_map: RwLock<HashMap<String, Arc<BucketMetadata>>>,
api: Arc<ECStore>,
initialized: RwLock<bool>,
}
impl BucketMetadataSys {
pub fn new(api: Arc<ECStore>) -> Self {
Self {
metadata_map: RwLock::new(HashMap::new()),
api,
initialized: RwLock::new(false),
}
}
pub async fn init(&mut self, buckets: Vec<String>) {
let _ = self.init_internal(buckets).await;
}
async fn init_internal(&self, buckets: Vec<String>) -> Result<()> {
let count = runtime_sources::endpoint_erasure_set_count()
.map(|count| count * 10)
.ok_or_else(|| Error::other("GLOBAL_Endpoints not init"))?;
let mut failed_buckets: HashSet<String> = HashSet::new();
let mut buckets = buckets.as_slice();
loop {
if buckets.len() < count {
self.concurrent_load(buckets, &mut failed_buckets).await;
break;
}
self.concurrent_load(&buckets[..count], &mut failed_buckets).await;
buckets = &buckets[count..]
}
let mut initialized = self.initialized.write().await;
*initialized = true;
if runtime_sources::setup_is_dist_erasure().await {
// TODO: refresh_buckets_metadata_loop
}
Ok(())
}
async fn concurrent_load(&self, buckets: &[String], failed_buckets: &mut HashSet<String>) {
let mut futures = Vec::new();
for bucket in buckets.iter() {
// TODO: HealBucket
let api = self.api.clone();
let bucket = bucket.clone();
futures.push(async move {
sleep(Duration::from_millis(30)).await;
let _ = api
.heal_bucket(
&bucket,
&HealOpts {
recreate: true,
..Default::default()
},
)
.await;
load_bucket_metadata(self.api.clone(), bucket.as_str()).await
});
}
let results = join_all(futures).await;
let mut idx = 0;
let mut mp = self.metadata_map.write().await;
// TODO:EventNotifier,BucketTargetSys
for res in results {
match res {
Ok(res) => {
if let Some(bucket) = buckets.get(idx) {
let x = Arc::new(res);
mp.insert(bucket.clone(), x.clone());
// TODO:EventNotifier,BucketTargetSys
BucketTargetSys::get().set(bucket, &x).await;
}
}
Err(e) => {
error!("Unable to load bucket metadata, will be retried: {:?}", e);
if let Some(bucket) = buckets.get(idx) {
failed_buckets.insert(bucket.clone());
}
}
}
idx += 1;
}
}
pub async fn get(&self, bucket: &str) -> Result<Arc<BucketMetadata>> {
if is_meta_bucketname(bucket) {
return Err(Error::ConfigNotFound);
}
let map = self.metadata_map.read().await;
if let Some(bm) = map.get(bucket) {
Ok(bm.clone())
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
if !is_meta_bucketname(&bucket) {
let mut map = self.metadata_map.write().await;
map.insert(bucket, bm);
}
}
async fn _reset(&mut self) {
let mut map = self.metadata_map.write().await;
map.clear();
}
pub async fn update(&mut self, bucket: &str, config_file: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
self.update_and_parse(bucket, config_file, data, true).await
}
pub async fn delete(&mut self, bucket: &str, config_file: &str) -> Result<OffsetDateTime> {
if config_file == BUCKET_LIFECYCLE_CONFIG {
let meta = match self.get_config_from_disk(bucket).await {
Ok(res) => res,
Err(err) => {
if err != Error::ConfigNotFound {
return Err(err);
} else {
BucketMetadata::new(bucket)
}
}
};
if !meta.lifecycle_config_xml.is_empty() {
if let Ok(cfg) = deserialize::<BucketLifecycleConfiguration>(&meta.lifecycle_config_xml) {
if let Some(_v) = cfg.rules.first() {}
} else {
tracing::warn!(
bucket = %bucket,
"delete: failed to parse lifecycle config XML"
);
}
}
// TODO: other lifecycle handle
}
self.update_and_parse(bucket, config_file, Vec::new(), false).await
}
async fn update_and_parse(&mut self, bucket: &str, config_file: &str, data: Vec<u8>, parse: bool) -> Result<OffsetDateTime> {
let Some(store) = runtime_sources::object_store_handle() else {
return Err(Error::other("errServerNotInitialized"));
};
if is_meta_bucketname(bucket) {
return Err(Error::other("errInvalidArgument"));
}
let mut bm = match load_bucket_metadata_parse(store, bucket, parse).await {
Ok(res) => res,
Err(err) => {
if !runtime_sources::setup_is_erasure().await
&& !runtime_sources::setup_is_dist_erasure().await
&& is_err_bucket_not_found(&err)
{
BucketMetadata::new(bucket)
} else {
error!("load bucket metadata failed: {}", err);
return Err(err);
}
}
};
let updated = bm.update_config(config_file, data)?;
self.save(bm).await?;
Ok(updated)
}
async fn save(&self, bm: BucketMetadata) -> Result<()> {
if is_meta_bucketname(&bm.name) {
return Err(Error::other("errInvalidArgument"));
}
let mut bm = bm;
bm.save().await?;
self.set(bm.name.clone(), Arc::new(bm)).await;
Ok(())
}
pub async fn get_config_from_disk(&self, bucket: &str) -> Result<BucketMetadata> {
if is_meta_bucketname(bucket) {
return Err(Error::other("errInvalidArgument"));
}
load_bucket_metadata(self.api.clone(), bucket).await
}
pub async fn get_config(&self, bucket: &str) -> Result<(Arc<BucketMetadata>, bool)> {
let has_bm = {
let map = self.metadata_map.read().await;
map.get(&bucket.to_string()).cloned()
};
if let Some(bm) = has_bm {
Ok((bm, false))
} else {
let bm = match load_bucket_metadata(self.api.clone(), bucket).await {
Ok(res) => res,
Err(err) => {
return if *self.initialized.read().await {
Err(Error::other("errBucketMetadataNotInitialized"))
} else {
Err(err)
};
}
};
let mut map = self.metadata_map.write().await;
let bm = Arc::new(bm);
map.insert(bucket.to_string(), bm.clone());
Ok((bm, true))
}
}
pub async fn get_versioning_config(&self, bucket: &str) -> Result<(VersioningConfiguration, OffsetDateTime)> {
let bm = match self.get_config(bucket).await {
Ok((res, _)) => res,
Err(err) => {
return if err == Error::ConfigNotFound {
Ok((VersioningConfiguration::default(), OffsetDateTime::UNIX_EPOCH))
} else {
Err(err)
};
}
};
if let Some(config) = &bm.versioning_config {
Ok((config.clone(), bm.versioning_config_updated_at))
} else {
Ok((VersioningConfiguration::default(), bm.versioning_config_updated_at))
}
}
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.policy_config {
Ok((config.clone(), bm.policy_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
/// Returns the raw JSON string of the bucket policy as originally stored.
/// This preserves the exact format of the policy document as it was PUT.
pub async fn get_bucket_policy_raw(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if bm.policy_config_json.is_empty() {
Err(Error::ConfigNotFound)
} else {
let policy_str = String::from_utf8(bm.policy_config_json.clone())
.map_err(|e| Error::other(format!("invalid UTF-8 in policy JSON: {}", e)))?;
Ok((policy_str, bm.policy_config_updated_at))
}
}
pub async fn get_bucket_acl_config(&self, bucket: &str) -> Result<(String, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.bucket_acl_config {
Ok((config.clone(), bm.bucket_acl_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_tagging_config(&self, bucket: &str) -> Result<(Tagging, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.tagging_config {
Ok((config.clone(), bm.tagging_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_public_access_block_config(&self, bucket: &str) -> Result<(PublicAccessBlockConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.public_access_block_config {
Ok((config.clone(), bm.public_access_block_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_object_lock_config(&self, bucket: &str) -> Result<(ObjectLockConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.object_lock_config {
Ok((config.clone(), bm.object_lock_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_lifecycle_config(&self, bucket: &str) -> Result<(BucketLifecycleConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.lifecycle_config {
if config.rules.is_empty() {
Err(Error::ConfigNotFound)
} else {
Ok((config.clone(), bm.lifecycle_config_updated_at))
}
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_notification_config(&self, bucket: &str) -> Result<Option<NotificationConfiguration>> {
let bm = match self.get_config(bucket).await {
Ok((bm, _)) => bm.notification_config.clone(),
Err(err) => {
if err == Error::ConfigNotFound {
None
} else {
return Err(err);
}
}
};
Ok(bm)
}
pub async fn get_sse_config(&self, bucket: &str) -> Result<(ServerSideEncryptionConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.sse_config {
Ok((config.clone(), bm.encryption_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_cors_config(&self, bucket: &str) -> Result<(CORSConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.cors_config {
Ok((config.clone(), bm.cors_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_website_config(&self, bucket: &str) -> Result<(WebsiteConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.website_config {
Ok((config.clone(), bm.website_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_logging_config(&self, bucket: &str) -> Result<(BucketLoggingStatus, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.logging_config {
Ok((config.clone(), bm.logging_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_accelerate_config(&self, bucket: &str) -> Result<(AccelerateConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.accelerate_config {
Ok((config.clone(), bm.accelerate_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_request_payment_config(&self, bucket: &str) -> Result<(RequestPaymentConfiguration, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.request_payment_config {
Ok((config.clone(), bm.request_payment_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn created_at(&self, bucket: &str) -> Result<OffsetDateTime> {
let bm = match self.get_config(bucket).await {
Ok((bm, _)) => bm.created,
Err(err) => {
return Err(err);
}
};
Ok(bm)
}
pub async fn get_quota_config(&self, bucket: &str) -> Result<(BucketQuota, OffsetDateTime)> {
let (bm, _) = self.get_config(bucket).await?;
if let Some(config) = &bm.quota_config {
Ok((config.clone(), bm.quota_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_replication_config(&self, bucket: &str) -> Result<(ReplicationConfiguration, OffsetDateTime)> {
let (bm, reload) = self.get_config(bucket).await?;
if let Some(config) = &bm.replication_config {
if reload {
// TODO: globalBucketTargetSys
}
//println!("549 {:?}", config.clone());
Ok((config.clone(), bm.replication_config_updated_at))
} else {
Err(Error::ConfigNotFound)
}
}
pub async fn get_bucket_targets_config(&self, bucket: &str) -> Result<BucketTargets> {
let (bm, reload) = self.get_config(bucket).await?;
if let Some(config) = &bm.bucket_target_config {
if reload {
// TODO: globalBucketTargetSys
//config.
}
Ok(config.clone())
} else {
Err(Error::ConfigNotFound)
}
}
}
crates/ecstore/src/metadata/bucket_test.rs
+398
View File
@@ -0,0 +1,398 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::metadata::BucketMetadata;
use time::OffsetDateTime;
/// Full BucketMetadata hex (all fields populated).
const TEST_BUCKET_METADATA_HEX: &str = "de0019a44e616d65b27275737466732d636f6d7061742d74657374a743726561746564c70c050000000065920080075bcd15ab4c6f636b456e61626c6564c3b0506f6c696379436f6e6669674a534f4ec4907b2256657273696f6e223a22323031322d31302d3137222c2253746174656d656e74223a5b7b22456666656374223a22416c6c6f77222c225072696e636970616c223a222a222c22416374696f6e223a2273333a4765744f626a656374222c225265736f75726365223a2261726e3a6177733a73333a3a3a7275737466732d636f6d7061742d746573742f2a227d5d7db54e6f74696669636174696f6e436f6e666967584d4cc4963c4e6f74696669636174696f6e436f6e66696775726174696f6e3e3c436c6f75645761746368436f6e66696775726174696f6e3e3c49643e6e313c2f49643e3c4576656e743e73333a4f626a656374437265617465643a2a3c2f4576656e743e3c2f436c6f75645761746368436f6e66696775726174696f6e3e3c2f4e6f74696669636174696f6e436f6e66696775726174696f6e3eb24c6966656379636c65436f6e666967584d4cc48c3c4c6966656379636c65436f6e66696775726174696f6e3e3c52756c653e3c49443e72756c65313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c45787069726174696f6e3e3c446179733e33303c2f446179733e3c2f45787069726174696f6e3e3c2f52756c653e3c2f4c6966656379636c65436f6e66696775726174696f6e3eb34f626a6563744c6f636b436f6e666967584d4cc4b83c4f626a6563744c6f636b436f6e66696775726174696f6e3e3c4f626a6563744c6f636b456e61626c65643e456e61626c65643c2f4f626a6563744c6f636b456e61626c65643e3c52756c653e3c44656661756c74526574656e74696f6e3e3c4d6f64653e474f5645524e414e43453c2f4d6f64653e3c446179733e373c2f446179733e3c2f44656661756c74526574656e74696f6e3e3c2f52756c653e3c2f4f626a6563744c6f636b436f6e66696775726174696f6e3eb356657273696f6e696e67436f6e666967584d4cc44b3c56657273696f6e696e67436f6e66696775726174696f6e3e3c5374617475733e456e61626c65643c2f5374617475733e3c2f56657273696f6e696e67436f6e66696775726174696f6e3eb3456e6372797074696f6e436f6e666967584d4cc4c03c53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3e3c52756c653e3c4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c535345416c676f726974686d3e4145533235363c2f535345416c676f726974686d3e3c2f4170706c7953657276657253696465456e6372797074696f6e427944656661756c743e3c2f52756c653e3c2f53657276657253696465456e6372797074696f6e436f6e66696775726174696f6e3eb054616767696e67436f6e666967584d4cc4503c54616767696e673e3c5461675365743e3c5461673e3c4b65793e456e763c2f4b65793e3c56616c75653e546573743c2f56616c75653e3c2f5461673e3c2f5461675365743e3c2f54616767696e673eaf51756f7461436f6e6669674a534f4ec4707b2271756f7461223a313037333734313832342c2271756f74615f74797065223a2248617264222c22637265617465645f6174223a22323032342d30312d30315430303a30303a30305a222c22757064617465645f6174223a22323032342d30312d30315430303a30303a30305a227db45265706c69636174696f6e436f6e666967584d4cc4e73c5265706c69636174696f6e436f6e66696775726174696f6e3e3c526f6c653e61726e3a6177733a69616d3a3a3132333435363738393031323a726f6c652f7265706c3c2f526f6c653e3c52756c653e3c49443e72313c2f49443e3c5374617475733e456e61626c65643c2f5374617475733e3c5072656669783e646f632f3c2f5072656669783e3c44657374696e6174696f6e3e3c4275636b65743e61726e3a6177733a73333a3a3a646573743c2f4275636b65743e3c2f44657374696e6174696f6e3e3c2f52756c653e3c2f5265706c69636174696f6e436f6e66696775726174696f6e3eb74275636b657454617267657473436f6e6669674a534f4ec4535b7b22656e64706f696e74223a22687474703a2f2f7461726765742e6578616d706c652e636f6d222c227461726765744275636b6574223a227462222c22726567696f6e223a2275732d656173742d31227d5dbb4275636b657454617267657473436f6e6669674d6574614a534f4ec42d7b227265706c69636174696f6e4964223a227265706c2d31222c2273796e634d6f6465223a226173796e63227db5506f6c696379436f6e666967557064617465644174c70c050000000065a5022000000000b94f626a6563744c6f636b436f6e666967557064617465644174c70c050000000065a5022000000000b9456e6372797074696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b654616767696e67436f6e666967557064617465644174c70c050000000065a5022000000000b451756f7461436f6e666967557064617465644174c70c050000000065a5022000000000ba5265706c69636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000b956657273696f6e696e67436f6e666967557064617465644174c70c050000000065a5022000000000b84c6966656379636c65436f6e666967557064617465644174c70c050000000065a5022000000000bb4e6f74696669636174696f6e436f6e666967557064617465644174c70c050000000065a5022000000000bc4275636b657454617267657473436f6e666967557064617465644174c70c050000000065a5022000000000d9204275636b657454617267657473436f6e6669674d657461557064617465644174c70c050000000065a5022000000000";
#[tokio::test]
async fn marshal_msg() {
let bm = BucketMetadata::new("dada");
let buf = bm.marshal_msg().unwrap();
let new = BucketMetadata::unmarshal(&buf).unwrap();
assert_eq!(bm.name, new.name);
}
/// Verifies that serialized time uses msgp ext type 5.
#[tokio::test]
async fn marshal_msg_uses_time_format() {
let mut bm = BucketMetadata::new("test-bucket");
bm.created = OffsetDateTime::from_unix_timestamp(1704067200).unwrap(); // 2024-01-01 00:00:00 UTC
let buf = bm.marshal_msg().unwrap();
// msgp uses ext8 (0xc7), len 12, type 5 for time
assert!(
buf.windows(3).any(|w| w == [0xc7, 0x0c, 0x05]),
"serialized data should contain msgp time ext (0xc7 0x0c 0x05)"
);
}
#[tokio::test]
async fn unmarshal_test_bucket_metadata() {
use faster_hex::hex_decode;
let mut bytes = vec![0u8; TEST_BUCKET_METADATA_HEX.len() / 2];
hex_decode(TEST_BUCKET_METADATA_HEX.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("RustFS must unmarshal MinIO format");
assert_eq!(bm.name, "rustfs-compat-test");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert_eq!(bm.created.nanosecond(), 123456789);
assert!(bm.lock_enabled);
assert!(!bm.policy_config_json.is_empty());
assert!(bm.policy_config_json.starts_with(b"{\"Version\""));
assert!(!bm.notification_config_xml.is_empty());
assert!(bm.notification_config_xml.starts_with(b"<Notification"));
assert!(!bm.lifecycle_config_xml.is_empty());
assert!(bm.lifecycle_config_xml.starts_with(b"<Lifecycle"));
assert!(!bm.object_lock_config_xml.is_empty());
assert!(bm.object_lock_config_xml.starts_with(b"<ObjectLock"));
assert!(!bm.versioning_config_xml.is_empty());
assert!(bm.versioning_config_xml.starts_with(b"<Versioning"));
assert!(!bm.encryption_config_xml.is_empty());
assert!(bm.encryption_config_xml.starts_with(b"<ServerSide"));
assert!(!bm.tagging_config_xml.is_empty());
assert!(bm.tagging_config_xml.starts_with(b"<Tagging"));
assert!(!bm.quota_config_json.is_empty());
assert!(bm.quota_config_json.starts_with(b"{\"quota\""));
assert!(!bm.replication_config_xml.is_empty());
assert!(bm.replication_config_xml.starts_with(b"<Replication"));
assert!(!bm.bucket_targets_config_json.is_empty());
assert!(bm.bucket_targets_config_json.starts_with(b"[{"));
assert!(!bm.bucket_targets_config_meta_json.is_empty());
assert!(bm.bucket_targets_config_meta_json.starts_with(b"{\"replication"));
let updated_sec = 1705312800; // 2024-01-15 12:00:00 UTC
assert_eq!(bm.policy_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.object_lock_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.encryption_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.tagging_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.quota_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.replication_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.versioning_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.lifecycle_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.notification_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_updated_at.unix_timestamp(), updated_sec);
assert_eq!(bm.bucket_targets_config_meta_updated_at.unix_timestamp(), updated_sec);
assert!(bm.cors_config_xml.is_empty());
assert!(bm.public_access_block_config_xml.is_empty());
assert!(bm.bucket_acl_config_json.is_empty());
}
#[test]
fn unmarshal_legacy_compact_time_bucket_metadata() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"83",
"a44e616d65",
"a474657374",
"a743726561746564",
"99cd07e9cd01100c1021ce2026b1fa000000",
"ab4c6f636b456e61626c6564",
"c2"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy compact time should decode");
assert_eq!(bm.name, "test");
assert_eq!(bm.created.unix_timestamp(), 1759148193);
assert_eq!(bm.created.nanosecond(), 539406842);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_bin_wrapped_ext_time_bucket_metadata() {
use faster_hex::hex_decode;
let wrapped_hex = concat!(
"83",
"a44e616d65",
"a464616461",
"a743726561746564",
"c40fc70c05fffffff1886e090000000000",
"ab4c6f636b456e61626c6564",
"c2"
);
let mut bytes = vec![0u8; wrapped_hex.len() / 2];
hex_decode(wrapped_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("bin-wrapped ext time should decode");
assert_eq!(bm.name, "dada");
assert_eq!(bm.created.unix_timestamp(), -62135596800);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_legacy_rmp_serde_field_aliases_and_byte_arrays() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"85",
"a44e616d65",
"a474657374",
"a743726561746564",
"99cd07e9cd01100c1021ce2026b1fa000000",
"ab4c6f636b456e61626c6564",
"c2",
"b0506f6c696379436f6e6669674a736f6e",
"93010203",
"bb4275636b657454617267657473436f6e6669674d6574614a736f6e",
"920405"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy field aliases and byte arrays should decode");
assert_eq!(bm.name, "test");
assert_eq!(bm.created.unix_timestamp(), 1759148193);
assert_eq!(bm.created.nanosecond(), 539406842);
assert_eq!(bm.policy_config_json, vec![1, 2, 3]);
assert_eq!(bm.bucket_targets_config_meta_json, vec![4, 5]);
assert!(!bm.lock_enabled);
}
#[test]
fn unmarshal_legacy_bin16_and_array16_bucket_metadata() {
let policy = vec![b'x'; 257];
let targets_meta: Vec<u8> = (0u8..=16).collect();
let mut bytes = Vec::new();
rmp::encode::write_map_len(&mut bytes, 5).unwrap();
rmp::encode::write_str(&mut bytes, "Name").unwrap();
rmp::encode::write_str(&mut bytes, "test-bucket").unwrap();
rmp::encode::write_str(&mut bytes, "Created").unwrap();
bytes.extend_from_slice(&[
0xc7, 0x0c, 0x05, 0x00, 0x00, 0x00, 0x00, 0x65, 0x92, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00,
]);
rmp::encode::write_str(&mut bytes, "LockEnabled").unwrap();
bytes.push(0x01);
rmp::encode::write_str(&mut bytes, "PolicyConfigJson").unwrap();
rmp::encode::write_bin(&mut bytes, &policy).unwrap();
rmp::encode::write_str(&mut bytes, "BucketTargetsConfigMetaJson").unwrap();
rmp::encode::write_array_len(&mut bytes, targets_meta.len() as u32).unwrap();
for byte in &targets_meta {
rmp::encode::write_uint(&mut bytes, u64::from(*byte)).unwrap();
}
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy bin16 and array16 should decode");
assert_eq!(bm.name, "test-bucket");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert!(bm.lock_enabled);
assert_eq!(bm.policy_config_json, policy);
assert_eq!(bm.bucket_targets_config_meta_json, targets_meta);
}
#[test]
fn unmarshal_legacy_numeric_bool_bucket_metadata() {
use faster_hex::hex_decode;
let legacy_hex = concat!(
"83",
"a44e616d65",
"ab746573742d6275636b6574",
"a743726561746564",
"c70c05000000006592008000000000",
"ab4c6f636b456e61626c6564",
"01"
);
let mut bytes = vec![0u8; legacy_hex.len() / 2];
hex_decode(legacy_hex.as_bytes(), &mut bytes).expect("valid hex");
let bm = BucketMetadata::unmarshal(&bytes).expect("legacy numeric bool should decode");
assert_eq!(bm.name, "test-bucket");
assert_eq!(bm.created.unix_timestamp(), 1704067200);
assert_eq!(bm.created.nanosecond(), 0);
assert!(bm.lock_enabled);
}
#[tokio::test]
async fn marshal_msg_complete_example() {
// Create a complete BucketMetadata with various configurations
let mut bm = BucketMetadata::new("test-bucket");
// Set creation time to current time
bm.created = OffsetDateTime::now_utc();
bm.lock_enabled = true;
// Add policy configuration
let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#;
bm.policy_config_json = policy_json.as_bytes().to_vec();
bm.policy_config_updated_at = OffsetDateTime::now_utc();
// Add lifecycle configuration
let lifecycle_xml = r#"<LifecycleConfiguration><Rule><ID>rule1</ID><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#;
bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec();
bm.lifecycle_config_updated_at = OffsetDateTime::now_utc();
// Add versioning configuration
let versioning_xml = r#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
bm.versioning_config_xml = versioning_xml.as_bytes().to_vec();
bm.versioning_config_updated_at = OffsetDateTime::now_utc();
// Add encryption configuration
let encryption_xml = r#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#;
bm.encryption_config_xml = encryption_xml.as_bytes().to_vec();
bm.encryption_config_updated_at = OffsetDateTime::now_utc();
// Add tagging configuration
let tagging_xml = r#"<Tagging><TagSet><Tag><Key>Environment</Key><Value>Test</Value></Tag><Tag><Key>Owner</Key><Value>RustFS</Value></Tag></TagSet></Tagging>"#;
bm.tagging_config_xml = tagging_xml.as_bytes().to_vec();
bm.tagging_config_updated_at = OffsetDateTime::now_utc();
// Add quota configuration
let quota_json =
r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota
bm.quota_config_json = quota_json.as_bytes().to_vec();
bm.quota_config_updated_at = OffsetDateTime::now_utc();
// Add object lock configuration
let object_lock_xml = r#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#;
bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec();
bm.object_lock_config_updated_at = OffsetDateTime::now_utc();
// Add notification configuration
let notification_xml = r#"<NotificationConfiguration><CloudWatchConfiguration><Id>notification1</Id><Event>s3:ObjectCreated:*</Event><CloudWatchConfiguration><LogGroupName>test-log-group</LogGroupName></CloudWatchConfiguration></CloudWatchConfiguration></NotificationConfiguration>"#;
bm.notification_config_xml = notification_xml.as_bytes().to_vec();
bm.notification_config_updated_at = OffsetDateTime::now_utc();
// Add replication configuration
let replication_xml = r#"<ReplicationConfiguration><Role>arn:aws:iam::123456789012:role/replication-role</Role><Rule><ID>rule1</ID><Status>Enabled</Status><Prefix>documents/</Prefix><Destination><Bucket>arn:aws:s3:::destination-bucket</Bucket></Destination></Rule></ReplicationConfiguration>"#;
bm.replication_config_xml = replication_xml.as_bytes().to_vec();
bm.replication_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets configuration
let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#;
bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec();
bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc();
// Add bucket targets meta configuration
let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#;
bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec();
bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc();
// Add public access block configuration
let public_access_block_xml = r#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#;
bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec();
bm.public_access_block_config_updated_at = OffsetDateTime::now_utc();
let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#;
bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec();
bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc();
// Test serialization
let buf = bm.marshal_msg().unwrap();
assert!(!buf.is_empty(), "Serialized buffer should not be empty");
// Test deserialization
let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap();
// Verify all fields are correctly serialized and deserialized
assert_eq!(bm.name, deserialized_bm.name);
assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp());
assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled);
// Verify configuration data
assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json);
assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml);
assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml);
assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml);
assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml);
assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json);
assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml);
assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json);
assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml);
assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml);
assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml);
assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json);
assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json);
// Verify timestamps (comparing unix timestamps to avoid precision issues)
assert_eq!(
bm.policy_config_updated_at.unix_timestamp(),
deserialized_bm.policy_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.lifecycle_config_updated_at.unix_timestamp(),
deserialized_bm.lifecycle_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.versioning_config_updated_at.unix_timestamp(),
deserialized_bm.versioning_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.encryption_config_updated_at.unix_timestamp(),
deserialized_bm.encryption_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.tagging_config_updated_at.unix_timestamp(),
deserialized_bm.tagging_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.quota_config_updated_at.unix_timestamp(),
deserialized_bm.quota_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.object_lock_config_updated_at.unix_timestamp(),
deserialized_bm.object_lock_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.notification_config_updated_at.unix_timestamp(),
deserialized_bm.notification_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.replication_config_updated_at.unix_timestamp(),
deserialized_bm.replication_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_updated_at.unix_timestamp()
);
assert_eq!(
bm.bucket_targets_config_meta_updated_at.unix_timestamp(),
deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp()
);
// Test that the serialized data contains expected content
let buf_str = String::from_utf8_lossy(&buf);
assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name");
// Verify the buffer size is reasonable (should be larger due to all the config data)
assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations");
println!("✅ Complete BucketMetadata serialization test passed");
println!(" - Bucket name: {}", deserialized_bm.name);
println!(" - Lock enabled: {}", deserialized_bm.lock_enabled);
println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len());
println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len());
println!(" - Serialized buffer size: {} bytes", buf.len());
}
crates/ecstore/src/metadata/set_disk.rs
+597
View File
@@ -0,0 +1,597 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
impl SetDisks {
pub(super) fn all_not_found_metadata(errs: &[Option<DiskError>]) -> bool {
!errs.is_empty()
&& errs.iter().all(|err| match err {
Some(err) => {
matches!(
err,
DiskError::FileNotFound
| DiskError::FileVersionNotFound
| DiskError::VolumeNotFound
| DiskError::DiskNotFound
) || OBJECT_OP_IGNORED_ERRS.contains(err)
}
None => false,
})
&& errs.iter().any(|err| {
matches!(
err,
Some(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound)
)
})
}
pub(super) fn reduce_common_data_dir(data_dirs: &[Option<Uuid>], write_quorum: usize) -> Option<Uuid> {
let mut data_dirs_count = HashMap::new();
for ddir in data_dirs.iter().flatten().copied() {
*data_dirs_count.entry(ddir).or_insert(0) += 1;
}
let mut max = 0;
let mut data_dir = None;
for (ddir, count) in data_dirs_count {
if count > max {
max = count;
data_dir = Some(ddir);
}
}
if max >= write_quorum { data_dir } else { None }
}
pub(super) fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String {
let upload_uuid = base64_simd::URL_SAFE_NO_PAD
.decode_to_vec(upload_id.as_bytes())
.and_then(|v| {
String::from_utf8(v).map_or_else(
|_| Ok(upload_id.to_owned()),
|v| {
let parts: Vec<_> = v.splitn(2, '.').collect();
if parts.len() == 2 {
Ok(parts[1].to_string())
} else {
Ok(upload_id.to_string())
}
},
)
})
.unwrap_or_default();
format!("{}/{}", Self::get_multipart_sha_dir(bucket, object), upload_uuid)
}
pub(super) fn get_multipart_sha_dir(bucket: &str, object: &str) -> String {
let path = format!("{bucket}/{object}");
let mut hasher = Sha256::new();
hasher.update(path);
hex(hasher.finalize())
}
pub(super) fn common_parity(parities: &[i32], default_parity_count: i32) -> i32 {
let n = parities.len() as i32;
let mut occ_map: HashMap<i32, i32> = HashMap::new();
for &p in parities {
*occ_map.entry(p).or_insert(0) += 1;
}
let mut max_occ = 0;
let mut cparity = 0;
for (&parity, &occ) in &occ_map {
if parity == -1 {
// Ignore non defined parity
continue;
}
let mut read_quorum = n - parity;
if default_parity_count > 0 && parity == 0 {
// In this case, parity == 0 implies that this object version is a
// delete marker
read_quorum = n / 2 + 1;
}
if occ < read_quorum {
// Ignore this parity since we don't have enough shards for read quorum
continue;
}
if occ > max_occ {
max_occ = occ;
cparity = parity;
}
}
if max_occ == 0 {
// Did not find anything useful
return -1;
}
cparity
}
pub(super) fn list_object_modtimes(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<Option<OffsetDateTime>> {
let mut times = vec![None; parts_metadata.len()];
for (i, metadata) in parts_metadata.iter().enumerate() {
if errs[i].is_some() {
continue;
}
times[i] = metadata.mod_time
}
times
}
pub(super) fn common_time(times: &[Option<OffsetDateTime>], quorum: usize) -> Option<OffsetDateTime> {
let (time, count) = Self::common_time_and_occurrence(times);
if count >= quorum { time } else { None }
}
pub(super) fn common_time_and_occurrence(times: &[Option<OffsetDateTime>]) -> (Option<OffsetDateTime>, usize) {
let mut time_occurrence_map = HashMap::new();
// Ignore the uuid sentinel and count the rest.
for time in times.iter().flatten() {
*time_occurrence_map.entry(time.unix_timestamp_nanos()).or_insert(0) += 1;
}
let mut maxima = 0; // Counter for remembering max occurrence of elements.
let mut latest = 0;
// Find the common cardinality from previously collected
// occurrences of elements.
for (&nano, &count) in &time_occurrence_map {
if count < maxima {
continue;
}
// We are at or above maxima
if count > maxima || nano > latest {
maxima = count;
latest = nano;
}
}
if latest == 0 {
return (None, maxima);
}
if let Ok(time) = OffsetDateTime::from_unix_timestamp_nanos(latest) {
(Some(time), maxima)
} else {
(None, maxima)
}
}
pub(super) fn common_etag(etags: &[Option<String>], quorum: usize) -> Option<String> {
let (etag, count) = Self::common_etags(etags);
if count >= quorum { etag } else { None }
}
pub(super) fn common_etags(etags: &[Option<String>]) -> (Option<String>, usize) {
let mut etags_map = HashMap::new();
for etag in etags.iter().flatten() {
*etags_map.entry(etag).or_insert(0) += 1;
}
let mut maxima = 0; // Counter for remembering max occurrence of elements.
let mut latest = None;
for (&etag, &count) in &etags_map {
if count < maxima {
continue;
}
// We are at or above maxima
if count > maxima {
maxima = count;
latest = Some(etag.clone());
}
}
(latest, maxima)
}
pub(super) fn list_object_etags(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<Option<String>> {
let mut etags = vec![None; parts_metadata.len()];
for (i, metadata) in parts_metadata.iter().enumerate() {
if errs[i].is_some() {
continue;
}
if let Some(etag) = metadata.metadata.get("etag") {
etags[i] = Some(etag.clone())
}
}
etags
}
pub(super) fn list_object_parities(parts_metadata: &[FileInfo], errs: &[Option<DiskError>]) -> Vec<i32> {
let total_shards = parts_metadata.len();
let half = total_shards as i32 / 2;
let mut parities: Vec<i32> = vec![-1; total_shards];
for (index, metadata) in parts_metadata.iter().enumerate() {
if errs[index].is_some() {
parities[index] = -1;
continue;
}
if !metadata.is_valid() {
parities[index] = -1;
continue;
}
if metadata.deleted || metadata.size == 0 {
parities[index] = half;
// } else if metadata.transition_status == "TransitionComplete" {
// TODO: metadata.transition_status
// parities[index] = total_shards - (total_shards / 2 + 1);
} else {
parities[index] = metadata.erasure.parity_blocks as i32;
}
}
parities
}
#[tracing::instrument(level = "debug", skip(parts_metadata))]
pub(super) fn object_quorum_from_meta(
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
default_parity_count: usize,
) -> disk::error::Result<(i32, i32)> {
if Self::all_not_found_metadata(errs) {
return Err(DiskError::FileNotFound);
}
let expected_rquorum = if default_parity_count == 0 {
parts_metadata.len()
} else {
parts_metadata.len() / 2
};
if let Some(err) = reduce_read_quorum_errs(errs, OBJECT_OP_IGNORED_ERRS, expected_rquorum) {
// let object = parts_metadata.first().map(|v| v.name.clone()).unwrap_or_default();
// error!("object_quorum_from_meta: {:?}, errs={:?}, object={:?}", err, errs, object);
return Err(err);
}
if default_parity_count == 0 {
return Ok((parts_metadata.len() as i32, parts_metadata.len() as i32));
}
let parities = Self::list_object_parities(parts_metadata, errs);
let parity_blocks = Self::common_parity(&parities, default_parity_count as i32);
if parity_blocks < 0 {
error!("object_quorum_from_meta: parity_blocks < 0, errs={:?}", errs);
return Err(DiskError::ErasureReadQuorum);
}
let data_blocks = parts_metadata.len() as i32 - parity_blocks;
let write_quorum = if data_blocks == parity_blocks {
data_blocks + 1
} else {
data_blocks
};
Ok((data_blocks, write_quorum))
}
#[tracing::instrument(level = "debug", skip(disks, parts_metadata))]
pub(super) fn list_online_disks(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
errs: &[Option<DiskError>],
quorum: usize,
) -> (Vec<Option<DiskStore>>, Option<OffsetDateTime>, Option<String>) {
let mod_times = Self::list_object_modtimes(parts_metadata, errs);
let mod_time = Self::common_time(&mod_times, quorum);
if mod_time.is_none() {
let etags = Self::list_object_etags(parts_metadata, errs);
let etag_op = Self::common_etag(&etags, quorum);
if let Some(etag) = etag_op {
let mut new_disk = vec![None; disks.len()];
for (i, etag_item) in etags.iter().enumerate() {
if let Some(etag_item) = etag_item
&& etag_item == &etag
&& parts_metadata[i].is_valid()
{
new_disk[i].clone_from(&disks[i]);
}
}
return (new_disk, None, Some(etag));
}
}
let mut new_disk = vec![None; disks.len()];
for (i, &t) in mod_times.iter().enumerate() {
if parts_metadata[i].is_valid() && mod_time == t {
new_disk[i].clone_from(&disks[i]);
}
}
(new_disk, mod_time, None)
}
pub(super) fn pick_valid_fileinfo(
metas: &[FileInfo],
mod_time: Option<OffsetDateTime>,
etag: Option<String>,
quorum: usize,
) -> disk::error::Result<FileInfo> {
Self::find_file_info_in_quorum(metas, &mod_time, &etag, quorum)
}
pub(super) fn find_file_info_in_quorum(
metas: &[FileInfo],
mod_time: &Option<OffsetDateTime>,
etag: &Option<String>,
quorum: usize,
) -> disk::error::Result<FileInfo> {
if quorum < 1 {
warn!("find_file_info_in_quorum: quorum < 1");
return Err(DiskError::ErasureReadQuorum);
}
let mut meta_hashes = vec![None; metas.len()];
let mut hasher = Sha256::new();
for (i, meta) in metas.iter().enumerate() {
if !meta.is_valid() {
debug!(
index = i,
valid = false,
version_id = ?meta.version_id,
mod_time = ?meta.mod_time,
"find_file_info_in_quorum: skipping invalid meta"
);
continue;
}
debug!(
index = i,
valid = true,
version_id = ?meta.version_id,
mod_time = ?meta.mod_time,
deleted = meta.deleted,
size = meta.size,
"find_file_info_in_quorum: inspecting meta"
);
let etag_only = mod_time.is_none() && etag.is_some() && meta.get_etag().is_some_and(|v| &v == etag.as_ref().unwrap());
let mod_valid = mod_time == &meta.mod_time;
if etag_only || mod_valid {
for part in meta.parts.iter() {
hasher.update(format!("part.{}", part.number).as_bytes());
hasher.update(format!("part.{}", part.size).as_bytes());
}
if !meta.deleted && meta.size != 0 {
hasher.update(format!("{}+{}", meta.erasure.data_blocks, meta.erasure.parity_blocks).as_bytes());
hasher.update(format!("{:?}", meta.erasure.distribution).as_bytes());
}
if meta.is_remote() {
// TODO:
}
// TODO: IsEncrypted
// TODO: IsCompressed
meta_hashes[i] = Some(hex(hasher.clone().finalize().as_slice()));
hasher.reset();
} else {
debug!(
index = i,
etag_only_match = etag_only,
mod_valid_match = mod_valid,
"find_file_info_in_quorum: meta does not match common etag or mod_time, skipping hash calculation"
);
}
}
let mut count_map = HashMap::new();
for hash in meta_hashes.iter().flatten() {
*count_map.entry(hash).or_insert(0) += 1;
}
let mut max_val = None;
let mut max_count = 0;
for (&val, &count) in &count_map {
if count > max_count {
max_val = Some(val);
max_count = count;
}
}
if max_count < quorum {
warn!("find_file_info_in_quorum: max_count < quorum, max_val={:?}", max_val);
return Err(DiskError::ErasureReadQuorum);
}
let mut found_fi = None;
let mut found = false;
let mut valid_obj_map = HashMap::new();
for (i, op_hash) in meta_hashes.iter().enumerate() {
if let Some(hash) = op_hash
&& let Some(max_hash) = max_val
&& hash == max_hash
&& metas[i].is_valid()
{
if !found {
found_fi = Some(metas[i].clone());
found = true;
}
let props = ObjProps {
mod_time: metas[i].mod_time,
num_versions: metas[i].num_versions,
};
*valid_obj_map.entry(props).or_insert(0) += 1;
}
}
if found {
let mut fi = found_fi.unwrap();
for (val, &count) in &valid_obj_map {
if count >= quorum {
fi.mod_time = val.mod_time;
fi.num_versions = val.num_versions;
fi.is_latest = val.mod_time.is_none();
break;
}
}
return Ok(fi);
}
warn!("find_file_info_in_quorum: fileinfo not found");
Err(DiskError::ErasureReadQuorum)
}
pub(super) fn shuffle_disks_and_parts_metadata_by_index(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
fi: &FileInfo,
) -> (Vec<Option<DiskStore>>, Vec<FileInfo>) {
let mut shuffled_disks = vec![None; disks.len()];
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
let distribution = &fi.erasure.distribution;
let mut inconsistent = 0;
for (k, v) in parts_metadata.iter().enumerate() {
if disks[k].is_none() {
inconsistent += 1;
continue;
}
if !v.is_valid() {
inconsistent += 1;
continue;
}
if distribution[k] != v.erasure.index {
inconsistent += 1;
continue;
}
let block_idx = distribution[k];
shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone();
shuffled_disks[block_idx - 1].clone_from(&disks[k]);
}
if inconsistent < fi.erasure.parity_blocks {
return (shuffled_disks, shuffled_parts_metadata);
}
Self::shuffle_disks_and_parts_metadata(disks, parts_metadata, fi)
}
pub(super) fn shuffle_disks_and_parts_metadata(
disks: &[Option<DiskStore>],
parts_metadata: &[FileInfo],
fi: &FileInfo,
) -> (Vec<Option<DiskStore>>, Vec<FileInfo>) {
let init = fi.mod_time.is_none();
let mut shuffled_disks = vec![None; disks.len()];
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
let distribution = &fi.erasure.distribution;
for (k, v) in disks.iter().enumerate() {
if v.is_none() {
continue;
}
if !init && !parts_metadata[k].is_valid() {
continue;
}
// if !init && fi.xlv1 != parts_metadata[k].xlv1 {
// continue;
// }
let block_idx = distribution[k];
shuffled_parts_metadata[block_idx - 1] = parts_metadata[k].clone();
shuffled_disks[block_idx - 1].clone_from(&disks[k]);
}
(shuffled_disks, shuffled_parts_metadata)
}
pub(super) fn shuffle_parts_metadata(parts_metadata: &[FileInfo], distribution: &[usize]) -> Vec<FileInfo> {
if distribution.is_empty() {
return parts_metadata.to_vec();
}
let mut shuffled_parts_metadata = vec![FileInfo::default(); parts_metadata.len()];
// Shuffle slice xl metadata for expected distribution.
for index in 0..parts_metadata.len() {
let block_index = distribution[index];
shuffled_parts_metadata[block_index - 1] = parts_metadata[index].clone();
}
shuffled_parts_metadata
}
pub(super) fn shuffle_disks(disks: &[Option<DiskStore>], distribution: &[usize]) -> Vec<Option<DiskStore>> {
if distribution.is_empty() {
return disks.to_vec();
}
let mut shuffled_disks = vec![None; disks.len()];
for (i, v) in disks.iter().enumerate() {
let idx = distribution[i];
shuffled_disks[idx - 1].clone_from(v);
}
shuffled_disks
}
pub(super) fn shuffle_check_parts(parts_errs: &[usize], distribution: &[usize]) -> Vec<usize> {
if distribution.is_empty() {
return parts_errs.to_vec();
}
let mut shuffled_parts_errs = vec![0; parts_errs.len()];
for (i, v) in parts_errs.iter().enumerate() {
let idx = distribution[i];
shuffled_parts_errs[idx - 1] = *v;
}
shuffled_parts_errs
}
}