// Copyright 2024 RustFS Team // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. use super::msgp_decode::{read_msgp_ext8_time, skip_msgp_value, write_msgp_time}; use super::object_lock::ObjectLockApi; use super::versioning::VersioningApi; use super::{quota::BucketQuota, target::BucketTargets}; use crate::bucket::utils::deserialize; use crate::config::com::{read_config, save_config}; use crate::disk::BUCKET_META_PREFIX; use crate::error::{Error, Result}; use crate::runtime_sources; use crate::store::ECStore; use byteorder::{BigEndian, ByteOrder, LittleEndian}; use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ AccelerateConfiguration, BucketLifecycleConfiguration, BucketLoggingStatus, CORSConfiguration, NotificationConfiguration, ObjectLockConfiguration, PublicAccessBlockConfiguration, ReplicationConfiguration, RequestPaymentConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, WebsiteConfiguration, }; use serde::Serializer; use sha2::{Digest, Sha256}; use std::collections::HashMap; use std::io::{Read, Write}; use std::sync::Arc; use time::{Date, OffsetDateTime, PrimitiveDateTime, Time as CivilTime, UtcOffset}; use tracing::error; fn read_msgp_str(rd: &mut R) -> Result { let len = rmp::decode::read_str_len(rd)? as usize; let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; Ok(String::from_utf8(buf)?) } fn read_msgp_bool(rd: &mut R) -> Result { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::True => Ok(true), rmp::Marker::False => Ok(false), rmp::Marker::FixPos(v) => Ok(v != 0), rmp::Marker::U8 => Ok(read_u8(rd)? != 0), rmp::Marker::U16 => Ok(read_u16_raw(rd)? != 0), rmp::Marker::U32 => Ok(read_u32_raw(rd)? != 0), rmp::Marker::U64 => Ok(read_u64_raw(rd)? != 0), rmp::Marker::I8 => Ok(read_i8_raw(rd)? != 0), rmp::Marker::I16 => Ok(read_i16_raw(rd)? != 0), rmp::Marker::I32 => Ok(read_i32_raw(rd)? != 0), rmp::Marker::I64 => Ok(read_i64_raw(rd)? != 0), rmp::Marker::FixNeg(v) => Ok(v != 0), _ => Err(Error::other(format!("expected bool or int-like bool, got marker: {marker:?}"))), } } fn read_msgp_time_value(rd: &mut R) -> Result { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::Null => Ok(OffsetDateTime::UNIX_EPOCH), rmp::Marker::Ext8 => read_msgp_ext8_time(rd), rmp::Marker::FixArray(len) => read_msgp_legacy_compact_time(rd, u32::from(len)), rmp::Marker::Array16 => { let len = read_u16_raw(rd)?; read_msgp_legacy_compact_time(rd, u32::from(len)) } rmp::Marker::Array32 => { let len = read_u32_raw(rd)?; read_msgp_legacy_compact_time(rd, len) } rmp::Marker::Bin8 => { let len = usize::from(read_u8(rd)?); read_msgp_time_value_from_embedded_bin(rd, len) } rmp::Marker::Bin16 => { let len = usize::from(read_u16_raw(rd)?); read_msgp_time_value_from_embedded_bin(rd, len) } rmp::Marker::Bin32 => { let len = read_u32_raw(rd)? as usize; read_msgp_time_value_from_embedded_bin(rd, len) } _ => Err(Error::other(format!("expected time ext or nil, got marker: {marker:?}"))), } } fn read_u8(rd: &mut R) -> Result { let mut buf = [0u8; 1]; rd.read_exact(&mut buf)?; Ok(buf[0]) } fn read_u16_raw(rd: &mut R) -> Result { let mut buf = [0u8; 2]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u16(&buf)) } fn read_u32_raw(rd: &mut R) -> Result { let mut buf = [0u8; 4]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u32(&buf)) } fn read_u64_raw(rd: &mut R) -> Result { let mut buf = [0u8; 8]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_u64(&buf)) } fn read_i8_raw(rd: &mut R) -> Result { Ok(read_u8(rd)? as i8) } fn read_i16_raw(rd: &mut R) -> Result { let mut buf = [0u8; 2]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i16(&buf)) } fn read_i32_raw(rd: &mut R) -> Result { let mut buf = [0u8; 4]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i32(&buf)) } fn read_i64_raw(rd: &mut R) -> Result { let mut buf = [0u8; 8]; rd.read_exact(&mut buf)?; Ok(BigEndian::read_i64(&buf)) } fn read_msgp_time_value_from_embedded_bin(rd: &mut R, len: usize) -> Result { let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; let mut cur = std::io::Cursor::new(buf); read_msgp_time_value(&mut cur) } fn read_msgp_legacy_compact_time(rd: &mut R, len: u32) -> Result { if len != 9 { return Err(Error::other(format!("invalid legacy compact time len: {len}"))); } let year: i32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let ordinal: u16 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let hour: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let minute: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let second: u8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let nanosecond: u32 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_hour: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_minute: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let offset_second: i8 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let date = Date::from_ordinal_date(year, ordinal).map_err(|e| Error::other(format!("invalid legacy compact time date: {e}")))?; let time = CivilTime::from_hms_nano(hour, minute, second, nanosecond) .map_err(|e| Error::other(format!("invalid legacy compact time time: {e}")))?; let offset = UtcOffset::from_hms(offset_hour, offset_minute, offset_second) .map_err(|e| Error::other(format!("invalid legacy compact time offset: {e}")))?; Ok(PrimitiveDateTime::new(date, time) .assume_offset(offset) .to_offset(UtcOffset::UTC)) } fn read_msgp_bin(rd: &mut R) -> Result> { let marker = rmp::decode::read_marker(rd).map_err(|e| Error::other(format!("{e:?}")))?; match marker { rmp::Marker::Null => Ok(Vec::new()), rmp::Marker::Bin8 => { let len = usize::from(read_u8(rd)?); read_exact_bytes(rd, len) } rmp::Marker::Bin16 => { let len = usize::from(read_u16_raw(rd)?); read_exact_bytes(rd, len) } rmp::Marker::Bin32 => { let len = read_u32_raw(rd)? as usize; read_exact_bytes(rd, len) } rmp::Marker::FixArray(len) => read_msgp_legacy_byte_array(rd, u32::from(len)), rmp::Marker::Array16 => { let len = read_u16_raw(rd)?; read_msgp_legacy_byte_array(rd, u32::from(len)) } rmp::Marker::Array32 => { let len = read_u32_raw(rd)?; read_msgp_legacy_byte_array(rd, len) } _ => Err(Error::other(format!("expected bin or byte array, got marker: {marker:?}"))), } } fn read_exact_bytes(rd: &mut R, len: usize) -> Result> { let mut buf = vec![0u8; len]; rd.read_exact(&mut buf)?; Ok(buf) } fn read_msgp_legacy_byte_array(rd: &mut R, len: u32) -> Result> { let mut buf = Vec::with_capacity(len as usize); for _ in 0..len { let value: i64 = rmp::decode::read_int(rd).map_err(|e| Error::other(format!("{e:?}")))?; let byte = u8::try_from(value).map_err(|_| Error::other(format!("byte value out of range: {value}")))?; buf.push(byte); } Ok(buf) } fn write_bin_field(wr: &mut W, key: &str, val: &[u8]) -> Result<()> { rmp::encode::write_str(wr, key)?; rmp::encode::write_bin(wr, val)?; Ok(()) } pub const BUCKET_METADATA_FILE: &str = ".metadata.bin"; pub const BUCKET_METADATA_FORMAT: u16 = 1; pub const BUCKET_METADATA_VERSION: u16 = 1; pub const BUCKET_POLICY_CONFIG: &str = "policy.json"; pub const BUCKET_NOTIFICATION_CONFIG: &str = "notification.xml"; pub const BUCKET_LIFECYCLE_CONFIG: &str = "lifecycle.xml"; pub const BUCKET_SSECONFIG: &str = "bucket-encryption.xml"; pub const BUCKET_TAGGING_CONFIG: &str = "tagging.xml"; pub const BUCKET_QUOTA_CONFIG_FILE: &str = "quota.json"; pub const OBJECT_LOCK_CONFIG: &str = "object-lock.xml"; pub const BUCKET_VERSIONING_CONFIG: &str = "versioning.xml"; pub const BUCKET_REPLICATION_CONFIG: &str = "replication.xml"; pub const BUCKET_TARGETS_FILE: &str = "bucket-targets.json"; pub const BUCKET_CORS_CONFIG: &str = "cors.xml"; pub const BUCKET_LOGGING_CONFIG: &str = "logging.xml"; pub const BUCKET_WEBSITE_CONFIG: &str = "website.xml"; pub const BUCKET_ACCELERATE_CONFIG: &str = "accelerate.xml"; pub const BUCKET_REQUEST_PAYMENT_CONFIG: &str = "request-payment.xml"; pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml"; pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json"; pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json"; pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table"; pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog"; pub const BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX: &str = "table-buckets"; pub fn table_catalog_path_hash(value: &str) -> String { let digest = Sha256::digest(value.as_bytes()); let mut output = String::with_capacity(digest.len() * 2); const HEX: &[u8; 16] = b"0123456789abcdef"; for byte in digest { output.push(char::from(HEX[usize::from(byte >> 4)])); output.push(char::from(HEX[usize::from(byte & 0x0f)])); } output } pub fn table_bucket_catalog_metadata_prefix(bucket: &str) -> String { format!( "{}/{}/{}", BUCKET_TABLE_CATALOG_META_PREFIX, BUCKET_TABLE_CATALOG_TABLE_BUCKETS_PREFIX, table_catalog_path_hash(bucket) ) } #[derive(Debug, Clone)] pub struct BucketMetadata { pub name: String, pub created: OffsetDateTime, pub lock_enabled: bool, // While marked as unused, it may need to be retained pub policy_config_json: Vec, pub notification_config_xml: Vec, pub lifecycle_config_xml: Vec, pub object_lock_config_xml: Vec, pub versioning_config_xml: Vec, pub encryption_config_xml: Vec, pub tagging_config_xml: Vec, pub quota_config_json: Vec, pub replication_config_xml: Vec, pub bucket_targets_config_json: Vec, pub bucket_targets_config_meta_json: Vec, pub cors_config_xml: Vec, pub logging_config_xml: Vec, pub website_config_xml: Vec, pub accelerate_config_xml: Vec, pub request_payment_config_xml: Vec, pub public_access_block_config_xml: Vec, pub bucket_acl_config_json: Vec, pub table_bucket_config_json: Vec, pub policy_config_updated_at: OffsetDateTime, pub object_lock_config_updated_at: OffsetDateTime, pub encryption_config_updated_at: OffsetDateTime, pub tagging_config_updated_at: OffsetDateTime, pub quota_config_updated_at: OffsetDateTime, pub replication_config_updated_at: OffsetDateTime, pub versioning_config_updated_at: OffsetDateTime, pub lifecycle_config_updated_at: OffsetDateTime, pub notification_config_updated_at: OffsetDateTime, pub bucket_targets_config_updated_at: OffsetDateTime, pub bucket_targets_config_meta_updated_at: OffsetDateTime, pub cors_config_updated_at: OffsetDateTime, pub logging_config_updated_at: OffsetDateTime, pub website_config_updated_at: OffsetDateTime, pub accelerate_config_updated_at: OffsetDateTime, pub request_payment_config_updated_at: OffsetDateTime, pub public_access_block_config_updated_at: OffsetDateTime, pub bucket_acl_config_updated_at: OffsetDateTime, pub table_bucket_config_updated_at: OffsetDateTime, pub new_field_updated_at: OffsetDateTime, pub policy_config: Option, pub notification_config: Option, pub lifecycle_config: Option, pub object_lock_config: Option, pub versioning_config: Option, pub sse_config: Option, pub tagging_config: Option, pub quota_config: Option, pub replication_config: Option, pub bucket_target_config: Option, pub bucket_target_config_meta: Option>, pub cors_config: Option, pub logging_config: Option, pub website_config: Option, pub accelerate_config: Option, pub request_payment_config: Option, pub public_access_block_config: Option, pub bucket_acl_config: Option, } impl Default for BucketMetadata { fn default() -> Self { Self { name: Default::default(), created: OffsetDateTime::UNIX_EPOCH, lock_enabled: Default::default(), policy_config_json: Default::default(), notification_config_xml: Default::default(), lifecycle_config_xml: Default::default(), object_lock_config_xml: Default::default(), versioning_config_xml: Default::default(), encryption_config_xml: Default::default(), tagging_config_xml: Default::default(), quota_config_json: Default::default(), replication_config_xml: Default::default(), bucket_targets_config_json: Default::default(), bucket_targets_config_meta_json: Default::default(), cors_config_xml: Default::default(), logging_config_xml: Default::default(), website_config_xml: Default::default(), accelerate_config_xml: Default::default(), request_payment_config_xml: Default::default(), public_access_block_config_xml: Default::default(), bucket_acl_config_json: Default::default(), table_bucket_config_json: Default::default(), policy_config_updated_at: OffsetDateTime::UNIX_EPOCH, object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH, encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH, tagging_config_updated_at: OffsetDateTime::UNIX_EPOCH, quota_config_updated_at: OffsetDateTime::UNIX_EPOCH, replication_config_updated_at: OffsetDateTime::UNIX_EPOCH, versioning_config_updated_at: OffsetDateTime::UNIX_EPOCH, lifecycle_config_updated_at: OffsetDateTime::UNIX_EPOCH, notification_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_targets_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_targets_config_meta_updated_at: OffsetDateTime::UNIX_EPOCH, cors_config_updated_at: OffsetDateTime::UNIX_EPOCH, logging_config_updated_at: OffsetDateTime::UNIX_EPOCH, website_config_updated_at: OffsetDateTime::UNIX_EPOCH, accelerate_config_updated_at: OffsetDateTime::UNIX_EPOCH, request_payment_config_updated_at: OffsetDateTime::UNIX_EPOCH, public_access_block_config_updated_at: OffsetDateTime::UNIX_EPOCH, bucket_acl_config_updated_at: OffsetDateTime::UNIX_EPOCH, table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH, new_field_updated_at: OffsetDateTime::UNIX_EPOCH, policy_config: Default::default(), notification_config: Default::default(), lifecycle_config: Default::default(), object_lock_config: Default::default(), versioning_config: Default::default(), sse_config: Default::default(), tagging_config: Default::default(), quota_config: Default::default(), replication_config: Default::default(), bucket_target_config: Default::default(), bucket_target_config_meta: Default::default(), cors_config: Default::default(), logging_config: Default::default(), website_config: Default::default(), accelerate_config: Default::default(), request_payment_config: Default::default(), public_access_block_config: Default::default(), bucket_acl_config: Default::default(), } } } impl BucketMetadata { pub fn new(name: &str) -> Self { BucketMetadata { name: name.to_string(), ..Default::default() } } pub fn save_file_path(&self) -> String { format!("{}/{}/{}", BUCKET_META_PREFIX, self.name.as_str(), BUCKET_METADATA_FILE) } pub fn versioning(&self) -> bool { self.lock_enabled || (self.object_lock_config.as_ref().is_some_and(|v| v.enabled()) || self.versioning_config.as_ref().is_some_and(|v| v.enabled())) } pub fn object_locking(&self) -> bool { self.lock_enabled || (self.versioning_config.as_ref().is_some_and(|v| v.enabled())) } pub fn table_bucket_enabled(&self) -> bool { !self.table_bucket_config_json.is_empty() } /// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn decode_from(&mut self, rd: &mut R) -> Result<()> { let mut fields = rmp::decode::read_map_len(rd)?; *self = Self::default(); while fields > 0 { fields -= 1; let key_len = rmp::decode::read_str_len(rd)?; let mut key_buf = vec![0u8; key_len as usize]; rd.read_exact(&mut key_buf)?; let key = String::from_utf8(key_buf)?; match key.as_str() { "Name" => self.name = read_msgp_str(rd)?, "Created" => self.created = read_msgp_time_value(rd)?, "LockEnabled" => self.lock_enabled = read_msgp_bool(rd)?, "PolicyConfigJSON" | "PolicyConfigJson" => self.policy_config_json = read_msgp_bin(rd)?, "NotificationConfigXML" | "NotificationConfigXml" => self.notification_config_xml = read_msgp_bin(rd)?, "LifecycleConfigXML" | "LifecycleConfigXml" => self.lifecycle_config_xml = read_msgp_bin(rd)?, "ObjectLockConfigXML" | "ObjectLockConfigXml" => self.object_lock_config_xml = read_msgp_bin(rd)?, "VersioningConfigXML" | "VersioningConfigXml" => self.versioning_config_xml = read_msgp_bin(rd)?, "EncryptionConfigXML" | "EncryptionConfigXml" => self.encryption_config_xml = read_msgp_bin(rd)?, "TaggingConfigXML" | "TaggingConfigXml" => self.tagging_config_xml = read_msgp_bin(rd)?, "QuotaConfigJSON" | "QuotaConfigJson" => self.quota_config_json = read_msgp_bin(rd)?, "ReplicationConfigXML" | "ReplicationConfigXml" => self.replication_config_xml = read_msgp_bin(rd)?, "BucketTargetsConfigJSON" | "BucketTargetsConfigJson" => self.bucket_targets_config_json = read_msgp_bin(rd)?, "BucketTargetsConfigMetaJSON" | "BucketTargetsConfigMetaJson" => { self.bucket_targets_config_meta_json = read_msgp_bin(rd)? } "PolicyConfigUpdatedAt" => self.policy_config_updated_at = read_msgp_time_value(rd)?, "ObjectLockConfigUpdatedAt" => self.object_lock_config_updated_at = read_msgp_time_value(rd)?, "EncryptionConfigUpdatedAt" => self.encryption_config_updated_at = read_msgp_time_value(rd)?, "TaggingConfigUpdatedAt" => self.tagging_config_updated_at = read_msgp_time_value(rd)?, "QuotaConfigUpdatedAt" => self.quota_config_updated_at = read_msgp_time_value(rd)?, "ReplicationConfigUpdatedAt" => self.replication_config_updated_at = read_msgp_time_value(rd)?, "VersioningConfigUpdatedAt" => self.versioning_config_updated_at = read_msgp_time_value(rd)?, "LifecycleConfigUpdatedAt" => self.lifecycle_config_updated_at = read_msgp_time_value(rd)?, "NotificationConfigUpdatedAt" => self.notification_config_updated_at = read_msgp_time_value(rd)?, "BucketTargetsConfigUpdatedAt" => self.bucket_targets_config_updated_at = read_msgp_time_value(rd)?, "BucketTargetsConfigMetaUpdatedAt" => self.bucket_targets_config_meta_updated_at = read_msgp_time_value(rd)?, "CorsConfigXML" | "CorsConfigXml" => self.cors_config_xml = read_msgp_bin(rd)?, "LoggingConfigXML" | "LoggingConfigXml" => self.logging_config_xml = read_msgp_bin(rd)?, "WebsiteConfigXML" | "WebsiteConfigXml" => self.website_config_xml = read_msgp_bin(rd)?, "AccelerateConfigXML" | "AccelerateConfigXml" => self.accelerate_config_xml = read_msgp_bin(rd)?, "RequestPaymentConfigXML" | "RequestPaymentConfigXml" => self.request_payment_config_xml = read_msgp_bin(rd)?, "PublicAccessBlockConfigXML" | "PublicAccessBlockConfigXml" => { self.public_access_block_config_xml = read_msgp_bin(rd)? } "BucketAclConfigJSON" | "BucketAclConfigJson" => self.bucket_acl_config_json = read_msgp_bin(rd)?, "TableBucketConfigJSON" | "TableBucketConfigJson" => self.table_bucket_config_json = read_msgp_bin(rd)?, "CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?, "LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?, "WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?, "AccelerateConfigUpdatedAt" => self.accelerate_config_updated_at = read_msgp_time_value(rd)?, "RequestPaymentConfigUpdatedAt" => self.request_payment_config_updated_at = read_msgp_time_value(rd)?, "PublicAccessBlockConfigUpdatedAt" => self.public_access_block_config_updated_at = read_msgp_time_value(rd)?, "BucketAclConfigUpdatedAt" => self.bucket_acl_config_updated_at = read_msgp_time_value(rd)?, "TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?, other => { tracing::debug!(field = %other, "BucketMetadata decode_from: skipping unknown field"); skip_msgp_value(rd)?; } } } Ok(()) } /// Encode to msgp bytes. Field order follows MinIO BucketMetadata for compatibility. pub fn encode_to(&self, wr: &mut W) -> Result<()> { // Map size: MinIO fields (25) + RustFS extensions (16) let map_len: u32 = 41; rmp::encode::write_map_len(wr, map_len)?; // MinIO field order (same as Go struct) rmp::encode::write_str(wr, "Name")?; rmp::encode::write_str(wr, &self.name)?; rmp::encode::write_str(wr, "Created")?; write_msgp_time(wr, self.created)?; rmp::encode::write_str(wr, "LockEnabled")?; rmp::encode::write_bool(wr, self.lock_enabled)?; write_bin_field(wr, "PolicyConfigJSON", &self.policy_config_json)?; write_bin_field(wr, "NotificationConfigXML", &self.notification_config_xml)?; write_bin_field(wr, "LifecycleConfigXML", &self.lifecycle_config_xml)?; write_bin_field(wr, "ObjectLockConfigXML", &self.object_lock_config_xml)?; write_bin_field(wr, "VersioningConfigXML", &self.versioning_config_xml)?; write_bin_field(wr, "EncryptionConfigXML", &self.encryption_config_xml)?; write_bin_field(wr, "TaggingConfigXML", &self.tagging_config_xml)?; write_bin_field(wr, "QuotaConfigJSON", &self.quota_config_json)?; write_bin_field(wr, "ReplicationConfigXML", &self.replication_config_xml)?; write_bin_field(wr, "BucketTargetsConfigJSON", &self.bucket_targets_config_json)?; write_bin_field(wr, "BucketTargetsConfigMetaJSON", &self.bucket_targets_config_meta_json)?; rmp::encode::write_str(wr, "PolicyConfigUpdatedAt")?; write_msgp_time(wr, self.policy_config_updated_at)?; rmp::encode::write_str(wr, "ObjectLockConfigUpdatedAt")?; write_msgp_time(wr, self.object_lock_config_updated_at)?; rmp::encode::write_str(wr, "EncryptionConfigUpdatedAt")?; write_msgp_time(wr, self.encryption_config_updated_at)?; rmp::encode::write_str(wr, "TaggingConfigUpdatedAt")?; write_msgp_time(wr, self.tagging_config_updated_at)?; rmp::encode::write_str(wr, "QuotaConfigUpdatedAt")?; write_msgp_time(wr, self.quota_config_updated_at)?; rmp::encode::write_str(wr, "ReplicationConfigUpdatedAt")?; write_msgp_time(wr, self.replication_config_updated_at)?; rmp::encode::write_str(wr, "VersioningConfigUpdatedAt")?; write_msgp_time(wr, self.versioning_config_updated_at)?; rmp::encode::write_str(wr, "LifecycleConfigUpdatedAt")?; write_msgp_time(wr, self.lifecycle_config_updated_at)?; rmp::encode::write_str(wr, "NotificationConfigUpdatedAt")?; write_msgp_time(wr, self.notification_config_updated_at)?; rmp::encode::write_str(wr, "BucketTargetsConfigUpdatedAt")?; write_msgp_time(wr, self.bucket_targets_config_updated_at)?; rmp::encode::write_str(wr, "BucketTargetsConfigMetaUpdatedAt")?; write_msgp_time(wr, self.bucket_targets_config_meta_updated_at)?; // RustFS extensions write_bin_field(wr, "CorsConfigXML", &self.cors_config_xml)?; write_bin_field(wr, "LoggingConfigXML", &self.logging_config_xml)?; write_bin_field(wr, "WebsiteConfigXML", &self.website_config_xml)?; write_bin_field(wr, "AccelerateConfigXML", &self.accelerate_config_xml)?; write_bin_field(wr, "RequestPaymentConfigXML", &self.request_payment_config_xml)?; write_bin_field(wr, "PublicAccessBlockConfigXML", &self.public_access_block_config_xml)?; write_bin_field(wr, "BucketAclConfigJSON", &self.bucket_acl_config_json)?; write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?; rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?; write_msgp_time(wr, self.cors_config_updated_at)?; rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?; write_msgp_time(wr, self.logging_config_updated_at)?; rmp::encode::write_str(wr, "WebsiteConfigUpdatedAt")?; write_msgp_time(wr, self.website_config_updated_at)?; rmp::encode::write_str(wr, "AccelerateConfigUpdatedAt")?; write_msgp_time(wr, self.accelerate_config_updated_at)?; rmp::encode::write_str(wr, "RequestPaymentConfigUpdatedAt")?; write_msgp_time(wr, self.request_payment_config_updated_at)?; rmp::encode::write_str(wr, "PublicAccessBlockConfigUpdatedAt")?; write_msgp_time(wr, self.public_access_block_config_updated_at)?; rmp::encode::write_str(wr, "BucketAclConfigUpdatedAt")?; write_msgp_time(wr, self.bucket_acl_config_updated_at)?; rmp::encode::write_str(wr, "TableBucketConfigUpdatedAt")?; write_msgp_time(wr, self.table_bucket_config_updated_at)?; Ok(()) } pub fn marshal_msg(&self) -> Result> { let mut buf = Vec::new(); self.encode_to(&mut buf)?; Ok(buf) } pub fn unmarshal(buf: &[u8]) -> Result { let mut bm = Self::default(); let mut cur = std::io::Cursor::new(buf); bm.decode_from(&mut cur)?; Ok(bm) } pub fn check_header(buf: &[u8]) -> Result<()> { if buf.len() <= 4 { return Err(Error::other("read_bucket_metadata: data invalid")); } let format = LittleEndian::read_u16(&buf[0..2]); let version = LittleEndian::read_u16(&buf[2..4]); match format { BUCKET_METADATA_FORMAT => {} _ => return Err(Error::other("read_bucket_metadata: format invalid")), } match version { BUCKET_METADATA_VERSION => {} _ => return Err(Error::other("read_bucket_metadata: version invalid")), } Ok(()) } fn default_timestamps(&mut self) { if self.policy_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.policy_config_updated_at = self.created } if self.encryption_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.encryption_config_updated_at = self.created } if self.tagging_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.tagging_config_updated_at = self.created } if self.object_lock_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.object_lock_config_updated_at = self.created } if self.quota_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.quota_config_updated_at = self.created } if self.replication_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.replication_config_updated_at = self.created } if self.versioning_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.versioning_config_updated_at = self.created } if self.lifecycle_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.lifecycle_config_updated_at = self.created } if self.notification_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.notification_config_updated_at = self.created } if self.bucket_targets_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_targets_config_updated_at = self.created } if self.bucket_targets_config_meta_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_targets_config_meta_updated_at = self.created } if self.public_access_block_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.public_access_block_config_updated_at = self.created } if self.logging_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.logging_config_updated_at = self.created } if self.website_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.website_config_updated_at = self.created } if self.accelerate_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.accelerate_config_updated_at = self.created } if self.request_payment_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.request_payment_config_updated_at = self.created } if self.bucket_acl_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.bucket_acl_config_updated_at = self.created } if self.table_bucket_config_updated_at == OffsetDateTime::UNIX_EPOCH { self.table_bucket_config_updated_at = self.created } } pub fn update_config(&mut self, config_file: &str, data: Vec) -> Result { let updated = OffsetDateTime::now_utc(); match config_file { BUCKET_POLICY_CONFIG => { self.policy_config_json = data; self.policy_config_updated_at = updated; } BUCKET_NOTIFICATION_CONFIG => { self.notification_config_xml = data; self.notification_config_updated_at = updated; } BUCKET_LIFECYCLE_CONFIG => { self.lifecycle_config_xml = data; self.lifecycle_config_updated_at = updated; } BUCKET_SSECONFIG => { self.encryption_config_xml = data; self.encryption_config_updated_at = updated; } BUCKET_TAGGING_CONFIG => { self.tagging_config_xml = data; self.tagging_config_updated_at = updated; } BUCKET_QUOTA_CONFIG_FILE => { self.quota_config_json = data; self.quota_config_updated_at = updated; } OBJECT_LOCK_CONFIG => { self.object_lock_config_xml = data; self.object_lock_config_updated_at = updated; } BUCKET_VERSIONING_CONFIG => { self.versioning_config_xml = data; self.versioning_config_updated_at = updated; } BUCKET_REPLICATION_CONFIG => { self.replication_config_xml = data; self.replication_config_updated_at = updated; } BUCKET_TARGETS_FILE => { // let x = data.clone(); // let str = std::str::from_utf8(&x).expect("Invalid UTF-8"); // println!("update config:{}", str); self.bucket_targets_config_json = data; self.bucket_targets_config_updated_at = updated; } BUCKET_CORS_CONFIG => { self.cors_config_xml = data; self.cors_config_updated_at = updated; } BUCKET_LOGGING_CONFIG => { self.logging_config_xml = data; self.logging_config_updated_at = updated; } BUCKET_WEBSITE_CONFIG => { self.website_config_xml = data; self.website_config_updated_at = updated; } BUCKET_ACCELERATE_CONFIG => { self.accelerate_config_xml = data; self.accelerate_config_updated_at = updated; } BUCKET_REQUEST_PAYMENT_CONFIG => { self.request_payment_config_xml = data; self.request_payment_config_updated_at = updated; } BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG => { self.public_access_block_config_xml = data; self.public_access_block_config_updated_at = updated; } BUCKET_ACL_CONFIG => { self.bucket_acl_config_json = data; self.bucket_acl_config_updated_at = updated; } BUCKET_TABLE_CONFIG => { self.table_bucket_config_json = data; self.table_bucket_config_updated_at = updated; } _ => return Err(Error::other(format!("config file not found : {config_file}"))), } Ok(updated) } pub fn set_created(&mut self, created: Option) { self.created = created.unwrap_or_else(OffsetDateTime::now_utc) } pub async fn save(&mut self) -> Result<()> { let Some(store) = runtime_sources::object_store_handle() else { return Err(Error::other("errServerNotInitialized")); }; self.parse_all_configs()?; let mut buf: Vec = vec![0; 4]; LittleEndian::write_u16(&mut buf[0..2], BUCKET_METADATA_FORMAT); LittleEndian::write_u16(&mut buf[2..4], BUCKET_METADATA_VERSION); let data = self .marshal_msg() .map_err(|e| Error::other(format!("save bucket metadata failed: {e}")))?; buf.extend_from_slice(&data); save_config(store, self.save_file_path().as_str(), buf).await?; Ok(()) } fn parse_policy_config(&mut self) -> Result<()> { if !self.policy_config_json.is_empty() { self.policy_config = Some(serde_json::from_slice(&self.policy_config_json)?); } else { self.policy_config = None; } Ok(()) } fn parse_all_configs(&mut self) -> Result<()> { if let Err(e) = self.parse_policy_config() { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "policy", error = %e, "Failed to parse bucket metadata config" ); } if !self.notification_config_xml.is_empty() && let Err(e) = deserialize::(&self.notification_config_xml) .map(|c| self.notification_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "notification", error = %e, "Failed to parse bucket metadata config" ); } if !self.lifecycle_config_xml.is_empty() && let Err(e) = deserialize::(&self.lifecycle_config_xml).map(|c| self.lifecycle_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "lifecycle", error = %e, "Failed to parse bucket metadata config" ); } if !self.object_lock_config_xml.is_empty() && let Err(e) = deserialize::(&self.object_lock_config_xml).map(|c| self.object_lock_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "object_lock", error = %e, "Failed to parse bucket metadata config" ); } if !self.versioning_config_xml.is_empty() && let Err(e) = deserialize::(&self.versioning_config_xml).map(|c| self.versioning_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "versioning", error = %e, "Failed to parse bucket metadata config" ); } if !self.encryption_config_xml.is_empty() && let Err(e) = deserialize::(&self.encryption_config_xml).map(|c| self.sse_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "encryption", error = %e, "Failed to parse bucket metadata config" ); } if !self.tagging_config_xml.is_empty() && let Err(e) = deserialize::(&self.tagging_config_xml).map(|c| self.tagging_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "tagging", error = %e, "Failed to parse bucket metadata config" ); } if !self.quota_config_json.is_empty() && let Err(e) = serde_json::from_slice(&self.quota_config_json).map(|c| self.quota_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "quota", error = %e, "Failed to parse bucket metadata config" ); } if !self.replication_config_xml.is_empty() && let Err(e) = deserialize::(&self.replication_config_xml).map(|c| self.replication_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "replication", error = %e, "Failed to parse bucket metadata config" ); } if !self.bucket_targets_config_json.is_empty() { if let Err(e) = serde_json::from_slice::(&self.bucket_targets_config_json) .map(|t| self.bucket_target_config = Some(t)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "bucket_targets", error = %e, "Failed to parse bucket metadata config" ); self.bucket_target_config = Some(BucketTargets::default()); } } else { self.bucket_target_config = Some(BucketTargets::default()); } if !self.cors_config_xml.is_empty() && let Err(e) = deserialize::(&self.cors_config_xml).map(|c| self.cors_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "cors", error = %e, "Failed to parse bucket metadata config" ); } if !self.logging_config_xml.is_empty() && let Err(e) = deserialize::(&self.logging_config_xml).map(|c| self.logging_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "logging", error = %e, "Failed to parse bucket metadata config" ); } if !self.website_config_xml.is_empty() && let Err(e) = deserialize::(&self.website_config_xml).map(|c| self.website_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "website", error = %e, "Failed to parse bucket metadata config" ); } if !self.accelerate_config_xml.is_empty() && let Err(e) = deserialize::(&self.accelerate_config_xml).map(|c| self.accelerate_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "accelerate", error = %e, "Failed to parse bucket metadata config" ); } if !self.request_payment_config_xml.is_empty() && let Err(e) = deserialize::(&self.request_payment_config_xml) .map(|c| self.request_payment_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "request_payment", error = %e, "Failed to parse bucket metadata config" ); } if !self.public_access_block_config_xml.is_empty() && let Err(e) = deserialize::(&self.public_access_block_config_xml) .map(|c| self.public_access_block_config = Some(c)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "public_access_block", error = %e, "Failed to parse bucket metadata config" ); } if !self.bucket_acl_config_json.is_empty() && let Err(e) = String::from_utf8(self.bucket_acl_config_json.clone()).map(|acl| self.bucket_acl_config = Some(acl)) { tracing::warn!( event = "bucket_metadata_parse_failed", component = "ecstore", subsystem = "bucket_metadata", bucket = %self.name, config = "bucket_acl", error = %e, "Failed to parse bucket metadata config" ); } Ok(()) } } pub async fn load_bucket_metadata(api: Arc, bucket: &str) -> Result { load_bucket_metadata_parse(api, bucket, true).await } pub async fn load_bucket_metadata_parse(api: Arc, bucket: &str, parse: bool) -> Result { let mut bm = match read_bucket_metadata(api.clone(), bucket).await { Ok(res) => res, Err(err) => { if err != Error::ConfigNotFound { return Err(err); } // info!("bucketmeta {} not found with err {:?}, start to init ", bucket, &err); BucketMetadata::new(bucket) } }; bm.default_timestamps(); if parse { bm.parse_all_configs()?; } Ok(bm) } async fn read_bucket_metadata(api: Arc, bucket: &str) -> Result { if bucket.is_empty() { error!("bucket name empty"); return Err(Error::other("invalid argument")); } let bm = BucketMetadata::new(bucket); let file_path = bm.save_file_path(); let data = read_config(api, &file_path).await?; BucketMetadata::check_header(&data)?; let bm = BucketMetadata::unmarshal(&data[4..])?; Ok(bm) } fn _write_time(t: &OffsetDateTime, s: S) -> std::result::Result where S: Serializer, { let mut buf = vec![0x0; 15]; let sec = t.unix_timestamp() - 62135596800; let nsec = t.nanosecond(); buf[0] = 0xc7; // mext8 buf[1] = 0x0c; // Length buf[2] = 0x05; // Time extension type BigEndian::write_u64(&mut buf[3..], sec as u64); BigEndian::write_u32(&mut buf[11..], nsec); s.serialize_bytes(&buf) } #[cfg(test)] mod test { use super::*; #[tokio::test] async fn marshal_msg() { // write_time(OffsetDateTime::UNIX_EPOCH).unwrap(); let bm = BucketMetadata::new("dada"); let buf = bm.marshal_msg().unwrap(); let new = BucketMetadata::unmarshal(&buf).unwrap(); assert_eq!(bm.name, new.name); } #[test] fn parse_all_configs_parses_stored_configs_without_store_dependency() { let mut bm = BucketMetadata::new("test-bucket"); bm.policy_config_json = br#"{"Version":"2012-10-17","Statement":[]}"#.to_vec(); bm.bucket_targets_config_json = br#"{"targets":[{"endpoint":"s3.amazonaws.com","targetbucket":"target-bucket","arn":"arn:aws:s3:::target-bucket"}]}"# .to_vec(); bm.parse_all_configs().unwrap(); assert!(bm.policy_config.is_some()); let bucket_targets = bm.bucket_target_config.unwrap(); assert_eq!(bucket_targets.targets.len(), 1); assert_eq!(bucket_targets.targets[0].endpoint, "s3.amazonaws.com"); assert_eq!(bucket_targets.targets[0].target_bucket, "target-bucket"); } #[tokio::test] async fn marshal_msg_complete_example() { // Create a complete BucketMetadata with various configurations let mut bm = BucketMetadata::new("test-bucket"); // Set creation time to current time bm.created = OffsetDateTime::now_utc(); bm.lock_enabled = true; // Add policy configuration let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::test-bucket/*"}]}"#; bm.policy_config_json = policy_json.as_bytes().to_vec(); bm.policy_config_updated_at = OffsetDateTime::now_utc(); // Add lifecycle configuration let lifecycle_xml = r#"rule1Enabled30"#; bm.lifecycle_config_xml = lifecycle_xml.as_bytes().to_vec(); bm.lifecycle_config_updated_at = OffsetDateTime::now_utc(); // Add versioning configuration let versioning_xml = r#"Enabled"#; bm.versioning_config_xml = versioning_xml.as_bytes().to_vec(); bm.versioning_config_updated_at = OffsetDateTime::now_utc(); // Add encryption configuration let encryption_xml = r#"AES256"#; bm.encryption_config_xml = encryption_xml.as_bytes().to_vec(); bm.encryption_config_updated_at = OffsetDateTime::now_utc(); // Add tagging configuration let tagging_xml = r#"EnvironmentTestOwnerRustFS"#; bm.tagging_config_xml = tagging_xml.as_bytes().to_vec(); bm.tagging_config_updated_at = OffsetDateTime::now_utc(); // Add quota configuration let quota_json = r#"{"quota":1073741824,"quota_type":"Hard","created_at":"2024-01-01T00:00:00Z","updated_at":"2024-01-01T00:00:00Z"}"#; // 1GB quota bm.quota_config_json = quota_json.as_bytes().to_vec(); bm.quota_config_updated_at = OffsetDateTime::now_utc(); // Add object lock configuration let object_lock_xml = r#"EnabledGOVERNANCE7"#; bm.object_lock_config_xml = object_lock_xml.as_bytes().to_vec(); bm.object_lock_config_updated_at = OffsetDateTime::now_utc(); // Add notification configuration let notification_xml = r#"notification1s3:ObjectCreated:*test-log-group"#; bm.notification_config_xml = notification_xml.as_bytes().to_vec(); bm.notification_config_updated_at = OffsetDateTime::now_utc(); // Add replication configuration let replication_xml = r#"arn:aws:iam::123456789012:role/replication-rolerule1Enableddocuments/arn:aws:s3:::destination-bucket"#; bm.replication_config_xml = replication_xml.as_bytes().to_vec(); bm.replication_config_updated_at = OffsetDateTime::now_utc(); // Add bucket targets configuration let bucket_targets_json = r#"[{"endpoint":"http://target1.example.com","credentials":{"accessKey":"key1","secretKey":"secret1"},"targetBucket":"target-bucket-1","region":"us-east-1"},{"endpoint":"http://target2.example.com","credentials":{"accessKey":"key2","secretKey":"secret2"},"targetBucket":"target-bucket-2","region":"us-west-2"}]"#; bm.bucket_targets_config_json = bucket_targets_json.as_bytes().to_vec(); bm.bucket_targets_config_updated_at = OffsetDateTime::now_utc(); // Add bucket targets meta configuration let bucket_targets_meta_json = r#"{"replicationId":"repl-123","syncMode":"async","bandwidth":"100MB"}"#; bm.bucket_targets_config_meta_json = bucket_targets_meta_json.as_bytes().to_vec(); bm.bucket_targets_config_meta_updated_at = OffsetDateTime::now_utc(); // Add public access block configuration let public_access_block_xml = r#"truetruetruefalse"#; bm.public_access_block_config_xml = public_access_block_xml.as_bytes().to_vec(); bm.public_access_block_config_updated_at = OffsetDateTime::now_utc(); let bucket_acl = r#"{"owner":{"id":"rustfsadmin","display_name":"RustFS Tester"},"grants":[{"grantee":{"grantee_type":"CanonicalUser","id":"rustfsadmin","display_name":"RustFS Tester","uri":null,"email_address":null},"permission":"FULL_CONTROL"}]}"#; bm.bucket_acl_config_json = bucket_acl.as_bytes().to_vec(); bm.bucket_acl_config_updated_at = OffsetDateTime::now_utc(); let table_bucket_marker = r#"{"enabled":true}"#; bm.table_bucket_config_json = table_bucket_marker.as_bytes().to_vec(); bm.table_bucket_config_updated_at = OffsetDateTime::now_utc(); // Test serialization let buf = bm.marshal_msg().unwrap(); assert!(!buf.is_empty(), "Serialized buffer should not be empty"); // Test deserialization let deserialized_bm = BucketMetadata::unmarshal(&buf).unwrap(); // Verify all fields are correctly serialized and deserialized assert_eq!(bm.name, deserialized_bm.name); assert_eq!(bm.created.unix_timestamp(), deserialized_bm.created.unix_timestamp()); assert_eq!(bm.lock_enabled, deserialized_bm.lock_enabled); // Verify configuration data assert_eq!(bm.policy_config_json, deserialized_bm.policy_config_json); assert_eq!(bm.lifecycle_config_xml, deserialized_bm.lifecycle_config_xml); assert_eq!(bm.versioning_config_xml, deserialized_bm.versioning_config_xml); assert_eq!(bm.encryption_config_xml, deserialized_bm.encryption_config_xml); assert_eq!(bm.tagging_config_xml, deserialized_bm.tagging_config_xml); assert_eq!(bm.quota_config_json, deserialized_bm.quota_config_json); assert_eq!(bm.public_access_block_config_xml, deserialized_bm.public_access_block_config_xml); assert_eq!(bm.bucket_acl_config_json, deserialized_bm.bucket_acl_config_json); assert_eq!(bm.table_bucket_config_json, deserialized_bm.table_bucket_config_json); assert_eq!(bm.object_lock_config_xml, deserialized_bm.object_lock_config_xml); assert_eq!(bm.notification_config_xml, deserialized_bm.notification_config_xml); assert_eq!(bm.replication_config_xml, deserialized_bm.replication_config_xml); assert_eq!(bm.bucket_targets_config_json, deserialized_bm.bucket_targets_config_json); assert_eq!(bm.bucket_targets_config_meta_json, deserialized_bm.bucket_targets_config_meta_json); // Verify timestamps (comparing unix timestamps to avoid precision issues) assert_eq!( bm.policy_config_updated_at.unix_timestamp(), deserialized_bm.policy_config_updated_at.unix_timestamp() ); assert_eq!( bm.lifecycle_config_updated_at.unix_timestamp(), deserialized_bm.lifecycle_config_updated_at.unix_timestamp() ); assert_eq!( bm.versioning_config_updated_at.unix_timestamp(), deserialized_bm.versioning_config_updated_at.unix_timestamp() ); assert_eq!( bm.encryption_config_updated_at.unix_timestamp(), deserialized_bm.encryption_config_updated_at.unix_timestamp() ); assert_eq!( bm.tagging_config_updated_at.unix_timestamp(), deserialized_bm.tagging_config_updated_at.unix_timestamp() ); assert_eq!( bm.quota_config_updated_at.unix_timestamp(), deserialized_bm.quota_config_updated_at.unix_timestamp() ); assert_eq!( bm.object_lock_config_updated_at.unix_timestamp(), deserialized_bm.object_lock_config_updated_at.unix_timestamp() ); assert_eq!( bm.notification_config_updated_at.unix_timestamp(), deserialized_bm.notification_config_updated_at.unix_timestamp() ); assert_eq!( bm.replication_config_updated_at.unix_timestamp(), deserialized_bm.replication_config_updated_at.unix_timestamp() ); assert_eq!( bm.bucket_targets_config_updated_at.unix_timestamp(), deserialized_bm.bucket_targets_config_updated_at.unix_timestamp() ); assert_eq!( bm.bucket_targets_config_meta_updated_at.unix_timestamp(), deserialized_bm.bucket_targets_config_meta_updated_at.unix_timestamp() ); assert_eq!( bm.table_bucket_config_updated_at.unix_timestamp(), deserialized_bm.table_bucket_config_updated_at.unix_timestamp() ); assert!(deserialized_bm.table_bucket_enabled()); // Test that the serialized data contains expected content let buf_str = String::from_utf8_lossy(&buf); assert!(buf_str.contains("test-bucket"), "Serialized data should contain bucket name"); // Verify the buffer size is reasonable (should be larger due to all the config data) assert!(buf.len() > 1000, "Buffer should be substantial in size due to all configurations"); println!("✅ Complete BucketMetadata serialization test passed"); println!(" - Bucket name: {}", deserialized_bm.name); println!(" - Lock enabled: {}", deserialized_bm.lock_enabled); println!(" - Policy config size: {} bytes", deserialized_bm.policy_config_json.len()); println!(" - Lifecycle config size: {} bytes", deserialized_bm.lifecycle_config_xml.len()); println!(" - Serialized buffer size: {} bytes", buf.len()); } #[test] fn table_bucket_marker_tracks_config_presence() { let mut bm = BucketMetadata::new("table-bucket"); assert!(!bm.table_bucket_enabled()); bm.update_config(BUCKET_TABLE_CONFIG, br#"{"enabled":true}"#.to_vec()) .unwrap(); assert!(bm.table_bucket_enabled()); assert!(!bm.table_bucket_config_json.is_empty()); bm.update_config(BUCKET_TABLE_CONFIG, Vec::new()).unwrap(); assert!(!bm.table_bucket_enabled()); } /// After policy deletion (policy_config_json cleared), parse_policy_config sets policy_config to None. #[test] fn test_parse_policy_config_clears_cache_when_json_empty() { let policy_json = r#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}"#; let mut bm = BucketMetadata::new("b"); bm.policy_config_json = policy_json.as_bytes().to_vec(); bm.parse_policy_config().unwrap(); assert!(bm.policy_config.is_some(), "policy_config should be set when JSON non-empty"); bm.policy_config_json.clear(); bm.parse_policy_config().unwrap(); assert!( bm.policy_config.is_none(), "policy_config should be None after JSON cleared (e.g. policy deleted)" ); } }